detect file moves in sync

This commit is contained in:
phernandez
2025-01-13 23:01:39 -06:00
parent ef13b3b30a
commit dd7a9da3e4
6 changed files with 184 additions and 22 deletions
+18 -1
View File
@@ -67,8 +67,19 @@ def group_changes_by_directory(changes: SyncReport) -> Dict[str, Dict[str, int]]
]:
for path in paths:
dir_name = path.split("/", 1)[0]
by_dir.setdefault(dir_name, {"new": 0, "modified": 0, "deleted": 0})
by_dir.setdefault(dir_name, {"new": 0, "modified": 0, "deleted": 0, "moved": 0})
by_dir[dir_name][change_type] += 1
# Handle moves - count in both source and destination directories
for old_path, new_path in changes.moves.items():
old_dir = old_path.split("/", 1)[0]
new_dir = new_path.split("/", 1)[0]
by_dir.setdefault(old_dir, {"new": 0, "modified": 0, "deleted": 0, "moved": 0})
by_dir.setdefault(new_dir, {"new": 0, "modified": 0, "deleted": 0, "moved": 0})
by_dir[old_dir]["moved"] += 1
if old_dir != new_dir:
by_dir[new_dir]["moved"] += 1
return by_dir
@@ -79,6 +90,8 @@ def build_directory_summary(counts: Dict[str, int]) -> str:
parts.append(f"[green]+{counts['new']} new[/green]")
if counts["modified"]:
parts.append(f"[yellow]~{counts['modified']} modified[/yellow]")
if counts["moved"]:
parts.append(f"[blue]↔{counts['moved']} moved[/blue]")
if counts["deleted"]:
parts.append(f"[red]-{counts['deleted']} deleted[/red]")
return " ".join(parts)
@@ -101,6 +114,10 @@ def display_changes(title: str, changes: SyncReport, verbose: bool = False):
if changes.modified:
mod_branch = tree.add("[yellow]Modified[/yellow]")
add_files_to_tree(mod_branch, changes.modified, "yellow", changes.checksums)
if changes.moves:
move_branch = tree.add("[blue]Moved[/blue]")
for old_path, new_path in sorted(changes.moves.items()):
move_branch.add(f"[blue]{old_path}[/blue] → [blue]{new_path}[/blue]")
if changes.deleted:
del_branch = tree.add("[red]Deleted[/red]")
add_files_to_tree(del_branch, changes.deleted, "red")
+19 -4
View File
@@ -25,6 +25,7 @@ from basic_memory.repository import (
RelationRepository,
)
from basic_memory.repository.search_repository import SearchRepository
from basic_memory.services.link_resolver import LinkResolver
from basic_memory.services.search_service import SearchService
from basic_memory.sync import SyncService, FileChangeScanner, EntitySyncService
from basic_memory.sync.utils import SyncReport
@@ -51,21 +52,28 @@ async def get_sync_service(db_type=DatabaseType.FILESYSTEM):
relation_repository = RelationRepository(session_maker)
search_repository = SearchRepository(session_maker)
# Initialize services
search_service = SearchService(search_repository, entity_repository)
link_resolver = LinkResolver(entity_repository, search_service)
# Initialize scanner
file_change_scanner = FileChangeScanner(entity_repository)
# Initialize services
knowledge_sync_service = EntitySyncService(
entity_repository, observation_repository, relation_repository
entity_repository,
observation_repository,
relation_repository,
link_resolver
)
entity_parser = EntityParser(config.home)
search_service = SearchService(search_repository, entity_repository)
# Create sync service
sync_service = SyncService(
scanner=file_change_scanner,
entity_sync_service=knowledge_sync_service,
entity_parser=entity_parser,
entity_repository=entity_repository,
search_service=search_service,
)
@@ -135,16 +143,19 @@ def display_sync_summary(knowledge: SyncReport):
console.print("[green]Everything up to date[/green]")
return
# Format as: "Synced X files (A new, B modified, C deleted)"
# Format as: "Synced X files (A new, B modified, C moved, D deleted)"
changes = []
new_count = len(knowledge.new)
mod_count = len(knowledge.modified)
move_count = len(knowledge.moves)
del_count = len(knowledge.deleted)
if new_count:
changes.append(f"[green]{new_count} new[/green]")
if mod_count:
changes.append(f"[yellow]{mod_count} modified[/yellow]")
if move_count:
changes.append(f"[blue]{move_count} moved[/blue]")
if del_count:
changes.append(f"[red]{del_count} deleted[/red]")
@@ -171,13 +182,17 @@ def display_detailed_sync_results(knowledge: SyncReport):
for path in sorted(knowledge.modified):
checksum = knowledge.checksums.get(path, "")
modified.add(f"[yellow]{path}[/yellow] ({checksum[:8]})")
if knowledge.moves:
moved = knowledge_tree.add("[blue]Moved[/blue]")
for old_path, new_path in sorted(knowledge.moves.items()):
checksum = knowledge.checksums.get(new_path, "")
moved.add(f"[blue]{old_path}[/blue] → [blue]{new_path}[/blue] ({checksum[:8]})")
if knowledge.deleted:
deleted = knowledge_tree.add("[red]Deleted[/red]")
for path in sorted(knowledge.deleted):
deleted.add(f"[red]{path}[/red]")
console.print(knowledge_tree)
async def validate_knowledge_files(
sync_service: SyncService, directory: Path
) -> List[ValidationIssue]:
@@ -62,7 +62,6 @@ class LinkResolver():
return await self.entity_repository.get_by_permalink(best_match.permalink)
# TODO replace with search ranking
def _normalize_link_text(self, link_text: str) -> Tuple[str, Optional[str]]:
"""Normalize link text and extract alias if present.
+19 -15
View File
@@ -87,18 +87,9 @@ class FileChangeScanner:
return result
async def find_changes(
self, directory: Path, db_file_state: Dict[str, FileState]
self, directory: Path, db_file_state: Dict[str, FileState]
) -> SyncReport:
"""
Find changes between filesystem and database.
Args:
directory: Directory to check
db_file_state: dict mapping file_path to DbState(permalink, checksum)
Returns:
SyncReport detailing changes
"""
"""Find changes between filesystem and database."""
# Get current files and checksums
scan_result = await self.scan_directory(directory)
current_files = scan_result.files
@@ -106,28 +97,41 @@ class FileChangeScanner:
# Build report
report = SyncReport()
# Track potentially moved files by checksum
files_by_checksum = {} # checksum -> (file_path, permalink)
# Find new and modified files
for file_path, checksum in current_files.items():
logger.debug(f"{file_path} ({checksum[:8]})")
if file_path not in db_file_state:
report.new.add(file_path)
# Track new file's checksum for move detection
files_by_checksum[checksum] = file_path
elif checksum != db_file_state[file_path].checksum:
report.modified.add(file_path)
report.checksums[file_path] = checksum
# Find deleted files - need to be deleted from db
# either not in current_files, or
# db row has no checksum
# Find deleted and moved files
for db_file_path, db_state in db_file_state.items():
if db_file_path not in current_files:
report.deleted.add(db_file_path)
# Check if this file was moved by looking for same checksum
if db_state.checksum in files_by_checksum:
new_path = files_by_checksum[db_state.checksum]
# Found a move - file exists at new path with same checksum
report.moves[db_file_path] = new_path
# Remove from new files since it's a move
report.new.remove(new_path)
else:
# Actually deleted
report.deleted.add(db_file_path)
# Log summary
logger.debug(f"Changes found: {report.total_changes}")
logger.debug(f" New: {len(report.new)}")
logger.debug(f" Modified: {len(report.modified)}")
logger.debug(f" Moved: {len(report.moves)}")
logger.debug(f" Deleted: {len(report.deleted)}")
if scan_result.errors:
+3 -1
View File
@@ -12,14 +12,16 @@ class SyncReport:
new: Files that exist on disk but not in database
modified: Files that exist in both but have different checksums
deleted: Files that exist in database but not on disk
moves: Files that have been moved from one location to another
checksums: Current checksums for files on disk
"""
new: Set[str] = field(default_factory=set)
modified: Set[str] = field(default_factory=set)
deleted: Set[str] = field(default_factory=set)
moves: Dict[str, str] = field(default_factory=dict) # old_path -> new_path
checksums: Dict[str, str] = field(default_factory=dict)
@property
def total_changes(self) -> int:
"""Total number of files that need attention."""
return len(self.new) + len(self.modified) + len(self.deleted)
return len(self.new) + len(self.modified) + len(self.deleted) + len(self.moves)