mirror of
https://github.com/basicmachines-co/basic-memory
synced 2026-06-21 13:47:35 +00:00
detect file moves in sync
This commit is contained in:
@@ -67,8 +67,19 @@ def group_changes_by_directory(changes: SyncReport) -> Dict[str, Dict[str, int]]
|
||||
]:
|
||||
for path in paths:
|
||||
dir_name = path.split("/", 1)[0]
|
||||
by_dir.setdefault(dir_name, {"new": 0, "modified": 0, "deleted": 0})
|
||||
by_dir.setdefault(dir_name, {"new": 0, "modified": 0, "deleted": 0, "moved": 0})
|
||||
by_dir[dir_name][change_type] += 1
|
||||
|
||||
# Handle moves - count in both source and destination directories
|
||||
for old_path, new_path in changes.moves.items():
|
||||
old_dir = old_path.split("/", 1)[0]
|
||||
new_dir = new_path.split("/", 1)[0]
|
||||
by_dir.setdefault(old_dir, {"new": 0, "modified": 0, "deleted": 0, "moved": 0})
|
||||
by_dir.setdefault(new_dir, {"new": 0, "modified": 0, "deleted": 0, "moved": 0})
|
||||
by_dir[old_dir]["moved"] += 1
|
||||
if old_dir != new_dir:
|
||||
by_dir[new_dir]["moved"] += 1
|
||||
|
||||
return by_dir
|
||||
|
||||
|
||||
@@ -79,6 +90,8 @@ def build_directory_summary(counts: Dict[str, int]) -> str:
|
||||
parts.append(f"[green]+{counts['new']} new[/green]")
|
||||
if counts["modified"]:
|
||||
parts.append(f"[yellow]~{counts['modified']} modified[/yellow]")
|
||||
if counts["moved"]:
|
||||
parts.append(f"[blue]↔{counts['moved']} moved[/blue]")
|
||||
if counts["deleted"]:
|
||||
parts.append(f"[red]-{counts['deleted']} deleted[/red]")
|
||||
return " ".join(parts)
|
||||
@@ -101,6 +114,10 @@ def display_changes(title: str, changes: SyncReport, verbose: bool = False):
|
||||
if changes.modified:
|
||||
mod_branch = tree.add("[yellow]Modified[/yellow]")
|
||||
add_files_to_tree(mod_branch, changes.modified, "yellow", changes.checksums)
|
||||
if changes.moves:
|
||||
move_branch = tree.add("[blue]Moved[/blue]")
|
||||
for old_path, new_path in sorted(changes.moves.items()):
|
||||
move_branch.add(f"[blue]{old_path}[/blue] → [blue]{new_path}[/blue]")
|
||||
if changes.deleted:
|
||||
del_branch = tree.add("[red]Deleted[/red]")
|
||||
add_files_to_tree(del_branch, changes.deleted, "red")
|
||||
|
||||
@@ -25,6 +25,7 @@ from basic_memory.repository import (
|
||||
RelationRepository,
|
||||
)
|
||||
from basic_memory.repository.search_repository import SearchRepository
|
||||
from basic_memory.services.link_resolver import LinkResolver
|
||||
from basic_memory.services.search_service import SearchService
|
||||
from basic_memory.sync import SyncService, FileChangeScanner, EntitySyncService
|
||||
from basic_memory.sync.utils import SyncReport
|
||||
@@ -51,21 +52,28 @@ async def get_sync_service(db_type=DatabaseType.FILESYSTEM):
|
||||
relation_repository = RelationRepository(session_maker)
|
||||
search_repository = SearchRepository(session_maker)
|
||||
|
||||
# Initialize services
|
||||
search_service = SearchService(search_repository, entity_repository)
|
||||
link_resolver = LinkResolver(entity_repository, search_service)
|
||||
|
||||
# Initialize scanner
|
||||
file_change_scanner = FileChangeScanner(entity_repository)
|
||||
|
||||
# Initialize services
|
||||
knowledge_sync_service = EntitySyncService(
|
||||
entity_repository, observation_repository, relation_repository
|
||||
entity_repository,
|
||||
observation_repository,
|
||||
relation_repository,
|
||||
link_resolver
|
||||
)
|
||||
entity_parser = EntityParser(config.home)
|
||||
search_service = SearchService(search_repository, entity_repository)
|
||||
|
||||
# Create sync service
|
||||
sync_service = SyncService(
|
||||
scanner=file_change_scanner,
|
||||
entity_sync_service=knowledge_sync_service,
|
||||
entity_parser=entity_parser,
|
||||
entity_repository=entity_repository,
|
||||
search_service=search_service,
|
||||
)
|
||||
|
||||
@@ -135,16 +143,19 @@ def display_sync_summary(knowledge: SyncReport):
|
||||
console.print("[green]Everything up to date[/green]")
|
||||
return
|
||||
|
||||
# Format as: "Synced X files (A new, B modified, C deleted)"
|
||||
# Format as: "Synced X files (A new, B modified, C moved, D deleted)"
|
||||
changes = []
|
||||
new_count = len(knowledge.new)
|
||||
mod_count = len(knowledge.modified)
|
||||
move_count = len(knowledge.moves)
|
||||
del_count = len(knowledge.deleted)
|
||||
|
||||
if new_count:
|
||||
changes.append(f"[green]{new_count} new[/green]")
|
||||
if mod_count:
|
||||
changes.append(f"[yellow]{mod_count} modified[/yellow]")
|
||||
if move_count:
|
||||
changes.append(f"[blue]{move_count} moved[/blue]")
|
||||
if del_count:
|
||||
changes.append(f"[red]{del_count} deleted[/red]")
|
||||
|
||||
@@ -171,13 +182,17 @@ def display_detailed_sync_results(knowledge: SyncReport):
|
||||
for path in sorted(knowledge.modified):
|
||||
checksum = knowledge.checksums.get(path, "")
|
||||
modified.add(f"[yellow]{path}[/yellow] ({checksum[:8]})")
|
||||
if knowledge.moves:
|
||||
moved = knowledge_tree.add("[blue]Moved[/blue]")
|
||||
for old_path, new_path in sorted(knowledge.moves.items()):
|
||||
checksum = knowledge.checksums.get(new_path, "")
|
||||
moved.add(f"[blue]{old_path}[/blue] → [blue]{new_path}[/blue] ({checksum[:8]})")
|
||||
if knowledge.deleted:
|
||||
deleted = knowledge_tree.add("[red]Deleted[/red]")
|
||||
for path in sorted(knowledge.deleted):
|
||||
deleted.add(f"[red]{path}[/red]")
|
||||
console.print(knowledge_tree)
|
||||
|
||||
|
||||
async def validate_knowledge_files(
|
||||
sync_service: SyncService, directory: Path
|
||||
) -> List[ValidationIssue]:
|
||||
|
||||
@@ -62,7 +62,6 @@ class LinkResolver():
|
||||
return await self.entity_repository.get_by_permalink(best_match.permalink)
|
||||
|
||||
|
||||
# TODO replace with search ranking
|
||||
def _normalize_link_text(self, link_text: str) -> Tuple[str, Optional[str]]:
|
||||
"""Normalize link text and extract alias if present.
|
||||
|
||||
|
||||
@@ -87,18 +87,9 @@ class FileChangeScanner:
|
||||
return result
|
||||
|
||||
async def find_changes(
|
||||
self, directory: Path, db_file_state: Dict[str, FileState]
|
||||
self, directory: Path, db_file_state: Dict[str, FileState]
|
||||
) -> SyncReport:
|
||||
"""
|
||||
Find changes between filesystem and database.
|
||||
|
||||
Args:
|
||||
directory: Directory to check
|
||||
db_file_state: dict mapping file_path to DbState(permalink, checksum)
|
||||
|
||||
Returns:
|
||||
SyncReport detailing changes
|
||||
"""
|
||||
"""Find changes between filesystem and database."""
|
||||
# Get current files and checksums
|
||||
scan_result = await self.scan_directory(directory)
|
||||
current_files = scan_result.files
|
||||
@@ -106,28 +97,41 @@ class FileChangeScanner:
|
||||
# Build report
|
||||
report = SyncReport()
|
||||
|
||||
# Track potentially moved files by checksum
|
||||
files_by_checksum = {} # checksum -> (file_path, permalink)
|
||||
|
||||
# Find new and modified files
|
||||
for file_path, checksum in current_files.items():
|
||||
logger.debug(f"{file_path} ({checksum[:8]})")
|
||||
|
||||
if file_path not in db_file_state:
|
||||
report.new.add(file_path)
|
||||
# Track new file's checksum for move detection
|
||||
files_by_checksum[checksum] = file_path
|
||||
elif checksum != db_file_state[file_path].checksum:
|
||||
report.modified.add(file_path)
|
||||
|
||||
report.checksums[file_path] = checksum
|
||||
|
||||
# Find deleted files - need to be deleted from db
|
||||
# either not in current_files, or
|
||||
# db row has no checksum
|
||||
# Find deleted and moved files
|
||||
for db_file_path, db_state in db_file_state.items():
|
||||
if db_file_path not in current_files:
|
||||
report.deleted.add(db_file_path)
|
||||
# Check if this file was moved by looking for same checksum
|
||||
if db_state.checksum in files_by_checksum:
|
||||
new_path = files_by_checksum[db_state.checksum]
|
||||
# Found a move - file exists at new path with same checksum
|
||||
report.moves[db_file_path] = new_path
|
||||
# Remove from new files since it's a move
|
||||
report.new.remove(new_path)
|
||||
else:
|
||||
# Actually deleted
|
||||
report.deleted.add(db_file_path)
|
||||
|
||||
# Log summary
|
||||
logger.debug(f"Changes found: {report.total_changes}")
|
||||
logger.debug(f" New: {len(report.new)}")
|
||||
logger.debug(f" Modified: {len(report.modified)}")
|
||||
logger.debug(f" Moved: {len(report.moves)}")
|
||||
logger.debug(f" Deleted: {len(report.deleted)}")
|
||||
|
||||
if scan_result.errors:
|
||||
|
||||
@@ -12,14 +12,16 @@ class SyncReport:
|
||||
new: Files that exist on disk but not in database
|
||||
modified: Files that exist in both but have different checksums
|
||||
deleted: Files that exist in database but not on disk
|
||||
moves: Files that have been moved from one location to another
|
||||
checksums: Current checksums for files on disk
|
||||
"""
|
||||
new: Set[str] = field(default_factory=set)
|
||||
modified: Set[str] = field(default_factory=set)
|
||||
deleted: Set[str] = field(default_factory=set)
|
||||
moves: Dict[str, str] = field(default_factory=dict) # old_path -> new_path
|
||||
checksums: Dict[str, str] = field(default_factory=dict)
|
||||
|
||||
@property
|
||||
def total_changes(self) -> int:
|
||||
"""Total number of files that need attention."""
|
||||
return len(self.new) + len(self.modified) + len(self.deleted)
|
||||
return len(self.new) + len(self.modified) + len(self.deleted) + len(self.moves)
|
||||
|
||||
Reference in New Issue
Block a user