From 4cb654a4fb7e5028e248fded194f26a6fb642721 Mon Sep 17 00:00:00 2001 From: phernandez Date: Wed, 8 Jan 2025 17:39:52 -0600 Subject: [PATCH] fix sync --- src/basic_memory/cli/commands/sync.py | 51 ++++----------- src/basic_memory/repository/repository.py | 7 +- src/basic_memory/sync/file_change_scanner.py | 68 +++++++++++--------- src/basic_memory/sync/utils.py | 14 ---- tests/cli/test_status.py | 10 +-- tests/sync/test_file_change_scanner.py | 22 +++---- 6 files changed, 72 insertions(+), 100 deletions(-) diff --git a/src/basic_memory/cli/commands/sync.py b/src/basic_memory/cli/commands/sync.py index 7101d524..43925836 100644 --- a/src/basic_memory/cli/commands/sync.py +++ b/src/basic_memory/cli/commands/sync.py @@ -20,18 +20,15 @@ from basic_memory.config import config from basic_memory.db import DatabaseType from basic_memory.markdown import KnowledgeParser from basic_memory.repository import ( - DocumentRepository, EntityRepository, ObservationRepository, RelationRepository, ) from basic_memory.repository.search_repository import SearchRepository from basic_memory.services import ( - DocumentService, EntityService, ObservationService, RelationService, - FileService, ) from basic_memory.services.search_service import SearchService from basic_memory.sync import SyncService, FileChangeScanner, KnowledgeSyncService @@ -54,17 +51,15 @@ async def get_sync_service(db_type=DatabaseType.FILESYSTEM): session_maker, ): # Initialize repositories - document_repository = DocumentRepository(session_maker) entity_repository = EntityRepository(session_maker) observation_repository = ObservationRepository(session_maker) relation_repository = RelationRepository(session_maker) search_repository = SearchRepository(session_maker) # Initialize scanner - file_change_scanner = FileChangeScanner(document_repository, entity_repository) + file_change_scanner = FileChangeScanner(entity_repository) # Initialize services - document_service = DocumentService(document_repository, config.documents_dir, FileService()) entity_service = EntityService(entity_repository) observation_service = ObservationService(observation_repository) relation_service = RelationService(relation_repository) @@ -73,13 +68,12 @@ async def get_sync_service(db_type=DatabaseType.FILESYSTEM): entity_service, observation_service, relation_service ) knowledge_parser = KnowledgeParser() - - search_service = SearchService(search_repository, document_service, entity_service) + + search_service = SearchService(search_repository, entity_service) # Create sync service sync_service = SyncService( scanner=file_change_scanner, - document_service=document_service, knowledge_sync_service=knowledge_sync_service, knowledge_parser=knowledge_parser, search_service=search_service, @@ -87,6 +81,7 @@ async def get_sync_service(db_type=DatabaseType.FILESYSTEM): return sync_service + def group_issues_by_directory(issues: List[ValidationIssue]) -> Dict[str, List[ValidationIssue]]: """Group validation issues by directory.""" grouped = defaultdict(list) @@ -143,18 +138,18 @@ def display_validation_errors(issues: List[ValidationIssue]): console.print() -def display_sync_summary(docs: SyncReport, knowledge: SyncReport): +def display_sync_summary(knowledge: SyncReport): """Display a one-line summary of sync changes.""" - total_changes = docs.total_changes + knowledge.total_changes + total_changes = knowledge.total_changes if total_changes == 0: console.print("[green]Everything up to date[/green]") return # Format as: "Synced X files (A new, B modified, C deleted)" changes = [] - new_count = len(docs.new) + len(knowledge.new) - mod_count = len(docs.modified) + len(knowledge.modified) - del_count = len(docs.deleted) + len(knowledge.deleted) + new_count = len(knowledge.new) + mod_count = len(knowledge.modified) + del_count = len(knowledge.deleted) if new_count: changes.append(f"[green]{new_count} new[/green]") @@ -166,32 +161,14 @@ def display_sync_summary(docs: SyncReport, knowledge: SyncReport): console.print(f"Synced {total_changes} files ({', '.join(changes)})") -def display_detailed_sync_results(docs: SyncReport, knowledge: SyncReport): +def display_detailed_sync_results(knowledge: SyncReport): """Display detailed sync results with trees.""" - if docs.total_changes == 0 and knowledge.total_changes == 0: + if knowledge.total_changes == 0: console.print("\n[green]Everything up to date[/green]") return console.print("\n[bold]Sync Results[/bold]") - if docs.total_changes > 0: - doc_tree = Tree("[bold]Documents[/bold]") - if docs.new: - created = doc_tree.add("[green]Created[/green]") - for path in sorted(docs.new): - checksum = docs.checksums.get(path, "") - created.add(f"[green]{path}[/green] ({checksum[:8]})") - if docs.modified: - modified = doc_tree.add("[yellow]Modified[/yellow]") - for path in sorted(docs.modified): - checksum = docs.checksums.get(path, "") - modified.add(f"[yellow]{path}[/yellow] ({checksum[:8]})") - if docs.deleted: - deleted = doc_tree.add("[red]Deleted[/red]") - for path in sorted(docs.deleted): - deleted.add(f"[red]{path}[/red]") - console.print(doc_tree) - if knowledge.total_changes > 0: knowledge_tree = Tree("[bold]Knowledge Files[/bold]") if knowledge.new: @@ -239,13 +216,13 @@ async def run_sync(verbose: bool = False): raise typer.Exit(1) # Sync - doc_changes, knowledge_changes = await sync_service.sync(config) + knowledge_changes = await sync_service.sync(config) # Display results if verbose: - display_detailed_sync_results(doc_changes, knowledge_changes) + display_detailed_sync_results(knowledge_changes) else: - display_sync_summary(doc_changes, knowledge_changes) + display_sync_summary(knowledge_changes) @app.command() diff --git a/src/basic_memory/repository/repository.py b/src/basic_memory/repository/repository.py index 211fbebe..d372c977 100644 --- a/src/basic_memory/repository/repository.py +++ b/src/basic_memory/repository/repository.py @@ -107,12 +107,15 @@ class Repository[T: Base]: await session.refresh(instance, relationships or []) logger.debug(f"Refreshed relationships: {relationships}") - async def find_all(self, skip: int = 0, limit: int = 100) -> Sequence[T]: + async def find_all(self, skip: int = 0, limit: Optional[int] = 0 ) -> Sequence[T]: """Fetch records from the database with pagination.""" logger.debug(f"Finding all {self.Model.__name__} (skip={skip}, limit={limit})") async with db.scoped_session(self.session_maker) as session: - query = select(self.Model).offset(skip).limit(limit).options(*self.get_load_options()) + + query = select(self.Model).offset(skip).options(*self.get_load_options()) + if limit: + query = query.limit(limit) result = await session.execute(query) diff --git a/src/basic_memory/sync/file_change_scanner.py b/src/basic_memory/sync/file_change_scanner.py index c32fc829..6445a1f4 100644 --- a/src/basic_memory/sync/file_change_scanner.py +++ b/src/basic_memory/sync/file_change_scanner.py @@ -1,15 +1,32 @@ """Service for detecting changes between filesystem and database.""" - +from dataclasses import dataclass, field from pathlib import Path -from typing import Dict, Sequence +from typing import Dict, Sequence, Any from loguru import logger from basic_memory.models import Entity from basic_memory.repository.entity_repository import EntityRepository -from basic_memory.sync.utils import DbState, SyncReport, ScanResult +from basic_memory.sync.utils import SyncReport from basic_memory.utils.file_utils import compute_checksum +@dataclass +class FileState: + """State of a file including file path, path_id and checksum info.""" + file_path: str + path_id: str + checksum: str + + +@dataclass +class ScanResult: + """Result of scanning a directory.""" + + # file_path -> checksum + files: Dict[str, str] = field(default_factory=dict) + # file_path -> error message + errors: Dict[str, str] = field(default_factory=dict) + class FileChangeScanner: """ @@ -53,8 +70,7 @@ class FileChangeScanner: checksum = await compute_checksum(content) if checksum: # Only store valid checksums - result.files[rel_path] = DbState(path=rel_path, checksum=checksum) - logger.debug(f"Found file: {rel_path} ({checksum[:8]})") + result.files[rel_path] = checksum else: result.errors[rel_path] = "Failed to compute checksum" @@ -69,13 +85,13 @@ class FileChangeScanner: return result - async def find_changes(self, directory: Path, db_records: Dict[str, DbState]) -> SyncReport: + async def find_changes(self, directory: Path, db_file_state: Dict[str, FileState]) -> SyncReport: """ Find changes between filesystem and database. Args: directory: Directory to check - db_records: dict mapping file_path to DbState(path_id, checksum) + db_file_state: dict mapping file_path to DbState(path_id, checksum) Returns: SyncReport detailing changes @@ -84,34 +100,24 @@ class FileChangeScanner: scan_result = await self.scan_directory(directory) current_files = scan_result.files - logger.debug("Current files from filesystem:") - for file_path, state in sorted(current_files.items()): - logger.debug(f" {file_path} ({state.checksum[:8]})") - - logger.debug("Files from database:") - for file_path, state in sorted(db_records.items()): - logger.debug( - f" {file_path} ({state.checksum[:8] if state.checksum else 'no checksum'})" - ) - # Build report report = SyncReport() - - # Add current checksums for display - for file_path, state in current_files.items(): - report.checksums[file_path] = state.checksum - + # Find new and modified files - for file_path, curr_state in current_files.items(): - if file_path not in db_records: + for file_path, checksum in current_files.items(): + logger.debug(f"{file_path} ({checksum[:8]})") + + if file_path not in db_file_state: report.new.add(file_path) - elif curr_state.checksum != db_records[file_path].checksum: + elif checksum != db_file_state[file_path].checksum: report.modified.add(file_path) + report.checksums[file_path] = checksum + # Find deleted files - need to be deleted from db # either not in current_files, or # db row has no checksum - for db_file_path, db_state in db_records.items(): + for db_file_path, db_state in db_file_state.items(): if db_file_path not in current_files: report.deleted.add(db_file_path) @@ -128,9 +134,9 @@ class FileChangeScanner: return report - async def get_db_file_paths( + async def get_db_file_state( self, db_records: Sequence[Entity] - ) -> Dict[str, DbState]: + ) -> Dict[str, FileState]: """Get file_path and checksums from database. Args: db_records: database records @@ -138,9 +144,9 @@ class FileChangeScanner: Dict mapping file paths to FileState :param db_records: the data from the db """ - return {r.file_path: DbState(path=r.path_id, checksum=r.checksum) for r in db_records} + return {r.file_path: FileState(file_path=r.file_path, path_id=r.path_id, checksum=r.checksum) for r in db_records} async def find_knowledge_changes(self, directory: Path) -> SyncReport: """Find changes in knowledge directory.""" - db_records = await self.get_db_file_paths(await self.entity_repository.find_all()) - return await self.find_changes(directory=directory, db_records=db_records) + db_file_state = await self.get_db_file_state(await self.entity_repository.find_all()) + return await self.find_changes(directory=directory, db_file_state=db_file_state) diff --git a/src/basic_memory/sync/utils.py b/src/basic_memory/sync/utils.py index df902b39..f25bc205 100644 --- a/src/basic_memory/sync/utils.py +++ b/src/basic_memory/sync/utils.py @@ -4,20 +4,6 @@ from dataclasses import dataclass, field from typing import Set, Dict, Optional -@dataclass -class DbState: - """State of a file including path and checksum info.""" - path: str - checksum: str - - -@dataclass -class ScanResult: - """Result of scanning a directory.""" - files: Dict[str, DbState] = field(default_factory=dict) - errors: Dict[str, str] = field(default_factory=dict) # path -> error message - - @dataclass class SyncReport: """Report of file changes found compared to database state. diff --git a/tests/cli/test_status.py b/tests/cli/test_status.py index 1eb1860a..3735aaf3 100644 --- a/tests/cli/test_status.py +++ b/tests/cli/test_status.py @@ -7,7 +7,7 @@ import pytest_asyncio from rich.console import Console from basic_memory.cli.commands.status import display_changes, run_status -from basic_memory.sync.utils import SyncReport, DbState +from basic_memory.sync.utils import SyncReport, FileState from basic_memory.utils.file_utils import compute_checksum @@ -36,8 +36,8 @@ async def test_display_compact_changes(console): modified={"docs/mod.md"}, deleted={"old/deleted.md"}, moved={ - "new/location.md": DbState( - path="new/location.md", checksum="abc123", moved_from="old/location.md" + "new/location.md": FileState( + path_id="new/location.md", checksum="abc123", moved_from="old/location.md" ) }, ) @@ -60,8 +60,8 @@ async def test_display_verbose_changes(console): modified={"docs/mod.md"}, deleted={"old/deleted.md"}, moved={ - "new/location.md": DbState( - path="new/location.md", + "new/location.md": FileState( + path_id="new/location.md", checksum="abc123def", # 8 chars for display moved_from="old/location.md", ) diff --git a/tests/sync/test_file_change_scanner.py b/tests/sync/test_file_change_scanner.py index 51dbfd25..9b419c63 100644 --- a/tests/sync/test_file_change_scanner.py +++ b/tests/sync/test_file_change_scanner.py @@ -6,7 +6,7 @@ import pytest from basic_memory.models import Entity from basic_memory.sync import FileChangeScanner -from basic_memory.sync.utils import DbState +from basic_memory.sync.utils import FileState from basic_memory.utils.file_utils import compute_checksum @@ -45,8 +45,8 @@ async def test_scan_with_mixed_files(file_change_scanner: FileChangeScanner, tem assert len(result.errors) == 0 # Verify FileState objects - assert isinstance(result.files["doc.md"], DbState) - assert result.files["doc.md"].path == "doc.md" + assert isinstance(result.files["doc.md"], FileState) + assert result.files["doc.md"].path_id == "doc.md" assert result.files["doc.md"].checksum is not None @@ -73,9 +73,9 @@ async def test_detect_new_files( await create_test_file(temp_dir / "new.md") # Empty DB state - db_records = await file_change_scanner.get_db_file_paths([]) + db_records = await file_change_scanner.get_db_file_state([]) - changes = await file_change_scanner.find_changes(directory=temp_dir, db_records=db_records) + changes = await file_change_scanner.find_changes(directory=temp_dir, db_file_state=db_records) assert len(changes.new) == 1 assert "new.md" in changes.new @@ -90,12 +90,12 @@ async def test_detect_modified_file(file_change_scanner: FileChangeScanner, temp # Create DB state with original checksum original_checksum = await compute_checksum(content) - db_records = {path: DbState(path=path, checksum=original_checksum)} + db_records = {path: FileState(path_id=path, checksum=original_checksum)} # Modify file await create_test_file(temp_dir / path, "modified") - changes = await file_change_scanner.find_changes(directory=temp_dir, db_records=db_records) + changes = await file_change_scanner.find_changes(directory=temp_dir, db_file_state=db_records) assert len(changes.modified) == 1 assert path in changes.modified @@ -107,9 +107,9 @@ async def test_detect_deleted_files(file_change_scanner: FileChangeScanner, temp path = "deleted.md" # Create DB state with file that doesn't exist - db_records = {path: DbState(path=path, checksum="any-checksum")} + db_records = {path: FileState(path_id=path, checksum="any-checksum")} - changes = await file_change_scanner.find_changes(directory=temp_dir, db_records=db_records) + changes = await file_change_scanner.find_changes(directory=temp_dir, db_file_state=db_records) assert len(changes.deleted) == 1 assert path in changes.deleted @@ -121,7 +121,7 @@ async def test_get_db_state_entities(file_change_scanner: FileChangeScanner): """Test converting entity records to file states.""" entity = Entity(path_id="concept/test", file_path="concept/test.md", checksum="test-checksum") - db_records = await file_change_scanner.get_db_file_paths([entity]) + db_records = await file_change_scanner.get_db_file_state([entity]) assert len(db_records) == 1 assert "concept/test.md" in db_records @@ -134,7 +134,7 @@ async def test_empty_directory(file_change_scanner: FileChangeScanner, temp_dir: """Test handling empty/nonexistent directory.""" nonexistent = temp_dir / "nonexistent" - changes = await file_change_scanner.find_changes(directory=nonexistent, db_records={}) + changes = await file_change_scanner.find_changes(directory=nonexistent, db_file_state={}) assert changes.total_changes == 0 assert not changes.new