knowledge sync tests wip

src/basic_memory/services/sync/file_change_scanner.py

@@ -8,7 +8,7 @@ from loguru import logger
 from basic_memory.models import Document, Entity
 from basic_memory.repository.document_repository import DocumentRepository
 from basic_memory.repository.entity_repository import EntityRepository
-from basic_memory.services.sync.utils import FileState, SyncReport, ScanResult
+from basic_memory.services.sync.utils import DbState, SyncReport, ScanResult
 from basic_memory.utils.file_utils import compute_checksum
 
 
@@ -58,7 +58,7 @@ class FileChangeScanner:
                 checksum = await compute_checksum(content)
                 
                 if checksum:  # Only store valid checksums
-                    result.files[rel_path] = FileState(
+                    result.files[rel_path] = DbState(
                         path=rel_path,
                         checksum=checksum
                     )
@@ -81,14 +81,14 @@ class FileChangeScanner:
     async def find_changes(
             self,
             directory: Path,
-            db_records: Dict[str, FileState]
+            db_records: Dict[str, DbState]
     ) -> SyncReport:
         """
         Find changes between filesystem and database.
 
         Args:
             directory: Directory to check
-            get_records: Function to get database records
+            db_records: dict mapping file_path to DbState(path_id, checksum)
 
         Returns:
             SyncReport detailing changes
@@ -98,34 +98,34 @@ class FileChangeScanner:
         current_files = scan_result.files
         
         logger.debug("Current files from filesystem:")
-        for path, state in sorted(current_files.items()):
-            logger.debug(f"  {path} ({state.checksum[:8]})")
+        for file_path, state in sorted(current_files.items()):
+            logger.debug(f"  {file_path} ({state.checksum[:8]})")
 
         
         logger.debug("Files from database:")
-        for path, state in sorted(db_records.items()):
-            logger.debug(f"  {path} ({state.checksum[:8] if state.checksum else 'no checksum'})")
+        for file_path, state in sorted(db_records.items()):
+            logger.debug(f"  {file_path} ({state.checksum[:8] if state.checksum else 'no checksum'})")
 
         # Build report
         report = SyncReport()
         
         # Add current checksums for display
-        for path, state in current_files.items():
-            report.checksums[path] = state.checksum
+        for file_path, state in current_files.items():
+            report.checksums[file_path] = state.checksum
         
         # Find new and modified files
-        for path, curr_state in current_files.items():
-            if path not in db_records:
-                report.new.add(path)
-            elif curr_state.checksum != db_records[path].checksum:
-                report.modified.add(path)
+        for file_path, curr_state in current_files.items():
+            if file_path not in db_records:
+                report.new.add(file_path)
+            elif curr_state.checksum != db_records[file_path].checksum:
+                report.modified.add(file_path)
 
         # Find deleted files - need to be deleted from db
         # either not in current_files, or
         # db row has no checksum
-        for db_path, file_state in db_records.items():
-            if db_path not in current_files or file_state.checksum is None:
-                report.deleted.add(db_path)
+        for db_file_path, db_state in db_records.items():
+            if db_file_path not in current_files:
+                report.deleted.add(db_file_path)
 
         # Log summary
         logger.debug(f"Changes found: {report.total_changes}")
@@ -135,12 +135,12 @@ class FileChangeScanner:
         
         if scan_result.errors:
             logger.warning("Files skipped due to errors:")
-            for path, error in scan_result.errors.items():
-                logger.warning(f"  {path}: {error}")
+            for file_path, error in scan_result.errors.items():
+                logger.warning(f"  {file_path}: {error}")
 
         return report
 
-    async def get_db_file_paths(self, db_records: Sequence[Document | Entity]) -> Dict[str, FileState]:
+    async def get_db_file_paths(self, db_records: Sequence[Document | Entity]) -> Dict[str, DbState]:
         """Get file_path and checksums from database.
         Args:
             db_records: database records
@@ -148,7 +148,7 @@ class FileChangeScanner:
             Dict mapping file paths to FileState
             :param db_records: the data from the db 
         """
-        return {r.file_path: FileState(path=r.path_id, checksum=r.checksum) for r in db_records}
+        return {r.file_path: DbState(path=r.path_id, checksum=r.checksum) for r in db_records}
 
     async def find_document_changes(self, directory: Path) -> SyncReport:
         """Find changes in document directory."""

src/basic_memory/services/sync/sync_service.py

@@ -54,29 +54,31 @@ class SyncService:
         logger.info(f"Found {changes.total_changes} knowledge changes")
 
         # Handle deletions first
-        for path_id in changes.deleted:
-            logger.debug(f"Deleting entity: {path_id}")
-            await self.knowledge_sync_service.delete_entity_by_file_path(path_id)
+        # remove rows from db for files no longer present
+        for file_path in changes.deleted:
+            logger.debug(f"Deleting entity from db: {file_path}")
+            await self.knowledge_sync_service.delete_entity_by_file_path(file_path)
 
         # Parse files that need updating
         parsed_entities = {}
-        for path_id in [*changes.new, *changes.modified]:
-            entity = await self.knowledge_parser.parse_file(directory / path_id)
-            parsed_entities[path_id] = entity
+        for file_path in [*changes.new, *changes.modified]:
+            entity_markdown = await self.knowledge_parser.parse_file(directory / file_path)
+            parsed_entities[file_path] = entity_markdown
 
         # First pass: Create/update entities
-        for path_id, entity in parsed_entities.items():
-            if path_id in changes.new:
-                logger.debug(f"Creating new entity: {path_id}")
-                await self.knowledge_sync_service.create_entity_and_observations(entity)
+        for file_path, entity_markdown in parsed_entities.items():
+            if file_path in changes.new:
+                logger.debug(f"Creating new entity_markdown: {file_path}")
+                await self.knowledge_sync_service.create_entity_and_observations(entity_markdown)
             else:
-                logger.debug(f"Updating entity: {path_id}")
-                await self.knowledge_sync_service.update_entity_and_observations(entity)
+                path_id = entity_markdown.frontmatter.id
+                logger.debug(f"Updating entity_markdown: {path_id}")
+                await self.knowledge_sync_service.update_entity_and_observations(path_id, entity_markdown)
 
         # Second pass: Process relations
-        for path_id, entity in parsed_entities.items():
-            logger.debug(f"Updating relations for: {path_id}")
-            await self.knowledge_sync_service.update_entity_relations(entity, checksum=changes.checksums[path_id])
+        for file_path, entity_markdown in parsed_entities.items():
+            logger.debug(f"Updating relations for: {file_path}")
+            await self.knowledge_sync_service.update_entity_relations(entity_markdown, checksum=changes.checksums[file_path])
 
     async def sync(self, root_dir: Path) -> None:
         """Sync all files with database."""

src/basic_memory/services/sync/utils.py

@@ -5,7 +5,7 @@ from typing import Set, Dict, Optional
 
 
 @dataclass
-class FileState:
+class DbState:
     """State of a file including path and checksum info."""
     path: str
     checksum: str
@@ -14,7 +14,7 @@ class FileState:
 @dataclass 
 class ScanResult:
     """Result of scanning a directory."""
-    files: Dict[str, FileState] = field(default_factory=dict)
+    files: Dict[str, DbState] = field(default_factory=dict)
     errors: Dict[str, str] = field(default_factory=dict)  # path -> error message