fix sync

src/basic_memory/cli/commands/sync.py

@@ -20,18 +20,15 @@ from basic_memory.config import config
 from basic_memory.db import DatabaseType
 from basic_memory.markdown import KnowledgeParser
 from basic_memory.repository import (
-    DocumentRepository,
     EntityRepository,
     ObservationRepository,
     RelationRepository,
 )
 from basic_memory.repository.search_repository import SearchRepository
 from basic_memory.services import (
-    DocumentService,
     EntityService,
     ObservationService,
     RelationService,
-    FileService,
 )
 from basic_memory.services.search_service import SearchService
 from basic_memory.sync import SyncService, FileChangeScanner, KnowledgeSyncService
@@ -54,17 +51,15 @@ async def get_sync_service(db_type=DatabaseType.FILESYSTEM):
         session_maker,
     ):
         # Initialize repositories
-        document_repository = DocumentRepository(session_maker)
         entity_repository = EntityRepository(session_maker)
         observation_repository = ObservationRepository(session_maker)
         relation_repository = RelationRepository(session_maker)
         search_repository = SearchRepository(session_maker)
 
         # Initialize scanner
-        file_change_scanner = FileChangeScanner(document_repository, entity_repository)
+        file_change_scanner = FileChangeScanner(entity_repository)
 
         # Initialize services
-        document_service = DocumentService(document_repository, config.documents_dir, FileService())
         entity_service = EntityService(entity_repository)
         observation_service = ObservationService(observation_repository)
         relation_service = RelationService(relation_repository)
@@ -73,13 +68,12 @@ async def get_sync_service(db_type=DatabaseType.FILESYSTEM):
             entity_service, observation_service, relation_service
         )
         knowledge_parser = KnowledgeParser()
-        
-        search_service = SearchService(search_repository, document_service, entity_service)
+
+        search_service = SearchService(search_repository, entity_service)
 
         # Create sync service
         sync_service = SyncService(
             scanner=file_change_scanner,
-            document_service=document_service,
             knowledge_sync_service=knowledge_sync_service,
             knowledge_parser=knowledge_parser,
             search_service=search_service,
@@ -87,6 +81,7 @@ async def get_sync_service(db_type=DatabaseType.FILESYSTEM):
 
         return sync_service
 
+
 def group_issues_by_directory(issues: List[ValidationIssue]) -> Dict[str, List[ValidationIssue]]:
     """Group validation issues by directory."""
     grouped = defaultdict(list)
@@ -143,18 +138,18 @@ def display_validation_errors(issues: List[ValidationIssue]):
     console.print()
 
 
-def display_sync_summary(docs: SyncReport, knowledge: SyncReport):
+def display_sync_summary(knowledge: SyncReport):
     """Display a one-line summary of sync changes."""
-    total_changes = docs.total_changes + knowledge.total_changes
+    total_changes = knowledge.total_changes
     if total_changes == 0:
         console.print("[green]Everything up to date[/green]")
         return
 
     # Format as: "Synced X files (A new, B modified, C deleted)"
     changes = []
-    new_count = len(docs.new) + len(knowledge.new)
-    mod_count = len(docs.modified) + len(knowledge.modified)
-    del_count = len(docs.deleted) + len(knowledge.deleted)
+    new_count =  len(knowledge.new)
+    mod_count =  len(knowledge.modified)
+    del_count =  len(knowledge.deleted)
 
     if new_count:
         changes.append(f"[green]{new_count} new[/green]")
@@ -166,32 +161,14 @@ def display_sync_summary(docs: SyncReport, knowledge: SyncReport):
     console.print(f"Synced {total_changes} files ({', '.join(changes)})")
 
 
-def display_detailed_sync_results(docs: SyncReport, knowledge: SyncReport):
+def display_detailed_sync_results(knowledge: SyncReport):
     """Display detailed sync results with trees."""
-    if docs.total_changes == 0 and knowledge.total_changes == 0:
+    if knowledge.total_changes == 0:
         console.print("\n[green]Everything up to date[/green]")
         return
 
     console.print("\n[bold]Sync Results[/bold]")
 
-    if docs.total_changes > 0:
-        doc_tree = Tree("[bold]Documents[/bold]")
-        if docs.new:
-            created = doc_tree.add("[green]Created[/green]")
-            for path in sorted(docs.new):
-                checksum = docs.checksums.get(path, "")
-                created.add(f"[green]{path}[/green] ({checksum[:8]})")
-        if docs.modified:
-            modified = doc_tree.add("[yellow]Modified[/yellow]")
-            for path in sorted(docs.modified):
-                checksum = docs.checksums.get(path, "")
-                modified.add(f"[yellow]{path}[/yellow] ({checksum[:8]})")
-        if docs.deleted:
-            deleted = doc_tree.add("[red]Deleted[/red]")
-            for path in sorted(docs.deleted):
-                deleted.add(f"[red]{path}[/red]")
-        console.print(doc_tree)
-
     if knowledge.total_changes > 0:
         knowledge_tree = Tree("[bold]Knowledge Files[/bold]")
         if knowledge.new:
@@ -239,13 +216,13 @@ async def run_sync(verbose: bool = False):
         raise typer.Exit(1)
 
     # Sync
-    doc_changes, knowledge_changes = await sync_service.sync(config)
+    knowledge_changes = await sync_service.sync(config)
 
     # Display results
     if verbose:
-        display_detailed_sync_results(doc_changes, knowledge_changes)
+        display_detailed_sync_results(knowledge_changes)
     else:
-        display_sync_summary(doc_changes, knowledge_changes)
+        display_sync_summary(knowledge_changes)
 
 
 @app.command()

src/basic_memory/repository/repository.py

@@ -107,12 +107,15 @@ class Repository[T: Base]:
             await session.refresh(instance, relationships or [])
             logger.debug(f"Refreshed relationships: {relationships}")
 
-    async def find_all(self, skip: int = 0, limit: int = 100) -> Sequence[T]:
+    async def find_all(self, skip: int = 0, limit: Optional[int] = 0 ) -> Sequence[T]:
         """Fetch records from the database with pagination."""
         logger.debug(f"Finding all {self.Model.__name__} (skip={skip}, limit={limit})")
 
         async with db.scoped_session(self.session_maker) as session:
-            query = select(self.Model).offset(skip).limit(limit).options(*self.get_load_options())
+            
+            query = select(self.Model).offset(skip).options(*self.get_load_options())
+            if limit:
+                query = query.limit(limit)
 
             result = await session.execute(query)
 

src/basic_memory/sync/file_change_scanner.py

@@ -1,15 +1,32 @@
 """Service for detecting changes between filesystem and database."""
-
+from dataclasses import dataclass, field
 from pathlib import Path
-from typing import Dict, Sequence
+from typing import Dict, Sequence, Any
 
 from loguru import logger
 
 from basic_memory.models import Entity
 from basic_memory.repository.entity_repository import EntityRepository
-from basic_memory.sync.utils import DbState, SyncReport, ScanResult
+from basic_memory.sync.utils import SyncReport
 from basic_memory.utils.file_utils import compute_checksum
 
+@dataclass
+class FileState:
+    """State of a file including file path, path_id and checksum info."""
+    file_path: str
+    path_id: str
+    checksum: str
+
+
+@dataclass 
+class ScanResult:
+    """Result of scanning a directory."""
+    
+    # file_path -> checksum
+    files: Dict[str, str] = field(default_factory=dict)
+    # file_path -> error message
+    errors: Dict[str, str] = field(default_factory=dict)  
+
 
 class FileChangeScanner:
     """
@@ -53,8 +70,7 @@ class FileChangeScanner:
                 checksum = await compute_checksum(content)
 
                 if checksum:  # Only store valid checksums
-                    result.files[rel_path] = DbState(path=rel_path, checksum=checksum)
-                    logger.debug(f"Found file: {rel_path} ({checksum[:8]})")
+                    result.files[rel_path] = checksum 
                 else:
                     result.errors[rel_path] = "Failed to compute checksum"
 
@@ -69,13 +85,13 @@ class FileChangeScanner:
 
         return result
 
-    async def find_changes(self, directory: Path, db_records: Dict[str, DbState]) -> SyncReport:
+    async def find_changes(self, directory: Path, db_file_state: Dict[str, FileState]) -> SyncReport:
         """
         Find changes between filesystem and database.
 
         Args:
             directory: Directory to check
-            db_records: dict mapping file_path to DbState(path_id, checksum)
+            db_file_state: dict mapping file_path to DbState(path_id, checksum)
 
         Returns:
             SyncReport detailing changes
@@ -84,34 +100,24 @@ class FileChangeScanner:
         scan_result = await self.scan_directory(directory)
         current_files = scan_result.files
 
-        logger.debug("Current files from filesystem:")
-        for file_path, state in sorted(current_files.items()):
-            logger.debug(f"  {file_path} ({state.checksum[:8]})")
-
-        logger.debug("Files from database:")
-        for file_path, state in sorted(db_records.items()):
-            logger.debug(
-                f"  {file_path} ({state.checksum[:8] if state.checksum else 'no checksum'})"
-            )
-
         # Build report
         report = SyncReport()
-
-        # Add current checksums for display
-        for file_path, state in current_files.items():
-            report.checksums[file_path] = state.checksum
-
+            
         # Find new and modified files
-        for file_path, curr_state in current_files.items():
-            if file_path not in db_records:
+        for file_path, checksum in current_files.items():
+            logger.debug(f"{file_path} ({checksum[:8]})")
+            
+            if file_path not in db_file_state:
                 report.new.add(file_path)
-            elif curr_state.checksum != db_records[file_path].checksum:
+            elif checksum != db_file_state[file_path].checksum:
                 report.modified.add(file_path)
 
+            report.checksums[file_path] = checksum
+
         # Find deleted files - need to be deleted from db
         # either not in current_files, or
         # db row has no checksum
-        for db_file_path, db_state in db_records.items():
+        for db_file_path, db_state in db_file_state.items():
             if db_file_path not in current_files:
                 report.deleted.add(db_file_path)
 
@@ -128,9 +134,9 @@ class FileChangeScanner:
 
         return report
 
-    async def get_db_file_paths(
+    async def get_db_file_state(
         self, db_records: Sequence[Entity]
-    ) -> Dict[str, DbState]:
+    ) -> Dict[str, FileState]:
         """Get file_path and checksums from database.
         Args:
             db_records: database records
@@ -138,9 +144,9 @@ class FileChangeScanner:
             Dict mapping file paths to FileState
             :param db_records: the data from the db
         """
-        return {r.file_path: DbState(path=r.path_id, checksum=r.checksum) for r in db_records}
+        return {r.file_path: FileState(file_path=r.file_path, path_id=r.path_id, checksum=r.checksum) for r in db_records}
 
     async def find_knowledge_changes(self, directory: Path) -> SyncReport:
         """Find changes in knowledge directory."""
-        db_records = await self.get_db_file_paths(await self.entity_repository.find_all())
-        return await self.find_changes(directory=directory, db_records=db_records)
+        db_file_state = await self.get_db_file_state(await self.entity_repository.find_all())
+        return await self.find_changes(directory=directory, db_file_state=db_file_state)

src/basic_memory/sync/utils.py

@@ -4,20 +4,6 @@ from dataclasses import dataclass, field
 from typing import Set, Dict, Optional
 
 
-@dataclass
-class DbState:
-    """State of a file including path and checksum info."""
-    path: str
-    checksum: str
-
-
-@dataclass 
-class ScanResult:
-    """Result of scanning a directory."""
-    files: Dict[str, DbState] = field(default_factory=dict)
-    errors: Dict[str, str] = field(default_factory=dict)  # path -> error message
-
-
 @dataclass
 class SyncReport:
     """Report of file changes found compared to database state.