diff --git a/src/basic_memory/cli/commands/status.py b/src/basic_memory/cli/commands/status.py index 90d42d9e..4176114d 100644 --- a/src/basic_memory/cli/commands/status.py +++ b/src/basic_memory/cli/commands/status.py @@ -12,7 +12,7 @@ from basic_memory import db from basic_memory.cli.app import app from basic_memory.config import config from basic_memory.db import DatabaseType -from basic_memory.repository import DocumentRepository +from basic_memory.repository import DocumentRepository, EntityRepository from basic_memory.services import FileSyncService from basic_memory.services.file_sync_service import SyncReport @@ -26,7 +26,8 @@ async def get_sync_service(db_type=DatabaseType.FILESYSTEM) -> FileSyncService: session_maker, ): document_repository = DocumentRepository(session_maker) - sync_service = FileSyncService(document_repository) + entity_repository = EntityRepository(session_maker) + sync_service = FileSyncService(document_repository, entity_repository) return sync_service @@ -131,13 +132,11 @@ async def run_status(sync_service: FileSyncService, verbose: bool = False): """Check sync status of files vs database.""" # Check knowledge/ directory - files = await sync_service.scan_files(config.knowledge_dir) - knowledge_changes = await sync_service.find_changes(files, config.knowledge_dir) + knowledge_changes = await sync_service.find_knowledge_changes(config.knowledge_dir) display_changes("Knowledge Files", knowledge_changes) # Check documents/ directory - files = await sync_service.scan_files(config.documents_dir) - document_changes = await sync_service.find_changes(files, config.documents_dir) + document_changes = await sync_service.find_document_changes(config.documents_dir) display_changes("Documents", document_changes) diff --git a/src/basic_memory/services/file_sync_service.py b/src/basic_memory/services/file_sync_service.py index a20da2be..e7ebd2e3 100644 --- a/src/basic_memory/services/file_sync_service.py +++ b/src/basic_memory/services/file_sync_service.py @@ -1,13 +1,13 @@ """Service for syncing files with the database.""" -import hashlib from dataclasses import dataclass from pathlib import Path -from typing import Set +from typing import Set, Dict, Protocol, TypeVar, Generic from loguru import logger from basic_memory.repository.document_repository import DocumentRepository +from basic_memory.repository.entity_repository import EntityRepository from basic_memory.utils.file_utils import compute_checksum @@ -22,192 +22,111 @@ class SyncReport: def total_changes(self) -> int: return len(self.new) + len(self.modified) + len(self.deleted) - def __str__(self) -> str: - return ( - f"Changes detected:\n" - f" New files: {len(self.new)}\n" - f" Modified: {len(self.modified)}\n" - f" Deleted: {len(self.deleted)}" - ) + +class DbRecord(Protocol): + """Protocol for database records with path and checksum.""" + @property + def path(self) -> str: ... + @property + def checksum(self) -> str: ... -class SyncError(Exception): - """Raised when sync operations fail.""" - pass +T = TypeVar('T', bound=DbRecord) class FileSyncService: - """Service for keeping files and database in sync.""" + """ + Service for keeping files and database in sync. + The filesystem is the source of truth. + """ - def __init__(self, document_repository: DocumentRepository): - self.repository = document_repository + def __init__( + self, + document_repository: DocumentRepository, + entity_repository: EntityRepository + ): + self.document_repository = document_repository + self.entity_repository = entity_repository - async def scan_files(self, directory: Path) -> dict[str, str]: + async def scan_directory(self, directory: Path) -> Dict[str, str]: """ Scan directory for markdown files and their checksums. - Only processes .md files, logs and skips other files. + Only processes .md files, logs and skips others. Args: - directory: Root directory to scan + directory: Directory to scan Returns: - Dict mapping paths to checksums - - Raises: - SyncError: If any markdown file cannot be read + Dict mapping relative paths to checksums """ logger.debug(f"Scanning directory: {directory}") files = {} - errors = [] - for path in directory.rglob('*'): - if path.is_file(): - if not path.name.endswith('.md'): - logger.debug(f"Skipping non-markdown file: '{path}'") - continue + for path in directory.rglob("*"): + if not path.is_file() or not path.name.endswith(".md"): + if path.is_file(): + logger.debug(f"Skipping non-markdown file: {path}") + continue - try: - content = path.read_text() - checksum = await compute_checksum(content) - # Store path relative to root directory - rel_path = str(path.relative_to(directory)) - files[rel_path] = checksum - except Exception as e: - errors.append(f"Failed to read {path}: {e}") - logger.debug(f"Scanned file: {path} checksum: {checksum}") - if errors: - raise SyncError("Failed to read files:\n" + "\n".join(errors)) + try: + content = path.read_text() + checksum = await compute_checksum(content) + rel_path = str(path.relative_to(directory)) + files[rel_path] = checksum + except Exception as e: + logger.error(f"Failed to read {path}: {e}") - logger.debug(f"Found {len(files)} markdown files in {directory}") + logger.debug(f"Found {len(files)} markdown files") return files - async def find_changes(self, file_system_files: dict[str, str], directory: Path) -> SyncReport: + async def find_changes( + self, + directory: Path, + get_records: callable, + get_path: callable = lambda x: x.path + ) -> SyncReport: """ Find changes between filesystem and database. - Only considers files that belong to the specified directory. Args: - file_system_files: Dict mapping paths to checksums - directory: Root directory being scanned (knowledge/ or documents/) + directory: Directory to check + get_records: Function to get database records + get_path: Function to get path from record (defaults to .path) Returns: SyncReport detailing changes """ - logger.debug(f"Finding changes in {directory}") - - # Get all documents from DB - db_documents = await self.repository.find_all() - - # Filter DB files to only those in this directory + # Get current files and checksums + current_files = await self.scan_directory(directory) + + # Get database records + db_records = await get_records() db_files = { - doc.path: doc.checksum - for doc in db_documents - if Path(doc.path).is_relative_to(directory.name) # e.g., 'knowledge/' or 'documents/' + get_path(record): record.checksum + for record in db_records } - logger.debug(f"Found {len(db_files)} files in DB for {directory}") - - # Find changes - new = set(file_system_files.keys()) - set(db_files.keys()) - deleted = set(db_files.keys()) - set(file_system_files.keys()) + # Compare current vs database state + new = set(current_files.keys()) - set(db_files.keys()) + deleted = set(db_files.keys()) - set(current_files.keys()) modified = { - path for path in file_system_files - if path in db_files and file_system_files[path] != db_files[path] + path for path in current_files + if path in db_files and current_files[path] != db_files[path] } return SyncReport(new=new, modified=modified, deleted=deleted) - async def sync_new_file(self, path: str, directory: Path) -> None: - """ - Sync a new file. + async def find_document_changes(self, directory: Path) -> SyncReport: + """Find changes in document directory.""" + return await self.find_changes( + directory=directory, + get_records=self.document_repository.find_all + ) - Args: - path: Relative path to file - directory: Root directory - - Raises: - SyncError: If sync fails - """ - full_path = directory / path - try: - content = full_path.read_text() - checksum = await self.compute_checksum(content) - await self.repository.create({ - "path": path, - "checksum": checksum - }) - except Exception as e: - raise SyncError(f"Failed to sync new file {path}: {e}") - - async def sync_modified_file(self, path: str, directory: Path) -> None: - """ - Sync a modified file. - - Args: - path: Relative path to file - directory: Root directory - - Raises: - SyncError: If sync fails - """ - full_path = directory / path - try: - content = full_path.read_text() - checksum = await self.compute_checksum(content) - doc = await self.repository.find_by_path(path) - if doc: - await self.repository.update(doc.id, {"checksum": checksum}) - else: - await self.repository.create({ - "path": path, - "checksum": checksum - }) - except Exception as e: - raise SyncError(f"Failed to sync modified file {path}: {e}") - - async def sync(self, directory: Path) -> SyncReport: - """ - Sync filesystem with database. - Filesystem is source of truth. - - Args: - directory: Root directory to sync - - Returns: - SyncReport detailing changes - - Raises: - SyncError: If sync fails - """ - logger.info(f"Starting sync of {directory}") - - # Get current state - current_files = await self.scan_files(directory) - - # Find changes - changes = await self.find_changes(current_files, directory) - logger.info(f"Found changes in {directory}: {changes}") - - if changes.total_changes == 0: - logger.info("No changes detected") - return changes - - # Process new files - for path in changes.new: - logger.debug(f"Processing new file: {path}") - await self.sync_new_file(path, directory) - - # Process modified files - for path in changes.modified: - logger.debug(f"Processing modified file: {path}") - await self.sync_modified_file(path, directory) - - # Process deleted files - for path in changes.deleted: - logger.debug(f"Processing deleted file: {path}") - doc = await self.repository.find_by_path(path) - if doc: - await self.repository.delete(doc.id) - - logger.info("Sync completed successfully") - return changes \ No newline at end of file + async def find_knowledge_changes(self, directory: Path) -> SyncReport: + """Find changes in knowledge directory.""" + return await self.find_changes( + directory=directory, + get_records=self.entity_repository.find_all, + get_path=lambda x: x.path_id + ) \ No newline at end of file diff --git a/tests/conftest.py b/tests/conftest.py index 6855539e..af69dc05 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -142,10 +142,10 @@ def knowledge_writer(): """Create writer instance.""" return KnowledgeWriter() -@pytest.fixture -def file_sync_service(document_repository: DocumentRepository): - """Create FileService instance.""" - return FileSyncService(document_repository) +@pytest_asyncio.fixture +def file_sync_service(document_repository, entity_repository) -> FileSyncService: + """Create FileSyncService instance.""" + return FileSyncService(document_repository, entity_repository) diff --git a/tests/services/test_file_sync_service.py b/tests/services/test_file_sync_service.py index dfa13260..f587fb79 100644 --- a/tests/services/test_file_sync_service.py +++ b/tests/services/test_file_sync_service.py @@ -5,13 +5,15 @@ from pathlib import Path import pytest import pytest_asyncio -from basic_memory.services.file_sync_service import FileSyncService, SyncError +from basic_memory.services.file_sync_service import FileSyncService + @pytest_asyncio.fixture -async def file_sync_service(document_repository) -> FileSyncService: - """Create FileSyncService instance.""" - return FileSyncService(document_repository) +async def knowledge_dir(test_config) -> Path: + """Get knowledge directory.""" + test_config.knowledge_dir.mkdir(parents=True) + return test_config.knowledge_dir @pytest_asyncio.fixture @@ -22,22 +24,19 @@ async def docs_dir(test_config) -> Path: @pytest_asyncio.fixture -async def sample_files(docs_dir) -> dict[str, str]: - """Create some sample test files.""" - # Create test structure +async def sample_documents(docs_dir) -> dict[str, str]: + """Create sample document files.""" design_dir = docs_dir / "design" notes_dir = docs_dir / "notes" design_dir.mkdir(parents=True, exist_ok=True) notes_dir.mkdir(parents=True, exist_ok=True) - # Map of relative paths to content files = { - "design/architecture.md": "# Architecture\nSome design notes", - "notes/meeting.md": "# Meeting Notes\nDiscussion points", - "README.md": "# Project\nOverview doc", + "design/architecture.md": "# Architecture\nDesign notes", + "notes/meeting.md": "# Meeting\nNotes from discussion", + "README.md": "# Project\nOverview" } - # Create files with full paths for rel_path, content in files.items(): full_path = docs_dir / rel_path full_path.write_text(content) @@ -45,132 +44,152 @@ async def sample_files(docs_dir) -> dict[str, str]: return files +@pytest_asyncio.fixture +async def sample_knowledge(knowledge_dir) -> dict[str, str]: + """Create sample knowledge files.""" + component_dir = knowledge_dir / "component" + concept_dir = knowledge_dir / "concept" + component_dir.mkdir(parents=True, exist_ok=True) + concept_dir.mkdir(parents=True, exist_ok=True) + + files = { + "component/memory_service.md": "# Memory Service\nCore service", + "component/file_service.md": "# File Service\nFile ops", + "concept/local_first.md": "# Local First\nDesign principle" + } + + for rel_path, content in files.items(): + full_path = knowledge_dir / rel_path + full_path.write_text(content) + + return files + + @pytest.mark.asyncio -async def test_scan_files(file_sync_service, docs_dir, sample_files): +async def test_scan_directory(file_sync_service, docs_dir, sample_documents): """Test scanning directory for files.""" - scanned = await file_sync_service.scan_files(docs_dir) + # Scan documents directory + scanned = await file_sync_service.scan_directory(docs_dir) # Should find all files - assert len(scanned) == len(sample_files) - - # Paths should be relative and match sample files - assert set(scanned.keys()) == set(sample_files.keys()) - - # All files should have checksums + assert len(scanned) == len(sample_documents) + assert set(scanned.keys()) == set(sample_documents.keys()) assert all(isinstance(checksum, str) for checksum in scanned.values()) - -@pytest.mark.asyncio -async def test_find_new_files(file_sync_service, docs_dir, sample_files): - """Test detecting new files.""" - changes = await file_sync_service.find_changes(await file_sync_service.scan_files(docs_dir)) - - # All files should be new - assert len(changes.new) == len(sample_files) - assert len(changes.modified) == 0 - assert len(changes.deleted) == 0 + # Checksums should be different for different content + checksums = list(scanned.values()) + assert len(set(checksums)) == len(checksums) # All unique @pytest.mark.asyncio -async def test_find_modified_files(file_sync_service, docs_dir, sample_files): - """Test detecting modified files.""" - # First sync to create DB records - await file_sync_service.sync(docs_dir) +async def test_document_changes_new_files(file_sync_service, docs_dir, sample_documents): + """Test detecting new document files.""" + # Check changes - all should be new since DB is empty + changes = await file_sync_service.find_document_changes(docs_dir) - # Modify a file - mod_path = docs_dir / "design/architecture.md" - mod_path.write_text("# Updated Architecture") - - # Check changes - changes = await file_sync_service.find_changes(await file_sync_service.scan_files(docs_dir)) - - assert len(changes.modified) == 1 - assert "design/architecture.md" in changes.modified - assert len(changes.new) == 0 - assert len(changes.deleted) == 0 + assert changes.new == set(sample_documents.keys()) + assert not changes.modified + assert not changes.deleted @pytest.mark.asyncio -async def test_find_deleted_files(file_sync_service, docs_dir, sample_files): - """Test detecting deleted files.""" - # First sync to create DB records - await file_sync_service.sync(docs_dir) +async def test_document_changes_modified_files( + file_sync_service, docs_dir, sample_documents, document_repository +): + """Test detecting modified document files.""" + # Create initial DB records + for path, content in sample_documents.items(): + await document_repository.create({ + "path": path, + "checksum": "old_checksum" # Different from actual file + }) - # Delete a file - del_path = docs_dir / "notes/meeting.md" - del_path.unlink() + # Check changes - all should be modified + changes = await file_sync_service.find_document_changes(docs_dir) - # Check changes - changes = await file_sync_service.find_changes(await file_sync_service.scan_files(docs_dir)) - - assert len(changes.deleted) == 1 - assert "notes/meeting.md" in changes.deleted - assert len(changes.new) == 0 - assert len(changes.modified) == 0 + assert not changes.new + assert changes.modified == set(sample_documents.keys()) + assert not changes.deleted @pytest.mark.asyncio -async def test_full_sync_process(file_sync_service, docs_dir, sample_files): - """Test full sync process with various changes.""" - # First sync to create initial state - initial_sync = await file_sync_service.sync(docs_dir) - assert initial_sync.total_changes == len(sample_files) +async def test_document_changes_deleted_files( + file_sync_service, docs_dir, document_repository +): + """Test detecting deleted document files.""" + # Create DB records for non-existent files + db_files = { + "old/doc1.md": "checksum1", + "old/doc2.md": "checksum2" + } + for path, checksum in db_files.items(): + await document_repository.create({ + "path": path, + "checksum": checksum + }) - # Make some changes: - # 1. Add new file - (docs_dir / "notes").mkdir(exist_ok=True) # Ensure parent exists - (docs_dir / "notes/todo.md").write_text("# TODO\n- First item") + # Check changes - all should be deleted + changes = await file_sync_service.find_document_changes(docs_dir) - # 2. Modify existing file - (docs_dir / "README.md").write_text("# Updated Project") - - # 3. Delete a file - (docs_dir / "design/architecture.md").unlink() - - # Run sync - changes = await file_sync_service.sync(docs_dir) - - # Verify changes - assert len(changes.new) == 1 - assert "notes/todo.md" in changes.new - - assert len(changes.modified) == 1 - assert "README.md" in changes.modified - - assert len(changes.deleted) == 1 - assert "design/architecture.md" in changes.deleted + assert not changes.new + assert not changes.modified + assert changes.deleted == set(db_files.keys()) @pytest.mark.asyncio -async def test_no_changes_sync(file_sync_service, docs_dir, sample_files): - """Test sync when no changes are present.""" - # First sync to create initial state - await file_sync_service.sync(docs_dir) +async def test_knowledge_changes_new_files(file_sync_service, knowledge_dir, sample_knowledge): + """Test detecting new knowledge files.""" + # Check changes - all should be new since DB is empty + changes = await file_sync_service.find_knowledge_changes(knowledge_dir) - # Sync again immediately - changes = await file_sync_service.sync(docs_dir) - - # Should detect no changes - assert changes.total_changes == 0 + assert changes.new == set(sample_knowledge.keys()) + assert not changes.modified + assert not changes.deleted @pytest.mark.asyncio -async def test_sync_empty_directory(file_sync_service, docs_dir): - """Test syncing an empty directory.""" - changes = await file_sync_service.sync(docs_dir) - assert changes.total_changes == 0 +async def test_knowledge_changes_modified_files( + file_sync_service, knowledge_dir, sample_knowledge, entity_repository +): + """Test detecting modified knowledge files.""" + # Create initial DB records + for path_id, content in sample_knowledge.items(): + await entity_repository.create({ + "path_id": path_id, + "name": path_id.split("/")[-1], + "entity_type": path_id.split("/")[0], + "checksum": "old_checksum" # Different from actual file + }) + + # Check changes - all should be modified + changes = await file_sync_service.find_knowledge_changes(knowledge_dir) + + assert not changes.new + assert changes.modified == set(sample_knowledge.keys()) + assert not changes.deleted @pytest.mark.asyncio -async def test_error_on_unreadable_file(file_sync_service, docs_dir): - """Test handling of unreadable files during sync.""" - # Create file without read permissions - bad_file = docs_dir / "bad.md" - bad_file.write_text("test") - bad_file.chmod(0o000) # Remove all permissions +async def test_knowledge_changes_deleted_files( + file_sync_service, knowledge_dir, entity_repository +): + """Test detecting deleted knowledge files.""" + # Create DB records for non-existent files + db_files = { + "component/old_service.md": "checksum1", + "concept/old_idea.md": "checksum2" + } + for path_id, checksum in db_files.items(): + await entity_repository.create({ + "path_id": path_id, + "name": path_id.split("/")[-1], + "entity_type": path_id.split("/")[0], + "checksum": checksum + }) - with pytest.raises(SyncError): - await file_sync_service.sync(docs_dir) + # Check changes - all should be deleted + changes = await file_sync_service.find_knowledge_changes(knowledge_dir) - # Clean up - bad_file.chmod(0o666) # Make readable/writable for cleanup + assert not changes.new + assert not changes.modified + assert changes.deleted == set(db_files.keys()) \ No newline at end of file