diff --git a/src/basic_memory/services/__init__.py b/src/basic_memory/services/__init__.py index b4a87349..482861c5 100644 --- a/src/basic_memory/services/__init__.py +++ b/src/basic_memory/services/__init__.py @@ -1,33 +1,15 @@ -"""Service layer exceptions and imports.""" +"""Services package.""" - -class ServiceError(Exception): - """Base exception for service errors""" - - pass - - -class DatabaseSyncError(ServiceError): - """Raised when database sync fails""" - - pass - - -class RelationError(ServiceError): - """Base exception for relation-specific errors""" - - pass - - -from .entity_service import EntityService -from .observation_service import ObservationService -from .relation_service import RelationService +from basic_memory.services.document_service import DocumentService +from basic_memory.services.entity_service import EntityService +from basic_memory.services.observation_service import ObservationService +from basic_memory.services.relation_service import RelationService +from basic_memory.services.service import BaseService __all__ = [ - "ServiceError", - "DatabaseSyncError", - "RelationError", - "EntityService", - "ObservationService", - "RelationService", -] + 'BaseService', + 'DocumentService', + 'EntityService', + 'ObservationService', + 'RelationService', +] \ No newline at end of file diff --git a/src/basic_memory/services/document_service.py b/src/basic_memory/services/document_service.py index 9dfa804b..52fe6b96 100644 --- a/src/basic_memory/services/document_service.py +++ b/src/basic_memory/services/document_service.py @@ -2,7 +2,7 @@ import hashlib from pathlib import Path -from typing import Optional, Dict, Any +from typing import Optional, Dict, Any, List from loguru import logger @@ -44,6 +44,32 @@ class DocumentService(BaseService[DocumentRepository]): """Compute SHA-256 checksum of content.""" return hashlib.sha256(content.encode()).hexdigest() + async def ensure_parent_directory(self, path: Path) -> None: + """ + Ensure parent directory exists and is writable. + + Args: + path: Path to check + + Raises: + DocumentWriteError: If directory cannot be created or is not writable + """ + parent = path.parent + try: + if not parent.exists(): + parent.mkdir(parents=True) + + # Verify we can write to it + test_file = parent / ".write_test" + test_file.touch() + test_file.unlink() + except Exception as e: + raise DocumentWriteError(f"Directory not writable: {parent}: {e}") + + async def list_documents(self) -> List[Document]: + """List all documents in the database.""" + return await self.repository.find_all() + async def create_document( self, path: str, content: str, metadata: Optional[Dict[str, Any]] = None ) -> Document: @@ -63,9 +89,9 @@ class DocumentService(BaseService[DocumentRepository]): """ logger.debug(f"Creating document at {path}") - # Ensure parent directories exist + # Ensure parent directories exist and are writable file_path = Path(path) - file_path.parent.mkdir(parents=True, exist_ok=True) + await self.ensure_parent_directory(file_path) # Write file first try: @@ -164,9 +190,11 @@ class DocumentService(BaseService[DocumentRepository]): checksum = await self.compute_checksum(content) update_data = {"checksum": checksum} if metadata is not None: - update_data["doc_metadata"] = metadata # pyright: ignore [reportArgumentType] + update_data["doc_metadata"] = metadata - return await self.repository.update(doc.id, update_data) + updated_document = await self.repository.update(doc.id, update_data) + assert updated_document is not None, f"Could not update document {doc.id}" + return updated_document async def delete_document(self, path: str) -> None: """ diff --git a/src/basic_memory/services/file_sync_service.py b/src/basic_memory/services/file_sync_service.py new file mode 100644 index 00000000..2288efaa --- /dev/null +++ b/src/basic_memory/services/file_sync_service.py @@ -0,0 +1,185 @@ +"""Service for syncing files with the database.""" + +from dataclasses import dataclass +from pathlib import Path +from typing import Set + +from loguru import logger + +from basic_memory.services.document_service import DocumentService + + +@dataclass +class SyncReport: + """Report of sync results.""" + new: Set[str] + modified: Set[str] + deleted: Set[str] + + @property + def total_changes(self) -> int: + return len(self.new) + len(self.modified) + len(self.deleted) + + def __str__(self) -> str: + return ( + f"Changes detected:\n" + f" New files: {len(self.new)}\n" + f" Modified: {len(self.modified)}\n" + f" Deleted: {len(self.deleted)}" + ) + + +class SyncError(Exception): + """Raised when sync operations fail.""" + pass + + +class FileSyncService: + """Service for keeping files and database in sync.""" + + def __init__(self, document_service: DocumentService): + self.document_service = document_service + + async def scan_files(self, directory: Path) -> dict[str, str]: + """ + Scan directory for files and their checksums. + Only processes files, ignores directories. + + Args: + directory: Root directory to scan + + Returns: + Dict mapping paths to checksums + + Raises: + SyncError: If any file cannot be read + """ + logger.debug(f"Scanning directory: {directory}") + files = {} + errors = [] + + for path in directory.rglob('*'): + if path.is_file(): + try: + content = path.read_text() + checksum = await self.document_service.compute_checksum(content) + rel_path = str(path.relative_to(directory)) + files[rel_path] = checksum + except Exception as e: + errors.append(f"Failed to read {path}: {e}") + + if errors: + raise SyncError("Failed to read files:\n" + "\n".join(errors)) + + logger.debug(f"Found {len(files)} files") + return files + + async def find_changes(self, current_files: dict[str, str]) -> SyncReport: + """ + Find changes between filesystem and database. + + Args: + current_files: Dict mapping paths to checksums + + Returns: + SyncReport detailing changes + """ + logger.debug("Finding changes") + + # Get all documents from DB + db_documents = await self.document_service.list_documents() + db_files = { + doc.path: doc.checksum + for doc in db_documents + } + + # Find changes + new = set(current_files.keys()) - set(db_files.keys()) + deleted = set(db_files.keys()) - set(current_files.keys()) + modified = { + path for path in current_files + if path in db_files and current_files[path] != db_files[path] + } + + return SyncReport(new=new, modified=modified, deleted=deleted) + + async def sync_new_file(self, path: str, directory: Path) -> None: + """ + Sync a new file. + + Args: + path: Relative path to file + directory: Root directory + + Raises: + SyncError: If sync fails + """ + full_path = directory / path + try: + content = full_path.read_text() + await self.document_service.create_document(path, content) + except Exception as e: + raise SyncError(f"Failed to sync new file {path}: {e}") + + async def sync_modified_file(self, path: str, directory: Path) -> None: + """ + Sync a modified file. + + Args: + path: Relative path to file + directory: Root directory + + Raises: + SyncError: If sync fails + """ + full_path = directory / path + try: + content = full_path.read_text() + await self.document_service.update_document(path, content) + except Exception as e: + raise SyncError(f"Failed to sync modified file {path}: {e}") + + async def sync(self, directory: Path) -> SyncReport: + """ + Sync filesystem with database. + Filesystem is source of truth. + + Args: + directory: Root directory to sync + + Returns: + SyncReport detailing changes + + Raises: + SyncError: If sync fails + """ + logger.info(f"Starting sync of {directory}") + + # Get current state + current_files = await self.scan_files(directory) + + # Find changes + changes = await self.find_changes(current_files) + logger.info(f"Found changes: {changes}") + + if changes.total_changes == 0: + logger.info("No changes detected") + return changes + + # Process new files + for path in changes.new: + logger.debug(f"Processing new file: {path}") + await self.sync_new_file(path, directory) + + # Process modified files + for path in changes.modified: + logger.debug(f"Processing modified file: {path}") + await self.sync_modified_file(path, directory) + + # Process deleted files + for path in changes.deleted: + logger.debug(f"Processing deleted file: {path}") + await self.document_service.delete_document(path) + + logger.info("Sync completed successfully") + return changes \ No newline at end of file diff --git a/tests/conftest.py b/tests/conftest.py index 6bffab54..13ef0ee5 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -25,6 +25,7 @@ from basic_memory.services import ( EntityService, ObservationService, RelationService, + DocumentService, ) @@ -69,11 +70,14 @@ async def session_maker(engine_factory) -> async_sessionmaker[AsyncSession]: @pytest_asyncio.fixture async def test_project_path(): - """Create a temporary project directory.""" + """Create a temporary project directory with standard subdirs.""" with tempfile.TemporaryDirectory() as temp_dir: project_path = Path(temp_dir) / "test-project" - entities_path = project_path / "entities" - entities_path.mkdir(parents=True) + + # Create standard directories + (project_path / "documents").mkdir(parents=True) + (project_path / "entities").mkdir(parents=True) + yield project_path @@ -83,6 +87,12 @@ async def document_repository(session_maker: async_sessionmaker[AsyncSession]) - return DocumentRepository(session_maker) +@pytest_asyncio.fixture(scope="function") +async def document_service(document_repository: DocumentRepository) -> DocumentService: + """Create a DocumentService instance.""" + return DocumentService(document_repository) + + @pytest_asyncio.fixture(scope="function") async def entity_repository(session_maker: async_sessionmaker[AsyncSession]) -> EntityRepository: """Create an EntityRepository instance.""" diff --git a/tests/services/test_document_service.py b/tests/services/test_document_service.py index b959273a..4083d6e2 100644 --- a/tests/services/test_document_service.py +++ b/tests/services/test_document_service.py @@ -1,8 +1,10 @@ """Tests for DocumentService.""" +import os import pytest import pytest_asyncio from pathlib import Path +import stat from basic_memory.services.document_service import ( DocumentService, @@ -45,18 +47,25 @@ async def test_create_document_file_first(document_service, test_doc_path): @pytest.mark.asyncio async def test_create_document_unwriteable_directory(document_service, tmp_path): """Test error when trying to write to an unwriteable directory.""" - # Try to create in non-existent parent - bad_path = tmp_path / "nonexistent" / "test.md" + # Create parent directory without write permissions + parent_dir = tmp_path / "unwriteable" + parent_dir.mkdir() + parent_dir.chmod(stat.S_IREAD) # Read-only + bad_path = parent_dir / "test.md" content = "# Test" + with pytest.raises(DocumentWriteError): await document_service.create_document(str(bad_path), content) - # Verify no file was created - assert not bad_path.exists() + # Verify directory is still read-only + assert not os.access(parent_dir, os.W_OK) # Verify no database record doc = await document_service.repository.find_by_path(str(bad_path)) assert doc is None + + # Clean up - make writable again so it can be deleted + parent_dir.chmod(stat.S_IWRITE | stat.S_IREAD | stat.S_IEXEC) @pytest.mark.asyncio @@ -114,4 +123,4 @@ async def test_update_file_exists_no_record(document_service, test_doc_path): assert doc is not None assert doc.path == str(test_doc_path) assert doc.doc_metadata == {"status": "updated"} - assert test_doc_path.read_text() == new_content + assert test_doc_path.read_text() == new_content \ No newline at end of file diff --git a/tests/services/test_file_sync_service.py b/tests/services/test_file_sync_service.py new file mode 100644 index 00000000..ae46c462 --- /dev/null +++ b/tests/services/test_file_sync_service.py @@ -0,0 +1,173 @@ +"""Tests for FileSyncService.""" + +from pathlib import Path + +import pytest +import pytest_asyncio + +from basic_memory.services.file_sync_service import FileSyncService, SyncError + + +@pytest_asyncio.fixture +async def file_sync_service(document_service) -> FileSyncService: + """Create FileSyncService instance.""" + return FileSyncService(document_service) + + +@pytest_asyncio.fixture +async def docs_dir(test_project_path) -> Path: + """Get documents directory.""" + return test_project_path / "documents" + + +@pytest_asyncio.fixture +async def sample_files(docs_dir) -> dict[str, str]: + """Create some sample files for testing.""" + # Create test structure + design_dir = docs_dir / "design" + notes_dir = docs_dir / "notes" + design_dir.mkdir() + notes_dir.mkdir() + + files = { + "design/architecture.md": "# Architecture\nSome design notes", + "notes/meeting.md": "# Meeting Notes\nDiscussion points", + "README.md": "# Project\nOverview doc", + } + + # Create files + for path, content in files.items(): + file_path = docs_dir / path + file_path.write_text(content) + + return files + + +@pytest.mark.asyncio +async def test_scan_files(file_sync_service, docs_dir, sample_files): + """Test scanning directory for files.""" + scanned = await file_sync_service.scan_files(docs_dir) + + # Should find all files + assert len(scanned) == len(sample_files) + + # Paths should be relative + assert all(str(docs_dir) not in path for path in scanned) + + # All files should have checksums + assert all(isinstance(checksum, str) for checksum in scanned.values()) + + +@pytest.mark.asyncio +async def test_find_new_files(file_sync_service, docs_dir, sample_files): + """Test detecting new files.""" + changes = await file_sync_service.find_changes(await file_sync_service.scan_files(docs_dir)) + + # All files should be new + assert len(changes.new) == len(sample_files) + assert len(changes.modified) == 0 + assert len(changes.deleted) == 0 + + +@pytest.mark.asyncio +async def test_find_modified_files(file_sync_service, docs_dir, sample_files): + """Test detecting modified files.""" + # First sync to create DB records + await file_sync_service.sync(docs_dir) + + # Modify a file + mod_path = docs_dir / "design/architecture.md" + mod_path.write_text("# Updated Architecture") + + # Check changes + changes = await file_sync_service.find_changes(await file_sync_service.scan_files(docs_dir)) + + assert len(changes.modified) == 1 + assert "design/architecture.md" in changes.modified + assert len(changes.new) == 0 + assert len(changes.deleted) == 0 + + +@pytest.mark.asyncio +async def test_find_deleted_files(file_sync_service, docs_dir, sample_files): + """Test detecting deleted files.""" + # First sync to create DB records + await file_sync_service.sync(docs_dir) + + # Delete a file + del_path = docs_dir / "notes/meeting.md" + del_path.unlink() + + # Check changes + changes = await file_sync_service.find_changes(await file_sync_service.scan_files(docs_dir)) + + assert len(changes.deleted) == 1 + assert "notes/meeting.md" in changes.deleted + assert len(changes.new) == 0 + assert len(changes.modified) == 0 + + +@pytest.mark.asyncio +async def test_full_sync_process(file_sync_service, docs_dir, sample_files): + """Test full sync process with various changes.""" + # First sync to create initial state + initial_sync = await file_sync_service.sync(docs_dir) + assert initial_sync.total_changes == len(sample_files) + + # Make some changes: + # 1. Add new file + new_file = docs_dir / "notes/todo.md" + new_file.write_text("# TODO\n- First item") + + # 2. Modify existing file + mod_file = docs_dir / "README.md" + mod_file.write_text("# Updated Project") + + # 3. Delete a file + del_file = docs_dir / "design/architecture.md" + del_file.unlink() + + # Run sync + changes = await file_sync_service.sync(docs_dir) + + # Verify changes + assert len(changes.new) == 1 + assert "notes/todo.md" in changes.new + + assert len(changes.modified) == 1 + assert "README.md" in changes.modified + + assert len(changes.deleted) == 1 + assert "design/architecture.md" in changes.deleted + + +@pytest.mark.asyncio +async def test_no_changes_sync(file_sync_service, docs_dir, sample_files): + """Test sync when no changes are present.""" + # First sync to create initial state + await file_sync_service.sync(docs_dir) + + # Sync again immediately + changes = await file_sync_service.sync(docs_dir) + + # Should detect no changes + assert changes.total_changes == 0 + + +@pytest.mark.asyncio +async def test_sync_empty_directory(file_sync_service, docs_dir): + """Test syncing an empty directory.""" + changes = await file_sync_service.sync(docs_dir) + assert changes.total_changes == 0 + + +@pytest.mark.asyncio +async def test_error_on_unreadable_file(file_sync_service, docs_dir): + """Test handling of unreadable files during sync.""" + # Create file without read permissions + bad_file = docs_dir / "bad.md" + bad_file.write_text("test") + bad_file.chmod(0o000) # Remove all permissions + + with pytest.raises(SyncError): + await file_sync_service.sync(docs_dir)