file_sync_service

src/basic_memory/services/__init__.py

@@ -1,33 +1,15 @@
-"""Service layer exceptions and imports."""
+"""Services package."""
 
-
-class ServiceError(Exception):
-    """Base exception for service errors"""
-
-    pass
-
-
-class DatabaseSyncError(ServiceError):
-    """Raised when database sync fails"""
-
-    pass
-
-
-class RelationError(ServiceError):
-    """Base exception for relation-specific errors"""
-
-    pass
-
-
-from .entity_service import EntityService
-from .observation_service import ObservationService
-from .relation_service import RelationService
+from basic_memory.services.document_service import DocumentService
+from basic_memory.services.entity_service import EntityService
+from basic_memory.services.observation_service import ObservationService
+from basic_memory.services.relation_service import RelationService
+from basic_memory.services.service import BaseService
 
 __all__ = [
-    "ServiceError",
-    "DatabaseSyncError",
-    "RelationError",
-    "EntityService",
-    "ObservationService",
-    "RelationService",
-]
+    'BaseService',
+    'DocumentService',
+    'EntityService',
+    'ObservationService',
+    'RelationService',
+]

src/basic_memory/services/document_service.py

@@ -2,7 +2,7 @@
 
 import hashlib
 from pathlib import Path
-from typing import Optional, Dict, Any
+from typing import Optional, Dict, Any, List
 
 from loguru import logger
 
@@ -44,6 +44,32 @@ class DocumentService(BaseService[DocumentRepository]):
         """Compute SHA-256 checksum of content."""
         return hashlib.sha256(content.encode()).hexdigest()
 
+    async def ensure_parent_directory(self, path: Path) -> None:
+        """
+        Ensure parent directory exists and is writable.
+
+        Args:
+            path: Path to check
+
+        Raises:
+            DocumentWriteError: If directory cannot be created or is not writable
+        """
+        parent = path.parent
+        try:
+            if not parent.exists():
+                parent.mkdir(parents=True)
+
+            # Verify we can write to it
+            test_file = parent / ".write_test"
+            test_file.touch()
+            test_file.unlink()
+        except Exception as e:
+            raise DocumentWriteError(f"Directory not writable: {parent}: {e}")
+
+    async def list_documents(self) -> List[Document]:
+        """List all documents in the database."""
+        return await self.repository.find_all()
+
     async def create_document(
         self, path: str, content: str, metadata: Optional[Dict[str, Any]] = None
     ) -> Document:
@@ -63,9 +89,9 @@ class DocumentService(BaseService[DocumentRepository]):
         """
         logger.debug(f"Creating document at {path}")
 
-        # Ensure parent directories exist
+        # Ensure parent directories exist and are writable
         file_path = Path(path)
-        file_path.parent.mkdir(parents=True, exist_ok=True)
+        await self.ensure_parent_directory(file_path)
 
         # Write file first
         try:
@@ -164,9 +190,11 @@ class DocumentService(BaseService[DocumentRepository]):
         checksum = await self.compute_checksum(content)
         update_data = {"checksum": checksum}
         if metadata is not None:
-            update_data["doc_metadata"] = metadata  # pyright: ignore [reportArgumentType]
+            update_data["doc_metadata"] = metadata
 
-        return await self.repository.update(doc.id, update_data)
+        updated_document = await self.repository.update(doc.id, update_data)
+        assert updated_document is not None, f"Could not update document {doc.id}"
+        return updated_document
 
     async def delete_document(self, path: str) -> None:
         """

src/basic_memory/services/file_sync_service.py

@@ -0,0 +1,185 @@
+"""Service for syncing files with the database."""
+
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Set
+
+from loguru import logger
+
+from basic_memory.services.document_service import DocumentService
+
+
+@dataclass
+class SyncReport:
+    """Report of sync results."""
+    new: Set[str]
+    modified: Set[str]
+    deleted: Set[str]
+
+    @property
+    def total_changes(self) -> int:
+        return len(self.new) + len(self.modified) + len(self.deleted)
+
+    def __str__(self) -> str:
+        return (
+            f"Changes detected:\n"
+            f"  New files: {len(self.new)}\n"
+            f"  Modified: {len(self.modified)}\n"
+            f"  Deleted: {len(self.deleted)}"
+        )
+
+
+class SyncError(Exception):
+    """Raised when sync operations fail."""
+    pass
+
+
+class FileSyncService:
+    """Service for keeping files and database in sync."""
+
+    def __init__(self, document_service: DocumentService):
+        self.document_service = document_service
+
+    async def scan_files(self, directory: Path) -> dict[str, str]:
+        """
+        Scan directory for files and their checksums.
+        Only processes files, ignores directories.
+
+        Args:
+            directory: Root directory to scan
+
+        Returns:
+            Dict mapping paths to checksums
+
+        Raises:
+            SyncError: If any file cannot be read
+        """
+        logger.debug(f"Scanning directory: {directory}")
+        files = {}
+        errors = []
+
+        for path in directory.rglob('*'):
+            if path.is_file():
+                try:
+                    content = path.read_text()
+                    checksum = await self.document_service.compute_checksum(content)
+                    rel_path = str(path.relative_to(directory))
+                    files[rel_path] = checksum
+                except Exception as e:
+                    errors.append(f"Failed to read {path}: {e}")
+
+        if errors:
+            raise SyncError("Failed to read files:\n" + "\n".join(errors))
+
+        logger.debug(f"Found {len(files)} files")
+        return files
+
+    async def find_changes(self, current_files: dict[str, str]) -> SyncReport:
+        """
+        Find changes between filesystem and database.
+
+        Args:
+            current_files: Dict mapping paths to checksums
+
+        Returns:
+            SyncReport detailing changes
+        """
+        logger.debug("Finding changes")
+        
+        # Get all documents from DB
+        db_documents = await self.document_service.list_documents()
+        db_files = {
+            doc.path: doc.checksum 
+            for doc in db_documents
+        }
+
+        # Find changes
+        new = set(current_files.keys()) - set(db_files.keys())
+        deleted = set(db_files.keys()) - set(current_files.keys())
+        modified = {
+            path for path in current_files
+            if path in db_files and current_files[path] != db_files[path]
+        }
+
+        return SyncReport(new=new, modified=modified, deleted=deleted)
+
+    async def sync_new_file(self, path: str, directory: Path) -> None:
+        """
+        Sync a new file.
+
+        Args:
+            path: Relative path to file
+            directory: Root directory
+
+        Raises:
+            SyncError: If sync fails
+        """
+        full_path = directory / path
+        try:
+            content = full_path.read_text()
+            await self.document_service.create_document(path, content)
+        except Exception as e:
+            raise SyncError(f"Failed to sync new file {path}: {e}")
+
+    async def sync_modified_file(self, path: str, directory: Path) -> None:
+        """
+        Sync a modified file.
+
+        Args:
+            path: Relative path to file
+            directory: Root directory
+
+        Raises:
+            SyncError: If sync fails
+        """
+        full_path = directory / path
+        try:
+            content = full_path.read_text()
+            await self.document_service.update_document(path, content)
+        except Exception as e:
+            raise SyncError(f"Failed to sync modified file {path}: {e}")
+
+    async def sync(self, directory: Path) -> SyncReport:
+        """
+        Sync filesystem with database.
+        Filesystem is source of truth.
+
+        Args:
+            directory: Root directory to sync
+
+        Returns:
+            SyncReport detailing changes
+
+        Raises:
+            SyncError: If sync fails
+        """
+        logger.info(f"Starting sync of {directory}")
+
+        # Get current state
+        current_files = await self.scan_files(directory)
+        
+        # Find changes
+        changes = await self.find_changes(current_files)
+        logger.info(f"Found changes: {changes}")
+
+        if changes.total_changes == 0:
+            logger.info("No changes detected")
+            return changes
+
+        # Process new files
+        for path in changes.new:
+            logger.debug(f"Processing new file: {path}")
+            await self.sync_new_file(path, directory)
+
+        # Process modified files
+        for path in changes.modified:
+            logger.debug(f"Processing modified file: {path}")
+            await self.sync_modified_file(path, directory)
+
+        # Process deleted files
+        for path in changes.deleted:
+            logger.debug(f"Processing deleted file: {path}")
+            await self.document_service.delete_document(path)
+
+        logger.info("Sync completed successfully")
+        return changes

tests/conftest.py

@@ -25,6 +25,7 @@ from basic_memory.services import (
     EntityService,
     ObservationService,
     RelationService,
+    DocumentService,
 )
 
 
@@ -69,11 +70,14 @@ async def session_maker(engine_factory) -> async_sessionmaker[AsyncSession]:
 
 @pytest_asyncio.fixture
 async def test_project_path():
-    """Create a temporary project directory."""
+    """Create a temporary project directory with standard subdirs."""
     with tempfile.TemporaryDirectory() as temp_dir:
         project_path = Path(temp_dir) / "test-project"
-        entities_path = project_path / "entities"
-        entities_path.mkdir(parents=True)
+        
+        # Create standard directories
+        (project_path / "documents").mkdir(parents=True)
+        (project_path / "entities").mkdir(parents=True)
+        
         yield project_path
 
 
@@ -83,6 +87,12 @@ async def document_repository(session_maker: async_sessionmaker[AsyncSession]) -
     return DocumentRepository(session_maker)
 
 
+@pytest_asyncio.fixture(scope="function")
+async def document_service(document_repository: DocumentRepository) -> DocumentService:
+    """Create a DocumentService instance."""
+    return DocumentService(document_repository)
+
+
 @pytest_asyncio.fixture(scope="function")
 async def entity_repository(session_maker: async_sessionmaker[AsyncSession]) -> EntityRepository:
     """Create an EntityRepository instance."""

tests/services/test_document_service.py

@@ -1,8 +1,10 @@
 """Tests for DocumentService."""
 
+import os
 import pytest
 import pytest_asyncio
 from pathlib import Path
+import stat
 
 from basic_memory.services.document_service import (
     DocumentService,
@@ -45,18 +47,25 @@ async def test_create_document_file_first(document_service, test_doc_path):
 @pytest.mark.asyncio
 async def test_create_document_unwriteable_directory(document_service, tmp_path):
     """Test error when trying to write to an unwriteable directory."""
-    # Try to create in non-existent parent
-    bad_path = tmp_path / "nonexistent" / "test.md"
+    # Create parent directory without write permissions
+    parent_dir = tmp_path / "unwriteable"
+    parent_dir.mkdir()
+    parent_dir.chmod(stat.S_IREAD)  # Read-only
     
+    bad_path = parent_dir / "test.md"
     content = "# Test"
+    
     with pytest.raises(DocumentWriteError):
         await document_service.create_document(str(bad_path), content)
     
-    # Verify no file was created
-    assert not bad_path.exists()
+    # Verify directory is still read-only
+    assert not os.access(parent_dir, os.W_OK)
     # Verify no database record
     doc = await document_service.repository.find_by_path(str(bad_path))
     assert doc is None
+    
+    # Clean up - make writable again so it can be deleted
+    parent_dir.chmod(stat.S_IWRITE | stat.S_IREAD | stat.S_IEXEC)
 
 
 @pytest.mark.asyncio
@@ -114,4 +123,4 @@ async def test_update_file_exists_no_record(document_service, test_doc_path):
     assert doc is not None
     assert doc.path == str(test_doc_path)
     assert doc.doc_metadata == {"status": "updated"}
-    assert test_doc_path.read_text() == new_content
+    assert test_doc_path.read_text() == new_content

tests/services/test_file_sync_service.py

@@ -0,0 +1,173 @@
+"""Tests for FileSyncService."""
+
+from pathlib import Path
+
+import pytest
+import pytest_asyncio
+
+from basic_memory.services.file_sync_service import FileSyncService, SyncError
+
+
+@pytest_asyncio.fixture
+async def file_sync_service(document_service) -> FileSyncService:
+    """Create FileSyncService instance."""
+    return FileSyncService(document_service)
+
+
+@pytest_asyncio.fixture
+async def docs_dir(test_project_path) -> Path:
+    """Get documents directory."""
+    return test_project_path / "documents"
+
+
+@pytest_asyncio.fixture
+async def sample_files(docs_dir) -> dict[str, str]:
+    """Create some sample files for testing."""
+    # Create test structure
+    design_dir = docs_dir / "design"
+    notes_dir = docs_dir / "notes"
+    design_dir.mkdir()
+    notes_dir.mkdir()
+
+    files = {
+        "design/architecture.md": "# Architecture\nSome design notes",
+        "notes/meeting.md": "# Meeting Notes\nDiscussion points",
+        "README.md": "# Project\nOverview doc",
+    }
+
+    # Create files
+    for path, content in files.items():
+        file_path = docs_dir / path
+        file_path.write_text(content)
+
+    return files
+
+
+@pytest.mark.asyncio
+async def test_scan_files(file_sync_service, docs_dir, sample_files):
+    """Test scanning directory for files."""
+    scanned = await file_sync_service.scan_files(docs_dir)
+
+    # Should find all files
+    assert len(scanned) == len(sample_files)
+
+    # Paths should be relative
+    assert all(str(docs_dir) not in path for path in scanned)
+
+    # All files should have checksums
+    assert all(isinstance(checksum, str) for checksum in scanned.values())
+
+
+@pytest.mark.asyncio
+async def test_find_new_files(file_sync_service, docs_dir, sample_files):
+    """Test detecting new files."""
+    changes = await file_sync_service.find_changes(await file_sync_service.scan_files(docs_dir))
+
+    # All files should be new
+    assert len(changes.new) == len(sample_files)
+    assert len(changes.modified) == 0
+    assert len(changes.deleted) == 0
+
+
+@pytest.mark.asyncio
+async def test_find_modified_files(file_sync_service, docs_dir, sample_files):
+    """Test detecting modified files."""
+    # First sync to create DB records
+    await file_sync_service.sync(docs_dir)
+
+    # Modify a file
+    mod_path = docs_dir / "design/architecture.md"
+    mod_path.write_text("# Updated Architecture")
+
+    # Check changes
+    changes = await file_sync_service.find_changes(await file_sync_service.scan_files(docs_dir))
+
+    assert len(changes.modified) == 1
+    assert "design/architecture.md" in changes.modified
+    assert len(changes.new) == 0
+    assert len(changes.deleted) == 0
+
+
+@pytest.mark.asyncio
+async def test_find_deleted_files(file_sync_service, docs_dir, sample_files):
+    """Test detecting deleted files."""
+    # First sync to create DB records
+    await file_sync_service.sync(docs_dir)
+
+    # Delete a file
+    del_path = docs_dir / "notes/meeting.md"
+    del_path.unlink()
+
+    # Check changes
+    changes = await file_sync_service.find_changes(await file_sync_service.scan_files(docs_dir))
+
+    assert len(changes.deleted) == 1
+    assert "notes/meeting.md" in changes.deleted
+    assert len(changes.new) == 0
+    assert len(changes.modified) == 0
+
+
+@pytest.mark.asyncio
+async def test_full_sync_process(file_sync_service, docs_dir, sample_files):
+    """Test full sync process with various changes."""
+    # First sync to create initial state
+    initial_sync = await file_sync_service.sync(docs_dir)
+    assert initial_sync.total_changes == len(sample_files)
+
+    # Make some changes:
+    # 1. Add new file
+    new_file = docs_dir / "notes/todo.md"
+    new_file.write_text("# TODO\n- First item")
+
+    # 2. Modify existing file
+    mod_file = docs_dir / "README.md"
+    mod_file.write_text("# Updated Project")
+
+    # 3. Delete a file
+    del_file = docs_dir / "design/architecture.md"
+    del_file.unlink()
+
+    # Run sync
+    changes = await file_sync_service.sync(docs_dir)
+
+    # Verify changes
+    assert len(changes.new) == 1
+    assert "notes/todo.md" in changes.new
+
+    assert len(changes.modified) == 1
+    assert "README.md" in changes.modified
+
+    assert len(changes.deleted) == 1
+    assert "design/architecture.md" in changes.deleted
+
+
+@pytest.mark.asyncio
+async def test_no_changes_sync(file_sync_service, docs_dir, sample_files):
+    """Test sync when no changes are present."""
+    # First sync to create initial state
+    await file_sync_service.sync(docs_dir)
+
+    # Sync again immediately
+    changes = await file_sync_service.sync(docs_dir)
+
+    # Should detect no changes
+    assert changes.total_changes == 0
+
+
+@pytest.mark.asyncio
+async def test_sync_empty_directory(file_sync_service, docs_dir):
+    """Test syncing an empty directory."""
+    changes = await file_sync_service.sync(docs_dir)
+    assert changes.total_changes == 0
+
+
+@pytest.mark.asyncio
+async def test_error_on_unreadable_file(file_sync_service, docs_dir):
+    """Test handling of unreadable files during sync."""
+    # Create file without read permissions
+    bad_file = docs_dir / "bad.md"
+    bad_file.write_text("test")
+    bad_file.chmod(0o000)  # Remove all permissions
+
+    with pytest.raises(SyncError):
+        await file_sync_service.sync(docs_dir)