watch service

pyproject.toml

@@ -23,6 +23,7 @@ dependencies = [
     "rich>=13.9.4",
     "unidecode>=1.3.8",
     "dateparser>=1.2.0",
+    "watchfiles>=1.0.4",
 ]
 
 [project.optional-dependencies]

src/basic_memory/config.py

@@ -22,6 +22,10 @@ class ProjectConfig(BaseSettings):
     # Name of the project
     project: str = Field(default="default", description="Project name")
 
+    # Watch service configuration
+    sync_delay: int = Field(
+        default=500, description="Milliseconds to wait after changes before syncing", gt=0
+    )
 
     model_config = SettingsConfigDict(
         env_prefix="BASIC_MEMORY_",

src/basic_memory/services/file_service.py

@@ -208,3 +208,6 @@ class FileService:
         except Exception as e:
             logger.error(f"Failed to delete file {path}: {e}")
             raise FileOperationError(f"Failed to delete file: {e}")
+
+    def path(self, path_string: str, absolute: bool = False):
+        return Path( self.base_path / path_string ) if absolute else Path(path_string).relative_to(self.base_path)

src/basic_memory/sync/sync_service.py

@@ -1,7 +1,7 @@
 """Service for syncing files between filesystem and database."""
 
 from pathlib import Path
-from typing import Dict
+from typing import Dict, Optional
 
 from loguru import logger
 
@@ -10,7 +10,8 @@ from basic_memory.repository import EntityRepository, RelationRepository
 from basic_memory.services import EntityService
 from basic_memory.services.search_service import SearchService
 from basic_memory.sync import FileChangeScanner
-from basic_memory.sync.utils import SyncReport
+from basic_memory.sync.utils import SyncReport, FileChange
+from watchfiles import Change
 
 
 class SyncService:
@@ -60,10 +61,28 @@ class SyncService:
         else:
             logger.debug(f"No entity found to delete: {file_path}")
 
-    async def sync(self, directory: Path) -> SyncReport:
+    async def sync(self, directory: Optional[Path] = None, file_changes: Optional[dict[str, FileChange]] = None) -> SyncReport:
         """Sync knowledge files with database."""
-        changes = await self.scanner.find_knowledge_changes(directory)
-        logger.info(f"Found {changes.total_changes} knowledge changes")
+        if file_changes is not None:
+            changes = SyncReport()
+            for path, file_change in file_changes.items():
+                logger.debug(f"path {path} file_change {file_change}")
+                match file_change.change_type:
+                    case Change.added:
+                        changes.new.add(path)
+                        changes.checksums[path] = file_change.checksum
+                    case Change.modified:
+                        changes.modified.add(path)
+                        changes.checksums[path] = file_change.checksum
+                    case Change.deleted:
+                        changes.deleted.add(path)
+        else:
+            # Traditional directory scan mode
+            if directory is None:
+                raise ValueError("Must provide either directory or file_changes")
+            
+            changes = await self.scanner.find_knowledge_changes(directory)
+            logger.info(f"Found {changes.total_changes} knowledge changes")
 
         # Handle moves first
         for old_path, new_path in changes.moves.items():
@@ -86,7 +105,7 @@ class SyncService:
         parsed_entities: Dict[str, EntityMarkdown] = {}
 
         for file_path in [*changes.new, *changes.modified]:
-            entity_markdown = await self.entity_parser.parse_file(directory / file_path)
+            entity_markdown = await self.entity_parser.parse_file(Path(file_path))
             parsed_entities[file_path] = entity_markdown
 
         # First pass: Create/update entities

src/basic_memory/sync/utils.py

@@ -2,6 +2,38 @@
 
 from dataclasses import dataclass, field
 from typing import Set, Dict, Optional
+from watchfiles import Change
+from basic_memory.services.file_service import FileService
+
+
+@dataclass
+class FileChange:
+    """A change to a file detected by the watch service.
+    
+    Attributes:
+        change_type: Type of change (added, modified, deleted)
+        path: Path to the file
+        checksum: File checksum (None for deleted files)
+    """
+    change_type: Change
+    path: str
+    checksum: Optional[str] = None
+
+    @classmethod
+    async def from_path(cls, path: str, change_type: Change, file_service: FileService) -> "FileChange":
+        """Create FileChange from a path, computing checksum if file exists.
+        
+        Args:
+            path: Path to the file
+            change_type: Type of change detected
+            file_service: Service to read file and compute checksum
+            
+        Returns:
+            FileChange with computed checksum for non-deleted files
+        """
+        file_path = file_service.path(path)
+        content, checksum = await file_service.read_file(file_path) if change_type != Change.deleted else (None, None)
+        return cls(path=file_path, change_type=change_type, checksum=checksum)
 
 
 @dataclass
@@ -18,7 +50,7 @@ class SyncReport:
     new: Set[str] = field(default_factory=set)
     modified: Set[str] = field(default_factory=set)
     deleted: Set[str] = field(default_factory=set)
-    moves: Dict[str, str] = field(default_factory=dict)  # old_path -> new_path
+    moves: Dict[str, str] = field(default_factory=dict)  # old_path -> new_path  
     checksums: Dict[str, str] = field(default_factory=dict)
 
     @property

src/basic_memory/sync/watch_service.py

@@ -0,0 +1,131 @@
+"""Watch service for Basic Memory."""
+
+import json
+import dataclasses
+
+from pydantic import BaseModel
+from pydantic.dataclasses import dataclass
+from datetime import datetime
+from pathlib import Path
+from typing import List, Optional
+from watchfiles import awatch, Change
+import os
+
+from basic_memory.config import ProjectConfig
+from basic_memory.sync.sync_service import SyncService
+from basic_memory.services.file_service import FileService
+from basic_memory.sync.utils import FileChange
+
+
+class WatchEvent(BaseModel):
+    timestamp: datetime
+    path: str
+    action: str  # sync, delete, etc
+    status: str  # success, error
+    error: Optional[str] = None
+
+
+class WatchServiceState(BaseModel):
+    # Service status
+    running: bool = False
+    start_time: datetime = dataclasses.field(default_factory=datetime.now)
+    pid: int = dataclasses.field(default_factory=os.getpid)
+
+    # Stats
+    files_synced: int = 0
+    bytes_processed: int = 0
+    error_count: int = 0
+    last_error: Optional[datetime] = None
+    last_scan: Optional[datetime] = None
+
+    # File counts
+    total_files: int = 0
+    markdown_files: int = 0
+
+    # Recent activity
+    recent_events: List[WatchEvent] = dataclasses.field(default_factory=list)
+
+    def add_event(self, path: str, action: str, status: str, error: Optional[str] = None):
+        event = WatchEvent(
+            timestamp=datetime.now(), path=path, action=action, status=status, error=error
+        )
+        self.recent_events.insert(0, event)
+        self.recent_events = self.recent_events[:100]  # Keep last 100
+
+    def record_error(self, error: str):
+        self.error_count += 1
+        self.last_error = datetime.now()
+
+
+class WatchService:
+    def __init__(self, sync_service: SyncService, file_service: FileService, config: ProjectConfig):
+        self.sync_service = sync_service
+        self.file_service = file_service
+        self.config = config
+        self.state = WatchServiceState()
+        self.status_path = config.home / ".basic-memory" / "watch-status.json"
+        self.status_path.parent.mkdir(parents=True, exist_ok=True)
+
+    async def run(self):
+        """Watch for file changes and sync them"""
+        self.state.running = True
+        self.state.start_time = datetime.now()
+        await self.write_status()
+
+        try:
+            async for changes in awatch(
+                self.config.home,
+                watch_filter=self.filter_changes,
+                debounce=self.config.sync_delay,
+                recursive=True,
+            ):
+                await self.handle_changes(changes)
+
+        except Exception as e:
+            self.state.record_error(str(e))
+            await self.write_status()
+            raise
+        finally:
+            self.state.running = False
+            await self.write_status()
+
+    async def write_status(self):
+        """Write current state to status file"""
+        self.status_path.write_text(WatchServiceState.model_dump_json(self.state, indent=2))
+
+    def filter_changes(self, change: Change, path: str) -> bool:
+        """Filter to only watch markdown files"""
+        return path.endswith(".md") and not Path(path).name.startswith(".")
+
+    async def handle_changes(self, changes: set[tuple[Change, str]]):
+        """Process a batch of file changes"""
+
+        # Group changes by file path
+        changes_by_file = {}
+        try:
+            for change_type, path in changes:
+                file_change = await FileChange.from_path(path, change_type, self.file_service)
+                
+                # store changes by relative path
+                changes_by_file[str(file_change.path)] = file_change
+
+            # Process changes with timeout
+            await self.sync_service.sync(file_changes=changes_by_file)
+
+            # Update stats
+            self.state.files_synced += len(changes_by_file)
+            for path, change in changes_by_file.items():
+                if change.change_type != Change.deleted:
+                    size = self.file_service.path(path,absolute=True).stat().st_size
+                    self.state.bytes_processed += size
+
+                self.state.add_event(path=path, action="sync", status="success")
+
+            await self.write_status()
+
+        except Exception as e:
+            self.state.record_error(str(e))
+            for path in changes_by_file:
+                self.state.add_event(path=path, action="sync", status="error", error=str(e))
+            await self.write_status()
+            raise

tests/sync/test_watch_service.py

@@ -0,0 +1,166 @@
+"""Tests for the watch service."""
+
+from dataclasses import asdict
+import pytest
+import pytest_asyncio
+from datetime import datetime
+from pathlib import Path
+from watchfiles import Change
+
+from basic_memory.sync.utils import FileChange
+from basic_memory.sync.watch_service import WatchService, WatchEvent, WatchServiceState
+
+
+@pytest_asyncio.fixture
+def watch_service(sync_service, file_service, test_config):
+    return WatchService(
+        sync_service=sync_service,
+        file_service=file_service,
+        config=test_config
+    )
+
+
+@pytest_asyncio.fixture
+async def sample_markdown(tmp_path):
+    content = """---
+title: Test Note
+type: note
+---
+# Test Note
+This is a test note.
+"""
+    file_path = tmp_path / "test.md"
+    file_path.write_text(content)
+    return file_path
+
+
+@pytest.mark.asyncio
+async def test_handle_file_added(watch_service, sync_service, sample_markdown):
+    """Test handling a new file event"""
+    changes = {(Change.added, str(sample_markdown))}
+    
+    await watch_service.handle_changes(changes)
+    
+    # Check stats updated
+    assert watch_service.state.files_synced == 1
+    assert watch_service.state.bytes_processed > 0
+    
+    # Check event recorded
+    assert len(watch_service.state.recent_events) == 1
+    event = watch_service.state.recent_events[0]
+    assert event.path == "test.md"
+    assert event.action == "sync"
+    assert event.status == "success"
+
+
+@pytest.mark.asyncio
+async def test_handle_file_modified(watch_service, sync_service, sample_markdown):
+    """Test handling a modified file event"""
+    # First add the file
+    await watch_service.handle_changes({(Change.added, str(sample_markdown))})
+    
+    # Modify the file
+    sample_markdown.write_text(sample_markdown.read_text() + "\nModified content")
+    await watch_service.handle_changes({(Change.modified, str(sample_markdown))})
+    
+    # Should have two events
+    assert len(watch_service.state.recent_events) == 2
+    assert watch_service.state.files_synced == 2
+
+
+@pytest.mark.asyncio
+async def test_handle_file_deleted(watch_service, sync_service, sample_markdown):
+    """Test handling a deleted file event"""
+    # First add the file
+    await watch_service.handle_changes({(Change.added, str(sample_markdown))})
+    
+    # Delete the file
+    sample_markdown.unlink()
+    await watch_service.handle_changes({(Change.deleted, str(sample_markdown))})
+    
+    # Should have two events
+    assert len(watch_service.state.recent_events) == 2
+    delete_event = watch_service.state.recent_events[0]
+    assert delete_event.action == "sync"
+    assert delete_event.path == "test.md"
+
+
+@pytest.mark.asyncio
+async def test_filter_changes(watch_service):
+    """Test change filtering"""
+    assert watch_service.filter_changes(Change.added, "test.md") is True
+    assert watch_service.filter_changes(Change.added, "test.txt") is False
+    assert watch_service.filter_changes(Change.added, ".test.md") is False
+
+
+@pytest.mark.asyncio
+async def test_handle_error(watch_service, sync_service, ):
+    """Test error handling during sync"""
+    
+    changes = {(Change.added, "test.md")}
+    
+    # ValueError raised because "test.md" path does not exist
+    with pytest.raises(ValueError):
+        await watch_service.handle_changes(changes)
+    
+    assert watch_service.state.error_count == 1
+    assert watch_service.state.last_error is not None
+    
+
+@pytest.mark.asyncio
+async def test_recent_events_limit(watch_service):
+    """Test that recent events are limited to 100"""
+    for i in range(150):
+        watch_service.state.add_event(
+            path=f"test{i}.md",
+            action="sync",
+            status="success"
+        )
+    
+    assert len(watch_service.state.recent_events) == 100
+    # Most recent should be at the start
+    assert watch_service.state.recent_events[0].path == "test149.md"
+
+
+@pytest.mark.asyncio
+async def test_state_serialization(watch_service):
+    """Test state serializes to dict correctly"""
+    # Add some test state
+    watch_service.state.running = True
+    watch_service.state.files_synced = 42
+    watch_service.state.add_event(
+        path="test.md",
+        action="sync",
+        status="success"
+    )
+    
+    data = WatchServiceState.model_dump(watch_service.state)
+    
+    # Check basic fields
+    assert data['running'] is True
+    assert data['files_synced'] == 42
+    assert isinstance(data['start_time'], datetime)
+    
+    # Check events serialized
+    assert len(data['recent_events']) == 1
+    event = data['recent_events'][0]
+    assert event['path'] == "test.md"
+    assert event['action'] == "sync"
+    assert isinstance(event['timestamp'], datetime)
+
+
+@pytest.mark.asyncio
+async def test_status_file(watch_service, tmp_path):
+    """Test status file writing"""
+    watch_service.state.running = True
+    await watch_service.write_status()
+    
+    status_file = watch_service.status_path
+    assert status_file.exists()
+    
+    # Should be valid JSON with our fields
+    import json
+    data = json.loads(status_file.read_text())
+    assert data['running'] is True
+    assert isinstance(data['start_time'], str)
+    assert data['pid'] > 0

tests/sync/test_watch_sync.py

@@ -0,0 +1,129 @@
+"""Tests for watch service integration with sync service."""
+
+import pytest
+import pytest_asyncio
+from pathlib import Path
+from watchfiles import Change
+
+from basic_memory.sync.utils import FileChange
+
+
+@pytest.mark.asyncio
+async def test_sync_with_file_changes(sync_service, tmp_path):
+    """Test sync with file changes from watch service"""
+    # Create a test file
+    content = """---
+title: Test Note
+type: note
+---
+# Test Note
+This is a test."""
+    
+    file_path = tmp_path / "test.md"
+    file_path.write_text(content)
+    
+    # Create FileChange for a new file
+    changes = {
+        str(file_path): FileChange(
+            change_type=Change.added,
+            path=str(file_path),
+            checksum="abc123"
+        )
+    }
+    
+    report = await sync_service.sync(file_changes=changes)
+    assert str(file_path) in report.new
+    assert report.checksums[str(file_path)] == "abc123"
+
+
+@pytest.mark.asyncio
+async def test_sync_mixed_changes(sync_service, tmp_path):
+    """Test sync with multiple types of changes"""
+    # Setup initial files
+    new_file = tmp_path / "new.md"
+    new_file.write_text("New file")
+    
+    mod_file = tmp_path / "modified.md"
+    mod_file.write_text("Original content")
+    
+    del_file = tmp_path / "deleted.md"
+    
+    changes = {
+        str(new_file): FileChange(
+            change_type=Change.added,
+            path=str(new_file),
+            checksum="new123"
+        ),
+        str(mod_file): FileChange(
+            change_type=Change.modified,
+            path=str(mod_file),
+            checksum="mod123"
+        ),
+        str(del_file): FileChange(
+            change_type=Change.deleted,
+            path=str(del_file)
+        )
+    }
+    
+    report = await sync_service.sync(file_changes=changes)
+    
+    # Verify report contains all changes
+    assert str(new_file) in report.new
+    assert str(mod_file) in report.modified
+    assert str(del_file) in report.deleted
+    
+    # Check checksums
+    assert report.checksums[str(new_file)] == "new123"
+    assert report.checksums[str(mod_file)] == "mod123"
+    assert str(del_file) not in report.checksums
+
+
+@pytest.mark.asyncio
+async def test_sync_requires_params(sync_service):
+    """Test sync requires either directory or file_changes"""
+    with pytest.raises(ValueError):
+        await sync_service.sync()
+
+
+@pytest.mark.asyncio
+async def test_sync_with_invalid_change_type(sync_service, tmp_path):
+    """Test handling of invalid change types"""
+    file_path = tmp_path / "test.md"
+    file_path.touch()
+    
+    # Type checker won't let us create this directly, so ignore the type
+    changes = {
+        str(file_path): FileChange(
+            change_type="invalid",  # type: ignore
+            path=str(file_path),
+            checksum="test123"
+        )
+    }
+    
+    with pytest.raises(Exception):
+        await sync_service.sync(file_changes=changes)
+
+
+@pytest.mark.asyncio
+async def test_end_to_end_watch_sync(watch_service, sync_service, tmp_path):
+    """Test complete watch -> sync flow"""
+    # Create a test file that will trigger a watch event
+    test_file = tmp_path / "test.md"
+    test_file.write_text("""---
+title: Test Note
+type: note
+---
+# Test
+Testing watch -> sync flow
+""")
+    
+    # Simulate watchfiles event
+    changes = {(Change.added, str(test_file))}
+    await watch_service.handle_changes(changes)
+    
+    # Check watch service state
+    assert watch_service.state.files_synced == 1
+    assert len(watch_service.state.recent_events) == 1
+    event = watch_service.state.recent_events[0]
+    assert event.path == str(test_file)
+    assert event.status == "success"

uv.lock

@@ -198,6 +198,7 @@ dependencies = [
     { name = "sqlalchemy" },
     { name = "typer" },
     { name = "unidecode" },
+    { name = "watchfiles" },
 ]
 
 [package.optional-dependencies]
@@ -238,6 +239,7 @@ requires-dist = [
     { name = "sqlalchemy", specifier = ">=2.0.0" },
     { name = "typer", specifier = ">=0.9.0" },
     { name = "unidecode", specifier = ">=1.3.8" },
+    { name = "watchfiles", specifier = ">=1.0.4" },
 ]
 
 [package.metadata.requires-dev]