watch service

src/basic_memory/config.py

@@ -22,6 +22,10 @@ class ProjectConfig(BaseSettings):
     # Name of the project
     project: str = Field(default="default", description="Project name")
 
+    # Watch service configuration
+    sync_delay: int = Field(
+        default=500, description="Milliseconds to wait after changes before syncing", gt=0
+    )
 
     model_config = SettingsConfigDict(
         env_prefix="BASIC_MEMORY_",

src/basic_memory/services/file_service.py

@@ -208,3 +208,6 @@ class FileService:
         except Exception as e:
             logger.error(f"Failed to delete file {path}: {e}")
             raise FileOperationError(f"Failed to delete file: {e}")
+
+    def path(self, path_string: str, absolute: bool = False):
+        return Path( self.base_path / path_string ) if absolute else Path(path_string).relative_to(self.base_path)

src/basic_memory/sync/sync_service.py

@@ -1,7 +1,7 @@
 """Service for syncing files between filesystem and database."""
 
 from pathlib import Path
-from typing import Dict
+from typing import Dict, Optional
 
 from loguru import logger
 
@@ -10,7 +10,8 @@ from basic_memory.repository import EntityRepository, RelationRepository
 from basic_memory.services import EntityService
 from basic_memory.services.search_service import SearchService
 from basic_memory.sync import FileChangeScanner
-from basic_memory.sync.utils import SyncReport
+from basic_memory.sync.utils import SyncReport, FileChange
+from watchfiles import Change
 
 
 class SyncService:
@@ -60,10 +61,28 @@ class SyncService:
         else:
             logger.debug(f"No entity found to delete: {file_path}")
 
-    async def sync(self, directory: Path) -> SyncReport:
+    async def sync(self, directory: Optional[Path] = None, file_changes: Optional[dict[str, FileChange]] = None) -> SyncReport:
         """Sync knowledge files with database."""
-        changes = await self.scanner.find_knowledge_changes(directory)
-        logger.info(f"Found {changes.total_changes} knowledge changes")
+        if file_changes is not None:
+            changes = SyncReport()
+            for path, file_change in file_changes.items():
+                logger.debug(f"path {path} file_change {file_change}")
+                match file_change.change_type:
+                    case Change.added:
+                        changes.new.add(path)
+                        changes.checksums[path] = file_change.checksum
+                    case Change.modified:
+                        changes.modified.add(path)
+                        changes.checksums[path] = file_change.checksum
+                    case Change.deleted:
+                        changes.deleted.add(path)
+        else:
+            # Traditional directory scan mode
+            if directory is None:
+                raise ValueError("Must provide either directory or file_changes")
+            
+            changes = await self.scanner.find_knowledge_changes(directory)
+            logger.info(f"Found {changes.total_changes} knowledge changes")
 
         # Handle moves first
         for old_path, new_path in changes.moves.items():
@@ -86,7 +105,7 @@ class SyncService:
         parsed_entities: Dict[str, EntityMarkdown] = {}
 
         for file_path in [*changes.new, *changes.modified]:
-            entity_markdown = await self.entity_parser.parse_file(directory / file_path)
+            entity_markdown = await self.entity_parser.parse_file(Path(file_path))
             parsed_entities[file_path] = entity_markdown
 
         # First pass: Create/update entities

src/basic_memory/sync/utils.py

@@ -2,6 +2,38 @@
 
 from dataclasses import dataclass, field
 from typing import Set, Dict, Optional
+from watchfiles import Change
+from basic_memory.services.file_service import FileService
+
+
+@dataclass
+class FileChange:
+    """A change to a file detected by the watch service.
+    
+    Attributes:
+        change_type: Type of change (added, modified, deleted)
+        path: Path to the file
+        checksum: File checksum (None for deleted files)
+    """
+    change_type: Change
+    path: str
+    checksum: Optional[str] = None
+
+    @classmethod
+    async def from_path(cls, path: str, change_type: Change, file_service: FileService) -> "FileChange":
+        """Create FileChange from a path, computing checksum if file exists.
+        
+        Args:
+            path: Path to the file
+            change_type: Type of change detected
+            file_service: Service to read file and compute checksum
+            
+        Returns:
+            FileChange with computed checksum for non-deleted files
+        """
+        file_path = file_service.path(path)
+        content, checksum = await file_service.read_file(file_path) if change_type != Change.deleted else (None, None)
+        return cls(path=file_path, change_type=change_type, checksum=checksum)
 
 
 @dataclass
@@ -18,7 +50,7 @@ class SyncReport:
     new: Set[str] = field(default_factory=set)
     modified: Set[str] = field(default_factory=set)
     deleted: Set[str] = field(default_factory=set)
-    moves: Dict[str, str] = field(default_factory=dict)  # old_path -> new_path
+    moves: Dict[str, str] = field(default_factory=dict)  # old_path -> new_path  
     checksums: Dict[str, str] = field(default_factory=dict)
 
     @property

src/basic_memory/sync/watch_service.py

@@ -0,0 +1,131 @@
+"""Watch service for Basic Memory."""
+
+import json
+import dataclasses
+
+from pydantic import BaseModel
+from pydantic.dataclasses import dataclass
+from datetime import datetime
+from pathlib import Path
+from typing import List, Optional
+from watchfiles import awatch, Change
+import os
+
+from basic_memory.config import ProjectConfig
+from basic_memory.sync.sync_service import SyncService
+from basic_memory.services.file_service import FileService
+from basic_memory.sync.utils import FileChange
+
+
+class WatchEvent(BaseModel):
+    timestamp: datetime
+    path: str
+    action: str  # sync, delete, etc
+    status: str  # success, error
+    error: Optional[str] = None
+
+
+class WatchServiceState(BaseModel):
+    # Service status
+    running: bool = False
+    start_time: datetime = dataclasses.field(default_factory=datetime.now)
+    pid: int = dataclasses.field(default_factory=os.getpid)
+
+    # Stats
+    files_synced: int = 0
+    bytes_processed: int = 0
+    error_count: int = 0
+    last_error: Optional[datetime] = None
+    last_scan: Optional[datetime] = None
+
+    # File counts
+    total_files: int = 0
+    markdown_files: int = 0
+
+    # Recent activity
+    recent_events: List[WatchEvent] = dataclasses.field(default_factory=list)
+
+    def add_event(self, path: str, action: str, status: str, error: Optional[str] = None):
+        event = WatchEvent(
+            timestamp=datetime.now(), path=path, action=action, status=status, error=error
+        )
+        self.recent_events.insert(0, event)
+        self.recent_events = self.recent_events[:100]  # Keep last 100
+
+    def record_error(self, error: str):
+        self.error_count += 1
+        self.last_error = datetime.now()
+
+
+class WatchService:
+    def __init__(self, sync_service: SyncService, file_service: FileService, config: ProjectConfig):
+        self.sync_service = sync_service
+        self.file_service = file_service
+        self.config = config
+        self.state = WatchServiceState()
+        self.status_path = config.home / ".basic-memory" / "watch-status.json"
+        self.status_path.parent.mkdir(parents=True, exist_ok=True)
+
+    async def run(self):
+        """Watch for file changes and sync them"""
+        self.state.running = True
+        self.state.start_time = datetime.now()
+        await self.write_status()
+
+        try:
+            async for changes in awatch(
+                self.config.home,
+                watch_filter=self.filter_changes,
+                debounce=self.config.sync_delay,
+                recursive=True,
+            ):
+                await self.handle_changes(changes)
+
+        except Exception as e:
+            self.state.record_error(str(e))
+            await self.write_status()
+            raise
+        finally:
+            self.state.running = False
+            await self.write_status()
+
+    async def write_status(self):
+        """Write current state to status file"""
+        self.status_path.write_text(WatchServiceState.model_dump_json(self.state, indent=2))
+
+    def filter_changes(self, change: Change, path: str) -> bool:
+        """Filter to only watch markdown files"""
+        return path.endswith(".md") and not Path(path).name.startswith(".")
+
+    async def handle_changes(self, changes: set[tuple[Change, str]]):
+        """Process a batch of file changes"""
+
+        # Group changes by file path
+        changes_by_file = {}
+        try:
+            for change_type, path in changes:
+                file_change = await FileChange.from_path(path, change_type, self.file_service)
+                
+                # store changes by relative path
+                changes_by_file[str(file_change.path)] = file_change
+
+            # Process changes with timeout
+            await self.sync_service.sync(file_changes=changes_by_file)
+
+            # Update stats
+            self.state.files_synced += len(changes_by_file)
+            for path, change in changes_by_file.items():
+                if change.change_type != Change.deleted:
+                    size = self.file_service.path(path,absolute=True).stat().st_size
+                    self.state.bytes_processed += size
+
+                self.state.add_event(path=path, action="sync", status="success")
+
+            await self.write_status()
+
+        except Exception as e:
+            self.state.record_error(str(e))
+            for path in changes_by_file:
+                self.state.add_event(path=path, action="sync", status="error", error=str(e))
+            await self.write_status()
+            raise