From a88fc996632b4fc14319afe0ce27b329d7f67c2d Mon Sep 17 00:00:00 2001 From: phernandez Date: Sun, 2 Feb 2025 20:07:04 -0600 Subject: [PATCH] add file watcher to sync command --- src/basic_memory/cli/commands/sync.py | 48 +++++++++++++------- src/basic_memory/sync/file_change_scanner.py | 3 +- src/basic_memory/sync/utils.py | 4 +- src/basic_memory/sync/watch_service.py | 8 ++-- tests/sync/test_watch_service.py | 6 +-- 5 files changed, 45 insertions(+), 24 deletions(-) diff --git a/src/basic_memory/cli/commands/sync.py b/src/basic_memory/cli/commands/sync.py index 70ac6eaf..801148cc 100644 --- a/src/basic_memory/cli/commands/sync.py +++ b/src/basic_memory/cli/commands/sync.py @@ -31,6 +31,7 @@ from basic_memory.services.link_resolver import LinkResolver from basic_memory.services.search_service import SearchService from basic_memory.sync import SyncService, FileChangeScanner from basic_memory.sync.utils import SyncReport +from basic_memory.sync.watch_service import WatchService console = Console() @@ -48,7 +49,7 @@ async def get_sync_service(db_type=DatabaseType.FILESYSTEM): session_maker, ): entity_parser = EntityParser(config.home) - markdown_processor = MarkdownProcessor(entity_parser) + markdown_processor = MarkdownProcessor(entity_parser) file_service = FileService(config.home, markdown_processor) # Initialize repositories @@ -64,15 +65,14 @@ async def get_sync_service(db_type=DatabaseType.FILESYSTEM): # Initialize scanner file_change_scanner = FileChangeScanner(entity_repository) - # Initialize services entity_service = EntityService( - entity_parser, - entity_repository, - observation_repository, + entity_parser, + entity_repository, + observation_repository, relation_repository, file_service, - link_resolver + link_resolver, ) # Create sync service @@ -114,7 +114,7 @@ def display_validation_errors(issues: List[ValidationIssue]): for dir_name, dir_issues in sorted(grouped_issues.items()): # Create branch for directory branch = tree.add( - f"[bold blue]{dir_name}/[/bold blue] " f"([yellow]{len(dir_issues)} files[/yellow])" + f"[bold blue]{dir_name}/[/bold blue] ([yellow]{len(dir_issues)} files[/yellow])" ) # Add each file issue @@ -202,19 +202,29 @@ def display_detailed_sync_results(knowledge: SyncReport): console.print(knowledge_tree) -async def run_sync(verbose: bool = False): +async def run_sync(verbose: bool = False, watch: bool = False): """Run sync operation.""" sync_service = await get_sync_service() - # Sync - knowledge_changes = await sync_service.sync(config.home) - - # Display results - if verbose: - display_detailed_sync_results(knowledge_changes) + # Start watching if requested + if watch: + console.print("\n[cyan]Starting watch service...[/cyan]") + watch_service = WatchService( + sync_service=sync_service, + file_service=sync_service.entity_service.file_service, + config=config + ) + await watch_service.handle_changes(config.home) + await watch_service.run() else: - display_sync_summary(knowledge_changes) + # one time sync + knowledge_changes = await sync_service.sync(config.home) + # Display results + if verbose: + display_detailed_sync_results(knowledge_changes) + else: + display_sync_summary(knowledge_changes) @app.command() @@ -225,11 +235,17 @@ def sync( "-v", help="Show detailed sync information.", ), + watch: bool = typer.Option( + False, + "--watch", + "-w", + help="Start watching for changes after sync.", + ), ) -> None: """Sync knowledge files with the database.""" try: # Run sync - asyncio.run(run_sync(verbose)) + asyncio.run(run_sync(verbose=verbose, watch=watch)) except Exception as e: if not isinstance(e, typer.Exit): diff --git a/src/basic_memory/sync/file_change_scanner.py b/src/basic_memory/sync/file_change_scanner.py index 7125897f..9109fe13 100644 --- a/src/basic_memory/sync/file_change_scanner.py +++ b/src/basic_memory/sync/file_change_scanner.py @@ -95,7 +95,7 @@ class FileChangeScanner: current_files = scan_result.files # Build report - report = SyncReport() + report = SyncReport(total=len(current_files)) # Track potentially moved files by checksum files_by_checksum = {} # checksum -> file_path @@ -127,6 +127,7 @@ class FileChangeScanner: report.deleted.add(db_file_path) # Log summary + logger.debug(f"Total files: {report.total}") logger.debug(f"Changes found: {report.total_changes}") logger.debug(f" New: {len(report.new)}") logger.debug(f" Modified: {len(report.modified)}") diff --git a/src/basic_memory/sync/utils.py b/src/basic_memory/sync/utils.py index 57d917ec..e0d5f94c 100644 --- a/src/basic_memory/sync/utils.py +++ b/src/basic_memory/sync/utils.py @@ -41,12 +41,14 @@ class SyncReport: """Report of file changes found compared to database state. Attributes: + total: Total number of files in directory being synced new: Files that exist on disk but not in database modified: Files that exist in both but have different checksums deleted: Files that exist in database but not on disk moves: Files that have been moved from one location to another checksums: Current checksums for files on disk """ + total: int = 0 new: Set[str] = field(default_factory=set) modified: Set[str] = field(default_factory=set) deleted: Set[str] = field(default_factory=set) @@ -59,6 +61,6 @@ class SyncReport: return len(self.new) + len(self.modified) + len(self.deleted) + len(self.moves) @property - def total_files(self) -> int: + def syned_files(self) -> int: """Total number of files synced.""" return len(self.new) + len(self.modified) + len(self.moves) diff --git a/src/basic_memory/sync/watch_service.py b/src/basic_memory/sync/watch_service.py index 096219aa..09b6d294 100644 --- a/src/basic_memory/sync/watch_service.py +++ b/src/basic_memory/sync/watch_service.py @@ -3,6 +3,7 @@ import json import dataclasses +from loguru import logger from pydantic import BaseModel from pydantic.dataclasses import dataclass from datetime import datetime @@ -37,8 +38,7 @@ class WatchServiceState(BaseModel): last_scan: Optional[datetime] = None # File counts - total_files: int = 0 - markdown_files: int = 0 + synced_files: int = 0 # Recent activity recent_events: List[WatchEvent] = dataclasses.field(default_factory=list) @@ -78,6 +78,7 @@ class WatchService: debounce=self.config.sync_delay, recursive=True, ): + # just sync the whole dir await self.handle_changes(self.config.home) except Exception as e: @@ -100,10 +101,11 @@ class WatchService: """Process a batch of file changes""" try: + logger.debug(f"handling change in directory: {directory} ...") # Process changes with timeout report = await self.sync_service.sync(directory) self.state.last_scan = datetime.now() - self.state.total_files = report.total_files + self.state.synced_files = report.total # Update stats for path in report.new: diff --git a/tests/sync/test_watch_service.py b/tests/sync/test_watch_service.py index a3e569b4..d2e60271 100644 --- a/tests/sync/test_watch_service.py +++ b/tests/sync/test_watch_service.py @@ -29,7 +29,7 @@ async def test_handle_file_added(test_config, watch_service, sync_service, sampl await watch_service.handle_changes(test_config.home) # Check stats updated - assert watch_service.state.total_files == 1 + assert watch_service.state.synced_files == 1 assert watch_service.state.last_scan is not None # Check event recorded @@ -52,7 +52,7 @@ async def test_handle_file_modified(test_config, watch_service, sync_service, sa # Should have two events assert len(watch_service.state.recent_events) == 2 - assert watch_service.state.total_files == 1 + assert watch_service.state.synced_files == 1 event = watch_service.state.recent_events[0] assert event.path == "test.md" assert event.action == "modified" @@ -72,7 +72,7 @@ async def test_handle_file_moved(test_config, watch_service, sync_service, sampl # Should have two events assert len(watch_service.state.recent_events) == 2 - assert watch_service.state.total_files == 1 + assert watch_service.state.synced_files == 1 event = watch_service.state.recent_events[0] assert event.path == "test.md" assert event.action == "moved"