add file watcher to sync command

This commit is contained in:
phernandez
2025-02-02 20:07:04 -06:00
parent b7b69ecd84
commit a88fc99663
5 changed files with 45 additions and 24 deletions
+32 -16
View File
@@ -31,6 +31,7 @@ from basic_memory.services.link_resolver import LinkResolver
from basic_memory.services.search_service import SearchService
from basic_memory.sync import SyncService, FileChangeScanner
from basic_memory.sync.utils import SyncReport
from basic_memory.sync.watch_service import WatchService
console = Console()
@@ -48,7 +49,7 @@ async def get_sync_service(db_type=DatabaseType.FILESYSTEM):
session_maker,
):
entity_parser = EntityParser(config.home)
markdown_processor = MarkdownProcessor(entity_parser)
markdown_processor = MarkdownProcessor(entity_parser)
file_service = FileService(config.home, markdown_processor)
# Initialize repositories
@@ -64,15 +65,14 @@ async def get_sync_service(db_type=DatabaseType.FILESYSTEM):
# Initialize scanner
file_change_scanner = FileChangeScanner(entity_repository)
# Initialize services
entity_service = EntityService(
entity_parser,
entity_repository,
observation_repository,
entity_parser,
entity_repository,
observation_repository,
relation_repository,
file_service,
link_resolver
link_resolver,
)
# Create sync service
@@ -114,7 +114,7 @@ def display_validation_errors(issues: List[ValidationIssue]):
for dir_name, dir_issues in sorted(grouped_issues.items()):
# Create branch for directory
branch = tree.add(
f"[bold blue]{dir_name}/[/bold blue] " f"([yellow]{len(dir_issues)} files[/yellow])"
f"[bold blue]{dir_name}/[/bold blue] ([yellow]{len(dir_issues)} files[/yellow])"
)
# Add each file issue
@@ -202,19 +202,29 @@ def display_detailed_sync_results(knowledge: SyncReport):
console.print(knowledge_tree)
async def run_sync(verbose: bool = False):
async def run_sync(verbose: bool = False, watch: bool = False):
"""Run sync operation."""
sync_service = await get_sync_service()
# Sync
knowledge_changes = await sync_service.sync(config.home)
# Display results
if verbose:
display_detailed_sync_results(knowledge_changes)
# Start watching if requested
if watch:
console.print("\n[cyan]Starting watch service...[/cyan]")
watch_service = WatchService(
sync_service=sync_service,
file_service=sync_service.entity_service.file_service,
config=config
)
await watch_service.handle_changes(config.home)
await watch_service.run()
else:
display_sync_summary(knowledge_changes)
# one time sync
knowledge_changes = await sync_service.sync(config.home)
# Display results
if verbose:
display_detailed_sync_results(knowledge_changes)
else:
display_sync_summary(knowledge_changes)
@app.command()
@@ -225,11 +235,17 @@ def sync(
"-v",
help="Show detailed sync information.",
),
watch: bool = typer.Option(
False,
"--watch",
"-w",
help="Start watching for changes after sync.",
),
) -> None:
"""Sync knowledge files with the database."""
try:
# Run sync
asyncio.run(run_sync(verbose))
asyncio.run(run_sync(verbose=verbose, watch=watch))
except Exception as e:
if not isinstance(e, typer.Exit):
+2 -1
View File
@@ -95,7 +95,7 @@ class FileChangeScanner:
current_files = scan_result.files
# Build report
report = SyncReport()
report = SyncReport(total=len(current_files))
# Track potentially moved files by checksum
files_by_checksum = {} # checksum -> file_path
@@ -127,6 +127,7 @@ class FileChangeScanner:
report.deleted.add(db_file_path)
# Log summary
logger.debug(f"Total files: {report.total}")
logger.debug(f"Changes found: {report.total_changes}")
logger.debug(f" New: {len(report.new)}")
logger.debug(f" Modified: {len(report.modified)}")
+3 -1
View File
@@ -41,12 +41,14 @@ class SyncReport:
"""Report of file changes found compared to database state.
Attributes:
total: Total number of files in directory being synced
new: Files that exist on disk but not in database
modified: Files that exist in both but have different checksums
deleted: Files that exist in database but not on disk
moves: Files that have been moved from one location to another
checksums: Current checksums for files on disk
"""
total: int = 0
new: Set[str] = field(default_factory=set)
modified: Set[str] = field(default_factory=set)
deleted: Set[str] = field(default_factory=set)
@@ -59,6 +61,6 @@ class SyncReport:
return len(self.new) + len(self.modified) + len(self.deleted) + len(self.moves)
@property
def total_files(self) -> int:
def syned_files(self) -> int:
"""Total number of files synced."""
return len(self.new) + len(self.modified) + len(self.moves)
+5 -3
View File
@@ -3,6 +3,7 @@
import json
import dataclasses
from loguru import logger
from pydantic import BaseModel
from pydantic.dataclasses import dataclass
from datetime import datetime
@@ -37,8 +38,7 @@ class WatchServiceState(BaseModel):
last_scan: Optional[datetime] = None
# File counts
total_files: int = 0
markdown_files: int = 0
synced_files: int = 0
# Recent activity
recent_events: List[WatchEvent] = dataclasses.field(default_factory=list)
@@ -78,6 +78,7 @@ class WatchService:
debounce=self.config.sync_delay,
recursive=True,
):
# just sync the whole dir
await self.handle_changes(self.config.home)
except Exception as e:
@@ -100,10 +101,11 @@ class WatchService:
"""Process a batch of file changes"""
try:
logger.debug(f"handling change in directory: {directory} ...")
# Process changes with timeout
report = await self.sync_service.sync(directory)
self.state.last_scan = datetime.now()
self.state.total_files = report.total_files
self.state.synced_files = report.total
# Update stats
for path in report.new: