diff --git a/src/basic_memory/cli/commands/status.py b/src/basic_memory/cli/commands/status.py index 10e79158..0fe33b93 100644 --- a/src/basic_memory/cli/commands/status.py +++ b/src/basic_memory/cli/commands/status.py @@ -4,6 +4,7 @@ import asyncio from typing import Set, Dict import typer +from loguru import logger from rich.console import Console from rich.panel import Panel from rich.tree import Tree @@ -179,5 +180,6 @@ def status( sync_service = asyncio.run(get_sync_service()) asyncio.run(run_status(sync_service, verbose)) except Exception as e: + logger.error(f"Error checking status: {e}") typer.echo(f"Error checking status: {e}", err=True) raise typer.Exit(1) \ No newline at end of file diff --git a/src/basic_memory/services/file_sync_service.py b/src/basic_memory/services/file_sync_service.py index 4f5296c0..825ae5a7 100644 --- a/src/basic_memory/services/file_sync_service.py +++ b/src/basic_memory/services/file_sync_service.py @@ -2,7 +2,7 @@ from dataclasses import dataclass, field from pathlib import Path -from typing import Set, Dict, Protocol, TypeVar, Generic, List, Optional +from typing import Set, Dict, Protocol, TypeVar, Generic, Optional from loguru import logger @@ -16,7 +16,6 @@ class FileState: """State of a file including path and checksum info.""" path: str checksum: str - normalized_path: str # For comparison moved_from: Optional[str] = None @@ -37,7 +36,9 @@ class SyncReport: class DbRecord(Protocol): """Protocol for database records with path and checksum.""" @property - def path(self) -> str: ... + def file_path(self) -> Optional[str]: ... + @property + def path_id(self) -> str: ... @property def checksum(self) -> str: ... @@ -94,9 +95,7 @@ class FileSyncService: async def find_changes( self, directory: Path, - get_records: callable, - normalize_path: callable = lambda x: x, - get_record_path: callable = lambda x: x.path_id + get_records: callable ) -> SyncReport: """ Find changes between filesystem and database. @@ -104,8 +103,6 @@ class FileSyncService: Args: directory: Directory to check get_records: Function to get database records - normalize_path: Function to normalize paths for comparison - get_record_path: Function to get path from record Returns: SyncReport detailing changes @@ -121,34 +118,32 @@ class FileSyncService: for path, checksum in current_files.items(): report.checksums[path] = checksum - # Build DB state with normalized paths + # Build DB state - use path_id if file_path is NULL db_records = await get_records() - db_files: Dict[str, tuple[str, str]] = { - normalize_path(get_record_path(record)): (get_record_path(record), record.checksum) - for record in db_records - } - - logger.debug("Files from database:") - for norm_path, (orig_path, checksum) in sorted(db_files.items()): - logger.debug(f" {norm_path} ({checksum[:8]}) [original: {orig_path}]") + db_files = {} + for record in db_records: + path = record.file_path if record.file_path is not None else record.path_id + db_files[path] = (path, record.checksum) - # Track files by checksum to detect moves - checksum_locations: Dict[str, List[str]] = {} - for path, checksum in current_files.items(): - norm_path = normalize_path(path) - locations = checksum_locations.setdefault(checksum, []) - locations.append(norm_path) + logger.debug("Files from database:") + for path, (_, checksum) in sorted(db_files.items()): + logger.debug(f" {path} ({checksum[:8]})") + + # Track files by checksum for move detection + db_paths_by_checksum = {} + for path, (_, checksum) in db_files.items(): + paths = db_paths_by_checksum.setdefault(checksum, []) + paths.append(path) processed_current = set() processed_db = set() # First pass - check for unchanged and modified files for curr_path, curr_checksum in current_files.items(): - norm_curr_path = normalize_path(curr_path) - if norm_curr_path in db_files: - db_orig_path, db_checksum = db_files[norm_curr_path] - processed_current.add(norm_curr_path) - processed_db.add(norm_curr_path) + if curr_path in db_files: + _, db_checksum = db_files[curr_path] + processed_current.add(curr_path) + processed_db.add(curr_path) if curr_checksum != db_checksum: logger.debug(f"Modified: {curr_path} (checksum changed)") @@ -156,38 +151,35 @@ class FileSyncService: # Second pass - look for moves for curr_path, curr_checksum in current_files.items(): - norm_curr_path = normalize_path(curr_path) - if norm_curr_path in processed_current: + if curr_path in processed_current: continue - # Look for files with same checksum in DB + # Look for any files with same checksum in DB was_move = False - for db_norm_path, (db_orig_path, db_checksum) in db_files.items(): - if db_norm_path in processed_db: - continue - if curr_checksum == db_checksum: - logger.debug(f"Moved: {db_orig_path} -> {curr_path}") - report.moved[curr_path] = FileState( - path=curr_path, - checksum=curr_checksum, - normalized_path=norm_curr_path, - moved_from=db_orig_path - ) - processed_current.add(norm_curr_path) - processed_db.add(db_norm_path) - was_move = True - break - + if curr_checksum in db_paths_by_checksum: + for db_path in db_paths_by_checksum[curr_checksum]: + if db_path not in processed_db: + logger.debug(f"Moved: {db_path} -> {curr_path}") + report.moved[curr_path] = FileState( + path=curr_path, + checksum=curr_checksum, + moved_from=db_path + ) + processed_current.add(curr_path) + processed_db.add(db_path) + was_move = True + break + if not was_move: logger.debug(f"New: {curr_path}") report.new.add(curr_path) - processed_current.add(norm_curr_path) + processed_current.add(curr_path) # Remaining DB files must be deleted - for db_norm_path, (db_orig_path, _) in db_files.items(): - if db_norm_path not in processed_db: - logger.debug(f"Deleted: {db_orig_path}") - report.deleted.add(db_orig_path) + for path, (db_path, _) in db_files.items(): + if path not in processed_db: + logger.debug(f"Deleted: {db_path}") + report.deleted.add(db_path) # Log summary logger.debug(f"Changes found: {report.total_changes}") @@ -200,35 +192,14 @@ class FileSyncService: async def find_document_changes(self, directory: Path) -> SyncReport: """Find changes in document directory.""" - def normalize_doc_path(path: str) -> str: - """Normalize document paths.""" - return str(Path(path)) - return await self.find_changes( directory=directory, - get_records=self.document_repository.find_all, - normalize_path=normalize_doc_path + get_records=self.document_repository.find_all ) async def find_knowledge_changes(self, directory: Path) -> SyncReport: """Find changes in knowledge directory.""" - def normalize_entity_path(path: str) -> str: - """Normalize entity paths.""" - path = path.lower() - if path.endswith('.md'): - path = path[:-3] - return path - - def get_entity_path(entity) -> str: - """Get path from entity record.""" - path = entity.path_id - if not path.endswith('.md'): - path = f"{path}.md" - return path - return await self.find_changes( directory=directory, - get_records=self.entity_repository.find_all, - normalize_path=normalize_entity_path, - get_record_path=get_entity_path + get_records=self.entity_repository.find_all ) \ No newline at end of file diff --git a/tests/services/test_file_sync_service.py b/tests/services/test_file_sync_service.py index e69de29b..9ac13a2c 100644 --- a/tests/services/test_file_sync_service.py +++ b/tests/services/test_file_sync_service.py @@ -0,0 +1,200 @@ +"""Tests for file sync service.""" + +import asyncio +from pathlib import Path +from typing import AsyncGenerator +import pytest +import pytest_asyncio +from sqlalchemy import text +from sqlalchemy.ext.asyncio import AsyncSession + +from basic_memory import db +from basic_memory.repository import DocumentRepository, EntityRepository +from basic_memory.services import FileSyncService +from basic_memory.utils.file_utils import compute_checksum + +@pytest_asyncio.fixture +def temp_dir(test_config): + return test_config.home + + +@pytest.mark.asyncio +async def create_test_file(path: Path, content: str = "test content") -> None: + """Create a test file with given content.""" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content) + +@pytest.mark.asyncio +async def test_scan_directory_empty( + file_sync_service: FileSyncService, + temp_dir: Path +): + """Test scanning empty directory.""" + files = await file_sync_service.scan_directory(temp_dir) + assert len(files) == 0 + +@pytest.mark.asyncio +async def test_scan_directory_with_files( + file_sync_service: FileSyncService, + temp_dir: Path +): + """Test scanning directory with markdown files.""" + # Create test files + await create_test_file(temp_dir / "test1.md", "content 1") + await create_test_file(temp_dir / "test2.md", "content 2") + await create_test_file(temp_dir / "not-markdown.txt", "ignore me") + + files = await file_sync_service.scan_directory(temp_dir) + assert len(files) == 2 + assert "test1.md" in files + assert "test2.md" in files + assert "not-markdown.txt" not in files + +@pytest.mark.asyncio +async def test_find_new_files( + file_sync_service: FileSyncService, + temp_dir: Path +): + """Test detection of new files.""" + # Create new file + await create_test_file(temp_dir / "new_file.md", "new content") + + changes = await file_sync_service.find_changes( + directory=temp_dir, + get_records=lambda: asyncio.sleep(0, []) # Empty DB + ) + + assert len(changes.new) == 1 + assert "new_file.md" in changes.new + assert len(changes.modified) == 0 + assert len(changes.deleted) == 0 + assert len(changes.moved) == 0 + +@pytest.mark.asyncio +async def test_find_case_sensitive_move( + file_sync_service: FileSyncService, + temp_dir: Path, + session_maker: AsyncGenerator[AsyncSession, None] +): + """Test detection of case-sensitive file moves.""" + # Create and track original file + original_path = "Original.md" + await create_test_file(temp_dir / original_path, "test content") + original_checksum = await compute_checksum("test content") + + # Add to DB with original path + async with db.scoped_session(session_maker) as session: + await session.execute(text( + f"INSERT INTO document (path_id, file_path, checksum) VALUES ('{original_path.lower()}', '{original_path}', '{original_checksum}')")) + await session.commit() + + # Rename file (case change only) + new_path = "ORIGINAL.md" + (temp_dir / original_path).rename(temp_dir / new_path) + + async def get_records(): + return session.execute( + text("SELECT * FROM document") + ).fetchall() + + changes = await file_sync_service.find_changes( + directory=temp_dir, + get_records=(await get_records), + normalize_path=lambda p: p.lower() + ) + + assert len(changes.moved) == 1 + assert new_path in changes.moved + assert changes.moved[new_path].moved_from == original_path + assert len(changes.new) == 0 + assert len(changes.modified) == 0 + assert len(changes.deleted) == 0 + + + + +@pytest.mark.asyncio +async def test_find_modified_files( + file_sync_service: FileSyncService, + temp_dir: Path, + session_maker: AsyncGenerator[AsyncSession, None] +): + """Test detection of modified files.""" + # Create and track original file + path = "test.md" + await create_test_file(temp_dir / path, "original content") + original_checksum = await compute_checksum("original content") + + # Add to DB + async with db.scoped_session(session_maker) as session: + await session.execute( + "INSERT INTO document (path_id, file_path, checksum) VALUES (?, ?, ?)", + [path.lower(), path, original_checksum] + ) + await session.commit() + + # Modify file + await create_test_file(temp_dir / path, "modified content") + + changes = await file_sync_service.find_changes( + directory=temp_dir, + get_records=lambda: session.execute( + "SELECT * FROM document" + ).fetchall(), + normalize_path=lambda p: p.lower() + ) + + assert len(changes.modified) == 1 + assert path in changes.modified + assert len(changes.new) == 0 + assert len(changes.deleted) == 0 + assert len(changes.moved) == 0 + +@pytest.mark.asyncio +async def test_find_deleted_files( + file_sync_service: FileSyncService, + temp_dir: Path, + session_maker: AsyncGenerator[AsyncSession, None] +): + """Test detection of deleted files.""" + # Add file to DB that doesn't exist + missing_path = "deleted.md" + async with db.scoped_session(session_maker) as session: + await session.execute( + "INSERT INTO document (path_id, file_path, checksum) VALUES (?, ?, ?)", + [missing_path.lower(), missing_path, "any-checksum"] + ) + await session.commit() + + changes = await file_sync_service.find_changes( + directory=temp_dir, + get_records=lambda: session.execute( + "SELECT * FROM document" + ).fetchall(), + normalize_path=lambda p: p.lower() + ) + + assert len(changes.deleted) == 1 + assert missing_path in changes.deleted + assert len(changes.new) == 0 + assert len(changes.modified) == 0 + assert len(changes.moved) == 0 + +@pytest.mark.asyncio +async def test_path_normalization( + file_sync_service: FileSyncService, + temp_dir: Path +): + """Test path normalization for different formats.""" + paths = [ + ("my file.md", "my_file"), # Spaces + ("MyFile.md", "myfile"), # Case + ("my/file.md", "my/file"), # Subdirectories + ("MY/FILE.md", "my/file"), # Mixed case dirs + ("my//file.md", "my/file"), # Extra slashes + ("./my/file.md", "my/file"), # Current dir + ] + + for file_path, expected_norm in paths: + norm_path = file_sync_service.normalize_path(file_path) + assert norm_path == expected_norm, f"Failed to normalize {file_path}" \ No newline at end of file