From 74adae506ef66ad9f3bab4e9eb0fb24a5a09e184 Mon Sep 17 00:00:00 2001 From: phernandez Date: Fri, 21 Feb 2025 21:36:39 -0600 Subject: [PATCH] fix tests --- src/basic_memory/sync/file_change_scanner.py | 162 ------------ src/basic_memory/sync/sync_service_old.py | 174 ------------- tests/sync/test_file_change_scanner.py | 245 ------------------- 3 files changed, 581 deletions(-) delete mode 100644 src/basic_memory/sync/file_change_scanner.py delete mode 100644 src/basic_memory/sync/sync_service_old.py delete mode 100644 tests/sync/test_file_change_scanner.py diff --git a/src/basic_memory/sync/file_change_scanner.py b/src/basic_memory/sync/file_change_scanner.py deleted file mode 100644 index d6da834a..00000000 --- a/src/basic_memory/sync/file_change_scanner.py +++ /dev/null @@ -1,162 +0,0 @@ -"""Service for detecting changes between filesystem and database.""" - -from dataclasses import dataclass, field -from pathlib import Path -from typing import Dict, Sequence - -from loguru import logger - -from basic_memory.file_utils import compute_checksum -from basic_memory.models import Entity -from basic_memory.repository.entity_repository import EntityRepository -from basic_memory.sync.utils import SyncReport - - -@dataclass -class FileState: - """State of a file including file path, permalink and checksum info.""" - - file_path: str - permalink: str - checksum: str - - -@dataclass -class ScanResult: - """Result of scanning a directory.""" - - # file_path -> checksum - files: Dict[str, str] = field(default_factory=dict) - - # checksum -> file_path - checksums: Dict[str, str] = field(default_factory=dict) - - # file_path -> error message - errors: Dict[str, str] = field(default_factory=dict) - - -class FileChangeScanner: - """ - Service for detecting changes between filesystem and database. - The filesystem is treated as the source of truth. - """ - - def __init__(self, entity_repository: EntityRepository): - self.entity_repository = entity_repository - - async def scan_directory(self, directory: Path) -> ScanResult: - """ - Scan directory for markdown files and their checksums. - Only processes .md files, logs and skips others. - - Args: - directory: Directory to scan - - Returns: - ScanResult containing found files and any errors - """ - logger.debug(f"Scanning directory: {directory}") - result = ScanResult() - - if not directory.exists(): - logger.debug(f"Directory does not exist: {directory}") - return result - - for path in directory.rglob("*"): - if not path.is_file() or not path.name.endswith(".md"): - if path.is_file(): - logger.debug(f"Skipping non-markdown file: {path}") - continue - - try: - # Get relative path first - used in error reporting if needed - rel_path = str(path.relative_to(directory)) - content = path.read_text() - checksum = await compute_checksum(content) - result.files[rel_path] = checksum - - except Exception as e: - rel_path = str(path.relative_to(directory)) - result.errors[rel_path] = str(e) - logger.error(f"Failed to read {rel_path}: {e}") - - logger.debug(f"Found {len(result.files)} markdown files") - if result.errors: - logger.warning(f"Encountered {len(result.errors)} errors while scanning") - - return result - - async def find_changes( - self, directory: Path, db_file_state: Dict[str, FileState] - ) -> SyncReport: - """Find changes between filesystem and database.""" - # Get current files and checksums - scan_result = await self.scan_directory(directory) - current_files = scan_result.files - - # Build report - report = SyncReport(total=len(current_files)) - - # Track potentially moved files by checksum - files_by_checksum = {} # checksum -> file_path - - # First find potential new files and record checksums - for file_path, checksum in current_files.items(): - logger.debug(f"{file_path} ({checksum[:8]})") - - if file_path not in db_file_state: - # Could be new or could be the destination of a move - report.new.add(file_path) - files_by_checksum[checksum] = file_path - elif checksum != db_file_state[file_path].checksum: - report.modified.add(file_path) - - report.checksums[file_path] = checksum - - # Now detect moves and deletions - for db_file_path, db_state in db_file_state.items(): - if db_file_path not in current_files: - if db_state.checksum in files_by_checksum: - # Found a move - file exists at new path with same checksum - new_path = files_by_checksum[db_state.checksum] - report.moves[db_file_path] = new_path - # Remove from new files since it's a move - report.new.remove(new_path) - else: - # Actually deleted - report.deleted.add(db_file_path) - - # Log summary - logger.debug(f"Total files: {report.total}") - logger.debug(f"Changes found: {report.total_changes}") - logger.debug(f" New: {len(report.new)}") - logger.debug(f" Modified: {len(report.modified)}") - logger.debug(f" Moved: {len(report.moves)}") - logger.debug(f" Deleted: {len(report.deleted)}") - - if scan_result.errors: # pragma: no cover - logger.warning("Files skipped due to errors:") - for file_path, error in scan_result.errors.items(): - logger.warning(f" {file_path}: {error}") - - return report - - async def get_db_file_state(self, db_records: Sequence[Entity]) -> Dict[str, FileState]: - """Get file_path and checksums from database. - Args: - db_records: database records - Returns: - Dict mapping file paths to FileState - :param db_records: the data from the db - """ - return { - r.file_path: FileState( - file_path=r.file_path, permalink=r.permalink, checksum=r.checksum or "" - ) - for r in db_records - } - - async def find_knowledge_changes(self, directory: Path) -> SyncReport: - """Find changes in knowledge directory.""" - db_file_state = await self.get_db_file_state(await self.entity_repository.find_all()) - return await self.find_changes(directory=directory, db_file_state=db_file_state) \ No newline at end of file diff --git a/src/basic_memory/sync/sync_service_old.py b/src/basic_memory/sync/sync_service_old.py deleted file mode 100644 index 5159231d..00000000 --- a/src/basic_memory/sync/sync_service_old.py +++ /dev/null @@ -1,174 +0,0 @@ -"""Service for syncing files between filesystem and database.""" - -from pathlib import Path -from typing import Dict - -import logfire -from loguru import logger -from sqlalchemy.exc import IntegrityError - -from basic_memory import file_utils -from basic_memory.markdown import EntityParser, EntityMarkdown -from basic_memory.repository import EntityRepository, RelationRepository -from basic_memory.services import EntityService -from basic_memory.services.search_service import SearchService -from basic_memory.sync import FileChangeScanner -from basic_memory.sync.utils import SyncReport - - -class SyncService: - """Syncs documents and knowledge files with database. - - Implements two-pass sync strategy for knowledge files to handle relations: - 1. First pass creates/updates entities without relations - 2. Second pass processes relations after all entities exist - """ - - def __init__( - self, - scanner: FileChangeScanner, - entity_service: EntityService, - entity_parser: EntityParser, - entity_repository: EntityRepository, - relation_repository: RelationRepository, - search_service: SearchService, - ): - self.scanner = scanner - self.entity_service = entity_service - self.entity_parser = entity_parser - self.entity_repository = entity_repository - self.relation_repository = relation_repository - self.search_service = search_service - - async def handle_entity_deletion(self, file_path: str): - """Handle complete entity deletion including search index cleanup.""" - # First get entity to get permalink before deletion - entity = await self.entity_repository.get_by_file_path(file_path) - if entity: - logger.debug(f"Deleting entity and cleaning up search index: {file_path}") - - # Delete from db (this cascades to observations/relations) - await self.entity_service.delete_entity_by_file_path(file_path) - - # Clean up search index - permalinks = ( - [entity.permalink] - + [o.permalink for o in entity.observations] - + [r.permalink for r in entity.relations] - ) - logger.debug(f"Deleting from search index: {permalinks}") - for permalink in permalinks: - await self.search_service.delete_by_permalink(permalink) - - async def sync(self, directory: Path) -> SyncReport: - """Sync knowledge files with database.""" - - with logfire.span("sync", directory=directory): # pyright: ignore [reportGeneralTypeIssues] - changes = await self.scanner.find_knowledge_changes(directory) - logger.info(f"Found {changes.total_changes} knowledge changes") - - # Handle moves first - for old_path, new_path in changes.moves.items(): - logger.debug(f"Moving entity: {old_path} -> {new_path}") - entity = await self.entity_repository.get_by_file_path(old_path) - if entity: - # Update file_path but keep the same permalink for link stability - updated = await self.entity_repository.update( - entity.id, {"file_path": new_path, "checksum": changes.checksums[new_path]} - ) - # update search index - if updated: - await self.search_service.index_entity(updated) - - # Handle deletions next - # remove rows from db for files no longer present - for path in changes.deleted: - await self.handle_entity_deletion(path) - - # Parse files that need updating - parsed_entities: Dict[str, EntityMarkdown] = {} - - for path in [*changes.new, *changes.modified]: - entity_markdown = await self.entity_parser.parse_file(directory / path) - parsed_entities[path] = entity_markdown - - # First pass: Create/update entities - # entities will have a null checksum to indicate they are not complete - for path, entity_markdown in parsed_entities.items(): - # Get unique permalink and update markdown if needed - permalink = await self.entity_service.resolve_permalink( - Path(path), markdown=entity_markdown - ) - - if permalink != entity_markdown.frontmatter.permalink: - # Add/update permalink in frontmatter - logger.info(f"Adding permalink '{permalink}' to file: {path}") - - # update markdown - entity_markdown.frontmatter.metadata["permalink"] = permalink - - # update file frontmatter - updated_checksum = await file_utils.update_frontmatter( - directory / path, {"permalink": permalink} - ) - - # Update checksum in changes report since file was modified - changes.checksums[path] = updated_checksum - - # if the file is new, create an entity - if path in changes.new: - # Create entity with final permalink - logger.debug(f"Creating new entity_markdown: {path}") - await self.entity_service.create_entity_from_markdown( - Path(path), entity_markdown - ) - # otherwise we need to update the entity and observations - else: - logger.debug(f"Updating entity_markdown: {path}") - await self.entity_service.update_entity_and_observations( - Path(path), entity_markdown - ) - - # Second pass - for path, entity_markdown in parsed_entities.items(): - logger.debug(f"Updating relations for: {path}") - - # Process relations - checksum = changes.checksums[path] - entity = await self.entity_service.update_entity_relations( - Path(path), entity_markdown - ) - - # add to search index - await self.search_service.index_entity(entity) - - # Set final checksum to mark sync complete - await self.entity_repository.update(entity.id, {"checksum": checksum}) - - # Third pass: Try to resolve any forward references - logger.debug("Attempting to resolve forward references") - for relation in await self.relation_repository.find_unresolved_relations(): - target_entity = await self.entity_service.link_resolver.resolve_link( - relation.to_name - ) - # check we found a link that is not the source - if target_entity and target_entity.id != relation.from_id: - logger.debug( - f"Resolved forward reference: {relation.to_name} -> {target_entity.permalink}" - ) - - try: - await self.relation_repository.update( - relation.id, - { - "to_id": target_entity.id, - "to_name": target_entity.title, # Update to actual title - }, - ) - except IntegrityError: - logger.debug(f"Ignoring duplicate relation {relation}") - - # update search index - await self.search_service.index_entity(target_entity) - - return changes diff --git a/tests/sync/test_file_change_scanner.py b/tests/sync/test_file_change_scanner.py deleted file mode 100644 index 60fd3cfc..00000000 --- a/tests/sync/test_file_change_scanner.py +++ /dev/null @@ -1,245 +0,0 @@ -"""Test file sync service.""" - -from pathlib import Path - -import pytest - -from basic_memory.file_utils import compute_checksum -from basic_memory.models import Entity -from basic_memory.sync import FileChangeScanner -from basic_memory.sync.file_change_scanner import FileState - - -@pytest.fixture -def temp_dir(tmp_path: Path) -> Path: - """Create temp directory for test files.""" - return tmp_path - - -async def create_test_file(path: Path, content: str = "test content") -> None: - """Create a test file with given content.""" - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(content) - - -@pytest.mark.asyncio -async def test_scan_empty_directory(file_change_scanner: FileChangeScanner, temp_dir: Path): - """Test scanning empty directory.""" - result = await file_change_scanner.scan_directory(temp_dir) - assert len(result.files) == 0 - assert len(result.errors) == 0 - - -@pytest.mark.asyncio -async def test_scan_with_mixed_files(file_change_scanner: FileChangeScanner, temp_dir: Path): - """Test scanning directory with markdown and non-markdown files.""" - # Create test files - await create_test_file(temp_dir / "doc.md", "markdown") - await create_test_file(temp_dir / "text.txt", "not markdown") - await create_test_file(temp_dir / "notes/deep.md", "nested markdown") - - result = await file_change_scanner.scan_directory(temp_dir) - assert len(result.files) == 2 - assert "doc.md" in result.files - assert "notes/deep.md" in result.files - assert len(result.errors) == 0 - - # Verify FileState objects - assert isinstance(result.files["doc.md"], str) - # checksum - assert result.files["doc.md"] is not None - - -@pytest.mark.asyncio -async def test_scan_with_unreadable_file(file_change_scanner: FileChangeScanner, temp_dir: Path): - """Test scanning directory with an unreadable file.""" - # Create a file we'll make unreadable - bad_file = temp_dir / "bad.md" - await create_test_file(bad_file) - bad_file.chmod(0o000) # Remove all permissions - - result = await file_change_scanner.scan_directory(temp_dir) - assert len(result.files) == 0 - assert len(result.errors) == 1 - assert "bad.md" in result.errors - - -@pytest.mark.asyncio -async def test_detect_new_files( - file_change_scanner: FileChangeScanner, - temp_dir: Path, -): - """Test detection of new files.""" - # Create new file - await create_test_file(temp_dir / "new.md") - - # Empty DB state - db_records = await file_change_scanner.get_db_file_state([]) - - changes = await file_change_scanner.find_changes(directory=temp_dir, db_file_state=db_records) - - assert len(changes.new) == 1 - assert "new.md" in changes.new - - -@pytest.mark.asyncio -async def test_detect_modified_file(file_change_scanner: FileChangeScanner, temp_dir: Path): - """Test detection of modified files.""" - file_path = "test.md" - content = "original" - await create_test_file(temp_dir / file_path, content) - - # Create DB state with original checksum - original_checksum = await compute_checksum(content) - db_records = { - file_path: FileState(file_path=file_path, permalink="test", checksum=original_checksum) - } - - # Modify file - await create_test_file(temp_dir / file_path, "modified") - - changes = await file_change_scanner.find_changes(directory=temp_dir, db_file_state=db_records) - - assert len(changes.modified) == 1 - assert file_path in changes.modified - - -@pytest.mark.asyncio -async def test_detect_deleted_files(file_change_scanner: FileChangeScanner, temp_dir: Path): - """Test detection of deleted files.""" - file_path = "deleted.md" - - # Create DB state with file that doesn't exist - db_records = { - file_path: FileState(file_path=file_path, permalink="deleted", checksum="any-checksum") - } - - changes = await file_change_scanner.find_changes(directory=temp_dir, db_file_state=db_records) - - assert len(changes.deleted) == 1 - assert file_path in changes.deleted - - -@pytest.mark.asyncio -async def test_get_db_state_entities(file_change_scanner: FileChangeScanner): - """Test converting entity records to file states.""" - entity = Entity(permalink="concept/test", file_path="concept/test.md", checksum="test-checksum") - - db_records = await file_change_scanner.get_db_file_state([entity]) - - assert len(db_records) == 1 - assert "concept/test.md" in db_records - assert db_records["concept/test.md"].checksum == "test-checksum" - - -@pytest.mark.asyncio -async def test_empty_directory(file_change_scanner: FileChangeScanner, temp_dir: Path): - """Test handling empty/nonexistent directory.""" - nonexistent = temp_dir / "nonexistent" - - changes = await file_change_scanner.find_changes(directory=nonexistent, db_file_state={}) - - assert changes.total_changes == 0 - assert not changes.new - assert not changes.modified - assert not changes.deleted - - -@pytest.mark.asyncio -async def test_detect_moved_file(file_change_scanner: FileChangeScanner, temp_dir: Path): - """Test detection of file moves.""" - # Create original file - old_path = "original/test.md" - new_path = "new/location/test.md" - content = "test content" - - await create_test_file(temp_dir / old_path, content) - original_checksum = await compute_checksum(content) - - # Set up DB state with original location - db_records = { - old_path: FileState(file_path=old_path, permalink="test", checksum=original_checksum) - } - - # Move file to new location - old_file = temp_dir / old_path - new_file = temp_dir / new_path - new_file.parent.mkdir(parents=True, exist_ok=True) - old_file.rename(new_file) - - # Check changes - changes = await file_change_scanner.find_changes(directory=temp_dir, db_file_state=db_records) - - # Should detect as move - assert len(changes.moves) == 1 - assert changes.moves[old_path] == new_path - # Should not be in new or deleted - assert old_path not in changes.new - assert old_path not in changes.deleted - assert new_path not in changes.new - - -@pytest.mark.asyncio -async def test_move_with_content_change(file_change_scanner: FileChangeScanner, temp_dir: Path): - """Test handling a file that is both moved and modified.""" - # Create original file - old_path = "original/test.md" - new_path = "new/location/test.md" - content = "original content" - - await create_test_file(temp_dir / old_path, content) - original_checksum = await compute_checksum(content) - - # Set up DB state with original location - db_records = { - old_path: FileState(file_path=old_path, permalink="test", checksum=original_checksum) - } - - # Move file and change content - old_file = temp_dir / old_path - new_file = temp_dir / new_path - new_file.parent.mkdir(parents=True, exist_ok=True) - await create_test_file(new_file, "modified content") - old_file.unlink() - - # Check changes - changes = await file_change_scanner.find_changes(directory=temp_dir, db_file_state=db_records) - - # Should be treated as delete + new, not move - assert old_path in changes.deleted - assert new_path in changes.new - assert len(changes.moves) == 0 - - -@pytest.mark.asyncio -async def test_multiple_moves(file_change_scanner: FileChangeScanner, temp_dir: Path): - """Test detecting multiple file moves at once.""" - # Create original files - files = {"a/test1.md": "content1", "b/test2.md": "content2"} - new_locations = {"a/test1.md": "new/test1.md", "b/test2.md": "new/nested/test2.md"} - - db_records = {} - # Create files and DB state - for old_path, content in files.items(): - await create_test_file(temp_dir / old_path, content) - checksum = await compute_checksum(content) - db_records[old_path] = FileState( - file_path=old_path, permalink=old_path.replace(".md", ""), checksum=checksum - ) - - # Move all files - for old_path, new_path in new_locations.items(): - old_file = temp_dir / old_path - new_file = temp_dir / new_path - new_file.parent.mkdir(parents=True, exist_ok=True) - old_file.rename(new_file) - - # Check changes - changes = await file_change_scanner.find_changes(directory=temp_dir, db_file_state=db_records) - - # Should detect both moves - assert len(changes.moves) == 2 - assert changes.moves["a/test1.md"] == "new/test1.md" - assert changes.moves["b/test2.md"] == "new/nested/test2.md" - assert not changes.new - assert not changes.deleted