From 82603be8fafdf395cc06adb6b9b1f20f21ce72e8 Mon Sep 17 00:00:00 2001 From: phernandez Date: Thu, 2 Jan 2025 21:42:11 -0600 Subject: [PATCH] file-sync v1 --- src/basic_memory/models/documents.py | 2 +- .../services/sync/file_change_scanner.py | 5 +- .../services/sync/sync_service.py | 12 +-- tests/conftest.py | 9 ++ .../services/sync/test_file_change_scanner.py | 4 +- .../sync/test_knowledge_sync_service.py | 8 -- tests/services/sync/test_sync_service.py | 82 ++++++++++--------- 7 files changed, 65 insertions(+), 57 deletions(-) diff --git a/src/basic_memory/models/documents.py b/src/basic_memory/models/documents.py index 6cb04921..6b1c5086 100644 --- a/src/basic_memory/models/documents.py +++ b/src/basic_memory/models/documents.py @@ -46,4 +46,4 @@ class Document(Base): ) def __repr__(self) -> str: - return f"Document(id={self.id}, path='{self.path}', checksum='{self.checksum}', created_at='{self.created_at}', updated_at='{self.updated_at}')" + return f"Document(id={self.id}, path_id='{self.path_id}', checksum='{self.checksum}', created_at='{self.created_at}', updated_at='{self.updated_at}')" diff --git a/src/basic_memory/services/sync/file_change_scanner.py b/src/basic_memory/services/sync/file_change_scanner.py index de80cf44..cdb37ecf 100644 --- a/src/basic_memory/services/sync/file_change_scanner.py +++ b/src/basic_memory/services/sync/file_change_scanner.py @@ -161,12 +161,13 @@ class FileChangeScanner: db_files = {} for record in db_records: + # TODO - why file_path? # Use file_path if available, otherwise use path_id path = record.file_path if record.file_path is not None else record.path_id if record.checksum: - db_files[path] = FileState( - path=path, + db_files[record.path_id] = FileState( + path=record.path_id, checksum=record.checksum ) diff --git a/src/basic_memory/services/sync/sync_service.py b/src/basic_memory/services/sync/sync_service.py index a014443c..abfaa988 100644 --- a/src/basic_memory/services/sync/sync_service.py +++ b/src/basic_memory/services/sync/sync_service.py @@ -20,12 +20,12 @@ class SyncService: self, scanner: FileChangeScanner, document_service: DocumentService, - entity_service: KnowledgeSyncService, + knowledge_sync_service: KnowledgeSyncService, knowledge_parser: KnowledgeParser, ): self.scanner = scanner self.document_service = document_service - self.entity_service = entity_service + self.knowledge_sync_service = knowledge_sync_service self.knowledge_parser = knowledge_parser async def sync_documents(self, directory: Path) -> None: @@ -56,7 +56,7 @@ class SyncService: # Handle deletions first for path_id in changes.deleted: logger.debug(f"Deleting entity: {path_id}") - await self.entity_service.delete_entity(path_id) + await self.knowledge_sync_service.delete_entity(path_id) # Parse files that need updating parsed_entities = {} @@ -68,15 +68,15 @@ class SyncService: for path_id, entity in parsed_entities.items(): if path_id in changes.new: logger.debug(f"Creating new entity: {path_id}") - await self.entity_service.create_entity_and_observations(entity) + await self.knowledge_sync_service.create_entity_and_observations(entity) else: logger.debug(f"Updating entity: {path_id}") - await self.entity_service.update_entity_and_observations(entity) + await self.knowledge_sync_service.update_entity_and_observations(entity) # Second pass: Process relations for path_id, entity in parsed_entities.items(): logger.debug(f"Updating relations for: {path_id}") - await self.entity_service.update_entity_relations(entity) + await self.knowledge_sync_service.update_entity_relations(entity, checksum=changes.checksums[path_id]) async def sync(self, root_dir: Path) -> None: """Sync all files with database.""" diff --git a/tests/conftest.py b/tests/conftest.py index 7b7f9704..e65e4b4e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -28,6 +28,7 @@ from basic_memory.services import ( from basic_memory.services.activity_service import ActivityService from basic_memory.services.file_service import FileService from basic_memory.services import KnowledgeService +from basic_memory.services.sync.knowledge_sync_service import KnowledgeSyncService @pytest_asyncio.fixture @@ -177,6 +178,14 @@ async def activity_service(document_service, entity_service, relation_service): """Create activity service with real dependencies.""" return ActivityService(entity_service, document_service, relation_service) +@pytest_asyncio.fixture +async def knowledge_sync_service( + entity_service: EntityService, + observation_service: ObservationService, + relation_service: RelationService, +) -> KnowledgeSyncService: + """Create EntitySyncService with repository.""" + return KnowledgeSyncService(entity_service, observation_service, relation_service) @pytest_asyncio.fixture(scope="function") diff --git a/tests/services/sync/test_file_change_scanner.py b/tests/services/sync/test_file_change_scanner.py index 8910cd37..385c8eab 100644 --- a/tests/services/sync/test_file_change_scanner.py +++ b/tests/services/sync/test_file_change_scanner.py @@ -177,8 +177,8 @@ async def test_get_db_state_entities( db_records = await file_change_scanner.get_db_state([entity]) assert len(db_records) == 1 - assert "concept/test.md" in db_records - assert db_records["concept/test.md"].checksum == "test-checksum" + assert "concept/test" in db_records + assert db_records["concept/test"].checksum == "test-checksum" @pytest.mark.asyncio diff --git a/tests/services/sync/test_knowledge_sync_service.py b/tests/services/sync/test_knowledge_sync_service.py index e06fd7ad..ff86b3ab 100644 --- a/tests/services/sync/test_knowledge_sync_service.py +++ b/tests/services/sync/test_knowledge_sync_service.py @@ -17,14 +17,6 @@ from basic_memory.markdown.schemas import ( ) -@pytest_asyncio.fixture -async def knowledge_sync_service( - entity_service: EntityService, - observation_service: ObservationService, - relation_service: RelationService, -) -> KnowledgeSyncService: - """Create EntitySyncService with repository.""" - return KnowledgeSyncService(entity_service, observation_service, relation_service) @pytest_asyncio.fixture diff --git a/tests/services/sync/test_sync_service.py b/tests/services/sync/test_sync_service.py index 7cd6dd96..1a463f13 100644 --- a/tests/services/sync/test_sync_service.py +++ b/tests/services/sync/test_sync_service.py @@ -2,17 +2,19 @@ from pathlib import Path import pytest +import pytest_asyncio from basic_memory.services import DocumentService, EntityService, FileChangeScanner +from basic_memory.services.sync.knowledge_sync_service import KnowledgeSyncService from basic_memory.services.sync.sync_service import SyncService from basic_memory.markdown import KnowledgeParser, EntityMarkdown from basic_memory.models import Document, Entity, Observation -@pytest.fixture +@pytest_asyncio.fixture async def sync_service( document_service: DocumentService, - entity_service: EntityService, + knowledge_sync_service: KnowledgeSyncService, file_change_scanner: FileChangeScanner, knowledge_parser: KnowledgeParser, ) -> SyncService: @@ -20,7 +22,7 @@ async def sync_service( return SyncService( scanner=file_change_scanner, document_service=document_service, - entity_service=entity_service, + knowledge_sync_service=knowledge_sync_service, knowledge_parser=knowledge_parser, ) @@ -65,25 +67,22 @@ async def test_sync_documents( doc = Document( path_id="modified.md", file_path="modified.md", - content="original content" + checksum="12345678" ) - await document_service.repository.add(doc) + added = await document_service.repository.add(doc) # Run sync await sync_service.sync(root_dir) # Verify results - async with document_service.repository.session() as session: - documents = await document_service.repository.find_all(session) - assert len(documents) == 2 + documents = await document_service.repository.find_all() + assert len(documents) == 2 + + paths = {d.path_id for d in documents} + assert "new.md" in paths + assert "modified.md" in paths - paths = {d.path_id for d in documents} - assert "new.md" in paths - assert "modified.md" in paths - - # Check content was updated - modified = next(d for d in documents if d.path_id == "modified.md") - assert modified.content == "modified document" + @pytest.mark.asyncio @@ -97,14 +96,20 @@ async def test_sync_knowledge( knowledge_dir = root_dir / "knowledge" # New entity with relation - new_content = """--- + new_content = """ +--- type: concept id: concept/test_concept +created: 2023-01-01 +modified: 2023-01-01 --- # Test Concept A test concept. +## Observations +- [design] Core feature + ## Relations - depends_on [[concept/other]] """ @@ -114,7 +119,8 @@ A test concept. other = Entity( path_id="concept/other", name="Other", - entity_type="concept" + entity_type="concept", + file_path="concept/other.md" ) await entity_service.repository.add(other) @@ -122,19 +128,19 @@ A test concept. await sync_service.sync(root_dir) # Verify results - async with entity_service.repository.session() as session: - entities = await entity_service.repository.find_all(session) - assert len(entities) == 2 - - # Find new entity - test_concept = next(e for e in entities if e.path_id == "concept/test_concept") - assert test_concept.entity_type == "concept" - - # Verify relation was created - relations = await entity_service.repository.get_relations(session, test_concept.id) - assert len(relations) == 1 - assert relations[0].relation_type == "depends_on" - assert relations[0].to_id == other.id + entities = await entity_service.repository.find_all() + assert len(entities) == 2 + + # Find new entity + test_concept: Entity = next(e for e in entities if e.path_id == "concept/test_concept") + assert test_concept.entity_type == "concept" + + # Verify relation was created + entity = await entity_service.get_by_path_id(test_concept.path_id) + relations = entity.relations + assert len(relations) == 1 + assert relations[0].relation_type == "depends_on" + assert relations[0].to_id == other.id @pytest.mark.asyncio @@ -149,14 +155,16 @@ async def test_sync_deletes( doc = Document( path_id="deleted.md", file_path="deleted.md", - content="deleted content" + checksum="12345678" ) await document_service.repository.add(doc) entity = Entity( path_id="concept/deleted", name="Deleted", - entity_type="concept" + entity_type="concept", + file_path="concept/deleted.md", + checksum = "12345678" ) await entity_service.repository.add(entity) @@ -164,10 +172,8 @@ async def test_sync_deletes( await sync_service.sync(root_dir) # Verify deletions - async with document_service.repository.session() as session: - docs = await document_service.repository.find_all(session) - assert len(docs) == 0 + docs = await document_service.repository.find_all() + assert len(docs) == 0 - async with entity_service.repository.session() as session: - entities = await entity_service.repository.find_all(session) - assert len(entities) == 0 \ No newline at end of file + entities = await entity_service.repository.find_all() + assert len(entities) == 0 \ No newline at end of file