diff --git a/src/basic_memory/cli/commands/sync.py b/src/basic_memory/cli/commands/sync.py index daebe195..30418321 100644 --- a/src/basic_memory/cli/commands/sync.py +++ b/src/basic_memory/cli/commands/sync.py @@ -187,7 +187,7 @@ async def validate_knowledge_files( for file_path in [*changes.new, *changes.modified]: try: - await sync_service.knowledge_parser.parse_file(directory / file_path) + await sync_service.entity_parser.parse_file(directory / file_path) except ParseError as e: issues.append(ValidationIssue(file_path=file_path, error=str(e))) diff --git a/src/basic_memory/repository/entity_repository.py b/src/basic_memory/repository/entity_repository.py index e546219f..8fb13077 100644 --- a/src/basic_memory/repository/entity_repository.py +++ b/src/basic_memory/repository/entity_repository.py @@ -28,6 +28,11 @@ class EntityRepository(Repository[Entity]): query = self.select().where(Entity.title == title).options(*self.get_load_options()) return await self.find_one(query) + async def get_by_file_path(self, file_path: str) -> Optional[Entity]: + """Get entity by file_path.""" + query = self.select().where(Entity.file_path == file_path).options(*self.get_load_options()) + return await self.find_one(query) + async def list_entities( self, entity_type: Optional[str] = None, diff --git a/src/basic_memory/sync/entity_sync_service.py b/src/basic_memory/sync/entity_sync_service.py index fdf2f0c3..fb4ac801 100644 --- a/src/basic_memory/sync/entity_sync_service.py +++ b/src/basic_memory/sync/entity_sync_service.py @@ -25,7 +25,7 @@ def entity_model_from_markdown(file_path: str, markdown: EntityMarkdown) -> Enti model = EntityModel( title=markdown.frontmatter.title, entity_type=markdown.frontmatter.type, - permalink=markdown.frontmatter.id, + permalink=markdown.frontmatter.permalink, file_path=file_path, content_type="text/markdown", summary=markdown.content.content, @@ -61,7 +61,7 @@ class EntitySyncService: Creates the entity with null checksum to indicate sync not complete. Relations will be added in second pass. """ - logger.debug(f"Creating entity without relations: {markdown.frontmatter.id}") + logger.debug(f"Creating entity without relations: {markdown.frontmatter.title}") model = entity_model_from_markdown(file_path, markdown) # Mark as incomplete sync @@ -69,17 +69,17 @@ class EntitySyncService: return await self.entity_repository.add(model) async def update_entity_and_observations( - self, permalink: str, markdown: EntityMarkdown + self, file_path: str, markdown: EntityMarkdown ) -> EntityModel: """First pass: Update entity fields and observations. Updates everything except relations and sets null checksum to indicate sync not complete. """ - logger.debug(f"Updating entity and observations: {permalink}") - db_entity = await self.entity_repository.get_by_permalink(permalink) + logger.debug(f"Updating entity and observations: {file_path}") + db_entity = await self.entity_repository.get_by_file_path(file_path) if not db_entity: - raise EntityNotFoundError(f"Entity not found: {permalink}") + raise EntityNotFoundError(f"Entity not found: {file_path}") # Update fields from markdown db_entity.title = markdown.frontmatter.title @@ -114,15 +114,11 @@ class EntitySyncService: }, ) - async def update_entity_relations(self, markdown: EntityMarkdown, checksum: str) -> EntityModel: + async def update_entity_relations(self, file_path: str, markdown: EntityMarkdown) -> EntityModel: """Second pass: Update relations and set checksum. - - Args: - markdown: Parsed markdown entity with relations - checksum: Final checksum to set after relations are updated """ - logger.debug(f"Updating relations for entity: {markdown.frontmatter.id}") - db_entity = await self.entity_repository.get_by_permalink(markdown.frontmatter.id) + logger.debug(f"Updating relations for entity: {file_path}") + db_entity = await self.entity_repository.get_by_file_path(file_path) # get all entities from relations target_entity_permalinks = [rel.target for rel in markdown.content.relations] @@ -159,5 +155,3 @@ class EntitySyncService: # Create unique relations await self.relation_repository.add_all(relation_dict.values()) - # Set final checksum to mark sync complete - return await self.entity_repository.update(db_entity.id, {"checksum": checksum}) diff --git a/src/basic_memory/sync/sync_service.py b/src/basic_memory/sync/sync_service.py index 68c1c1a4..c8c66e7d 100644 --- a/src/basic_memory/sync/sync_service.py +++ b/src/basic_memory/sync/sync_service.py @@ -1,10 +1,12 @@ """Service for syncing files between filesystem and database.""" from pathlib import Path +from typing import Dict from loguru import logger -from basic_memory.markdown import EntityParser +from basic_memory.markdown import EntityParser, EntityMarkdown +from basic_memory.repository import EntityRepository from basic_memory.services.search_service import SearchService from basic_memory.sync import FileChangeScanner from basic_memory.sync.entity_sync_service import EntitySyncService @@ -24,11 +26,13 @@ class SyncService: scanner: FileChangeScanner, entity_sync_service: EntitySyncService, entity_parser: EntityParser, + entity_repository: EntityRepository, search_service: SearchService, ): self.scanner = scanner - self.knowledge_sync_service = entity_sync_service - self.knowledge_parser = entity_parser + self.entity_sync_service = entity_sync_service + self.entity_parser = entity_parser + self.entity_repository = entity_repository self.search_service = search_service async def sync(self, directory: Path) -> SyncReport: @@ -40,35 +44,42 @@ class SyncService: # remove rows from db for files no longer present for file_path in changes.deleted: logger.debug(f"Deleting entity from db: {file_path}") - await self.knowledge_sync_service.delete_entity_by_file_path(file_path) + await self.entity_sync_service.delete_entity_by_file_path(file_path) # Parse files that need updating - parsed_entities = {} + parsed_entities: Dict[str, EntityMarkdown] = {} + for file_path in [*changes.new, *changes.modified]: - entity_markdown = await self.knowledge_parser.parse_file(directory / file_path) + entity_markdown = await self.entity_parser.parse_file(directory / file_path) parsed_entities[file_path] = entity_markdown # First pass: Create/update entities for file_path, entity_markdown in parsed_entities.items(): + # if the file is new, create an entity if file_path in changes.new: logger.debug(f"Creating new entity_markdown: {file_path}") - await self.knowledge_sync_service.create_entity_from_markdown( + await self.entity_sync_service.create_entity_from_markdown( file_path, entity_markdown ) + # otherwise we need to update the entity and observations else: - permalink = entity_markdown.frontmatter.id - logger.debug(f"Updating entity_markdown: {permalink}") - await self.knowledge_sync_service.update_entity_and_observations( - permalink, entity_markdown + logger.debug(f"Updating entity_markdown: {file_path}") + await self.entity_sync_service.update_entity_and_observations( + file_path, entity_markdown ) - # Second pass: Process relations + # Second pass for file_path, entity_markdown in parsed_entities.items(): logger.debug(f"Updating relations for: {file_path}") - entity = await self.knowledge_sync_service.update_entity_relations( - entity_markdown, checksum=changes.checksums[file_path] - ) + + # Process relations + checksum = changes.checksums[file_path] + entity = await self.entity_sync_service.update_entity_relations(file_path, entity_markdown) + # add to search index await self.search_service.index_entity(entity) + # Set final checksum to mark sync complete + return await self.entity_repository.update(entity.id, {"checksum": checksum}) + return changes diff --git a/tests/repository/test_entity_repository.py b/tests/repository/test_entity_repository.py index 3f9ab463..1b5477b8 100644 --- a/tests/repository/test_entity_repository.py +++ b/tests/repository/test_entity_repository.py @@ -526,3 +526,29 @@ async def test_get_by_title(entity_repository: EntityRepository, session_maker): found = await entity_repository.get_by_title("Non Existent") assert found is None + +@pytest.mark.asyncio +async def test_get_by_file_path(entity_repository: EntityRepository, session_maker): + """Test getting an entity by title.""" + # Create test entities + async with db.scoped_session(session_maker) as session: + entities = [ + Entity( + title="Unique Title", + entity_type="test", + permalink="test/unique-title", + file_path="test/unique-title.md", + content_type="text/markdown", + ), + ] + session.add_all(entities) + await session.flush() + + # Test getting by file_path + found = await entity_repository.get_by_file_path("test/unique-title.md") + assert found is not None + assert found.title == "Unique Title" + + # Test non-existent file_path + found = await entity_repository.get_by_file_path("not/a/real/file.md") + assert found is None diff --git a/tests/sync/test_sync_knowledge.py b/tests/sync/test_sync_knowledge.py index 93bbfc34..326cd82b 100644 --- a/tests/sync/test_sync_knowledge.py +++ b/tests/sync/test_sync_knowledge.py @@ -103,7 +103,7 @@ modified: 2024-01-01 await sync_service.sync(test_config.home) # Verify entity created but no relations - entity = await sync_service.knowledge_sync_service.entity_repository.get_by_permalink( + entity = await sync_service.entity_sync_service.entity_repository.get_by_permalink( "concept/depends_on_future" ) assert entity is not None @@ -157,10 +157,10 @@ modified: 2024-01-01 await sync_service.sync(test_config.home) # Verify both entities and their relations - entity_a = await sync_service.knowledge_sync_service.entity_repository.get_by_permalink( + entity_a = await sync_service.entity_sync_service.entity_repository.get_by_permalink( "concept/entity_a" ) - entity_b = await sync_service.knowledge_sync_service.entity_repository.get_by_permalink( + entity_b = await sync_service.entity_sync_service.entity_repository.get_by_permalink( "concept/entity_b" ) @@ -232,7 +232,7 @@ modified: 2024-01-01 await sync_service.sync(test_config.home) # Verify duplicates are handled - entity = await sync_service.knowledge_sync_service.entity_repository.get_by_permalink( + entity = await sync_service.entity_sync_service.entity_repository.get_by_permalink( "concept/duplicate_relations" ) @@ -274,7 +274,7 @@ modified: 2024-01-01 await sync_service.sync(test_config.home) # Verify observations - entity = await sync_service.knowledge_sync_service.entity_repository.get_by_permalink( + entity = await sync_service.entity_sync_service.entity_repository.get_by_permalink( "concept/invalid_category" ) @@ -353,13 +353,13 @@ modified: 2024-01-01 await sync_service.sync(test_config.home) # Verify all relations are created correctly regardless of order - entity_a = await sync_service.knowledge_sync_service.entity_repository.get_by_permalink( + entity_a = await sync_service.entity_sync_service.entity_repository.get_by_permalink( "concept/entity_a" ) - entity_b = await sync_service.knowledge_sync_service.entity_repository.get_by_permalink( + entity_b = await sync_service.entity_sync_service.entity_repository.get_by_permalink( "concept/entity_b" ) - entity_c = await sync_service.knowledge_sync_service.entity_repository.get_by_permalink( + entity_c = await sync_service.entity_sync_service.entity_repository.get_by_permalink( "concept/entity_c" ) diff --git a/tests/sync/test_sync_service.py b/tests/sync/test_sync_service.py index beafe2a7..24b931af 100644 --- a/tests/sync/test_sync_service.py +++ b/tests/sync/test_sync_service.py @@ -58,7 +58,7 @@ modified: 2024-01-01 await asyncio.gather(sync_service.sync(test_config.home), modify_file()) # Verify final state - doc = await sync_service.knowledge_sync_service.entity_repository.get_by_permalink("changing") + doc = await sync_service.entity_sync_service.entity_repository.get_by_permalink("changing") assert doc is not None # File should have a checksum, even if it's from either version assert doc.checksum is not None