From e2e979e705a3f0142a6944f96dbbd17930156bc3 Mon Sep 17 00:00:00 2001 From: phernandez Date: Sun, 26 Jan 2025 21:59:44 -0600 Subject: [PATCH] create observations/relations form entity content on create --- src/basic_memory/markdown/entity_parser.py | 21 +++---- src/basic_memory/markdown/utils.py | 15 ++++- src/basic_memory/services/entity_service.py | 52 +++++++++++++--- src/basic_memory/services/file_service.py | 2 +- src/basic_memory/services/link_resolver.py | 2 +- tests/services/test_entity_service.py | 66 ++++++++++++++++++--- 6 files changed, 126 insertions(+), 32 deletions(-) diff --git a/src/basic_memory/markdown/entity_parser.py b/src/basic_memory/markdown/entity_parser.py index f15e4fc2..cd7a0176 100644 --- a/src/basic_memory/markdown/entity_parser.py +++ b/src/basic_memory/markdown/entity_parser.py @@ -35,16 +35,17 @@ def parse(content: str) -> EntityContent: observations = [] relations = [] - for token in md.parse(content): - # check for observations and relations - if token.meta: - if "observation" in token.meta: - obs = token.meta["observation"] - observation = Observation.model_validate(obs) - observations.append(observation) - if "relations" in token.meta: - rels = token.meta["relations"] - relations.extend([Relation.model_validate(r) for r in rels]) + if content: + for token in md.parse(content): + # check for observations and relations + if token.meta: + if "observation" in token.meta: + obs = token.meta["observation"] + observation = Observation.model_validate(obs) + observations.append(observation) + if "relations" in token.meta: + rels = token.meta["relations"] + relations.extend([Relation.model_validate(r) for r in rels]) return EntityContent( content=content, diff --git a/src/basic_memory/markdown/utils.py b/src/basic_memory/markdown/utils.py index 609208a1..805e476e 100644 --- a/src/basic_memory/markdown/utils.py +++ b/src/basic_memory/markdown/utils.py @@ -35,12 +35,21 @@ def entity_model_to_markdown(entity: Entity, content: Optional[str] = None) -> E # convert model to markdown entity_observations = [ - Observation(category=obs.category, content=obs.content, tags=obs.tags if obs.tags else None, context=obs.context) + Observation( + category=obs.category, + content=obs.content, + tags=obs.tags if obs.tags else None, + context=obs.context, + ) for obs in entity.observations ] entity_relations = [ - Relation(type=r.relation_type, target=r.to_entity.title, context=r.context) + Relation( + type=r.relation_type, + target=r.to_entity.title if r.to_entity else r.to_name, + context=r.context, + ) for r in entity.outgoing_relations ] @@ -49,7 +58,7 @@ def entity_model_to_markdown(entity: Entity, content: Optional[str] = None) -> E # parse the content to see if it has semantic info (observations/relations) entity_content = parse(content) if content else None - + if entity_content: # remove if they are already in the content observations = [o for o in entity_observations if o not in entity_content.observations] diff --git a/src/basic_memory/services/entity_service.py b/src/basic_memory/services/entity_service.py index dea3b55c..ff927d4e 100644 --- a/src/basic_memory/services/entity_service.py +++ b/src/basic_memory/services/entity_service.py @@ -1,16 +1,18 @@ """Service for managing entities in the database.""" + from datetime import datetime, timezone from typing import Dict, Any, Sequence, List, Optional from loguru import logger -from basic_memory.models import Entity as EntityModel, Observation +from basic_memory.models import Entity as EntityModel, Observation, Relation from basic_memory.repository.entity_repository import EntityRepository from basic_memory.schemas import Entity as EntitySchema from basic_memory.services.exceptions import EntityNotFoundError from . import FileService from . import BaseService from .link_resolver import LinkResolver +from ..markdown.entity_parser import parse def entity_model(entity: EntitySchema): @@ -70,25 +72,59 @@ class EntityService(BaseService[EntityModel]): """Create a new entity and write to filesystem.""" logger.debug(f"Creating entity: {schema}") + # if content is provided use that, otherwise write the entity info + content = schema.content or None + + # create model from schema + model = entity_model(schema) + + # parse content to find semantic info + entity_content = parse(schema.content) + if entity_content.observations: + model.observations = [ + Observation( + category=o.category, + content=o.content, + tags=o.tags, + context=o.context, + created_at=datetime.now(timezone.utc), + updated_at=datetime.now(timezone.utc), + ) + for o in entity_content.observations + ] + db_entity = None try: - # 1. Create entity in DB - model = entity_model(schema) - # set timestamps for observations if present for observation in model.observations: observation.created_at = observation.created_at or datetime.now(timezone.utc) observation.updated_at = observation.updated_at or datetime.now(timezone.utc) + # Create entity in DB db_entity = await self.repository.add(model) - # if content is provided use that, otherwise write the entity info - content = schema.content or None + # if the content contains relations, add them to the model + if entity_content.relations: + db_entity.outgoing_relations = [] + for r in entity_content.relations: + target_entity = await self.link_resolver.resolve_link(r.target) + db_entity.outgoing_relations.append( + Relation( + from_id=db_entity.id, + to_id=target_entity.id if target_entity else None, + to_name=r.target, + relation_type=r.type, + context=r.context, + ) + ) + # save relations + await self.repository.add_all(db_entity.outgoing_relations) - # 2. Write file and get checksum + # Write file and get checksum + db_entity = await self.repository.find_by_id(db_entity.id) _, checksum = await self.file_service.write_entity_file(db_entity, content=content) - # 3. Update DB with checksum + # Update DB with checksum updated = await self.repository.update(db_entity.id, {"checksum": checksum}) return updated diff --git a/src/basic_memory/services/file_service.py b/src/basic_memory/services/file_service.py index a97f5adc..cf8bbc47 100644 --- a/src/basic_memory/services/file_service.py +++ b/src/basic_memory/services/file_service.py @@ -86,7 +86,7 @@ class FileService: return path, checksum except Exception as e: - logger.error(f"Failed to write entity file: {e}") + logger.exception(f"Failed to write entity file: {e}") raise FileOperationError(f"Failed to write entity file: {e}") async def read_entity_content(self, entity: EntityModel) -> str: diff --git a/src/basic_memory/services/link_resolver.py b/src/basic_memory/services/link_resolver.py index be9b29de..c99f26b1 100644 --- a/src/basic_memory/services/link_resolver.py +++ b/src/basic_memory/services/link_resolver.py @@ -29,7 +29,7 @@ class LinkResolver: async def resolve_link( self, link_text: str, - ) -> Entity: + ) -> Optional[Entity]: """Resolve a markdown link to a permalink.""" logger.debug(f"Resolving link: {link_text}") diff --git a/tests/services/test_entity_service.py b/tests/services/test_entity_service.py index ccd5b701..44f05cf9 100644 --- a/tests/services/test_entity_service.py +++ b/tests/services/test_entity_service.py @@ -138,7 +138,6 @@ async def test_get_by_permalink(entity_service: EntityService): await entity_service.get_by_permalink("nonexistent/test_entity") - async def test_get_entity_success(entity_service: EntityService): """Test successful entity retrieval.""" entity_data = EntitySchema( @@ -200,7 +199,6 @@ async def test_create_entity_with_special_chars(entity_service: EntityService): retrieved = await entity_service.get_by_permalink(entity_data.permalink) - async def test_open_nodes_by_permalinks(entity_service: EntityService): """Test opening multiple nodes by path IDs.""" # Create test entities @@ -303,7 +301,6 @@ async def test_get_entity_path(entity_service: EntityService): assert path == Path(entity_service.file_service.base_path / "test-entity.md") - @pytest.mark.asyncio async def test_update_note_entity_content(entity_service: EntityService, file_service: FileService): """Should update note content directly.""" @@ -333,8 +330,6 @@ async def test_update_note_entity_content(entity_service: EntityService, file_se assert metadata.get("status") == "draft" - - @pytest.mark.asyncio async def test_create_or_update_new(entity_service: EntityService, file_service: FileService): """Should create a new entity.""" @@ -348,7 +343,7 @@ async def test_create_or_update_new(entity_service: EntityService, file_service: ) assert entity.title == "test" assert created is True - + @pytest.mark.asyncio async def test_create_or_update_existing(entity_service: EntityService, file_service: FileService): @@ -362,13 +357,66 @@ async def test_create_or_update_existing(entity_service: EntityService, file_ser entity_metadata={"status": "final"}, ) ) - + entity.content = "Updated content" # Update name updated, created = await entity_service.create_or_update_entity(entity) - + assert updated.title == "test" assert updated.entity_metadata["status"] == "final" assert created is False - + + +@pytest.mark.asyncio +async def test_create_or_update_with_content( + entity_service: EntityService, file_service: FileService +): + content = """# Git Workflow Guide + +A guide to our [[Git]] workflow. This uses some ideas from [[Trunk Based Development]]. + +## Best Practices +Use branches effectively: +- [design] Keep feature branches short-lived #git #workflow (Reduces merge conflicts) +- implements [[Branch Strategy]] (Our standard workflow) + +## Common Commands +See the [[Git Cheat Sheet]] for reference. +""" + + # Create test entity + entity, created = await entity_service.create_or_update_entity( + EntitySchema( + title="Git Workflow Guide", + entity_type="test", + content=content, + ) + ) + + assert created is True + assert entity.title == "Git Workflow Guide" + + assert len(entity.observations) == 1 + assert entity.observations[0].category == "design" + assert entity.observations[0].content == "Keep feature branches short-lived" + assert set(entity.observations[0].tags) == {"git", "workflow"} + assert entity.observations[0].context == "Reduces merge conflicts" + + assert len(entity.relations) == 4 + assert entity.relations[0].relation_type == "links to" + assert entity.relations[0].to_name == "Git" + assert entity.relations[1].relation_type == "links to" + assert entity.relations[1].to_name == "Trunk Based Development" + assert entity.relations[2].relation_type == "implements" + assert entity.relations[2].to_name == "Branch Strategy" + assert entity.relations[2].context == "Our standard workflow" + assert entity.relations[3].relation_type == "links to" + assert entity.relations[3].to_name == "Git Cheat Sheet" + + # Verify file has new content but preserved metadata + file_path = file_service.get_entity_path(entity) + file_content, _ = await file_service.read_file(file_path) + + # assert content is in file + assert content.strip() in file_content