diff --git a/src/basic_memory/markdown/__init__.py b/src/basic_memory/markdown/__init__.py index 06bffc88..e9b6c8b1 100644 --- a/src/basic_memory/markdown/__init__.py +++ b/src/basic_memory/markdown/__init__.py @@ -4,7 +4,6 @@ from basic_memory.file_utils import ParseError from basic_memory.markdown.entity_parser import EntityParser from basic_memory.markdown.schemas import ( EntityMarkdown, - EntityContent, EntityFrontmatter, Observation, Relation, @@ -12,7 +11,6 @@ from basic_memory.markdown.schemas import ( __all__ = [ "EntityMarkdown", - "EntityContent", "EntityFrontmatter", "EntityParser", "Observation", diff --git a/src/basic_memory/markdown/entity_parser.py b/src/basic_memory/markdown/entity_parser.py index c9df4f62..313763ca 100644 --- a/src/basic_memory/markdown/entity_parser.py +++ b/src/basic_memory/markdown/entity_parser.py @@ -15,7 +15,6 @@ from basic_memory.markdown.plugins import observation_plugin, relation_plugin from basic_memory.markdown.schemas import ( EntityMarkdown, EntityFrontmatter, - EntityContent, Observation, Relation, ) @@ -103,18 +102,13 @@ class EntityParser: rels = token.meta["relations"] relations.extend([Relation.model_validate(r) for r in rels]) - # Create EntityContent - entity_content = EntityContent( + return EntityMarkdown( + frontmatter=entity_frontmatter, content=post.content, observations=observations, relations=relations, ) - return EntityMarkdown( - frontmatter=entity_frontmatter, - content=entity_content, - ) - def parse_tags(self, tags: Any) -> list[str]: """Parse tags into list of strings.""" if isinstance(tags, str): diff --git a/src/basic_memory/markdown/markdown_processor.py b/src/basic_memory/markdown/markdown_processor.py index 240a62ed..fa043539 100644 --- a/src/basic_memory/markdown/markdown_processor.py +++ b/src/basic_memory/markdown/markdown_processor.py @@ -104,23 +104,28 @@ class MarkdownProcessor: frontmatter_dict = { "type": markdown.frontmatter.type, "permalink": markdown.frontmatter.permalink, - "created": markdown.frontmatter.created.isoformat() if markdown.frontmatter.created else None, - "modified": markdown.frontmatter.modified.isoformat() if markdown.frontmatter.modified else None, - **metadata + "created": markdown.frontmatter.created.isoformat() + if markdown.frontmatter.created + else None, + "modified": markdown.frontmatter.modified.isoformat() + if markdown.frontmatter.modified + else None, + **metadata, } frontmatter_dict = {k: v for k, v in frontmatter_dict.items() if v is not None} # Start with user content (or minimal title for new files) - content = markdown.content.content or f"# {markdown.frontmatter.title}\n" + content = markdown.content or f"# {markdown.frontmatter.title}\n" - # Add structured sections if present - if markdown.content.observations: - content += ( - "\n\n## Observations\n\n" - + self.format_observations(markdown.content.observations) + # Add structured sections with proper spacing + content = content.rstrip() # Remove trailing whitespace + + if markdown.observations: + content += "\n\n## Observations\n\n" + self.format_observations( + markdown.observations ) - if markdown.content.relations: - content += "\n## Relations\n\n" + self.format_relations(markdown.content.relations) + if markdown.relations: + content += "\n\n## Relations\n\n" + self.format_relations(markdown.relations) # Create Post object for frontmatter post = Post(content, **frontmatter_dict) diff --git a/src/basic_memory/markdown/schemas.py b/src/basic_memory/markdown/schemas.py index 20a5bc83..d6902a25 100644 --- a/src/basic_memory/markdown/schemas.py +++ b/src/basic_memory/markdown/schemas.py @@ -66,16 +66,11 @@ class EntityFrontmatter(BaseModel): def modified(self) -> datetime: return self.metadata.get("modified") if self.metadata else None -class EntityContent(BaseModel): - """Content sections of an entity markdown file.""" - - content: Optional[str] = None - observations: List[Observation] = [] - relations: List[Relation] = [] - class EntityMarkdown(BaseModel): """Complete entity combining frontmatter, content, and metadata.""" frontmatter: EntityFrontmatter - content: EntityContent + content: Optional[str] = None + observations: List[Observation] = [] + relations: List[Relation] = [] diff --git a/src/basic_memory/markdown/utils.py b/src/basic_memory/markdown/utils.py index 0890b8a9..0d82568e 100644 --- a/src/basic_memory/markdown/utils.py +++ b/src/basic_memory/markdown/utils.py @@ -1,13 +1,10 @@ from typing import Optional -from basic_memory.markdown import EntityMarkdown, EntityFrontmatter, EntityContent, Observation, Relation +from basic_memory.markdown import EntityMarkdown, EntityFrontmatter, Observation, Relation +from basic_memory.models import Entity -class EntityModel: - pass - - -def entity_model_to_markdown(entity: EntityModel, content: Optional[str] = None) -> EntityMarkdown: +def entity_model_to_markdown(entity: Entity, content: Optional[str] = None) -> EntityMarkdown: """Convert entity model to markdown schema. Args: @@ -17,18 +14,15 @@ def entity_model_to_markdown(entity: EntityModel, content: Optional[str] = None) Returns: EntityMarkdown schema """ - metadata=entity.entity_metadata or {} + metadata = entity.entity_metadata or {} metadata["permalink"] = entity.permalink metadata["type"] = entity.entity_type or "note" metadata["title"] = entity.title metadata["created"] = entity.created_at metadata["modified"] = entity.updated_at - - entity_frontmatter = EntityFrontmatter( - metadata=metadata - ) - entity_content = EntityContent( + return EntityMarkdown( + frontmatter=EntityFrontmatter(metadata=metadata), content=content, # Use provided content observations=[ Observation( @@ -37,11 +31,7 @@ def entity_model_to_markdown(entity: EntityModel, content: Optional[str] = None) for obs in entity.observations ], relations=[ - Relation(type=r.relation_type, target=r.to_entity.title, context=r.context) for r in entity.outgoing_relations + Relation(type=r.relation_type, target=r.to_entity.title, context=r.context) + for r in entity.outgoing_relations ], ) - - return EntityMarkdown( - frontmatter=entity_frontmatter, - content=entity_content, - ) diff --git a/src/basic_memory/services/file_service.py b/src/basic_memory/services/file_service.py index 65456672..a97f5adc 100644 --- a/src/basic_memory/services/file_service.py +++ b/src/basic_memory/services/file_service.py @@ -6,13 +6,10 @@ from typing import Optional, Tuple from loguru import logger from basic_memory import file_utils -from basic_memory.markdown import EntityFrontmatter, EntityContent, EntityMarkdown, Observation, Relation from basic_memory.markdown.markdown_processor import MarkdownProcessor from basic_memory.markdown.utils import entity_model_to_markdown -from basic_memory.services.exceptions import FileOperationError from basic_memory.models import Entity as EntityModel - - +from basic_memory.services.exceptions import FileOperationError class FileService: @@ -75,7 +72,7 @@ class FileService: # merge content with entity # if content is supplied use it or existing content markdown = entity_model_to_markdown( - entity, content=content or existing_markdown.content.content + entity, content=content or existing_markdown.content ) else: # Create new file structure with provided content @@ -109,7 +106,7 @@ class FileService: try: file_path = self.get_entity_path(entity) markdown = await self.markdown_processor.read_file(file_path) - return markdown.content.content or "" + return markdown.content or "" except Exception as e: logger.error(f"Failed to read entity content: {e}") diff --git a/src/basic_memory/sync/entity_sync_service.py b/src/basic_memory/sync/entity_sync_service.py index d4c9d240..0a60a1cd 100644 --- a/src/basic_memory/sync/entity_sync_service.py +++ b/src/basic_memory/sync/entity_sync_service.py @@ -1,15 +1,16 @@ """Service for managing entities in the database.""" + from pathlib import Path from loguru import logger from sqlalchemy.exc import IntegrityError -from basic_memory.models import Entity as EntityModel, Observation, Relation, ObservationCategory from basic_memory.markdown.schemas import EntityMarkdown -from basic_memory.utils import generate_permalink +from basic_memory.models import Entity as EntityModel, Observation, Relation, ObservationCategory from basic_memory.repository import EntityRepository, ObservationRepository, RelationRepository from basic_memory.services.exceptions import EntityNotFoundError from basic_memory.services.link_resolver import LinkResolver +from basic_memory.utils import generate_permalink def entity_model_from_markdown(file_path: str, markdown: EntityMarkdown) -> EntityModel: @@ -38,7 +39,7 @@ def entity_model_from_markdown(file_path: str, markdown: EntityMarkdown) -> Enti updated_at=markdown.frontmatter.modified, observations=[ Observation(content=obs.content, category=get_valid_category(obs), context=obs.context) - for obs in markdown.content.observations + for obs in markdown.observations ], ) return model @@ -78,18 +79,18 @@ class EntitySyncService: # Set timestamps from frontmatter created_at = markdown.frontmatter.created updated_at = markdown.frontmatter.modified - + model.created_at = created_at model.updated_at = updated_at - + for obs in model.observations: obs.created_at = created_at obs.updated_at = updated_at - + for rel in model.relations: rel.created_at = created_at rel.updated_at = updated_at - + return await self.entity_repository.add(model) async def update_entity_and_observations( @@ -108,7 +109,7 @@ class EntitySyncService: # Update fields from markdown db_entity.title = markdown.frontmatter.title db_entity.entity_type = markdown.frontmatter.type - db_entity.summary = markdown.content.content + db_entity.summary = markdown.content # Clear observations for entity await self.observation_repository.delete_by_fields(entity_id=db_entity.id) @@ -121,7 +122,7 @@ class EntitySyncService: category=obs.category, context=obs.context, ) - for obs in markdown.content.observations + for obs in markdown.observations ] await self.observation_repository.add_all(observations) @@ -153,13 +154,13 @@ class EntitySyncService: await self.relation_repository.delete_outgoing_relations_from_entity(db_entity.id) # Process each relation - for rel in markdown.content.relations: + for rel in markdown.relations: # Resolve the target permalink target_entity = await self.link_resolver.resolve_link( rel.target, ) - # if the target is found, store the id + # if the target is found, store the id target_id = target_entity.id if target_entity else None # if the target is found, store the title, otherwise add the target for a "forward link" target_name = target_entity.title if target_entity else rel.target diff --git a/tests/markdown/test_entity_parser.py b/tests/markdown/test_entity_parser.py index aef56ae6..769ea7e1 100644 --- a/tests/markdown/test_entity_parser.py +++ b/tests/markdown/test_entity_parser.py @@ -5,8 +5,7 @@ from textwrap import dedent import pytest -from basic_memory.markdown.entity_parser import EntityParser -from basic_memory.markdown.schemas import EntityMarkdown, EntityFrontmatter, EntityContent, Relation +from basic_memory.markdown.schemas import EntityMarkdown, EntityFrontmatter, Relation @pytest.fixture @@ -50,7 +49,7 @@ async def test_parse_complete_file(test_config, entity_parser, valid_entity_cont # Verify entity structure assert isinstance(entity, EntityMarkdown) assert isinstance(entity.frontmatter, EntityFrontmatter) - assert isinstance(entity.content, EntityContent) + assert isinstance(entity.content, str) # Check frontmatter assert entity.frontmatter.title == "Auth Service" @@ -59,40 +58,39 @@ async def test_parse_complete_file(test_config, entity_parser, valid_entity_cont assert set(entity.frontmatter.tags) == {"authentication", "security", "core"} # Check content - assert "Core authentication service that handles user authentication." in entity.content.content + assert "Core authentication service that handles user authentication." in entity.content # Check observations - assert len(entity.content.observations) == 3 - obs = entity.content.observations[0] + assert len(entity.observations) == 3 + obs = entity.observations[0] assert obs.category == "design" assert obs.content == "Stateless authentication" assert set(obs.tags or []) == {"security", "architecture"} assert obs.context == "JWT based" # Check relations - assert len(entity.content.relations) == 5 - assert Relation( - type="implements", - target="OAuth Implementation", - context="Core auth flows") in entity.content.relations,"missing [[OAuth Implementation]]" - assert Relation( - type="uses", - target="Redis Cache", - context="Token caching") in entity.content.relations,"missing [[Redis Cache]]" - assert Relation( - type="specified_by", - target="Auth API Spec", - context="OpenAPI spec") in entity.content.relations,"missing [[Auth API Spec]]" + assert len(entity.relations) == 5 + assert ( + Relation(type="implements", target="OAuth Implementation", context="Core auth flows") + in entity.relations + ), "missing [[OAuth Implementation]]" + assert ( + Relation(type="uses", target="Redis Cache", context="Token caching") + in entity.relations + ), "missing [[Redis Cache]]" + assert ( + Relation(type="specified_by", target="Auth API Spec", context="OpenAPI spec") + in entity.relations + ), "missing [[Auth API Spec]]" # inline links in content - assert Relation( - type="links to", - target="Random Link", - context=None) in entity.content.relations,"missing [[Random Link]]" - assert Relation( - type="links to", - target="Random Link with Title|Titled Link", - context=None) in entity.content.relations,"missing [[Random Link with Title|Titled Link]]" + assert ( + Relation(type="links to", target="Random Link", context=None) in entity.relations + ), "missing [[Random Link]]" + assert ( + Relation(type="links to", target="Random Link with Title|Titled Link", context=None) + in entity.relations + ), "missing [[Random Link with Title|Titled Link]]" @pytest.mark.asyncio @@ -122,8 +120,8 @@ async def test_parse_minimal_file(test_config, entity_parser): assert entity.frontmatter.type == "component" assert entity.frontmatter.permalink is None - assert len(entity.content.observations) == 1 - assert len(entity.content.relations) == 1 + assert len(entity.observations) == 1 + assert len(entity.relations) == 1 @pytest.mark.asyncio diff --git a/tests/markdown/test_markdown_processor.py b/tests/markdown/test_markdown_processor.py index ee8ff499..e6c700e2 100644 --- a/tests/markdown/test_markdown_processor.py +++ b/tests/markdown/test_markdown_processor.py @@ -8,23 +8,20 @@ from pathlib import Path import pytest -from basic_memory.markdown.entity_parser import EntityParser from basic_memory.markdown.markdown_processor import MarkdownProcessor, DirtyFileError from basic_memory.markdown.schemas import ( EntityMarkdown, EntityFrontmatter, - EntityContent, Observation, Relation, ) - @pytest.mark.asyncio async def test_write_new_minimal_file(markdown_processor: MarkdownProcessor, tmp_path: Path): """Test creating new file with just title.""" path = tmp_path / "test.md" - + # Create minimal markdown schema metadata = {} metadata["title"] = "Test Note" @@ -37,12 +34,12 @@ async def test_write_new_minimal_file(markdown_processor: MarkdownProcessor, tmp frontmatter=EntityFrontmatter( metadata=metadata, ), - content=EntityContent(content=""), + content="", ) - + # Write file checksum = await markdown_processor.write_file(path, markdown) - + # Read back and verify content = path.read_text() assert "---" in content # Has frontmatter @@ -61,7 +58,7 @@ async def test_write_new_minimal_file(markdown_processor: MarkdownProcessor, tmp async def test_write_new_file_with_content(markdown_processor: MarkdownProcessor, tmp_path: Path): """Test creating new file with content and sections.""" path = tmp_path / "test.md" - + # Create markdown with content and sections markdown = EntityMarkdown( frontmatter=EntityFrontmatter( @@ -71,41 +68,39 @@ async def test_write_new_file_with_content(markdown_processor: MarkdownProcessor created=datetime(2024, 1, 1), modified=datetime(2024, 1, 1), ), - content=EntityContent( - content="# Custom Title\n\nMy content here.\nMultiple lines.", - observations=[ - Observation( - content="Test observation", - category="tech", - tags=["test"], - context="test context", - ), - ], - relations=[ - Relation( - type="relates_to", - target="other-note", - context="test relation", - ), - ], - ), + content="# Custom Title\n\nMy content here.\nMultiple lines.", + observations=[ + Observation( + content="Test observation", + category="tech", + tags=["test"], + context="test context", + ), + ], + relations=[ + Relation( + type="relates_to", + target="other-note", + context="test relation", + ), + ], ) - + # Write file checksum = await markdown_processor.write_file(path, markdown) - + # Read back and verify content = path.read_text() - + # Check content preserved exactly assert "# Custom Title" in content assert "My content here." in content assert "Multiple lines." in content - + # Check sections formatted correctly assert "## Observations" in content assert "- [tech] Test observation #test (test context)" in content - + assert "## Relations" in content assert "- relates_to [[other-note]] (test relation)" in content @@ -114,7 +109,7 @@ async def test_write_new_file_with_content(markdown_processor: MarkdownProcessor async def test_update_preserves_content(markdown_processor: MarkdownProcessor, tmp_path: Path): """Test that updating file preserves existing content.""" path = tmp_path / "test.md" - + # Create initial file initial = EntityMarkdown( frontmatter=EntityFrontmatter( @@ -124,48 +119,44 @@ async def test_update_preserves_content(markdown_processor: MarkdownProcessor, t created=datetime(2024, 1, 1), modified=datetime(2024, 1, 1), ), - content=EntityContent( - content="# My Note\n\nOriginal content here.", - observations=[ - Observation(content="First observation", category="note"), - ], - ), + content="# My Note\n\nOriginal content here.", + observations=[ + Observation(content="First observation", category="note"), + ], ) - + checksum = await markdown_processor.write_file(path, initial) - + # Update with new observation updated = EntityMarkdown( frontmatter=initial.frontmatter, - content=EntityContent( - content=initial.content.content, # Preserve original content - observations=[ - initial.content.observations[0], # Keep original observation - Observation(content="Second observation", category="tech"), # Add new one - ], - ), + content=initial.content, # Preserve original content + observations=[ + initial.observations[0], # Keep original observation + Observation(content="Second observation", category="tech"), # Add new one + ], ) - + # Update file new_checksum = await markdown_processor.write_file(path, updated, expected_checksum=checksum) - + # Read back and verify result = await markdown_processor.read_file(path) - + # Original content preserved - assert "Original content here." in result.content.content - + assert "Original content here." in result.content + # Both observations present - assert len(result.content.observations) == 2 - assert any(o.content == "First observation" for o in result.content.observations) - assert any(o.content == "Second observation" for o in result.content.observations) + assert len(result.observations) == 2 + assert any(o.content == "First observation" for o in result.observations) + assert any(o.content == "Second observation" for o in result.observations) @pytest.mark.asyncio async def test_dirty_file_detection(markdown_processor: MarkdownProcessor, tmp_path: Path): """Test detection of file modifications.""" path = tmp_path / "test.md" - + # Create initial file initial = EntityMarkdown( frontmatter=EntityFrontmatter( @@ -175,24 +166,24 @@ async def test_dirty_file_detection(markdown_processor: MarkdownProcessor, tmp_p created=datetime(2024, 1, 1), modified=datetime(2024, 1, 1), ), - content=EntityContent(content="Initial content"), + content="Initial content", ) - + checksum = await markdown_processor.write_file(path, initial) - + # Modify file directly path.write_text(path.read_text() + "\nModified!") - + # Try to update with old checksum update = EntityMarkdown( frontmatter=initial.frontmatter, - content=EntityContent(content="New content"), + content="New content", ) - + # Should raise DirtyFileError with pytest.raises(DirtyFileError): await markdown_processor.write_file(path, update, expected_checksum=checksum) - + # Should succeed without checksum new_checksum = await markdown_processor.write_file(path, update) - assert new_checksum != checksum \ No newline at end of file + assert new_checksum != checksum diff --git a/tests/markdown/test_parser_edge_cases.py b/tests/markdown/test_parser_edge_cases.py index 1682d653..d25c8b62 100644 --- a/tests/markdown/test_parser_edge_cases.py +++ b/tests/markdown/test_parser_edge_cases.py @@ -41,16 +41,16 @@ async def test_unicode_content(tmp_path): assert "测试" in entity.frontmatter.metadata["tags"] assert "chinese" not in entity.frontmatter.metadata["tags"] - assert "🧪" in entity.content.content + assert "🧪" in entity.content # Verify Unicode in observations - assert any(o.content == "Emoji test 👍" for o in entity.content.observations) - assert any(o.category == "中文" for o in entity.content.observations) - assert any(o.category == "русский" for o in entity.content.observations) + assert any(o.content == "Emoji test 👍" for o in entity.observations) + assert any(o.category == "中文" for o in entity.observations) + assert any(o.category == "русский" for o in entity.observations) # Verify Unicode in relations - assert any(r.target == "测试组件" for r in entity.content.relations) - assert any(r.target == "компонент" for r in entity.content.relations) + assert any(r.target == "测试组件" for r in entity.relations) + assert any(r.target == "компонент" for r in entity.relations) @pytest.mark.asyncio @@ -61,8 +61,8 @@ async def test_empty_file(tmp_path): parser = EntityParser(tmp_path) entity = await parser.parse_file(empty_file) - assert entity.content.observations == [] - assert entity.content.relations == [] + assert entity.observations == [] + assert entity.relations == [] @pytest.mark.asyncio @@ -86,9 +86,9 @@ async def test_missing_sections(tmp_path): parser = EntityParser(tmp_path) entity = await parser.parse_file(test_file) - assert len(entity.content.relations) == 1 - assert entity.content.relations[0].target == "links" - assert entity.content.relations[0].type == "links to" + assert len(entity.relations) == 1 + assert entity.relations[0].target == "links" + assert entity.relations[0].type == "links to" @pytest.mark.asyncio @@ -114,7 +114,7 @@ async def test_tasks_are_not_observations(tmp_path): parser = EntityParser(tmp_path) entity = await parser.parse_file(test_file) - assert len(entity.content.observations) == 0 + assert len(entity.observations) == 0 @pytest.mark.asyncio @@ -151,9 +151,9 @@ async def test_nested_content(tmp_path): entity = await parser.parse_file(test_file) # Should find all observations and relations regardless of nesting - assert len(entity.content.observations) == 3 - assert len(entity.content.relations) == 3 - assert {r.target for r in entity.content.relations} == {"One", "Two", "Three"} + assert len(entity.observations) == 3 + assert len(entity.relations) == 3 + assert {r.target for r in entity.relations} == {"One", "Two", "Three"} @pytest.mark.asyncio diff --git a/tests/sync/test_entity_sync_service.py b/tests/sync/test_entity_sync_service.py index e17003f1..e4d7d9ec 100644 --- a/tests/sync/test_entity_sync_service.py +++ b/tests/sync/test_entity_sync_service.py @@ -7,7 +7,6 @@ import pytest_asyncio from basic_memory.markdown.schemas import ( EntityMarkdown, - EntityContent, EntityFrontmatter, Observation as MarkdownObservation, Relation as MarkdownRelation, @@ -31,9 +30,10 @@ def test_frontmatter() -> EntityFrontmatter: @pytest_asyncio.fixture -def test_content() -> EntityContent: - """Create test content with observations and relations.""" - return EntityContent( +def test_markdown(test_frontmatter) -> EntityMarkdown: + """Create complete test markdown entity.""" + return EntityMarkdown( + frontmatter=test_frontmatter, content="A test entity description", observations=[ MarkdownObservation(content="First observation"), @@ -46,12 +46,6 @@ def test_content() -> EntityContent: ) -@pytest_asyncio.fixture -def test_markdown(test_frontmatter, test_content) -> EntityMarkdown: - """Create complete test markdown entity.""" - return EntityMarkdown(frontmatter=test_frontmatter, content=test_content) - - @pytest.mark.asyncio async def test_create_entity_without_relations( entity_sync_service: EntitySyncService, test_markdown: EntityMarkdown @@ -87,8 +81,8 @@ async def test_update_entity_without_relations( # Modify markdown content test_markdown.frontmatter.metadata["title"] = "Updated Title" - test_markdown.content.content = "Updated description" - test_markdown.content.observations = [MarkdownObservation(content="Updated observation")] + test_markdown.content = "Updated description" + test_markdown.observations = [MarkdownObservation(content="Updated observation")] # Update entity updated = await entity_sync_service.update_entity_and_observations( @@ -111,7 +105,7 @@ async def test_update_entity_relations( """Test second pass relation updates.""" # add a forward link to the markdown (entity does not exist) - test_markdown.content.relations.append( + test_markdown.relations.append( MarkdownRelation(type="depends_on", target="concept/doesnt-exist") )