diff --git a/src/basic_memory/markdown/entity_parser.py b/src/basic_memory/markdown/entity_parser.py index 06db138c..314103ee 100644 --- a/src/basic_memory/markdown/entity_parser.py +++ b/src/basic_memory/markdown/entity_parser.py @@ -53,6 +53,13 @@ def parse(content: str) -> EntityContent: relations=relations, ) +def parse_tags(tags: Any) -> list[str]: + """Parse tags into list of strings.""" + if isinstance(tags, str): + return [t.strip() for t in tags.split(",") if t.strip()] + if isinstance(tags, (list, tuple)): + return [str(t).strip() for t in tags if str(t).strip()] + return [] class EntityParser: """Parser for markdown files into Entity objects.""" @@ -107,17 +114,10 @@ class EntityParser: # Extract file stat info file_stats = absolute_path.stat() - metadata = post.metadata metadata["title"] = post.metadata.get("title", file_path.name) metadata["type"] = metadata.get("type", "note") - metadata["created"] = self.parse_date( - post.metadata.get("created") - ) or datetime.fromtimestamp(file_stats.st_ctime) - metadata["modified"] = self.parse_date( - post.metadata.get("modified") - ) or datetime.fromtimestamp(file_stats.st_mtime) - metadata["tags"] = self.parse_tags(post.metadata.get("tags", [])) + metadata["tags"] = parse_tags(post.metadata.get("tags", [])) # frontmatter entity_frontmatter = EntityFrontmatter( @@ -131,12 +131,7 @@ class EntityParser: content=post.content, observations=entity_content.observations, relations=entity_content.relations, + created=datetime.fromtimestamp(file_stats.st_ctime), + modified=datetime.fromtimestamp(file_stats.st_mtime), ) - def parse_tags(self, tags: Any) -> list[str]: - """Parse tags into list of strings.""" - if isinstance(tags, str): - return [t.strip() for t in tags.split(",") if t.strip()] - if isinstance(tags, (list, tuple)): - return [str(t).strip() for t in tags if str(t).strip()] - return [] diff --git a/src/basic_memory/markdown/markdown_processor.py b/src/basic_memory/markdown/markdown_processor.py index ad9e1de2..314a615e 100644 --- a/src/basic_memory/markdown/markdown_processor.py +++ b/src/basic_memory/markdown/markdown_processor.py @@ -105,10 +105,10 @@ class MarkdownProcessor: "type": markdown.frontmatter.type, "permalink": markdown.frontmatter.permalink, "created": markdown.frontmatter.created.isoformat() - if markdown.frontmatter.created + if markdown.created else None, "modified": markdown.frontmatter.modified.isoformat() - if markdown.frontmatter.modified + if markdown.modified else None, **metadata, } diff --git a/src/basic_memory/markdown/schemas.py b/src/basic_memory/markdown/schemas.py index 52257f77..f055d94b 100644 --- a/src/basic_memory/markdown/schemas.py +++ b/src/basic_memory/markdown/schemas.py @@ -58,13 +58,6 @@ class EntityFrontmatter(BaseModel): def permalink(self) -> str: return self.metadata.get("permalink") if self.metadata else None - @property - def created(self) -> datetime: - return self.metadata.get("created") if self.metadata else None - - @property - def modified(self) -> datetime: - return self.metadata.get("modified") if self.metadata else None class EntityMarkdown(BaseModel): @@ -74,3 +67,7 @@ class EntityMarkdown(BaseModel): content: Optional[str] = None observations: List[Observation] = [] relations: List[Relation] = [] + + # created, updated will have values after a read + created: Optional[datetime] = None + modified: Optional[datetime] = None diff --git a/src/basic_memory/markdown/utils.py b/src/basic_memory/markdown/utils.py index 03fb5682..6a335f79 100644 --- a/src/basic_memory/markdown/utils.py +++ b/src/basic_memory/markdown/utils.py @@ -1,6 +1,8 @@ from pathlib import Path from typing import Optional +from frontmatter import Post + from basic_memory.markdown import EntityMarkdown, EntityFrontmatter, Observation, Relation from basic_memory.markdown.entity_parser import parse from basic_memory.models import Entity, ObservationCategory, Observation as ObservationModel @@ -105,8 +107,8 @@ def entity_model_from_markdown(file_path: Path, markdown: EntityMarkdown, entity model.permalink=permalink model.file_path=str(file_path) model.content_type="text/markdown" - model.created_at=markdown.frontmatter.created - model.updated_at=markdown.frontmatter.modified + model.created_at=markdown.created + model.updated_at=markdown.modified model.entity_metadata={k:str(v) for k,v in markdown.frontmatter.metadata.items()} model.observations=[ ObservationModel( @@ -119,3 +121,21 @@ def entity_model_from_markdown(file_path: Path, markdown: EntityMarkdown, entity ] return model + +async def schema_to_markdown(schema): + """ + Convert schema to markdown. + :param schema: the schema to convert + :return: Post + """ + # Add metadata to dict + frontmatter_dict = schema.entity_metadata or {} + + # set permalink and type + frontmatter_dict["permalink"] = schema.permalink + frontmatter_dict["type"] = schema.entity_type + + # Create Post object + content = schema.content or "" + post = Post(content, **frontmatter_dict) + return post diff --git a/src/basic_memory/services/entity_service.py b/src/basic_memory/services/entity_service.py index 30e2e40b..42aa84cf 100644 --- a/src/basic_memory/services/entity_service.py +++ b/src/basic_memory/services/entity_service.py @@ -9,7 +9,7 @@ from loguru import logger from sqlalchemy.exc import IntegrityError from basic_memory.markdown import EntityMarkdown -from basic_memory.markdown.utils import entity_model_from_markdown +from basic_memory.markdown.utils import entity_model_from_markdown, schema_to_markdown from basic_memory.models import Entity as EntityModel, Observation, Relation from basic_memory.repository import ObservationRepository, RelationRepository from basic_memory.repository.entity_repository import EntityRepository @@ -69,14 +69,7 @@ class EntityService(BaseService[EntityModel]): f"file_path {file_path} for entity {schema.permalink} already exists: {file_path}" ) - # Convert frontmatter to dict - frontmatter_dict = schema.entity_metadata or {} - frontmatter_dict["permalink"] = schema.permalink - frontmatter_dict["type"] = schema.entity_type - - # Create Post object for frontmatter - content = schema.content or "" - post = Post(content, **frontmatter_dict) + post = await schema_to_markdown(schema) # write file final_content = frontmatter.dumps(post) @@ -84,6 +77,8 @@ class EntityService(BaseService[EntityModel]): # parse entity from file entity_markdown = await self.entity_parser.parse_file(file_path) + + # create entity created_entity = await self.create_entity_from_markdown( file_path, entity_markdown ) @@ -94,6 +89,7 @@ class EntityService(BaseService[EntityModel]): # Set final checksum to mark complete return await self.repository.update(entity.id, {"checksum": checksum}) + async def update_entity(self, schema: EntitySchema) -> EntityModel: """Update an entity's content and metadata.""" logger.debug(f"Updating entity with permalink: {schema.permalink}") @@ -101,14 +97,7 @@ class EntityService(BaseService[EntityModel]): # get file path file_path = Path(schema.file_path) - # Convert frontmatter to dict - frontmatter_dict = schema.entity_metadata or {} - frontmatter_dict["permalink"] = schema.permalink - frontmatter_dict["type"] = schema.entity_type - - # Create Post object for frontmatter - content = schema.content or "" - post = Post(content, **frontmatter_dict) + post = await schema_to_markdown(schema) # write file final_content = frontmatter.dumps(post) diff --git a/tests/markdown/test_entity_parser.py b/tests/markdown/test_entity_parser.py index 9d73d451..a8773cf1 100644 --- a/tests/markdown/test_entity_parser.py +++ b/tests/markdown/test_entity_parser.py @@ -1,5 +1,6 @@ """Tests for entity markdown parsing.""" - +import os +from datetime import datetime, timedelta, UTC from pathlib import Path from textwrap import dedent @@ -99,8 +100,6 @@ async def test_parse_minimal_file(test_config, entity_parser): content = dedent(""" --- type: component - created: 2024-12-21T14:00:00Z - modified: 2024-12-21T14:00:00Z tags: [] --- @@ -123,8 +122,8 @@ async def test_parse_minimal_file(test_config, entity_parser): assert len(entity.observations) == 1 assert len(entity.relations) == 1 - assert entity.frontmatter.created.isoformat().startswith("2024-12-21T14:00:00") - assert entity.frontmatter.modified.isoformat().startswith("2024-12-21T14:00:00") + assert entity.created is not None + assert entity.modified is not None @pytest.mark.asyncio @@ -149,8 +148,6 @@ async def test_parse_file_without_section_headers(test_config, entity_parser): --- type: component permalink: minimal_entity - created: 2024-12-21T14:00:00Z - modified: 2024-12-21T14:00:00Z status: draft tags: [] --- diff --git a/tests/sync/test_sync_service.py b/tests/sync/test_sync_service.py index ca596da3..fd2ec09d 100644 --- a/tests/sync/test_sync_service.py +++ b/tests/sync/test_sync_service.py @@ -1077,8 +1077,6 @@ async def test_sync_preserves_timestamps( frontmatter_content = """ --- type: knowledge -created: Jan 15, 2024 10:00 AM -modified: Jan 15, 2024 11:00 AM --- # Explicit Dates Testing frontmatter dates @@ -1101,8 +1099,8 @@ Testing file timestamps # Check explicit frontmatter dates explicit_entity = await entity_service.get_by_permalink("explicit-dates") - assert explicit_entity.created_at.isoformat().startswith("2024-01-15T10:00:00") - assert explicit_entity.updated_at.isoformat().startswith("2024-01-15T11:00:00") + assert explicit_entity.created_at is not None + assert explicit_entity.updated_at is not None # Check file timestamps file_entity = await entity_service.get_by_permalink("file-dates")