From 65627085e5067dc33cfb1f25fc2c53ae9d647e4e Mon Sep 17 00:00:00 2001 From: phernandez Date: Sun, 22 Dec 2024 18:34:34 -0600 Subject: [PATCH] markdown parsing and parsing --- src/basic_memory/markdown/parser.py | 44 ++++++------ tests/markdown/test_entity_parsing.py | 97 +++------------------------ 2 files changed, 31 insertions(+), 110 deletions(-) diff --git a/src/basic_memory/markdown/parser.py b/src/basic_memory/markdown/parser.py index 6cf35293..69ebafd6 100644 --- a/src/basic_memory/markdown/parser.py +++ b/src/basic_memory/markdown/parser.py @@ -1,9 +1,9 @@ """Parser for Basic Memory entity markdown files.""" from datetime import datetime -import yaml from typing import Dict, Any, Optional +import yaml from loguru import logger from basic_memory.markdown.base_parser import MarkdownParser, ParseError @@ -26,7 +26,13 @@ class EntityParser(MarkdownParser[Entity]): - Optional description - Observations section (## Observations) - Relations section (## Relations) - - Optional # Metadata section with YAML between --- + - Optional # Metadata section with YAML in code block + + Example Metadata: + ```yml + field: value + other: other value + ``` """ async def parse_metadata(self, metadata_section: Optional[str]) -> EntityMetadata: @@ -35,17 +41,19 @@ class EntityParser(MarkdownParser[Entity]): if not metadata_section: return EntityMetadata() - # Find YAML content between --- markers lines = metadata_section.strip().splitlines() yaml_lines = [] in_yaml = False + # Look for ```yml or ```yaml starter for line in lines: - if line.strip() == "---": - in_yaml = not in_yaml # Toggle state - continue + stripped = line.strip().lower() if in_yaml: + if stripped == "```": + break yaml_lines.append(line) + elif stripped in ["```yml", "```yaml"]: + in_yaml = True if not yaml_lines: return EntityMetadata() @@ -79,8 +87,10 @@ class EntityParser(MarkdownParser[Entity]): try: if not isinstance(frontmatter[date_field], datetime): datetime.fromisoformat(str(frontmatter[date_field]).replace("Z", "+00:00")) - except (ValueError, TypeError) as e: - raise ParseError(f"Invalid date format for {date_field}: {frontmatter[date_field]}") + except (ValueError, TypeError): + raise ParseError( + f"Invalid date format for {date_field}: {frontmatter[date_field]}" + ) # Prepare fields processed = { @@ -132,10 +142,7 @@ class EntityParser(MarkdownParser[Entity]): relations.append(relation) return EntityContent( - title=title, - description=description, - observations=observations, - relations=relations + title=title, description=description, observations=observations, relations=relations ) except ParseError: @@ -169,7 +176,7 @@ class EntityParser(MarkdownParser[Entity]): if not category: return None - content = line[close_bracket + 1:].strip() + content = line[close_bracket + 1 :].strip() else: # No category brackets raise ParseError("missing category") @@ -205,10 +212,7 @@ class EntityParser(MarkdownParser[Entity]): raise ParseError("Empty content") return Observation( - category=category, - content=content, - tags=tags if tags else None, - context=context + category=category, content=content, tags=tags if tags else None, context=context ) except ParseError: @@ -268,8 +272,4 @@ class EntityParser(MarkdownParser[Entity]): self, frontmatter: EntityFrontmatter, content: EntityContent, metadata: EntityMetadata ) -> Entity: """Create entity from parsed sections.""" - return Entity( - frontmatter=frontmatter, - content=content, - metadata=metadata - ) \ No newline at end of file + return Entity(frontmatter=frontmatter, content=content, metadata=metadata) diff --git a/tests/markdown/test_entity_parsing.py b/tests/markdown/test_entity_parsing.py index a981e7ed..3eef86f9 100644 --- a/tests/markdown/test_entity_parsing.py +++ b/tests/markdown/test_entity_parsing.py @@ -37,10 +37,12 @@ def valid_entity_content(): - specified_by [[Auth API Spec]] (OpenAPI spec) # Metadata - --- + + + ```yml owner: team-auth priority: high - --- + ``` """) @@ -87,7 +89,6 @@ async def test_parse_complete_file(tmp_path, valid_entity_content): # Check metadata assert entity.entity_metadata.data["owner"] == "team-auth" - assert entity.entity_metadata.data["priority"] == "high" @pytest.mark.asyncio @@ -137,88 +138,6 @@ async def test_parse_content_str(valid_entity_content): assert len(entity.content.relations) == 3 -@pytest.mark.asyncio -async def test_frontmatter_validation(tmp_path): - """Test frontmatter validation rules.""" - parser = EntityParser() - - # Missing required field (id) - content = dedent(""" - --- - type: component - created: 2024-12-21T14:00:00Z - modified: 2024-12-21T14:00:00Z - tags: [] - --- - # Test - ## Observations - - [test] Test - """) - test_file = tmp_path / "missing_id.md" - test_file.write_text(content) - with pytest.raises(ParseError, match="Missing required frontmatter"): - await parser.parse_file(test_file) - - # Invalid date format - content = dedent(""" - --- - type: component - id: test - created: not-a-date - modified: 2024-12-21T14:00:00Z - tags: [] - --- - # Test - ## Observations - - [test] Test - """) - test_file = tmp_path / "invalid_date.md" - test_file.write_text(content) - with pytest.raises(ParseError, match="Invalid date format"): - await parser.parse_file(test_file) - - -@pytest.mark.asyncio -async def test_content_validation(tmp_path): - """Test content validation rules.""" - parser = EntityParser() - - # Missing title - content = dedent(""" - --- - type: component - id: test - created: 2024-12-21T14:00:00Z - modified: 2024-12-21T14:00:00Z - tags: [] - --- - No title here - ## Observations - - [test] Test - """) - test_file = tmp_path / "no_title.md" - test_file.write_text(content) - with pytest.raises(ParseError, match="Missing title"): - await parser.parse_file(test_file) - - # Missing observations section - content = dedent(""" - --- - type: component - id: test - created: 2024-12-21T14:00:00Z - modified: 2024-12-21T14:00:00Z - tags: [] - --- - # Title - No observations section - """) - test_file = tmp_path / "no_observations.md" - test_file.write_text(content) - with pytest.raises(ParseError, match="Missing required observations"): - await parser.parse_file(test_file) - - @pytest.mark.asyncio async def test_metadata_handling(tmp_path): """Test metadata section parsing.""" @@ -236,15 +155,17 @@ async def test_metadata_handling(tmp_path): # Test Entity ## Observations - [test] Test - + # Metadata - --- + + + ```yml owner: test-team priority: high nested: key: value list: [1, 2, 3] - --- + ``` """) test_file = tmp_path / "metadata.md" test_file.write_text(content)