mirror of
https://github.com/basicmachines-co/basic-memory
synced 2026-06-21 13:47:35 +00:00
markdown parsing and parsing
This commit is contained in:
phernandez
@@ -1,9 +1,9 @@
|
||||
"""Parser for Basic Memory entity markdown files."""
|
||||
|
||||
from datetime import datetime
|
||||
import yaml
|
||||
from typing import Dict, Any, Optional
|
||||
|
||||
import yaml
|
||||
from loguru import logger
|
||||
|
||||
from basic_memory.markdown.base_parser import MarkdownParser, ParseError
|
||||
@@ -26,7 +26,13 @@ class EntityParser(MarkdownParser[Entity]):
|
||||
- Optional description
|
||||
- Observations section (## Observations)
|
||||
- Relations section (## Relations)
|
||||
- Optional # Metadata section with YAML between ---
|
||||
- Optional # Metadata section with YAML in code block
|
||||
|
||||
Example Metadata:
|
||||
```yml
|
||||
field: value
|
||||
other: other value
|
||||
```
|
||||
"""
|
||||
|
||||
async def parse_metadata(self, metadata_section: Optional[str]) -> EntityMetadata:
|
||||
@@ -35,17 +41,19 @@ class EntityParser(MarkdownParser[Entity]):
|
||||
if not metadata_section:
|
||||
return EntityMetadata()
|
||||
|
||||
# Find YAML content between --- markers
|
||||
lines = metadata_section.strip().splitlines()
|
||||
yaml_lines = []
|
||||
in_yaml = False
|
||||
|
||||
# Look for ```yml or ```yaml starter
|
||||
for line in lines:
|
||||
if line.strip() == "---":
|
||||
in_yaml = not in_yaml # Toggle state
|
||||
continue
|
||||
stripped = line.strip().lower()
|
||||
if in_yaml:
|
||||
if stripped == "```":
|
||||
break
|
||||
yaml_lines.append(line)
|
||||
elif stripped in ["```yml", "```yaml"]:
|
||||
in_yaml = True
|
||||
|
||||
if not yaml_lines:
|
||||
return EntityMetadata()
|
||||
@@ -79,8 +87,10 @@ class EntityParser(MarkdownParser[Entity]):
|
||||
try:
|
||||
if not isinstance(frontmatter[date_field], datetime):
|
||||
datetime.fromisoformat(str(frontmatter[date_field]).replace("Z", "+00:00"))
|
||||
except (ValueError, TypeError) as e:
|
||||
raise ParseError(f"Invalid date format for {date_field}: {frontmatter[date_field]}")
|
||||
except (ValueError, TypeError):
|
||||
raise ParseError(
|
||||
f"Invalid date format for {date_field}: {frontmatter[date_field]}"
|
||||
)
|
||||
|
||||
# Prepare fields
|
||||
processed = {
|
||||
@@ -132,10 +142,7 @@ class EntityParser(MarkdownParser[Entity]):
|
||||
relations.append(relation)
|
||||
|
||||
return EntityContent(
|
||||
title=title,
|
||||
description=description,
|
||||
observations=observations,
|
||||
relations=relations
|
||||
title=title, description=description, observations=observations, relations=relations
|
||||
)
|
||||
|
||||
except ParseError:
|
||||
@@ -169,7 +176,7 @@ class EntityParser(MarkdownParser[Entity]):
|
||||
if not category:
|
||||
return None
|
||||
|
||||
content = line[close_bracket + 1:].strip()
|
||||
content = line[close_bracket + 1 :].strip()
|
||||
else:
|
||||
# No category brackets
|
||||
raise ParseError("missing category")
|
||||
@@ -205,10 +212,7 @@ class EntityParser(MarkdownParser[Entity]):
|
||||
raise ParseError("Empty content")
|
||||
|
||||
return Observation(
|
||||
category=category,
|
||||
content=content,
|
||||
tags=tags if tags else None,
|
||||
context=context
|
||||
category=category, content=content, tags=tags if tags else None, context=context
|
||||
)
|
||||
|
||||
except ParseError:
|
||||
@@ -268,8 +272,4 @@ class EntityParser(MarkdownParser[Entity]):
|
||||
self, frontmatter: EntityFrontmatter, content: EntityContent, metadata: EntityMetadata
|
||||
) -> Entity:
|
||||
"""Create entity from parsed sections."""
|
||||
return Entity(
|
||||
frontmatter=frontmatter,
|
||||
content=content,
|
||||
metadata=metadata
|
||||
)
|
||||
return Entity(frontmatter=frontmatter, content=content, metadata=metadata)
|
||||
|
||||
@@ -37,10 +37,12 @@ def valid_entity_content():
|
||||
- specified_by [[Auth API Spec]] (OpenAPI spec)
|
||||
|
||||
# Metadata
|
||||
---
|
||||
<!-- anything below this line is for AI -->
|
||||
|
||||
```yml
|
||||
owner: team-auth
|
||||
priority: high
|
||||
---
|
||||
```
|
||||
""")
|
||||
|
||||
|
||||
@@ -87,7 +89,6 @@ async def test_parse_complete_file(tmp_path, valid_entity_content):
|
||||
|
||||
# Check metadata
|
||||
assert entity.entity_metadata.data["owner"] == "team-auth"
|
||||
assert entity.entity_metadata.data["priority"] == "high"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -137,88 +138,6 @@ async def test_parse_content_str(valid_entity_content):
|
||||
assert len(entity.content.relations) == 3
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_frontmatter_validation(tmp_path):
|
||||
"""Test frontmatter validation rules."""
|
||||
parser = EntityParser()
|
||||
|
||||
# Missing required field (id)
|
||||
content = dedent("""
|
||||
---
|
||||
type: component
|
||||
created: 2024-12-21T14:00:00Z
|
||||
modified: 2024-12-21T14:00:00Z
|
||||
tags: []
|
||||
---
|
||||
# Test
|
||||
## Observations
|
||||
- [test] Test
|
||||
""")
|
||||
test_file = tmp_path / "missing_id.md"
|
||||
test_file.write_text(content)
|
||||
with pytest.raises(ParseError, match="Missing required frontmatter"):
|
||||
await parser.parse_file(test_file)
|
||||
|
||||
# Invalid date format
|
||||
content = dedent("""
|
||||
---
|
||||
type: component
|
||||
id: test
|
||||
created: not-a-date
|
||||
modified: 2024-12-21T14:00:00Z
|
||||
tags: []
|
||||
---
|
||||
# Test
|
||||
## Observations
|
||||
- [test] Test
|
||||
""")
|
||||
test_file = tmp_path / "invalid_date.md"
|
||||
test_file.write_text(content)
|
||||
with pytest.raises(ParseError, match="Invalid date format"):
|
||||
await parser.parse_file(test_file)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_content_validation(tmp_path):
|
||||
"""Test content validation rules."""
|
||||
parser = EntityParser()
|
||||
|
||||
# Missing title
|
||||
content = dedent("""
|
||||
---
|
||||
type: component
|
||||
id: test
|
||||
created: 2024-12-21T14:00:00Z
|
||||
modified: 2024-12-21T14:00:00Z
|
||||
tags: []
|
||||
---
|
||||
No title here
|
||||
## Observations
|
||||
- [test] Test
|
||||
""")
|
||||
test_file = tmp_path / "no_title.md"
|
||||
test_file.write_text(content)
|
||||
with pytest.raises(ParseError, match="Missing title"):
|
||||
await parser.parse_file(test_file)
|
||||
|
||||
# Missing observations section
|
||||
content = dedent("""
|
||||
---
|
||||
type: component
|
||||
id: test
|
||||
created: 2024-12-21T14:00:00Z
|
||||
modified: 2024-12-21T14:00:00Z
|
||||
tags: []
|
||||
---
|
||||
# Title
|
||||
No observations section
|
||||
""")
|
||||
test_file = tmp_path / "no_observations.md"
|
||||
test_file.write_text(content)
|
||||
with pytest.raises(ParseError, match="Missing required observations"):
|
||||
await parser.parse_file(test_file)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_metadata_handling(tmp_path):
|
||||
"""Test metadata section parsing."""
|
||||
@@ -236,15 +155,17 @@ async def test_metadata_handling(tmp_path):
|
||||
# Test Entity
|
||||
## Observations
|
||||
- [test] Test
|
||||
|
||||
|
||||
# Metadata
|
||||
---
|
||||
<!-- anything below this line is for AI -->
|
||||
|
||||
```yml
|
||||
owner: test-team
|
||||
priority: high
|
||||
nested:
|
||||
key: value
|
||||
list: [1, 2, 3]
|
||||
---
|
||||
```
|
||||
""")
|
||||
test_file = tmp_path / "metadata.md"
|
||||
test_file.write_text(content)
|
||||
|
||||
Reference in New Issue
Block a user