mirror of
https://github.com/basicmachines-co/basic-memory
synced 2026-06-21 13:47:35 +00:00
markdown processing for entity
This commit is contained in:
phernandez
@@ -0,0 +1,5 @@
|
||||
"""Markdown parsing and handling for basic-memory."""
|
||||
|
||||
from .parser import EntityParser, ParseError
|
||||
|
||||
__all__ = ["EntityParser", "ParseError"]
|
||||
@@ -0,0 +1,240 @@
|
||||
"""
|
||||
Parser for Basic Memory entity markdown files.
|
||||
"""
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
import frontmatter
|
||||
from markdown_it import MarkdownIt
|
||||
from pydantic import BaseModel
|
||||
|
||||
class ParseError(Exception):
|
||||
"""Raised when parsing fails"""
|
||||
pass
|
||||
|
||||
class Observation(BaseModel):
|
||||
"""An observation about an entity."""
|
||||
category: str
|
||||
content: str
|
||||
tags: List[str]
|
||||
context: Optional[str] = None
|
||||
|
||||
class Relation(BaseModel):
|
||||
"""A relation between entities."""
|
||||
target: str # The entity being linked to
|
||||
type: str # The type of relation
|
||||
context: Optional[str] = None
|
||||
|
||||
class EntityFrontmatter(BaseModel):
|
||||
"""Frontmatter metadata for an entity."""
|
||||
type: str
|
||||
created: datetime
|
||||
modified: datetime
|
||||
tags: List[str]
|
||||
status: Optional[str] = None
|
||||
version: Optional[int] = None
|
||||
priority: Optional[str] = None
|
||||
domain: Optional[str] = None
|
||||
maturity: Optional[str] = None
|
||||
owner: Optional[str] = None
|
||||
review_interval: Optional[str] = None
|
||||
last_reviewed: Optional[datetime] = None
|
||||
confidence: Optional[str] = None
|
||||
aliases: Optional[List[str]] = None
|
||||
|
||||
class EntityContent(BaseModel):
|
||||
"""Content sections of an entity markdown file."""
|
||||
title: str
|
||||
description: Optional[str] = None
|
||||
observations: List[Observation] = []
|
||||
relations: List[Relation] = []
|
||||
context: Optional[str] = None
|
||||
metadata: Dict[str, Any] = {}
|
||||
|
||||
class Entity(BaseModel):
|
||||
"""Complete entity combining frontmatter and content."""
|
||||
frontmatter: EntityFrontmatter
|
||||
content: EntityContent
|
||||
|
||||
class EntityParser:
|
||||
"""Parser for entity markdown files."""
|
||||
|
||||
def __init__(self):
|
||||
self.md = MarkdownIt()
|
||||
|
||||
def _parse_observation(self, line: str) -> Observation:
|
||||
"""Parse a single observation line.
|
||||
|
||||
Format: - [category] content #tag1 #tag2 "optional context"
|
||||
"""
|
||||
# Remove leading "- " if present
|
||||
content = line.strip()
|
||||
if content.startswith("- "):
|
||||
content = content[2:].strip()
|
||||
|
||||
# Extract category
|
||||
if not content.startswith("["):
|
||||
raise ParseError(f"Invalid observation format, missing category: {line}")
|
||||
category_end = content.find("]")
|
||||
if category_end == -1:
|
||||
raise ParseError(f"Invalid observation format, unclosed category: {line}")
|
||||
category = content[1:category_end].strip()
|
||||
|
||||
# Remove category from content
|
||||
content = content[category_end + 1:].strip()
|
||||
|
||||
# Extract tags
|
||||
tags = []
|
||||
words = content.split()
|
||||
filtered_words = []
|
||||
for word in words:
|
||||
if word.startswith("#"):
|
||||
tags.append(word[1:]) # Remove # from tag
|
||||
else:
|
||||
filtered_words.append(word)
|
||||
content = " ".join(filtered_words)
|
||||
|
||||
# Extract context if present
|
||||
context = None
|
||||
if content.endswith('"'):
|
||||
last_quote = content.rfind('"')
|
||||
second_last_quote = content.rfind('"', 0, last_quote)
|
||||
if second_last_quote != -1:
|
||||
context = content[second_last_quote + 1:last_quote]
|
||||
content = content[:second_last_quote].strip()
|
||||
|
||||
return Observation(
|
||||
category=category,
|
||||
content=content,
|
||||
tags=tags,
|
||||
context=context
|
||||
)
|
||||
|
||||
def _parse_relation(self, line: str) -> Relation:
|
||||
"""Parse a single relation line.
|
||||
|
||||
Format: - [[Entity]] #relation_type "optional context"
|
||||
"""
|
||||
# Remove leading "- " if present
|
||||
content = line.strip()
|
||||
if content.startswith("- "):
|
||||
content = content[2:].strip()
|
||||
|
||||
# Extract target
|
||||
if not content.startswith("[["):
|
||||
raise ParseError(f"Invalid relation format, missing [[ : {line}")
|
||||
link_end = content.find("]]")
|
||||
if link_end == -1:
|
||||
raise ParseError(f"Invalid relation format, missing ]] : {line}")
|
||||
target = content[2:link_end].strip()
|
||||
|
||||
# Move past ]]
|
||||
content = content[link_end + 2:].strip()
|
||||
|
||||
# Extract relation type
|
||||
if not content.startswith("#"):
|
||||
raise ParseError(f"Invalid relation format, missing relation type: {line}")
|
||||
|
||||
words = content.split()
|
||||
rel_type = words[0][1:] # Remove # from type
|
||||
|
||||
# Extract context if present
|
||||
context = None
|
||||
remaining = " ".join(words[1:])
|
||||
if remaining:
|
||||
if remaining.startswith('"') and remaining.endswith('"'):
|
||||
context = remaining[1:-1]
|
||||
|
||||
return Relation(
|
||||
target=target,
|
||||
type=rel_type,
|
||||
context=context
|
||||
)
|
||||
|
||||
def _parse_metadata_line(self, line: str) -> tuple[str, str]:
|
||||
"""Parse a single metadata line."""
|
||||
if ":" not in line:
|
||||
return None, None
|
||||
|
||||
key, value = line.split(":", 1)
|
||||
return key.strip(), value.strip()
|
||||
|
||||
def parse_file(self, path: Path) -> Entity:
|
||||
"""Parse an entity markdown file."""
|
||||
if not path.exists():
|
||||
raise ParseError(f"File does not exist: {path}")
|
||||
|
||||
try:
|
||||
# Parse frontmatter and content
|
||||
post = frontmatter.load(path)
|
||||
|
||||
# Parse frontmatter
|
||||
frontmatter_data = EntityFrontmatter(**post.metadata)
|
||||
|
||||
# Parse markdown
|
||||
tokens = self.md.parse(post.content)
|
||||
|
||||
# Extract title, description, observations, etc
|
||||
title = ""
|
||||
description = ""
|
||||
observations = []
|
||||
relations = []
|
||||
context = ""
|
||||
metadata = {}
|
||||
|
||||
current_section = None
|
||||
collecting_description = False
|
||||
|
||||
for token in tokens:
|
||||
if token.type == "heading_open" and token.tag == "h1":
|
||||
# Next token will be title
|
||||
current_section = "title"
|
||||
elif token.type == "heading_open" and token.tag == "h2":
|
||||
# Next token will be section name
|
||||
current_section = "section_name"
|
||||
collecting_description = False
|
||||
elif token.type == "inline":
|
||||
if current_section == "title":
|
||||
title = token.content
|
||||
current_section = None
|
||||
elif current_section == "section_name":
|
||||
section_name = token.content.lower()
|
||||
if section_name == "description":
|
||||
collecting_description = True
|
||||
current_section = section_name
|
||||
elif collecting_description:
|
||||
description = token.content
|
||||
elif current_section == "observations":
|
||||
if token.content.strip(): # Skip empty lines
|
||||
observations.append(self._parse_observation(token.content))
|
||||
elif current_section == "relations":
|
||||
if token.content.strip(): # Skip empty lines
|
||||
relations.append(self._parse_relation(token.content))
|
||||
elif current_section == "context":
|
||||
context = token.content
|
||||
elif current_section == "metadata":
|
||||
# Process each line of metadata separately
|
||||
for line in token.content.split("\n"):
|
||||
key, value = self._parse_metadata_line(line.strip())
|
||||
if key and value:
|
||||
metadata[key] = value
|
||||
|
||||
# Create EntityContent
|
||||
content_data = EntityContent(
|
||||
title=title,
|
||||
description=description,
|
||||
observations=observations,
|
||||
relations=relations,
|
||||
context=context,
|
||||
metadata=metadata
|
||||
)
|
||||
|
||||
# Return complete Entity
|
||||
return Entity(
|
||||
frontmatter=frontmatter_data,
|
||||
content=content_data
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
raise ParseError(f"Failed to parse {path}: {str(e)}") from e
|
||||
@@ -0,0 +1,187 @@
|
||||
"""Tests for markdown entity parser."""
|
||||
import pytest
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from textwrap import dedent
|
||||
|
||||
from basic_memory.markdown.parser import EntityParser, ParseError
|
||||
|
||||
def test_parse_observation():
|
||||
"""Test parsing individual observation lines."""
|
||||
parser = EntityParser()
|
||||
|
||||
# Basic observation with category and content
|
||||
obs = parser._parse_observation("- [design] Simple observation")
|
||||
assert obs.category == "design"
|
||||
assert obs.content == "Simple observation"
|
||||
assert obs.tags == []
|
||||
assert obs.context is None
|
||||
|
||||
# Observation with tags
|
||||
obs = parser._parse_observation("- [feature] Added search #important #core")
|
||||
assert obs.category == "feature"
|
||||
assert obs.content == "Added search"
|
||||
assert set(obs.tags) == {"important", "core"}
|
||||
assert obs.context is None
|
||||
|
||||
# Observation with context
|
||||
obs = parser._parse_observation('- [design] Core system design #architecture "Initial version"')
|
||||
assert obs.category == "design"
|
||||
assert obs.content == "Core system design"
|
||||
assert obs.tags == ["architecture"]
|
||||
assert obs.context == "Initial version"
|
||||
|
||||
def test_parse_observation_errors():
|
||||
"""Test error handling in observation parsing."""
|
||||
parser = EntityParser()
|
||||
|
||||
# Missing category brackets
|
||||
with pytest.raises(ParseError, match="missing category"):
|
||||
parser._parse_observation("- Design without brackets")
|
||||
|
||||
# Unclosed category
|
||||
with pytest.raises(ParseError, match="unclosed category"):
|
||||
parser._parse_observation("- [design Core system")
|
||||
|
||||
def test_parse_relation():
|
||||
"""Test parsing individual relation lines."""
|
||||
parser = EntityParser()
|
||||
|
||||
# Basic relation
|
||||
rel = parser._parse_relation("- [[EntityA]] #depends_on")
|
||||
assert rel.target == "EntityA"
|
||||
assert rel.type == "depends_on"
|
||||
assert rel.context is None
|
||||
|
||||
# Relation with context
|
||||
rel = parser._parse_relation('- [[Component]] #implements "Core functionality"')
|
||||
assert rel.target == "Component"
|
||||
assert rel.type == "implements"
|
||||
assert rel.context == "Core functionality"
|
||||
|
||||
def test_parse_relation_errors():
|
||||
"""Test error handling in relation parsing."""
|
||||
parser = EntityParser()
|
||||
|
||||
# Missing [[ prefix
|
||||
with pytest.raises(ParseError, match="missing \\[\\["):
|
||||
parser._parse_relation("- EntityA #depends_on")
|
||||
|
||||
# Missing ]] suffix
|
||||
with pytest.raises(ParseError, match="missing \\]\\]"):
|
||||
parser._parse_relation("- [[EntityA #depends_on")
|
||||
|
||||
# Missing relation type
|
||||
with pytest.raises(ParseError, match="missing relation type"):
|
||||
parser._parse_relation("- [[EntityA]] depends_on")
|
||||
|
||||
def test_parse_complete_file(tmp_path):
|
||||
"""Test parsing a complete entity file."""
|
||||
content = dedent('''
|
||||
---
|
||||
type: concept
|
||||
created: 2024-12-21T10:00:00Z
|
||||
modified: 2024-12-21T10:00:00Z
|
||||
tags: [testing, validation]
|
||||
status: active
|
||||
version: 1
|
||||
priority: high
|
||||
---
|
||||
|
||||
# Test Entity
|
||||
|
||||
This is a test entity for validation.
|
||||
|
||||
## Description
|
||||
A more detailed description of the test entity.
|
||||
|
||||
## Observations
|
||||
- [test] First observation #testing
|
||||
- [design] Second observation #important "With context"
|
||||
|
||||
## Relations
|
||||
- [[EntityA]] #depends_on "Required dependency"
|
||||
- [[EntityB]] #implements
|
||||
|
||||
## Context
|
||||
Additional context information.
|
||||
|
||||
## Metadata
|
||||
schema_version: 1.0
|
||||
validation_status: verified
|
||||
''').lstrip()
|
||||
|
||||
# Write test file
|
||||
test_file = tmp_path / "test_entity.md"
|
||||
test_file.write_text(content)
|
||||
|
||||
# Parse file
|
||||
parser = EntityParser()
|
||||
entity = parser.parse_file(test_file)
|
||||
|
||||
# Check frontmatter
|
||||
assert entity.frontmatter.type == "concept"
|
||||
assert entity.frontmatter.tags == ["testing", "validation"]
|
||||
assert entity.frontmatter.status == "active"
|
||||
assert entity.frontmatter.priority == "high"
|
||||
|
||||
# Check content
|
||||
assert entity.content.title == "Test Entity"
|
||||
assert "detailed description" in entity.content.description
|
||||
assert "context information" in entity.content.context
|
||||
|
||||
# Check observations
|
||||
assert len(entity.content.observations) == 2
|
||||
first_obs = entity.content.observations[0]
|
||||
assert first_obs.category == "test"
|
||||
assert first_obs.content == "First observation"
|
||||
assert first_obs.tags == ["testing"]
|
||||
|
||||
# Check relations
|
||||
assert len(entity.content.relations) == 2
|
||||
first_rel = entity.content.relations[0]
|
||||
assert first_rel.target == "EntityA"
|
||||
assert first_rel.type == "depends_on"
|
||||
assert first_rel.context == "Required dependency"
|
||||
|
||||
# Check metadata
|
||||
assert entity.content.metadata["schema_version"] == "1.0"
|
||||
assert entity.content.metadata["validation_status"] == "verified"
|
||||
|
||||
def test_parse_missing_file():
|
||||
"""Test handling of missing files."""
|
||||
parser = EntityParser()
|
||||
with pytest.raises(ParseError, match="File does not exist"):
|
||||
parser.parse_file(Path("nonexistent.md"))
|
||||
|
||||
def test_parse_minimal_file(tmp_path):
|
||||
"""Test parsing a minimal valid entity file."""
|
||||
content = dedent('''
|
||||
---
|
||||
type: concept
|
||||
created: 2024-12-21T10:00:00Z
|
||||
modified: 2024-12-21T10:00:00Z
|
||||
tags: []
|
||||
---
|
||||
|
||||
# Minimal Entity
|
||||
|
||||
## Description
|
||||
Minimal valid entity.
|
||||
''').lstrip()
|
||||
|
||||
test_file = tmp_path / "minimal.md"
|
||||
test_file.write_text(content)
|
||||
|
||||
parser = EntityParser()
|
||||
entity = parser.parse_file(test_file)
|
||||
|
||||
# Check required fields
|
||||
assert entity.frontmatter.type == "concept"
|
||||
assert isinstance(entity.frontmatter.created, datetime)
|
||||
assert entity.content.title == "Minimal Entity"
|
||||
|
||||
# Optional fields should have default values
|
||||
assert entity.content.observations == []
|
||||
assert entity.content.relations == []
|
||||
assert entity.content.metadata == {}
|
||||
Reference in New Issue
Block a user