diff --git a/src/basic_memory/markdown/__init__.py b/src/basic_memory/markdown/__init__.py new file mode 100644 index 00000000..805b353e --- /dev/null +++ b/src/basic_memory/markdown/__init__.py @@ -0,0 +1,5 @@ +"""Markdown parsing and handling for basic-memory.""" + +from .parser import EntityParser, ParseError + +__all__ = ["EntityParser", "ParseError"] \ No newline at end of file diff --git a/src/basic_memory/markdown/parser.py b/src/basic_memory/markdown/parser.py new file mode 100644 index 00000000..8621754f --- /dev/null +++ b/src/basic_memory/markdown/parser.py @@ -0,0 +1,240 @@ +""" +Parser for Basic Memory entity markdown files. +""" +from datetime import datetime +from pathlib import Path +from typing import Any, Dict, List, Optional + +import frontmatter +from markdown_it import MarkdownIt +from pydantic import BaseModel + +class ParseError(Exception): + """Raised when parsing fails""" + pass + +class Observation(BaseModel): + """An observation about an entity.""" + category: str + content: str + tags: List[str] + context: Optional[str] = None + +class Relation(BaseModel): + """A relation between entities.""" + target: str # The entity being linked to + type: str # The type of relation + context: Optional[str] = None + +class EntityFrontmatter(BaseModel): + """Frontmatter metadata for an entity.""" + type: str + created: datetime + modified: datetime + tags: List[str] + status: Optional[str] = None + version: Optional[int] = None + priority: Optional[str] = None + domain: Optional[str] = None + maturity: Optional[str] = None + owner: Optional[str] = None + review_interval: Optional[str] = None + last_reviewed: Optional[datetime] = None + confidence: Optional[str] = None + aliases: Optional[List[str]] = None + +class EntityContent(BaseModel): + """Content sections of an entity markdown file.""" + title: str + description: Optional[str] = None + observations: List[Observation] = [] + relations: List[Relation] = [] + context: Optional[str] = None + metadata: Dict[str, Any] = {} + +class Entity(BaseModel): + """Complete entity combining frontmatter and content.""" + frontmatter: EntityFrontmatter + content: EntityContent + +class EntityParser: + """Parser for entity markdown files.""" + + def __init__(self): + self.md = MarkdownIt() + + def _parse_observation(self, line: str) -> Observation: + """Parse a single observation line. + + Format: - [category] content #tag1 #tag2 "optional context" + """ + # Remove leading "- " if present + content = line.strip() + if content.startswith("- "): + content = content[2:].strip() + + # Extract category + if not content.startswith("["): + raise ParseError(f"Invalid observation format, missing category: {line}") + category_end = content.find("]") + if category_end == -1: + raise ParseError(f"Invalid observation format, unclosed category: {line}") + category = content[1:category_end].strip() + + # Remove category from content + content = content[category_end + 1:].strip() + + # Extract tags + tags = [] + words = content.split() + filtered_words = [] + for word in words: + if word.startswith("#"): + tags.append(word[1:]) # Remove # from tag + else: + filtered_words.append(word) + content = " ".join(filtered_words) + + # Extract context if present + context = None + if content.endswith('"'): + last_quote = content.rfind('"') + second_last_quote = content.rfind('"', 0, last_quote) + if second_last_quote != -1: + context = content[second_last_quote + 1:last_quote] + content = content[:second_last_quote].strip() + + return Observation( + category=category, + content=content, + tags=tags, + context=context + ) + + def _parse_relation(self, line: str) -> Relation: + """Parse a single relation line. + + Format: - [[Entity]] #relation_type "optional context" + """ + # Remove leading "- " if present + content = line.strip() + if content.startswith("- "): + content = content[2:].strip() + + # Extract target + if not content.startswith("[["): + raise ParseError(f"Invalid relation format, missing [[ : {line}") + link_end = content.find("]]") + if link_end == -1: + raise ParseError(f"Invalid relation format, missing ]] : {line}") + target = content[2:link_end].strip() + + # Move past ]] + content = content[link_end + 2:].strip() + + # Extract relation type + if not content.startswith("#"): + raise ParseError(f"Invalid relation format, missing relation type: {line}") + + words = content.split() + rel_type = words[0][1:] # Remove # from type + + # Extract context if present + context = None + remaining = " ".join(words[1:]) + if remaining: + if remaining.startswith('"') and remaining.endswith('"'): + context = remaining[1:-1] + + return Relation( + target=target, + type=rel_type, + context=context + ) + + def _parse_metadata_line(self, line: str) -> tuple[str, str]: + """Parse a single metadata line.""" + if ":" not in line: + return None, None + + key, value = line.split(":", 1) + return key.strip(), value.strip() + + def parse_file(self, path: Path) -> Entity: + """Parse an entity markdown file.""" + if not path.exists(): + raise ParseError(f"File does not exist: {path}") + + try: + # Parse frontmatter and content + post = frontmatter.load(path) + + # Parse frontmatter + frontmatter_data = EntityFrontmatter(**post.metadata) + + # Parse markdown + tokens = self.md.parse(post.content) + + # Extract title, description, observations, etc + title = "" + description = "" + observations = [] + relations = [] + context = "" + metadata = {} + + current_section = None + collecting_description = False + + for token in tokens: + if token.type == "heading_open" and token.tag == "h1": + # Next token will be title + current_section = "title" + elif token.type == "heading_open" and token.tag == "h2": + # Next token will be section name + current_section = "section_name" + collecting_description = False + elif token.type == "inline": + if current_section == "title": + title = token.content + current_section = None + elif current_section == "section_name": + section_name = token.content.lower() + if section_name == "description": + collecting_description = True + current_section = section_name + elif collecting_description: + description = token.content + elif current_section == "observations": + if token.content.strip(): # Skip empty lines + observations.append(self._parse_observation(token.content)) + elif current_section == "relations": + if token.content.strip(): # Skip empty lines + relations.append(self._parse_relation(token.content)) + elif current_section == "context": + context = token.content + elif current_section == "metadata": + # Process each line of metadata separately + for line in token.content.split("\n"): + key, value = self._parse_metadata_line(line.strip()) + if key and value: + metadata[key] = value + + # Create EntityContent + content_data = EntityContent( + title=title, + description=description, + observations=observations, + relations=relations, + context=context, + metadata=metadata + ) + + # Return complete Entity + return Entity( + frontmatter=frontmatter_data, + content=content_data + ) + + except Exception as e: + raise ParseError(f"Failed to parse {path}: {str(e)}") from e \ No newline at end of file diff --git a/tests/markdown/__init__.py b/tests/markdown/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/markdown/test_parser.py b/tests/markdown/test_parser.py new file mode 100644 index 00000000..1815bab5 --- /dev/null +++ b/tests/markdown/test_parser.py @@ -0,0 +1,187 @@ +"""Tests for markdown entity parser.""" +import pytest +from datetime import datetime +from pathlib import Path +from textwrap import dedent + +from basic_memory.markdown.parser import EntityParser, ParseError + +def test_parse_observation(): + """Test parsing individual observation lines.""" + parser = EntityParser() + + # Basic observation with category and content + obs = parser._parse_observation("- [design] Simple observation") + assert obs.category == "design" + assert obs.content == "Simple observation" + assert obs.tags == [] + assert obs.context is None + + # Observation with tags + obs = parser._parse_observation("- [feature] Added search #important #core") + assert obs.category == "feature" + assert obs.content == "Added search" + assert set(obs.tags) == {"important", "core"} + assert obs.context is None + + # Observation with context + obs = parser._parse_observation('- [design] Core system design #architecture "Initial version"') + assert obs.category == "design" + assert obs.content == "Core system design" + assert obs.tags == ["architecture"] + assert obs.context == "Initial version" + +def test_parse_observation_errors(): + """Test error handling in observation parsing.""" + parser = EntityParser() + + # Missing category brackets + with pytest.raises(ParseError, match="missing category"): + parser._parse_observation("- Design without brackets") + + # Unclosed category + with pytest.raises(ParseError, match="unclosed category"): + parser._parse_observation("- [design Core system") + +def test_parse_relation(): + """Test parsing individual relation lines.""" + parser = EntityParser() + + # Basic relation + rel = parser._parse_relation("- [[EntityA]] #depends_on") + assert rel.target == "EntityA" + assert rel.type == "depends_on" + assert rel.context is None + + # Relation with context + rel = parser._parse_relation('- [[Component]] #implements "Core functionality"') + assert rel.target == "Component" + assert rel.type == "implements" + assert rel.context == "Core functionality" + +def test_parse_relation_errors(): + """Test error handling in relation parsing.""" + parser = EntityParser() + + # Missing [[ prefix + with pytest.raises(ParseError, match="missing \\[\\["): + parser._parse_relation("- EntityA #depends_on") + + # Missing ]] suffix + with pytest.raises(ParseError, match="missing \\]\\]"): + parser._parse_relation("- [[EntityA #depends_on") + + # Missing relation type + with pytest.raises(ParseError, match="missing relation type"): + parser._parse_relation("- [[EntityA]] depends_on") + +def test_parse_complete_file(tmp_path): + """Test parsing a complete entity file.""" + content = dedent(''' + --- + type: concept + created: 2024-12-21T10:00:00Z + modified: 2024-12-21T10:00:00Z + tags: [testing, validation] + status: active + version: 1 + priority: high + --- + + # Test Entity + + This is a test entity for validation. + + ## Description + A more detailed description of the test entity. + + ## Observations + - [test] First observation #testing + - [design] Second observation #important "With context" + + ## Relations + - [[EntityA]] #depends_on "Required dependency" + - [[EntityB]] #implements + + ## Context + Additional context information. + + ## Metadata + schema_version: 1.0 + validation_status: verified + ''').lstrip() + + # Write test file + test_file = tmp_path / "test_entity.md" + test_file.write_text(content) + + # Parse file + parser = EntityParser() + entity = parser.parse_file(test_file) + + # Check frontmatter + assert entity.frontmatter.type == "concept" + assert entity.frontmatter.tags == ["testing", "validation"] + assert entity.frontmatter.status == "active" + assert entity.frontmatter.priority == "high" + + # Check content + assert entity.content.title == "Test Entity" + assert "detailed description" in entity.content.description + assert "context information" in entity.content.context + + # Check observations + assert len(entity.content.observations) == 2 + first_obs = entity.content.observations[0] + assert first_obs.category == "test" + assert first_obs.content == "First observation" + assert first_obs.tags == ["testing"] + + # Check relations + assert len(entity.content.relations) == 2 + first_rel = entity.content.relations[0] + assert first_rel.target == "EntityA" + assert first_rel.type == "depends_on" + assert first_rel.context == "Required dependency" + + # Check metadata + assert entity.content.metadata["schema_version"] == "1.0" + assert entity.content.metadata["validation_status"] == "verified" + +def test_parse_missing_file(): + """Test handling of missing files.""" + parser = EntityParser() + with pytest.raises(ParseError, match="File does not exist"): + parser.parse_file(Path("nonexistent.md")) + +def test_parse_minimal_file(tmp_path): + """Test parsing a minimal valid entity file.""" + content = dedent(''' + --- + type: concept + created: 2024-12-21T10:00:00Z + modified: 2024-12-21T10:00:00Z + tags: [] + --- + + # Minimal Entity + + ## Description + Minimal valid entity. + ''').lstrip() + + test_file = tmp_path / "minimal.md" + test_file.write_text(content) + + parser = EntityParser() + entity = parser.parse_file(test_file) + + # Check required fields + assert entity.frontmatter.type == "concept" + assert isinstance(entity.frontmatter.created, datetime) + assert entity.content.title == "Minimal Entity" + + # Optional fields should have default values + assert entity.content.observations == [] + assert entity.content.relations == [] + assert entity.content.metadata == {} \ No newline at end of file