diff --git a/src/basic_memory/markdown/exceptions.py b/src/basic_memory/markdown/exceptions.py new file mode 100644 index 00000000..04697d06 --- /dev/null +++ b/src/basic_memory/markdown/exceptions.py @@ -0,0 +1,4 @@ +class ParseError(Exception): + """Raised when parsing fails""" + + pass diff --git a/src/basic_memory/markdown/models.py b/src/basic_memory/markdown/models.py new file mode 100644 index 00000000..4798e952 --- /dev/null +++ b/src/basic_memory/markdown/models.py @@ -0,0 +1,63 @@ +import logging +from datetime import datetime +from typing import Any, Dict, List, Optional + +from pydantic import BaseModel + +logging.basicConfig(level=logging.DEBUG) +logger = logging.getLogger(__name__) + + +class Observation(BaseModel): + """An observation about an entity.""" + + category: str + content: str + tags: List[str] + context: Optional[str] = None + + +class Relation(BaseModel): + """A relation between entities.""" + + target: str # The entity being linked to + type: str # The type of relation + context: Optional[str] = None + + +class EntityFrontmatter(BaseModel): + """Frontmatter metadata for an entity.""" + + type: str + id: str + created: datetime + modified: datetime + tags: List[str] + status: Optional[str] = None + version: Optional[int] = None + priority: Optional[str] = None + domain: Optional[str] = None + maturity: Optional[str] = None + owner: Optional[str] = None + review_interval: Optional[str] = None + last_reviewed: Optional[datetime] = None + confidence: Optional[str] = None + aliases: Optional[List[str]] = None + + +class EntityContent(BaseModel): + """Content sections of an entity markdown file.""" + + title: str + description: Optional[str] = None + observations: List[Observation] = [] + relations: List[Relation] = [] + context: Optional[str] = None + metadata: Dict[str, Any] = {} + + +class Entity(BaseModel): + """Complete entity combining frontmatter and content.""" + + frontmatter: EntityFrontmatter + content: EntityContent diff --git a/src/basic_memory/markdown/parser.py b/src/basic_memory/markdown/parser.py index 116fecf2..f697e450 100644 --- a/src/basic_memory/markdown/parser.py +++ b/src/basic_memory/markdown/parser.py @@ -1,69 +1,29 @@ """Parser for Basic Memory entity markdown files.""" -import re + import logging -from datetime import datetime +import re from pathlib import Path -from typing import Any, Dict, List, Optional +from typing import Optional import frontmatter from markdown_it import MarkdownIt -from pydantic import BaseModel + +from basic_memory.markdown.exceptions import ParseError +from basic_memory.markdown.models import ( + Observation, + Relation, + Entity, + EntityFrontmatter, + EntityContent, +) logging.basicConfig(level=logging.DEBUG) logger = logging.getLogger(__name__) -class ParseError(Exception): - """Raised when parsing fails""" - pass - -class Observation(BaseModel): - """An observation about an entity.""" - category: str - content: str - tags: List[str] - context: Optional[str] = None - -class Relation(BaseModel): - """A relation between entities.""" - target: str # The entity being linked to - type: str # The type of relation - context: Optional[str] = None - -class EntityFrontmatter(BaseModel): - """Frontmatter metadata for an entity.""" - type: str - id: str - created: datetime - modified: datetime - tags: List[str] - status: Optional[str] = None - version: Optional[int] = None - priority: Optional[str] = None - domain: Optional[str] = None - maturity: Optional[str] = None - owner: Optional[str] = None - review_interval: Optional[str] = None - last_reviewed: Optional[datetime] = None - confidence: Optional[str] = None - aliases: Optional[List[str]] = None - -class EntityContent(BaseModel): - """Content sections of an entity markdown file.""" - title: str - description: Optional[str] = None - observations: List[Observation] = [] - relations: List[Relation] = [] - context: Optional[str] = None - metadata: Dict[str, Any] = {} - -class Entity(BaseModel): - """Complete entity combining frontmatter and content.""" - frontmatter: EntityFrontmatter - content: EntityContent class EntityParser: """Parser for entity markdown files.""" - + def __init__(self): self.md = MarkdownIt() @@ -74,7 +34,7 @@ class EntityParser: return None # Parse category [type] - match = re.match(r'^\s*(?:-\s*)?\[([^\]]+)\](.*)', content) + match = re.match(r"^\s*(?:-\s*)?\[([^\]]+)\](.*)", content) if not match: return None category = match.group(1).strip() @@ -84,31 +44,26 @@ class EntityParser: tags = [] words = [] for word in content.split(): - if word.startswith('#'): + if word.startswith("#"): # Handle #tag1#tag2#tag3 - for tag in word.lstrip('#').split('#'): + for tag in word.lstrip("#").split("#"): if tag: tags.append(tag) else: words.append(word) - content = ' '.join(words) - + content = " ".join(words) + # Extract context in parentheses context = None - if content.endswith(')'): - ctx_start = content.rfind('(') + if content.endswith(")"): + ctx_start = content.rfind("(") if ctx_start != -1: - context = content[ctx_start + 1:-1].strip() + context = content[ctx_start + 1 : -1].strip() content = content[:ctx_start].strip() - return Observation( - category=category, - content=content, - tags=tags, - context=context - ) - except Exception as e: + return Observation(category=category, content=content, tags=tags, context=context) + except Exception: logger.exception("Failed to parse observation: %s", content) return None @@ -119,14 +74,14 @@ class EntityParser: return None # Find the link - match = re.search(r'\[\[([^\]]+)\]\]', content) + match = re.search(r"\[\[([^\]]+)\]\]", content) if not match: return None - + target = match.group(1).strip() - before_link = content[:match.start()].strip(' -') - after_link = content[match.end():].strip() - + before_link = content[: match.start()].strip(" -") + after_link = content[match.end() :].strip() + # Everything before the link is the type rel_type = before_link.strip() if not rel_type: @@ -134,82 +89,79 @@ class EntityParser: # Check for context in parentheses context = None - if after_link.startswith('(') and after_link.endswith(')'): + if after_link.startswith("(") and after_link.endswith(")"): context = after_link[1:-1].strip() - return Relation( - target=target, - type=rel_type, - context=context - ) - except Exception as e: + return Relation(target=target, type=rel_type, context=context) + except Exception: logger.exception("Failed to parse relation: %s", content) return None - def parse_file(self, path: Path, encoding: str = 'utf-8') -> Entity: + def parse_file(self, path: Path, encoding: str = "utf-8") -> Entity: """Parse an entity markdown file.""" if not path.exists(): raise ParseError(f"File does not exist: {path}") - + try: # Read and parse frontmatter - with open(path, 'r', encoding=encoding) as f: + with open(path, "r", encoding=encoding) as f: content = f.read() post = frontmatter.loads(content) - + # Handle frontmatter metadata = dict(post.metadata) - if isinstance(metadata.get('tags'), str): - metadata['tags'] = [t.strip() for t in metadata['tags'].split(',')] - frontmatter_data = EntityFrontmatter(**metadata) - + if isinstance(metadata.get("tags"), str): + metadata["tags"] = [t.strip() for t in metadata["tags"].split(",")] # pyright: ignore [reportAttributeAccessIssue] + frontmatter_data = EntityFrontmatter(**metadata) # pyright: ignore [reportArgumentType] + # Parse markdown tokens = self.md.parse(post.content) - + # State for parsing title = "" description = "" observations = [] relations = [] + context = "" metadata = {} - + current_section = None current_list = [] in_list = False - + # Process tokens for token in tokens: - if token.type == 'heading_open': - if token.tag == 'h1': - current_section = 'title' - elif token.tag == 'h2': - current_section = 'section_name' - elif token.type == 'inline': - if current_section == 'title': + if token.type == "heading_open": + if token.tag == "h1": + current_section = "title" + elif token.tag == "h2": + current_section = "section_name" + elif token.type == "inline": + if current_section == "title": title = token.content current_section = None - elif current_section == 'section_name': + elif current_section == "section_name": section = token.content.lower() current_section = section - if section in ['observations', 'relations']: + if section in ["observations", "relations"]: in_list = False current_list = [] - elif current_section == 'description': + elif current_section == "description": description = token.content - elif current_section == 'observations' and token.content.strip(): + elif current_section == "observations" and token.content.strip(): if obs := self._parse_observation(token.content): observations.append(obs) - elif current_section == 'relations' and token.content.strip(): + elif current_section == "relations" and token.content.strip(): if rel := self._parse_relation(token.content): relations.append(rel) - elif current_section == 'context': + elif current_section == "context": context = token.content - elif token.type == 'bullet_list_open': + elif token.type == "bullet_list_open": in_list = True - elif token.type == 'bullet_list_close': + elif token.type == "bullet_list_close": in_list = False - + # Create entity content_data = EntityContent( title=title, @@ -217,17 +169,14 @@ class EntityParser: observations=observations, relations=relations, context=context, - metadata=metadata + metadata=metadata, ) - - return Entity( - frontmatter=frontmatter_data, - content=content_data - ) - + + return Entity(frontmatter=frontmatter_data, content=content_data) + except UnicodeError as e: - if encoding == 'utf-8': - return self.parse_file(path, encoding='utf-16') + if encoding == "utf-8": + return self.parse_file(path, encoding="utf-16") raise ParseError(f"Failed to read {path} with encoding {encoding}: {str(e)}") except Exception as e: - raise ParseError(f"Failed to parse {path}: {str(e)}") from e \ No newline at end of file + raise ParseError(f"Failed to parse {path}: {str(e)}") from e diff --git a/tests/markdown/test_parse_entity_file.py b/tests/markdown/test_parse_entity_file.py new file mode 100644 index 00000000..4c4b9c4b --- /dev/null +++ b/tests/markdown/test_parse_entity_file.py @@ -0,0 +1,139 @@ +"""Tests for the markdown entity parser.""" + +from pathlib import Path +from textwrap import dedent + +import pytest + +from basic_memory.markdown.parser import EntityParser, ParseError + + +def test_parse_complete_file(tmp_path): + """Test parsing a complete entity file.""" + content = dedent(""" + --- + type: component + id: component/auth_service + created: 2024-12-21T14:00:00Z + modified: 2024-12-21T14:00:00Z + tags: authentication, security, core + status: active + version: 1 + --- + + # Auth Service + + Core authentication service. + + + + ## Observations + - [design] Stateless authentication #security #architecture (JWT based) + - [feature] Mobile client support #mobile #oauth (Required for App Store) + - [tech] Caching layer #performance (Redis implementation) + + ## Relations + - implements [[OAuth Implementation]] (Core auth flows) + - uses [[Redis Cache]] (Token caching) + - specified_by [[Auth API Spec]] (OpenAPI spec) + + ## Metadata + owner: team-auth + priority: high + """) + + test_file = tmp_path / "test_entity.md" + test_file.write_text(content) + + parser = EntityParser() + entity = parser.parse_file(test_file) + + # Check frontmatter + assert entity.frontmatter.type == "component" + assert entity.frontmatter.id == "component/auth_service" + assert "authentication" in entity.frontmatter.tags + assert entity.frontmatter.status == "active" + + # Check content + assert entity.content.title == "Auth Service" + assert len(entity.content.observations) == 3 + assert len(entity.content.relations) == 3 + + # Check specific observation + obs = entity.content.observations[0] + assert obs.category == "design" + assert "security" in obs.tags + assert obs.context == "JWT based" + + # Check specific relation + rel = entity.content.relations[0] + assert rel.type == "implements" + assert rel.target == "OAuth Implementation" + assert rel.context == "Core auth flows" + + # Check metadata + assert entity.content.metadata["owner"] == "team-auth" + assert entity.content.metadata["priority"] == "high" + + +def test_parse_minimal_file(tmp_path): + """Test parsing a minimal valid entity file.""" + content = dedent(""" + --- + type: component + id: minimal + created: 2024-12-21T14:00:00Z + modified: 2024-12-21T14:00:00Z + tags: [] + --- + + # Minimal Entity + + ## Observations + - [note] Basic observation #test + + ## Relations + - references [[Other Entity]] + """) + + test_file = tmp_path / "minimal.md" + test_file.write_text(content) + + parser = EntityParser() + entity = parser.parse_file(test_file) + + assert entity.frontmatter.type == "component" + assert len(entity.content.observations) == 1 + assert len(entity.content.relations) == 1 + + +def test_parse_file_errors(tmp_path): + """Test error handling for invalid files.""" + parser = EntityParser() + + # Missing file + with pytest.raises(ParseError, match="does not exist"): + parser.parse_file(Path("nonexistent.md")) + + # Invalid frontmatter + content = dedent(""" + --- + invalid: yaml: [ + --- + # Title + """) + test_file = tmp_path / "invalid.md" + test_file.write_text(content) + with pytest.raises(ParseError): + parser.parse_file(test_file) + + # Missing required frontmatter + content = dedent(""" + --- + type: component + --- + # Title + """) + test_file.write_text(content) + with pytest.raises(ParseError): + parser.parse_file(test_file) diff --git a/tests/markdown/test_parse_observation.py b/tests/markdown/test_parse_observation.py new file mode 100644 index 00000000..a83e71e8 --- /dev/null +++ b/tests/markdown/test_parse_observation.py @@ -0,0 +1,73 @@ +"""Tests for the markdown entity parser.""" + +import pytest + +from basic_memory.markdown.parser import EntityParser, ParseError + + +def test_parse_observation_basic(): + """Test basic observation parsing with category and tags.""" + parser = EntityParser() + + obs = parser._parse_observation("- [design] Core feature #important #mvp") + + assert obs is not None + assert obs.category == "design" + assert obs.content == "Core feature" + assert set(obs.tags) == {"important", "mvp"} + assert obs.context is None + + +def test_parse_observation_with_context(): + """Test observation parsing with context in parentheses.""" + parser = EntityParser() + + obs = parser._parse_observation( + "- [feature] Authentication system #security #auth (Required for MVP)" + ) + assert obs is not None + assert obs.category == "feature" + assert obs.content == "Authentication system" + assert set(obs.tags) == {"security", "auth"} + assert obs.context == "Required for MVP" + + +def test_parse_observation_edge_cases(): + """Test observation parsing edge cases.""" + parser = EntityParser() + + # Multiple word tags + obs = parser._parse_observation("- [tech] Database #high-priority #needs-review") + + assert obs is not None + + assert set(obs.tags) == {"high-priority", "needs-review"} + + # Multiple word category + obs = parser._parse_observation("- [user experience] Design #ux") + assert obs is not None + assert obs.category == "user experience" + + # Parentheses in content shouldn't be treated as context + obs = parser._parse_observation("- [code] Function (x) returns y #function") + assert obs is not None + assert obs.content == "Function (x) returns y" + assert obs.context is None + + # Multiple hashtags together + obs = parser._parse_observation("- [test] Feature #important#urgent#now") + assert obs is not None + assert set(obs.tags) == {"important", "urgent", "now"} + + +def test_parse_observation_errors(): + """Test error handling in observation parsing.""" + parser = EntityParser() + + # Missing category brackets + with pytest.raises(ParseError, match="missing category"): + parser._parse_observation("- Design without brackets #test") + + # Unclosed category + with pytest.raises(ParseError, match="unclosed category"): + parser._parse_observation("- [design Core feature #test") diff --git a/tests/markdown/test_parse_relation.py b/tests/markdown/test_parse_relation.py new file mode 100644 index 00000000..f59c266c --- /dev/null +++ b/tests/markdown/test_parse_relation.py @@ -0,0 +1,61 @@ +"""Tests for the markdown entity parser.""" + +import pytest + +from basic_memory.markdown.parser import EntityParser, ParseError + + +def test_parse_relation_basic(): + """Test basic relation parsing.""" + parser = EntityParser() + + rel = parser._parse_relation("- implements [[Auth Service]]") + assert rel is not None + assert rel.type == "implements" + assert rel.target == "Auth Service" + assert rel.context is None + + +def test_parse_relation_with_context(): + """Test relation parsing with context.""" + parser = EntityParser() + + rel = parser._parse_relation("- depends_on [[Database]] (Required for persistence)") + assert rel is not None + assert rel.type == "depends_on" + assert rel.target == "Database" + assert rel.context == "Required for persistence" + + +def test_parse_relation_edge_cases(): + """Test relation parsing edge cases.""" + parser = EntityParser() + + # Multiple word type + rel = parser._parse_relation("- is used by [[Client App]] (Primary consumer)") + assert rel is not None + assert rel.type == "is used by" + + # Brackets in context + rel = parser._parse_relation("- implements [[API]] (Follows [OpenAPI] spec)") + assert rel is not None + assert rel.context == "Follows [OpenAPI] spec" + + # Extra spaces + rel = parser._parse_relation("- specifies [[Format]] (Documentation)") + assert rel is not None + assert rel.type == "specifies" + assert rel.target == "Format" + + +def test_parse_relation_errors(): + """Test error handling in relation parsing.""" + parser = EntityParser() + + # Missing target brackets + with pytest.raises(ParseError, match="missing \\[\\["): + parser._parse_relation("- implements Auth Service") + + # Unclosed target + with pytest.raises(ParseError, match="missing ]]"): + parser._parse_relation("- implements [[Auth Service") diff --git a/tests/markdown/test_parser.py b/tests/markdown/test_parser.py deleted file mode 100644 index 4af3990b..00000000 --- a/tests/markdown/test_parser.py +++ /dev/null @@ -1,235 +0,0 @@ -"""Tests for the markdown entity parser.""" -import pytest -from datetime import datetime -from pathlib import Path -from textwrap import dedent - -from basic_memory.markdown.parser import EntityParser, ParseError - -def test_parse_observation_basic(): - """Test basic observation parsing with category and tags.""" - parser = EntityParser() - - obs = parser._parse_observation("- [design] Core feature #important #mvp") - assert obs.category == "design" - assert obs.content == "Core feature" - assert set(obs.tags) == {"important", "mvp"} - assert obs.context is None - -def test_parse_observation_with_context(): - """Test observation parsing with context in parentheses.""" - parser = EntityParser() - - obs = parser._parse_observation("- [feature] Authentication system #security #auth (Required for MVP)") - assert obs.category == "feature" - assert obs.content == "Authentication system" - assert set(obs.tags) == {"security", "auth"} - assert obs.context == "Required for MVP" - -def test_parse_observation_edge_cases(): - """Test observation parsing edge cases.""" - parser = EntityParser() - - # Multiple word tags - obs = parser._parse_observation("- [tech] Database #high-priority #needs-review") - assert set(obs.tags) == {"high-priority", "needs-review"} - - # Multiple word category - obs = parser._parse_observation("- [user experience] Design #ux") - assert obs.category == "user experience" - - # Parentheses in content shouldn't be treated as context - obs = parser._parse_observation("- [code] Function (x) returns y #function") - assert obs.content == "Function (x) returns y" - assert obs.context is None - - # Multiple hashtags together - obs = parser._parse_observation("- [test] Feature #important#urgent#now") - assert set(obs.tags) == {"important", "urgent", "now"} - -def test_parse_observation_errors(): - """Test error handling in observation parsing.""" - parser = EntityParser() - - # Missing category brackets - with pytest.raises(ParseError, match="missing category"): - parser._parse_observation("- Design without brackets #test") - - # Unclosed category - with pytest.raises(ParseError, match="unclosed category"): - parser._parse_observation("- [design Core feature #test") - -def test_parse_relation_basic(): - """Test basic relation parsing.""" - parser = EntityParser() - - rel = parser._parse_relation("- implements [[Auth Service]]") - assert rel.type == "implements" - assert rel.target == "Auth Service" - assert rel.context is None - -def test_parse_relation_with_context(): - """Test relation parsing with context.""" - parser = EntityParser() - - rel = parser._parse_relation("- depends_on [[Database]] (Required for persistence)") - assert rel.type == "depends_on" - assert rel.target == "Database" - assert rel.context == "Required for persistence" - -def test_parse_relation_edge_cases(): - """Test relation parsing edge cases.""" - parser = EntityParser() - - # Multiple word type - rel = parser._parse_relation("- is used by [[Client App]] (Primary consumer)") - assert rel.type == "is used by" - - # Brackets in context - rel = parser._parse_relation("- implements [[API]] (Follows [OpenAPI] spec)") - assert rel.context == "Follows [OpenAPI] spec" - - # Extra spaces - rel = parser._parse_relation("- specifies [[Format]] (Documentation)") - assert rel.type == "specifies" - assert rel.target == "Format" - -def test_parse_relation_errors(): - """Test error handling in relation parsing.""" - parser = EntityParser() - - # Missing target brackets - with pytest.raises(ParseError, match="missing \\[\\["): - parser._parse_relation("- implements Auth Service") - - # Unclosed target - with pytest.raises(ParseError, match="missing ]]"): - parser._parse_relation("- implements [[Auth Service") - -def test_parse_complete_file(tmp_path): - """Test parsing a complete entity file.""" - content = dedent(''' - --- - type: component - id: component/auth_service - created: 2024-12-21T14:00:00Z - modified: 2024-12-21T14:00:00Z - tags: authentication, security, core - status: active - version: 1 - --- - - # Auth Service - - Core authentication service. - - - - ## Observations - - [design] Stateless authentication #security #architecture (JWT based) - - [feature] Mobile client support #mobile #oauth (Required for App Store) - - [tech] Caching layer #performance (Redis implementation) - - ## Relations - - implements [[OAuth Implementation]] (Core auth flows) - - uses [[Redis Cache]] (Token caching) - - specified_by [[Auth API Spec]] (OpenAPI spec) - - ## Metadata - owner: team-auth - priority: high - ''') - - test_file = tmp_path / "test_entity.md" - test_file.write_text(content) - - parser = EntityParser() - entity = parser.parse_file(test_file) - - # Check frontmatter - assert entity.frontmatter.type == "component" - assert entity.frontmatter.id == "component/auth_service" - assert "authentication" in entity.frontmatter.tags - assert entity.frontmatter.status == "active" - - # Check content - assert entity.content.title == "Auth Service" - assert len(entity.content.observations) == 3 - assert len(entity.content.relations) == 3 - - # Check specific observation - obs = entity.content.observations[0] - assert obs.category == "design" - assert "security" in obs.tags - assert obs.context == "JWT based" - - # Check specific relation - rel = entity.content.relations[0] - assert rel.type == "implements" - assert rel.target == "OAuth Implementation" - assert rel.context == "Core auth flows" - - # Check metadata - assert entity.content.metadata["owner"] == "team-auth" - assert entity.content.metadata["priority"] == "high" - -def test_parse_minimal_file(tmp_path): - """Test parsing a minimal valid entity file.""" - content = dedent(''' - --- - type: component - id: minimal - created: 2024-12-21T14:00:00Z - modified: 2024-12-21T14:00:00Z - tags: [] - --- - - # Minimal Entity - - ## Observations - - [note] Basic observation #test - - ## Relations - - references [[Other Entity]] - ''') - - test_file = tmp_path / "minimal.md" - test_file.write_text(content) - - parser = EntityParser() - entity = parser.parse_file(test_file) - - assert entity.frontmatter.type == "component" - assert len(entity.content.observations) == 1 - assert len(entity.content.relations) == 1 - -def test_parse_file_errors(tmp_path): - """Test error handling for invalid files.""" - parser = EntityParser() - - # Missing file - with pytest.raises(ParseError, match="does not exist"): - parser.parse_file(Path("nonexistent.md")) - - # Invalid frontmatter - content = dedent(''' - --- - invalid: yaml: [ - --- - # Title - ''') - test_file = tmp_path / "invalid.md" - test_file.write_text(content) - with pytest.raises(ParseError): - parser.parse_file(test_file) - - # Missing required frontmatter - content = dedent(''' - --- - type: component - --- - # Title - ''') - test_file.write_text(content) - with pytest.raises(ParseError): - parser.parse_file(test_file) \ No newline at end of file