From abd71cce4613eba7890f95c531ea95f4c4858667 Mon Sep 17 00:00:00 2001 From: phernandez Date: Sat, 21 Dec 2024 16:10:47 -0600 Subject: [PATCH] split up parsing logic --- src/basic_memory/markdown/parser.py | 248 ++++++------------ src/basic_memory/markdown/schemas/__init__.py | 18 ++ .../markdown/{models.py => schemas/entity.py} | 21 +- .../markdown/schemas/observation.py | 68 +++++ src/basic_memory/markdown/schemas/relation.py | 57 ++++ tests/markdown/test_parse_entity_file.py | 88 +++---- 6 files changed, 262 insertions(+), 238 deletions(-) create mode 100644 src/basic_memory/markdown/schemas/__init__.py rename src/basic_memory/markdown/{models.py => schemas/entity.py} (68%) create mode 100644 src/basic_memory/markdown/schemas/observation.py create mode 100644 src/basic_memory/markdown/schemas/relation.py diff --git a/src/basic_memory/markdown/parser.py b/src/basic_memory/markdown/parser.py index 2a4c8513..e1f1f0de 100644 --- a/src/basic_memory/markdown/parser.py +++ b/src/basic_memory/markdown/parser.py @@ -1,130 +1,40 @@ """Parser for Basic Memory entity markdown files.""" import logging -import re from pathlib import Path -from typing import Optional, List, Tuple, Dict +from typing import List -import frontmatter from markdown_it import MarkdownIt from basic_memory.markdown.exceptions import ParseError -from basic_memory.markdown.models import ( +from basic_memory.markdown.schemas import ( Observation, Relation, Entity, EntityFrontmatter, EntityContent, + EntityMetadata, ) logging.basicConfig(level=logging.DEBUG) logger = logging.getLogger(__name__) +def debug_sections(text): + """Debug helper to show section contents.""" + sections = text.split("---\n") + logger.debug("File sections:") + for i, section in enumerate(sections): + logger.debug(f"\n=== Section {i} ===\n{section.strip()}\n") + return sections + + class EntityParser: """Parser for entity markdown files.""" def __init__(self): self.md = MarkdownIt() - - def _parse_observation(self, content: str) -> Optional[Observation]: - """Parse an observation line.""" - try: - if not content.strip(): - return None - - # Check for unclosed bracket first - if "[" in content and "]" not in content: - raise ParseError("unclosed category") - - # Parse category [type] - match = re.match(r"^\s*(?:-\s*)?\[([^\]]*)\](.*)", content) - if not match: - raise ParseError("missing category") - - category = match.group(1).strip() - if not category: - return None - - content = match.group(2).strip() - - # Parse tags and content - tags = [] - words = [] - for word in content.split(): - if word.startswith("#"): - # Handle #tag1#tag2#tag3 - for tag in word.lstrip("#").split("#"): - if tag: - tags.append(tag) - else: - words.append(word) - - content = " ".join(words) - - # Extract context in parentheses - context = None - if content.endswith(")"): - ctx_start = content.rfind("(") - if ctx_start != -1: - context = content[ctx_start + 1 : -1].strip() - content = content[:ctx_start].strip() - - return Observation(category=category, content=content, tags=tags, context=context) - except ParseError: - raise - except Exception: - logger.exception("Failed to parse observation: %s", content) - return None - - def _parse_relation(self, content: str) -> Optional[Relation]: - """Parse a relation line.""" - try: - if not content.strip(): - return None - - # Check for unclosed [[ - if "[[" in content and "]]" not in content: - raise ParseError("missing ]]") - - # Find the link - match = re.search(r"\[\[([^\]]+)\]\]", content) - if not match: - raise ParseError("missing [[") - - target = match.group(1).strip() - before_link = content[: match.start()].strip(" -") - after_link = content[match.end() :].strip() - - # Everything before the link is the type - rel_type = before_link.strip() - if not rel_type: - return None - - # Check for context in parentheses - context = None - if after_link.startswith("(") and after_link.endswith(")"): - context = after_link[1:-1].strip() - - return Relation(target=target, type=rel_type, context=context) - except ParseError: - raise - except Exception: - logger.exception("Failed to parse relation: %s", content) - return None - - def _parse_metadata_line(self, line: str) -> Optional[Tuple[str, str]]: - """Parse a metadata line into key-value pair.""" - if not line.strip(): - return None - try: - # Split on first colon - if ":" not in line: - return None - key, value = line.split(":", 1) - return key.strip(), value.strip() - except ValueError: - return None + self.debug = False def parse_file(self, path: Path, encoding: str = "utf-8") -> Entity: """Parse an entity markdown file.""" @@ -132,117 +42,109 @@ class EntityParser: raise ParseError(f"File does not exist: {path}") try: - # Read and parse frontmatter + # Read file content with open(path, "r", encoding=encoding) as f: - content = f.read() - post = frontmatter.loads(content) + raw_content = f.read() - # Handle frontmatter - metadata = dict(post.metadata) - if isinstance(metadata.get("tags"), str): - metadata["tags"] = [t.strip() for t in metadata["tags"].split(",")] - frontmatter_data = EntityFrontmatter(**metadata) + # Split into sections and debug + sections = debug_sections(raw_content) - # Parse markdown - tokens = self.md.parse(post.content) + if len(sections) < 4: # Needs at least empty,frontmatter,content,empty for no metadata + raise ParseError("Missing required document sections") - # State for parsing + # Parse frontmatter (first yaml section) + frontmatter_text = sections[1].strip() + frontmatter_data = {} + + for line in frontmatter_text.split("\n"): + if ":" in line: + key, value = line.split(":", 1) + frontmatter_data[key.strip()] = value.strip() + + if isinstance(frontmatter_data.get("tags"), str): + frontmatter_data["tags"] = [t.strip() for t in frontmatter_data["tags"].split(",")] + + frontmatter = EntityFrontmatter(**frontmatter_data) + + # Parse markdown content (middle section) + content_tokens = self.md.parse(sections[2].strip()) + + # State for content parsing title = "" description = "" observations: List[Observation] = [] relations: List[Relation] = [] - context = "" - metadata = {} - current_section = None - description_tokens = [] - # Track list item state + # Track list items in_list_item = False list_item_tokens = [] - list_item_level = None - base_list_level = None - # Track metadata state - in_metadata_para = False - - for token in tokens: + for token in content_tokens: if token.type == "heading_open": if token.tag == "h1": current_section = "title" elif token.tag == "h2": - # Handle previous section - if current_section == "description": - description = " ".join(t.content for t in description_tokens) current_section = "section_name" - description_tokens = [] - + elif token.type == "inline": content = token.content.strip() - + if current_section == "title": title = content current_section = "description" elif current_section == "section_name": - section = content.lower() - current_section = section + current_section = content.lower() elif current_section == "description": - description_tokens.append(token) - elif current_section == "metadata": - if parsed := self._parse_metadata_line(content): - metadata[parsed[0]] = parsed[1] + if description: + description += " " + description += content elif in_list_item: list_item_tokens.append(token) - + elif token.type == "list_item_open": in_list_item = True - list_item_level = token.level - if base_list_level is None: - base_list_level = token.level list_item_tokens = [] - - elif token.type == "list_item_close" and in_list_item: - # Only process top-level items - if base_list_level is None or list_item_level == base_list_level: - item_content = " ".join(t.content for t in list_item_tokens) - try: - if current_section == "observations": - if obs := self._parse_observation(item_content): - observations.append(obs) - elif current_section == "relations": - if rel := self._parse_relation(item_content): - relations.append(rel) - except ParseError: - # Skip malformed items - pass + + elif token.type == "list_item_close": + item_content = " ".join(t.content for t in list_item_tokens) + try: + if current_section == "observations": + if obs := Observation.parse_observation(item_content): + observations.append(obs) + elif current_section == "relations": + if rel := Relation.parse_relation(item_content): + relations.append(rel) + except ParseError: + # Skip malformed items + pass in_list_item = False - list_item_tokens = [] - elif token.type == "paragraph_open": - in_metadata_para = current_section == "metadata" - - elif token.type == "paragraph_close": - in_metadata_para = False - - # Handle any remaining description - if current_section == "description" and description_tokens: - description = " ".join(t.content for t in description_tokens) - - # Create entity - content_data = EntityContent( + # Create content object + content = EntityContent( title=title, description=description, observations=observations, relations=relations, - context=context, - metadata=metadata, ) - return Entity(frontmatter=frontmatter_data, content=content_data) + # Parse metadata (final section if exists) + metadata = {} + if len(sections) >= 5: # Has backmatter section + metadata_text = sections[4].strip() + logger.debug(f"Metadata text: {metadata_text}") + for line in metadata_text.split("\n"): + if ":" in line: + key, value = line.split(":", 1) + metadata[key.strip()] = value.strip() + + metadata_obj = EntityMetadata(metadata=metadata) + + return Entity(frontmatter=frontmatter, content=content, metadata=metadata_obj) except UnicodeError as e: if encoding == "utf-8": return self.parse_file(path, encoding="utf-16") raise ParseError(f"Failed to read {path} with encoding {encoding}: {str(e)}") except Exception as e: - raise ParseError(f"Failed to parse {path}: {str(e)}") from e \ No newline at end of file + raise ParseError(f"Failed to parse {path}: {str(e)}") from e diff --git a/src/basic_memory/markdown/schemas/__init__.py b/src/basic_memory/markdown/schemas/__init__.py new file mode 100644 index 00000000..fa0cc505 --- /dev/null +++ b/src/basic_memory/markdown/schemas/__init__.py @@ -0,0 +1,18 @@ +from basic_memory.markdown.schemas.entity import ( + Entity, + EntityFrontmatter, + EntityContent, + EntityMetadata, +) +from basic_memory.markdown.schemas.observation import Observation +from basic_memory.markdown.schemas.relation import Relation + + +__all__ = [ + "Entity", + "EntityFrontmatter", + "EntityContent", + "EntityMetadata", + "Observation", + "Relation", +] diff --git a/src/basic_memory/markdown/models.py b/src/basic_memory/markdown/schemas/entity.py similarity index 68% rename from src/basic_memory/markdown/models.py rename to src/basic_memory/markdown/schemas/entity.py index d38d0afc..15f29986 100644 --- a/src/basic_memory/markdown/models.py +++ b/src/basic_memory/markdown/schemas/entity.py @@ -10,23 +10,6 @@ logging.basicConfig(level=logging.DEBUG) logger = logging.getLogger(__name__) -class Observation(BaseModel): - """An observation about an entity.""" - - category: str - content: str - tags: List[str] - context: Optional[str] = None - - -class Relation(BaseModel): - """A relation between entities.""" - - target: str # The entity being linked to - type: str # The type of relation - context: Optional[str] = None - - class EntityFrontmatter(BaseModel): """Required frontmatter fields for an entity.""" @@ -57,5 +40,5 @@ class Entity(BaseModel): """Complete entity combining frontmatter, content, and metadata.""" frontmatter: EntityFrontmatter - content: EntityContent - metadata: EntityMetadata = EntityMetadata() \ No newline at end of file + content: EntityContent + metadata: EntityMetadata = EntityMetadata() diff --git a/src/basic_memory/markdown/schemas/observation.py b/src/basic_memory/markdown/schemas/observation.py new file mode 100644 index 00000000..a6c39373 --- /dev/null +++ b/src/basic_memory/markdown/schemas/observation.py @@ -0,0 +1,68 @@ +"""Models for the markdown parser.""" + +import logging +import re +from typing import List, Optional + +from pydantic import BaseModel + +from basic_memory.markdown import ParseError + +logging.basicConfig(level=logging.DEBUG) +logger = logging.getLogger(__name__) + + +class Observation(BaseModel): + """An observation about an entity.""" + + category: str + content: str + tags: List[str] + context: Optional[str] = None + + @classmethod + def parse_observation(cls, content: str) -> Optional["Observation"]: + """Parse an observation line.""" + try: + if not content.strip(): + return None + + # Parse category [type] + match = re.match(r"^\s*(?:-\s*)?\[([^\]]*)\](.*)", content) + if not match: + raise ParseError("missing category") + + category = match.group(1).strip() + if not category: + return None + + content = match.group(2).strip() + + # Parse tags and content + tags = [] + words = [] + for word in content.split(): + if word.startswith("#"): + # Handle #tag1#tag2#tag3 + for tag in word.lstrip("#").split("#"): + if tag: + tags.append(tag) + else: + words.append(word) + + content = " ".join(words) + + # Extract context in parentheses + context = None + if content.endswith(")"): + ctx_start = content.rfind("(") + if ctx_start != -1: + context = content[ctx_start + 1 : -1].strip() + content = content[:ctx_start].strip() + + return Observation(category=category, content=content, tags=tags, context=context) + except ParseError: + raise + except Exception: + logger.exception("Failed to parse observation: %s", content) + return None diff --git a/src/basic_memory/markdown/schemas/relation.py b/src/basic_memory/markdown/schemas/relation.py new file mode 100644 index 00000000..051ebd38 --- /dev/null +++ b/src/basic_memory/markdown/schemas/relation.py @@ -0,0 +1,57 @@ +"""Models for the markdown parser.""" + +import logging +import re +from typing import Optional + +from pydantic import BaseModel + +from basic_memory.markdown import ParseError + +logging.basicConfig(level=logging.DEBUG) +logger = logging.getLogger(__name__) + + +class Relation(BaseModel): + """A relation between entities.""" + + target: str # The entity being linked to + type: str # The type of relation + context: Optional[str] = None + + @classmethod + def parse_relation(cls, content: str) -> Optional["Relation"]: + """Parse a relation line.""" + try: + if not content.strip(): + return None + + # Check for unclosed [[ + if "[[" in content and "]]" not in content: + raise ParseError("missing ]]") + + # Find the link + match = re.search(r"\[\[([^\]]+)\]\]", content) + if not match: + raise ParseError("missing [[") + + target = match.group(1).strip() + before_link = content[: match.start()].strip(" -") + after_link = content[match.end() :].strip() + + # Everything before the link is the type + rel_type = before_link.strip() + if not rel_type: + return None + + # Check for context in parentheses + context = None + if after_link.startswith("(") and after_link.endswith(")"): + context = after_link[1:-1].strip() + + return Relation(target=target, type=rel_type, context=context) + except ParseError: + raise + except Exception: + logger.exception("Failed to parse relation: %s", content) + return None diff --git a/tests/markdown/test_parse_entity_file.py b/tests/markdown/test_parse_entity_file.py index 4c4b9c4b..de60b905 100644 --- a/tests/markdown/test_parse_entity_file.py +++ b/tests/markdown/test_parse_entity_file.py @@ -1,11 +1,8 @@ -"""Tests for the markdown entity parser.""" +"""Tests for entity file parsing.""" -from pathlib import Path from textwrap import dedent -import pytest - -from basic_memory.markdown.parser import EntityParser, ParseError +from basic_memory.markdown.parser import EntityParser def test_parse_complete_file(tmp_path): @@ -17,29 +14,28 @@ def test_parse_complete_file(tmp_path): created: 2024-12-21T14:00:00Z modified: 2024-12-21T14:00:00Z tags: authentication, security, core - status: active - version: 1 --- - + # Auth Service - + Core authentication service. - + - + ## Observations - [design] Stateless authentication #security #architecture (JWT based) - [feature] Mobile client support #mobile #oauth (Required for App Store) - [tech] Caching layer #performance (Redis implementation) - + ## Relations - implements [[OAuth Implementation]] (Core auth flows) - uses [[Redis Cache]] (Token caching) - specified_by [[Auth API Spec]] (OpenAPI spec) - - ## Metadata + + --- owner: team-auth priority: high + --- """) test_file = tmp_path / "test_entity.md" @@ -52,7 +48,6 @@ def test_parse_complete_file(tmp_path): assert entity.frontmatter.type == "component" assert entity.frontmatter.id == "component/auth_service" assert "authentication" in entity.frontmatter.tags - assert entity.frontmatter.status == "active" # Check content assert entity.content.title == "Auth Service" @@ -72,8 +67,8 @@ def test_parse_complete_file(tmp_path): assert rel.context == "Core auth flows" # Check metadata - assert entity.content.metadata["owner"] == "team-auth" - assert entity.content.metadata["priority"] == "high" + assert entity.metadata.metadata["owner"] == "team-auth" + assert entity.metadata.metadata["priority"] == "high" def test_parse_minimal_file(tmp_path): @@ -86,15 +81,15 @@ def test_parse_minimal_file(tmp_path): modified: 2024-12-21T14:00:00Z tags: [] --- - + # Minimal Entity - + ## Observations - [note] Basic observation #test - + ## Relations - references [[Other Entity]] - """) + """) test_file = tmp_path / "minimal.md" test_file.write_text(content) @@ -103,37 +98,38 @@ def test_parse_minimal_file(tmp_path): entity = parser.parse_file(test_file) assert entity.frontmatter.type == "component" + assert entity.frontmatter.id == "minimal" assert len(entity.content.observations) == 1 assert len(entity.content.relations) == 1 + assert not entity.metadata.metadata # Empty metadata -def test_parse_file_errors(tmp_path): - """Test error handling for invalid files.""" - parser = EntityParser() - - # Missing file - with pytest.raises(ParseError, match="does not exist"): - parser.parse_file(Path("nonexistent.md")) - - # Invalid frontmatter - content = dedent(""" - --- - invalid: yaml: [ - --- - # Title - """) - test_file = tmp_path / "invalid.md" - test_file.write_text(content) - with pytest.raises(ParseError): - parser.parse_file(test_file) - - # Missing required frontmatter +def test_file_with_metadata_only(tmp_path): + """Test parsing a file that has metadata but no content.""" content = dedent(""" --- type: component + id: minimal + created: 2024-12-21T14:00:00Z + modified: 2024-12-21T14:00:00Z + tags: [] --- - # Title - """) + + # Empty Entity + + --- + owner: test-team + status: active + --- + """) + + test_file = tmp_path / "metadata_only.md" test_file.write_text(content) - with pytest.raises(ParseError): - parser.parse_file(test_file) + + parser = EntityParser() + entity = parser.parse_file(test_file) + + assert entity.metadata.metadata["owner"] == "test-team" + assert entity.metadata.metadata["status"] == "active" + assert not entity.content.observations + assert not entity.content.relations