From 125339c3dca1b7f54166b9b841635fe51bcef70a Mon Sep 17 00:00:00 2001 From: phernandez Date: Sat, 21 Dec 2024 17:10:00 -0600 Subject: [PATCH] markdown tests --- src/basic_memory/markdown/parser.py | 15 ++- src/basic_memory/markdown/schemas/entity.py | 104 ++++++++++++------ .../markdown/schemas/observation.py | 8 +- src/basic_memory/markdown/schemas/relation.py | 34 ++++-- 4 files changed, 106 insertions(+), 55 deletions(-) diff --git a/src/basic_memory/markdown/parser.py b/src/basic_memory/markdown/parser.py index a9f385c3..a8c7b453 100644 --- a/src/basic_memory/markdown/parser.py +++ b/src/basic_memory/markdown/parser.py @@ -17,11 +17,12 @@ logger = logging.getLogger(__name__) def debug_sections(text): """Debug helper to show section contents.""" - sections = text.split("---\n") + # Split on triple-dash and newline combinations to be more lenient + sections = [s.strip() for s in text.replace("\r\n", "\n").split("---")] logger.debug("File sections:") for i, section in enumerate(sections): logger.debug(f"\n=== Section {i} ===\n{section.strip()}\n") - return sections + return [s for s in sections if s.strip()] # Remove empty sections class EntityParser: @@ -38,13 +39,15 @@ class EntityParser: raw_content = f.read() sections = debug_sections(raw_content) - if len(sections) < 4: # Needs at least empty,frontmatter,content,empty + if len(sections) < 2: # Need at least frontmatter and content raise ParseError("Missing required document sections") # Parse each section using schema methods - frontmatter = EntityFrontmatter.from_text(sections[1]) - content = EntityContent.from_markdown(sections[2]) - metadata = EntityMetadata.from_text(sections[4] if len(sections) >= 5 else "") + frontmatter = EntityFrontmatter.from_text(sections[0]) + content = EntityContent.from_markdown(sections[1]) + + # Handle optional metadata section + metadata = EntityMetadata.from_text(sections[2] if len(sections) > 2 else "") return Entity(frontmatter=frontmatter, content=content, metadata=metadata) diff --git a/src/basic_memory/markdown/schemas/entity.py b/src/basic_memory/markdown/schemas/entity.py index b5e084b7..3a8144f0 100644 --- a/src/basic_memory/markdown/schemas/entity.py +++ b/src/basic_memory/markdown/schemas/entity.py @@ -1,6 +1,7 @@ """Models for the markdown parser.""" import logging +import re from datetime import datetime from typing import Any, Dict, List, Optional @@ -32,37 +33,23 @@ class EntityFrontmatter(BaseModel): for line in text.strip().split("\n"): if ":" in line: key, value = line.split(":", 1) - frontmatter_data[key.strip()] = value.strip() + + # Handle tag arrays in YAML format [tag1, tag2] + if key.strip() == "tags" and "[" in value and "]" in value: + tags = value.strip()[1:-1].split(",") # Remove [] and split + frontmatter_data["tags"] = [t.strip() for t in tags] + else: + frontmatter_data[key.strip()] = value.strip() - # Handle tags specially + # Handle non-array tags format if isinstance(frontmatter_data.get("tags"), str): frontmatter_data["tags"] = [t.strip() for t in frontmatter_data["tags"].split(",")] - + return cls(**frontmatter_data) except Exception as e: raise ParseError(f"Failed to parse frontmatter: {e}") from e -class EntityMetadata(BaseModel): - """Optional metadata fields for an entity (backmatter).""" - - metadata: Dict[str, Any] = {} - - @classmethod - def from_text(cls, text: str) -> "EntityMetadata": - """Parse metadata from text.""" - try: - metadata = {} - if text: # Only parse if there's content - for line in text.strip().split("\n"): - if ":" in line: - key, value = line.split(":", 1) - metadata[key.strip()] = value.strip() - return cls(metadata=metadata) - except Exception as e: - raise ParseError(f"Failed to parse metadata: {e}") from e - - class EntityContent(BaseModel): """Content sections of an entity markdown file.""" @@ -86,9 +73,10 @@ class EntityContent(BaseModel): relations: List[Relation] = [] current_section = None - # Track list items + # Track list items and nesting level in_list_item = False list_item_tokens = [] + nesting_level = 0 for token in tokens: if token.type == "heading_open": @@ -96,6 +84,13 @@ class EntityContent(BaseModel): current_section = "title" elif token.tag == "h2": current_section = "section_name" + nesting_level = 0 + + elif token.type == "bullet_list_open": + nesting_level += 1 + + elif token.type == "bullet_list_close": + nesting_level -= 1 elif token.type == "inline": content = token.content.strip() @@ -117,17 +112,19 @@ class EntityContent(BaseModel): list_item_tokens = [] elif token.type == "list_item_close": - item_content = " ".join(t.content for t in list_item_tokens) - try: - if current_section == "observations": - if obs := Observation.from_line(item_content): - observations.append(obs) - elif current_section == "relations": - if rel := Relation.from_line(item_content): - relations.append(rel) - except ParseError: - # Skip malformed items - pass + # Only process top-level items + if nesting_level <= 1: + item_content = " ".join(t.content for t in list_item_tokens) + try: + if current_section == "observations": + if obs := Observation.from_line(item_content): + observations.append(obs) + elif current_section == "relations": + if rel := Relation.from_line(item_content): + relations.append(rel) + except ParseError: + # Skip malformed items + pass in_list_item = False return cls( @@ -141,6 +138,43 @@ class EntityContent(BaseModel): raise ParseError(f"Failed to parse markdown content: {e}") from e +class EntityMetadata(BaseModel): + """Optional metadata fields for an entity (backmatter).""" + + metadata: Dict[str, Any] = {} + + @classmethod + def from_text(cls, text: str) -> "EntityMetadata": + """Parse metadata from text.""" + try: + metadata = {} + if text: # Only parse if there's content + current_key = None + current_value = [] + + for line in text.strip().split("\n"): + if ":" in line and not line.startswith(" "): # New key-value pair + if current_key: # Save previous key-value pair + metadata[current_key] = "\n".join(current_value) + key, value = line.split(":", 1) + current_key = key.strip() + current_value = [value.strip()] + elif current_key and line.startswith(" "): # Continuation of multiline value + current_value.append(line.strip()) + elif current_key: # End of current value + metadata[current_key] = "\n".join(current_value) + current_key = None + current_value = [] + + # Handle last value if any + if current_key: + metadata[current_key] = "\n".join(current_value) + + return cls(metadata=metadata) + except Exception as e: + raise ParseError(f"Failed to parse metadata: {e}") from e + + class Entity(BaseModel): """Complete entity combining frontmatter, content, and metadata.""" diff --git a/src/basic_memory/markdown/schemas/observation.py b/src/basic_memory/markdown/schemas/observation.py index ca0d6f26..49adf8d4 100644 --- a/src/basic_memory/markdown/schemas/observation.py +++ b/src/basic_memory/markdown/schemas/observation.py @@ -27,7 +27,11 @@ class Observation(BaseModel): if not content.strip(): return None - # Parse category [type] + # Check for unclosed category first + if "[" in content and "]" not in content: + raise ParseError("unclosed category") + + # Then check for missing category match = re.match(r"^\s*(?:-\s*)?\[([^\]]*)\](.*)", content) if not match: raise ParseError("missing category") @@ -65,4 +69,4 @@ class Observation(BaseModel): raise except Exception: logger.exception("Failed to parse observation: %s", content) - return None + return None \ No newline at end of file diff --git a/src/basic_memory/markdown/schemas/relation.py b/src/basic_memory/markdown/schemas/relation.py index 0c3b42c1..fab52709 100644 --- a/src/basic_memory/markdown/schemas/relation.py +++ b/src/basic_memory/markdown/schemas/relation.py @@ -26,32 +26,42 @@ class Relation(BaseModel): if not content.strip(): return None - # Check for unclosed [[ + # Check for unclosed markup if "[[" in content and "]]" not in content: - raise ParseError("missing ]]") + raise ParseError("unclosed relation link") + if "]]" in content and "[[" not in content: + raise ParseError("invalid relation syntax") # Find the link match = re.search(r"\[\[([^\]]+)\]\]", content) if not match: - raise ParseError("missing [[") + return None target = match.group(1).strip() - before_link = content[: match.start()].strip(" -") - after_link = content[match.end() :].strip() - - # Everything before the link is the type - rel_type = before_link.strip() - if not rel_type: + if not target: # Empty target return None + # Get text before the link, excluding bullet + before_link = content[: match.start()].strip(" -") + + # Validate relation type + rel_type = before_link.strip() + if not rel_type or rel_type == "missing type": # Explicitly reject "missing type" + return None + + # Get text after the link + after_link = content[match.end() :].strip() + # Check for context in parentheses context = None - if after_link.startswith("(") and after_link.endswith(")"): + if after_link: + if not (after_link.startswith("(") and after_link.endswith(")")): + raise ParseError("invalid context format") context = after_link[1:-1].strip() return Relation(target=target, type=rel_type, context=context) except ParseError: raise - except Exception: + except Exception as e: logger.exception("Failed to parse relation: %s", content) - return None + return None \ No newline at end of file