mirror of
https://github.com/basicmachines-co/basic-memory
synced 2026-06-21 13:47:35 +00:00
fixing parser tests
This commit is contained in:
phernandez
@@ -1,58 +1,25 @@
|
||||
"""Models for the markdown parser."""
|
||||
|
||||
import logging
|
||||
import re
|
||||
from datetime import datetime
|
||||
from typing import Any, Dict, List, Optional
|
||||
from typing import List, Optional
|
||||
|
||||
from markdown_it import MarkdownIt
|
||||
from pydantic import BaseModel
|
||||
|
||||
from basic_memory.markdown.exceptions import ParseError
|
||||
from basic_memory.utils.file_utils import ParseError
|
||||
from basic_memory.markdown.schemas.observation import Observation
|
||||
from basic_memory.markdown.schemas.relation import Relation
|
||||
|
||||
logging.basicConfig(level=logging.DEBUG) # pragma: no cover
|
||||
logger = logging.getLogger(__name__) # pragma: no cover
|
||||
|
||||
|
||||
class EntityFrontmatter(BaseModel):
|
||||
"""Required frontmatter fields for an entity."""
|
||||
|
||||
type: str
|
||||
id: str
|
||||
created: datetime
|
||||
modified: datetime
|
||||
tags: List[str]
|
||||
|
||||
@classmethod
|
||||
def from_text(cls, text: str) -> "EntityFrontmatter":
|
||||
"""Parse frontmatter from YAML-style text."""
|
||||
try:
|
||||
frontmatter_data = {}
|
||||
for line in text.strip().split("\n"):
|
||||
if ":" in line:
|
||||
key, value = line.split(":", 1)
|
||||
|
||||
# Handle tag arrays in YAML format [tag1, tag2]
|
||||
if key.strip() == "tags" and "[" in value and "]" in value:
|
||||
tags = value.strip()[1:-1].split(",") # Remove [] and split
|
||||
frontmatter_data["tags"] = [t.strip() for t in tags]
|
||||
else:
|
||||
frontmatter_data[key.strip()] = value.strip()
|
||||
|
||||
# Handle non-array tags format
|
||||
if isinstance(frontmatter_data.get("tags"), str):
|
||||
frontmatter_data["tags"] = [t.strip() for t in frontmatter_data["tags"].split(",")]
|
||||
|
||||
return cls(**frontmatter_data)
|
||||
except Exception as e:
|
||||
raise ParseError(f"Failed to parse frontmatter: {e}") from e # pragma: no cover
|
||||
|
||||
|
||||
class EntityContent(BaseModel):
|
||||
"""Content sections of an entity markdown file."""
|
||||
|
||||
title: str
|
||||
description: Optional[str] = None
|
||||
observations: List[Observation] = []
|
||||
@@ -61,124 +28,87 @@ class EntityContent(BaseModel):
|
||||
|
||||
@classmethod
|
||||
def from_markdown(cls, text: str) -> "EntityContent":
|
||||
"""Parse content from markdown text."""
|
||||
"""
|
||||
Parse content sections from markdown.
|
||||
|
||||
Required sections:
|
||||
- Title (# Title)
|
||||
- Observations (## Observations)
|
||||
"""
|
||||
try:
|
||||
md = MarkdownIt()
|
||||
tokens = md.parse(text.strip())
|
||||
lines = text.strip().split("\n")
|
||||
if not lines:
|
||||
raise ParseError("Content is empty")
|
||||
|
||||
# State for parsing
|
||||
# Parse title (must start with # )
|
||||
title = ""
|
||||
for i, line in enumerate(lines):
|
||||
if line.startswith("# "):
|
||||
title = line[2:].strip()
|
||||
description_start = i + 1
|
||||
break
|
||||
if not title:
|
||||
raise ParseError("Missing title section (must start with '# ')")
|
||||
|
||||
# Find section boundaries
|
||||
desc_lines = []
|
||||
observations: List[Observation] = []
|
||||
relations: List[Relation] = []
|
||||
current_section = None
|
||||
obs_lines = []
|
||||
rel_lines = []
|
||||
|
||||
current_section = "description"
|
||||
|
||||
for line in lines[description_start:]:
|
||||
if line.startswith("## Observations"):
|
||||
current_section = "observations"
|
||||
continue
|
||||
elif line.startswith("## Relations"):
|
||||
current_section = "relations"
|
||||
continue
|
||||
|
||||
if line.strip(): # Skip empty lines
|
||||
if current_section == "description":
|
||||
desc_lines.append(line)
|
||||
elif current_section == "observations":
|
||||
obs_lines.append(line)
|
||||
elif current_section == "relations":
|
||||
rel_lines.append(line)
|
||||
|
||||
# Track list items and nesting level
|
||||
in_list_item = False
|
||||
list_item_tokens = []
|
||||
nesting_level = 0
|
||||
# Parse observations
|
||||
observations = []
|
||||
for line in obs_lines:
|
||||
if obs := Observation.from_line(line):
|
||||
observations.append(obs)
|
||||
|
||||
for token in tokens:
|
||||
if token.type == "heading_open":
|
||||
if token.tag == "h1":
|
||||
current_section = "title"
|
||||
elif token.tag == "h2":
|
||||
current_section = "section_name"
|
||||
nesting_level = 0
|
||||
# Parse relations
|
||||
relations = []
|
||||
for line in rel_lines:
|
||||
if rel := Relation.from_line(line):
|
||||
relations.append(rel)
|
||||
|
||||
elif token.type == "bullet_list_open":
|
||||
nesting_level += 1
|
||||
|
||||
elif token.type == "bullet_list_close":
|
||||
nesting_level -= 1
|
||||
|
||||
elif token.type == "inline":
|
||||
content = token.content.strip()
|
||||
|
||||
if current_section == "title":
|
||||
title = content
|
||||
current_section = "description"
|
||||
elif current_section == "section_name":
|
||||
current_section = content.lower()
|
||||
elif current_section == "description":
|
||||
if content:
|
||||
desc_lines.append(content)
|
||||
elif in_list_item:
|
||||
list_item_tokens.append(token)
|
||||
|
||||
elif token.type == "list_item_open":
|
||||
in_list_item = True
|
||||
list_item_tokens = []
|
||||
|
||||
elif token.type == "list_item_close":
|
||||
# Only process top-level items
|
||||
if nesting_level <= 1:
|
||||
item_content = " ".join(t.content for t in list_item_tokens)
|
||||
try:
|
||||
if current_section == "observations":
|
||||
if obs := Observation.from_line(item_content):
|
||||
observations.append(obs)
|
||||
elif current_section == "relations":
|
||||
if rel := Relation.from_line(item_content):
|
||||
relations.append(rel)
|
||||
except ParseError:
|
||||
# Skip malformed items
|
||||
pass
|
||||
in_list_item = False
|
||||
|
||||
description = " ".join(desc_lines) if desc_lines else None
|
||||
# Must have at least an observations section
|
||||
if not obs_lines:
|
||||
raise ParseError("Missing observations section")
|
||||
|
||||
return cls(
|
||||
title=title,
|
||||
description=description,
|
||||
description="\n".join(desc_lines).strip() if desc_lines else None,
|
||||
observations=observations,
|
||||
relations=relations,
|
||||
)
|
||||
|
||||
except Exception as e: # pragma: no cover
|
||||
raise ParseError(f"Failed to parse markdown content: {e}") from e # pragma: no cover
|
||||
except Exception as e:
|
||||
if not isinstance(e, ParseError):
|
||||
raise ParseError(f"Failed to parse content: {str(e)}") from e
|
||||
raise
|
||||
|
||||
|
||||
class EntityMetadata(BaseModel):
|
||||
"""Optional metadata fields for an entity (backmatter)."""
|
||||
|
||||
metadata: Dict[str, Any] = {}
|
||||
|
||||
@classmethod
|
||||
def from_text(cls, text: str) -> "EntityMetadata":
|
||||
"""Parse metadata from text."""
|
||||
try:
|
||||
metadata = {}
|
||||
if text: # Only parse if there's content
|
||||
current_key = None
|
||||
current_value = []
|
||||
|
||||
for line in text.strip().split("\n"):
|
||||
if ":" in line and not line.startswith(" "): # New key-value pair
|
||||
if current_key: # Save previous key-value pair
|
||||
metadata[current_key] = "\n".join(current_value)
|
||||
key, value = line.split(":", 1)
|
||||
current_key = key.strip()
|
||||
current_value = [value.strip()]
|
||||
elif current_key and line.startswith(" "): # Continuation of multiline value
|
||||
current_value.append(line.strip())
|
||||
elif current_key: # End of current value
|
||||
metadata[current_key] = "\n".join(current_value) # pragma: no cover
|
||||
current_key = None # pragma: no cover
|
||||
current_value = [] # pragma: no cover
|
||||
|
||||
# Handle last value if any
|
||||
if current_key:
|
||||
metadata[current_key] = "\n".join(current_value)
|
||||
|
||||
return cls(metadata=metadata)
|
||||
except Exception as e: # pragma: no cover
|
||||
raise ParseError(f"Failed to parse metadata: {e}") from e # pragma: no cover
|
||||
metadata: dict = {}
|
||||
|
||||
|
||||
class Entity(BaseModel):
|
||||
"""Complete entity combining frontmatter, content, and metadata."""
|
||||
|
||||
frontmatter: EntityFrontmatter
|
||||
content: EntityContent
|
||||
content: EntityContent
|
||||
metadata: EntityMetadata = EntityMetadata()
|
||||
@@ -1,93 +1,86 @@
|
||||
"""Models for the markdown parser."""
|
||||
|
||||
import logging
|
||||
import re
|
||||
from typing import List, Optional
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
from basic_memory.markdown import ParseError
|
||||
from basic_memory.utils.file_utils import ParseError
|
||||
|
||||
logging.basicConfig(level=logging.DEBUG) # pragma: no cover
|
||||
logger = logging.getLogger(__name__) # pragma: no cover
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Observation(BaseModel):
|
||||
"""An observation about an entity."""
|
||||
|
||||
category: str
|
||||
content: str
|
||||
tags: List[str]
|
||||
context: Optional[str] = None
|
||||
|
||||
@classmethod
|
||||
def from_line(cls, content: str) -> Optional["Observation"]:
|
||||
"""Parse an observation line."""
|
||||
def from_line(cls, line: str) -> Optional["Observation"]:
|
||||
"""
|
||||
Parse an observation from a line.
|
||||
|
||||
Format must be:
|
||||
- [category] Content text #tag1 #tag2 (optional context)
|
||||
|
||||
Leading spaces before bullet are allowed.
|
||||
"""
|
||||
try:
|
||||
if not content.strip():
|
||||
line = line.strip()
|
||||
|
||||
# Skip empty or non-bullet lines
|
||||
if not line or not line.startswith("-"):
|
||||
return None
|
||||
|
||||
try:
|
||||
if "\xff" in content or "\xfe" in content:
|
||||
return None
|
||||
except UnicodeError: # pragma: no cover
|
||||
return None
|
||||
# Remove bullet and trim
|
||||
line = line[1:].lstrip()
|
||||
|
||||
# Only allow valid printable Unicode
|
||||
for char in content:
|
||||
# Skip normal whitespace
|
||||
if char in {" ", "\t", "\n", "\r"}:
|
||||
continue
|
||||
if not char.isprintable():
|
||||
return None
|
||||
|
||||
# Break up extremely long content
|
||||
if len(content) > 10000: # Arbitrary large limit
|
||||
logger.warning("Content too long, truncating: %s", content[:100])
|
||||
return None
|
||||
|
||||
# Check for unclosed category first
|
||||
if "[" in content and "]" not in content:
|
||||
raise ParseError("unclosed category")
|
||||
|
||||
# Then check for missing category
|
||||
match = re.match(r"^\s*(?:-\s*)?\[([^\]]*)\](.*)", content)
|
||||
# Parse category [category]
|
||||
match = re.match(r"\[([^\]]+)\](.*)", line)
|
||||
if not match:
|
||||
raise ParseError("missing category")
|
||||
raise ParseError("Invalid format - must start with '[category]'")
|
||||
|
||||
category = match.group(1).strip()
|
||||
if not category:
|
||||
return None
|
||||
raise ParseError("Category cannot be empty")
|
||||
|
||||
content = match.group(2).strip()
|
||||
rest = match.group(2).strip()
|
||||
|
||||
# Parse tags and content
|
||||
# Parse content and tags
|
||||
content_parts = []
|
||||
tags = []
|
||||
words = []
|
||||
for word in content.split():
|
||||
if word.startswith("#"):
|
||||
# Handle #tag1#tag2#tag3
|
||||
for tag in word.lstrip("#").split("#"):
|
||||
if tag:
|
||||
tags.append(tag)
|
||||
else:
|
||||
words.append(word)
|
||||
|
||||
content = " ".join(words)
|
||||
|
||||
# Extract context
|
||||
context = None
|
||||
if content.endswith(")"):
|
||||
pos = content.find("(")
|
||||
if pos > 0: # Must have content before paren
|
||||
before = content[:pos].strip()
|
||||
if before:
|
||||
context = content[pos + 1 : -1].strip()
|
||||
content = before
|
||||
|
||||
return Observation(category=category, content=content, tags=tags, context=context)
|
||||
except ParseError:
|
||||
raise
|
||||
except Exception:
|
||||
logger.exception("Failed to parse observation: %s", content) # pragma: no cover
|
||||
return None # pragma: no cover
|
||||
# Extract context if exists
|
||||
if rest.endswith(")"):
|
||||
context_start = rest.rfind("(")
|
||||
if context_start > 0:
|
||||
context = rest[context_start + 1:-1].strip()
|
||||
rest = rest[:context_start].strip()
|
||||
|
||||
# Split remaining text and collect tags
|
||||
for word in rest.split():
|
||||
if word.startswith("#"):
|
||||
tag = word[1:].strip()
|
||||
if tag:
|
||||
tags.append(tag)
|
||||
else:
|
||||
content_parts.append(word)
|
||||
|
||||
content = " ".join(content_parts)
|
||||
if not content:
|
||||
raise ParseError("Content cannot be empty")
|
||||
|
||||
return cls(
|
||||
category=category,
|
||||
content=content,
|
||||
tags=tags,
|
||||
context=context,
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
if not isinstance(e, ParseError):
|
||||
raise ParseError(f"Failed to parse observation: {line}: {str(e)}") from e
|
||||
raise
|
||||
@@ -1,70 +1,69 @@
|
||||
"""Models for the markdown parser."""
|
||||
|
||||
import logging
|
||||
import re
|
||||
from typing import Optional
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
from basic_memory.markdown import ParseError
|
||||
from basic_memory.utils.file_utils import ParseError
|
||||
|
||||
logging.basicConfig(level=logging.DEBUG) # pragma: no cover
|
||||
logger = logging.getLogger(__name__) # pragma: no cover
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Relation(BaseModel):
|
||||
"""A relation between entities."""
|
||||
|
||||
target: str # The entity being linked to
|
||||
type: str # The type of relation
|
||||
type: str
|
||||
target: str
|
||||
context: Optional[str] = None
|
||||
|
||||
@classmethod
|
||||
def from_line(cls, content: str) -> Optional["Relation"]:
|
||||
"""Parse a relation line."""
|
||||
def from_line(cls, line: str) -> Optional["Relation"]:
|
||||
"""
|
||||
Parse a relation from a line.
|
||||
|
||||
Format must be:
|
||||
- relation_type [[Target Entity]] (optional context)
|
||||
|
||||
Leading spaces before bullet are allowed.
|
||||
"""
|
||||
try:
|
||||
if not content.strip():
|
||||
line = line.strip()
|
||||
|
||||
# Skip empty or non-bullet lines
|
||||
if not line or not line.startswith("-"):
|
||||
return None
|
||||
|
||||
# Check for unclosed markup
|
||||
if "[[" in content and "]]" not in content:
|
||||
raise ParseError("missing ]]")
|
||||
if "]]" in content and "[[" not in content:
|
||||
raise ParseError("invalid relation syntax")
|
||||
# Remove bullet and trim
|
||||
line = line[1:].lstrip()
|
||||
|
||||
# Find the link - must have [[target]] with content inside
|
||||
match = re.search(r"\[\[([^\]]*)\]\]", content)
|
||||
if not match:
|
||||
raise ParseError("missing [[")
|
||||
|
||||
target = match.group(1).strip()
|
||||
if not target: # Empty target
|
||||
return None
|
||||
|
||||
# Get text before the link, excluding bullet
|
||||
before_link = content[: match.start()].strip(" -")
|
||||
|
||||
# Validate relation type
|
||||
rel_type = before_link.strip()
|
||||
if not rel_type or rel_type == "missing type": # Explicitly reject "missing type"
|
||||
return None
|
||||
|
||||
# Get text after the link
|
||||
after_link = content[match.end():].strip()
|
||||
|
||||
# Check for context in parentheses
|
||||
# Extract context from parens at end if present
|
||||
context = None
|
||||
if after_link:
|
||||
if not (after_link.startswith("(") and after_link.endswith(")")):
|
||||
raise ParseError("invalid context format")
|
||||
# Handle invalid context formats
|
||||
if ")" in after_link[1:-1]:
|
||||
raise ParseError("invalid context format")
|
||||
context = after_link[1:-1].strip()
|
||||
if line.endswith(")"):
|
||||
context_start = line.rfind("(")
|
||||
if context_start > 0:
|
||||
context = line[context_start + 1:-1].strip()
|
||||
line = line[:context_start].strip()
|
||||
|
||||
# Look for [[target]]
|
||||
match = re.match(r"^(\w+)\s+\[\[([^\]]+)\]\]", line)
|
||||
if not match:
|
||||
raise ParseError("Invalid format - must be 'relation_type [[Target]]'")
|
||||
|
||||
rel_type = match.group(1).strip()
|
||||
target = match.group(2).strip()
|
||||
|
||||
if not rel_type:
|
||||
raise ParseError("Relation type cannot be empty")
|
||||
if not target:
|
||||
raise ParseError("Target cannot be empty")
|
||||
|
||||
return cls(
|
||||
type=rel_type,
|
||||
target=target,
|
||||
context=context
|
||||
)
|
||||
|
||||
return Relation(target=target, type=rel_type, context=context)
|
||||
except ParseError:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.exception("Failed to parse relation: %s", content) # pragma: no cover
|
||||
return None # pragma: no cover
|
||||
if not isinstance(e, ParseError):
|
||||
raise ParseError(f"Failed to parse relation: {line}: {str(e)}") from e
|
||||
raise
|
||||
@@ -116,19 +116,26 @@ async def parse_frontmatter(content: str) -> Tuple[Dict[str, Any], str]:
|
||||
ParseError: If frontmatter parsing fails
|
||||
"""
|
||||
try:
|
||||
if not content.startswith("---\n"):
|
||||
return {}, content
|
||||
# Ensure we have frontmatter
|
||||
if not content.strip().startswith("---"):
|
||||
return {}, content.strip()
|
||||
|
||||
try:
|
||||
_, fm, content = content.split("---\n", 2)
|
||||
except ValueError as e:
|
||||
raise ParseError("Invalid frontmatter format") from e
|
||||
# Split on first two occurrences of ---
|
||||
parts = content.split("---", 2)
|
||||
if len(parts) < 3:
|
||||
raise ParseError("Invalid frontmatter format")
|
||||
|
||||
# Parse YAML (skipping empty first part)
|
||||
try:
|
||||
metadata = yaml.safe_load(fm)
|
||||
return metadata or {}, content.strip()
|
||||
frontmatter = yaml.safe_load(parts[1])
|
||||
if not isinstance(frontmatter, dict):
|
||||
raise ParseError("Frontmatter must be a YAML dictionary")
|
||||
|
||||
# Return parsed frontmatter and rest of content
|
||||
return frontmatter, parts[2].strip()
|
||||
|
||||
except yaml.YAMLError as e:
|
||||
raise ParseError(f"Invalid YAML in frontmatter: {e}") from e
|
||||
raise ParseError(f"Invalid YAML in frontmatter: {e}")
|
||||
|
||||
except Exception as e:
|
||||
if not isinstance(e, ParseError):
|
||||
|
||||
Reference in New Issue
Block a user