mirror of
https://github.com/basicmachines-co/basic-memory
synced 2026-06-21 13:47:35 +00:00
markdown parsing
This commit is contained in:
phernandez
+127
-134
@@ -1,6 +1,6 @@
|
||||
"""
|
||||
Parser for Basic Memory entity markdown files.
|
||||
"""
|
||||
"""Parser for Basic Memory entity markdown files."""
|
||||
import re
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional
|
||||
@@ -9,6 +9,9 @@ import frontmatter
|
||||
from markdown_it import MarkdownIt
|
||||
from pydantic import BaseModel
|
||||
|
||||
logging.basicConfig(level=logging.DEBUG)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
class ParseError(Exception):
|
||||
"""Raised when parsing fails"""
|
||||
pass
|
||||
@@ -29,6 +32,7 @@ class Relation(BaseModel):
|
||||
class EntityFrontmatter(BaseModel):
|
||||
"""Frontmatter metadata for an entity."""
|
||||
type: str
|
||||
id: str
|
||||
created: datetime
|
||||
modified: datetime
|
||||
tags: List[str]
|
||||
@@ -63,119 +67,106 @@ class EntityParser:
|
||||
def __init__(self):
|
||||
self.md = MarkdownIt()
|
||||
|
||||
def _parse_observation(self, line: str) -> Observation:
|
||||
"""Parse a single observation line.
|
||||
|
||||
Format: - [category] content #tag1 #tag2 "optional context"
|
||||
"""
|
||||
# Remove leading "- " if present
|
||||
content = line.strip()
|
||||
if content.startswith("- "):
|
||||
content = content[2:].strip()
|
||||
|
||||
# Extract category
|
||||
if not content.startswith("["):
|
||||
raise ParseError(f"Invalid observation format, missing category: {line}")
|
||||
category_end = content.find("]")
|
||||
if category_end == -1:
|
||||
raise ParseError(f"Invalid observation format, unclosed category: {line}")
|
||||
category = content[1:category_end].strip()
|
||||
|
||||
# Remove category from content
|
||||
content = content[category_end + 1:].strip()
|
||||
|
||||
# Extract tags
|
||||
tags = []
|
||||
words = content.split()
|
||||
filtered_words = []
|
||||
for word in words:
|
||||
if word.startswith("#"):
|
||||
tags.append(word[1:]) # Remove # from tag
|
||||
else:
|
||||
filtered_words.append(word)
|
||||
content = " ".join(filtered_words)
|
||||
|
||||
# Extract context if present
|
||||
context = None
|
||||
if content.endswith('"'):
|
||||
last_quote = content.rfind('"')
|
||||
second_last_quote = content.rfind('"', 0, last_quote)
|
||||
if second_last_quote != -1:
|
||||
context = content[second_last_quote + 1:last_quote]
|
||||
content = content[:second_last_quote].strip()
|
||||
|
||||
return Observation(
|
||||
category=category,
|
||||
content=content,
|
||||
tags=tags,
|
||||
context=context
|
||||
)
|
||||
def _parse_observation(self, content: str) -> Optional[Observation]:
|
||||
"""Parse an observation line."""
|
||||
try:
|
||||
if not content.strip():
|
||||
return None
|
||||
|
||||
def _parse_relation(self, line: str) -> Relation:
|
||||
"""Parse a single relation line.
|
||||
|
||||
Format: - [[Entity]] #relation_type "optional context"
|
||||
"""
|
||||
# Remove leading "- " if present
|
||||
content = line.strip()
|
||||
if content.startswith("- "):
|
||||
content = content[2:].strip()
|
||||
|
||||
# Extract target
|
||||
if not content.startswith("[["):
|
||||
raise ParseError(f"Invalid relation format, missing [[ : {line}")
|
||||
link_end = content.find("]]")
|
||||
if link_end == -1:
|
||||
raise ParseError(f"Invalid relation format, missing ]] : {line}")
|
||||
target = content[2:link_end].strip()
|
||||
|
||||
# Move past ]]
|
||||
content = content[link_end + 2:].strip()
|
||||
|
||||
# Extract relation type
|
||||
if not content.startswith("#"):
|
||||
raise ParseError(f"Invalid relation format, missing relation type: {line}")
|
||||
|
||||
words = content.split()
|
||||
rel_type = words[0][1:] # Remove # from type
|
||||
|
||||
# Extract context if present
|
||||
context = None
|
||||
remaining = " ".join(words[1:])
|
||||
if remaining:
|
||||
if remaining.startswith('"') and remaining.endswith('"'):
|
||||
context = remaining[1:-1]
|
||||
|
||||
return Relation(
|
||||
target=target,
|
||||
type=rel_type,
|
||||
context=context
|
||||
)
|
||||
# Parse category [type]
|
||||
match = re.match(r'^\s*(?:-\s*)?\[([^\]]+)\](.*)', content)
|
||||
if not match:
|
||||
return None
|
||||
category = match.group(1).strip()
|
||||
content = match.group(2).strip()
|
||||
|
||||
def _parse_metadata_line(self, line: str) -> tuple[str, str]:
|
||||
"""Parse a single metadata line."""
|
||||
if ":" not in line:
|
||||
return None, None
|
||||
# Parse tags and content
|
||||
tags = []
|
||||
words = []
|
||||
for word in content.split():
|
||||
if word.startswith('#'):
|
||||
# Handle #tag1#tag2#tag3
|
||||
for tag in word.lstrip('#').split('#'):
|
||||
if tag:
|
||||
tags.append(tag)
|
||||
else:
|
||||
words.append(word)
|
||||
|
||||
content = ' '.join(words)
|
||||
|
||||
key, value = line.split(":", 1)
|
||||
return key.strip(), value.strip()
|
||||
# Extract context in parentheses
|
||||
context = None
|
||||
if content.endswith(')'):
|
||||
ctx_start = content.rfind('(')
|
||||
if ctx_start != -1:
|
||||
context = content[ctx_start + 1:-1].strip()
|
||||
content = content[:ctx_start].strip()
|
||||
|
||||
def parse_file(self, path: Path) -> Entity:
|
||||
return Observation(
|
||||
category=category,
|
||||
content=content,
|
||||
tags=tags,
|
||||
context=context
|
||||
)
|
||||
except Exception as e:
|
||||
logger.exception("Failed to parse observation: %s", content)
|
||||
return None
|
||||
|
||||
def _parse_relation(self, content: str) -> Optional[Relation]:
|
||||
"""Parse a relation line."""
|
||||
try:
|
||||
if not content.strip():
|
||||
return None
|
||||
|
||||
# Find the link
|
||||
match = re.search(r'\[\[([^\]]+)\]\]', content)
|
||||
if not match:
|
||||
return None
|
||||
|
||||
target = match.group(1).strip()
|
||||
before_link = content[:match.start()].strip(' -')
|
||||
after_link = content[match.end():].strip()
|
||||
|
||||
# Everything before the link is the type
|
||||
rel_type = before_link.strip()
|
||||
if not rel_type:
|
||||
return None
|
||||
|
||||
# Check for context in parentheses
|
||||
context = None
|
||||
if after_link.startswith('(') and after_link.endswith(')'):
|
||||
context = after_link[1:-1].strip()
|
||||
|
||||
return Relation(
|
||||
target=target,
|
||||
type=rel_type,
|
||||
context=context
|
||||
)
|
||||
except Exception as e:
|
||||
logger.exception("Failed to parse relation: %s", content)
|
||||
return None
|
||||
|
||||
def parse_file(self, path: Path, encoding: str = 'utf-8') -> Entity:
|
||||
"""Parse an entity markdown file."""
|
||||
if not path.exists():
|
||||
raise ParseError(f"File does not exist: {path}")
|
||||
|
||||
try:
|
||||
# Parse frontmatter and content
|
||||
post = frontmatter.load(path)
|
||||
# Read and parse frontmatter
|
||||
with open(path, 'r', encoding=encoding) as f:
|
||||
content = f.read()
|
||||
post = frontmatter.loads(content)
|
||||
|
||||
# Parse frontmatter
|
||||
frontmatter_data = EntityFrontmatter(**post.metadata)
|
||||
# Handle frontmatter
|
||||
metadata = dict(post.metadata)
|
||||
if isinstance(metadata.get('tags'), str):
|
||||
metadata['tags'] = [t.strip() for t in metadata['tags'].split(',')]
|
||||
frontmatter_data = EntityFrontmatter(**metadata)
|
||||
|
||||
# Parse markdown
|
||||
tokens = self.md.parse(post.content)
|
||||
|
||||
# Extract title, description, observations, etc
|
||||
# State for parsing
|
||||
title = ""
|
||||
description = ""
|
||||
observations = []
|
||||
@@ -184,43 +175,42 @@ class EntityParser:
|
||||
metadata = {}
|
||||
|
||||
current_section = None
|
||||
collecting_description = False
|
||||
current_list = []
|
||||
in_list = False
|
||||
|
||||
# Process tokens
|
||||
for token in tokens:
|
||||
if token.type == "heading_open" and token.tag == "h1":
|
||||
# Next token will be title
|
||||
current_section = "title"
|
||||
elif token.type == "heading_open" and token.tag == "h2":
|
||||
# Next token will be section name
|
||||
current_section = "section_name"
|
||||
collecting_description = False
|
||||
elif token.type == "inline":
|
||||
if current_section == "title":
|
||||
if token.type == 'heading_open':
|
||||
if token.tag == 'h1':
|
||||
current_section = 'title'
|
||||
elif token.tag == 'h2':
|
||||
current_section = 'section_name'
|
||||
elif token.type == 'inline':
|
||||
if current_section == 'title':
|
||||
title = token.content
|
||||
current_section = None
|
||||
elif current_section == "section_name":
|
||||
section_name = token.content.lower()
|
||||
if section_name == "description":
|
||||
collecting_description = True
|
||||
current_section = section_name
|
||||
elif collecting_description:
|
||||
elif current_section == 'section_name':
|
||||
section = token.content.lower()
|
||||
current_section = section
|
||||
if section in ['observations', 'relations']:
|
||||
in_list = False
|
||||
current_list = []
|
||||
elif current_section == 'description':
|
||||
description = token.content
|
||||
elif current_section == "observations":
|
||||
if token.content.strip(): # Skip empty lines
|
||||
observations.append(self._parse_observation(token.content))
|
||||
elif current_section == "relations":
|
||||
if token.content.strip(): # Skip empty lines
|
||||
relations.append(self._parse_relation(token.content))
|
||||
elif current_section == "context":
|
||||
elif current_section == 'observations' and token.content.strip():
|
||||
if obs := self._parse_observation(token.content):
|
||||
observations.append(obs)
|
||||
elif current_section == 'relations' and token.content.strip():
|
||||
if rel := self._parse_relation(token.content):
|
||||
relations.append(rel)
|
||||
elif current_section == 'context':
|
||||
context = token.content
|
||||
elif current_section == "metadata":
|
||||
# Process each line of metadata separately
|
||||
for line in token.content.split("\n"):
|
||||
key, value = self._parse_metadata_line(line.strip())
|
||||
if key and value:
|
||||
metadata[key] = value
|
||||
|
||||
# Create EntityContent
|
||||
elif token.type == 'bullet_list_open':
|
||||
in_list = True
|
||||
elif token.type == 'bullet_list_close':
|
||||
in_list = False
|
||||
|
||||
# Create entity
|
||||
content_data = EntityContent(
|
||||
title=title,
|
||||
description=description,
|
||||
@@ -230,11 +220,14 @@ class EntityParser:
|
||||
metadata=metadata
|
||||
)
|
||||
|
||||
# Return complete Entity
|
||||
return Entity(
|
||||
frontmatter=frontmatter_data,
|
||||
content=content_data
|
||||
)
|
||||
|
||||
except UnicodeError as e:
|
||||
if encoding == 'utf-8':
|
||||
return self.parse_file(path, encoding='utf-16')
|
||||
raise ParseError(f"Failed to read {path} with encoding {encoding}: {str(e)}")
|
||||
except Exception as e:
|
||||
raise ParseError(f"Failed to parse {path}: {str(e)}") from e
|
||||
@@ -0,0 +1,25 @@
|
||||
"""Test to explore markdown-it token structure."""
|
||||
from textwrap import dedent
|
||||
from markdown_it import MarkdownIt
|
||||
|
||||
def test_token_structure():
|
||||
"""Analyze markdown-it token structure."""
|
||||
content = dedent('''
|
||||
# Title
|
||||
|
||||
## Observations
|
||||
- [test] First line #tag
|
||||
- [another] Second line #tag2
|
||||
''')
|
||||
|
||||
md = MarkdownIt()
|
||||
tokens = md.parse(content)
|
||||
|
||||
# Print full token structure
|
||||
print("\nToken structure:")
|
||||
for i, token in enumerate(tokens):
|
||||
attrs = {key: getattr(token, key) for key in dir(token)
|
||||
if not key.startswith('_') and not callable(getattr(token, key))}
|
||||
print(f"\nToken {i}:")
|
||||
for key, value in attrs.items():
|
||||
print(f" {key}: {value!r}")
|
||||
+152
-104
@@ -1,4 +1,4 @@
|
||||
"""Tests for markdown entity parser."""
|
||||
"""Tests for the markdown entity parser."""
|
||||
import pytest
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
@@ -6,169 +6,192 @@ from textwrap import dedent
|
||||
|
||||
from basic_memory.markdown.parser import EntityParser, ParseError
|
||||
|
||||
def test_parse_observation():
|
||||
"""Test parsing individual observation lines."""
|
||||
def test_parse_observation_basic():
|
||||
"""Test basic observation parsing with category and tags."""
|
||||
parser = EntityParser()
|
||||
|
||||
# Basic observation with category and content
|
||||
obs = parser._parse_observation("- [design] Simple observation")
|
||||
obs = parser._parse_observation("- [design] Core feature #important #mvp")
|
||||
assert obs.category == "design"
|
||||
assert obs.content == "Simple observation"
|
||||
assert obs.tags == []
|
||||
assert obs.content == "Core feature"
|
||||
assert set(obs.tags) == {"important", "mvp"}
|
||||
assert obs.context is None
|
||||
|
||||
def test_parse_observation_with_context():
|
||||
"""Test observation parsing with context in parentheses."""
|
||||
parser = EntityParser()
|
||||
|
||||
# Observation with tags
|
||||
obs = parser._parse_observation("- [feature] Added search #important #core")
|
||||
obs = parser._parse_observation("- [feature] Authentication system #security #auth (Required for MVP)")
|
||||
assert obs.category == "feature"
|
||||
assert obs.content == "Added search"
|
||||
assert set(obs.tags) == {"important", "core"}
|
||||
assert obs.content == "Authentication system"
|
||||
assert set(obs.tags) == {"security", "auth"}
|
||||
assert obs.context == "Required for MVP"
|
||||
|
||||
def test_parse_observation_edge_cases():
|
||||
"""Test observation parsing edge cases."""
|
||||
parser = EntityParser()
|
||||
|
||||
# Multiple word tags
|
||||
obs = parser._parse_observation("- [tech] Database #high-priority #needs-review")
|
||||
assert set(obs.tags) == {"high-priority", "needs-review"}
|
||||
|
||||
# Multiple word category
|
||||
obs = parser._parse_observation("- [user experience] Design #ux")
|
||||
assert obs.category == "user experience"
|
||||
|
||||
# Parentheses in content shouldn't be treated as context
|
||||
obs = parser._parse_observation("- [code] Function (x) returns y #function")
|
||||
assert obs.content == "Function (x) returns y"
|
||||
assert obs.context is None
|
||||
|
||||
# Observation with context
|
||||
obs = parser._parse_observation('- [design] Core system design #architecture "Initial version"')
|
||||
assert obs.category == "design"
|
||||
assert obs.content == "Core system design"
|
||||
assert obs.tags == ["architecture"]
|
||||
assert obs.context == "Initial version"
|
||||
|
||||
# Multiple hashtags together
|
||||
obs = parser._parse_observation("- [test] Feature #important#urgent#now")
|
||||
assert set(obs.tags) == {"important", "urgent", "now"}
|
||||
|
||||
def test_parse_observation_errors():
|
||||
"""Test error handling in observation parsing."""
|
||||
parser = EntityParser()
|
||||
|
||||
# Missing category brackets
|
||||
with pytest.raises(ParseError, match="missing category"):
|
||||
parser._parse_observation("- Design without brackets")
|
||||
|
||||
parser._parse_observation("- Design without brackets #test")
|
||||
|
||||
# Unclosed category
|
||||
with pytest.raises(ParseError, match="unclosed category"):
|
||||
parser._parse_observation("- [design Core system")
|
||||
parser._parse_observation("- [design Core feature #test")
|
||||
|
||||
def test_parse_relation():
|
||||
"""Test parsing individual relation lines."""
|
||||
def test_parse_relation_basic():
|
||||
"""Test basic relation parsing."""
|
||||
parser = EntityParser()
|
||||
|
||||
# Basic relation
|
||||
rel = parser._parse_relation("- [[EntityA]] #depends_on")
|
||||
assert rel.target == "EntityA"
|
||||
assert rel.type == "depends_on"
|
||||
assert rel.context is None
|
||||
|
||||
# Relation with context
|
||||
rel = parser._parse_relation('- [[Component]] #implements "Core functionality"')
|
||||
assert rel.target == "Component"
|
||||
rel = parser._parse_relation("- implements [[Auth Service]]")
|
||||
assert rel.type == "implements"
|
||||
assert rel.context == "Core functionality"
|
||||
assert rel.target == "Auth Service"
|
||||
assert rel.context is None
|
||||
|
||||
def test_parse_relation_with_context():
|
||||
"""Test relation parsing with context."""
|
||||
parser = EntityParser()
|
||||
|
||||
rel = parser._parse_relation("- depends_on [[Database]] (Required for persistence)")
|
||||
assert rel.type == "depends_on"
|
||||
assert rel.target == "Database"
|
||||
assert rel.context == "Required for persistence"
|
||||
|
||||
def test_parse_relation_edge_cases():
|
||||
"""Test relation parsing edge cases."""
|
||||
parser = EntityParser()
|
||||
|
||||
# Multiple word type
|
||||
rel = parser._parse_relation("- is used by [[Client App]] (Primary consumer)")
|
||||
assert rel.type == "is used by"
|
||||
|
||||
# Brackets in context
|
||||
rel = parser._parse_relation("- implements [[API]] (Follows [OpenAPI] spec)")
|
||||
assert rel.context == "Follows [OpenAPI] spec"
|
||||
|
||||
# Extra spaces
|
||||
rel = parser._parse_relation("- specifies [[Format]] (Documentation)")
|
||||
assert rel.type == "specifies"
|
||||
assert rel.target == "Format"
|
||||
|
||||
def test_parse_relation_errors():
|
||||
"""Test error handling in relation parsing."""
|
||||
parser = EntityParser()
|
||||
|
||||
# Missing [[ prefix
|
||||
# Missing target brackets
|
||||
with pytest.raises(ParseError, match="missing \\[\\["):
|
||||
parser._parse_relation("- EntityA #depends_on")
|
||||
|
||||
# Missing ]] suffix
|
||||
with pytest.raises(ParseError, match="missing \\]\\]"):
|
||||
parser._parse_relation("- [[EntityA #depends_on")
|
||||
|
||||
# Missing relation type
|
||||
with pytest.raises(ParseError, match="missing relation type"):
|
||||
parser._parse_relation("- [[EntityA]] depends_on")
|
||||
parser._parse_relation("- implements Auth Service")
|
||||
|
||||
# Unclosed target
|
||||
with pytest.raises(ParseError, match="missing ]]"):
|
||||
parser._parse_relation("- implements [[Auth Service")
|
||||
|
||||
def test_parse_complete_file(tmp_path):
|
||||
"""Test parsing a complete entity file."""
|
||||
content = dedent('''
|
||||
---
|
||||
type: concept
|
||||
created: 2024-12-21T10:00:00Z
|
||||
modified: 2024-12-21T10:00:00Z
|
||||
tags: [testing, validation]
|
||||
type: component
|
||||
id: component/auth_service
|
||||
created: 2024-12-21T14:00:00Z
|
||||
modified: 2024-12-21T14:00:00Z
|
||||
tags: authentication, security, core
|
||||
status: active
|
||||
version: 1
|
||||
priority: high
|
||||
---
|
||||
|
||||
# Test Entity
|
||||
# Auth Service
|
||||
|
||||
This is a test entity for validation.
|
||||
Core authentication service.
|
||||
|
||||
## Description
|
||||
A more detailed description of the test entity.
|
||||
<!-- Some comments that should be ignored -->
|
||||
|
||||
## Observations
|
||||
- [test] First observation #testing
|
||||
- [design] Second observation #important "With context"
|
||||
- [design] Stateless authentication #security #architecture (JWT based)
|
||||
- [feature] Mobile client support #mobile #oauth (Required for App Store)
|
||||
- [tech] Caching layer #performance (Redis implementation)
|
||||
|
||||
## Relations
|
||||
- [[EntityA]] #depends_on "Required dependency"
|
||||
- [[EntityB]] #implements
|
||||
|
||||
## Context
|
||||
Additional context information.
|
||||
- implements [[OAuth Implementation]] (Core auth flows)
|
||||
- uses [[Redis Cache]] (Token caching)
|
||||
- specified_by [[Auth API Spec]] (OpenAPI spec)
|
||||
|
||||
## Metadata
|
||||
schema_version: 1.0
|
||||
validation_status: verified
|
||||
''').lstrip()
|
||||
owner: team-auth
|
||||
priority: high
|
||||
''')
|
||||
|
||||
# Write test file
|
||||
test_file = tmp_path / "test_entity.md"
|
||||
test_file.write_text(content)
|
||||
|
||||
# Parse file
|
||||
parser = EntityParser()
|
||||
entity = parser.parse_file(test_file)
|
||||
|
||||
# Check frontmatter
|
||||
assert entity.frontmatter.type == "concept"
|
||||
assert entity.frontmatter.tags == ["testing", "validation"]
|
||||
assert entity.frontmatter.type == "component"
|
||||
assert entity.frontmatter.id == "component/auth_service"
|
||||
assert "authentication" in entity.frontmatter.tags
|
||||
assert entity.frontmatter.status == "active"
|
||||
assert entity.frontmatter.priority == "high"
|
||||
|
||||
# Check content
|
||||
assert entity.content.title == "Test Entity"
|
||||
assert "detailed description" in entity.content.description
|
||||
assert "context information" in entity.content.context
|
||||
assert entity.content.title == "Auth Service"
|
||||
assert len(entity.content.observations) == 3
|
||||
assert len(entity.content.relations) == 3
|
||||
|
||||
# Check observations
|
||||
assert len(entity.content.observations) == 2
|
||||
first_obs = entity.content.observations[0]
|
||||
assert first_obs.category == "test"
|
||||
assert first_obs.content == "First observation"
|
||||
assert first_obs.tags == ["testing"]
|
||||
# Check specific observation
|
||||
obs = entity.content.observations[0]
|
||||
assert obs.category == "design"
|
||||
assert "security" in obs.tags
|
||||
assert obs.context == "JWT based"
|
||||
|
||||
# Check relations
|
||||
assert len(entity.content.relations) == 2
|
||||
first_rel = entity.content.relations[0]
|
||||
assert first_rel.target == "EntityA"
|
||||
assert first_rel.type == "depends_on"
|
||||
assert first_rel.context == "Required dependency"
|
||||
# Check specific relation
|
||||
rel = entity.content.relations[0]
|
||||
assert rel.type == "implements"
|
||||
assert rel.target == "OAuth Implementation"
|
||||
assert rel.context == "Core auth flows"
|
||||
|
||||
# Check metadata
|
||||
assert entity.content.metadata["schema_version"] == "1.0"
|
||||
assert entity.content.metadata["validation_status"] == "verified"
|
||||
|
||||
def test_parse_missing_file():
|
||||
"""Test handling of missing files."""
|
||||
parser = EntityParser()
|
||||
with pytest.raises(ParseError, match="File does not exist"):
|
||||
parser.parse_file(Path("nonexistent.md"))
|
||||
assert entity.content.metadata["owner"] == "team-auth"
|
||||
assert entity.content.metadata["priority"] == "high"
|
||||
|
||||
def test_parse_minimal_file(tmp_path):
|
||||
"""Test parsing a minimal valid entity file."""
|
||||
content = dedent('''
|
||||
---
|
||||
type: concept
|
||||
created: 2024-12-21T10:00:00Z
|
||||
modified: 2024-12-21T10:00:00Z
|
||||
type: component
|
||||
id: minimal
|
||||
created: 2024-12-21T14:00:00Z
|
||||
modified: 2024-12-21T14:00:00Z
|
||||
tags: []
|
||||
---
|
||||
|
||||
# Minimal Entity
|
||||
|
||||
## Description
|
||||
Minimal valid entity.
|
||||
''').lstrip()
|
||||
## Observations
|
||||
- [note] Basic observation #test
|
||||
|
||||
## Relations
|
||||
- references [[Other Entity]]
|
||||
''')
|
||||
|
||||
test_file = tmp_path / "minimal.md"
|
||||
test_file.write_text(content)
|
||||
@@ -176,12 +199,37 @@ def test_parse_minimal_file(tmp_path):
|
||||
parser = EntityParser()
|
||||
entity = parser.parse_file(test_file)
|
||||
|
||||
# Check required fields
|
||||
assert entity.frontmatter.type == "concept"
|
||||
assert isinstance(entity.frontmatter.created, datetime)
|
||||
assert entity.content.title == "Minimal Entity"
|
||||
assert entity.frontmatter.type == "component"
|
||||
assert len(entity.content.observations) == 1
|
||||
assert len(entity.content.relations) == 1
|
||||
|
||||
def test_parse_file_errors(tmp_path):
|
||||
"""Test error handling for invalid files."""
|
||||
parser = EntityParser()
|
||||
|
||||
# Optional fields should have default values
|
||||
assert entity.content.observations == []
|
||||
assert entity.content.relations == []
|
||||
assert entity.content.metadata == {}
|
||||
# Missing file
|
||||
with pytest.raises(ParseError, match="does not exist"):
|
||||
parser.parse_file(Path("nonexistent.md"))
|
||||
|
||||
# Invalid frontmatter
|
||||
content = dedent('''
|
||||
---
|
||||
invalid: yaml: [
|
||||
---
|
||||
# Title
|
||||
''')
|
||||
test_file = tmp_path / "invalid.md"
|
||||
test_file.write_text(content)
|
||||
with pytest.raises(ParseError):
|
||||
parser.parse_file(test_file)
|
||||
|
||||
# Missing required frontmatter
|
||||
content = dedent('''
|
||||
---
|
||||
type: component
|
||||
---
|
||||
# Title
|
||||
''')
|
||||
test_file.write_text(content)
|
||||
with pytest.raises(ParseError):
|
||||
parser.parse_file(test_file)
|
||||
@@ -0,0 +1,279 @@
|
||||
"""Tests for edge cases and tricky inputs in the markdown parser."""
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from textwrap import dedent
|
||||
import pytest
|
||||
|
||||
from basic_memory.markdown.parser import EntityParser, ParseError, EntityFrontmatter, EntityContent, Entity
|
||||
|
||||
def test_unicode_content(tmp_path):
|
||||
"""Test handling of Unicode content including emoji and non-Latin scripts."""
|
||||
content = dedent('''
|
||||
---
|
||||
type: test
|
||||
id: test/unicode
|
||||
created: 2024-12-21T14:00:00Z
|
||||
modified: 2024-12-21T14:00:00Z
|
||||
tags: [unicode, 测试]
|
||||
---
|
||||
|
||||
# Unicode Test 🧪
|
||||
|
||||
## Observations
|
||||
- [test] Emoji test 👍 #emoji #test
|
||||
- [中文] Chinese text 测试 #language
|
||||
- [русский] Russian привет #language
|
||||
- [😀] Emoji category #meta (Category test)
|
||||
|
||||
## Relations
|
||||
- implements [[测试组件]] (Unicode test)
|
||||
- used_by [[компонент]] (Another test)
|
||||
''')
|
||||
|
||||
test_file = tmp_path / "unicode.md"
|
||||
test_file.write_text(content, encoding='utf-8')
|
||||
|
||||
parser = EntityParser()
|
||||
entity = parser.parse_file(test_file)
|
||||
|
||||
assert "测试" in entity.frontmatter.tags
|
||||
assert entity.content.title == "Unicode Test 🧪"
|
||||
assert "👍" in entity.content.observations[0].content
|
||||
assert entity.content.observations[2].category == "русский"
|
||||
assert entity.content.observations[3].category == "😀"
|
||||
assert entity.content.relations[0].target == "测试组件"
|
||||
|
||||
def test_long_content(tmp_path):
|
||||
"""Test handling of very long content at our limits."""
|
||||
# Create a long observation right at our length limit
|
||||
long_obs = "x" * 995 + " #tag" # 1000 chars with tag
|
||||
|
||||
content = dedent(f'''
|
||||
---
|
||||
type: test
|
||||
id: test/long
|
||||
created: 2024-12-21T14:00:00Z
|
||||
modified: 2024-12-21T14:00:00Z
|
||||
tags: [long]
|
||||
---
|
||||
|
||||
# Long Content Test
|
||||
|
||||
## Description
|
||||
{"Very long description " * 100}
|
||||
|
||||
## Observations
|
||||
- [test] {long_obs}
|
||||
|
||||
## Relations
|
||||
- related_to [[{"Very long entity name " * 10}]] (Long context test)
|
||||
''')
|
||||
|
||||
test_file = tmp_path / "long.md"
|
||||
test_file.write_text(content)
|
||||
|
||||
parser = EntityParser()
|
||||
entity = parser.parse_file(test_file)
|
||||
|
||||
# Check that long content is preserved
|
||||
assert len(entity.content.observations[0].content) == 995
|
||||
assert entity.content.observations[0].tags == ["tag"]
|
||||
|
||||
def test_mixed_newlines(tmp_path):
|
||||
"""Test handling of different newline styles (\\n, \\r\\n, \\r)."""
|
||||
content = "---\\ntype: test\\r\\nid: test/newlines\\ncreated: 2024-12-21T14:00:00Z\\rmodified: 2024-12-21T14:00:00Z\\ntags: [test]\\n---\\n\\r\\n# Test\\r\\n## Observations\\n- [test] Line 1\\r- [test] Line 2\\n".replace('\\n', '\n').replace('\\r', '\r')
|
||||
|
||||
test_file = tmp_path / "newlines.md"
|
||||
test_file.write_text(content, encoding='utf-8')
|
||||
|
||||
parser = EntityParser()
|
||||
entity = parser.parse_file(test_file)
|
||||
|
||||
assert len(entity.content.observations) == 2
|
||||
|
||||
def test_malformed_frontmatter(tmp_path):
|
||||
"""Test various malformed frontmatter cases."""
|
||||
cases = [
|
||||
# Invalid YAML syntax
|
||||
'''---
|
||||
type: : test: :
|
||||
id: test/bad
|
||||
created: 2024-12-21T14:00:00Z
|
||||
modified: 2024-12-21T14:00:00Z
|
||||
tags: [test]
|
||||
---''',
|
||||
|
||||
# Invalid datetime format
|
||||
'''---
|
||||
type: test
|
||||
id: test/bad
|
||||
created: not-a-date
|
||||
modified: 2024-12-21T14:00:00Z
|
||||
tags: [test]
|
||||
---''',
|
||||
|
||||
# Missing required field
|
||||
'''---
|
||||
type: test
|
||||
created: 2024-12-21T14:00:00Z
|
||||
modified: 2024-12-21T14:00:00Z
|
||||
tags: [test]
|
||||
---''',
|
||||
|
||||
# Extra fields
|
||||
'''---
|
||||
type: test
|
||||
id: test/extra
|
||||
created: 2024-12-21T14:00:00Z
|
||||
modified: 2024-12-21T14:00:00Z
|
||||
tags: [test]
|
||||
nonexistent_field: value
|
||||
---'''
|
||||
]
|
||||
|
||||
parser = EntityParser()
|
||||
test_file = tmp_path / "bad.md"
|
||||
|
||||
for i, case in enumerate(cases):
|
||||
content = case + "\n# Test"
|
||||
test_file.write_text(content)
|
||||
with pytest.raises(ParseError):
|
||||
parser.parse_file(test_file)
|
||||
|
||||
def test_nested_structures(tmp_path):
|
||||
"""Test handling of nested markdown structures."""
|
||||
content = dedent('''
|
||||
---
|
||||
type: test
|
||||
id: test/nested
|
||||
created: 2024-12-21T14:00:00Z
|
||||
modified: 2024-12-21T14:00:00Z
|
||||
tags: [test]
|
||||
---
|
||||
|
||||
# Nested Test
|
||||
|
||||
## Observations
|
||||
- [test] Main point #main
|
||||
- [sub] Subpoint #sub
|
||||
- [subsub] Sub-subpoint #detail
|
||||
|
||||
## Relations
|
||||
- contains [[Sub Entity]]
|
||||
- and [[Another Entity]]
|
||||
- also [[Third Entity]]
|
||||
''')
|
||||
|
||||
test_file = tmp_path / "nested.md"
|
||||
test_file.write_text(content)
|
||||
|
||||
parser = EntityParser()
|
||||
entity = parser.parse_file(test_file)
|
||||
|
||||
# Only top-level items should be parsed
|
||||
assert len(entity.content.observations) == 1
|
||||
assert len(entity.content.relations) == 1
|
||||
|
||||
def test_file_encodings(tmp_path):
|
||||
"""Test different file encodings."""
|
||||
encodings = ['utf-8', 'utf-16', 'latin1']
|
||||
content = dedent('''
|
||||
---
|
||||
type: test
|
||||
id: test/encoding
|
||||
created: 2024-12-21T14:00:00Z
|
||||
modified: 2024-12-21T14:00:00Z
|
||||
tags: [test]
|
||||
---
|
||||
|
||||
# Encoding Test
|
||||
|
||||
## Observations
|
||||
- [test] ASCII content #test
|
||||
- [utf8] UTF-8 content 测试 #unicode
|
||||
''')
|
||||
|
||||
parser = EntityParser()
|
||||
|
||||
for encoding in encodings:
|
||||
test_file = tmp_path / f"encoding_{encoding}.md"
|
||||
test_file.write_text(content, encoding=encoding)
|
||||
|
||||
try:
|
||||
entity = parser.parse_file(test_file)
|
||||
assert len(entity.content.observations) == 2
|
||||
except UnicodeError:
|
||||
# Some encodings might not handle all characters
|
||||
pass
|
||||
|
||||
def test_empty_sections(tmp_path):
|
||||
"""Test handling of empty sections."""
|
||||
content = dedent('''
|
||||
---
|
||||
type: test
|
||||
id: test/empty
|
||||
created: 2024-12-21T14:00:00Z
|
||||
modified: 2024-12-21T14:00:00Z
|
||||
tags: []
|
||||
---
|
||||
|
||||
# Empty Test
|
||||
|
||||
## Description
|
||||
|
||||
## Observations
|
||||
|
||||
## Relations
|
||||
|
||||
## Context
|
||||
|
||||
## Metadata
|
||||
''')
|
||||
|
||||
test_file = tmp_path / "empty.md"
|
||||
test_file.write_text(content)
|
||||
|
||||
parser = EntityParser()
|
||||
entity = parser.parse_file(test_file)
|
||||
|
||||
assert entity.content.description == ""
|
||||
assert entity.content.observations == []
|
||||
assert entity.content.relations == []
|
||||
assert entity.content.context == ""
|
||||
assert entity.content.metadata == {}
|
||||
|
||||
def test_malformed_sections(tmp_path):
|
||||
"""Test various malformed section contents."""
|
||||
content = dedent('''
|
||||
---
|
||||
type: test
|
||||
id: test/malformed
|
||||
created: 2024-12-21T14:00:00Z
|
||||
modified: 2024-12-21T14:00:00Z
|
||||
tags: [test]
|
||||
---
|
||||
|
||||
# Malformed Test
|
||||
|
||||
## Observations
|
||||
- not a valid observation
|
||||
- [unclosed category content
|
||||
- no content]
|
||||
- [] empty category
|
||||
|
||||
## Relations
|
||||
- not a valid relation
|
||||
- missing type [[Entity]]
|
||||
- incomplete [[
|
||||
- ]] backwards
|
||||
''')
|
||||
|
||||
test_file = tmp_path / "malformed.md"
|
||||
test_file.write_text(content)
|
||||
|
||||
parser = EntityParser()
|
||||
entity = parser.parse_file(test_file)
|
||||
|
||||
# Should skip invalid entries but not fail completely
|
||||
assert len(entity.content.observations) == 0
|
||||
assert len(entity.content.relations) == 0
|
||||
Reference in New Issue
Block a user