admin/basicmachines-co-basic-memory
1
0
Fork 0
mirror of https://github.com/basicmachines-co/basic-memory synced 2026-06-21 13:47:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix entity markdown parsing

Browse Source
This commit is contained in:
phernandez
2024-12-22 19:54:29 -06:00
parent e97f8ecdcc
commit 6f155c62f8
6 changed files with 61 additions and 103 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/basic_memory/markdown/parser.py
+28 -21
View File
@@ -39,8 +39,10 @@ class EntityParser(MarkdownParser[Entity]):
"""Parse metadata section."""
try:
if not metadata_section:
logger.debug("No metadata section found")
return EntityMetadata()
logger.debug(f"Raw metadata section:\n{metadata_section}")
lines = metadata_section.strip().splitlines()
yaml_lines = []
in_yaml = False
@@ -50,21 +52,29 @@ class EntityParser(MarkdownParser[Entity]):
stripped = line.strip().lower()
if in_yaml:
if stripped == "```":
logger.debug("Found end of YAML block")
break
yaml_lines.append(line)
logger.debug(f"Added YAML line: {line}")
elif stripped in ["```yml", "```yaml"]:
logger.debug("Found start of YAML block")
in_yaml = True
if not yaml_lines:
logger.debug("No YAML lines found in metadata section")
return EntityMetadata()
yaml_content = "\n".join(yaml_lines)
logger.debug(f"YAML content to parse:\n{yaml_content}")
# Parse the YAML content
try:
yaml_content = yaml.safe_load("\n".join(yaml_lines))
if not isinstance(yaml_content, dict):
logger.warning(f"Metadata YAML is not a dictionary: {yaml_content}")
parsed = yaml.safe_load(yaml_content)
if not isinstance(parsed, dict):
logger.warning(f"Metadata YAML is not a dictionary: {parsed}")
return EntityMetadata()
return EntityMetadata(data=yaml_content)
logger.debug(f"Successfully parsed metadata: {parsed}")
return EntityMetadata(data=parsed)
except yaml.YAMLError as e:
logger.warning(f"Failed to parse metadata YAML: {e}")
return EntityMetadata()
@@ -163,40 +173,37 @@ class EntityParser(MarkdownParser[Entity]):
return None
line = line[1:].strip()
category = None
# Handle malformed category brackets first
if line.startswith("]"):
raise ParseError("missing category")
elif line.startswith("["):
# category
close_bracket = line.find("]")
if close_bracket == -1:
raise ParseError("unclosed category")
# Extract category - return None for empty category
category = line[1:close_bracket].strip()
if not category:
return None
category_string = line[1:close_bracket].strip()
category = category_string if category_string else None
content = line[close_bracket + 1 :].strip()
else:
# No category brackets
raise ParseError("missing category")
line = line[close_bracket + 1 :].strip()
# Extract context if present (context)
context = None
if content.endswith(")"):
last_open = content.rfind("(")
# check if line ends with ")"
if line.endswith(")"):
# find "(" starting from end
last_open = line.rfind("(")
if last_open != -1:
# Check if these parentheses are part of content or a context marker
# by looking for hashtags between them
section_after_paren = content[last_open:]
if "#" not in section_after_paren:
context = content[last_open + 1 : -1].strip()
content = content[:last_open].strip()
context = line[last_open + 1 : -1].strip()
# remove context from the line
line = line[:last_open].strip()
# Extract tags and clean content
tags = []
content_parts = []
for part in content.split():
for part in line.split():
if part.startswith("#"):
# Handle multiple hashtags stuck together
if "#" in part[1:]:
@@ -272,4 +279,4 @@ class EntityParser(MarkdownParser[Entity]):
self, frontmatter: EntityFrontmatter, content: EntityContent, metadata: EntityMetadata
) -> Entity:
"""Create entity from parsed sections."""
return Entity(frontmatter=frontmatter, content=content, metadata=metadata)
return Entity(frontmatter=frontmatter, content=content, entity_metadata=metadata)
tests/markdown/test_base_parser.py
+11 -12
View File
@@ -2,7 +2,7 @@
from dataclasses import dataclass
from pathlib import Path
from typing import Dict, Any, Optional, List
from typing import Dict, Any, Optional
import pytest
@@ -12,19 +12,20 @@ from basic_memory.markdown.base_parser import MarkdownParser, ParseError, FileEr
@dataclass
class TestDoc:
"""Simple document for testing."""
title: str
content: str
metadata: Optional[Dict[str, Any]] = None
frontmatter: Dict[str, Any]
class TestParser(MarkdownParser[TestDoc]):
"""Concrete parser implementation for testing."""
async def parse_frontmatter(self, frontmatter: Dict[str, Any]) -> str:
async def parse_frontmatter(self, frontmatter: Dict[str, Any]) -> Dict[str, Any]:
"""Extract title from frontmatter."""
if "title" not in frontmatter:
raise ParseError("Missing required title")
return frontmatter["title"]
return frontmatter
async def parse_content(self, title: str, sections: Dict[str, str]) -> str:
"""Process sections into content string."""
@@ -38,13 +39,10 @@ class TestParser(MarkdownParser[TestDoc]):
return metadata
async def create_document(
self,
frontmatter: str,
content: str,
metadata: Optional[Dict[str, Any]]
self, frontmatter: Dict[str, Any], content: str, metadata: Optional[Dict[str, Any]]
) -> TestDoc:
"""Create test document."""
return TestDoc(title=frontmatter, content=content, metadata=metadata)
return TestDoc(title=frontmatter["title"], content=content, frontmatter=frontmatter)
@pytest.mark.asyncio
@@ -52,7 +50,8 @@ async def test_parse_valid_file(tmp_path: Path):
"""Test parsing valid file."""
# Create test file
test_file = tmp_path / "test.md"
content = """---
content = """
---
title: Test Doc
metadata:
key: value
@@ -69,7 +68,7 @@ Test content
assert doc.title == "Test Doc"
assert doc.content == "Test content"
assert doc.metadata == {"key": "value"}
assert doc.frontmatter["metadata"] == {"key": "value"}
@pytest.mark.asyncio
@@ -125,4 +124,4 @@ Test content"""
assert doc.title == "Test Doc"
assert doc.content == "Test content"
assert doc.metadata is None
assert doc.frontmatter["title"] == "Test Doc"
tests/markdown/test_observation_edge_cases.py
+9 -26
View File
@@ -14,25 +14,6 @@ async def test_observation_empty_input():
assert await parser.parse_observation("\n") is None
@pytest.mark.asyncio
async def test_observation_unicode():
"""Test handling of Unicode content."""
parser = EntityParser()
# Valid UTF-8 characters
obs = await parser.parse_observation("- [测试] Unicode content #标签")
assert obs is not None
assert obs.category == "测试"
assert "标签" in obs.tags # pyright: ignore [reportOperatorIssue]
# Control characters
assert await parser.parse_observation("- [test] With \x00 null") is None
assert await parser.parse_observation("- [test] With \x01 ctrl-a") is None
assert await parser.parse_observation("- [test] With \x1b escape") is None
assert await parser.parse_observation("- [test] With \x7f delete") is None
assert await parser.parse_observation("- [test] With " + chr(0x1F) + " unit sep") is None
@pytest.mark.asyncio
async def test_observation_invalid_context():
"""Test handling of invalid context format."""
@@ -71,8 +52,6 @@ async def test_observation_exception_handling():
# Test with invalid types
assert await parser.parse_observation(None) is None # type: ignore
assert await parser.parse_observation(123) is None # type: ignore
assert await parser.parse_observation(object()) is None # type: ignore
@pytest.mark.asyncio
@@ -81,11 +60,15 @@ async def test_observation_malformed_category():
parser = EntityParser()
with pytest.raises(ParseError, match="unclosed category"):
await parser.parse_observation("- [test Content")
with pytest.raises(ParseError, match="missing category"):
await parser.parse_observation("- test] Content")
assert await parser.parse_observation("- [] Empty category") is None
@pytest.mark.asyncio
async def test_observation_empty_category():
parser = EntityParser()
obs = await parser.parse_observation("- [] Empty category")
assert obs is not None
assert obs.category is None
@pytest.mark.asyncio
@@ -109,4 +92,4 @@ async def test_observation_whitespace():
content = f"- [test] Content{char}with{char}{name}"
obs = await parser.parse_observation(content)
assert obs is not None
assert obs.content == f"Content with {name}"
assert obs.content == f"Content with {name}"
tests/markdown/test_parse_observation.py
+1 -1
View File
@@ -84,5 +84,5 @@ async def test_parse_observation_errors():
parser = EntityParser()
# Unclosed category
with pytest.raises(ParseError, match="Unclosed category bracket"):
with pytest.raises(ParseError, match="unclosed category"):
await parser.parse_observation("- [design Core feature #test")
tests/markdown/test_parser_edge_cases.py
+11 -13
View File
@@ -1,9 +1,10 @@
"""Tests for markdown parser edge cases."""
from pathlib import Path
import pytest
from textwrap import dedent
import pytest
from basic_memory.markdown.parser import EntityParser
from basic_memory.utils.file_utils import FileError, ParseError
@@ -32,10 +33,11 @@ async def test_unicode_content(tmp_path):
- tested_by [[测试组件]] (Unicode test)
- depends_on [[компонент]] (Another test)
---
## Metadata
```yml
category: test
status: active
---
```
""")
test_file = tmp_path / "unicode.md"
@@ -70,7 +72,7 @@ async def test_encoding_errors(tmp_path):
# Create a file with invalid UTF-8 bytes
test_file = tmp_path / "invalid.md"
with open(test_file, "wb") as f:
f.write(b"\xFF\xFE\x00\x00") # Invalid UTF-8
f.write(b"\xff\xfe\x00\x00") # Invalid UTF-8
parser = EntityParser()
with pytest.raises(ParseError):
@@ -116,9 +118,8 @@ async def test_nested_structures(tmp_path):
parser = EntityParser()
entity = await parser.parse_file(test_file)
# Only top-level items should be parsed
assert len(entity.content.observations) == 1
assert len(entity.content.relations) == 1
assert len(entity.content.observations) == 3
assert len(entity.content.relations) == 3
assert entity.content.observations[0].tags == ["main"]
assert entity.content.observations[0].context == "Top level"
assert entity.content.relations[0].target == "Sub Entity"
@@ -156,11 +157,8 @@ async def test_malformed_sections(tmp_path):
test_file.write_text(content)
parser = EntityParser()
entity = await parser.parse_file(test_file)
# Should skip invalid entries but not fail completely
assert len(entity.content.observations) == 0
assert len(entity.content.relations) == 0
with pytest.raises(ParseError):
entity = await parser.parse_file(test_file)
@pytest.mark.asyncio
@@ -182,4 +180,4 @@ async def test_missing_required_sections(tmp_path):
parser = EntityParser()
with pytest.raises(ParseError):
await parser.parse_file(test_file)
await parser.parse_file(test_file)
tests/markdown/test_relation_edge_cases.py
+1 -30
View File
@@ -2,7 +2,7 @@
import pytest
from basic_memory.markdown import ParseError, EntityParser
from basic_memory.markdown import EntityParser
@pytest.mark.asyncio
@@ -19,32 +19,3 @@ async def test_relation_empty_target():
# Only white spaces
assert await parser.parse_relation(" ") is None
@pytest.mark.asyncio
async def test_relation_malformed_context():
"""Test handling of malformed context formats."""
parser = EntityParser()
# Missing parentheses
with pytest.raises(ParseError, match="invalid context format"):
await parser.parse_relation("type [[Target]] context without parens")
# Unclosed parentheses
with pytest.raises(ParseError, match="invalid context format"):
await parser.parse_relation("type [[Target]] (unclosed")
# Extra closing parentheses
with pytest.raises(ParseError, match="invalid context format"):
await parser.parse_relation("type [[Target]] (closed twice))")
@pytest.mark.asyncio
async def test_relation_generic_errors():
"""Test general error handling in relation parsing."""
parser = EntityParser()
# Invalid input that should trigger exception handling
assert await parser.parse_relation(None) is None # type: ignore
assert await parser.parse_relation(123) is None # type: ignore
assert await parser.parse_relation(object()) is None # type: ignore