mirror of
https://github.com/basicmachines-co/basic-memory
synced 2026-06-21 13:47:35 +00:00
rename EntityParser to KnowledgeParser
This commit is contained in:
@@ -1,6 +1,6 @@
|
||||
"""Base package for markdown parsing."""
|
||||
|
||||
from basic_memory.markdown.parser import EntityParser
|
||||
from basic_memory.markdown.knowledge_parser import KnowledgeParser
|
||||
from basic_memory.markdown.schemas import (
|
||||
Entity,
|
||||
EntityContent,
|
||||
@@ -16,7 +16,7 @@ __all__ = [
|
||||
"EntityContent",
|
||||
"EntityFrontmatter",
|
||||
"EntityMetadata",
|
||||
"EntityParser",
|
||||
"KnowledgeParser",
|
||||
"Observation",
|
||||
"Relation",
|
||||
"ParseError",
|
||||
|
||||
@@ -80,9 +80,7 @@ class MarkdownParser(ABC, Generic[T]):
|
||||
|
||||
# Create final document
|
||||
return await self.create_document(
|
||||
frontmatter=parsed_frontmatter,
|
||||
content=parsed_content,
|
||||
metadata=parsed_metadata
|
||||
frontmatter=parsed_frontmatter, content=parsed_content, metadata=parsed_metadata
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
@@ -164,11 +162,11 @@ class MarkdownParser(ABC, Generic[T]):
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
async def parse_metadata(self, metadata: Optional[str]) -> Any:
|
||||
async def parse_metadata(self, metadata_section: Optional[str]) -> Any:
|
||||
"""Parse metadata section."""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
async def create_document(self, frontmatter: Any, content: Any, metadata: Any) -> T:
|
||||
"""Create document from parsed sections."""
|
||||
pass
|
||||
pass
|
||||
|
||||
@@ -17,7 +17,7 @@ from basic_memory.markdown.schemas import (
|
||||
)
|
||||
|
||||
|
||||
class EntityParser(MarkdownParser[Entity]):
|
||||
class KnowledgeParser(MarkdownParser[Entity]):
|
||||
"""Parser for entity markdown files.
|
||||
|
||||
Entity files must have:
|
||||
@@ -34,7 +34,7 @@ class TestParser(MarkdownParser[TestDoc]):
|
||||
# Join all section content if no direct content section
|
||||
return "\n".join(sections.values())
|
||||
|
||||
async def parse_metadata(self, metadata: Optional[Dict[str, Any]]) -> Optional[Dict[str, Any]]:
|
||||
async def parse_metadata(self, metadata: Optional[Dict[str, Any]]) -> Optional[Dict[str, Any]]: # pyright: ignore [reportIncompatibleMethodOverride]
|
||||
"""Pass through metadata."""
|
||||
return metadata
|
||||
|
||||
|
||||
@@ -1,23 +1,22 @@
|
||||
"""Tests for entity frontmatter parsing."""
|
||||
|
||||
from textwrap import dedent
|
||||
|
||||
import pytest
|
||||
from datetime import datetime
|
||||
|
||||
from basic_memory.markdown.parser import EntityParser, ParseError
|
||||
import pytest
|
||||
|
||||
from basic_memory.markdown.knowledge_parser import KnowledgeParser, ParseError
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_parse_frontmatter():
|
||||
"""Test parsing valid frontmatter."""
|
||||
parser = EntityParser()
|
||||
parser = KnowledgeParser()
|
||||
frontmatter_dict = {
|
||||
"type": "test",
|
||||
"id": "test/123",
|
||||
"created": "2024-12-22T10:00:00Z",
|
||||
"modified": "2024-12-22T10:00:00Z",
|
||||
"tags": ["test", "example"]
|
||||
"tags": ["test", "example"],
|
||||
}
|
||||
|
||||
result = await parser.parse_frontmatter(frontmatter_dict)
|
||||
@@ -31,13 +30,13 @@ async def test_parse_frontmatter():
|
||||
@pytest.mark.asyncio
|
||||
async def test_parse_frontmatter_comma_tags():
|
||||
"""Test parsing tags with commas."""
|
||||
parser = EntityParser()
|
||||
parser = KnowledgeParser()
|
||||
frontmatter_dict = {
|
||||
"type": "test",
|
||||
"id": "test/tags",
|
||||
"created": "2024-12-22T10:00:00Z",
|
||||
"modified": "2024-12-22T10:00:00Z",
|
||||
"tags": "tag1, tag2, tag3" # String format
|
||||
"tags": "tag1, tag2, tag3", # String format
|
||||
}
|
||||
|
||||
result = await parser.parse_frontmatter(frontmatter_dict)
|
||||
@@ -47,13 +46,13 @@ async def test_parse_frontmatter_comma_tags():
|
||||
@pytest.mark.asyncio
|
||||
async def test_parse_frontmatter_missing_required():
|
||||
"""Test error on missing required fields."""
|
||||
parser = EntityParser()
|
||||
parser = KnowledgeParser()
|
||||
frontmatter_dict = {
|
||||
"type": "test",
|
||||
# missing id
|
||||
"created": "2024-12-22T10:00:00Z",
|
||||
"modified": "2024-12-22T10:00:00Z",
|
||||
"tags": []
|
||||
"tags": [],
|
||||
}
|
||||
|
||||
with pytest.raises(ParseError, match="Missing required frontmatter fields: id"):
|
||||
@@ -63,13 +62,13 @@ async def test_parse_frontmatter_missing_required():
|
||||
@pytest.mark.asyncio
|
||||
async def test_parse_frontmatter_invalid_date():
|
||||
"""Test error on invalid date format."""
|
||||
parser = EntityParser()
|
||||
parser = KnowledgeParser()
|
||||
frontmatter_dict = {
|
||||
"type": "test",
|
||||
"id": "test/invalid",
|
||||
"created": "not-a-date",
|
||||
"modified": "2024-12-22T10:00:00Z",
|
||||
"tags": []
|
||||
"tags": [],
|
||||
}
|
||||
|
||||
with pytest.raises(ParseError, match="Invalid date format for created"):
|
||||
@@ -79,13 +78,13 @@ async def test_parse_frontmatter_invalid_date():
|
||||
@pytest.mark.asyncio
|
||||
async def test_parse_frontmatter_whitespace():
|
||||
"""Test handling of extra whitespace."""
|
||||
parser = EntityParser()
|
||||
parser = KnowledgeParser()
|
||||
frontmatter_dict = {
|
||||
"type": " test ",
|
||||
"id": " test/spaces ",
|
||||
"created": "2024-12-22T10:00:00Z",
|
||||
"modified": "2024-12-22T10:00:00Z",
|
||||
"tags": " tag1 , tag2 "
|
||||
"tags": " tag1 , tag2 ",
|
||||
}
|
||||
|
||||
result = await parser.parse_frontmatter(frontmatter_dict)
|
||||
@@ -97,14 +96,14 @@ async def test_parse_frontmatter_whitespace():
|
||||
@pytest.mark.asyncio
|
||||
async def test_parse_frontmatter_empty_tags():
|
||||
"""Test handling of empty tags field."""
|
||||
parser = EntityParser()
|
||||
parser = KnowledgeParser()
|
||||
frontmatter_dict = {
|
||||
"type": "test",
|
||||
"id": "test/notags",
|
||||
"created": "2024-12-22T10:00:00Z",
|
||||
"modified": "2024-12-22T10:00:00Z"
|
||||
"modified": "2024-12-22T10:00:00Z",
|
||||
# No tags field
|
||||
}
|
||||
|
||||
result = await parser.parse_frontmatter(frontmatter_dict)
|
||||
assert result.tags == [] # Should default to empty list
|
||||
assert result.tags == [] # Should default to empty list
|
||||
|
||||
@@ -5,7 +5,7 @@ from textwrap import dedent
|
||||
|
||||
import pytest
|
||||
|
||||
from basic_memory.markdown.parser import EntityParser
|
||||
from basic_memory.markdown.knowledge_parser import KnowledgeParser
|
||||
from basic_memory.markdown.schemas import Entity, EntityFrontmatter, EntityContent
|
||||
from basic_memory.utils.file_utils import ParseError, FileError
|
||||
|
||||
@@ -52,7 +52,7 @@ async def test_parse_complete_file(tmp_path, valid_entity_content):
|
||||
test_file = tmp_path / "test_entity.md"
|
||||
test_file.write_text(valid_entity_content)
|
||||
|
||||
parser = EntityParser()
|
||||
parser = KnowledgeParser()
|
||||
entity = await parser.parse_file(test_file)
|
||||
|
||||
# Verify entity structure
|
||||
@@ -115,7 +115,7 @@ async def test_parse_minimal_file(tmp_path):
|
||||
test_file = tmp_path / "minimal.md"
|
||||
test_file.write_text(content)
|
||||
|
||||
parser = EntityParser()
|
||||
parser = KnowledgeParser()
|
||||
entity = await parser.parse_file(test_file)
|
||||
|
||||
assert entity.frontmatter.type == "component"
|
||||
@@ -128,7 +128,7 @@ async def test_parse_minimal_file(tmp_path):
|
||||
@pytest.mark.asyncio
|
||||
async def test_parse_content_str(valid_entity_content):
|
||||
"""Test parsing content string directly."""
|
||||
parser = EntityParser()
|
||||
parser = KnowledgeParser()
|
||||
entity = await parser.parse_content_str(valid_entity_content)
|
||||
|
||||
assert isinstance(entity, Entity)
|
||||
@@ -141,7 +141,7 @@ async def test_parse_content_str(valid_entity_content):
|
||||
@pytest.mark.asyncio
|
||||
async def test_metadata_handling(tmp_path):
|
||||
"""Test metadata section parsing."""
|
||||
parser = EntityParser()
|
||||
parser = KnowledgeParser()
|
||||
|
||||
# Multiple metadata fields
|
||||
content = dedent("""
|
||||
@@ -199,7 +199,7 @@ async def test_metadata_handling(tmp_path):
|
||||
@pytest.mark.asyncio
|
||||
async def test_error_handling(tmp_path):
|
||||
"""Test error handling."""
|
||||
parser = EntityParser()
|
||||
parser = KnowledgeParser()
|
||||
|
||||
# Missing file
|
||||
with pytest.raises(FileError):
|
||||
|
||||
@@ -2,13 +2,13 @@
|
||||
|
||||
import pytest
|
||||
|
||||
from basic_memory.markdown import ParseError, EntityParser
|
||||
from basic_memory.markdown import ParseError, KnowledgeParser
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_observation_empty_input():
|
||||
"""Test handling of empty input."""
|
||||
parser = EntityParser()
|
||||
parser = KnowledgeParser()
|
||||
assert await parser.parse_observation("") is None
|
||||
assert await parser.parse_observation(" ") is None
|
||||
assert await parser.parse_observation("\n") is None
|
||||
@@ -17,7 +17,7 @@ async def test_observation_empty_input():
|
||||
@pytest.mark.asyncio
|
||||
async def test_observation_invalid_context():
|
||||
"""Test handling of invalid context format."""
|
||||
parser = EntityParser()
|
||||
parser = KnowledgeParser()
|
||||
obs = await parser.parse_observation("- [test] Content (unclosed")
|
||||
assert obs is not None
|
||||
assert obs.content == "Content (unclosed"
|
||||
@@ -33,7 +33,7 @@ async def test_observation_invalid_context():
|
||||
async def test_observation_complex_format():
|
||||
"""Test parsing complex observation formats."""
|
||||
# Test multiple nested tags and spaces
|
||||
parser = EntityParser()
|
||||
parser = KnowledgeParser()
|
||||
obs = await parser.parse_observation("- [complex test] This is #tag1#tag2 with #tag3 content")
|
||||
assert obs is not None
|
||||
assert obs.category == "complex test"
|
||||
@@ -46,7 +46,7 @@ async def test_observation_exception_handling():
|
||||
"""Test general error handling in observation parsing."""
|
||||
# Test with a problematic regex pattern that could cause catastrophic backtracking
|
||||
long_input = "[test] " + "a" * 1000000 # Very long input
|
||||
parser = EntityParser()
|
||||
parser = KnowledgeParser()
|
||||
|
||||
assert await parser.parse_observation(long_input) is None
|
||||
|
||||
@@ -57,14 +57,14 @@ async def test_observation_exception_handling():
|
||||
@pytest.mark.asyncio
|
||||
async def test_observation_malformed_category():
|
||||
"""Test handling of malformed category brackets."""
|
||||
parser = EntityParser()
|
||||
parser = KnowledgeParser()
|
||||
with pytest.raises(ParseError, match="unclosed category"):
|
||||
await parser.parse_observation("- [test Content")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_observation_empty_category():
|
||||
parser = EntityParser()
|
||||
parser = KnowledgeParser()
|
||||
obs = await parser.parse_observation("- [] Empty category")
|
||||
|
||||
assert obs is not None
|
||||
@@ -75,7 +75,7 @@ async def test_observation_empty_category():
|
||||
async def test_observation_whitespace():
|
||||
"""Test handling of whitespace."""
|
||||
# Valid whitespace cases
|
||||
parser = EntityParser()
|
||||
parser = KnowledgeParser()
|
||||
obs = await parser.parse_observation("- [test] Content")
|
||||
assert obs is not None
|
||||
assert obs.content == "Content"
|
||||
|
||||
@@ -2,13 +2,13 @@
|
||||
|
||||
import pytest
|
||||
|
||||
from basic_memory.markdown.parser import EntityParser, ParseError
|
||||
from basic_memory.markdown.knowledge_parser import KnowledgeParser, ParseError
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_parse_observation_basic():
|
||||
"""Test basic observation parsing with category and tags."""
|
||||
parser = EntityParser()
|
||||
parser = KnowledgeParser()
|
||||
|
||||
obs = await parser.parse_observation("- [design] Core feature #important #mvp")
|
||||
|
||||
@@ -22,7 +22,7 @@ async def test_parse_observation_basic():
|
||||
@pytest.mark.asyncio
|
||||
async def test_parse_observation_with_context():
|
||||
"""Test observation parsing with context in parentheses."""
|
||||
parser = EntityParser()
|
||||
parser = KnowledgeParser()
|
||||
|
||||
obs = await parser.parse_observation(
|
||||
"- [feature] Authentication system #security #auth (Required for MVP)"
|
||||
@@ -37,7 +37,7 @@ async def test_parse_observation_with_context():
|
||||
@pytest.mark.asyncio
|
||||
async def test_parse_observation_without_category():
|
||||
"""Test observation parsing with context in parentheses."""
|
||||
parser = EntityParser()
|
||||
parser = KnowledgeParser()
|
||||
|
||||
obs = await parser.parse_observation(
|
||||
"- Authentication system #security #auth (Required for MVP)"
|
||||
@@ -52,7 +52,7 @@ async def test_parse_observation_without_category():
|
||||
@pytest.mark.asyncio
|
||||
async def test_parse_observation_edge_cases():
|
||||
"""Test observation parsing edge cases."""
|
||||
parser = EntityParser()
|
||||
parser = KnowledgeParser()
|
||||
|
||||
# Multiple word tags
|
||||
obs = await parser.parse_observation("- [tech] Database #high-priority #needs-review")
|
||||
@@ -81,7 +81,7 @@ async def test_parse_observation_edge_cases():
|
||||
@pytest.mark.asyncio
|
||||
async def test_parse_observation_errors():
|
||||
"""Test error handling in observation parsing."""
|
||||
parser = EntityParser()
|
||||
parser = KnowledgeParser()
|
||||
|
||||
# Unclosed category
|
||||
with pytest.raises(ParseError, match="unclosed category"):
|
||||
|
||||
@@ -2,13 +2,13 @@
|
||||
|
||||
import pytest
|
||||
|
||||
from basic_memory.markdown.parser import EntityParser, ParseError
|
||||
from basic_memory.markdown.knowledge_parser import KnowledgeParser, ParseError
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_parse_relation_basic():
|
||||
"""Test basic relation parsing."""
|
||||
parser = EntityParser()
|
||||
parser = KnowledgeParser()
|
||||
|
||||
rel = await parser.parse_relation("- implements [[Auth Service]]")
|
||||
assert rel is not None
|
||||
@@ -20,7 +20,7 @@ async def test_parse_relation_basic():
|
||||
@pytest.mark.asyncio
|
||||
async def test_parse_relation_with_context():
|
||||
"""Test relation parsing with context."""
|
||||
parser = EntityParser()
|
||||
parser = KnowledgeParser()
|
||||
|
||||
rel = await parser.parse_relation("- depends_on [[Database]] (Required for persistence)")
|
||||
assert rel is not None
|
||||
@@ -32,7 +32,7 @@ async def test_parse_relation_with_context():
|
||||
@pytest.mark.asyncio
|
||||
async def test_parse_relation_edge_cases():
|
||||
"""Test relation parsing edge cases."""
|
||||
parser = EntityParser()
|
||||
parser = KnowledgeParser()
|
||||
|
||||
# Multiple word type
|
||||
rel = await parser.parse_relation("- is used by [[Client App]] (Primary consumer)")
|
||||
@@ -54,7 +54,7 @@ async def test_parse_relation_edge_cases():
|
||||
@pytest.mark.asyncio
|
||||
async def test_parse_relation_errors():
|
||||
"""Test error handling in relation parsing."""
|
||||
parser = EntityParser()
|
||||
parser = KnowledgeParser()
|
||||
|
||||
# Missing target brackets
|
||||
with pytest.raises(ParseError, match="missing \\[\\["):
|
||||
|
||||
@@ -5,7 +5,7 @@ from textwrap import dedent
|
||||
|
||||
import pytest
|
||||
|
||||
from basic_memory.markdown.parser import EntityParser
|
||||
from basic_memory.markdown.knowledge_parser import KnowledgeParser
|
||||
from basic_memory.utils.file_utils import FileError, ParseError
|
||||
|
||||
|
||||
@@ -43,7 +43,7 @@ async def test_unicode_content(tmp_path):
|
||||
test_file = tmp_path / "unicode.md"
|
||||
test_file.write_text(content, encoding="utf-8")
|
||||
|
||||
parser = EntityParser()
|
||||
parser = KnowledgeParser()
|
||||
entity = await parser.parse_file(test_file)
|
||||
|
||||
assert "测试" in entity.frontmatter.tags
|
||||
@@ -61,7 +61,7 @@ async def test_fallback_encoding(tmp_path):
|
||||
test_file = tmp_path / "unicode_file.md"
|
||||
test_file.write_text(content, encoding="utf-16")
|
||||
|
||||
parser = EntityParser()
|
||||
parser = KnowledgeParser()
|
||||
with pytest.raises(ParseError):
|
||||
await parser.parse_file(test_file)
|
||||
|
||||
@@ -74,7 +74,7 @@ async def test_encoding_errors(tmp_path):
|
||||
with open(test_file, "wb") as f:
|
||||
f.write(b"\xff\xfe\x00\x00") # Invalid UTF-8
|
||||
|
||||
parser = EntityParser()
|
||||
parser = KnowledgeParser()
|
||||
with pytest.raises(ParseError):
|
||||
await parser.parse_file(test_file, encoding="ascii")
|
||||
|
||||
@@ -82,7 +82,7 @@ async def test_encoding_errors(tmp_path):
|
||||
@pytest.mark.asyncio
|
||||
async def test_file_not_found():
|
||||
"""Test handling of non-existent files."""
|
||||
parser = EntityParser()
|
||||
parser = KnowledgeParser()
|
||||
with pytest.raises(FileError):
|
||||
await parser.parse_file(Path("nonexistent.md"))
|
||||
|
||||
@@ -115,7 +115,7 @@ async def test_nested_structures(tmp_path):
|
||||
test_file = tmp_path / "nested.md"
|
||||
test_file.write_text(content)
|
||||
|
||||
parser = EntityParser()
|
||||
parser = KnowledgeParser()
|
||||
entity = await parser.parse_file(test_file)
|
||||
|
||||
assert len(entity.content.observations) == 3
|
||||
@@ -156,7 +156,7 @@ async def test_malformed_sections(tmp_path):
|
||||
test_file = tmp_path / "malformed.md"
|
||||
test_file.write_text(content)
|
||||
|
||||
parser = EntityParser()
|
||||
parser = KnowledgeParser()
|
||||
with pytest.raises(ParseError):
|
||||
entity = await parser.parse_file(test_file)
|
||||
|
||||
@@ -178,6 +178,6 @@ async def test_missing_required_sections(tmp_path):
|
||||
test_file = tmp_path / "incomplete.md"
|
||||
test_file.write_text(content)
|
||||
|
||||
parser = EntityParser()
|
||||
parser = KnowledgeParser()
|
||||
with pytest.raises(ParseError):
|
||||
await parser.parse_file(test_file)
|
||||
|
||||
@@ -2,14 +2,14 @@
|
||||
|
||||
import pytest
|
||||
|
||||
from basic_memory.markdown import EntityParser
|
||||
from basic_memory.markdown import KnowledgeParser
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_relation_empty_target():
|
||||
"""Test handling of empty targets."""
|
||||
# Empty brackets
|
||||
parser = EntityParser()
|
||||
parser = KnowledgeParser()
|
||||
|
||||
assert await parser.parse_relation("type [[]]") is None
|
||||
assert await parser.parse_relation("type [[ ]]") is None
|
||||
|
||||
Reference in New Issue
Block a user