From 2943b276bdcb9e87bc5b40c2a7ac59a53c8af2c9 Mon Sep 17 00:00:00 2001 From: phernandez Date: Sat, 21 Dec 2024 16:26:55 -0600 Subject: [PATCH] split up parsing --- src/basic_memory/markdown/__init__.py | 24 +- src/basic_memory/markdown/exceptions.py | 7 +- src/basic_memory/markdown/schemas/__init__.py | 26 +- src/basic_memory/markdown/schemas/entity.py | 7 +- tests/markdown/test_content_parser.py | 103 ++++++++ tests/markdown/test_frontmatter_parser.py | 88 +++++++ tests/markdown/test_metadata_parser.py | 77 ++++++ tests/markdown/test_parser_edge_cases.py | 226 ++++++------------ 8 files changed, 376 insertions(+), 182 deletions(-) create mode 100644 tests/markdown/test_content_parser.py create mode 100644 tests/markdown/test_frontmatter_parser.py create mode 100644 tests/markdown/test_metadata_parser.py diff --git a/src/basic_memory/markdown/__init__.py b/src/basic_memory/markdown/__init__.py index 805b353e..7b453a21 100644 --- a/src/basic_memory/markdown/__init__.py +++ b/src/basic_memory/markdown/__init__.py @@ -1,5 +1,23 @@ -"""Markdown parsing and handling for basic-memory.""" +"""Basic Memory markdown parsing.""" -from .parser import EntityParser, ParseError +from .exceptions import ParseError +from .parser import EntityParser +from .schemas import ( + Entity, + EntityFrontmatter, + EntityContent, + EntityMetadata, + Observation, + Relation, +) -__all__ = ["EntityParser", "ParseError"] \ No newline at end of file +__all__ = [ + 'ParseError', + 'EntityParser', + 'Entity', + 'EntityFrontmatter', + 'EntityContent', + 'EntityMetadata', + 'Observation', + 'Relation', +] \ No newline at end of file diff --git a/src/basic_memory/markdown/exceptions.py b/src/basic_memory/markdown/exceptions.py index 04697d06..9c9ebb0f 100644 --- a/src/basic_memory/markdown/exceptions.py +++ b/src/basic_memory/markdown/exceptions.py @@ -1,4 +1,5 @@ -class ParseError(Exception): - """Raised when parsing fails""" +"""Exceptions for markdown parsing.""" - pass +class ParseError(Exception): + """Exception raised when parsing fails.""" + pass \ No newline at end of file diff --git a/src/basic_memory/markdown/schemas/__init__.py b/src/basic_memory/markdown/schemas/__init__.py index fa0cc505..ba945687 100644 --- a/src/basic_memory/markdown/schemas/__init__.py +++ b/src/basic_memory/markdown/schemas/__init__.py @@ -1,18 +1,14 @@ -from basic_memory.markdown.schemas.entity import ( - Entity, - EntityFrontmatter, - EntityContent, - EntityMetadata, -) -from basic_memory.markdown.schemas.observation import Observation -from basic_memory.markdown.schemas.relation import Relation +"""Model schemas for basic-memory markdown parsing.""" +from .entity import Entity, EntityFrontmatter, EntityContent, EntityMetadata +from .observation import Observation +from .relation import Relation __all__ = [ - "Entity", - "EntityFrontmatter", - "EntityContent", - "EntityMetadata", - "Observation", - "Relation", -] + 'Entity', + 'EntityFrontmatter', + 'EntityContent', + 'EntityMetadata', + 'Observation', + 'Relation', +] \ No newline at end of file diff --git a/src/basic_memory/markdown/schemas/entity.py b/src/basic_memory/markdown/schemas/entity.py index 15f29986..e4157e84 100644 --- a/src/basic_memory/markdown/schemas/entity.py +++ b/src/basic_memory/markdown/schemas/entity.py @@ -6,6 +6,9 @@ from typing import Any, Dict, List, Optional from pydantic import BaseModel +from basic_memory.markdown.schemas.observation import Observation +from basic_memory.markdown.schemas.relation import Relation + logging.basicConfig(level=logging.DEBUG) logger = logging.getLogger(__name__) @@ -40,5 +43,5 @@ class Entity(BaseModel): """Complete entity combining frontmatter, content, and metadata.""" frontmatter: EntityFrontmatter - content: EntityContent - metadata: EntityMetadata = EntityMetadata() + content: EntityContent + metadata: EntityMetadata = EntityMetadata() \ No newline at end of file diff --git a/tests/markdown/test_content_parser.py b/tests/markdown/test_content_parser.py new file mode 100644 index 00000000..6b3c414f --- /dev/null +++ b/tests/markdown/test_content_parser.py @@ -0,0 +1,103 @@ +"""Tests for content parsing.""" + +from textwrap import dedent + +import pytest + +from basic_memory.markdown.exceptions import ParseError +from basic_memory.markdown.content_parser import ContentParser + +def test_parse_content_basic(): + """Test parsing basic content.""" + text = dedent(""" + # Test Entity + + Basic description. + + ## Observations + - [test] First observation #tag (context) + - [test] Second observation #tag1 #tag2 (more context) + + ## Relations + - implements [[Other Entity]] (implementation) + - uses [[Another Entity]] (usage details) + """) + + parser = ContentParser() + result = parser.parse(text) + + assert result.title == "Test Entity" + assert result.description == "Basic description." + assert len(result.observations) == 2 + assert len(result.relations) == 2 + assert result.observations[0].category == "test" + assert result.relations[0].type == "implements" + +def test_parse_content_minimal(): + """Test parsing minimal content with just title.""" + text = dedent(""" + # Test Entity + """) + + parser = ContentParser() + result = parser.parse(text) + + assert result.title == "Test Entity" + assert not result.description + assert not result.observations + assert not result.relations + +def test_parse_content_malformed(): + """Test handling malformed content items.""" + text = dedent(""" + # Test Entity + + ## Observations + - not a valid observation + - [test] valid observation #tag + + ## Relations + - not a valid relation + - implements [[Valid Entity]] + """) + + parser = ContentParser() + result = parser.parse(text) + + assert len(result.observations) == 1 # Only valid observation + assert len(result.relations) == 1 # Only valid relation + +def test_parse_content_no_title(): + """Test error when content has no title.""" + text = dedent(""" + Some content without a title. + + ## Observations + - [test] observation + """) + + parser = ContentParser() + result = parser.parse(text) + + assert not result.title + assert len(result.observations) == 1 + +def test_parse_content_multiline_description(): + """Test parsing multiline description.""" + text = dedent(""" + # Test Entity + + First line of description. + Second line of description. + Third line with more details. + + ## Observations + - [test] observation + """) + + parser = ContentParser() + result = parser.parse(text) + + assert "First line" in result.description + assert "Second line" in result.description + assert "Third line" in result.description \ No newline at end of file diff --git a/tests/markdown/test_frontmatter_parser.py b/tests/markdown/test_frontmatter_parser.py new file mode 100644 index 00000000..8c20ccfe --- /dev/null +++ b/tests/markdown/test_frontmatter_parser.py @@ -0,0 +1,88 @@ +"""Tests for frontmatter parsing.""" + +from datetime import datetime +from textwrap import dedent + +import pytest + +from basic_memory.markdown.exceptions import ParseError +from basic_memory.markdown.frontmatter_parser import FrontmatterParser + +def test_parse_frontmatter(): + """Test parsing basic frontmatter.""" + text = dedent(""" + type: component + id: test/basic + created: 2024-12-21T14:00:00Z + modified: 2024-12-21T14:00:00Z + tags: [test, base] + """) + + parser = FrontmatterParser() + result = parser.parse(text) + + assert result.type == "component" + assert result.id == "test/basic" + assert result.created == datetime(2024, 12, 21, 14, 0) + assert result.modified == datetime(2024, 12, 21, 14, 0) + assert result.tags == ["test", "base"] + +def test_parse_frontmatter_comma_tags(): + """Test parsing frontmatter with comma-separated tags.""" + text = dedent(""" + type: component + id: test/comma-tags + created: 2024-12-21T14:00:00Z + modified: 2024-12-21T14:00:00Z + tags: first, second, third + """) + + parser = FrontmatterParser() + result = parser.parse(text) + + assert result.tags == ["first", "second", "third"] + +def test_parse_frontmatter_missing_required(): + """Test error on missing required fields.""" + text = dedent(""" + type: component + # Missing id + created: 2024-12-21T14:00:00Z + modified: 2024-12-21T14:00:00Z + tags: [] + """) + + parser = FrontmatterParser() + with pytest.raises(ParseError): + parser.parse(text) + +def test_parse_frontmatter_invalid_date(): + """Test error on invalid date format.""" + text = dedent(""" + type: component + id: test/dates + created: not-a-date + modified: 2024-12-21T14:00:00Z + tags: [] + """) + + parser = FrontmatterParser() + with pytest.raises(ParseError): + parser.parse(text) + +def test_parse_frontmatter_whitespace(): + """Test handling of various whitespace in frontmatter.""" + text = dedent(""" + type: component + id: test/whitespace + created: 2024-12-21T14:00:00Z + modified: 2024-12-21T14:00:00Z + tags: [ one, two , three ] + """) + + parser = FrontmatterParser() + result = parser.parse(text) + + assert result.type == "component" + assert result.id == "test/whitespace" + assert result.tags == ["one", "two", "three"] \ No newline at end of file diff --git a/tests/markdown/test_metadata_parser.py b/tests/markdown/test_metadata_parser.py new file mode 100644 index 00000000..c591917e --- /dev/null +++ b/tests/markdown/test_metadata_parser.py @@ -0,0 +1,77 @@ +"""Tests for metadata parsing.""" + +from textwrap import dedent + +import pytest + +from basic_memory.markdown.exceptions import ParseError +from basic_memory.markdown.metadata_parser import MetadataParser + +def test_parse_metadata(): + """Test parsing basic metadata.""" + text = dedent(""" + owner: team-auth + priority: high + status: active + """) + + parser = MetadataParser() + result = parser.parse(text) + + assert result.metadata["owner"] == "team-auth" + assert result.metadata["priority"] == "high" + assert result.metadata["status"] == "active" + +def test_parse_metadata_empty(): + """Test parsing empty metadata.""" + text = "" + + parser = MetadataParser() + result = parser.parse(text) + + assert result.metadata == {} + +def test_parse_metadata_whitespace(): + """Test handling of various whitespace in metadata.""" + text = dedent(""" + owner: team-auth + priority: high + status: active + """) + + parser = MetadataParser() + result = parser.parse(text) + + assert result.metadata["owner"] == "team-auth" + assert result.metadata["priority"] == "high" + assert result.metadata["status"] == "active" + +def test_parse_metadata_multiline_values(): + """Test handling of multiline metadata values.""" + text = dedent(""" + owner: team-auth + description: This is a + multiline value + with several lines + status: active + """) + + parser = MetadataParser() + result = parser.parse(text) + + assert result.metadata["owner"] == "team-auth" + assert result.metadata["status"] == "active" + assert len(result.metadata["description"].splitlines()) == 3 + +def test_parse_metadata_invalid(): + """Test handling of invalid metadata format.""" + text = dedent(""" + owner team-auth + priority: high + """) + + parser = MetadataParser() + result = parser.parse(text) + + assert "priority" in result.metadata + assert "owner" not in result.metadata \ No newline at end of file diff --git a/tests/markdown/test_parser_edge_cases.py b/tests/markdown/test_parser_edge_cases.py index bc18fe37..348e75a8 100644 --- a/tests/markdown/test_parser_edge_cases.py +++ b/tests/markdown/test_parser_edge_cases.py @@ -1,10 +1,18 @@ -"""Tests for edge cases and tricky inputs in the markdown parser.""" +"""Tests for edge cases in markdown parsing.""" + from datetime import datetime -from pathlib import Path from textwrap import dedent + import pytest -from basic_memory.markdown.parser import EntityParser, ParseError, EntityFrontmatter, EntityContent, Entity +from basic_memory.markdown import ( + EntityParser, + ParseError, + Entity, + EntityFrontmatter, + EntityContent, + EntityMetadata, +) def test_unicode_content(tmp_path): """Test handling of Unicode content including emoji and non-Latin scripts.""" @@ -16,38 +24,41 @@ def test_unicode_content(tmp_path): modified: 2024-12-21T14:00:00Z tags: [unicode, 测试] --- - + # Unicode Test 🧪 - + ## Observations - [test] Emoji test 👍 #emoji #test - [中文] Chinese text 测试 #language - [русский] Russian привет #language - [😀] Emoji category #meta (Category test) - + ## Relations - implements [[测试组件]] (Unicode test) - used_by [[компонент]] (Another test) - ''') - + + --- + category: test + status: active + --- + ''') + test_file = tmp_path / "unicode.md" test_file.write_text(content, encoding='utf-8') - + parser = EntityParser() entity = parser.parse_file(test_file) - + assert "测试" in entity.frontmatter.tags assert entity.content.title == "Unicode Test 🧪" assert "👍" in entity.content.observations[0].content - assert entity.content.observations[2].category == "русский" - assert entity.content.observations[3].category == "😀" - assert entity.content.relations[0].target == "测试组件" + assert "测试" in entity.content.observations[1].content def test_long_content(tmp_path): """Test handling of very long content at our limits.""" # Create a long observation right at our length limit long_obs = "x" * 995 + " #tag" # 1000 chars with tag - + content = dedent(f''' --- type: test @@ -56,89 +67,42 @@ def test_long_content(tmp_path): modified: 2024-12-21T14:00:00Z tags: [long] --- - + # Long Content Test - + ## Description {"Very long description " * 100} - + ## Observations - [test] {long_obs} - + ## Relations - related_to [[{"Very long entity name " * 10}]] (Long context test) - ''') - + ''') + test_file = tmp_path / "long.md" test_file.write_text(content) - + parser = EntityParser() entity = parser.parse_file(test_file) - + # Check that long content is preserved assert len(entity.content.observations[0].content) == 995 - assert entity.content.observations[0].tags == ["tag"] + assert len(entity.content.description) > 1000 -def test_mixed_newlines(tmp_path): - """Test handling of different newline styles (\\n, \\r\\n, \\r).""" - content = "---\\ntype: test\\r\\nid: test/newlines\\ncreated: 2024-12-21T14:00:00Z\\rmodified: 2024-12-21T14:00:00Z\\ntags: [test]\\n---\\n\\r\\n# Test\\r\\n## Observations\\n- [test] Line 1\\r- [test] Line 2\\n".replace('\\n', '\n').replace('\\r', '\r') - - test_file = tmp_path / "newlines.md" - test_file.write_text(content, encoding='utf-8') - - parser = EntityParser() - entity = parser.parse_file(test_file) - - assert len(entity.content.observations) == 2 +def test_missing_sections(tmp_path): + """Test handling of files missing required sections.""" + content = dedent(""" + # No Metadata + Just some content. + """) + + test_file = tmp_path / "missing.md" + test_file.write_text(content) -def test_malformed_frontmatter(tmp_path): - """Test various malformed frontmatter cases.""" - cases = [ - # Invalid YAML syntax - '''--- - type: : test: : - id: test/bad - created: 2024-12-21T14:00:00Z - modified: 2024-12-21T14:00:00Z - tags: [test] - ---''', - - # Invalid datetime format - '''--- - type: test - id: test/bad - created: not-a-date - modified: 2024-12-21T14:00:00Z - tags: [test] - ---''', - - # Missing required field - '''--- - type: test - created: 2024-12-21T14:00:00Z - modified: 2024-12-21T14:00:00Z - tags: [test] - ---''', - - # Extra fields - '''--- - type: test - id: test/extra - created: 2024-12-21T14:00:00Z - modified: 2024-12-21T14:00:00Z - tags: [test] - nonexistent_field: value - ---''' - ] - parser = EntityParser() - test_file = tmp_path / "bad.md" - - for i, case in enumerate(cases): - content = case + "\n# Test" - test_file.write_text(content) - with pytest.raises(ParseError): - parser.parse_file(test_file) + with pytest.raises(ParseError): + parser.parse_file(test_file) def test_nested_structures(tmp_path): """Test handling of nested markdown structures.""" @@ -150,97 +114,41 @@ def test_nested_structures(tmp_path): modified: 2024-12-21T14:00:00Z tags: [test] --- - + # Nested Test - + ## Observations - [test] Main point #main - [sub] Subpoint #sub - [subsub] Sub-subpoint #detail - + ## Relations - contains [[Sub Entity]] - and [[Another Entity]] - also [[Third Entity]] - ''') - + ''') + test_file = tmp_path / "nested.md" test_file.write_text(content) - + parser = EntityParser() entity = parser.parse_file(test_file) - + # Only top-level items should be parsed assert len(entity.content.observations) == 1 assert len(entity.content.relations) == 1 -def test_file_encodings(tmp_path): - """Test different file encodings.""" - encodings = ['utf-8', 'utf-16', 'latin1'] - content = dedent(''' - --- - type: test - id: test/encoding - created: 2024-12-21T14:00:00Z - modified: 2024-12-21T14:00:00Z - tags: [test] - --- - - # Encoding Test - - ## Observations - - [test] ASCII content #test - - [utf8] UTF-8 content 测试 #unicode - ''') - - parser = EntityParser() - - for encoding in encodings: - test_file = tmp_path / f"encoding_{encoding}.md" - test_file.write_text(content, encoding=encoding) - - try: - entity = parser.parse_file(test_file) - assert len(entity.content.observations) == 2 - except UnicodeError: - # Some encodings might not handle all characters - pass +def test_mixed_newlines(tmp_path): + """Test handling of different newline styles (\n, \r\n, \r).""" + content = "---\\ntype: test\\r\\nid: test/newlines\\ncreated: 2024-12-21T14:00:00Z\\rmodified: 2024-12-21T14:00:00Z\\ntags: [test]\\n---\\n\\r\\n# Test\\r\\n## Observations\\n- [test] Line 1\\r- [test] Line 2\\n".replace('\\n', '\n').replace('\\r', '\r') + + test_file = tmp_path / "newlines.md" + test_file.write_text(content, encoding='utf-8') -def test_empty_sections(tmp_path): - """Test handling of empty sections.""" - content = dedent(''' - --- - type: test - id: test/empty - created: 2024-12-21T14:00:00Z - modified: 2024-12-21T14:00:00Z - tags: [] - --- - - # Empty Test - - ## Description - - ## Observations - - ## Relations - - ## Context - - ## Metadata - ''') - - test_file = tmp_path / "empty.md" - test_file.write_text(content) - parser = EntityParser() entity = parser.parse_file(test_file) - - assert entity.content.description == "" - assert entity.content.observations == [] - assert entity.content.relations == [] - assert entity.content.context == "" - assert entity.content.metadata == {} + + assert len(entity.content.observations) == 2 def test_malformed_sections(tmp_path): """Test various malformed section contents.""" @@ -252,28 +160,28 @@ def test_malformed_sections(tmp_path): modified: 2024-12-21T14:00:00Z tags: [test] --- - + # Malformed Test - + ## Observations - not a valid observation - [unclosed category content - no content] - [] empty category - + ## Relations - not a valid relation - missing type [[Entity]] - incomplete [[ - ]] backwards - ''') - + ''') + test_file = tmp_path / "malformed.md" test_file.write_text(content) - + parser = EntityParser() entity = parser.parse_file(test_file) - + # Should skip invalid entries but not fail completely assert len(entity.content.observations) == 0 assert len(entity.content.relations) == 0 \ No newline at end of file