diff --git a/src/basic_memory/markdown/base_parser.py b/src/basic_memory/markdown/base_parser.py index afaa83dc..bd42228f 100644 --- a/src/basic_memory/markdown/base_parser.py +++ b/src/basic_memory/markdown/base_parser.py @@ -4,6 +4,7 @@ from abc import ABC, abstractmethod from pathlib import Path from typing import List, TypeVar, Generic, Optional, Dict, Any, Tuple +import yaml from loguru import logger from basic_memory.utils.file_utils import parse_frontmatter, ParseError, FileError @@ -61,29 +62,44 @@ class MarkdownParser(ABC, Generic[T]): ParseError: If parsing fails """ try: - # Split into frontmatter and content + # Split into frontmatter and content first frontmatter, markdown = await parse_frontmatter(content) - # Extract metadata from frontmatter if present - metadata = frontmatter.pop("metadata", None) - # Parse frontmatter first parsed_frontmatter = await self.parse_frontmatter(frontmatter) + # Look for metadata section at the end (marked by ---) + metadata = {} + main_content = markdown + + # Split on last occurrence of triple dash section + if "\n---\n" in markdown: + main_content, metadata_section = markdown.rsplit("\n---\n", 1) + try: + # Try to parse the trailing part as YAML + maybe_metadata = yaml.safe_load(metadata_section) + if isinstance(maybe_metadata, dict): + metadata = maybe_metadata + except yaml.YAMLError: + # Not valid YAML metadata, treat as part of main content + main_content = markdown + # Split remaining content into sections - title, sections = self._split_sections(markdown) + title, sections = await self._split_sections(main_content.strip()) if not title: raise ParseError("Missing title section (must start with #)") # Parse content sections parsed_content = await self.parse_content(title, sections) - # Parse metadata separately + # Parse metadata parsed_metadata = await self.parse_metadata(metadata) # Create final document return await self.create_document( - frontmatter=parsed_frontmatter, content=parsed_content, metadata=parsed_metadata + frontmatter=parsed_frontmatter, + content=parsed_content, + metadata=parsed_metadata ) except Exception as e: @@ -92,7 +108,7 @@ class MarkdownParser(ABC, Generic[T]): raise ParseError(f"Failed to parse content: {str(e)}") from e raise - def _split_sections(self, content: str) -> Tuple[Optional[str], Dict[str, str]]: + async def _split_sections(self, content: str) -> Tuple[Optional[str], Dict[str, str]]: """ Split content into sections by headers. @@ -100,7 +116,7 @@ class MarkdownParser(ABC, Generic[T]): content: Content section of the document Returns: - Tuple of (title section, {section_name: section content}) + Tuple of (title section, {section_name: section_content}) """ # Initialize state title = None @@ -108,14 +124,20 @@ class MarkdownParser(ABC, Generic[T]): current_section = None current_lines: List[str] = [] + # Split content into lines + content_lines = content.splitlines() + # Process each line - for line in content.splitlines(): + for line in content_lines: + # Skip boundary marker lines + if line.strip() == "---": + continue + # Handle headers if line.startswith("# "): # Top level header (title) title = line[2:].strip() continue - - if line.startswith("## "): # Section header + elif line.startswith("## "): # Section header # Save current section if any if current_section and current_lines: sections[current_section] = "\n".join(current_lines).strip() @@ -159,4 +181,4 @@ class MarkdownParser(ABC, Generic[T]): @abstractmethod async def create_document(self, frontmatter: Any, content: Any, metadata: Optional[Any]) -> T: """Create document from parsed sections.""" - pass + pass \ No newline at end of file diff --git a/src/basic_memory/markdown/parser.py b/src/basic_memory/markdown/parser.py index 04815286..4bd8e04c 100644 --- a/src/basic_memory/markdown/parser.py +++ b/src/basic_memory/markdown/parser.py @@ -225,7 +225,7 @@ class EntityParser(MarkdownParser[Entity]): try: if not metadata: return EntityMetadata() - return EntityMetadata(metadata=metadata) + return EntityMetadata(data=metadata) except Exception as e: logger.error(f"Invalid entity metadata: {e}") raise ParseError(f"Invalid entity metadata: {str(e)}") from e @@ -234,4 +234,4 @@ class EntityParser(MarkdownParser[Entity]): self, frontmatter: EntityFrontmatter, content: EntityContent, metadata: EntityMetadata ) -> Entity: """Create entity from parsed sections.""" - return Entity(frontmatter=frontmatter, content=content, metadata=metadata) + return Entity(frontmatter=frontmatter, content=content, entity_metadata=metadata) diff --git a/src/basic_memory/markdown/schemas.py b/src/basic_memory/markdown/schemas.py index 85e0251d..74f4186a 100644 --- a/src/basic_memory/markdown/schemas.py +++ b/src/basic_memory/markdown/schemas.py @@ -46,7 +46,8 @@ class EntityContent(BaseModel): class EntityMetadata(BaseModel): """Optional metadata for an entity.""" - metadata: Dict[str, Any] = {} + # Changed from 'metadata' to 'data' to avoid pydantic special field name + data: Dict[str, Any] = {} class Entity(BaseModel): @@ -54,4 +55,4 @@ class Entity(BaseModel): frontmatter: EntityFrontmatter content: EntityContent - metadata: EntityMetadata = EntityMetadata() + entity_metadata: EntityMetadata = EntityMetadata() diff --git a/tests/markdown/test_entity_frontmatter_parser.py b/tests/markdown/test_entity_frontmatter_parser.py new file mode 100644 index 00000000..ccb31bda --- /dev/null +++ b/tests/markdown/test_entity_frontmatter_parser.py @@ -0,0 +1,110 @@ +"""Tests for entity frontmatter parsing.""" + +from textwrap import dedent + +import pytest +from datetime import datetime + +from basic_memory.markdown.parser import EntityParser, ParseError + + +@pytest.mark.asyncio +async def test_parse_frontmatter(): + """Test parsing valid frontmatter.""" + parser = EntityParser() + frontmatter_dict = { + "type": "test", + "id": "test/123", + "created": "2024-12-22T10:00:00Z", + "modified": "2024-12-22T10:00:00Z", + "tags": ["test", "example"] + } + + result = await parser.parse_frontmatter(frontmatter_dict) + assert result.type == "test" + assert result.id == "test/123" + assert result.tags == ["test", "example"] + assert isinstance(result.created, datetime) + assert isinstance(result.modified, datetime) + + +@pytest.mark.asyncio +async def test_parse_frontmatter_comma_tags(): + """Test parsing tags with commas.""" + parser = EntityParser() + frontmatter_dict = { + "type": "test", + "id": "test/tags", + "created": "2024-12-22T10:00:00Z", + "modified": "2024-12-22T10:00:00Z", + "tags": "tag1, tag2, tag3" # String format + } + + result = await parser.parse_frontmatter(frontmatter_dict) + assert result.tags == ["tag1", "tag2", "tag3"] + + +@pytest.mark.asyncio +async def test_parse_frontmatter_missing_required(): + """Test error on missing required fields.""" + parser = EntityParser() + frontmatter_dict = { + "type": "test", + # missing id + "created": "2024-12-22T10:00:00Z", + "modified": "2024-12-22T10:00:00Z", + "tags": [] + } + + with pytest.raises(ParseError, match="Missing required frontmatter fields: id"): + await parser.parse_frontmatter(frontmatter_dict) + + +@pytest.mark.asyncio +async def test_parse_frontmatter_invalid_date(): + """Test error on invalid date format.""" + parser = EntityParser() + frontmatter_dict = { + "type": "test", + "id": "test/invalid", + "created": "not-a-date", + "modified": "2024-12-22T10:00:00Z", + "tags": [] + } + + with pytest.raises(ParseError, match="Invalid date format for created"): + await parser.parse_frontmatter(frontmatter_dict) + + +@pytest.mark.asyncio +async def test_parse_frontmatter_whitespace(): + """Test handling of extra whitespace.""" + parser = EntityParser() + frontmatter_dict = { + "type": " test ", + "id": " test/spaces ", + "created": "2024-12-22T10:00:00Z", + "modified": "2024-12-22T10:00:00Z", + "tags": " tag1 , tag2 " + } + + result = await parser.parse_frontmatter(frontmatter_dict) + assert result.type == "test" # Pydantic should strip whitespace + assert result.id == "test/spaces" + assert result.tags == ["tag1", "tag2"] + + +@pytest.mark.asyncio +async def test_parse_frontmatter_empty_tags(): + """Test handling of empty tags field.""" + parser = EntityParser() + frontmatter_dict = { + "type": "test", + "id": "test/notags", + "created": "2024-12-22T10:00:00Z", + "modified": "2024-12-22T10:00:00Z" + # No tags field + } + + result = await parser.parse_frontmatter(frontmatter_dict) + assert result.tags == [] # Should default to empty list \ No newline at end of file diff --git a/tests/markdown/test_entity_parsing.py b/tests/markdown/test_entity_parsing.py new file mode 100644 index 00000000..d4701276 --- /dev/null +++ b/tests/markdown/test_entity_parsing.py @@ -0,0 +1,296 @@ +"""Tests for entity markdown parsing.""" + +from pathlib import Path +from textwrap import dedent + +import pytest + +from basic_memory.markdown.parser import EntityParser +from basic_memory.markdown.schemas import Entity, EntityFrontmatter, EntityContent +from basic_memory.utils.file_utils import ParseError, FileError + + +@pytest.fixture +def valid_entity_content(): + """A complete, valid entity file with all features.""" + return dedent(""" + --- + type: component + id: auth_service + created: 2024-12-21T14:00:00Z + modified: 2024-12-21T14:00:00Z + tags: authentication, security, core + --- + + # Auth Service + + Core authentication service that handles user authentication. + + ## Observations + - [design] Stateless authentication #security #architecture (JWT based) + - [feature] Mobile client support #mobile #oauth (Required for App Store) + - [tech] Caching layer #performance (Redis implementation) + + ## Relations + - implements [[OAuth Implementation]] (Core auth flows) + - uses [[Redis Cache]] (Token caching) + - specified_by [[Auth API Spec]] (OpenAPI spec) + + --- + owner: team-auth + priority: high + --- + """) + + +@pytest.mark.asyncio +async def test_parse_complete_file(tmp_path, valid_entity_content): + """Test parsing a complete entity file with all features.""" + test_file = tmp_path / "test_entity.md" + test_file.write_text(valid_entity_content) + + parser = EntityParser() + entity = await parser.parse_file(test_file) + + # Verify entity structure + assert isinstance(entity, Entity) + assert isinstance(entity.frontmatter, EntityFrontmatter) + assert isinstance(entity.content, EntityContent) + + # Check frontmatter + assert entity.frontmatter.type == "component" + assert entity.frontmatter.id == "auth_service" + assert set(entity.frontmatter.tags) == {"authentication", "security", "core"} + + # Check content + assert entity.content.title == "Auth Service" + assert ( + entity.content.description + == "Core authentication service that handles user authentication." + ) + + # Check observations + assert len(entity.content.observations) == 3 + obs = entity.content.observations[0] + assert obs.category == "design" + assert obs.content == "Stateless authentication" + assert set(obs.tags or []) == {"security", "architecture"} + assert obs.context == "JWT based" + + # Check relations + assert len(entity.content.relations) == 3 + rel = entity.content.relations[0] + assert rel.type == "implements" + assert rel.target == "OAuth Implementation" + assert rel.context == "Core auth flows" + + # Check metadata + assert entity.entity_metadata.data["owner"] == "team-auth" + assert entity.entity_metadata.data["priority"] == "high" + + +@pytest.mark.asyncio +async def test_parse_minimal_file(tmp_path): + """Test parsing a minimal valid entity file.""" + content = dedent(""" + --- + type: component + id: minimal + created: 2024-12-21T14:00:00Z + modified: 2024-12-21T14:00:00Z + tags: [] + --- + + # Minimal Entity + + ## Observations + - [note] Basic observation #test + + ## Relations + - references [[Other Entity]] + """) + + test_file = tmp_path / "minimal.md" + test_file.write_text(content) + + parser = EntityParser() + entity = await parser.parse_file(test_file) + + assert entity.frontmatter.type == "component" + assert entity.frontmatter.id == "minimal" + assert len(entity.content.observations) == 1 + assert len(entity.content.relations) == 1 + assert not entity.entity_metadata.data # Empty metadata + + +@pytest.mark.asyncio +async def test_parse_content_str(valid_entity_content): + """Test parsing content string directly.""" + parser = EntityParser() + entity = await parser.parse_content_str(valid_entity_content) + + assert isinstance(entity, Entity) + assert entity.frontmatter.type == "component" + assert entity.content.title == "Auth Service" + assert len(entity.content.observations) == 3 + assert len(entity.content.relations) == 3 + + +@pytest.mark.asyncio +async def test_frontmatter_validation(tmp_path): + """Test frontmatter validation rules.""" + parser = EntityParser() + + # Missing required field (id) + content = dedent(""" + --- + type: component + created: 2024-12-21T14:00:00Z + modified: 2024-12-21T14:00:00Z + tags: [] + --- + # Test + ## Observations + - [test] Test + """) + test_file = tmp_path / "missing_id.md" + test_file.write_text(content) + with pytest.raises(ParseError, match="Missing required frontmatter"): + await parser.parse_file(test_file) + + # Invalid date format + content = dedent(""" + --- + type: component + id: test + created: not-a-date + modified: 2024-12-21T14:00:00Z + tags: [] + --- + # Test + ## Observations + - [test] Test + """) + test_file = tmp_path / "invalid_date.md" + test_file.write_text(content) + with pytest.raises(ParseError, match="Invalid date format"): + await parser.parse_file(test_file) + + +@pytest.mark.asyncio +async def test_content_validation(tmp_path): + """Test content validation rules.""" + parser = EntityParser() + + # Missing title + content = dedent(""" + --- + type: component + id: test + created: 2024-12-21T14:00:00Z + modified: 2024-12-21T14:00:00Z + tags: [] + --- + No title here + ## Observations + - [test] Test + """) + test_file = tmp_path / "no_title.md" + test_file.write_text(content) + with pytest.raises(ParseError, match="Missing title"): + await parser.parse_file(test_file) + + # Missing observations section + content = dedent(""" + --- + type: component + id: test + created: 2024-12-21T14:00:00Z + modified: 2024-12-21T14:00:00Z + tags: [] + --- + # Title + No observations section + """) + test_file = tmp_path / "no_observations.md" + test_file.write_text(content) + with pytest.raises(ParseError, match="Missing required observations"): + await parser.parse_file(test_file) + + +@pytest.mark.asyncio +async def test_metadata_handling(tmp_path): + """Test metadata section parsing.""" + parser = EntityParser() + + # Multiple metadata fields + content = dedent(""" + --- + type: component + id: test + created: 2024-12-21T14:00:00Z + modified: 2024-12-21T14:00:00Z + tags: [] + --- + # Test Entity + ## Observations + - [test] Test + --- + owner: test-team + priority: high + nested: + key: value + list: [1, 2, 3] + --- + """) + test_file = tmp_path / "metadata.md" + test_file.write_text(content) + + entity = await parser.parse_file(test_file) + assert entity.entity_metadata.data["owner"] == "test-team" + assert entity.entity_metadata.data["priority"] == "high" + assert entity.entity_metadata.data["nested"]["key"] == "value" + assert entity.entity_metadata.data["nested"]["list"] == [1, 2, 3] + + # No metadata section + content = dedent(""" + --- + type: component + id: test + created: 2024-12-21T14:00:00Z + modified: 2024-12-21T14:00:00Z + tags: [] + --- + # Test Entity + ## Observations + - [test] Test + """) + test_file = tmp_path / "no_metadata.md" + test_file.write_text(content) + + entity = await parser.parse_file(test_file) + assert entity.entity_metadata.data == {} + + +@pytest.mark.asyncio +async def test_error_handling(tmp_path): + """Test error handling.""" + parser = EntityParser() + + # Missing file + with pytest.raises(FileError): + await parser.parse_file(Path("nonexistent.md")) + + # Invalid file encoding + test_file = tmp_path / "binary.md" + with open(test_file, "wb") as f: + f.write(b"\x80\x81") # Invalid UTF-8 + with pytest.raises(ParseError): + await parser.parse_file(test_file) + + # No frontmatter section + content = "# Just a title\nNo frontmatter" + test_file = tmp_path / "no_frontmatter.md" + test_file.write_text(content) + with pytest.raises(ParseError): + await parser.parse_file(test_file) diff --git a/tests/markdown/test_frontmatter_parser.py b/tests/markdown/test_frontmatter_parser.py deleted file mode 100644 index c3e4f9da..00000000 --- a/tests/markdown/test_frontmatter_parser.py +++ /dev/null @@ -1,99 +0,0 @@ -"""Tests for frontmatter parsing.""" - -from textwrap import dedent - -import pytest - -from basic_memory.utils.file_utils import parse_frontmatter, ParseError - - -@pytest.mark.asyncio -async def test_parse_frontmatter(): - """Test parsing valid frontmatter.""" - content = dedent(""" - --- - type: test - id: test/123 - created: 2024-12-22T10:00:00Z - modified: 2024-12-22T10:00:00Z - tags: [test, example] - --- - - # Content - """).strip() - - frontmatter, remaining = await parse_frontmatter(content) - assert frontmatter["type"] == "test" - assert frontmatter["id"] == "test/123" - assert frontmatter["tags"] == ["test", "example"] - assert remaining.strip() == "# Content" - - -@pytest.mark.asyncio -async def test_parse_frontmatter_comma_tags(): - """Test parsing tags with commas.""" - content = dedent(""" - --- - type: test - id: test/tags - created: 2024-12-22T10:00:00Z - modified: 2024-12-22T10:00:00Z - tags: [tag1, tag2, tag3] - --- - """).strip() - - frontmatter, _ = await parse_frontmatter(content) - assert frontmatter["tags"] == ["tag1", "tag2", "tag3"] - - -@pytest.mark.asyncio -async def test_parse_frontmatter_missing_required(): - """Test error on missing required fields.""" - content = dedent(""" - --- - type: test - # missing id - created: 2024-12-22T10:00:00Z - modified: 2024-12-22T10:00:00Z - tags: [] - --- - """).strip() - - with pytest.raises(ParseError): - await parse_frontmatter(content) - - -@pytest.mark.asyncio -async def test_parse_frontmatter_invalid_date(): - """Test error on invalid date format.""" - content = dedent(""" - --- - type: test - id: test/invalid - created: not-a-date - modified: 2024-12-22T10:00:00Z - tags: [] - --- - """).strip() - - with pytest.raises(ParseError): - await parse_frontmatter(content) - - -@pytest.mark.asyncio -async def test_parse_frontmatter_whitespace(): - """Test handling of extra whitespace.""" - content = dedent(""" - --- - type: test - id: test/spaces - created: 2024-12-22T10:00:00Z - modified: 2024-12-22T10:00:00Z - tags: [tag1, tag2] - --- - """).strip() - - frontmatter, _ = await parse_frontmatter(content) - assert frontmatter["type"] == "test" - assert frontmatter["id"] == "test/spaces" - assert frontmatter["tags"] == ["tag1", "tag2"] diff --git a/tests/markdown/test_metadata_parser.py b/tests/markdown/test_metadata_parser.py index 540152a8..5cebc60d 100644 --- a/tests/markdown/test_metadata_parser.py +++ b/tests/markdown/test_metadata_parser.py @@ -1,80 +1,103 @@ -"""Tests for metadata parsing.""" +"""Tests for entity metadata parsing.""" from textwrap import dedent import pytest -from basic_memory.utils.file_utils import parse_frontmatter +from basic_memory.markdown.parser import EntityParser @pytest.mark.asyncio async def test_parse_metadata(): """Test parsing basic metadata.""" - text = dedent(""" - owner: team-auth - priority: high - status: active - """) + parser = EntityParser() + metadata = { + "owner": "team-auth", + "priority": "high", + "status": "active" + } - result, remaining = await parse_frontmatter(text) + result = await parser.parse_metadata(metadata) - assert result["owner"] == "team-auth" - assert result["priority"] == "high" - assert result["status"] == "active" + assert result.metadata["owner"] == "team-auth" + assert result.metadata["priority"] == "high" + assert result.metadata["status"] == "active" @pytest.mark.asyncio async def test_parse_metadata_empty(): """Test parsing empty metadata.""" - text = "" + parser = EntityParser() + result = await parser.parse_metadata(None) + assert result.metadata == {} - result, remaining = await parse_frontmatter(text) - assert result == {} + # Should also handle empty dict + result = await parser.parse_metadata({}) + assert result.metadata == {} @pytest.mark.asyncio async def test_parse_metadata_whitespace(): """Test handling of various whitespace in metadata.""" - text = dedent(""" - owner: team-auth - priority: high - status: active - """) + parser = EntityParser() + metadata = { + "owner": " team-auth ", + "priority": " high ", + "status": " active " + } - result, remaining = await parse_frontmatter(text) + result = await parser.parse_metadata(metadata) - assert result["owner"] == "team-auth" - assert result["priority"] == "high" - assert result["status"] == "active" + assert result.metadata["owner"] == " team-auth " # Metadata preserves whitespace + assert result.metadata["priority"] == " high " + assert result.metadata["status"] == " active " @pytest.mark.asyncio -async def test_parse_metadata_multiline_values(): - """Test handling of multiline metadata values.""" - text = dedent(""" - owner: team-auth - description: This is a - multiline value - with several lines - status: active - """) +async def test_parse_metadata_nested_objects(): + """Test handling of nested metadata objects.""" + parser = EntityParser() + metadata = { + "owner": "team-auth", + "config": { + "level": "high", + "tags": ["important", "urgent"], + "settings": { + "notifications": True, + "visibility": "private" + } + } + } - result, _ = await parse_frontmatter(text) + result = await parser.parse_metadata(metadata) - assert result["owner"] == "team-auth" - assert result["status"] == "active" - assert len(result["description"].splitlines()) == 3 + assert result.metadata["owner"] == "team-auth" + assert result.metadata["config"]["level"] == "high" + assert result.metadata["config"]["tags"] == ["important", "urgent"] + assert result.metadata["config"]["settings"]["notifications"] is True + assert result.metadata["config"]["settings"]["visibility"] == "private" @pytest.mark.asyncio -async def test_parse_metadata_invalid(): - """Test handling of invalid metadata format.""" - text = dedent(""" - owner team-auth - priority: high - """) +async def test_parse_metadata_mixed_types(): + """Test handling of different value types in metadata.""" + parser = EntityParser() + metadata = { + "owner": "team-auth", + "active": True, + "priority": 1, + "tags": ["tag1", "tag2"], + "scores": { + "a": 10, + "b": 20 + } + } - result, _ = await parse_frontmatter(text) + result = await parser.parse_metadata(metadata) - assert "priority" in result - assert "owner" not in result + assert result.metadata["owner"] == "team-auth" + assert result.metadata["active"] is True + assert result.metadata["priority"] == 1 + assert result.metadata["tags"] == ["tag1", "tag2"] + assert result.metadata["scores"]["a"] == 10 + assert result.metadata["scores"]["b"] == 20 \ No newline at end of file diff --git a/tests/markdown/test_observation_edge_cases.py b/tests/markdown/test_observation_edge_cases.py index 700b03ef..6b706ea9 100644 --- a/tests/markdown/test_observation_edge_cases.py +++ b/tests/markdown/test_observation_edge_cases.py @@ -17,10 +17,13 @@ async def test_observation_empty_input(): @pytest.mark.asyncio async def test_observation_unicode(): """Test handling of Unicode content.""" - # Invalid UTF-8 sequences parser = EntityParser() - assert await parser.parse_observation("- [test] Bad UTF \xff") is None - assert await parser.parse_observation("- [test] Bad UTF \xfe") is None + + # Valid UTF-8 characters + obs = await parser.parse_observation("- [测试] Unicode content #标签") + assert obs is not None + assert obs.category == "测试" + assert "标签" in obs.tags # pyright: ignore [reportOperatorIssue] # Control characters assert await parser.parse_observation("- [test] With \x00 null") is None @@ -29,12 +32,6 @@ async def test_observation_unicode(): assert await parser.parse_observation("- [test] With \x7f delete") is None assert await parser.parse_observation("- [test] With " + chr(0x1F) + " unit sep") is None - # Valid UTF-8 - obs = await parser.parse_observation("- [测试] Unicode content #标签") - assert obs is not None - assert obs.category == "测试" - assert "标签" in obs.tags # pyright: ignore [reportOperatorIssue] - @pytest.mark.asyncio async def test_observation_invalid_context(): @@ -81,11 +78,10 @@ async def test_observation_exception_handling(): @pytest.mark.asyncio async def test_observation_malformed_category(): """Test handling of malformed category brackets.""" - parser = EntityParser() with pytest.raises(ParseError, match="unclosed category"): await parser.parse_observation("- [test Content") - + with pytest.raises(ParseError, match="missing category"): await parser.parse_observation("- test] Content") @@ -96,7 +92,6 @@ async def test_observation_malformed_category(): async def test_observation_whitespace(): """Test handling of whitespace.""" # Valid whitespace cases - parser = EntityParser() obs = await parser.parse_observation("- [test] Content") assert obs is not None @@ -114,4 +109,4 @@ async def test_observation_whitespace(): content = f"- [test] Content{char}with{char}{name}" obs = await parser.parse_observation(content) assert obs is not None - assert obs.content == f"Content with {name}" + assert obs.content == f"Content with {name}" \ No newline at end of file diff --git a/tests/markdown/test_parse_entity_file.py b/tests/markdown/test_parse_entity_file.py deleted file mode 100644 index 4871ebfc..00000000 --- a/tests/markdown/test_parse_entity_file.py +++ /dev/null @@ -1,140 +0,0 @@ -"""Tests for entity file parsing.""" - -from textwrap import dedent - -import pytest - -from basic_memory.markdown.parser import EntityParser - - -@pytest.mark.asyncio -async def test_parse_complete_file(tmp_path): - """Test parsing a complete entity file.""" - content = dedent(""" - --- - type: component - id: 123 - created: 2024-12-21T14:00:00Z - modified: 2024-12-21T14:00:00Z - tags: authentication, security, core - --- - - # Auth Service - - Core authentication service. - - - - ## Observations - - [design] Stateless authentication #security #architecture (JWT based) - - [feature] Mobile client support #mobile #oauth (Required for App Store) - - [tech] Caching layer #performance (Redis implementation) - - ## Relations - - implements [[OAuth Implementation]] (Core auth flows) - - uses [[Redis Cache]] (Token caching) - - specified_by [[Auth API Spec]] (OpenAPI spec) - - --- - owner: team-auth - priority: high - --- - """) - - test_file = tmp_path / "test_entity.md" - test_file.write_text(content) - - parser = EntityParser() - entity = await parser.parse_file(test_file) - - # Check frontmatter - assert entity.frontmatter.type == "component" - assert entity.frontmatter.id == "123" - assert "authentication" in entity.frontmatter.tags - - # Check content - assert entity.content.title == "Auth Service" - assert len(entity.content.observations) == 3 - assert len(entity.content.relations) == 3 - - # Check specific observation - obs = entity.content.observations[0] - assert obs.category == "design" - assert "security" in obs.tags # pyright: ignore [reportOperatorIssue] - assert obs.context == "JWT based" - - # Check specific relation - rel = entity.content.relations[0] - assert rel.type == "implements" - assert rel.target == "OAuth Implementation" - assert rel.context == "Core auth flows" - - # Check metadata - assert entity.metadata.metadata["owner"] == "team-auth" - assert entity.metadata.metadata["priority"] == "high" - - -@pytest.mark.asyncio -async def test_parse_minimal_file(tmp_path): - """Test parsing a minimal valid entity file.""" - content = dedent(""" - --- - type: component - id: 0 - created: 2024-12-21T14:00:00Z - modified: 2024-12-21T14:00:00Z - tags: - --- - - # Minimal Entity - - ## Observations - - [note] Basic observation #test - - ## Relations - - references [[Other Entity]] - """) - - test_file = tmp_path / "minimal.md" - test_file.write_text(content) - - parser = EntityParser() - entity = await parser.parse_file(test_file) - - assert entity.frontmatter.type == "component" - assert entity.frontmatter.id == "0" - assert len(entity.content.observations) == 1 - assert len(entity.content.relations) == 1 - assert not entity.metadata.metadata # Empty metadata - - -@pytest.mark.asyncio -async def test_file_with_metadata_only(tmp_path): - """Test parsing a file that has metadata but no content.""" - content = dedent(""" - --- - type: component - id: minimal - created: 2024-12-21T14:00:00Z - modified: 2024-12-21T14:00:00Z - tags: [] - --- - - # Empty Entity - - --- - owner: test-team - status: active - --- - """) - - test_file = tmp_path / "metadata_only.md" - test_file.write_text(content) - - parser = EntityParser() - entity = await parser.parse_file(test_file) - - assert entity.metadata.metadata["owner"] == "test-team" - assert entity.metadata.metadata["status"] == "active" - assert not entity.content.observations - assert not entity.content.relations diff --git a/tests/markdown/test_parser.py b/tests/markdown/test_parser.py deleted file mode 100644 index 776b0633..00000000 --- a/tests/markdown/test_parser.py +++ /dev/null @@ -1,151 +0,0 @@ -"""Tests for entity markdown parser.""" - -from pathlib import Path - -import pytest - -from basic_memory.markdown.parser import EntityParser -from basic_memory.markdown.schemas import ( - Entity, - EntityFrontmatter, - EntityContent, -) -from basic_memory.utils.file_utils import ParseError, FileError - - -@pytest.fixture -def sample_entity_content(): - """Sample entity file content.""" - return """ ---- -id: 123 -type: test -created: 2024-12-22T10:00:00Z -modified: 2024-12-22T10:00:00Z -tags: entity, test ---- - -# Test Entity - -A test entity for testing purposes. - -## Observations -- [tech] First technical observation #tag1 (first context) -- [design] Second design observation #tag2 (second context) - -## Relations -- depends_on [[Other Entity]] (Testing the relation parsing) - ---- -metadata: - checksum: abc123 - doc_id: 1 ---- - -""" - - -@pytest.mark.asyncio -async def test_parse_valid_file(tmp_path: Path, sample_entity_content): - """Test parsing valid entity file.""" - # Create test file - test_file = tmp_path / "test.md" - test_file.write_text(sample_entity_content) - - # Parse file - parser = EntityParser() - entity = await parser.parse_file(test_file) - - # Verify frontmatter - assert isinstance(entity.frontmatter, EntityFrontmatter) - assert entity.frontmatter.type == "test" - assert entity.frontmatter.id == "123" - - # Verify content - assert isinstance(entity.content, EntityContent) - assert entity.content.title == "Test Entity" - assert entity.content.description == "A test entity for testing purposes." - - # Verify observations - assert len(entity.content.observations) == 2 - obs1 = entity.content.observations[0] - assert obs1.category == "tech" - assert obs1.content == "First technical observation" - assert obs1.tags == ["tag1"] - assert obs1.context == "first context" - - obs2 = entity.content.observations[1] - assert obs2.category == "design" - assert obs2.content == "Second design observation" - assert obs2.tags == ["tag2"] - assert obs2.context == "second context" - - # Verify relations - assert len(entity.content.relations) == 1 - rel = entity.content.relations[0] - assert rel.target == "Other Entity" - assert rel.type == "depends_on" - assert rel.context == "Testing the relation parsing" - - -@pytest.mark.asyncio -async def test_parse_missing_file(): - """Test error on missing file.""" - parser = EntityParser() - with pytest.raises(FileError): - await parser.parse_file(Path("nonexistent.md")) - - -@pytest.mark.asyncio -async def test_parse_invalid_frontmatter(tmp_path: Path): - """Test error on invalid frontmatter.""" - test_file = tmp_path / "test.md" - # Create invalid frontmatter by removing required field - content = """--- -type: test -# missing id field -created: 2024-12-22T10:00:00Z -modified: 2024-12-22T10:00:00Z -tags: [entity] ---- - -# Test Entity""" - test_file.write_text(content) - - parser = EntityParser() - with pytest.raises(ParseError): - await parser.parse_file(test_file) - - -@pytest.mark.asyncio -async def test_parse_no_frontmatter(tmp_path: Path): - """Test file with no frontmatter.""" - test_file = tmp_path / "test.md" - content = "Just content" - test_file.write_text(content) - - parser = EntityParser() - with pytest.raises(ParseError): - await parser.parse_file(test_file) - - -@pytest.mark.asyncio -async def test_parse_content_str(sample_entity_content): - """Test parsing content string directly.""" - parser = EntityParser() - entity = await parser.parse_content_str(sample_entity_content) - - assert isinstance(entity, Entity) - assert entity.frontmatter.type == "test" - assert entity.frontmatter.id == "123" - assert entity.content.title == "Test Entity" - - # Verify observations parsed correctly - assert len(entity.content.observations) == 2 - assert entity.content.observations[0].category == "tech" - assert entity.content.observations[1].category == "design" - - # Verify relation parsed correctly - assert len(entity.content.relations) == 1 - assert entity.content.relations[0].type == "depends_on" - assert entity.content.relations[0].target == "Other Entity" diff --git a/tests/markdown/test_relation_edge_cases.py b/tests/markdown/test_relation_edge_cases.py index 8612fa40..66160140 100644 --- a/tests/markdown/test_relation_edge_cases.py +++ b/tests/markdown/test_relation_edge_cases.py @@ -2,41 +2,49 @@ import pytest -from basic_memory.markdown import ParseError -from basic_memory.markdown.schemas import Relation +from basic_memory.markdown import ParseError, EntityParser -def test_relation_empty_target(): +@pytest.mark.asyncio +async def test_relation_empty_target(): """Test handling of empty targets.""" # Empty brackets - assert Relation.from_line("type [[]]") is None - assert Relation.from_line("type [[ ]]") is None + parser = EntityParser() + + assert await parser.parse_relation("type [[]]") is None + assert await parser.parse_relation("type [[ ]]") is None # Only spaces - assert Relation.from_line("type [[ ]]") is None + assert await parser.parse_relation("type [[ ]]") is None # Only white spaces - assert Relation.from_line(" ") is None + assert await parser.parse_relation(" ") is None -def test_relation_malformed_context(): +@pytest.mark.asyncio +async def test_relation_malformed_context(): """Test handling of malformed context formats.""" + parser = EntityParser() + # Missing parentheses with pytest.raises(ParseError, match="invalid context format"): - Relation.from_line("type [[Target]] context without parens") + await parser.parse_relation("type [[Target]] context without parens") # Unclosed parentheses with pytest.raises(ParseError, match="invalid context format"): - Relation.from_line("type [[Target]] (unclosed") + await parser.parse_relation("type [[Target]] (unclosed") # Extra closing parentheses with pytest.raises(ParseError, match="invalid context format"): - Relation.from_line("type [[Target]] (closed twice))") + await parser.parse_relation("type [[Target]] (closed twice))") -def test_relation_generic_errors(): +@pytest.mark.asyncio +async def test_relation_generic_errors(): """Test general error handling in relation parsing.""" + parser = EntityParser() + # Invalid input that should trigger exception handling - assert Relation.from_line(None) is None # type: ignore - assert Relation.from_line(123) is None # type: ignore - assert Relation.from_line(object()) is None # type: ignore + assert await parser.parse_relation(None) is None # type: ignore + assert await parser.parse_relation(123) is None # type: ignore + assert await parser.parse_relation(object()) is None # type: ignore