entity markdown parsing

src/basic_memory/markdown/base_parser.py

@@ -4,6 +4,7 @@ from abc import ABC, abstractmethod
 from pathlib import Path
 from typing import List, TypeVar, Generic, Optional, Dict, Any, Tuple
 
+import yaml
 from loguru import logger
 
 from basic_memory.utils.file_utils import parse_frontmatter, ParseError, FileError
@@ -61,29 +62,44 @@ class MarkdownParser(ABC, Generic[T]):
             ParseError: If parsing fails
         """
         try:
-            # Split into frontmatter and content
+            # Split into frontmatter and content first
             frontmatter, markdown = await parse_frontmatter(content)
 
-            # Extract metadata from frontmatter if present
-            metadata = frontmatter.pop("metadata", None)
-
             # Parse frontmatter first
             parsed_frontmatter = await self.parse_frontmatter(frontmatter)
 
+            # Look for metadata section at the end (marked by ---)
+            metadata = {}
+            main_content = markdown
+
+            # Split on last occurrence of triple dash section
+            if "\n---\n" in markdown:
+                main_content, metadata_section = markdown.rsplit("\n---\n", 1)
+                try:
+                    # Try to parse the trailing part as YAML
+                    maybe_metadata = yaml.safe_load(metadata_section)
+                    if isinstance(maybe_metadata, dict):
+                        metadata = maybe_metadata
+                except yaml.YAMLError:
+                    # Not valid YAML metadata, treat as part of main content
+                    main_content = markdown
+
             # Split remaining content into sections
-            title, sections = self._split_sections(markdown)
+            title, sections = await self._split_sections(main_content.strip())
             if not title:
                 raise ParseError("Missing title section (must start with #)")
 
             # Parse content sections
             parsed_content = await self.parse_content(title, sections)
 
-            # Parse metadata separately
+            # Parse metadata
             parsed_metadata = await self.parse_metadata(metadata)
 
             # Create final document
             return await self.create_document(
-                frontmatter=parsed_frontmatter, content=parsed_content, metadata=parsed_metadata
+                frontmatter=parsed_frontmatter,
+                content=parsed_content,
+                metadata=parsed_metadata
             )
 
         except Exception as e:
@@ -92,7 +108,7 @@ class MarkdownParser(ABC, Generic[T]):
                 raise ParseError(f"Failed to parse content: {str(e)}") from e
             raise
 
-    def _split_sections(self, content: str) -> Tuple[Optional[str], Dict[str, str]]:
+    async def _split_sections(self, content: str) -> Tuple[Optional[str], Dict[str, str]]:
         """
         Split content into sections by headers.
 
@@ -100,7 +116,7 @@ class MarkdownParser(ABC, Generic[T]):
             content: Content section of the document
 
         Returns:
-            Tuple of (title section, {section_name: section content})
+            Tuple of (title section, {section_name: section_content})
         """
         # Initialize state
         title = None
@@ -108,14 +124,20 @@ class MarkdownParser(ABC, Generic[T]):
         current_section = None
         current_lines: List[str] = []
 
+        # Split content into lines
+        content_lines = content.splitlines()
+
         # Process each line
-        for line in content.splitlines():
+        for line in content_lines:
+            # Skip boundary marker lines
+            if line.strip() == "---":
+                continue
+
             # Handle headers
             if line.startswith("# "):  # Top level header (title)
                 title = line[2:].strip()
                 continue
-
-            if line.startswith("## "):  # Section header
+            elif line.startswith("## "):  # Section header
                 # Save current section if any
                 if current_section and current_lines:
                     sections[current_section] = "\n".join(current_lines).strip()
@@ -159,4 +181,4 @@ class MarkdownParser(ABC, Generic[T]):
     @abstractmethod
     async def create_document(self, frontmatter: Any, content: Any, metadata: Optional[Any]) -> T:
         """Create document from parsed sections."""
-        pass
+        pass

src/basic_memory/markdown/parser.py

@@ -225,7 +225,7 @@ class EntityParser(MarkdownParser[Entity]):
         try:
             if not metadata:
                 return EntityMetadata()
-            return EntityMetadata(metadata=metadata)
+            return EntityMetadata(data=metadata)
         except Exception as e:
             logger.error(f"Invalid entity metadata: {e}")
             raise ParseError(f"Invalid entity metadata: {str(e)}") from e
@@ -234,4 +234,4 @@ class EntityParser(MarkdownParser[Entity]):
         self, frontmatter: EntityFrontmatter, content: EntityContent, metadata: EntityMetadata
     ) -> Entity:
         """Create entity from parsed sections."""
-        return Entity(frontmatter=frontmatter, content=content, metadata=metadata)
+        return Entity(frontmatter=frontmatter, content=content, entity_metadata=metadata)

src/basic_memory/markdown/schemas.py

@@ -46,7 +46,8 @@ class EntityContent(BaseModel):
 class EntityMetadata(BaseModel):
     """Optional metadata for an entity."""
 
-    metadata: Dict[str, Any] = {}
+    # Changed from 'metadata' to 'data' to avoid pydantic special field name
+    data: Dict[str, Any] = {}
 
 
 class Entity(BaseModel):
@@ -54,4 +55,4 @@ class Entity(BaseModel):
 
     frontmatter: EntityFrontmatter
     content: EntityContent
-    metadata: EntityMetadata = EntityMetadata()
+    entity_metadata: EntityMetadata = EntityMetadata()

tests/markdown/test_entity_frontmatter_parser.py

@@ -0,0 +1,110 @@
+"""Tests for entity frontmatter parsing."""
+
+from textwrap import dedent
+
+import pytest
+from datetime import datetime
+
+from basic_memory.markdown.parser import EntityParser, ParseError
+
+
+@pytest.mark.asyncio
+async def test_parse_frontmatter():
+    """Test parsing valid frontmatter."""
+    parser = EntityParser()
+    frontmatter_dict = {
+        "type": "test",
+        "id": "test/123",
+        "created": "2024-12-22T10:00:00Z",
+        "modified": "2024-12-22T10:00:00Z",
+        "tags": ["test", "example"]
+    }
+
+    result = await parser.parse_frontmatter(frontmatter_dict)
+    assert result.type == "test"
+    assert result.id == "test/123"
+    assert result.tags == ["test", "example"]
+    assert isinstance(result.created, datetime)
+    assert isinstance(result.modified, datetime)
+
+
+@pytest.mark.asyncio
+async def test_parse_frontmatter_comma_tags():
+    """Test parsing tags with commas."""
+    parser = EntityParser()
+    frontmatter_dict = {
+        "type": "test",
+        "id": "test/tags",
+        "created": "2024-12-22T10:00:00Z",
+        "modified": "2024-12-22T10:00:00Z",
+        "tags": "tag1, tag2, tag3"  # String format
+    }
+
+    result = await parser.parse_frontmatter(frontmatter_dict)
+    assert result.tags == ["tag1", "tag2", "tag3"]
+
+
+@pytest.mark.asyncio
+async def test_parse_frontmatter_missing_required():
+    """Test error on missing required fields."""
+    parser = EntityParser()
+    frontmatter_dict = {
+        "type": "test",
+        # missing id
+        "created": "2024-12-22T10:00:00Z",
+        "modified": "2024-12-22T10:00:00Z",
+        "tags": []
+    }
+
+    with pytest.raises(ParseError, match="Missing required frontmatter fields: id"):
+        await parser.parse_frontmatter(frontmatter_dict)
+
+
+@pytest.mark.asyncio
+async def test_parse_frontmatter_invalid_date():
+    """Test error on invalid date format."""
+    parser = EntityParser()
+    frontmatter_dict = {
+        "type": "test",
+        "id": "test/invalid",
+        "created": "not-a-date",
+        "modified": "2024-12-22T10:00:00Z",
+        "tags": []
+    }
+
+    with pytest.raises(ParseError, match="Invalid date format for created"):
+        await parser.parse_frontmatter(frontmatter_dict)
+
+
+@pytest.mark.asyncio
+async def test_parse_frontmatter_whitespace():
+    """Test handling of extra whitespace."""
+    parser = EntityParser()
+    frontmatter_dict = {
+        "type": "   test   ",
+        "id": "   test/spaces   ",
+        "created": "2024-12-22T10:00:00Z",
+        "modified": "2024-12-22T10:00:00Z",
+        "tags": "   tag1   ,    tag2   "
+    }
+
+    result = await parser.parse_frontmatter(frontmatter_dict)
+    assert result.type == "test"  # Pydantic should strip whitespace
+    assert result.id == "test/spaces"
+    assert result.tags == ["tag1", "tag2"]
+
+
+@pytest.mark.asyncio
+async def test_parse_frontmatter_empty_tags():
+    """Test handling of empty tags field."""
+    parser = EntityParser()
+    frontmatter_dict = {
+        "type": "test",
+        "id": "test/notags",
+        "created": "2024-12-22T10:00:00Z",
+        "modified": "2024-12-22T10:00:00Z"
+        # No tags field
+    }
+
+    result = await parser.parse_frontmatter(frontmatter_dict)
+    assert result.tags == []  # Should default to empty list

tests/markdown/test_entity_parsing.py

@@ -0,0 +1,296 @@
+"""Tests for entity markdown parsing."""
+
+from pathlib import Path
+from textwrap import dedent
+
+import pytest
+
+from basic_memory.markdown.parser import EntityParser
+from basic_memory.markdown.schemas import Entity, EntityFrontmatter, EntityContent
+from basic_memory.utils.file_utils import ParseError, FileError
+
+
+@pytest.fixture
+def valid_entity_content():
+    """A complete, valid entity file with all features."""
+    return dedent("""
+        ---
+        type: component
+        id: auth_service
+        created: 2024-12-21T14:00:00Z
+        modified: 2024-12-21T14:00:00Z
+        tags: authentication, security, core
+        ---
+
+        # Auth Service
+
+        Core authentication service that handles user authentication.
+
+        ## Observations
+        - [design] Stateless authentication #security #architecture (JWT based)
+        - [feature] Mobile client support #mobile #oauth (Required for App Store)
+        - [tech] Caching layer #performance (Redis implementation)
+
+        ## Relations
+        - implements [[OAuth Implementation]] (Core auth flows)
+        - uses [[Redis Cache]] (Token caching)
+        - specified_by [[Auth API Spec]] (OpenAPI spec)
+
+        ---
+        owner: team-auth
+        priority: high
+        ---
+        """)
+
+
+@pytest.mark.asyncio
+async def test_parse_complete_file(tmp_path, valid_entity_content):
+    """Test parsing a complete entity file with all features."""
+    test_file = tmp_path / "test_entity.md"
+    test_file.write_text(valid_entity_content)
+
+    parser = EntityParser()
+    entity = await parser.parse_file(test_file)
+
+    # Verify entity structure
+    assert isinstance(entity, Entity)
+    assert isinstance(entity.frontmatter, EntityFrontmatter)
+    assert isinstance(entity.content, EntityContent)
+
+    # Check frontmatter
+    assert entity.frontmatter.type == "component"
+    assert entity.frontmatter.id == "auth_service"
+    assert set(entity.frontmatter.tags) == {"authentication", "security", "core"}
+
+    # Check content
+    assert entity.content.title == "Auth Service"
+    assert (
+        entity.content.description
+        == "Core authentication service that handles user authentication."
+    )
+
+    # Check observations
+    assert len(entity.content.observations) == 3
+    obs = entity.content.observations[0]
+    assert obs.category == "design"
+    assert obs.content == "Stateless authentication"
+    assert set(obs.tags or []) == {"security", "architecture"}
+    assert obs.context == "JWT based"
+
+    # Check relations
+    assert len(entity.content.relations) == 3
+    rel = entity.content.relations[0]
+    assert rel.type == "implements"
+    assert rel.target == "OAuth Implementation"
+    assert rel.context == "Core auth flows"
+
+    # Check metadata
+    assert entity.entity_metadata.data["owner"] == "team-auth"
+    assert entity.entity_metadata.data["priority"] == "high"
+
+
+@pytest.mark.asyncio
+async def test_parse_minimal_file(tmp_path):
+    """Test parsing a minimal valid entity file."""
+    content = dedent("""
+        ---
+        type: component
+        id: minimal
+        created: 2024-12-21T14:00:00Z
+        modified: 2024-12-21T14:00:00Z
+        tags: []
+        ---
+
+        # Minimal Entity
+
+        ## Observations
+        - [note] Basic observation #test
+
+        ## Relations
+        - references [[Other Entity]]
+        """)
+
+    test_file = tmp_path / "minimal.md"
+    test_file.write_text(content)
+
+    parser = EntityParser()
+    entity = await parser.parse_file(test_file)
+
+    assert entity.frontmatter.type == "component"
+    assert entity.frontmatter.id == "minimal"
+    assert len(entity.content.observations) == 1
+    assert len(entity.content.relations) == 1
+    assert not entity.entity_metadata.data  # Empty metadata
+
+
+@pytest.mark.asyncio
+async def test_parse_content_str(valid_entity_content):
+    """Test parsing content string directly."""
+    parser = EntityParser()
+    entity = await parser.parse_content_str(valid_entity_content)
+
+    assert isinstance(entity, Entity)
+    assert entity.frontmatter.type == "component"
+    assert entity.content.title == "Auth Service"
+    assert len(entity.content.observations) == 3
+    assert len(entity.content.relations) == 3
+
+
+@pytest.mark.asyncio
+async def test_frontmatter_validation(tmp_path):
+    """Test frontmatter validation rules."""
+    parser = EntityParser()
+
+    # Missing required field (id)
+    content = dedent("""
+        ---
+        type: component
+        created: 2024-12-21T14:00:00Z
+        modified: 2024-12-21T14:00:00Z
+        tags: []
+        ---
+        # Test
+        ## Observations
+        - [test] Test
+        """)
+    test_file = tmp_path / "missing_id.md"
+    test_file.write_text(content)
+    with pytest.raises(ParseError, match="Missing required frontmatter"):
+        await parser.parse_file(test_file)
+
+    # Invalid date format
+    content = dedent("""
+        ---
+        type: component
+        id: test
+        created: not-a-date
+        modified: 2024-12-21T14:00:00Z
+        tags: []
+        ---
+        # Test
+        ## Observations
+        - [test] Test
+        """)
+    test_file = tmp_path / "invalid_date.md"
+    test_file.write_text(content)
+    with pytest.raises(ParseError, match="Invalid date format"):
+        await parser.parse_file(test_file)
+
+
+@pytest.mark.asyncio
+async def test_content_validation(tmp_path):
+    """Test content validation rules."""
+    parser = EntityParser()
+
+    # Missing title
+    content = dedent("""
+        ---
+        type: component
+        id: test
+        created: 2024-12-21T14:00:00Z
+        modified: 2024-12-21T14:00:00Z
+        tags: []
+        ---
+        No title here
+        ## Observations
+        - [test] Test
+        """)
+    test_file = tmp_path / "no_title.md"
+    test_file.write_text(content)
+    with pytest.raises(ParseError, match="Missing title"):
+        await parser.parse_file(test_file)
+
+    # Missing observations section
+    content = dedent("""
+        ---
+        type: component
+        id: test
+        created: 2024-12-21T14:00:00Z
+        modified: 2024-12-21T14:00:00Z
+        tags: []
+        ---
+        # Title
+        No observations section
+        """)
+    test_file = tmp_path / "no_observations.md"
+    test_file.write_text(content)
+    with pytest.raises(ParseError, match="Missing required observations"):
+        await parser.parse_file(test_file)
+
+
+@pytest.mark.asyncio
+async def test_metadata_handling(tmp_path):
+    """Test metadata section parsing."""
+    parser = EntityParser()
+
+    # Multiple metadata fields
+    content = dedent("""
+        ---
+        type: component
+        id: test
+        created: 2024-12-21T14:00:00Z
+        modified: 2024-12-21T14:00:00Z
+        tags: []
+        ---
+        # Test Entity
+        ## Observations
+        - [test] Test
+        ---
+        owner: test-team
+        priority: high
+        nested:
+          key: value
+          list: [1, 2, 3]
+        ---
+        """)
+    test_file = tmp_path / "metadata.md"
+    test_file.write_text(content)
+
+    entity = await parser.parse_file(test_file)
+    assert entity.entity_metadata.data["owner"] == "test-team"
+    assert entity.entity_metadata.data["priority"] == "high"
+    assert entity.entity_metadata.data["nested"]["key"] == "value"
+    assert entity.entity_metadata.data["nested"]["list"] == [1, 2, 3]
+
+    # No metadata section
+    content = dedent("""
+        ---
+        type: component
+        id: test
+        created: 2024-12-21T14:00:00Z
+        modified: 2024-12-21T14:00:00Z
+        tags: []
+        ---
+        # Test Entity
+        ## Observations
+        - [test] Test
+        """)
+    test_file = tmp_path / "no_metadata.md"
+    test_file.write_text(content)
+
+    entity = await parser.parse_file(test_file)
+    assert entity.entity_metadata.data == {}
+
+
+@pytest.mark.asyncio
+async def test_error_handling(tmp_path):
+    """Test error handling."""
+    parser = EntityParser()
+
+    # Missing file
+    with pytest.raises(FileError):
+        await parser.parse_file(Path("nonexistent.md"))
+
+    # Invalid file encoding
+    test_file = tmp_path / "binary.md"
+    with open(test_file, "wb") as f:
+        f.write(b"\x80\x81")  # Invalid UTF-8
+    with pytest.raises(ParseError):
+        await parser.parse_file(test_file)
+
+    # No frontmatter section
+    content = "# Just a title\nNo frontmatter"
+    test_file = tmp_path / "no_frontmatter.md"
+    test_file.write_text(content)
+    with pytest.raises(ParseError):
+        await parser.parse_file(test_file)

tests/markdown/test_frontmatter_parser.py

@@ -1,99 +0,0 @@
-"""Tests for frontmatter parsing."""
-
-from textwrap import dedent
-
-import pytest
-
-from basic_memory.utils.file_utils import parse_frontmatter, ParseError
-
-
-@pytest.mark.asyncio
-async def test_parse_frontmatter():
-    """Test parsing valid frontmatter."""
-    content = dedent("""
-        ---
-        type: test
-        id: test/123
-        created: 2024-12-22T10:00:00Z
-        modified: 2024-12-22T10:00:00Z
-        tags: [test, example]
-        ---
-
-        # Content
-        """).strip()
-
-    frontmatter, remaining = await parse_frontmatter(content)
-    assert frontmatter["type"] == "test"
-    assert frontmatter["id"] == "test/123"
-    assert frontmatter["tags"] == ["test", "example"]
-    assert remaining.strip() == "# Content"
-
-
-@pytest.mark.asyncio
-async def test_parse_frontmatter_comma_tags():
-    """Test parsing tags with commas."""
-    content = dedent("""
-        ---
-        type: test
-        id: test/tags
-        created: 2024-12-22T10:00:00Z
-        modified: 2024-12-22T10:00:00Z
-        tags: [tag1, tag2, tag3]
-        ---
-        """).strip()
-
-    frontmatter, _ = await parse_frontmatter(content)
-    assert frontmatter["tags"] == ["tag1", "tag2", "tag3"]
-
-
-@pytest.mark.asyncio
-async def test_parse_frontmatter_missing_required():
-    """Test error on missing required fields."""
-    content = dedent("""
-        ---
-        type: test
-        # missing id
-        created: 2024-12-22T10:00:00Z
-        modified: 2024-12-22T10:00:00Z
-        tags: []
-        ---
-        """).strip()
-
-    with pytest.raises(ParseError):
-        await parse_frontmatter(content)
-
-
-@pytest.mark.asyncio
-async def test_parse_frontmatter_invalid_date():
-    """Test error on invalid date format."""
-    content = dedent("""
-        ---
-        type: test
-        id: test/invalid
-        created: not-a-date
-        modified: 2024-12-22T10:00:00Z
-        tags: []
-        ---
-        """).strip()
-
-    with pytest.raises(ParseError):
-        await parse_frontmatter(content)
-
-
-@pytest.mark.asyncio
-async def test_parse_frontmatter_whitespace():
-    """Test handling of extra whitespace."""
-    content = dedent("""
-        ---
-        type:    test   
-        id:   test/spaces   
-        created:    2024-12-22T10:00:00Z    
-        modified:    2024-12-22T10:00:00Z    
-        tags:    [tag1,    tag2]   
-        ---
-        """).strip()
-
-    frontmatter, _ = await parse_frontmatter(content)
-    assert frontmatter["type"] == "test"
-    assert frontmatter["id"] == "test/spaces"
-    assert frontmatter["tags"] == ["tag1", "tag2"]

tests/markdown/test_metadata_parser.py

@@ -1,80 +1,103 @@
-"""Tests for metadata parsing."""
+"""Tests for entity metadata parsing."""
 
 from textwrap import dedent
 
 import pytest
 
-from basic_memory.utils.file_utils import parse_frontmatter
+from basic_memory.markdown.parser import EntityParser
 
 
 @pytest.mark.asyncio
 async def test_parse_metadata():
     """Test parsing basic metadata."""
-    text = dedent("""
-        owner: team-auth
-        priority: high
-        status: active
-    """)
+    parser = EntityParser()
+    metadata = {
+        "owner": "team-auth",
+        "priority": "high",
+        "status": "active"
+    }
 
-    result, remaining = await parse_frontmatter(text)
+    result = await parser.parse_metadata(metadata)
 
-    assert result["owner"] == "team-auth"
-    assert result["priority"] == "high"
-    assert result["status"] == "active"
+    assert result.metadata["owner"] == "team-auth"
+    assert result.metadata["priority"] == "high"
+    assert result.metadata["status"] == "active"
 
 
 @pytest.mark.asyncio
 async def test_parse_metadata_empty():
     """Test parsing empty metadata."""
-    text = ""
+    parser = EntityParser()
+    result = await parser.parse_metadata(None)
+    assert result.metadata == {}
 
-    result, remaining = await parse_frontmatter(text)
-    assert result == {}
+    # Should also handle empty dict
+    result = await parser.parse_metadata({})
+    assert result.metadata == {}
 
 
 @pytest.mark.asyncio
 async def test_parse_metadata_whitespace():
     """Test handling of various whitespace in metadata."""
-    text = dedent("""
-        owner:     team-auth    
-        priority:      high     
-        status:   active     
-    """)
+    parser = EntityParser()
+    metadata = {
+        "owner": "   team-auth    ",
+        "priority": "    high     ",
+        "status": "   active     "
+    }
 
-    result, remaining = await parse_frontmatter(text)
+    result = await parser.parse_metadata(metadata)
 
-    assert result["owner"] == "team-auth"
-    assert result["priority"] == "high"
-    assert result["status"] == "active"
+    assert result.metadata["owner"] == "   team-auth    "  # Metadata preserves whitespace
+    assert result.metadata["priority"] == "    high     "
+    assert result.metadata["status"] == "   active     "
 
 
 @pytest.mark.asyncio
-async def test_parse_metadata_multiline_values():
-    """Test handling of multiline metadata values."""
-    text = dedent("""
-        owner: team-auth
-        description: This is a
-         multiline value
-         with several lines
-        status: active
-    """)
+async def test_parse_metadata_nested_objects():
+    """Test handling of nested metadata objects."""
+    parser = EntityParser()
+    metadata = {
+        "owner": "team-auth",
+        "config": {
+            "level": "high",
+            "tags": ["important", "urgent"],
+            "settings": {
+                "notifications": True,
+                "visibility": "private"
+            }
+        }
+    }
 
-    result, _ = await parse_frontmatter(text)
+    result = await parser.parse_metadata(metadata)
 
-    assert result["owner"] == "team-auth"
-    assert result["status"] == "active"
-    assert len(result["description"].splitlines()) == 3
+    assert result.metadata["owner"] == "team-auth"
+    assert result.metadata["config"]["level"] == "high"
+    assert result.metadata["config"]["tags"] == ["important", "urgent"]
+    assert result.metadata["config"]["settings"]["notifications"] is True
+    assert result.metadata["config"]["settings"]["visibility"] == "private"
 
 
 @pytest.mark.asyncio
-async def test_parse_metadata_invalid():
-    """Test handling of invalid metadata format."""
-    text = dedent("""
-        owner team-auth
-        priority: high
-    """)
+async def test_parse_metadata_mixed_types():
+    """Test handling of different value types in metadata."""
+    parser = EntityParser()
+    metadata = {
+        "owner": "team-auth",
+        "active": True,
+        "priority": 1,
+        "tags": ["tag1", "tag2"],
+        "scores": {
+            "a": 10,
+            "b": 20
+        }
+    }
 
-    result, _ = await parse_frontmatter(text)
+    result = await parser.parse_metadata(metadata)
 
-    assert "priority" in result
-    assert "owner" not in result
+    assert result.metadata["owner"] == "team-auth"
+    assert result.metadata["active"] is True
+    assert result.metadata["priority"] == 1
+    assert result.metadata["tags"] == ["tag1", "tag2"]
+    assert result.metadata["scores"]["a"] == 10
+    assert result.metadata["scores"]["b"] == 20

tests/markdown/test_observation_edge_cases.py

@@ -17,10 +17,13 @@ async def test_observation_empty_input():
 @pytest.mark.asyncio
 async def test_observation_unicode():
     """Test handling of Unicode content."""
-    # Invalid UTF-8 sequences
     parser = EntityParser()
-    assert await parser.parse_observation("- [test] Bad UTF \xff") is None
-    assert await parser.parse_observation("- [test] Bad UTF \xfe") is None
+
+    # Valid UTF-8 characters
+    obs = await parser.parse_observation("- [测试] Unicode content #标签")
+    assert obs is not None
+    assert obs.category == "测试"
+    assert "标签" in obs.tags  # pyright: ignore [reportOperatorIssue]
 
     # Control characters
     assert await parser.parse_observation("- [test] With \x00 null") is None
@@ -29,12 +32,6 @@ async def test_observation_unicode():
     assert await parser.parse_observation("- [test] With \x7f delete") is None
     assert await parser.parse_observation("- [test] With " + chr(0x1F) + " unit sep") is None
 
-    # Valid UTF-8
-    obs = await parser.parse_observation("- [测试] Unicode content #标签")
-    assert obs is not None
-    assert obs.category == "测试"
-    assert "标签" in obs.tags  # pyright: ignore [reportOperatorIssue]
-
 
 @pytest.mark.asyncio
 async def test_observation_invalid_context():
@@ -81,11 +78,10 @@ async def test_observation_exception_handling():
 @pytest.mark.asyncio
 async def test_observation_malformed_category():
     """Test handling of malformed category brackets."""
-
     parser = EntityParser()
     with pytest.raises(ParseError, match="unclosed category"):
         await parser.parse_observation("- [test Content")
-
+    
     with pytest.raises(ParseError, match="missing category"):
         await parser.parse_observation("- test] Content")
 
@@ -96,7 +92,6 @@ async def test_observation_malformed_category():
 async def test_observation_whitespace():
     """Test handling of whitespace."""
     # Valid whitespace cases
-
     parser = EntityParser()
     obs = await parser.parse_observation("- [test] Content")
     assert obs is not None
@@ -114,4 +109,4 @@ async def test_observation_whitespace():
         content = f"- [test] Content{char}with{char}{name}"
         obs = await parser.parse_observation(content)
         assert obs is not None
-        assert obs.content == f"Content with {name}"
+        assert obs.content == f"Content with {name}"

tests/markdown/test_parse_entity_file.py

@@ -1,140 +0,0 @@
-"""Tests for entity file parsing."""
-
-from textwrap import dedent
-
-import pytest
-
-from basic_memory.markdown.parser import EntityParser
-
-
-@pytest.mark.asyncio
-async def test_parse_complete_file(tmp_path):
-    """Test parsing a complete entity file."""
-    content = dedent("""
-        ---
-        type: component
-        id: 123
-        created: 2024-12-21T14:00:00Z
-        modified: 2024-12-21T14:00:00Z
-        tags: authentication, security, core
-        ---
-
-        # Auth Service
-
-        Core authentication service.
-
-        <!-- Some comments that should be ignored -->
-
-        ## Observations
-        - [design] Stateless authentication #security #architecture (JWT based)
-        - [feature] Mobile client support #mobile #oauth (Required for App Store)
-        - [tech] Caching layer #performance (Redis implementation)
-
-        ## Relations
-        - implements [[OAuth Implementation]] (Core auth flows)
-        - uses [[Redis Cache]] (Token caching)
-        - specified_by [[Auth API Spec]] (OpenAPI spec)
-
-        ---
-        owner: team-auth
-        priority: high
-        ---
-        """)
-
-    test_file = tmp_path / "test_entity.md"
-    test_file.write_text(content)
-
-    parser = EntityParser()
-    entity = await parser.parse_file(test_file)
-
-    # Check frontmatter
-    assert entity.frontmatter.type == "component"
-    assert entity.frontmatter.id == "123"
-    assert "authentication" in entity.frontmatter.tags
-
-    # Check content
-    assert entity.content.title == "Auth Service"
-    assert len(entity.content.observations) == 3
-    assert len(entity.content.relations) == 3
-
-    # Check specific observation
-    obs = entity.content.observations[0]
-    assert obs.category == "design"
-    assert "security" in obs.tags  # pyright: ignore [reportOperatorIssue]
-    assert obs.context == "JWT based"
-
-    # Check specific relation
-    rel = entity.content.relations[0]
-    assert rel.type == "implements"
-    assert rel.target == "OAuth Implementation"
-    assert rel.context == "Core auth flows"
-
-    # Check metadata
-    assert entity.metadata.metadata["owner"] == "team-auth"
-    assert entity.metadata.metadata["priority"] == "high"
-
-
-@pytest.mark.asyncio
-async def test_parse_minimal_file(tmp_path):
-    """Test parsing a minimal valid entity file."""
-    content = dedent("""
-        ---
-        type: component
-        id: 0
-        created: 2024-12-21T14:00:00Z
-        modified: 2024-12-21T14:00:00Z
-        tags: 
-        ---
-
-        # Minimal Entity
-
-        ## Observations
-        - [note] Basic observation #test
-
-        ## Relations
-        - references [[Other Entity]]
-        """)
-
-    test_file = tmp_path / "minimal.md"
-    test_file.write_text(content)
-
-    parser = EntityParser()
-    entity = await parser.parse_file(test_file)
-
-    assert entity.frontmatter.type == "component"
-    assert entity.frontmatter.id == "0"
-    assert len(entity.content.observations) == 1
-    assert len(entity.content.relations) == 1
-    assert not entity.metadata.metadata  # Empty metadata
-
-
-@pytest.mark.asyncio
-async def test_file_with_metadata_only(tmp_path):
-    """Test parsing a file that has metadata but no content."""
-    content = dedent("""
-        ---
-        type: component
-        id: minimal
-        created: 2024-12-21T14:00:00Z
-        modified: 2024-12-21T14:00:00Z
-        tags: []
-        ---
-
-        # Empty Entity
-
-        ---
-        owner: test-team
-        status: active
-        ---
-        """)
-
-    test_file = tmp_path / "metadata_only.md"
-    test_file.write_text(content)
-
-    parser = EntityParser()
-    entity = await parser.parse_file(test_file)
-
-    assert entity.metadata.metadata["owner"] == "test-team"
-    assert entity.metadata.metadata["status"] == "active"
-    assert not entity.content.observations
-    assert not entity.content.relations

tests/markdown/test_parser.py

@@ -1,151 +0,0 @@
-"""Tests for entity markdown parser."""
-
-from pathlib import Path
-
-import pytest
-
-from basic_memory.markdown.parser import EntityParser
-from basic_memory.markdown.schemas import (
-    Entity,
-    EntityFrontmatter,
-    EntityContent,
-)
-from basic_memory.utils.file_utils import ParseError, FileError
-
-
-@pytest.fixture
-def sample_entity_content():
-    """Sample entity file content."""
-    return """
----
-id: 123
-type: test
-created: 2024-12-22T10:00:00Z
-modified: 2024-12-22T10:00:00Z
-tags: entity, test
----
-
-# Test Entity
-
-A test entity for testing purposes.
-
-## Observations
-- [tech] First technical observation #tag1 (first context)
-- [design] Second design observation #tag2 (second context)
-
-## Relations
-- depends_on [[Other Entity]] (Testing the relation parsing)
-
----
-metadata:
-  checksum: abc123
-  doc_id: 1
----
-
-"""
-
-
-@pytest.mark.asyncio
-async def test_parse_valid_file(tmp_path: Path, sample_entity_content):
-    """Test parsing valid entity file."""
-    # Create test file
-    test_file = tmp_path / "test.md"
-    test_file.write_text(sample_entity_content)
-
-    # Parse file
-    parser = EntityParser()
-    entity = await parser.parse_file(test_file)
-
-    # Verify frontmatter
-    assert isinstance(entity.frontmatter, EntityFrontmatter)
-    assert entity.frontmatter.type == "test"
-    assert entity.frontmatter.id == "123"
-
-    # Verify content
-    assert isinstance(entity.content, EntityContent)
-    assert entity.content.title == "Test Entity"
-    assert entity.content.description == "A test entity for testing purposes."
-
-    # Verify observations
-    assert len(entity.content.observations) == 2
-    obs1 = entity.content.observations[0]
-    assert obs1.category == "tech"
-    assert obs1.content == "First technical observation"
-    assert obs1.tags == ["tag1"]
-    assert obs1.context == "first context"
-
-    obs2 = entity.content.observations[1]
-    assert obs2.category == "design"
-    assert obs2.content == "Second design observation"
-    assert obs2.tags == ["tag2"]
-    assert obs2.context == "second context"
-
-    # Verify relations
-    assert len(entity.content.relations) == 1
-    rel = entity.content.relations[0]
-    assert rel.target == "Other Entity"
-    assert rel.type == "depends_on"
-    assert rel.context == "Testing the relation parsing"
-
-
-@pytest.mark.asyncio
-async def test_parse_missing_file():
-    """Test error on missing file."""
-    parser = EntityParser()
-    with pytest.raises(FileError):
-        await parser.parse_file(Path("nonexistent.md"))
-
-
-@pytest.mark.asyncio
-async def test_parse_invalid_frontmatter(tmp_path: Path):
-    """Test error on invalid frontmatter."""
-    test_file = tmp_path / "test.md"
-    # Create invalid frontmatter by removing required field
-    content = """---
-type: test
-# missing id field
-created: 2024-12-22T10:00:00Z
-modified: 2024-12-22T10:00:00Z
-tags: [entity]
----
-
-# Test Entity"""
-    test_file.write_text(content)
-
-    parser = EntityParser()
-    with pytest.raises(ParseError):
-        await parser.parse_file(test_file)
-
-
-@pytest.mark.asyncio
-async def test_parse_no_frontmatter(tmp_path: Path):
-    """Test file with no frontmatter."""
-    test_file = tmp_path / "test.md"
-    content = "Just content"
-    test_file.write_text(content)
-
-    parser = EntityParser()
-    with pytest.raises(ParseError):
-        await parser.parse_file(test_file)
-
-
-@pytest.mark.asyncio
-async def test_parse_content_str(sample_entity_content):
-    """Test parsing content string directly."""
-    parser = EntityParser()
-    entity = await parser.parse_content_str(sample_entity_content)
-
-    assert isinstance(entity, Entity)
-    assert entity.frontmatter.type == "test"
-    assert entity.frontmatter.id == "123"
-    assert entity.content.title == "Test Entity"
-
-    # Verify observations parsed correctly
-    assert len(entity.content.observations) == 2
-    assert entity.content.observations[0].category == "tech"
-    assert entity.content.observations[1].category == "design"
-
-    # Verify relation parsed correctly
-    assert len(entity.content.relations) == 1
-    assert entity.content.relations[0].type == "depends_on"
-    assert entity.content.relations[0].target == "Other Entity"

tests/markdown/test_relation_edge_cases.py

@@ -2,41 +2,49 @@
 
 import pytest
 
-from basic_memory.markdown import ParseError
-from basic_memory.markdown.schemas import Relation
+from basic_memory.markdown import ParseError, EntityParser
 
 
-def test_relation_empty_target():
+@pytest.mark.asyncio
+async def test_relation_empty_target():
     """Test handling of empty targets."""
     # Empty brackets
-    assert Relation.from_line("type [[]]") is None
-    assert Relation.from_line("type [[ ]]") is None
+    parser = EntityParser()
+
+    assert await parser.parse_relation("type [[]]") is None
+    assert await parser.parse_relation("type [[ ]]") is None
 
     # Only spaces
-    assert Relation.from_line("type [[   ]]") is None
+    assert await parser.parse_relation("type [[   ]]") is None
 
     # Only white spaces
-    assert Relation.from_line("  ") is None
+    assert await parser.parse_relation("  ") is None
 
 
-def test_relation_malformed_context():
+@pytest.mark.asyncio
+async def test_relation_malformed_context():
     """Test handling of malformed context formats."""
+    parser = EntityParser()
+
     # Missing parentheses
     with pytest.raises(ParseError, match="invalid context format"):
-        Relation.from_line("type [[Target]] context without parens")
+        await parser.parse_relation("type [[Target]] context without parens")
 
     # Unclosed parentheses
     with pytest.raises(ParseError, match="invalid context format"):
-        Relation.from_line("type [[Target]] (unclosed")
+        await parser.parse_relation("type [[Target]] (unclosed")
 
     # Extra closing parentheses
     with pytest.raises(ParseError, match="invalid context format"):
-        Relation.from_line("type [[Target]] (closed twice))")
+        await parser.parse_relation("type [[Target]] (closed twice))")
 
 
-def test_relation_generic_errors():
+@pytest.mark.asyncio
+async def test_relation_generic_errors():
     """Test general error handling in relation parsing."""
+    parser = EntityParser()
+
     # Invalid input that should trigger exception handling
-    assert Relation.from_line(None) is None  # type: ignore
-    assert Relation.from_line(123) is None  # type: ignore
-    assert Relation.from_line(object()) is None  # type: ignore
+    assert await parser.parse_relation(None) is None  # type: ignore
+    assert await parser.parse_relation(123) is None  # type: ignore
+    assert await parser.parse_relation(object()) is None  # type: ignore