markdown parsing

tests/markdown/test_debug.py

@@ -0,0 +1,25 @@
+"""Test to explore markdown-it token structure."""
+from textwrap import dedent
+from markdown_it import MarkdownIt
+
+def test_token_structure():
+    """Analyze markdown-it token structure."""
+    content = dedent('''
+    # Title
+
+    ## Observations
+    - [test] First line #tag
+    - [another] Second line #tag2
+    ''')
+
+    md = MarkdownIt()
+    tokens = md.parse(content)
+
+    # Print full token structure
+    print("\nToken structure:")
+    for i, token in enumerate(tokens):
+        attrs = {key: getattr(token, key) for key in dir(token) 
+                if not key.startswith('_') and not callable(getattr(token, key))}
+        print(f"\nToken {i}:")
+        for key, value in attrs.items():
+            print(f"  {key}: {value!r}")

tests/markdown/test_parser.py

@@ -1,4 +1,4 @@
-"""Tests for markdown entity parser."""
+"""Tests for the markdown entity parser."""
 import pytest
 from datetime import datetime
 from pathlib import Path
@@ -6,169 +6,192 @@ from textwrap import dedent
 
 from basic_memory.markdown.parser import EntityParser, ParseError
 
-def test_parse_observation():
-    """Test parsing individual observation lines."""
+def test_parse_observation_basic():
+    """Test basic observation parsing with category and tags."""
     parser = EntityParser()
     
-    # Basic observation with category and content
-    obs = parser._parse_observation("- [design] Simple observation")
+    obs = parser._parse_observation("- [design] Core feature #important #mvp")
     assert obs.category == "design"
-    assert obs.content == "Simple observation"
-    assert obs.tags == []
+    assert obs.content == "Core feature"
+    assert set(obs.tags) == {"important", "mvp"}
     assert obs.context is None
+
+def test_parse_observation_with_context():
+    """Test observation parsing with context in parentheses."""
+    parser = EntityParser()
     
-    # Observation with tags
-    obs = parser._parse_observation("- [feature] Added search #important #core")
+    obs = parser._parse_observation("- [feature] Authentication system #security #auth (Required for MVP)")
     assert obs.category == "feature"
-    assert obs.content == "Added search"
-    assert set(obs.tags) == {"important", "core"}
+    assert obs.content == "Authentication system"
+    assert set(obs.tags) == {"security", "auth"}
+    assert obs.context == "Required for MVP"
+
+def test_parse_observation_edge_cases():
+    """Test observation parsing edge cases."""
+    parser = EntityParser()
+    
+    # Multiple word tags
+    obs = parser._parse_observation("- [tech] Database #high-priority #needs-review")
+    assert set(obs.tags) == {"high-priority", "needs-review"}
+    
+    # Multiple word category
+    obs = parser._parse_observation("- [user experience] Design #ux")
+    assert obs.category == "user experience"
+    
+    # Parentheses in content shouldn't be treated as context
+    obs = parser._parse_observation("- [code] Function (x) returns y #function")
+    assert obs.content == "Function (x) returns y"
     assert obs.context is None
     
-    # Observation with context
-    obs = parser._parse_observation('- [design] Core system design #architecture "Initial version"')
-    assert obs.category == "design"
-    assert obs.content == "Core system design"
-    assert obs.tags == ["architecture"]
-    assert obs.context == "Initial version"
-    
+    # Multiple hashtags together
+    obs = parser._parse_observation("- [test] Feature #important#urgent#now")
+    assert set(obs.tags) == {"important", "urgent", "now"}
+
 def test_parse_observation_errors():
     """Test error handling in observation parsing."""
     parser = EntityParser()
     
     # Missing category brackets
     with pytest.raises(ParseError, match="missing category"):
-        parser._parse_observation("- Design without brackets")
-        
+        parser._parse_observation("- Design without brackets #test")
+    
     # Unclosed category
     with pytest.raises(ParseError, match="unclosed category"):
-        parser._parse_observation("- [design Core system")
+        parser._parse_observation("- [design Core feature #test")
 
-def test_parse_relation():
-    """Test parsing individual relation lines."""
+def test_parse_relation_basic():
+    """Test basic relation parsing."""
     parser = EntityParser()
     
-    # Basic relation
-    rel = parser._parse_relation("- [[EntityA]] #depends_on")
-    assert rel.target == "EntityA"
-    assert rel.type == "depends_on"
-    assert rel.context is None
-    
-    # Relation with context
-    rel = parser._parse_relation('- [[Component]] #implements "Core functionality"')
-    assert rel.target == "Component"
+    rel = parser._parse_relation("- implements [[Auth Service]]")
     assert rel.type == "implements"
-    assert rel.context == "Core functionality"
+    assert rel.target == "Auth Service"
+    assert rel.context is None
+
+def test_parse_relation_with_context():
+    """Test relation parsing with context."""
+    parser = EntityParser()
+    
+    rel = parser._parse_relation("- depends_on [[Database]] (Required for persistence)")
+    assert rel.type == "depends_on"
+    assert rel.target == "Database"
+    assert rel.context == "Required for persistence"
+
+def test_parse_relation_edge_cases():
+    """Test relation parsing edge cases."""
+    parser = EntityParser()
+    
+    # Multiple word type
+    rel = parser._parse_relation("- is used by [[Client App]] (Primary consumer)")
+    assert rel.type == "is used by"
+    
+    # Brackets in context
+    rel = parser._parse_relation("- implements [[API]] (Follows [OpenAPI] spec)")
+    assert rel.context == "Follows [OpenAPI] spec"
+    
+    # Extra spaces
+    rel = parser._parse_relation("-   specifies   [[Format]]   (Documentation)")
+    assert rel.type == "specifies"
+    assert rel.target == "Format"
 
 def test_parse_relation_errors():
     """Test error handling in relation parsing."""
     parser = EntityParser()
     
-    # Missing [[ prefix
+    # Missing target brackets
     with pytest.raises(ParseError, match="missing \\[\\["):
-        parser._parse_relation("- EntityA #depends_on")
-        
-    # Missing ]] suffix
-    with pytest.raises(ParseError, match="missing \\]\\]"):
-        parser._parse_relation("- [[EntityA #depends_on")
-        
-    # Missing relation type
-    with pytest.raises(ParseError, match="missing relation type"):
-        parser._parse_relation("- [[EntityA]] depends_on")
+        parser._parse_relation("- implements Auth Service")
+    
+    # Unclosed target
+    with pytest.raises(ParseError, match="missing ]]"):
+        parser._parse_relation("- implements [[Auth Service")
 
 def test_parse_complete_file(tmp_path):
     """Test parsing a complete entity file."""
     content = dedent('''
         ---
-        type: concept
-        created: 2024-12-21T10:00:00Z
-        modified: 2024-12-21T10:00:00Z
-        tags: [testing, validation]
+        type: component
+        id: component/auth_service
+        created: 2024-12-21T14:00:00Z
+        modified: 2024-12-21T14:00:00Z
+        tags: authentication, security, core
         status: active
         version: 1
-        priority: high
         ---
         
-        # Test Entity
+        # Auth Service
         
-        This is a test entity for validation.
+        Core authentication service.
         
-        ## Description
-        A more detailed description of the test entity.
+        <!-- Some comments that should be ignored -->
         
         ## Observations
-        - [test] First observation #testing
-        - [design] Second observation #important "With context"
+        - [design] Stateless authentication #security #architecture (JWT based)
+        - [feature] Mobile client support #mobile #oauth (Required for App Store)
+        - [tech] Caching layer #performance (Redis implementation)
         
         ## Relations
-        - [[EntityA]] #depends_on "Required dependency"
-        - [[EntityB]] #implements
-        
-        ## Context
-        Additional context information.
+        - implements [[OAuth Implementation]] (Core auth flows)
+        - uses [[Redis Cache]] (Token caching)
+        - specified_by [[Auth API Spec]] (OpenAPI spec)
         
         ## Metadata
-        schema_version: 1.0
-        validation_status: verified
-    ''').lstrip()
+        owner: team-auth
+        priority: high
+        ''')
     
-    # Write test file
     test_file = tmp_path / "test_entity.md"
     test_file.write_text(content)
     
-    # Parse file
     parser = EntityParser()
     entity = parser.parse_file(test_file)
     
     # Check frontmatter
-    assert entity.frontmatter.type == "concept"
-    assert entity.frontmatter.tags == ["testing", "validation"]
+    assert entity.frontmatter.type == "component"
+    assert entity.frontmatter.id == "component/auth_service"
+    assert "authentication" in entity.frontmatter.tags
     assert entity.frontmatter.status == "active"
-    assert entity.frontmatter.priority == "high"
     
     # Check content
-    assert entity.content.title == "Test Entity"
-    assert "detailed description" in entity.content.description
-    assert "context information" in entity.content.context
+    assert entity.content.title == "Auth Service"
+    assert len(entity.content.observations) == 3
+    assert len(entity.content.relations) == 3
     
-    # Check observations
-    assert len(entity.content.observations) == 2
-    first_obs = entity.content.observations[0]
-    assert first_obs.category == "test"
-    assert first_obs.content == "First observation"
-    assert first_obs.tags == ["testing"]
+    # Check specific observation
+    obs = entity.content.observations[0]
+    assert obs.category == "design"
+    assert "security" in obs.tags
+    assert obs.context == "JWT based"
     
-    # Check relations
-    assert len(entity.content.relations) == 2
-    first_rel = entity.content.relations[0]
-    assert first_rel.target == "EntityA"
-    assert first_rel.type == "depends_on"
-    assert first_rel.context == "Required dependency"
+    # Check specific relation
+    rel = entity.content.relations[0]
+    assert rel.type == "implements"
+    assert rel.target == "OAuth Implementation"
+    assert rel.context == "Core auth flows"
     
     # Check metadata
-    assert entity.content.metadata["schema_version"] == "1.0"
-    assert entity.content.metadata["validation_status"] == "verified"
-
-def test_parse_missing_file():
-    """Test handling of missing files."""
-    parser = EntityParser()
-    with pytest.raises(ParseError, match="File does not exist"):
-        parser.parse_file(Path("nonexistent.md"))
+    assert entity.content.metadata["owner"] == "team-auth"
+    assert entity.content.metadata["priority"] == "high"
 
 def test_parse_minimal_file(tmp_path):
     """Test parsing a minimal valid entity file."""
     content = dedent('''
         ---
-        type: concept
-        created: 2024-12-21T10:00:00Z
-        modified: 2024-12-21T10:00:00Z
+        type: component
+        id: minimal
+        created: 2024-12-21T14:00:00Z
+        modified: 2024-12-21T14:00:00Z
         tags: []
         ---
         
         # Minimal Entity
         
-        ## Description
-        Minimal valid entity.
-    ''').lstrip()
+        ## Observations
+        - [note] Basic observation #test
+        
+        ## Relations
+        - references [[Other Entity]]
+    ''')
     
     test_file = tmp_path / "minimal.md"
     test_file.write_text(content)
@@ -176,12 +199,37 @@ def test_parse_minimal_file(tmp_path):
     parser = EntityParser()
     entity = parser.parse_file(test_file)
     
-    # Check required fields
-    assert entity.frontmatter.type == "concept"
-    assert isinstance(entity.frontmatter.created, datetime)
-    assert entity.content.title == "Minimal Entity"
+    assert entity.frontmatter.type == "component"
+    assert len(entity.content.observations) == 1
+    assert len(entity.content.relations) == 1
+
+def test_parse_file_errors(tmp_path):
+    """Test error handling for invalid files."""
+    parser = EntityParser()
     
-    # Optional fields should have default values
-    assert entity.content.observations == []
-    assert entity.content.relations == []
-    assert entity.content.metadata == {}
+    # Missing file
+    with pytest.raises(ParseError, match="does not exist"):
+        parser.parse_file(Path("nonexistent.md"))
+    
+    # Invalid frontmatter
+    content = dedent('''
+        ---
+        invalid: yaml: [
+        ---
+        # Title
+    ''')
+    test_file = tmp_path / "invalid.md"
+    test_file.write_text(content)
+    with pytest.raises(ParseError):
+        parser.parse_file(test_file)
+    
+    # Missing required frontmatter
+    content = dedent('''
+        ---
+        type: component
+        ---
+        # Title
+    ''')
+    test_file.write_text(content)
+    with pytest.raises(ParseError):
+        parser.parse_file(test_file)

tests/markdown/test_parser_edge_cases.py

@@ -0,0 +1,279 @@
+"""Tests for edge cases and tricky inputs in the markdown parser."""
+from datetime import datetime
+from pathlib import Path
+from textwrap import dedent
+import pytest
+
+from basic_memory.markdown.parser import EntityParser, ParseError, EntityFrontmatter, EntityContent, Entity
+
+def test_unicode_content(tmp_path):
+    """Test handling of Unicode content including emoji and non-Latin scripts."""
+    content = dedent('''
+        ---
+        type: test
+        id: test/unicode
+        created: 2024-12-21T14:00:00Z
+        modified: 2024-12-21T14:00:00Z
+        tags: [unicode, 测试]
+        ---
+        
+        # Unicode Test 🧪
+        
+        ## Observations
+        - [test] Emoji test 👍 #emoji #test
+        - [中文] Chinese text 测试 #language
+        - [русский] Russian привет #language
+        - [😀] Emoji category #meta (Category test)
+        
+        ## Relations
+        - implements [[测试组件]] (Unicode test)
+        - used_by [[компонент]] (Another test)
+    ''')
+    
+    test_file = tmp_path / "unicode.md"
+    test_file.write_text(content, encoding='utf-8')
+    
+    parser = EntityParser()
+    entity = parser.parse_file(test_file)
+    
+    assert "测试" in entity.frontmatter.tags
+    assert entity.content.title == "Unicode Test 🧪"
+    assert "👍" in entity.content.observations[0].content
+    assert entity.content.observations[2].category == "русский"
+    assert entity.content.observations[3].category == "😀"
+    assert entity.content.relations[0].target == "测试组件"
+
+def test_long_content(tmp_path):
+    """Test handling of very long content at our limits."""
+    # Create a long observation right at our length limit
+    long_obs = "x" * 995 + " #tag"  # 1000 chars with tag
+    
+    content = dedent(f'''
+        ---
+        type: test
+        id: test/long
+        created: 2024-12-21T14:00:00Z
+        modified: 2024-12-21T14:00:00Z
+        tags: [long]
+        ---
+        
+        # Long Content Test
+        
+        ## Description
+        {"Very long description " * 100}
+        
+        ## Observations
+        - [test] {long_obs}
+        
+        ## Relations
+        - related_to [[{"Very long entity name " * 10}]] (Long context test)
+    ''')
+    
+    test_file = tmp_path / "long.md"
+    test_file.write_text(content)
+    
+    parser = EntityParser()
+    entity = parser.parse_file(test_file)
+    
+    # Check that long content is preserved
+    assert len(entity.content.observations[0].content) == 995
+    assert entity.content.observations[0].tags == ["tag"]
+
+def test_mixed_newlines(tmp_path):
+    """Test handling of different newline styles (\\n, \\r\\n, \\r)."""
+    content = "---\\ntype: test\\r\\nid: test/newlines\\ncreated: 2024-12-21T14:00:00Z\\rmodified: 2024-12-21T14:00:00Z\\ntags: [test]\\n---\\n\\r\\n# Test\\r\\n## Observations\\n- [test] Line 1\\r- [test] Line 2\\n".replace('\\n', '\n').replace('\\r', '\r')
+    
+    test_file = tmp_path / "newlines.md"
+    test_file.write_text(content, encoding='utf-8')
+    
+    parser = EntityParser()
+    entity = parser.parse_file(test_file)
+    
+    assert len(entity.content.observations) == 2
+
+def test_malformed_frontmatter(tmp_path):
+    """Test various malformed frontmatter cases."""
+    cases = [
+        # Invalid YAML syntax
+        '''---
+        type: : test: :
+        id: test/bad
+        created: 2024-12-21T14:00:00Z
+        modified: 2024-12-21T14:00:00Z
+        tags: [test]
+        ---''',
+        
+        # Invalid datetime format
+        '''---
+        type: test
+        id: test/bad
+        created: not-a-date
+        modified: 2024-12-21T14:00:00Z
+        tags: [test]
+        ---''',
+        
+        # Missing required field
+        '''---
+        type: test
+        created: 2024-12-21T14:00:00Z
+        modified: 2024-12-21T14:00:00Z
+        tags: [test]
+        ---''',
+        
+        # Extra fields
+        '''---
+        type: test
+        id: test/extra
+        created: 2024-12-21T14:00:00Z
+        modified: 2024-12-21T14:00:00Z
+        tags: [test]
+        nonexistent_field: value
+        ---'''
+    ]
+    
+    parser = EntityParser()
+    test_file = tmp_path / "bad.md"
+    
+    for i, case in enumerate(cases):
+        content = case + "\n# Test"
+        test_file.write_text(content)
+        with pytest.raises(ParseError):
+            parser.parse_file(test_file)
+
+def test_nested_structures(tmp_path):
+    """Test handling of nested markdown structures."""
+    content = dedent('''
+        ---
+        type: test
+        id: test/nested
+        created: 2024-12-21T14:00:00Z
+        modified: 2024-12-21T14:00:00Z
+        tags: [test]
+        ---
+        
+        # Nested Test
+        
+        ## Observations
+        - [test] Main point #main
+            - [sub] Subpoint #sub
+                - [subsub] Sub-subpoint #detail
+        
+        ## Relations
+        - contains [[Sub Entity]]
+            - and [[Another Entity]]
+                - also [[Third Entity]]
+    ''')
+    
+    test_file = tmp_path / "nested.md"
+    test_file.write_text(content)
+    
+    parser = EntityParser()
+    entity = parser.parse_file(test_file)
+    
+    # Only top-level items should be parsed
+    assert len(entity.content.observations) == 1
+    assert len(entity.content.relations) == 1
+
+def test_file_encodings(tmp_path):
+    """Test different file encodings."""
+    encodings = ['utf-8', 'utf-16', 'latin1']
+    content = dedent('''
+        ---
+        type: test
+        id: test/encoding
+        created: 2024-12-21T14:00:00Z
+        modified: 2024-12-21T14:00:00Z
+        tags: [test]
+        ---
+        
+        # Encoding Test
+        
+        ## Observations
+        - [test] ASCII content #test
+        - [utf8] UTF-8 content 测试 #unicode
+    ''')
+    
+    parser = EntityParser()
+    
+    for encoding in encodings:
+        test_file = tmp_path / f"encoding_{encoding}.md"
+        test_file.write_text(content, encoding=encoding)
+        
+        try:
+            entity = parser.parse_file(test_file)
+            assert len(entity.content.observations) == 2
+        except UnicodeError:
+            # Some encodings might not handle all characters
+            pass
+
+def test_empty_sections(tmp_path):
+    """Test handling of empty sections."""
+    content = dedent('''
+        ---
+        type: test
+        id: test/empty
+        created: 2024-12-21T14:00:00Z
+        modified: 2024-12-21T14:00:00Z
+        tags: []
+        ---
+        
+        # Empty Test
+        
+        ## Description
+        
+        ## Observations
+        
+        ## Relations
+        
+        ## Context
+        
+        ## Metadata
+    ''')
+    
+    test_file = tmp_path / "empty.md"
+    test_file.write_text(content)
+    
+    parser = EntityParser()
+    entity = parser.parse_file(test_file)
+    
+    assert entity.content.description == ""
+    assert entity.content.observations == []
+    assert entity.content.relations == []
+    assert entity.content.context == ""
+    assert entity.content.metadata == {}
+
+def test_malformed_sections(tmp_path):
+    """Test various malformed section contents."""
+    content = dedent('''
+        ---
+        type: test
+        id: test/malformed
+        created: 2024-12-21T14:00:00Z
+        modified: 2024-12-21T14:00:00Z
+        tags: [test]
+        ---
+        
+        # Malformed Test
+        
+        ## Observations
+        - not a valid observation
+        - [unclosed category content
+        - no content]
+        - [] empty category
+        
+        ## Relations
+        - not a valid relation
+        - missing type [[Entity]]
+        - incomplete [[
+        - ]] backwards
+    ''')
+    
+    test_file = tmp_path / "malformed.md"
+    test_file.write_text(content)
+    
+    parser = EntityParser()
+    entity = parser.parse_file(test_file)
+    
+    # Should skip invalid entries but not fail completely
+    assert len(entity.content.observations) == 0
+    assert len(entity.content.relations) == 0