basicmachines-co-basic-memory/tests/markdown/test_parser_edge_cases.py

"""Tests for markdown parser edge cases."""

from pathlib import Path
from textwrap import dedent

import pytest

from basic_memory.markdown.knowledge_parser import KnowledgeParser
from basic_memory.utils.file_utils import FileError, ParseError


@pytest.mark.asyncio
async def test_unicode_content(tmp_path):
    """Test handling of Unicode content including emoji and non-Latin scripts."""
    content = dedent("""
        ---
        type: test
        id: test/unicode
        created: 2024-12-21T14:00:00Z
        modified: 2024-12-21T14:00:00Z
        tags: [unicode, 测试]
        ---

        # Unicode Test 🧪

        ## Observations
        - [test] Emoji test 👍 #emoji #test (Testing emoji)
        - [中文] Chinese text 测试 #language (Script test)
        - [русский] Russian привет #language (More scripts)
        - [note] Emoji in text 😀 #meta (Category test)

        ## Relations
        - tested_by [[测试组件]] (Unicode test)
        - depends_on [[компонент]] (Another test)

        ## Metadata
        ```yml
        category: test
        status: active
        ```
        """)

    test_file = tmp_path / "unicode.md"
    test_file.write_text(content, encoding="utf-8")

    parser = KnowledgeParser()
    entity = await parser.parse_file(test_file)

    assert "测试" in entity.frontmatter.tags
    assert "китайский" not in entity.frontmatter.tags
    assert entity.content.title == "Unicode Test 🧪"


@pytest.mark.asyncio
async def test_fallback_encoding(tmp_path):
    """Test UTF-16 fallback when UTF-8 fails."""
    content = dedent("""
        Hello 世界
        No proper sections here
        """)
    test_file = tmp_path / "unicode_file.md"
    test_file.write_text(content, encoding="utf-16")

    parser = KnowledgeParser()
    with pytest.raises(ParseError):
        await parser.parse_file(test_file)


@pytest.mark.asyncio
async def test_encoding_errors(tmp_path):
    """Test handling of encoding errors."""
    # Create a file with invalid UTF-8 bytes
    test_file = tmp_path / "invalid.md"
    with open(test_file, "wb") as f:
        f.write(b"\xff\xfe\x00\x00")  # Invalid UTF-8

    parser = KnowledgeParser()
    with pytest.raises(ParseError):
        await parser.parse_file(test_file, encoding="ascii")


@pytest.mark.asyncio
async def test_file_not_found():
    """Test handling of non-existent files."""
    parser = KnowledgeParser()
    with pytest.raises(FileError):
        await parser.parse_file(Path("nonexistent.md"))


@pytest.mark.asyncio
async def test_nested_structures(tmp_path):
    """Test handling of nested markdown structures."""
    content = dedent("""
        ---
        type: test
        id: test/nested
        created: 2024-12-21T14:00:00Z
        modified: 2024-12-21T14:00:00Z
        tags: [test]
        ---

        # Nested Test

        ## Observations
        - [test] Main point #main (Top level)
            - [test] Subpoint #sub (Should be ignored)
                - [test] Sub-subpoint #detail (Also ignored)

        ## Relations
        - depends_on [[Sub Entity]] (Top level)
            - uses [[Another Entity]] (Should be ignored)
                - implements [[Third Entity]] (Also ignored)
        """)

    test_file = tmp_path / "nested.md"
    test_file.write_text(content)

    parser = KnowledgeParser()
    entity = await parser.parse_file(test_file)

    assert len(entity.content.observations) == 3
    assert len(entity.content.relations) == 3
    assert entity.content.observations[0].tags == ["main"]
    assert entity.content.observations[0].context == "Top level"
    assert entity.content.relations[0].target == "Sub Entity"
    assert entity.content.relations[0].type == "depends_on"


@pytest.mark.asyncio
async def test_malformed_sections(tmp_path):
    """Test various malformed section contents."""
    content = dedent("""
        ---
        type: test
        id: test/malformed
        created: 2024-12-21T14:00:00Z
        modified: 2024-12-21T14:00:00Z
        tags: [test]
        ---

        # Malformed Test

        ## Observations
        - not a valid observation
        - [unclosed category content
        - no content]
        - [] empty category

        ## Relations
        - not a valid relation
        - missing_brackets Entity
        - implements incomplete [[
        - implements ]] backwards
        """)

    test_file = tmp_path / "malformed.md"
    test_file.write_text(content)

    parser = KnowledgeParser()
    with pytest.raises(ParseError):
        entity = await parser.parse_file(test_file)


@pytest.mark.asyncio
async def test_missing_required_sections(tmp_path):
    """Test handling of missing required sections."""
    # Test file with only frontmatter
    content = dedent("""
        ---
        type: test
        id: test/incomplete
        created: 2024-12-21T14:00:00Z
        modified: 2024-12-21T14:00:00Z
        tags: [test]
        ---
        """)

    test_file = tmp_path / "incomplete.md"
    test_file.write_text(content)

    parser = KnowledgeParser()
    with pytest.raises(ParseError):
        await parser.parse_file(test_file)