From fb4f0ba30d8323f7b9d9e88a84c7bcd995d664e8 Mon Sep 17 00:00:00 2001 From: phernandez Date: Sun, 22 Dec 2024 14:40:12 -0600 Subject: [PATCH] breaking up parsing logic for reuse --- src/basic_memory/markdown/base_parser.py | 141 ++++++++++++++++++++++ src/basic_memory/markdown/parser.py | 136 ++++++++++++++------- tests/markdown/test_base_parser.py | 122 +++++++++++++++++++ tests/markdown/test_parser.py | 145 +++++++++++++++++++++++ 4 files changed, 500 insertions(+), 44 deletions(-) create mode 100644 src/basic_memory/markdown/base_parser.py create mode 100644 tests/markdown/test_base_parser.py create mode 100644 tests/markdown/test_parser.py diff --git a/src/basic_memory/markdown/base_parser.py b/src/basic_memory/markdown/base_parser.py new file mode 100644 index 00000000..baa71e7a --- /dev/null +++ b/src/basic_memory/markdown/base_parser.py @@ -0,0 +1,141 @@ +"""Base parser for markdown files with frontmatter.""" +from abc import ABC, abstractmethod +from pathlib import Path +from typing import List, TypeVar, Generic, Optional, Dict, Any, Tuple + +from loguru import logger + +from basic_memory.utils.file_utils import parse_frontmatter, ParseError, FileError + +T = TypeVar('T') # The parsed document type + + +class MarkdownParser(ABC, Generic[T]): + """ + Base parser for markdown files with frontmatter. + + Every supported document type should subclass this with their specific + parsing and document creation logic. + """ + + async def parse_file(self, path: Path, encoding: str = "utf-8") -> T: + """ + Parse a markdown file with frontmatter. + + Args: + path: Path to markdown file + encoding: File encoding to use + + Returns: + Parsed document of type T + + Raises: + FileError: If file cannot be read + ParseError: If content cannot be parsed + """ + if not path.exists(): + raise FileError(f"File does not exist: {path}") + + try: + content = path.read_text(encoding=encoding) + return await self.parse_content_str(content) + + except UnicodeError as e: + if encoding == "utf-8": + return await self.parse_file(path, encoding="utf-16") + raise ParseError(f"Failed to decode {path}: {str(e)}") from e + except Exception as e: + if not isinstance(e, (FileError, ParseError)): + logger.error(f"Failed to parse {path}: {e}") + raise ParseError(f"Failed to parse {path}: {str(e)}") from e + raise + + async def parse_content_str(self, content: str) -> T: + """ + Parse raw content string into document. + + Args: + content: Raw file content + + Returns: + Parsed document + + Raises: + ParseError: If parsing fails + """ + try: + # Split into frontmatter and content + frontmatter, remaining = await parse_frontmatter(content) + + # Parse sections with concrete implementation + parsed_frontmatter = await self.parse_frontmatter(frontmatter) + parsed_content = await self.parse_content(remaining) + parsed_metadata = await self.parse_metadata(frontmatter.get("metadata")) + + # Create document from parts + return await self.create_document(parsed_frontmatter, parsed_content, parsed_metadata) + + except Exception as e: + if not isinstance(e, ParseError): + logger.error(f"Failed to parse content: {e}") + raise ParseError(f"Failed to parse content: {str(e)}") from e + raise + + @abstractmethod + async def parse_frontmatter(self, frontmatter: Dict[str, Any]) -> Any: + """ + Parse frontmatter section. + + Args: + frontmatter: Parsed YAML frontmatter + + Returns: + Parsed frontmatter in format needed by document + """ + pass + + @abstractmethod + async def parse_content(self, content: str) -> Any: + """ + Parse main content section. + + Args: + content: Content section text + + Returns: + Parsed content in format needed by document + """ + pass + + @abstractmethod + async def parse_metadata(self, metadata: Optional[Dict[str, Any]]) -> Any: + """ + Parse optional metadata section. + + Args: + metadata: Optional metadata dictionary + + Returns: + Parsed metadata in format needed by document + """ + pass + + @abstractmethod + async def create_document( + self, + frontmatter: Any, + content: Any, + metadata: Optional[Any] + ) -> T: + """ + Create document from parsed sections. + + Args: + frontmatter: Parsed frontmatter + content: Parsed content + metadata: Optional parsed metadata + + Returns: + Complete document of type T + """ + pass \ No newline at end of file diff --git a/src/basic_memory/markdown/parser.py b/src/basic_memory/markdown/parser.py index a8c7b453..65aa4375 100644 --- a/src/basic_memory/markdown/parser.py +++ b/src/basic_memory/markdown/parser.py @@ -1,9 +1,11 @@ """Parser for Basic Memory entity markdown files.""" -import logging from pathlib import Path +from typing import Dict, Any, Optional -from basic_memory.markdown.exceptions import ParseError +from loguru import logger + +from basic_memory.markdown.base_parser import MarkdownParser, ParseError from basic_memory.markdown.schemas import ( Entity, EntityFrontmatter, @@ -11,49 +13,95 @@ from basic_memory.markdown.schemas import ( EntityMetadata, ) -logging.basicConfig(level=logging.DEBUG) -logger = logging.getLogger(__name__) +class EntityParser(MarkdownParser[Entity]): + """ + Parser for entity markdown files. + + Parses files in the Knowledge Format, which includes: + - YAML frontmatter + - Markdown content with observations + - Optional metadata section + """ -def debug_sections(text): - """Debug helper to show section contents.""" - # Split on triple-dash and newline combinations to be more lenient - sections = [s.strip() for s in text.replace("\r\n", "\n").split("---")] - logger.debug("File sections:") - for i, section in enumerate(sections): - logger.debug(f"\n=== Section {i} ===\n{section.strip()}\n") - return [s for s in sections if s.strip()] # Remove empty sections - - -class EntityParser: - """Parser for entity markdown files.""" - - def parse_file(self, path: Path, encoding: str = "utf-8") -> Entity: - """Parse an entity markdown file.""" - if not path.exists(): - raise ParseError(f"File does not exist: {path}") - - try: - # Read file content and split sections - with open(path, "r", encoding=encoding) as f: - raw_content = f.read() - sections = debug_sections(raw_content) - - if len(sections) < 2: # Need at least frontmatter and content - raise ParseError("Missing required document sections") - - # Parse each section using schema methods - frontmatter = EntityFrontmatter.from_text(sections[0]) - content = EntityContent.from_markdown(sections[1]) + async def parse_frontmatter(self, frontmatter: Dict[str, Any]) -> EntityFrontmatter: + """ + Parse entity frontmatter. + + Args: + frontmatter: Parsed YAML frontmatter - # Handle optional metadata section - metadata = EntityMetadata.from_text(sections[2] if len(sections) > 2 else "") - - return Entity(frontmatter=frontmatter, content=content, metadata=metadata) - - except UnicodeError as e: - if encoding == "utf-8": - return self.parse_file(path, encoding="utf-16") - raise ParseError(f"Failed to parse {path}: {str(e)}") from e + Returns: + Parsed EntityFrontmatter + + Raises: + ParseError: If frontmatter doesn't match schema + """ + try: + return EntityFrontmatter(**frontmatter) except Exception as e: - raise ParseError(f"Failed to parse {path}: {str(e)}") from e \ No newline at end of file + logger.error(f"Invalid entity frontmatter: {e}") + raise ParseError(f"Invalid entity frontmatter: {str(e)}") from e + + async def parse_content(self, content: str) -> EntityContent: + """ + Parse entity content section. + + Args: + content: Content section text + + Returns: + Parsed EntityContent + + Raises: + ParseError: If content doesn't match schema + """ + try: + return EntityContent.from_markdown(content) + except Exception as e: + logger.error(f"Invalid entity content: {e}") + raise ParseError(f"Invalid entity content: {str(e)}") from e + + async def parse_metadata(self, metadata: Optional[Dict[str, Any]]) -> EntityMetadata: + """ + Parse entity metadata section. + + Args: + metadata: Optional metadata dictionary + + Returns: + Parsed EntityMetadata + + Raises: + ParseError: If metadata doesn't match schema + """ + try: + if not metadata: + return EntityMetadata() + return EntityMetadata(**metadata) + except Exception as e: + logger.error(f"Invalid entity metadata: {e}") + raise ParseError(f"Invalid entity metadata: {str(e)}") from e + + async def create_document( + self, + frontmatter: EntityFrontmatter, + content: EntityContent, + metadata: EntityMetadata + ) -> Entity: + """ + Create entity from parsed sections. + + Args: + frontmatter: Parsed frontmatter + content: Parsed content + metadata: Parsed metadata + + Returns: + Complete entity + """ + return Entity( + frontmatter=frontmatter, + content=content, + metadata=metadata + ) \ No newline at end of file diff --git a/tests/markdown/test_base_parser.py b/tests/markdown/test_base_parser.py new file mode 100644 index 00000000..f3e2f452 --- /dev/null +++ b/tests/markdown/test_base_parser.py @@ -0,0 +1,122 @@ +"""Tests for base markdown parser.""" +import pytest +from dataclasses import dataclass +from pathlib import Path +from typing import Dict, Any, Optional + +from basic_memory.markdown.base_parser import MarkdownParser, ParseError, FileError + + +@dataclass +class TestDoc: + """Simple document for testing.""" + title: str + content: str + metadata: Optional[Dict[str, Any]] = None + + +class TestParser(MarkdownParser[TestDoc]): + """Concrete parser implementation for testing.""" + + async def parse_frontmatter(self, frontmatter: Dict[str, Any]) -> str: + """Extract title from frontmatter.""" + if "title" not in frontmatter: + raise ParseError("Missing required title") + return frontmatter["title"] + + async def parse_content(self, content: str) -> str: + """Just return content as-is.""" + return content.strip() + + async def parse_metadata(self, metadata: Optional[Dict[str, Any]]) -> Optional[Dict[str, Any]]: + """Pass through metadata.""" + return metadata + + async def create_document( + self, + frontmatter: str, + content: str, + metadata: Optional[Dict[str, Any]] + ) -> TestDoc: + """Create test document.""" + return TestDoc( + title=frontmatter, + content=content, + metadata=metadata + ) + + +@pytest.mark.asyncio +async def test_parse_valid_file(tmp_path: Path): + """Test parsing valid file.""" + # Create test file + test_file = tmp_path / "test.md" + content = """--- +title: Test Doc +metadata: + key: value +--- + +Test content""" + test_file.write_text(content) + + # Parse file + parser = TestParser() + doc = await parser.parse_file(test_file) + + assert doc.title == "Test Doc" + assert doc.content == "Test content" + assert doc.metadata == {"key": "value"} + + +@pytest.mark.asyncio +async def test_parse_missing_file(): + """Test error on missing file.""" + parser = TestParser() + with pytest.raises(FileError): + await parser.parse_file(Path("nonexistent.md")) + + +@pytest.mark.asyncio +async def test_parse_invalid_frontmatter(tmp_path: Path): + """Test error on invalid frontmatter.""" + test_file = tmp_path / "test.md" + content = """--- +not_title: Test Doc +--- + +content""" + test_file.write_text(content) + + parser = TestParser() + with pytest.raises(ParseError): + await parser.parse_file(test_file) + + +@pytest.mark.asyncio +async def test_parse_no_frontmatter(tmp_path: Path): + """Test file with no frontmatter.""" + test_file = tmp_path / "test.md" + content = "Just content" + test_file.write_text(content) + + parser = TestParser() + with pytest.raises(ParseError): + await parser.parse_file(test_file) + + +@pytest.mark.asyncio +async def test_parse_content_str(): + """Test parsing content string directly.""" + content = """--- +title: Test Doc +--- + +Test content""" + + parser = TestParser() + doc = await parser.parse_content_str(content) + + assert doc.title == "Test Doc" + assert doc.content == "Test content" + assert doc.metadata is None diff --git a/tests/markdown/test_parser.py b/tests/markdown/test_parser.py new file mode 100644 index 00000000..ef65f07a --- /dev/null +++ b/tests/markdown/test_parser.py @@ -0,0 +1,145 @@ +"""Tests for entity markdown parser.""" +import pytest +from pathlib import Path + +from basic_memory.markdown.parser import EntityParser +from basic_memory.utils.file_utils import ParseError, FileError +from basic_memory.markdown.schemas import ( + Entity, + EntityFrontmatter, + EntityContent, + EntityMetadata, +) + + +@pytest.fixture +def sample_entity_content(): + """Sample entity file content.""" + return """--- +type: test +id: test/test_entity +created: 2024-12-22T10:00:00Z +modified: 2024-12-22T10:00:00Z +tags: [entity, test] +metadata: + checksum: abc123 + doc_id: 1 +--- + +# Test Entity + +A test entity for testing purposes. + +## Observations +- [tech] First technical observation #tag1 (first context) +- [design] Second design observation #tag2 (second context) + +## Relations +- depends_on [[Other Entity]] (Testing the relation parsing) +""" + + +@pytest.mark.asyncio +async def test_parse_valid_file(tmp_path: Path, sample_entity_content): + """Test parsing valid entity file.""" + # Create test file + test_file = tmp_path / "test.md" + test_file.write_text(sample_entity_content) + + # Parse file + parser = EntityParser() + entity = await parser.parse_file(test_file) + + # Verify frontmatter + assert isinstance(entity.frontmatter, EntityFrontmatter) + assert entity.frontmatter.type == "test" + assert entity.frontmatter.id == "test/test_entity" + + # Verify content + assert isinstance(entity.content, EntityContent) + assert entity.content.title == "Test Entity" + assert entity.content.description == "A test entity for testing purposes." + + # Verify observations + assert len(entity.content.observations) == 2 + obs1 = entity.content.observations[0] + assert obs1.category == "tech" + assert obs1.content == "First technical observation" + assert obs1.tags == ["tag1"] + assert obs1.context == "first context" + + obs2 = entity.content.observations[1] + assert obs2.category == "design" + assert obs2.content == "Second design observation" + assert obs2.tags == ["tag2"] + assert obs2.context == "second context" + + # Verify relations + assert len(entity.content.relations) == 1 + rel = entity.content.relations[0] + assert rel.target == "Other Entity" + assert rel.type == "depends_on" + assert rel.context == "Testing the relation parsing" + + +@pytest.mark.asyncio +async def test_parse_missing_file(): + """Test error on missing file.""" + parser = EntityParser() + with pytest.raises(FileError): + await parser.parse_file(Path("nonexistent.md")) + + +@pytest.mark.asyncio +async def test_parse_invalid_frontmatter(tmp_path: Path): + """Test error on invalid frontmatter.""" + test_file = tmp_path / "test.md" + # Create invalid frontmatter by removing required field + content = """--- +type: test +# missing id field +created: 2024-12-22T10:00:00Z +modified: 2024-12-22T10:00:00Z +tags: [entity] +--- + +# Test Entity""" + test_file.write_text(content) + + parser = EntityParser() + with pytest.raises(ParseError): + await parser.parse_file(test_file) + + +@pytest.mark.asyncio +async def test_parse_no_frontmatter(tmp_path: Path): + """Test file with no frontmatter.""" + test_file = tmp_path / "test.md" + content = "Just content" + test_file.write_text(content) + + parser = EntityParser() + with pytest.raises(ParseError): + await parser.parse_file(test_file) + + +@pytest.mark.asyncio +async def test_parse_content_str(sample_entity_content): + """Test parsing content string directly.""" + parser = EntityParser() + entity = await parser.parse_content_str(sample_entity_content) + + assert isinstance(entity, Entity) + assert entity.frontmatter.type == "test" + assert entity.frontmatter.id == "test/test_entity" + assert entity.content.title == "Test Entity" + + # Verify observations parsed correctly + assert len(entity.content.observations) == 2 + assert entity.content.observations[0].category == "tech" + assert entity.content.observations[1].category == "design" + + # Verify relation parsed correctly + assert len(entity.content.relations) == 1 + assert entity.content.relations[0].type == "depends_on" + assert entity.content.relations[0].target == "Other Entity" \ No newline at end of file