diff --git a/src/basic_memory/markdown/base_parser.py b/src/basic_memory/markdown/base_parser.py index ec63f592..9c2b43cb 100644 --- a/src/basic_memory/markdown/base_parser.py +++ b/src/basic_memory/markdown/base_parser.py @@ -62,20 +62,31 @@ class MarkdownParser(ABC, Generic[T]): """ try: # Split into frontmatter and content - frontmatter, remaining = await parse_frontmatter(content) + frontmatter, markdown = await parse_frontmatter(content) + + # Extract metadata from frontmatter if present + metadata = frontmatter.pop("metadata", None) - # Split main content into sections - title, sections = self.split_sections(remaining) + # Parse frontmatter first + parsed_frontmatter = await self.parse_frontmatter(frontmatter) + + # Split remaining content into sections + title, sections = self._split_sections(markdown) if not title: raise ParseError("Missing title section (must start with #)") - # Parse each section - parsed_frontmatter = await self.parse_frontmatter(frontmatter) + # Parse content sections parsed_content = await self.parse_content(title, sections) - parsed_metadata = await self.parse_metadata(frontmatter.get("metadata")) - # Create document from parts - return await self.create_document(parsed_frontmatter, parsed_content, parsed_metadata) + # Parse metadata separately + parsed_metadata = await self.parse_metadata(metadata) + + # Create final document + return await self.create_document( + frontmatter=parsed_frontmatter, + content=parsed_content, + metadata=parsed_metadata + ) except Exception as e: if not isinstance(e, ParseError): @@ -83,7 +94,7 @@ class MarkdownParser(ABC, Generic[T]): raise ParseError(f"Failed to parse content: {str(e)}") from e raise - def split_sections(self, content: str) -> Tuple[str, Dict[str, List[str]]]: + def _split_sections(self, content: str) -> Tuple[Optional[str], Dict[str, str]]: """ Split content into sections by headers. @@ -91,37 +102,46 @@ class MarkdownParser(ABC, Generic[T]): content: Content section of the document Returns: - Tuple of (title section, {section_name: list of section lines}) + Tuple of (title section, {section_name: section content}) """ - current_section = None - sections = {} + # Initialize state title = None + sections: Dict[str, List[str]] = {} + current_section = None + current_lines: List[str] = [] + # Process each line for line in content.splitlines(): - line = line.strip() - - # Skip empty lines - if not line: + # Handle headers + if line.startswith("# "): # Top level header (title) + title = line[2:].strip() continue - # Handle headers - if line.startswith("#"): - # Top level header is title - if line.startswith("# "): - title = line[2:].strip() - continue + if line.startswith("## "): # Section header + # Save current section if any + if current_section and current_lines: + sections[current_section] = "\n".join(current_lines).strip() + current_lines = [] + + # Start new section + current_section = line[3:].strip().lower() + continue - # Other headers start new sections - if line.startswith("## "): - current_section = line[3:].strip().lower() - sections[current_section] = [] - continue + # Add line to current section + if current_section is not None: + current_lines.append(line) + elif line.strip() and title: # Non-empty line after title but before first section + # Default section for content right after title + if "content" not in sections: + sections["content"] = line.strip() + else: + sections["content"] += f"\n{line.strip()}" - # Add non-header lines to current section - if current_section: - sections[current_section].append(line) + # Save last section + if current_section and current_lines: + sections[current_section] = "\n".join(current_lines).strip() - return title, sections # pyright: ignore [reportReturnType] + return title, sections @abstractmethod async def parse_frontmatter(self, frontmatter: Dict[str, Any]) -> Any: @@ -141,4 +161,4 @@ class MarkdownParser(ABC, Generic[T]): @abstractmethod async def create_document(self, frontmatter: Any, content: Any, metadata: Optional[Any]) -> T: """Create document from parsed sections.""" - pass + pass \ No newline at end of file diff --git a/src/basic_memory/markdown/parser.py b/src/basic_memory/markdown/parser.py index 09326975..41d4cd23 100644 --- a/src/basic_memory/markdown/parser.py +++ b/src/basic_memory/markdown/parser.py @@ -10,6 +10,8 @@ from basic_memory.markdown.schemas import ( EntityFrontmatter, EntityContent, EntityMetadata, + Observation, + Relation ) @@ -21,15 +23,9 @@ class EntityParser(MarkdownParser[Entity]): - YAML frontmatter (type, id, created, modified, tags) - Title (# Title) - Optional description - - Observations section (## Observations) with at least one observation + - Observations section (## Observations) - Relations section (## Relations) - Optional metadata section - - Observations format: - - [category] Content text #tag1 #tag2 (optional context) - - Relations format: - - relation_type [[Target Entity]] (optional context) """ async def parse_frontmatter(self, frontmatter: Dict[str, Any]) -> EntityFrontmatter: @@ -46,7 +42,21 @@ class EntityParser(MarkdownParser[Entity]): ParseError: If frontmatter doesn't match schema """ try: - return EntityFrontmatter(**frontmatter) + # Preprocess fields for schema validation + processed = frontmatter.copy() + + # Ensure id is string + if 'id' in processed: + processed['id'] = str(processed['id']) + + # Handle tags field + if 'tags' in processed: + if isinstance(processed['tags'], str): + # Split comma-separated tags and strip whitespace + processed['tags'] = [tag.strip() for tag in processed['tags'].split(',')] + + return EntityFrontmatter(**processed) + except Exception as e: logger.error(f"Invalid entity frontmatter: {e}") raise ParseError(f"Invalid entity frontmatter: {str(e)}") from e @@ -71,11 +81,138 @@ class EntityParser(MarkdownParser[Entity]): if "description" in sections: description = " ".join(sections["description"]) - # Parse observations (required) + # Parse observations (required) + observations = [] + if "observations" not in sections: + raise ParseError("Missing required observations section") + + for line in sections["observations"]: + if line and not line.isspace(): + observation = await self._parse_observation(line) + if observation: + observations.append(observation) + + # Parse relations (optional) + relations = [] + if "relations" in sections: + for line in sections["relations"]: + if line and not line.isspace(): + relation = await self._parse_relation(line) + if relation: + relations.append(relation) + + return EntityContent( + title=title, + description=description, + observations=observations, + relations=relations + ) + + except ParseError: + raise except Exception as e: logger.error(f"Invalid entity content: {e}") raise ParseError(f"Invalid entity content: {str(e)}") from e + async def _parse_observation(self, line: str) -> Optional[Observation]: + """ + Parse a single observation line. + + Format: [category] Content text #tag1 #tag2 (optional context) + """ + if not line or line.isspace(): + return None + + try: + # Extract category if present [category] + category = None + content = line + if line.startswith('['): + end_bracket = line.find(']') + if end_bracket != -1: + category = line[1:end_bracket].strip() + content = line[end_bracket + 1:].strip() + + # Extract context if present (context) + context = None + if content.endswith(')'): + context_start = content.rfind('(') + if context_start != -1: + context = content[context_start + 1:-1].strip() + content = content[:context_start].strip() + + # Extract tags #tag1 #tag2 + tags = [] + content_parts = [] + for part in content.split(): + if part.startswith('#'): + tags.append(part[1:]) # Remove # prefix + else: + content_parts.append(part) + + content = ' '.join(content_parts).strip() + + if not content: + logger.warning(f"Skipping observation with no content: {line}") + return None + + return Observation( + category=category, + content=content, + context=context, + tags=tags if tags else None + ) + + except Exception as e: + logger.warning(f"Failed to parse observation '{line}': {e}") + return None + + async def _parse_relation(self, line: str) -> Optional[Relation]: + """ + Parse a single relation line. + + Format: relation_type [[Target Entity]] (optional context) + """ + if not line or line.isspace(): + return None + + try: + # Extract context if present (context) + context = None + main_part = line + if line.endswith(')'): + context_start = line.rfind('(') + if context_start != -1: + context = line[context_start + 1:-1].strip() + main_part = line[:context_start].strip() + + # Extract relation type and target [[Entity]] + if '[[' not in main_part or ']]' not in main_part: + logger.warning(f"Invalid relation format (missing [[]]): {line}") + return None + + # Split into relation type and target + relation_parts = main_part.split('[[', 1) + relation_type = relation_parts[0].strip() + if not relation_type: + logger.warning(f"Missing relation type: {line}") + return None + + target = relation_parts[1].split(']]')[0].strip() + if not target: + logger.warning(f"Missing target entity: {line}") + return None + + return Relation( + relation_type=relation_type, + target=target, + context=context + ) + + except Exception as e: + logger.warning(f"Failed to parse relation '{line}': {e}") + return None + async def parse_metadata(self, metadata: Optional[Dict[str, Any]]) -> EntityMetadata: """ Parse entity metadata section. @@ -92,13 +229,16 @@ class EntityParser(MarkdownParser[Entity]): try: if not metadata: return EntityMetadata() - return EntityMetadata(**metadata) + return EntityMetadata(metadata=metadata) except Exception as e: logger.error(f"Invalid entity metadata: {e}") raise ParseError(f"Invalid entity metadata: {str(e)}") from e - async def create_document( # pyright: ignore [reportIncompatibleMethodOverride] - self, frontmatter: EntityFrontmatter, content: EntityContent, metadata: EntityMetadata + async def create_document( + self, + frontmatter: EntityFrontmatter, + content: EntityContent, + metadata: EntityMetadata ) -> Entity: """ Create entity from parsed sections. @@ -111,4 +251,8 @@ class EntityParser(MarkdownParser[Entity]): Returns: Complete entity """ - return Entity(frontmatter=frontmatter, content=content, metadata=metadata) + return Entity( + frontmatter=frontmatter, + content=content, + metadata=metadata + ) \ No newline at end of file diff --git a/tests/markdown/test_base_parser.py b/tests/markdown/test_base_parser.py index eb2acb87..0db1b873 100644 --- a/tests/markdown/test_base_parser.py +++ b/tests/markdown/test_base_parser.py @@ -12,7 +12,6 @@ from basic_memory.markdown.base_parser import MarkdownParser, ParseError, FileEr @dataclass class TestDoc: """Simple document for testing.""" - title: str content: str metadata: Optional[Dict[str, Any]] = None @@ -27,18 +26,22 @@ class TestParser(MarkdownParser[TestDoc]): raise ParseError("Missing required title") return frontmatter["title"] - async def parse_content( - self, title: str, sections: Dict[str, List[str]] - ) -> Dict[str, List[str]]: - """Just return content as-is.""" - return sections + async def parse_content(self, title: str, sections: Dict[str, str]) -> str: + """Process sections into content string.""" + if "content" in sections: + return sections["content"] + # Join all section content if no direct content section + return "\n".join(sections.values()) async def parse_metadata(self, metadata: Optional[Dict[str, Any]]) -> Optional[Dict[str, Any]]: """Pass through metadata.""" return metadata async def create_document( - self, frontmatter: str, content: str, metadata: Optional[Dict[str, Any]] + self, + frontmatter: str, + content: str, + metadata: Optional[Dict[str, Any]] ) -> TestDoc: """Create test document.""" return TestDoc(title=frontmatter, content=content, metadata=metadata) @@ -49,8 +52,7 @@ async def test_parse_valid_file(tmp_path: Path): """Test parsing valid file.""" # Create test file test_file = tmp_path / "test.md" - content = """ ---- + content = """--- title: Test Doc metadata: key: value @@ -86,6 +88,7 @@ async def test_parse_invalid_frontmatter(tmp_path: Path): not_title: Test Doc --- +# Title content""" test_file.write_text(content) @@ -98,7 +101,8 @@ content""" async def test_parse_no_frontmatter(tmp_path: Path): """Test file with no frontmatter.""" test_file = tmp_path / "test.md" - content = "Just content" + content = """# Title +Just content""" test_file.write_text(content) parser = TestParser() @@ -113,6 +117,7 @@ async def test_parse_content_str(): title: Test Doc --- +# Title Test content""" parser = TestParser() @@ -120,4 +125,4 @@ Test content""" assert doc.title == "Test Doc" assert doc.content == "Test content" - assert doc.metadata is None + assert doc.metadata is None \ No newline at end of file