fixing base_parser tests

src/basic_memory/markdown/base_parser.py

@@ -62,20 +62,31 @@ class MarkdownParser(ABC, Generic[T]):
         """
         try:
             # Split into frontmatter and content
-            frontmatter, remaining = await parse_frontmatter(content)
+            frontmatter, markdown = await parse_frontmatter(content)
+            
+            # Extract metadata from frontmatter if present
+            metadata = frontmatter.pop("metadata", None)
 
-            # Split main content into sections
-            title, sections = self.split_sections(remaining)
+            # Parse frontmatter first
+            parsed_frontmatter = await self.parse_frontmatter(frontmatter)
+
+            # Split remaining content into sections
+            title, sections = self._split_sections(markdown)
             if not title:
                 raise ParseError("Missing title section (must start with #)")
 
-            # Parse each section
-            parsed_frontmatter = await self.parse_frontmatter(frontmatter)
+            # Parse content sections
             parsed_content = await self.parse_content(title, sections)
-            parsed_metadata = await self.parse_metadata(frontmatter.get("metadata"))
 
-            # Create document from parts
-            return await self.create_document(parsed_frontmatter, parsed_content, parsed_metadata)
+            # Parse metadata separately
+            parsed_metadata = await self.parse_metadata(metadata)
+
+            # Create final document
+            return await self.create_document(
+                frontmatter=parsed_frontmatter,
+                content=parsed_content,
+                metadata=parsed_metadata
+            )
 
         except Exception as e:
             if not isinstance(e, ParseError):
@@ -83,7 +94,7 @@ class MarkdownParser(ABC, Generic[T]):
                 raise ParseError(f"Failed to parse content: {str(e)}") from e
             raise
 
-    def split_sections(self, content: str) -> Tuple[str, Dict[str, List[str]]]:
+    def _split_sections(self, content: str) -> Tuple[Optional[str], Dict[str, str]]:
         """
         Split content into sections by headers.
 
@@ -91,37 +102,46 @@ class MarkdownParser(ABC, Generic[T]):
             content: Content section of the document
 
         Returns:
-            Tuple of (title section, {section_name: list of section lines})
+            Tuple of (title section, {section_name: section content})
         """
-        current_section = None
-        sections = {}
+        # Initialize state
         title = None
+        sections: Dict[str, List[str]] = {}
+        current_section = None
+        current_lines: List[str] = []
 
+        # Process each line
         for line in content.splitlines():
-            line = line.strip()
-
-            # Skip empty lines
-            if not line:
+            # Handle headers
+            if line.startswith("# "):  # Top level header (title)
+                title = line[2:].strip()
                 continue
 
-            # Handle headers
-            if line.startswith("#"):
-                # Top level header is title
-                if line.startswith("# "):
-                    title = line[2:].strip()
-                    continue
+            if line.startswith("## "):  # Section header
+                # Save current section if any
+                if current_section and current_lines:
+                    sections[current_section] = "\n".join(current_lines).strip()
+                    current_lines = []
+                
+                # Start new section
+                current_section = line[3:].strip().lower()
+                continue
 
-                # Other headers start new sections
-                if line.startswith("## "):
-                    current_section = line[3:].strip().lower()
-                    sections[current_section] = []
-                    continue
+            # Add line to current section
+            if current_section is not None:
+                current_lines.append(line)
+            elif line.strip() and title:  # Non-empty line after title but before first section
+                # Default section for content right after title
+                if "content" not in sections:
+                    sections["content"] = line.strip()
+                else:
+                    sections["content"] += f"\n{line.strip()}"
 
-            # Add non-header lines to current section
-            if current_section:
-                sections[current_section].append(line)
+        # Save last section
+        if current_section and current_lines:
+            sections[current_section] = "\n".join(current_lines).strip()
 
-        return title, sections  # pyright: ignore [reportReturnType]
+        return title, sections
 
     @abstractmethod
     async def parse_frontmatter(self, frontmatter: Dict[str, Any]) -> Any:
@@ -141,4 +161,4 @@ class MarkdownParser(ABC, Generic[T]):
     @abstractmethod
     async def create_document(self, frontmatter: Any, content: Any, metadata: Optional[Any]) -> T:
         """Create document from parsed sections."""
-        pass
+        pass

src/basic_memory/markdown/parser.py

@@ -10,6 +10,8 @@ from basic_memory.markdown.schemas import (
     EntityFrontmatter,
     EntityContent,
     EntityMetadata,
+    Observation,
+    Relation
 )
 
 
@@ -21,15 +23,9 @@ class EntityParser(MarkdownParser[Entity]):
     - YAML frontmatter (type, id, created, modified, tags)
     - Title (# Title)
     - Optional description
-    - Observations section (## Observations) with at least one observation
+    - Observations section (## Observations)
     - Relations section (## Relations)
     - Optional metadata section
-
-    Observations format:
-    - [category] Content text #tag1 #tag2 (optional context)
-
-    Relations format:
-    - relation_type [[Target Entity]] (optional context)
     """
 
     async def parse_frontmatter(self, frontmatter: Dict[str, Any]) -> EntityFrontmatter:
@@ -46,7 +42,21 @@ class EntityParser(MarkdownParser[Entity]):
             ParseError: If frontmatter doesn't match schema
         """
         try:
-            return EntityFrontmatter(**frontmatter)
+            # Preprocess fields for schema validation
+            processed = frontmatter.copy()
+            
+            # Ensure id is string
+            if 'id' in processed:
+                processed['id'] = str(processed['id'])
+
+            # Handle tags field
+            if 'tags' in processed:
+                if isinstance(processed['tags'], str):
+                    # Split comma-separated tags and strip whitespace
+                    processed['tags'] = [tag.strip() for tag in processed['tags'].split(',')]
+
+            return EntityFrontmatter(**processed)
+            
         except Exception as e:
             logger.error(f"Invalid entity frontmatter: {e}")
             raise ParseError(f"Invalid entity frontmatter: {str(e)}") from e
@@ -71,11 +81,138 @@ class EntityParser(MarkdownParser[Entity]):
             if "description" in sections:
                 description = " ".join(sections["description"])
 
-        # Parse observations (required)
+            # Parse observations (required)
+            observations = []
+            if "observations" not in sections:
+                raise ParseError("Missing required observations section")
+            
+            for line in sections["observations"]:
+                if line and not line.isspace():
+                    observation = await self._parse_observation(line)
+                    if observation:
+                        observations.append(observation)
+
+            # Parse relations (optional)
+            relations = []
+            if "relations" in sections:
+                for line in sections["relations"]:
+                    if line and not line.isspace():
+                        relation = await self._parse_relation(line)
+                        if relation:
+                            relations.append(relation)
+
+            return EntityContent(
+                title=title,
+                description=description,
+                observations=observations,
+                relations=relations
+            )
+
+        except ParseError:
+            raise
         except Exception as e:
             logger.error(f"Invalid entity content: {e}")
             raise ParseError(f"Invalid entity content: {str(e)}") from e
 
+    async def _parse_observation(self, line: str) -> Optional[Observation]:
+        """
+        Parse a single observation line.
+
+        Format: [category] Content text #tag1 #tag2 (optional context)
+        """
+        if not line or line.isspace():
+            return None
+
+        try:
+            # Extract category if present [category]
+            category = None
+            content = line
+            if line.startswith('['):
+                end_bracket = line.find(']')
+                if end_bracket != -1:
+                    category = line[1:end_bracket].strip()
+                    content = line[end_bracket + 1:].strip()
+
+            # Extract context if present (context)
+            context = None
+            if content.endswith(')'):
+                context_start = content.rfind('(')
+                if context_start != -1:
+                    context = content[context_start + 1:-1].strip()
+                    content = content[:context_start].strip()
+
+            # Extract tags #tag1 #tag2
+            tags = []
+            content_parts = []
+            for part in content.split():
+                if part.startswith('#'):
+                    tags.append(part[1:])  # Remove # prefix
+                else:
+                    content_parts.append(part)
+            
+            content = ' '.join(content_parts).strip()
+            
+            if not content:
+                logger.warning(f"Skipping observation with no content: {line}")
+                return None
+
+            return Observation(
+                category=category,
+                content=content,
+                context=context,
+                tags=tags if tags else None
+            )
+
+        except Exception as e:
+            logger.warning(f"Failed to parse observation '{line}': {e}")
+            return None
+
+    async def _parse_relation(self, line: str) -> Optional[Relation]:
+        """
+        Parse a single relation line.
+
+        Format: relation_type [[Target Entity]] (optional context)
+        """
+        if not line or line.isspace():
+            return None
+
+        try:
+            # Extract context if present (context)
+            context = None
+            main_part = line
+            if line.endswith(')'):
+                context_start = line.rfind('(')
+                if context_start != -1:
+                    context = line[context_start + 1:-1].strip()
+                    main_part = line[:context_start].strip()
+
+            # Extract relation type and target [[Entity]]
+            if '[[' not in main_part or ']]' not in main_part:
+                logger.warning(f"Invalid relation format (missing [[]]): {line}")
+                return None
+
+            # Split into relation type and target
+            relation_parts = main_part.split('[[', 1)
+            relation_type = relation_parts[0].strip()
+            if not relation_type:
+                logger.warning(f"Missing relation type: {line}")
+                return None
+
+            target = relation_parts[1].split(']]')[0].strip()
+            if not target:
+                logger.warning(f"Missing target entity: {line}")
+                return None
+
+            return Relation(
+                relation_type=relation_type,
+                target=target,
+                context=context
+            )
+
+        except Exception as e:
+            logger.warning(f"Failed to parse relation '{line}': {e}")
+            return None
+
     async def parse_metadata(self, metadata: Optional[Dict[str, Any]]) -> EntityMetadata:
         """
         Parse entity metadata section.
@@ -92,13 +229,16 @@ class EntityParser(MarkdownParser[Entity]):
         try:
             if not metadata:
                 return EntityMetadata()
-            return EntityMetadata(**metadata)
+            return EntityMetadata(metadata=metadata)
         except Exception as e:
             logger.error(f"Invalid entity metadata: {e}")
             raise ParseError(f"Invalid entity metadata: {str(e)}") from e
 
-    async def create_document(  # pyright: ignore [reportIncompatibleMethodOverride]
-        self, frontmatter: EntityFrontmatter, content: EntityContent, metadata: EntityMetadata
+    async def create_document(
+        self,
+        frontmatter: EntityFrontmatter,
+        content: EntityContent,
+        metadata: EntityMetadata
     ) -> Entity:
         """
         Create entity from parsed sections.
@@ -111,4 +251,8 @@ class EntityParser(MarkdownParser[Entity]):
         Returns:
             Complete entity
         """
-        return Entity(frontmatter=frontmatter, content=content, metadata=metadata)
+        return Entity(
+            frontmatter=frontmatter,
+            content=content,
+            metadata=metadata
+        )