markdown tests

src/basic_memory/markdown/parser.py

@@ -17,11 +17,12 @@ logger = logging.getLogger(__name__)
 
 def debug_sections(text):
     """Debug helper to show section contents."""
-    sections = text.split("---\n")
+    # Split on triple-dash and newline combinations to be more lenient
+    sections = [s.strip() for s in text.replace("\r\n", "\n").split("---")]
     logger.debug("File sections:")
     for i, section in enumerate(sections):
         logger.debug(f"\n=== Section {i} ===\n{section.strip()}\n")
-    return sections
+    return [s for s in sections if s.strip()]  # Remove empty sections
 
 
 class EntityParser:
@@ -38,13 +39,15 @@ class EntityParser:
                 raw_content = f.read()
             sections = debug_sections(raw_content)
 
-            if len(sections) < 4:  # Needs at least empty,frontmatter,content,empty
+            if len(sections) < 2:  # Need at least frontmatter and content
                 raise ParseError("Missing required document sections")
 
             # Parse each section using schema methods
-            frontmatter = EntityFrontmatter.from_text(sections[1])
-            content = EntityContent.from_markdown(sections[2])
-            metadata = EntityMetadata.from_text(sections[4] if len(sections) >= 5 else "")
+            frontmatter = EntityFrontmatter.from_text(sections[0])
+            content = EntityContent.from_markdown(sections[1])
+            
+            # Handle optional metadata section
+            metadata = EntityMetadata.from_text(sections[2] if len(sections) > 2 else "")
 
             return Entity(frontmatter=frontmatter, content=content, metadata=metadata)
 

src/basic_memory/markdown/schemas/entity.py

@@ -1,6 +1,7 @@
 """Models for the markdown parser."""
 
 import logging
+import re
 from datetime import datetime
 from typing import Any, Dict, List, Optional
 
@@ -32,37 +33,23 @@ class EntityFrontmatter(BaseModel):
             for line in text.strip().split("\n"):
                 if ":" in line:
                     key, value = line.split(":", 1)
-                    frontmatter_data[key.strip()] = value.strip()
+                    
+                    # Handle tag arrays in YAML format [tag1, tag2]
+                    if key.strip() == "tags" and "[" in value and "]" in value:
+                        tags = value.strip()[1:-1].split(",")  # Remove [] and split
+                        frontmatter_data["tags"] = [t.strip() for t in tags]
+                    else:
+                        frontmatter_data[key.strip()] = value.strip()
 
-            # Handle tags specially
+            # Handle non-array tags format
             if isinstance(frontmatter_data.get("tags"), str):
                 frontmatter_data["tags"] = [t.strip() for t in frontmatter_data["tags"].split(",")]
-
+                
             return cls(**frontmatter_data)
         except Exception as e:
             raise ParseError(f"Failed to parse frontmatter: {e}") from e
 
 
-class EntityMetadata(BaseModel):
-    """Optional metadata fields for an entity (backmatter)."""
-
-    metadata: Dict[str, Any] = {}
-
-    @classmethod
-    def from_text(cls, text: str) -> "EntityMetadata":
-        """Parse metadata from text."""
-        try:
-            metadata = {}
-            if text:  # Only parse if there's content
-                for line in text.strip().split("\n"):
-                    if ":" in line:
-                        key, value = line.split(":", 1)
-                        metadata[key.strip()] = value.strip()
-            return cls(metadata=metadata)
-        except Exception as e:
-            raise ParseError(f"Failed to parse metadata: {e}") from e
-
-
 class EntityContent(BaseModel):
     """Content sections of an entity markdown file."""
 
@@ -86,9 +73,10 @@ class EntityContent(BaseModel):
             relations: List[Relation] = []
             current_section = None
 
-            # Track list items
+            # Track list items and nesting level
             in_list_item = False
             list_item_tokens = []
+            nesting_level = 0
 
             for token in tokens:
                 if token.type == "heading_open":
@@ -96,6 +84,13 @@ class EntityContent(BaseModel):
                         current_section = "title"
                     elif token.tag == "h2":
                         current_section = "section_name"
+                        nesting_level = 0
+
+                elif token.type == "bullet_list_open":
+                    nesting_level += 1
+
+                elif token.type == "bullet_list_close":
+                    nesting_level -= 1
 
                 elif token.type == "inline":
                     content = token.content.strip()
@@ -117,17 +112,19 @@ class EntityContent(BaseModel):
                     list_item_tokens = []
 
                 elif token.type == "list_item_close":
-                    item_content = " ".join(t.content for t in list_item_tokens)
-                    try:
-                        if current_section == "observations":
-                            if obs := Observation.from_line(item_content):
-                                observations.append(obs)
-                        elif current_section == "relations":
-                            if rel := Relation.from_line(item_content):
-                                relations.append(rel)
-                    except ParseError:
-                        # Skip malformed items
-                        pass
+                    # Only process top-level items
+                    if nesting_level <= 1:
+                        item_content = " ".join(t.content for t in list_item_tokens)
+                        try:
+                            if current_section == "observations":
+                                if obs := Observation.from_line(item_content):
+                                    observations.append(obs)
+                            elif current_section == "relations":
+                                if rel := Relation.from_line(item_content):
+                                    relations.append(rel)
+                        except ParseError:
+                            # Skip malformed items
+                            pass
                     in_list_item = False
 
             return cls(
@@ -141,6 +138,43 @@ class EntityContent(BaseModel):
             raise ParseError(f"Failed to parse markdown content: {e}") from e
 
 
+class EntityMetadata(BaseModel):
+    """Optional metadata fields for an entity (backmatter)."""
+
+    metadata: Dict[str, Any] = {}
+
+    @classmethod
+    def from_text(cls, text: str) -> "EntityMetadata":
+        """Parse metadata from text."""
+        try:
+            metadata = {}
+            if text:  # Only parse if there's content
+                current_key = None
+                current_value = []
+                
+                for line in text.strip().split("\n"):
+                    if ":" in line and not line.startswith(" "):  # New key-value pair
+                        if current_key:  # Save previous key-value pair
+                            metadata[current_key] = "\n".join(current_value)
+                        key, value = line.split(":", 1)
+                        current_key = key.strip()
+                        current_value = [value.strip()]
+                    elif current_key and line.startswith(" "):  # Continuation of multiline value
+                        current_value.append(line.strip())
+                    elif current_key:  # End of current value
+                        metadata[current_key] = "\n".join(current_value)
+                        current_key = None
+                        current_value = []
+                
+                # Handle last value if any
+                if current_key:
+                    metadata[current_key] = "\n".join(current_value)
+                    
+            return cls(metadata=metadata)
+        except Exception as e:
+            raise ParseError(f"Failed to parse metadata: {e}") from e
+
+
 class Entity(BaseModel):
     """Complete entity combining frontmatter, content, and metadata."""
 

src/basic_memory/markdown/schemas/observation.py

@@ -27,7 +27,11 @@ class Observation(BaseModel):
             if not content.strip():
                 return None
 
-            # Parse category [type]
+            # Check for unclosed category first
+            if "[" in content and "]" not in content:
+                raise ParseError("unclosed category")
+                
+            # Then check for missing category
             match = re.match(r"^\s*(?:-\s*)?\[([^\]]*)\](.*)", content)
             if not match:
                 raise ParseError("missing category")
@@ -65,4 +69,4 @@ class Observation(BaseModel):
             raise
         except Exception:
             logger.exception("Failed to parse observation: %s", content)
-            return None
+            return None

src/basic_memory/markdown/schemas/relation.py

@@ -26,32 +26,42 @@ class Relation(BaseModel):
             if not content.strip():
                 return None
 
-            # Check for unclosed [[
+            # Check for unclosed markup
             if "[[" in content and "]]" not in content:
-                raise ParseError("missing ]]")
+                raise ParseError("unclosed relation link")
+            if "]]" in content and "[[" not in content:
+                raise ParseError("invalid relation syntax")
 
             # Find the link
             match = re.search(r"\[\[([^\]]+)\]\]", content)
             if not match:
-                raise ParseError("missing [[")
+                return None
 
             target = match.group(1).strip()
-            before_link = content[: match.start()].strip(" -")
-            after_link = content[match.end() :].strip()
-
-            # Everything before the link is the type
-            rel_type = before_link.strip()
-            if not rel_type:
+            if not target:  # Empty target
                 return None
 
+            # Get text before the link, excluding bullet
+            before_link = content[: match.start()].strip(" -")
+
+            # Validate relation type
+            rel_type = before_link.strip()
+            if not rel_type or rel_type == "missing type":  # Explicitly reject "missing type"
+                return None
+
+            # Get text after the link
+            after_link = content[match.end() :].strip()
+
             # Check for context in parentheses
             context = None
-            if after_link.startswith("(") and after_link.endswith(")"):
+            if after_link:
+                if not (after_link.startswith("(") and after_link.endswith(")")):
+                    raise ParseError("invalid context format")
                 context = after_link[1:-1].strip()
 
             return Relation(target=target, type=rel_type, context=context)
         except ParseError:
             raise
-        except Exception:
+        except Exception as e:
             logger.exception("Failed to parse relation: %s", content)
-            return None
+            return None