From a0b7e7095110c19bdf91cbaf9833147d6fad9447 Mon Sep 17 00:00:00 2001 From: phernandez Date: Thu, 9 Jan 2025 20:49:49 -0600 Subject: [PATCH] enity parsing tests --- src/basic_memory/markdown/__init__.py | 7 +- src/basic_memory/markdown/entity_parser.py | 377 ++++-------------- src/basic_memory/markdown/knowledge_parser.py | 282 ------------- src/basic_memory/markdown/plugins.py | 230 +++++++++++ src/basic_memory/markdown/schemas.py | 13 +- src/basic_memory/sync/sync_service.py | 6 +- tests/conftest.py | 10 +- tests/markdown/test_debug.py | 49 ++- ...ntity_parsing.py => test_entity_parser.py} | 114 +----- tests/markdown/test_markdown_plugins.py | 163 ++++++++ tests/markdown/test_observation_edge_cases.py | 197 +++++---- tests/markdown/test_parse_observation.py | 88 ---- tests/markdown/test_parse_relation.py | 65 --- tests/markdown/test_parser_edge_cases.py | 257 ++++++------ tests/markdown/test_relation_edge_cases.py | 158 +++++++- 15 files changed, 924 insertions(+), 1092 deletions(-) delete mode 100644 src/basic_memory/markdown/knowledge_parser.py create mode 100644 src/basic_memory/markdown/plugins.py rename tests/markdown/{test_entity_parsing.py => test_entity_parser.py} (52%) create mode 100644 tests/markdown/test_markdown_plugins.py delete mode 100644 tests/markdown/test_parse_observation.py delete mode 100644 tests/markdown/test_parse_relation.py diff --git a/src/basic_memory/markdown/__init__.py b/src/basic_memory/markdown/__init__.py index 72eb0627..5c14d43b 100644 --- a/src/basic_memory/markdown/__init__.py +++ b/src/basic_memory/markdown/__init__.py @@ -1,11 +1,9 @@ """Base package for markdown parsing.""" - -from basic_memory.markdown.knowledge_parser import KnowledgeParser +from basic_memory.markdown.entity_parser import EntityParser from basic_memory.markdown.schemas import ( EntityMarkdown, EntityContent, EntityFrontmatter, - EntityMetadata, Observation, Relation, ) @@ -15,8 +13,7 @@ __all__ = [ "EntityMarkdown", "EntityContent", "EntityFrontmatter", - "EntityMetadata", - "KnowledgeParser", + "EntityParser", "Observation", "Relation", "ParseError", diff --git a/src/basic_memory/markdown/entity_parser.py b/src/basic_memory/markdown/entity_parser.py index 4e04c615..e8909488 100644 --- a/src/basic_memory/markdown/entity_parser.py +++ b/src/basic_memory/markdown/entity_parser.py @@ -1,317 +1,112 @@ -"""Universal parser for markdown files with optional frontmatter, observations, and relations. +"""Parser for markdown files into Entity objects. -The id field in frontmatter is derived from the filename, converted to snake_case with .md extension removed. -For example: - 'My Project Notes.md' -> 'my_project_notes' - 'API-Design.md' -> 'api_design' +Uses markdown-it with plugins to parse structured data from markdown content. """ -import re -from datetime import datetime from pathlib import Path -from typing import Dict, Any, Optional, Tuple, List +from datetime import datetime +from typing import Dict, Any, Optional -from loguru import logger +from markdown_it import MarkdownIt +import frontmatter -from basic_memory.markdown.base_parser import MarkdownParser, ParseError +from basic_memory.markdown.plugins import observation_plugin, relation_plugin from basic_memory.markdown.schemas import ( EntityMarkdown, EntityFrontmatter, - EntityContent, - EntityMetadata, - Observation, - Relation, + EntityContent, Observation, Relation, ) -from basic_memory.schemas.base import to_snake_case -class EntityParser(MarkdownParser[EntityMarkdown]): - """A forgiving parser that extracts as much structure as it can find. - - Generates entity IDs from filenames by: - 1. Removing .md extension - 2. Converting to snake_case - 3. Removing any invalid characters - """ +class EntityParser: + """Parser for markdown files into Entity objects.""" - def convert_to_id(self, filename: str) -> str: - """Convert a filename to a valid entity ID. - - Args: - filename: Name of the file (with or without .md extension) - - Returns: - Snake case version of filename without extension - - Examples: - 'My Project Notes.md' -> 'my_project_notes' - 'API-Design.md' -> 'api_design' + def __init__(self, base_path: Path): + """Initialize parser with base path for relative path_id generation.""" + self.base_path = base_path.resolve() + self.md = (MarkdownIt() + .use(observation_plugin) + .use(relation_plugin)) + + def get_path_id(self, file_path: Path) -> str: + """Get path_id from file path relative to base_path. + + Example: + base_path: /project/root + file_path: /project/root/design/models/data.md + returns: "design/models/data" """ - # Remove .md extension if present - if filename.lower().endswith('.md'): - filename = filename[:-3] - - return to_snake_case(filename) - - def parse_dates(self, frontmatter: Dict[str, Any], file_path: Path) -> Tuple[datetime, datetime]: - """Parse created and updated dates from frontmatter or file system. - - Args: - frontmatter: Dictionary containing frontmatter fields - file_path: Path to the source file for fallback dates - - Returns: - Tuple of (created_date, updated_date) - - Priority: - 1. Valid frontmatter dates - 2. File system dates (created/modified) - """ - created = None - updated = None - - # Try frontmatter first - try: - if 'created' in frontmatter: - created = self.parse_date(frontmatter['created']) - if 'updated' in frontmatter or 'modified' in frontmatter: - updated = self.parse_date(frontmatter.get('updated') or frontmatter.get('modified')) - except Exception as e: - logger.warning(f"Error parsing frontmatter dates: {e}") - - # Fall back to file system dates if needed - try: - stats = file_path.stat() - if not created: - created = datetime.fromtimestamp(stats.st_ctime) - if not updated: - updated = datetime.fromtimestamp(stats.st_mtime) - except Exception as e: - logger.warning(f"Error getting file stats: {e}") - # Last resort - use current time - now = datetime.now() - created = created or now - updated = updated or now - - return created, updated + # Get relative path and remove .md extension + rel_path = file_path.resolve().relative_to(self.base_path) + if rel_path.suffix.lower() == '.md': + return str(rel_path.with_suffix('')) + return str(rel_path) def parse_date(self, value: Any) -> Optional[datetime]: - """Convert various date formats to datetime.""" + """Parse various date formats into datetime.""" if isinstance(value, datetime): return value - try: - if isinstance(value, str): + if isinstance(value, str): + try: return datetime.fromisoformat(value.replace("Z", "+00:00")) - except (ValueError, TypeError): - pass + except (ValueError, TypeError): + pass return None - def parse_tags(self, tags: Any) -> List[str]: - """Convert various tag formats to list of strings.""" + async def parse_file(self, file_path: Path) -> EntityMarkdown: + """Parse markdown file into EntityMarkdown.""" + # Parse frontmatter and content using python-frontmatter + post = frontmatter.load(str(file_path)) + + # Extract or generate required fields + path_id = post.metadata.get("id") or self.get_path_id(file_path) + stats = file_path.stat() + + # Parse frontmatter + entity_frontmatter = EntityFrontmatter( + type=str(post.metadata.get("type", "note")), + id=path_id, + title=str(post.metadata.get("title", file_path.name)), + created=self.parse_date(post.metadata.get("created")) + or datetime.fromtimestamp(stats.st_ctime), + modified=self.parse_date(post.metadata.get("modified")) + or datetime.fromtimestamp(stats.st_mtime), + tags=self.parse_tags(post.metadata.get("tags", [])), + ) + + # Parse content for observations and relations using markdown-it + tokens = self.md.parse(post.content) + + # Extract observations and relations from token meta + observations = [] + relations = [] + + for token in tokens: + if token.meta: # Token might not have meta + if 'observation' in token.meta: + obs = token.meta['observation'] + observation = Observation.model_validate(obs) + observations.append(observation) + if 'relations' in token.meta: + rels = token.meta['relations'] + relations.extend([Relation.model_validate(r) for r in rels]) + + # Create EntityContent + entity_content = EntityContent( + content=post.content, + observations=observations, + relations=relations, + ) + + return EntityMarkdown( + frontmatter=entity_frontmatter, + content=entity_content, + ) + + def parse_tags(self, tags: Any) -> list[str]: + """Parse tags into list of strings.""" if isinstance(tags, str): return [t.strip() for t in tags.split(",") if t.strip()] if isinstance(tags, (list, tuple)): return [str(t).strip() for t in tags if str(t).strip()] return [] - - async def parse_frontmatter(self, frontmatter: Dict[str, Any], file_path: Optional[Path] = None) -> EntityFrontmatter: - """Parse frontmatter with sensible defaults for missing fields. - - Args: - frontmatter: Dictionary of frontmatter fields - file_path: Optional path to source file, used for id and dates - """ - try: - # Get or generate ID from filename - entity_name = None - if file_path: - entity_name = self.convert_to_id(file_path.name) - - # Get dates from frontmatter or file - created, updated = self.parse_dates(frontmatter, file_path) if file_path else (datetime.now(), datetime.now()) - - # Ensure we have minimum required fields - processed = { - "type": str(frontmatter.get("type", "document")).strip(), - "id": str(frontmatter.get("id", entity_name or "document")).strip(), - "created": created, - "modified": updated, - "tags": self.parse_tags(frontmatter.get("tags", [])) - } - - return EntityFrontmatter(**processed) - - except Exception as e: - logger.warning(f"Error parsing frontmatter, using defaults: {e}") - return EntityFrontmatter( - type="document", - id=entity_name or "document", - created=created, - modified=updated, - tags=[] - ) - - async def parse_content(self, title: str, sections: Dict[str, str]) -> EntityContent: - """Parse content sections without requiring any particular structure.""" - try: - # Get content from content section if it exists - content = sections.get("content", "").strip() or None - - # Try to parse observations if they exist - observations = [] - if "observations" in sections: - for line in sections["observations"].splitlines(): - try: - obs = await self.parse_observation(line) - if obs: - observations.append(obs) - except ParseError as e: - logger.warning(f"Skipping invalid observation: {e}") - - # Try to parse relations if they exist - relations = [] - if "relations" in sections: - for line in sections["relations"].splitlines(): - try: - rel = await self.parse_relation(line) - if rel: - relations.append(rel) - except ParseError as e: - logger.warning(f"Skipping invalid relation: {e}") - - # Also look for wiki-links in content as implicit relations - try: - content_relations = await self.parse_content_relations(sections.get("content", "")) - relations.extend(content_relations) - except Exception as e: - logger.warning(f"Error parsing content relations: {e}") - - return EntityContent( - title=title or "Untitled", - summary=content, - observations=observations, - relations=relations - ) - - except Exception as e: - logger.error(f"Error parsing content, using minimal structure: {e}") - return EntityContent( - title=title or "Untitled", - summary=None, - observations=[], - relations=[] - ) - - async def parse_observation(self, line: str) -> Optional[Observation]: - """Parse a single observation line.""" - if not line or not line.strip().startswith("-"): - return None - - line = line.strip()[1:].strip() # Remove leading "-" and whitespace - - # Extract category if present - category = None - if line.startswith("["): - end = line.find("]") - if end != -1: - category = line[1:end].strip() - line = line[end + 1:].strip() - - # Extract context if present - context = None - if line.endswith(")"): - start = line.rfind("(") - if start != -1: - context = line[start + 1:-1].strip() - line = line[:start].strip() - - # Extract tags and content - parts = line.split() - content_parts = [] - tags = [] - - for part in parts: - if part.startswith("#"): - tags.append(part[1:]) - else: - content_parts.append(part) - - content = " ".join(content_parts).strip() - if not content: - return None - - return Observation( - category=category, - content=content, - tags=tags if tags else None, - context=context - ) - - async def parse_relation(self, line: str) -> Optional[Relation]: - """Parse a single relation line.""" - if not line or not line.strip().startswith("-"): - return None - - line = line.strip()[1:].strip() - - # Look for [[target]] - start = line.find("[[") - end = line.find("]]") - if start == -1 or end == -1: - return None - - # Extract parts - rel_type = line[:start].strip() or "relates_to" # Default type if none specified - target = line[start + 2:end].strip() - - # Extract context if present - context = None - remaining = line[end + 2:].strip() - if remaining.startswith("(") and remaining.endswith(")"): - context = remaining[1:-1].strip() - - if not target: - return None - - return Relation( - type=rel_type, - target=target, - context=context - ) - - async def parse_content_relations(self, content: str) -> List[Relation]: - """Extract wiki-style links from content as relations.""" - relations = [] - if not content: - return relations - - import re - pattern = r'\[\[([^\]]+)\]\]' - - for match in re.finditer(pattern, content): - target = match.group(1).strip() - if target: - relations.append(Relation( - type="mentions", - target=target, - context=None - )) - - return relations - - async def parse_metadata(self, metadata_section: Optional[str]) -> EntityMetadata: - """Metadata section is no longer used.""" - return EntityMetadata() - - async def create_document( - self, - frontmatter: EntityFrontmatter, - content: EntityContent, - metadata: EntityMetadata - ) -> EntityMarkdown: - """Create the final EntityMarkdown document.""" - return EntityMarkdown( - frontmatter=frontmatter, - content=content, - entity_metadata=metadata - ) \ No newline at end of file diff --git a/src/basic_memory/markdown/knowledge_parser.py b/src/basic_memory/markdown/knowledge_parser.py deleted file mode 100644 index 6d8fcd4a..00000000 --- a/src/basic_memory/markdown/knowledge_parser.py +++ /dev/null @@ -1,282 +0,0 @@ -"""Parser for Basic Memory entity markdown files.""" - -from datetime import datetime -from typing import Dict, Any, Optional - -import yaml -from loguru import logger - -from basic_memory.markdown.base_parser import MarkdownParser, ParseError -from basic_memory.markdown.schemas import ( - EntityMarkdown, - EntityFrontmatter, - EntityContent, - EntityMetadata, - Observation, - Relation, -) - - -class KnowledgeParser(MarkdownParser[EntityMarkdown]): - """Parser for entity markdown files. - - Entity files must have: - - YAML frontmatter (type, id, created, modified, tags) - - Title (# Title) - - Optional description - - Observations section (## Observations) - - Relations section (## Relations) - - Optional # Metadata section with YAML in code block - - Example Metadata: - ```yml - field: value - other: other value - ``` - """ - - async def parse_metadata(self, metadata_section: Optional[str]) -> EntityMetadata: - """Parse metadata section.""" - try: - if not metadata_section: - logger.debug("No metadata section found") - return EntityMetadata() - - logger.debug(f"Raw metadata section:\n{metadata_section}") - lines = metadata_section.strip().splitlines() - yaml_lines = [] - in_yaml = False - - # Look for ```yml or ```yaml starter - for line in lines: - stripped = line.strip().lower() - if in_yaml: - if stripped == "```": - logger.debug("Found end of YAML block") - break - yaml_lines.append(line) - logger.debug(f"Added YAML line: {line}") - elif stripped in ["```yml", "```yaml"]: - logger.debug("Found start of YAML block") - in_yaml = True - - if not yaml_lines: - logger.debug("No YAML lines found in metadata section") - return EntityMetadata() - - yaml_content = "\n".join(yaml_lines) - logger.debug(f"YAML content to parse:\n{yaml_content}") - - # Parse the YAML content - try: - parsed = yaml.safe_load(yaml_content) - if not isinstance(parsed, dict): - logger.warning(f"Metadata YAML is not a dictionary: {parsed}") - return EntityMetadata() - logger.debug(f"Successfully parsed metadata: {parsed}") - return EntityMetadata(data=parsed) - except yaml.YAMLError as e: - logger.warning(f"Failed to parse metadata YAML: {e}") - return EntityMetadata() - - except Exception as e: - logger.error(f"Failed to parse metadata: {e}") - return EntityMetadata() - - async def parse_frontmatter(self, frontmatter: Dict[str, Any]) -> EntityFrontmatter: - """Parse entity frontmatter.""" - try: - # Check required fields - required_fields = {"type", "id", "created", "modified"} - missing = required_fields - set(frontmatter.keys()) - if missing: - raise ParseError(f"Missing required frontmatter fields: {', '.join(missing)}") - - # Validate date fields - for date_field in ["created", "modified"]: - try: - if not isinstance(frontmatter[date_field], datetime): - datetime.fromisoformat(str(frontmatter[date_field]).replace("Z", "+00:00")) - except (ValueError, TypeError): - raise ParseError( - f"Invalid date format for {date_field}: {frontmatter[date_field]}" - ) - - # Prepare fields - processed = { - "type": frontmatter["type"].strip(), - "id": str(frontmatter["id"]).strip(), - "created": frontmatter["created"], - "modified": frontmatter["modified"], - "tags": ( - [tag.strip() for tag in frontmatter.get("tags", "").split(",")] - if isinstance(frontmatter.get("tags"), str) - else [t.strip() for t in frontmatter.get("tags", [])] - ), - } - - return EntityFrontmatter(**processed) - - except Exception as e: - if isinstance(e, ParseError): - raise - logger.error(f"Invalid entity frontmatter: {e}") - raise ParseError(f"Invalid entity frontmatter: {str(e)}") - - async def parse_content(self, title: str, sections: Dict[str, str]) -> EntityContent: - """Parse entity content section.""" - try: - # Get description (if any) - description = None - if "content" in sections: - description = sections["content"] - - # Parse observations (required) - observations = [] - if "observations" not in sections: - raise ParseError("Missing required observations section") - - for line in sections["observations"].splitlines(): - if line and not line.isspace(): - observation = await self.parse_observation(line) - if observation: - observations.append(observation) - - # Parse relations (optional) - relations = [] - if "relations" in sections: - for line in sections["relations"].splitlines(): - if line and not line.isspace(): - relation = await self.parse_relation(line) - if relation: - relations.append(relation) - - return EntityContent( - title=title, summary=description, observations=observations, relations=relations - ) - - except ParseError: - raise - except Exception as e: - logger.error(f"Invalid entity content: {e}") - raise ParseError(f"Invalid entity content: {str(e)}") - - async def parse_observation(self, line: str) -> Optional[Observation]: - """Parse a single observation line.""" - if not line or line.isspace(): - return None - - try: - # Remove leading/trailing whitespace and bullet - line = line.strip() - if not line.startswith("-"): - return None - line = line[1:].strip() - - category = None - # Handle malformed category brackets first - if line.startswith("]"): - raise ParseError("missing category") - elif line.startswith("["): - # category - close_bracket = line.find("]") - if close_bracket == -1: - raise ParseError("unclosed category") - - # Extract category - return None for empty category - category_string = line[1:close_bracket].strip() - category = category_string if category_string else None - - line = line[close_bracket + 1 :].strip() - - # Extract context if present (context) - context = None - # check if line ends with ")" - if line.endswith(")"): - # find "(" starting from end - last_open = line.rfind("(") - if last_open != -1: - context = line[last_open + 1 : -1].strip() - # remove context from the line - line = line[:last_open].strip() - - # Extract tags and clean content - tags = [] - content_parts = [] - for part in line.split(): - if part.startswith("#"): - # Handle multiple hashtags stuck together - if "#" in part[1:]: - multi_tags = part.split("#") - tags.extend(t for t in multi_tags if t) - else: - tags.append(part[1:]) - else: - content_parts.append(part) - - content = " ".join(content_parts).strip() - if not content: - raise ParseError("Empty content") - - return Observation( - category=category, content=content, tags=tags if tags else None, context=context - ) - - except ParseError: - raise - except Exception as e: - logger.error(f"Failed to parse observation: {e}") - raise ParseError(f"Failed to parse observation: {str(e)}") - - async def parse_relation(self, line: str) -> Optional[Relation]: - """Parse a single relation line.""" - if not line or line.isspace(): - return None - - try: - # Remove leading/trailing whitespace and bullet - line = line.strip() - if not line.startswith("-"): - return None - line = line[1:].strip() - - # Extract context if present (context) - context = None - if line.endswith(")"): - context_start = line.rfind("(") - if context_start != -1: - context = line[context_start + 1 : -1].strip() - line = line[:context_start].strip() - - # Extract relation type and target [[Entity]] - if "[[" not in line or "]]" not in line: - raise ParseError("Invalid relation format - missing [[entity]]") - - # Split into relation type and target - parts = line.split("[[", 1) - rel_type = parts[0].strip() - if not rel_type: - raise ParseError("Missing relation type") - - target_part = parts[1] - close_pos = target_part.find("]]") - if close_pos == -1: - raise ParseError("Unclosed [[ ]] in target") - - target = target_part[:close_pos].strip() - if not target: - raise ParseError("Empty target entity") - - return Relation(type=rel_type, target=target, context=context) - - except ParseError: - raise - except Exception as e: - logger.error(f"Failed to parse relation: {e}") - raise ParseError(f"Failed to parse relation: {str(e)}") - - async def create_document( - self, frontmatter: EntityFrontmatter, content: EntityContent, metadata: EntityMetadata - ) -> EntityMarkdown: - """Create entity from parsed sections.""" - return EntityMarkdown(frontmatter=frontmatter, content=content, entity_metadata=metadata) diff --git a/src/basic_memory/markdown/plugins.py b/src/basic_memory/markdown/plugins.py new file mode 100644 index 00000000..8c918e3c --- /dev/null +++ b/src/basic_memory/markdown/plugins.py @@ -0,0 +1,230 @@ +"""Markdown-it plugins for Basic Memory markdown parsing.""" + +from typing import List, Any, Dict +from markdown_it import MarkdownIt +from markdown_it.token import Token + + +# Observation handling functions +def is_observation(token: Token) -> bool: + """Check if token looks like our observation format.""" + if token.type != 'inline': + return False + + content = token.content.strip() + return ((content.startswith('[') and ']' in content) or # Has category + '#' in content) # Has tags + + +def parse_observation(token: Token) -> Dict[str, Any]: + """Extract observation parts from token.""" + # Strip bullet point if present + content = token.content.strip() + if content.startswith('- '): + content = content[2:].strip() + elif content.startswith('-'): + content = content[1:].strip() + + # Parse [category] + category = None + if content.startswith('['): + end = content.find(']') + if end != -1: + category = content[1:end].strip() + content = content[end + 1:].strip() + + # Parse (context) + context = None + if content.endswith(')'): + start = content.rfind('(') + if start != -1: + context = content[start + 1:-1].strip() + content = content[:start].strip() + + # Parse #tags and content + parts = content.split() + content_parts = [] + tags = set() # Use set to avoid duplicates + + for part in parts: + if part.startswith('#'): + # Handle multiple #tags stuck together + if '#' in part[1:]: + # Split on # but keep non-empty tags + subtags = [t for t in part.split('#') if t] + tags.update(subtags) + else: + tags.add(part[1:]) + else: + content_parts.append(part) + + return { + 'category': category, + 'content': ' '.join(content_parts).strip(), + 'tags': list(tags) if tags else None, + 'context': context + } + + +# Relation handling functions +def is_explicit_relation(token: Token) -> bool: + """Check if token looks like our relation format.""" + if token.type != 'inline': + return False + + content = token.content.strip() + return ('[[' in content and + ']]' in content and + content.index('[[') < content.index(']]')) + + +def parse_relation(token: Token) -> Dict[str, Any]: + """Extract relation parts from token.""" + # Remove bullet point if present + content = token.content.strip() + if content.startswith('- '): + content = content[2:].strip() + elif content.startswith('-'): + content = content[1:].strip() + + # Extract [[target]] + target = None + rel_type = 'relates_to' # default + context = None + + start = content.find('[[') + end = content.find(']]') + + if start != -1 and end != -1: + # Get text before link as relation type + before = content[:start].strip() + if before: + rel_type = before + + # Get target + target = content[start + 2:end].strip() + + # Look for context after + after = content[end + 2:].strip() + if after.startswith('(') and after.endswith(')'): + context = after[1:-1].strip() + + if not target: + return None + + return { + 'type': rel_type, + 'target': target, + 'context': context + } + + +def parse_inline_relations(content: str) -> List[Dict[str, Any]]: + """Find wiki-style links in regular content.""" + relations = [] + + import re + pattern = r'\[\[([^\]]+)\]\]' + + for match in re.finditer(pattern, content): + target = match.group(1).strip() + if target and not target.startswith('[['): # Avoid nested matches + relations.append({ + 'type': 'mentions', + 'target': target, + 'context': None + }) + + return relations + + +def observation_plugin(md: MarkdownIt) -> None: + """Plugin for parsing observation format: + - [category] Content text #tag1 #tag2 (context) + - Content text #tag1 (context) # No category is also valid + """ + + def observation_rule(state: Any) -> None: + """Process observations in token stream.""" + tokens = state.tokens + current_section = None + in_list_item = False + + for idx in range(len(tokens)): + token = tokens[idx] + + # Track current section by headings + if token.type == 'heading_open': + next_token = tokens[idx + 1] if idx + 1 < len(tokens) else None + if next_token and next_token.type == 'inline': + current_section = next_token.content.lower() + + # Track list nesting + elif token.type == 'list_item_open': + in_list_item = True + elif token.type == 'list_item_close': + in_list_item = False + + # Initialize meta for all tokens + token.meta = token.meta or {} + + # Parse observations in list items + if token.type == 'inline' and in_list_item and is_observation(token): + obs = parse_observation(token) + if obs['content']: # Only store if we have content + token.meta['observation'] = obs + + # Add the rule after inline processing + md.core.ruler.after('inline', 'observations', observation_rule) + + +def relation_plugin(md: MarkdownIt) -> None: + """Plugin for parsing relation formats: + + Explicit relations: + - relation_type [[target]] (context) + + Implicit relations (links in content): + Some text with [[target]] reference + """ + + def relation_rule(state: Any) -> None: + """Process relations in token stream.""" + tokens = state.tokens + current_section = None + in_list_item = False + + for idx in range(len(tokens)): + token = tokens[idx] + + # Track current section by headings + if token.type == 'heading_open': + next_token = tokens[idx + 1] if idx + 1 < len(tokens) else None + if next_token and next_token.type == 'inline': + current_section = next_token.content.lower() + + # Track list nesting + elif token.type == 'list_item_open': + in_list_item = True + elif token.type == 'list_item_close': + in_list_item = False + + # Initialize meta for all tokens + token.meta = token.meta or {} + + # Only process inline tokens + if token.type == 'inline': + # Check for explicit relations in list items + if in_list_item and is_explicit_relation(token): + rel = parse_relation(token) + if rel: + token.meta['relations'] = [rel] + + # Always check for inline relations in any text + elif '[[' in token.content: + rels = parse_inline_relations(token.content) + if rels: + token.meta['relations'] = token.meta.get('relations', []) + rels + + # Add the rule after inline processing + md.core.ruler.after('inline', 'relations', relation_rule) diff --git a/src/basic_memory/markdown/schemas.py b/src/basic_memory/markdown/schemas.py index e069e340..34fa75c8 100644 --- a/src/basic_memory/markdown/schemas.py +++ b/src/basic_memory/markdown/schemas.py @@ -26,6 +26,7 @@ class Relation(BaseModel): class EntityFrontmatter(BaseModel): """Required frontmatter fields for an entity.""" + title: str type: str id: str created: datetime @@ -36,18 +37,9 @@ class EntityFrontmatter(BaseModel): class EntityContent(BaseModel): """Content sections of an entity markdown file.""" - title: str - summary: Optional[str] = None + content: Optional[str] = None observations: List[Observation] = [] relations: List[Relation] = [] - context: Optional[str] = None - - -class EntityMetadata(BaseModel): - """Optional metadata for an entity.""" - - # Changed from 'metadata' to 'data' to avoid pydantic special field name - data: Dict[str, Any] = {} class EntityMarkdown(BaseModel): @@ -55,4 +47,3 @@ class EntityMarkdown(BaseModel): frontmatter: EntityFrontmatter content: EntityContent - entity_metadata: EntityMetadata = EntityMetadata() diff --git a/src/basic_memory/sync/sync_service.py b/src/basic_memory/sync/sync_service.py index 4844521d..339e2584 100644 --- a/src/basic_memory/sync/sync_service.py +++ b/src/basic_memory/sync/sync_service.py @@ -5,7 +5,7 @@ from pathlib import Path from loguru import logger from basic_memory.config import ProjectConfig -from basic_memory.markdown import KnowledgeParser +from basic_memory.markdown import EntityParser from basic_memory.services.search_service import SearchService from basic_memory.sync import FileChangeScanner from basic_memory.sync.knowledge_sync_service import KnowledgeSyncService @@ -24,12 +24,12 @@ class SyncService: self, scanner: FileChangeScanner, knowledge_sync_service: KnowledgeSyncService, - knowledge_parser: KnowledgeParser, + entity_parser: EntityParser, search_service: SearchService, ): self.scanner = scanner self.knowledge_sync_service = knowledge_sync_service - self.knowledge_parser = knowledge_parser + self.knowledge_parser = entity_parser self.search_service = search_service diff --git a/tests/conftest.py b/tests/conftest.py index 121d1f1f..1fa6e979 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -10,7 +10,7 @@ from sqlalchemy.ext.asyncio import AsyncSession, AsyncEngine, async_sessionmaker from basic_memory import db from basic_memory.config import ProjectConfig from basic_memory.db import DatabaseType -from basic_memory.markdown.knowledge_parser import KnowledgeParser +from basic_memory.markdown import EntityParser from basic_memory.markdown.knowledge_writer import KnowledgeWriter from basic_memory.models import Base from basic_memory.models.knowledge import Entity @@ -130,9 +130,9 @@ def knowledge_writer(): @pytest.fixture -def knowledge_parser(): +def entity_parser(test_config): """Create parser instance.""" - return KnowledgeParser() + return EntityParser(test_config.home) @pytest_asyncio.fixture @@ -181,14 +181,14 @@ async def knowledge_sync_service( async def sync_service( knowledge_sync_service: KnowledgeSyncService, file_change_scanner: FileChangeScanner, - knowledge_parser: KnowledgeParser, + entity_parser: EntityParser, search_service: SearchService, ) -> SyncService: """Create sync service for testing.""" return SyncService( scanner=file_change_scanner, knowledge_sync_service=knowledge_sync_service, - knowledge_parser=knowledge_parser, + knowledge_parser=entity_parser, search_service=search_service, ) diff --git a/tests/markdown/test_debug.py b/tests/markdown/test_debug.py index a671e2fd..aad27a83 100644 --- a/tests/markdown/test_debug.py +++ b/tests/markdown/test_debug.py @@ -1,25 +1,34 @@ -"""Test to explore markdown-it token structure.""" -from textwrap import dedent +"""Debug markdown-it token structure.""" + from markdown_it import MarkdownIt +from basic_memory.markdown.plugins import observation_plugin, parse_observation, is_observation -def test_token_structure(): - """Analyze markdown-it token structure.""" - content = dedent(''' - # Title - - ## Observations - - [test] First line #tag - - [another] Second line #tag2 - ''') - - md = MarkdownIt() +def test_debug_observations(): + """Debug observation parsing.""" + md = MarkdownIt().use(observation_plugin) + + content = "- [design] Core feature #important #mvp" tokens = md.parse(content) - - # Print full token structure - print("\nToken structure:") + + # Print token info + print("\nTokens:") for i, token in enumerate(tokens): - attrs = {key: getattr(token, key) for key in dir(token) - if not key.startswith('_') and not callable(getattr(token, key))} print(f"\nToken {i}:") - for key, value in attrs.items(): - print(f" {key}: {value!r}") + print(f" Type: {token.type}") + print(f" Tag: {token.tag}") + print(f" Content: {token.content}") + print(f" Nesting: {token.nesting}") + if hasattr(token, 'meta'): + print(f" Meta: {token.meta}") + + # Try the functions directly + token = next(t for t in tokens if t.type == 'inline') + print("\nTesting observation functions:") + print(f"Is observation: {is_observation(token)}") + obs = parse_observation(token) + print(f"Parsed observation: {obs}") + + # Verify meta was set + print("\nVerifying meta:") + token = next(t for t in tokens if t.meta and 'observation' in t.meta) + print(f"Meta observation: {token.meta}") diff --git a/tests/markdown/test_entity_parsing.py b/tests/markdown/test_entity_parser.py similarity index 52% rename from tests/markdown/test_entity_parsing.py rename to tests/markdown/test_entity_parser.py index 3c4d706b..b8c5ded2 100644 --- a/tests/markdown/test_entity_parsing.py +++ b/tests/markdown/test_entity_parser.py @@ -5,7 +5,7 @@ from textwrap import dedent import pytest -from basic_memory.markdown.knowledge_parser import KnowledgeParser +from basic_memory.markdown.entity_parser import EntityParser from basic_memory.markdown.schemas import EntityMarkdown, EntityFrontmatter, EntityContent from basic_memory.utils.file_utils import ParseError, FileError @@ -15,6 +15,7 @@ def valid_entity_content(): """A complete, valid entity file with all features.""" return dedent(""" --- + title: Auth Service type: component id: auth_service created: 2024-12-21T14:00:00Z @@ -22,8 +23,6 @@ def valid_entity_content(): tags: authentication, security, core --- - # Auth Service - Core authentication service that handles user authentication. ## Observations @@ -35,24 +34,16 @@ def valid_entity_content(): - implements [[OAuth Implementation]] (Core auth flows) - uses [[Redis Cache]] (Token caching) - specified_by [[Auth API Spec]] (OpenAPI spec) - - # Metadata - - - ```yml - owner: team-auth - priority: high - ``` """) @pytest.mark.asyncio -async def test_parse_complete_file(tmp_path, valid_entity_content): +async def test_parse_complete_file(test_config, valid_entity_content): """Test parsing a complete entity file with all features.""" - test_file = tmp_path / "test_entity.md" + test_file = test_config.home / "test_entity.md" test_file.write_text(valid_entity_content) - parser = KnowledgeParser() + parser = EntityParser(test_config.home) entity = await parser.parse_file(test_file) # Verify entity structure @@ -61,16 +52,13 @@ async def test_parse_complete_file(tmp_path, valid_entity_content): assert isinstance(entity.content, EntityContent) # Check frontmatter + assert entity.frontmatter.title == "Auth Service" assert entity.frontmatter.type == "component" assert entity.frontmatter.id == "auth_service" assert set(entity.frontmatter.tags) == {"authentication", "security", "core"} # Check content - assert entity.content.title == "Auth Service" - assert ( - entity.content.summary - == "Core authentication service that handles user authentication." - ) + assert "Core authentication service that handles user authentication." in entity.content.content # Check observations assert len(entity.content.observations) == 3 @@ -87,9 +75,6 @@ async def test_parse_complete_file(tmp_path, valid_entity_content): assert rel.target == "OAuth Implementation" assert rel.context == "Core auth flows" - # Check metadata - assert entity.entity_metadata.data["owner"] == "team-auth" - @pytest.mark.asyncio async def test_parse_minimal_file(tmp_path): @@ -115,106 +100,29 @@ async def test_parse_minimal_file(tmp_path): test_file = tmp_path / "minimal.md" test_file.write_text(content) - parser = KnowledgeParser() + parser = EntityParser(tmp_path) entity = await parser.parse_file(test_file) assert entity.frontmatter.type == "component" assert entity.frontmatter.id == "minimal" assert len(entity.content.observations) == 1 assert len(entity.content.relations) == 1 - assert not entity.entity_metadata.data # Empty metadata -@pytest.mark.asyncio -async def test_parse_content_str(valid_entity_content): - """Test parsing content string directly.""" - parser = KnowledgeParser() - entity = await parser.parse_content_str(valid_entity_content) - - assert isinstance(entity, EntityMarkdown) - assert entity.frontmatter.type == "component" - assert entity.content.title == "Auth Service" - assert len(entity.content.observations) == 3 - assert len(entity.content.relations) == 3 - - -@pytest.mark.asyncio -async def test_metadata_handling(tmp_path): - """Test metadata section parsing.""" - parser = KnowledgeParser() - - # Multiple metadata fields - content = dedent(""" - --- - type: component - id: test - created: 2024-12-21T14:00:00Z - modified: 2024-12-21T14:00:00Z - tags: [] - --- - # Test Entity - ## Observations - - [test] Test - - # Metadata - - - ```yml - owner: test-team - priority: high - nested: - key: value - list: [1, 2, 3] - ``` - """) - test_file = tmp_path / "metadata.md" - test_file.write_text(content) - - entity = await parser.parse_file(test_file) - assert entity.entity_metadata.data["owner"] == "test-team" - assert entity.entity_metadata.data["priority"] == "high" - assert entity.entity_metadata.data["nested"]["key"] == "value" - assert entity.entity_metadata.data["nested"]["list"] == [1, 2, 3] - - # No metadata section - content = dedent(""" - --- - type: component - id: test - created: 2024-12-21T14:00:00Z - modified: 2024-12-21T14:00:00Z - tags: [] - --- - # Test Entity - ## Observations - - [test] Test - """) - test_file = tmp_path / "no_metadata.md" - test_file.write_text(content) - - entity = await parser.parse_file(test_file) - assert entity.entity_metadata.data == {} @pytest.mark.asyncio async def test_error_handling(tmp_path): """Test error handling.""" - parser = KnowledgeParser() + parser = EntityParser(tmp_path) # Missing file - with pytest.raises(FileError): + with pytest.raises(FileNotFoundError): await parser.parse_file(Path("nonexistent.md")) # Invalid file encoding test_file = tmp_path / "binary.md" with open(test_file, "wb") as f: f.write(b"\x80\x81") # Invalid UTF-8 - with pytest.raises(ParseError): - await parser.parse_file(test_file) - - # No frontmatter section - content = "# Just a title\nNo frontmatter" - test_file = tmp_path / "no_frontmatter.md" - test_file.write_text(content) - with pytest.raises(ParseError): + with pytest.raises(UnicodeDecodeError): await parser.parse_file(test_file) diff --git a/tests/markdown/test_markdown_plugins.py b/tests/markdown/test_markdown_plugins.py new file mode 100644 index 00000000..a2b64cae --- /dev/null +++ b/tests/markdown/test_markdown_plugins.py @@ -0,0 +1,163 @@ +"""Tests for markdown-it plugins.""" + +import pytest +from markdown_it import MarkdownIt + +from basic_memory.markdown.plugins import observation_plugin, relation_plugin + + +def test_observation_plugin(): + """Test observation plugin parsing.""" + md = MarkdownIt().use(observation_plugin) + + # Basic observation + tokens = md.parse("- [design] Core feature #important #mvp") + obs_token = next(t for t in tokens if t.meta and 'observation' in t.meta) + obs = obs_token.meta['observation'] + + assert obs['category'] == 'design' + assert obs['content'] == 'Core feature' + assert set(obs['tags']) == {'important', 'mvp'} + assert obs['context'] is None + + # With context + tokens = md.parse("- [feature] Authentication system #security (Required for MVP)") + obs_token = next(t for t in tokens if t.meta and 'observation' in t.meta) + obs = obs_token.meta['observation'] + + assert obs['category'] == 'feature' + assert obs['content'] == 'Authentication system' + assert set(obs['tags']) == {'security'} + assert obs['context'] == 'Required for MVP' + + # Without category + tokens = md.parse("- Authentication system #security (Required for MVP)") + obs_token = next(t for t in tokens if t.meta and 'observation' in t.meta) + obs = obs_token.meta['observation'] + + assert obs['category'] is None + assert obs['content'] == 'Authentication system' + assert set(obs['tags']) == {'security'} + assert obs['context'] == 'Required for MVP' + + +def test_observation_edge_cases(): + """Test observation plugin edge cases.""" + md = MarkdownIt().use(observation_plugin) + + # Multiple word tags + tokens = md.parse("- [tech] Database #high-priority #needs-review") + obs_token = next(t for t in tokens if t.meta and 'observation' in t.meta) + obs = obs_token.meta['observation'] + assert set(obs['tags']) == {'high-priority', 'needs-review'} + + # Multiple word category + tokens = md.parse("- [user experience] Design #ux") + obs_token = next(t for t in tokens if t.meta and 'observation' in t.meta) + obs = obs_token.meta['observation'] + assert obs['category'] == 'user experience' + + # Parentheses in content + tokens = md.parse("- [code] Function (x) returns y #function") + obs_token = next(t for t in tokens if t.meta and 'observation' in t.meta) + obs = obs_token.meta['observation'] + assert obs['content'] == 'Function (x) returns y' + assert obs['context'] is None + + # Multiple hashtags together + tokens = md.parse("- [test] Feature #important#urgent#now") + obs_token = next(t for t in tokens if t.meta and 'observation' in t.meta) + obs = obs_token.meta['observation'] + assert set(obs['tags']) == {'important', 'urgent', 'now'} + + +def test_relation_plugin(): + """Test relation plugin parsing.""" + md = MarkdownIt().use(relation_plugin) + + # Basic relation + tokens = md.parse("- implements [[Auth Service]]") + rel_token = next(t for t in tokens if t.meta and 'relations' in t.meta) + rel = rel_token.meta['relations'][0] + + assert rel['type'] == 'implements' + assert rel['target'] == 'Auth Service' + assert rel['context'] is None + + # With context + tokens = md.parse("- depends_on [[Database]] (Required for persistence)") + rel_token = next(t for t in tokens if t.meta and 'relations' in t.meta) + rel = rel_token.meta['relations'][0] + + assert rel['type'] == 'depends_on' + assert rel['target'] == 'Database' + assert rel['context'] == 'Required for persistence' + + +def test_relation_edge_cases(): + """Test relation plugin edge cases.""" + md = MarkdownIt().use(relation_plugin) + + # Multiple word type + tokens = md.parse("- is used by [[Client App]] (Primary consumer)") + rel_token = next(t for t in tokens if t.meta and 'relations' in t.meta) + rel = rel_token.meta['relations'][0] + assert rel['type'] == 'is used by' + + # Extra spaces + tokens = md.parse("- specifies [[Format]] (Documentation)") + rel_token = next(t for t in tokens if t.meta and 'relations' in t.meta) + rel = rel_token.meta['relations'][0] + assert rel['type'] == 'specifies' + assert rel['target'] == 'Format' + + +def test_inline_relations(): + """Test finding relations in regular content.""" + md = MarkdownIt().use(relation_plugin) + + # Single inline link + tokens = md.parse("This references [[Another Doc]].") + rel_token = next(t for t in tokens if t.meta and 'relations' in t.meta) + rel = rel_token.meta['relations'][0] + assert rel['type'] == 'mentions' + assert rel['target'] == 'Another Doc' + + # Multiple inline links + tokens = md.parse("Links to [[Doc1]] and [[Doc2]].") + rel_token = next(t for t in tokens if t.meta and 'relations' in t.meta) + rels = rel_token.meta['relations'] + assert len(rels) == 2 + assert {r['target'] for r in rels} == {'Doc1', 'Doc2'} + + +def test_combined_plugins(): + """Test both plugins working together.""" + md = (MarkdownIt() + .use(observation_plugin) + .use(relation_plugin)) + + content = """# Document + +Some text with a [[Link]]. + +## Observations +- [tech] Implements [[Feature]] #important +- [design] References [[AnotherDoc]] (For context) +""" + + tokens = md.parse(content) + + # Should find both observations and relations + observations = [t.meta['observation'] for t in tokens if t.meta and 'observation' in t.meta] + relations = [r for t in tokens if t.meta and 'relations' in t.meta for r in t.meta['relations']] + + assert len(observations) == 2 + assert any(o['category'] == 'tech' and 'important' in o['tags'] for o in observations) + assert any(o['category'] == 'design' and o['context'] == 'For context' for o in observations) + + # Should find all relations (Feature, AnotherDoc, and Link) + assert len(relations) == 3 + assert any(r['target'] == 'Link' and r['type'] == 'mentions' for r in relations) + assert any(r['target'] == 'Feature' for r in relations) + assert any(r['target'] == 'AnotherDoc' for r in relations) \ No newline at end of file diff --git a/tests/markdown/test_observation_edge_cases.py b/tests/markdown/test_observation_edge_cases.py index 485df4d8..9dc68fd9 100644 --- a/tests/markdown/test_observation_edge_cases.py +++ b/tests/markdown/test_observation_edge_cases.py @@ -1,95 +1,142 @@ """Tests for edge cases in observation parsing.""" import pytest +from markdown_it import MarkdownIt -from basic_memory.markdown import ParseError, KnowledgeParser +from basic_memory.markdown.plugins import ( + observation_plugin, + is_observation, + parse_observation +) +from basic_memory.markdown.schemas import Observation -@pytest.mark.asyncio -async def test_observation_empty_input(): +def test_empty_input(): """Test handling of empty input.""" - parser = KnowledgeParser() - assert await parser.parse_observation("") is None - assert await parser.parse_observation(" ") is None - assert await parser.parse_observation("\n") is None + md = MarkdownIt().use(observation_plugin) + + tokens = md.parse("") + assert not any(t.meta and 'observation' in t.meta for t in tokens) + + tokens = md.parse(" ") + assert not any(t.meta and 'observation' in t.meta for t in tokens) + + tokens = md.parse("\n") + assert not any(t.meta and 'observation' in t.meta for t in tokens) -@pytest.mark.asyncio -async def test_observation_invalid_context(): +def test_invalid_context(): """Test handling of invalid context format.""" - parser = KnowledgeParser() - obs = await parser.parse_observation("- [test] Content (unclosed") - assert obs is not None - assert obs.content == "Content (unclosed" - assert obs.context is None - - obs = await parser.parse_observation("- [test] Content (with) extra) parens)") - assert obs is not None - assert obs.content == "Content" - assert obs.context == "with) extra) parens" + md = MarkdownIt().use(observation_plugin) + + # Unclosed context + tokens = md.parse("- [test] Content (unclosed") + token = next(t for t in tokens if t.type == 'inline') + obs = parse_observation(token) + assert obs['content'] == "Content (unclosed" + assert obs['context'] is None + + # Multiple parens + tokens = md.parse("- [test] Content (with) extra) parens)") + token = next(t for t in tokens if t.type == 'inline') + obs = parse_observation(token) + assert obs['content'] == "Content" + assert obs['context'] == "with) extra) parens" -@pytest.mark.asyncio -async def test_observation_complex_format(): +def test_complex_format(): """Test parsing complex observation formats.""" - # Test multiple nested tags and spaces - parser = KnowledgeParser() - obs = await parser.parse_observation("- [complex test] This is #tag1#tag2 with #tag3 content") - assert obs is not None - assert obs.category == "complex test" - assert set(obs.tags) == {"tag1", "tag2", "tag3"} # pyright: ignore [reportArgumentType] - assert obs.content == "This is with content" + md = MarkdownIt().use(observation_plugin) + + # Multiple hashtags together + tokens = md.parse("- [complex test] This is #tag1#tag2 with #tag3 content") + token = next(t for t in tokens if t.type == 'inline') + + obs = parse_observation(token) + assert obs['category'] == "complex test" + assert set(obs['tags']) == {"tag1", "tag2", "tag3"} + assert obs['content'] == "This is with content" + + # Pydantic model validation + observation = Observation.model_validate(obs) + assert observation.category == "complex test" + assert set(observation.tags) == {"tag1", "tag2", "tag3"} + assert observation.content == "This is with content" -@pytest.mark.asyncio -async def test_observation_exception_handling(): - """Test general error handling in observation parsing.""" - # Test with a problematic regex pattern that could cause catastrophic backtracking - long_input = "[test] " + "a" * 1000000 # Very long input - parser = KnowledgeParser() - - assert await parser.parse_observation(long_input) is None - - # Test with invalid types - assert await parser.parse_observation(None) is None # type: ignore - - -@pytest.mark.asyncio -async def test_observation_malformed_category(): +def test_malformed_category(): """Test handling of malformed category brackets.""" - parser = KnowledgeParser() - with pytest.raises(ParseError, match="unclosed category"): - await parser.parse_observation("- [test Content") + md = MarkdownIt().use(observation_plugin) + + # Empty category + tokens = md.parse("- [] Empty category") + token = next(t for t in tokens if t.type == 'inline') + obs = parse_observation(token) + assert obs['category'] is None + assert obs['content'] == "Empty category" + + # Missing close bracket + tokens = md.parse("- [test Content") + token = next(t for t in tokens if t.type == 'inline') + obs = parse_observation(token) + # Should treat whole thing as content + assert obs['category'] is None + assert "test Content" in obs['content'] -@pytest.mark.asyncio -async def test_observation_empty_category(): - parser = KnowledgeParser() - obs = await parser.parse_observation("- [] Empty category") - - assert obs is not None - assert obs.category is None - - -@pytest.mark.asyncio -async def test_observation_whitespace(): - """Test handling of whitespace.""" - # Valid whitespace cases - parser = KnowledgeParser() - obs = await parser.parse_observation("- [test] Content") - assert obs is not None - assert obs.content == "Content" - - # Test individual whitespace chars - test_chars = { - " ": "space", - "\t": "tab", - #'\n': 'newline', # newline should be end of content - "\r": "return", - } - +def test_whitespace_handling(): + """Test handling of various whitespace.""" + md = MarkdownIt().use(observation_plugin) + + # Various whitespace in content + test_chars = {" ": "space", "\t": "tab", "\r": "return"} for char, name in test_chars.items(): content = f"- [test] Content{char}with{char}{name}" - obs = await parser.parse_observation(content) - assert obs is not None - assert obs.content == f"Content with {name}" + tokens = md.parse(content) + token = next(t for t in tokens if t.type == 'inline') + obs = parse_observation(token) + assert obs['content'] == f"Content with {name}" + + # Indented observation + tokens = md.parse(" - [test] Indented") + assert any(t.meta and 'observation' in t.meta for t in tokens) + + # Multiple blank lines + text = """ + + - [test] After blanks + + """ + tokens = md.parse(text) + assert any(t.meta and 'observation' in t.meta for t in tokens) + + +def test_unicode_content(): + """Test handling of Unicode content.""" + md = MarkdownIt().use(observation_plugin) + + # Emoji + tokens = md.parse("- [test] Emoji test 👍 #emoji #test (Testing emoji)") + token = next(t for t in tokens if t.type == 'inline') + obs = parse_observation(token) + assert "👍" in obs['content'] + assert "emoji" in obs['tags'] + + # Non-Latin scripts + tokens = md.parse("- [中文] Chinese text 测试 #language (Script test)") + token = next(t for t in tokens if t.type == 'inline') + obs = parse_observation(token) + assert obs['category'] == "中文" + assert "测试" in obs['content'] + + # Mixed scripts and emoji + tokens = md.parse("- [test] Mixed 中文 and 👍 #mixed") + token = next(t for t in tokens if t.type == 'inline') + obs = parse_observation(token) + assert "中文" in obs['content'] + assert "👍" in obs['content'] + + # Model validation with Unicode + observation = Observation.model_validate(obs) + assert "中文" in observation.content + assert "👍" in observation.content \ No newline at end of file diff --git a/tests/markdown/test_parse_observation.py b/tests/markdown/test_parse_observation.py deleted file mode 100644 index 7dc42c05..00000000 --- a/tests/markdown/test_parse_observation.py +++ /dev/null @@ -1,88 +0,0 @@ -"""Tests for the markdown entity parser.""" - -import pytest - -from basic_memory.markdown.knowledge_parser import KnowledgeParser, ParseError - - -@pytest.mark.asyncio -async def test_parse_observation_basic(): - """Test basic observation parsing with category and tags.""" - parser = KnowledgeParser() - - obs = await parser.parse_observation("- [design] Core feature #important #mvp") - - assert obs is not None - assert obs.category == "design" - assert obs.content == "Core feature" - assert set(obs.tags) == {"important", "mvp"} # pyright: ignore [reportArgumentType] - assert obs.context is None - - -@pytest.mark.asyncio -async def test_parse_observation_with_context(): - """Test observation parsing with context in parentheses.""" - parser = KnowledgeParser() - - obs = await parser.parse_observation( - "- [feature] Authentication system #security #auth (Required for MVP)" - ) - assert obs is not None - assert obs.category == "feature" - assert obs.content == "Authentication system" - assert set(obs.tags) == {"security", "auth"} # pyright: ignore [reportArgumentType] - assert obs.context == "Required for MVP" - - -@pytest.mark.asyncio -async def test_parse_observation_without_category(): - """Test observation parsing with context in parentheses.""" - parser = KnowledgeParser() - - obs = await parser.parse_observation( - "- Authentication system #security #auth (Required for MVP)" - ) - assert obs is not None - assert obs.category is None - assert obs.content == "Authentication system" - assert set(obs.tags) == {"security", "auth"} # pyright: ignore [reportArgumentType] - assert obs.context == "Required for MVP" - - -@pytest.mark.asyncio -async def test_parse_observation_edge_cases(): - """Test observation parsing edge cases.""" - parser = KnowledgeParser() - - # Multiple word tags - obs = await parser.parse_observation("- [tech] Database #high-priority #needs-review") - - assert obs is not None - - assert set(obs.tags) == {"high-priority", "needs-review"} # pyright: ignore [reportArgumentType] - - # Multiple word category - obs = await parser.parse_observation("- [user experience] Design #ux") - assert obs is not None - assert obs.category == "user experience" - - # Parentheses in content shouldn't be treated as context - obs = await parser.parse_observation("- [code] Function (x) returns y #function") - assert obs is not None - assert obs.content == "Function (x) returns y" - assert obs.context is None - - # Multiple hashtags together - obs = await parser.parse_observation("- [test] Feature #important#urgent#now") - assert obs is not None - assert set(obs.tags) == {"important", "urgent", "now"} # pyright: ignore [reportArgumentType] - - -@pytest.mark.asyncio -async def test_parse_observation_errors(): - """Test error handling in observation parsing.""" - parser = KnowledgeParser() - - # Unclosed category - with pytest.raises(ParseError, match="unclosed category"): - await parser.parse_observation("- [design Core feature #test") diff --git a/tests/markdown/test_parse_relation.py b/tests/markdown/test_parse_relation.py deleted file mode 100644 index 1160889f..00000000 --- a/tests/markdown/test_parse_relation.py +++ /dev/null @@ -1,65 +0,0 @@ -"""Tests for the markdown entity parser.""" - -import pytest - -from basic_memory.markdown.knowledge_parser import KnowledgeParser, ParseError - - -@pytest.mark.asyncio -async def test_parse_relation_basic(): - """Test basic relation parsing.""" - parser = KnowledgeParser() - - rel = await parser.parse_relation("- implements [[Auth Service]]") - assert rel is not None - assert rel.type == "implements" - assert rel.target == "Auth Service" - assert rel.context is None - - -@pytest.mark.asyncio -async def test_parse_relation_with_context(): - """Test relation parsing with context.""" - parser = KnowledgeParser() - - rel = await parser.parse_relation("- depends_on [[Database]] (Required for persistence)") - assert rel is not None - assert rel.type == "depends_on" - assert rel.target == "Database" - assert rel.context == "Required for persistence" - - -@pytest.mark.asyncio -async def test_parse_relation_edge_cases(): - """Test relation parsing edge cases.""" - parser = KnowledgeParser() - - # Multiple word type - rel = await parser.parse_relation("- is used by [[Client App]] (Primary consumer)") - assert rel is not None - assert rel.type == "is used by" - - # Brackets in context - rel = await parser.parse_relation("- implements [[API]] (Follows [OpenAPI] spec)") - assert rel is not None - assert rel.context == "Follows [OpenAPI] spec" - - # Extra spaces - rel = await parser.parse_relation("- specifies [[Format]] (Documentation)") - assert rel is not None - assert rel.type == "specifies" - assert rel.target == "Format" - - -@pytest.mark.asyncio -async def test_parse_relation_errors(): - """Test error handling in relation parsing.""" - parser = KnowledgeParser() - - # Missing target brackets - with pytest.raises(ParseError, match="missing \\[\\["): - await parser.parse_relation("- implements Auth Service") - - # Unclosed target - with pytest.raises(ParseError, match="Invalid relation format - missing \\[\\[entity\\]\\]"): - await parser.parse_relation("- implements [[Auth Service") diff --git a/tests/markdown/test_parser_edge_cases.py b/tests/markdown/test_parser_edge_cases.py index 2172e75e..9f0f0cc6 100644 --- a/tests/markdown/test_parser_edge_cases.py +++ b/tests/markdown/test_parser_edge_cases.py @@ -4,9 +4,10 @@ from pathlib import Path from textwrap import dedent import pytest +from markdown_it import MarkdownIt -from basic_memory.markdown.knowledge_parser import KnowledgeParser -from basic_memory.utils.file_utils import FileError, ParseError +from basic_memory.markdown.entity_parser import EntityParser +from basic_memory.utils.file_utils import FileError @pytest.mark.asyncio @@ -20,164 +21,154 @@ async def test_unicode_content(tmp_path): modified: 2024-12-21T14:00:00Z tags: [unicode, 测试] --- - + # Unicode Test 🧪 - + ## Observations - [test] Emoji test 👍 #emoji #test (Testing emoji) - [中文] Chinese text 测试 #language (Script test) - [русский] Russian привет #language (More scripts) - [note] Emoji in text 😀 #meta (Category test) - + ## Relations - tested_by [[测试组件]] (Unicode test) - depends_on [[компонент]] (Another test) - - ## Metadata - ```yml - category: test - status: active - ``` """) - + test_file = tmp_path / "unicode.md" test_file.write_text(content, encoding="utf-8") - - parser = KnowledgeParser() + + parser = EntityParser(tmp_path) entity = await parser.parse_file(test_file) - + assert "测试" in entity.frontmatter.tags - assert "китайский" not in entity.frontmatter.tags - assert entity.content.title == "Unicode Test 🧪" + assert "chinese" not in entity.frontmatter.tags + assert "🧪" in entity.content.content + + # Verify Unicode in observations + assert any(o.content == "Emoji test 👍" for o in entity.content.observations) + assert any(o.category == "中文" for o in entity.content.observations) + assert any(o.category == "русский" for o in entity.content.observations) + + # Verify Unicode in relations + assert any(r.target == "测试组件" for r in entity.content.relations) + assert any(r.target == "компонент" for r in entity.content.relations) @pytest.mark.asyncio -async def test_fallback_encoding(tmp_path): - """Test UTF-16 fallback when UTF-8 fails.""" +async def test_empty_file(tmp_path): + """Test handling of empty files.""" + empty_file = tmp_path / "empty.md" + empty_file.write_text("") + + parser = EntityParser(tmp_path) + entity = await parser.parse_file(empty_file) + assert entity.content.observations == [] + assert entity.content.relations == [] + + +@pytest.mark.asyncio +async def test_missing_sections(tmp_path): + """Test handling of files with missing sections.""" content = dedent(""" - Hello 世界 - No proper sections here + --- + type: test + id: test/missing + created: 2024-01-09 + modified: 2024-01-09 + tags: [] + --- + + Just some content + with [[links]] but no sections """) - test_file = tmp_path / "unicode_file.md" - test_file.write_text(content, encoding="utf-16") - - parser = KnowledgeParser() - with pytest.raises(ParseError): - await parser.parse_file(test_file) + + test_file = tmp_path / "missing.md" + test_file.write_text(content) + + parser = EntityParser(tmp_path) + entity = await parser.parse_file(test_file) + assert len(entity.content.relations) == 1 + assert entity.content.relations[0].target == "links" + assert entity.content.relations[0].type == "mentions" @pytest.mark.asyncio -async def test_encoding_errors(tmp_path): - """Test handling of encoding errors.""" - # Create a file with invalid UTF-8 bytes - test_file = tmp_path / "invalid.md" - with open(test_file, "wb") as f: - f.write(b"\xff\xfe\x00\x00") # Invalid UTF-8 +async def test_nested_content(tmp_path): + """Test handling of deeply nested content.""" + content = dedent(""" + --- + type: test + id: test/nested + created: 2024-01-09 + modified: 2024-01-09 + tags: [] + --- + + # Test + + ## Level 1 + - [test] Level 1 #test (First level) + - implements [[One]] + + ### Level 2 + - [test] Level 2 #test (Second level) + - uses [[Two]] + + #### Level 3 + - [test] Level 3 #test (Third level) + - needs [[Three]] + """) + + test_file = tmp_path / "nested.md" + test_file.write_text(content) + + parser = EntityParser(tmp_path) + entity = await parser.parse_file(test_file) + + # Should find all observations and relations regardless of nesting + assert len(entity.content.observations) == 3 + assert len(entity.content.relations) == 3 + assert {r.target for r in entity.content.relations} == {"One", "Two", "Three"} - parser = KnowledgeParser() - with pytest.raises(ParseError): - await parser.parse_file(test_file, encoding="ascii") + +@pytest.mark.asyncio +async def test_malformed_frontmatter(tmp_path): + """Test handling of malformed frontmatter.""" + # Missing fields + content = dedent(""" + --- + type: test + --- + + # Test + """) + + test_file = tmp_path / "malformed.md" + test_file.write_text(content) + + parser = EntityParser(tmp_path) + entity = await parser.parse_file(test_file) + assert entity.frontmatter.id == "malformed" # Generated from filename + + # Invalid YAML + content = dedent(""" + --- + type: test + tags: [unclosed + --- + + # Test + """) + + test_file.write_text(content) + entity = await parser.parse_file(test_file) + assert entity.frontmatter.tags == [] # Default to empty list @pytest.mark.asyncio async def test_file_not_found(): """Test handling of non-existent files.""" - parser = KnowledgeParser() + parser = EntityParser(Path("/tmp")) with pytest.raises(FileError): await parser.parse_file(Path("nonexistent.md")) - - -@pytest.mark.asyncio -async def test_nested_structures(tmp_path): - """Test handling of nested markdown structures.""" - content = dedent(""" - --- - type: test - id: test/nested - created: 2024-12-21T14:00:00Z - modified: 2024-12-21T14:00:00Z - tags: [test] - --- - - # Nested Test - - ## Observations - - [test] Main point #main (Top level) - - [test] Subpoint #sub (Should be ignored) - - [test] Sub-subpoint #detail (Also ignored) - - ## Relations - - depends_on [[Sub Entity]] (Top level) - - uses [[Another Entity]] (Should be ignored) - - implements [[Third Entity]] (Also ignored) - """) - - test_file = tmp_path / "nested.md" - test_file.write_text(content) - - parser = KnowledgeParser() - entity = await parser.parse_file(test_file) - - assert len(entity.content.observations) == 3 - assert len(entity.content.relations) == 3 - assert entity.content.observations[0].tags == ["main"] - assert entity.content.observations[0].context == "Top level" - assert entity.content.relations[0].target == "Sub Entity" - assert entity.content.relations[0].type == "depends_on" - - -@pytest.mark.asyncio -async def test_malformed_sections(tmp_path): - """Test various malformed section contents.""" - content = dedent(""" - --- - type: test - id: test/malformed - created: 2024-12-21T14:00:00Z - modified: 2024-12-21T14:00:00Z - tags: [test] - --- - - # Malformed Test - - ## Observations - - not a valid observation - - [unclosed category content - - no content] - - [] empty category - - ## Relations - - not a valid relation - - missing_brackets Entity - - implements incomplete [[ - - implements ]] backwards - """) - - test_file = tmp_path / "malformed.md" - test_file.write_text(content) - - parser = KnowledgeParser() - with pytest.raises(ParseError): - entity = await parser.parse_file(test_file) - - -@pytest.mark.asyncio -async def test_missing_required_sections(tmp_path): - """Test handling of missing required sections.""" - # Test file with only frontmatter - content = dedent(""" - --- - type: test - id: test/incomplete - created: 2024-12-21T14:00:00Z - modified: 2024-12-21T14:00:00Z - tags: [test] - --- - """) - - test_file = tmp_path / "incomplete.md" - test_file.write_text(content) - - parser = KnowledgeParser() - with pytest.raises(ParseError): - await parser.parse_file(test_file) diff --git a/tests/markdown/test_relation_edge_cases.py b/tests/markdown/test_relation_edge_cases.py index dbbb0334..62f4862e 100644 --- a/tests/markdown/test_relation_edge_cases.py +++ b/tests/markdown/test_relation_edge_cases.py @@ -1,21 +1,157 @@ """Tests for edge cases in relation parsing.""" import pytest +from markdown_it import MarkdownIt -from basic_memory.markdown import KnowledgeParser +from basic_memory.markdown.plugins import ( + relation_plugin, + is_explicit_relation, + parse_relation, + parse_inline_relations +) +from basic_memory.markdown.schemas import Relation -@pytest.mark.asyncio -async def test_relation_empty_target(): +def test_empty_targets(): """Test handling of empty targets.""" + md = MarkdownIt().use(relation_plugin) + # Empty brackets - parser = KnowledgeParser() - - assert await parser.parse_relation("type [[]]") is None - assert await parser.parse_relation("type [[ ]]") is None - + tokens = md.parse("- type [[]]") + token = next(t for t in tokens if t.type == 'inline') + assert parse_relation(token) is None + # Only spaces - assert await parser.parse_relation("type [[ ]]") is None + tokens = md.parse("- type [[ ]]") + token = next(t for t in tokens if t.type == 'inline') + assert parse_relation(token) is None + + # Whitespace in brackets + tokens = md.parse("- type [[ ]]") + token = next(t for t in tokens if t.type == 'inline') + assert parse_relation(token) is None - # Only white spaces - assert await parser.parse_relation(" ") is None + +def test_malformed_links(): + """Test handling of malformed wiki links.""" + md = MarkdownIt().use(relation_plugin) + + # Missing close brackets + tokens = md.parse("- type [[Target") + assert not any(t.meta and 'relations' in t.meta for t in tokens) + + # Missing open brackets + tokens = md.parse("- type Target]]") + assert not any(t.meta and 'relations' in t.meta for t in tokens) + + # Backwards brackets + tokens = md.parse("- type ]]Target[[") + assert not any(t.meta and 'relations' in t.meta for t in tokens) + + # Nested brackets + tokens = md.parse("- type [[Outer [[Inner]] ]]") + token = next(t for t in tokens if t.type == 'inline') + rel = parse_relation(token) + assert rel is not None + assert "Outer" in rel['target'] + + +def test_context_handling(): + """Test handling of contexts.""" + md = MarkdownIt().use(relation_plugin) + + # Unclosed context + tokens = md.parse("- type [[Target]] (unclosed") + token = next(t for t in tokens if t.type == 'inline') + rel = parse_relation(token) + assert rel['context'] is None + + # Multiple parens + tokens = md.parse("- type [[Target]] (with (nested) parens)") + token = next(t for t in tokens if t.type == 'inline') + rel = parse_relation(token) + assert rel['context'] == "with (nested) parens" + + # Empty context + tokens = md.parse("- type [[Target]] ()") + token = next(t for t in tokens if t.type == 'inline') + rel = parse_relation(token) + assert rel['context'] is None + + +def test_inline_relations(): + """Test inline relation detection.""" + md = MarkdownIt().use(relation_plugin) + + # Multiple links in text + text = "Text with [[Link1]] and [[Link2]] and [[Link3]]" + rels = parse_inline_relations(text) + assert len(rels) == 3 + assert {r['target'] for r in rels} == {'Link1', 'Link2', 'Link3'} + + # Links with surrounding text + text = "Before [[Target]] After" + rels = parse_inline_relations(text) + assert len(rels) == 1 + assert rels[0]['target'] == 'Target' + + # Multiple links on same line + tokens = md.parse("[[One]] [[Two]] [[Three]]") + token = next(t for t in tokens if t.type == 'inline') + assert len(token.meta['relations']) == 3 + + +def test_unicode_targets(): + """Test handling of Unicode in targets.""" + md = MarkdownIt().use(relation_plugin) + + # Unicode in target + tokens = md.parse("- type [[测试]]") + token = next(t for t in tokens if t.type == 'inline') + rel = parse_relation(token) + assert rel['target'] == "测试" + + # Unicode in type + tokens = md.parse("- 使用 [[Target]]") + token = next(t for t in tokens if t.type == 'inline') + rel = parse_relation(token) + assert rel['type'] == "使用" + + # Unicode in context + tokens = md.parse("- type [[Target]] (测试)") + token = next(t for t in tokens if t.type == 'inline') + rel = parse_relation(token) + assert rel['context'] == "测试" + + # Model validation with Unicode + relation = Relation.model_validate(rel) + assert relation.type == "type" + assert relation.target == "Target" + assert relation.context == "测试" + + +def test_nested_content(): + """Test handling of nested content structures.""" + md = MarkdownIt().use(relation_plugin) + + content = """ + # Section + + Text with [[Link]] + + ## Subsection + - implements [[Component]] (In subsection) + - uses [[Other]] (Nested, should be separate) + + More text with [[Another Link]] + """ + + tokens = md.parse(content) + relations = [r for t in tokens if t.meta and 'relations' in t.meta for r in t.meta['relations']] + + # Should find all relations at any nesting level + assert len(relations) >= 4 # All links should be found + assert any(r['target'] == 'Link' for r in relations) + assert any(r['target'] == 'Component' for r in relations) + assert any(r['target'] == 'Other' for r in relations) + assert any(r['target'] == 'Another Link' for r in relations) \ No newline at end of file