mirror of
https://github.com/basicmachines-co/basic-memory
synced 2026-06-21 13:47:35 +00:00
enity parsing tests
This commit is contained in:
phernandez
@@ -1,11 +1,9 @@
|
||||
"""Base package for markdown parsing."""
|
||||
|
||||
from basic_memory.markdown.knowledge_parser import KnowledgeParser
|
||||
from basic_memory.markdown.entity_parser import EntityParser
|
||||
from basic_memory.markdown.schemas import (
|
||||
EntityMarkdown,
|
||||
EntityContent,
|
||||
EntityFrontmatter,
|
||||
EntityMetadata,
|
||||
Observation,
|
||||
Relation,
|
||||
)
|
||||
@@ -15,8 +13,7 @@ __all__ = [
|
||||
"EntityMarkdown",
|
||||
"EntityContent",
|
||||
"EntityFrontmatter",
|
||||
"EntityMetadata",
|
||||
"KnowledgeParser",
|
||||
"EntityParser",
|
||||
"Observation",
|
||||
"Relation",
|
||||
"ParseError",
|
||||
|
||||
@@ -1,317 +1,112 @@
|
||||
"""Universal parser for markdown files with optional frontmatter, observations, and relations.
|
||||
"""Parser for markdown files into Entity objects.
|
||||
|
||||
The id field in frontmatter is derived from the filename, converted to snake_case with .md extension removed.
|
||||
For example:
|
||||
'My Project Notes.md' -> 'my_project_notes'
|
||||
'API-Design.md' -> 'api_design'
|
||||
Uses markdown-it with plugins to parse structured data from markdown content.
|
||||
"""
|
||||
|
||||
import re
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Dict, Any, Optional, Tuple, List
|
||||
from datetime import datetime
|
||||
from typing import Dict, Any, Optional
|
||||
|
||||
from loguru import logger
|
||||
from markdown_it import MarkdownIt
|
||||
import frontmatter
|
||||
|
||||
from basic_memory.markdown.base_parser import MarkdownParser, ParseError
|
||||
from basic_memory.markdown.plugins import observation_plugin, relation_plugin
|
||||
from basic_memory.markdown.schemas import (
|
||||
EntityMarkdown,
|
||||
EntityFrontmatter,
|
||||
EntityContent,
|
||||
EntityMetadata,
|
||||
Observation,
|
||||
Relation,
|
||||
EntityContent, Observation, Relation,
|
||||
)
|
||||
from basic_memory.schemas.base import to_snake_case
|
||||
|
||||
|
||||
class EntityParser(MarkdownParser[EntityMarkdown]):
|
||||
"""A forgiving parser that extracts as much structure as it can find.
|
||||
|
||||
Generates entity IDs from filenames by:
|
||||
1. Removing .md extension
|
||||
2. Converting to snake_case
|
||||
3. Removing any invalid characters
|
||||
"""
|
||||
class EntityParser:
|
||||
"""Parser for markdown files into Entity objects."""
|
||||
|
||||
def convert_to_id(self, filename: str) -> str:
|
||||
"""Convert a filename to a valid entity ID.
|
||||
|
||||
Args:
|
||||
filename: Name of the file (with or without .md extension)
|
||||
|
||||
Returns:
|
||||
Snake case version of filename without extension
|
||||
|
||||
Examples:
|
||||
'My Project Notes.md' -> 'my_project_notes'
|
||||
'API-Design.md' -> 'api_design'
|
||||
def __init__(self, base_path: Path):
|
||||
"""Initialize parser with base path for relative path_id generation."""
|
||||
self.base_path = base_path.resolve()
|
||||
self.md = (MarkdownIt()
|
||||
.use(observation_plugin)
|
||||
.use(relation_plugin))
|
||||
|
||||
def get_path_id(self, file_path: Path) -> str:
|
||||
"""Get path_id from file path relative to base_path.
|
||||
|
||||
Example:
|
||||
base_path: /project/root
|
||||
file_path: /project/root/design/models/data.md
|
||||
returns: "design/models/data"
|
||||
"""
|
||||
# Remove .md extension if present
|
||||
if filename.lower().endswith('.md'):
|
||||
filename = filename[:-3]
|
||||
|
||||
return to_snake_case(filename)
|
||||
|
||||
def parse_dates(self, frontmatter: Dict[str, Any], file_path: Path) -> Tuple[datetime, datetime]:
|
||||
"""Parse created and updated dates from frontmatter or file system.
|
||||
|
||||
Args:
|
||||
frontmatter: Dictionary containing frontmatter fields
|
||||
file_path: Path to the source file for fallback dates
|
||||
|
||||
Returns:
|
||||
Tuple of (created_date, updated_date)
|
||||
|
||||
Priority:
|
||||
1. Valid frontmatter dates
|
||||
2. File system dates (created/modified)
|
||||
"""
|
||||
created = None
|
||||
updated = None
|
||||
|
||||
# Try frontmatter first
|
||||
try:
|
||||
if 'created' in frontmatter:
|
||||
created = self.parse_date(frontmatter['created'])
|
||||
if 'updated' in frontmatter or 'modified' in frontmatter:
|
||||
updated = self.parse_date(frontmatter.get('updated') or frontmatter.get('modified'))
|
||||
except Exception as e:
|
||||
logger.warning(f"Error parsing frontmatter dates: {e}")
|
||||
|
||||
# Fall back to file system dates if needed
|
||||
try:
|
||||
stats = file_path.stat()
|
||||
if not created:
|
||||
created = datetime.fromtimestamp(stats.st_ctime)
|
||||
if not updated:
|
||||
updated = datetime.fromtimestamp(stats.st_mtime)
|
||||
except Exception as e:
|
||||
logger.warning(f"Error getting file stats: {e}")
|
||||
# Last resort - use current time
|
||||
now = datetime.now()
|
||||
created = created or now
|
||||
updated = updated or now
|
||||
|
||||
return created, updated
|
||||
# Get relative path and remove .md extension
|
||||
rel_path = file_path.resolve().relative_to(self.base_path)
|
||||
if rel_path.suffix.lower() == '.md':
|
||||
return str(rel_path.with_suffix(''))
|
||||
return str(rel_path)
|
||||
|
||||
def parse_date(self, value: Any) -> Optional[datetime]:
|
||||
"""Convert various date formats to datetime."""
|
||||
"""Parse various date formats into datetime."""
|
||||
if isinstance(value, datetime):
|
||||
return value
|
||||
try:
|
||||
if isinstance(value, str):
|
||||
if isinstance(value, str):
|
||||
try:
|
||||
return datetime.fromisoformat(value.replace("Z", "+00:00"))
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
return None
|
||||
|
||||
def parse_tags(self, tags: Any) -> List[str]:
|
||||
"""Convert various tag formats to list of strings."""
|
||||
async def parse_file(self, file_path: Path) -> EntityMarkdown:
|
||||
"""Parse markdown file into EntityMarkdown."""
|
||||
# Parse frontmatter and content using python-frontmatter
|
||||
post = frontmatter.load(str(file_path))
|
||||
|
||||
# Extract or generate required fields
|
||||
path_id = post.metadata.get("id") or self.get_path_id(file_path)
|
||||
stats = file_path.stat()
|
||||
|
||||
# Parse frontmatter
|
||||
entity_frontmatter = EntityFrontmatter(
|
||||
type=str(post.metadata.get("type", "note")),
|
||||
id=path_id,
|
||||
title=str(post.metadata.get("title", file_path.name)),
|
||||
created=self.parse_date(post.metadata.get("created"))
|
||||
or datetime.fromtimestamp(stats.st_ctime),
|
||||
modified=self.parse_date(post.metadata.get("modified"))
|
||||
or datetime.fromtimestamp(stats.st_mtime),
|
||||
tags=self.parse_tags(post.metadata.get("tags", [])),
|
||||
)
|
||||
|
||||
# Parse content for observations and relations using markdown-it
|
||||
tokens = self.md.parse(post.content)
|
||||
|
||||
# Extract observations and relations from token meta
|
||||
observations = []
|
||||
relations = []
|
||||
|
||||
for token in tokens:
|
||||
if token.meta: # Token might not have meta
|
||||
if 'observation' in token.meta:
|
||||
obs = token.meta['observation']
|
||||
observation = Observation.model_validate(obs)
|
||||
observations.append(observation)
|
||||
if 'relations' in token.meta:
|
||||
rels = token.meta['relations']
|
||||
relations.extend([Relation.model_validate(r) for r in rels])
|
||||
|
||||
# Create EntityContent
|
||||
entity_content = EntityContent(
|
||||
content=post.content,
|
||||
observations=observations,
|
||||
relations=relations,
|
||||
)
|
||||
|
||||
return EntityMarkdown(
|
||||
frontmatter=entity_frontmatter,
|
||||
content=entity_content,
|
||||
)
|
||||
|
||||
def parse_tags(self, tags: Any) -> list[str]:
|
||||
"""Parse tags into list of strings."""
|
||||
if isinstance(tags, str):
|
||||
return [t.strip() for t in tags.split(",") if t.strip()]
|
||||
if isinstance(tags, (list, tuple)):
|
||||
return [str(t).strip() for t in tags if str(t).strip()]
|
||||
return []
|
||||
|
||||
async def parse_frontmatter(self, frontmatter: Dict[str, Any], file_path: Optional[Path] = None) -> EntityFrontmatter:
|
||||
"""Parse frontmatter with sensible defaults for missing fields.
|
||||
|
||||
Args:
|
||||
frontmatter: Dictionary of frontmatter fields
|
||||
file_path: Optional path to source file, used for id and dates
|
||||
"""
|
||||
try:
|
||||
# Get or generate ID from filename
|
||||
entity_name = None
|
||||
if file_path:
|
||||
entity_name = self.convert_to_id(file_path.name)
|
||||
|
||||
# Get dates from frontmatter or file
|
||||
created, updated = self.parse_dates(frontmatter, file_path) if file_path else (datetime.now(), datetime.now())
|
||||
|
||||
# Ensure we have minimum required fields
|
||||
processed = {
|
||||
"type": str(frontmatter.get("type", "document")).strip(),
|
||||
"id": str(frontmatter.get("id", entity_name or "document")).strip(),
|
||||
"created": created,
|
||||
"modified": updated,
|
||||
"tags": self.parse_tags(frontmatter.get("tags", []))
|
||||
}
|
||||
|
||||
return EntityFrontmatter(**processed)
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"Error parsing frontmatter, using defaults: {e}")
|
||||
return EntityFrontmatter(
|
||||
type="document",
|
||||
id=entity_name or "document",
|
||||
created=created,
|
||||
modified=updated,
|
||||
tags=[]
|
||||
)
|
||||
|
||||
async def parse_content(self, title: str, sections: Dict[str, str]) -> EntityContent:
|
||||
"""Parse content sections without requiring any particular structure."""
|
||||
try:
|
||||
# Get content from content section if it exists
|
||||
content = sections.get("content", "").strip() or None
|
||||
|
||||
# Try to parse observations if they exist
|
||||
observations = []
|
||||
if "observations" in sections:
|
||||
for line in sections["observations"].splitlines():
|
||||
try:
|
||||
obs = await self.parse_observation(line)
|
||||
if obs:
|
||||
observations.append(obs)
|
||||
except ParseError as e:
|
||||
logger.warning(f"Skipping invalid observation: {e}")
|
||||
|
||||
# Try to parse relations if they exist
|
||||
relations = []
|
||||
if "relations" in sections:
|
||||
for line in sections["relations"].splitlines():
|
||||
try:
|
||||
rel = await self.parse_relation(line)
|
||||
if rel:
|
||||
relations.append(rel)
|
||||
except ParseError as e:
|
||||
logger.warning(f"Skipping invalid relation: {e}")
|
||||
|
||||
# Also look for wiki-links in content as implicit relations
|
||||
try:
|
||||
content_relations = await self.parse_content_relations(sections.get("content", ""))
|
||||
relations.extend(content_relations)
|
||||
except Exception as e:
|
||||
logger.warning(f"Error parsing content relations: {e}")
|
||||
|
||||
return EntityContent(
|
||||
title=title or "Untitled",
|
||||
summary=content,
|
||||
observations=observations,
|
||||
relations=relations
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error parsing content, using minimal structure: {e}")
|
||||
return EntityContent(
|
||||
title=title or "Untitled",
|
||||
summary=None,
|
||||
observations=[],
|
||||
relations=[]
|
||||
)
|
||||
|
||||
async def parse_observation(self, line: str) -> Optional[Observation]:
|
||||
"""Parse a single observation line."""
|
||||
if not line or not line.strip().startswith("-"):
|
||||
return None
|
||||
|
||||
line = line.strip()[1:].strip() # Remove leading "-" and whitespace
|
||||
|
||||
# Extract category if present
|
||||
category = None
|
||||
if line.startswith("["):
|
||||
end = line.find("]")
|
||||
if end != -1:
|
||||
category = line[1:end].strip()
|
||||
line = line[end + 1:].strip()
|
||||
|
||||
# Extract context if present
|
||||
context = None
|
||||
if line.endswith(")"):
|
||||
start = line.rfind("(")
|
||||
if start != -1:
|
||||
context = line[start + 1:-1].strip()
|
||||
line = line[:start].strip()
|
||||
|
||||
# Extract tags and content
|
||||
parts = line.split()
|
||||
content_parts = []
|
||||
tags = []
|
||||
|
||||
for part in parts:
|
||||
if part.startswith("#"):
|
||||
tags.append(part[1:])
|
||||
else:
|
||||
content_parts.append(part)
|
||||
|
||||
content = " ".join(content_parts).strip()
|
||||
if not content:
|
||||
return None
|
||||
|
||||
return Observation(
|
||||
category=category,
|
||||
content=content,
|
||||
tags=tags if tags else None,
|
||||
context=context
|
||||
)
|
||||
|
||||
async def parse_relation(self, line: str) -> Optional[Relation]:
|
||||
"""Parse a single relation line."""
|
||||
if not line or not line.strip().startswith("-"):
|
||||
return None
|
||||
|
||||
line = line.strip()[1:].strip()
|
||||
|
||||
# Look for [[target]]
|
||||
start = line.find("[[")
|
||||
end = line.find("]]")
|
||||
if start == -1 or end == -1:
|
||||
return None
|
||||
|
||||
# Extract parts
|
||||
rel_type = line[:start].strip() or "relates_to" # Default type if none specified
|
||||
target = line[start + 2:end].strip()
|
||||
|
||||
# Extract context if present
|
||||
context = None
|
||||
remaining = line[end + 2:].strip()
|
||||
if remaining.startswith("(") and remaining.endswith(")"):
|
||||
context = remaining[1:-1].strip()
|
||||
|
||||
if not target:
|
||||
return None
|
||||
|
||||
return Relation(
|
||||
type=rel_type,
|
||||
target=target,
|
||||
context=context
|
||||
)
|
||||
|
||||
async def parse_content_relations(self, content: str) -> List[Relation]:
|
||||
"""Extract wiki-style links from content as relations."""
|
||||
relations = []
|
||||
if not content:
|
||||
return relations
|
||||
|
||||
import re
|
||||
pattern = r'\[\[([^\]]+)\]\]'
|
||||
|
||||
for match in re.finditer(pattern, content):
|
||||
target = match.group(1).strip()
|
||||
if target:
|
||||
relations.append(Relation(
|
||||
type="mentions",
|
||||
target=target,
|
||||
context=None
|
||||
))
|
||||
|
||||
return relations
|
||||
|
||||
async def parse_metadata(self, metadata_section: Optional[str]) -> EntityMetadata:
|
||||
"""Metadata section is no longer used."""
|
||||
return EntityMetadata()
|
||||
|
||||
async def create_document(
|
||||
self,
|
||||
frontmatter: EntityFrontmatter,
|
||||
content: EntityContent,
|
||||
metadata: EntityMetadata
|
||||
) -> EntityMarkdown:
|
||||
"""Create the final EntityMarkdown document."""
|
||||
return EntityMarkdown(
|
||||
frontmatter=frontmatter,
|
||||
content=content,
|
||||
entity_metadata=metadata
|
||||
)
|
||||
@@ -1,282 +0,0 @@
|
||||
"""Parser for Basic Memory entity markdown files."""
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Dict, Any, Optional
|
||||
|
||||
import yaml
|
||||
from loguru import logger
|
||||
|
||||
from basic_memory.markdown.base_parser import MarkdownParser, ParseError
|
||||
from basic_memory.markdown.schemas import (
|
||||
EntityMarkdown,
|
||||
EntityFrontmatter,
|
||||
EntityContent,
|
||||
EntityMetadata,
|
||||
Observation,
|
||||
Relation,
|
||||
)
|
||||
|
||||
|
||||
class KnowledgeParser(MarkdownParser[EntityMarkdown]):
|
||||
"""Parser for entity markdown files.
|
||||
|
||||
Entity files must have:
|
||||
- YAML frontmatter (type, id, created, modified, tags)
|
||||
- Title (# Title)
|
||||
- Optional description
|
||||
- Observations section (## Observations)
|
||||
- Relations section (## Relations)
|
||||
- Optional # Metadata section with YAML in code block
|
||||
|
||||
Example Metadata:
|
||||
```yml
|
||||
field: value
|
||||
other: other value
|
||||
```
|
||||
"""
|
||||
|
||||
async def parse_metadata(self, metadata_section: Optional[str]) -> EntityMetadata:
|
||||
"""Parse metadata section."""
|
||||
try:
|
||||
if not metadata_section:
|
||||
logger.debug("No metadata section found")
|
||||
return EntityMetadata()
|
||||
|
||||
logger.debug(f"Raw metadata section:\n{metadata_section}")
|
||||
lines = metadata_section.strip().splitlines()
|
||||
yaml_lines = []
|
||||
in_yaml = False
|
||||
|
||||
# Look for ```yml or ```yaml starter
|
||||
for line in lines:
|
||||
stripped = line.strip().lower()
|
||||
if in_yaml:
|
||||
if stripped == "```":
|
||||
logger.debug("Found end of YAML block")
|
||||
break
|
||||
yaml_lines.append(line)
|
||||
logger.debug(f"Added YAML line: {line}")
|
||||
elif stripped in ["```yml", "```yaml"]:
|
||||
logger.debug("Found start of YAML block")
|
||||
in_yaml = True
|
||||
|
||||
if not yaml_lines:
|
||||
logger.debug("No YAML lines found in metadata section")
|
||||
return EntityMetadata()
|
||||
|
||||
yaml_content = "\n".join(yaml_lines)
|
||||
logger.debug(f"YAML content to parse:\n{yaml_content}")
|
||||
|
||||
# Parse the YAML content
|
||||
try:
|
||||
parsed = yaml.safe_load(yaml_content)
|
||||
if not isinstance(parsed, dict):
|
||||
logger.warning(f"Metadata YAML is not a dictionary: {parsed}")
|
||||
return EntityMetadata()
|
||||
logger.debug(f"Successfully parsed metadata: {parsed}")
|
||||
return EntityMetadata(data=parsed)
|
||||
except yaml.YAMLError as e:
|
||||
logger.warning(f"Failed to parse metadata YAML: {e}")
|
||||
return EntityMetadata()
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to parse metadata: {e}")
|
||||
return EntityMetadata()
|
||||
|
||||
async def parse_frontmatter(self, frontmatter: Dict[str, Any]) -> EntityFrontmatter:
|
||||
"""Parse entity frontmatter."""
|
||||
try:
|
||||
# Check required fields
|
||||
required_fields = {"type", "id", "created", "modified"}
|
||||
missing = required_fields - set(frontmatter.keys())
|
||||
if missing:
|
||||
raise ParseError(f"Missing required frontmatter fields: {', '.join(missing)}")
|
||||
|
||||
# Validate date fields
|
||||
for date_field in ["created", "modified"]:
|
||||
try:
|
||||
if not isinstance(frontmatter[date_field], datetime):
|
||||
datetime.fromisoformat(str(frontmatter[date_field]).replace("Z", "+00:00"))
|
||||
except (ValueError, TypeError):
|
||||
raise ParseError(
|
||||
f"Invalid date format for {date_field}: {frontmatter[date_field]}"
|
||||
)
|
||||
|
||||
# Prepare fields
|
||||
processed = {
|
||||
"type": frontmatter["type"].strip(),
|
||||
"id": str(frontmatter["id"]).strip(),
|
||||
"created": frontmatter["created"],
|
||||
"modified": frontmatter["modified"],
|
||||
"tags": (
|
||||
[tag.strip() for tag in frontmatter.get("tags", "").split(",")]
|
||||
if isinstance(frontmatter.get("tags"), str)
|
||||
else [t.strip() for t in frontmatter.get("tags", [])]
|
||||
),
|
||||
}
|
||||
|
||||
return EntityFrontmatter(**processed)
|
||||
|
||||
except Exception as e:
|
||||
if isinstance(e, ParseError):
|
||||
raise
|
||||
logger.error(f"Invalid entity frontmatter: {e}")
|
||||
raise ParseError(f"Invalid entity frontmatter: {str(e)}")
|
||||
|
||||
async def parse_content(self, title: str, sections: Dict[str, str]) -> EntityContent:
|
||||
"""Parse entity content section."""
|
||||
try:
|
||||
# Get description (if any)
|
||||
description = None
|
||||
if "content" in sections:
|
||||
description = sections["content"]
|
||||
|
||||
# Parse observations (required)
|
||||
observations = []
|
||||
if "observations" not in sections:
|
||||
raise ParseError("Missing required observations section")
|
||||
|
||||
for line in sections["observations"].splitlines():
|
||||
if line and not line.isspace():
|
||||
observation = await self.parse_observation(line)
|
||||
if observation:
|
||||
observations.append(observation)
|
||||
|
||||
# Parse relations (optional)
|
||||
relations = []
|
||||
if "relations" in sections:
|
||||
for line in sections["relations"].splitlines():
|
||||
if line and not line.isspace():
|
||||
relation = await self.parse_relation(line)
|
||||
if relation:
|
||||
relations.append(relation)
|
||||
|
||||
return EntityContent(
|
||||
title=title, summary=description, observations=observations, relations=relations
|
||||
)
|
||||
|
||||
except ParseError:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error(f"Invalid entity content: {e}")
|
||||
raise ParseError(f"Invalid entity content: {str(e)}")
|
||||
|
||||
async def parse_observation(self, line: str) -> Optional[Observation]:
|
||||
"""Parse a single observation line."""
|
||||
if not line or line.isspace():
|
||||
return None
|
||||
|
||||
try:
|
||||
# Remove leading/trailing whitespace and bullet
|
||||
line = line.strip()
|
||||
if not line.startswith("-"):
|
||||
return None
|
||||
line = line[1:].strip()
|
||||
|
||||
category = None
|
||||
# Handle malformed category brackets first
|
||||
if line.startswith("]"):
|
||||
raise ParseError("missing category")
|
||||
elif line.startswith("["):
|
||||
# category
|
||||
close_bracket = line.find("]")
|
||||
if close_bracket == -1:
|
||||
raise ParseError("unclosed category")
|
||||
|
||||
# Extract category - return None for empty category
|
||||
category_string = line[1:close_bracket].strip()
|
||||
category = category_string if category_string else None
|
||||
|
||||
line = line[close_bracket + 1 :].strip()
|
||||
|
||||
# Extract context if present (context)
|
||||
context = None
|
||||
# check if line ends with ")"
|
||||
if line.endswith(")"):
|
||||
# find "(" starting from end
|
||||
last_open = line.rfind("(")
|
||||
if last_open != -1:
|
||||
context = line[last_open + 1 : -1].strip()
|
||||
# remove context from the line
|
||||
line = line[:last_open].strip()
|
||||
|
||||
# Extract tags and clean content
|
||||
tags = []
|
||||
content_parts = []
|
||||
for part in line.split():
|
||||
if part.startswith("#"):
|
||||
# Handle multiple hashtags stuck together
|
||||
if "#" in part[1:]:
|
||||
multi_tags = part.split("#")
|
||||
tags.extend(t for t in multi_tags if t)
|
||||
else:
|
||||
tags.append(part[1:])
|
||||
else:
|
||||
content_parts.append(part)
|
||||
|
||||
content = " ".join(content_parts).strip()
|
||||
if not content:
|
||||
raise ParseError("Empty content")
|
||||
|
||||
return Observation(
|
||||
category=category, content=content, tags=tags if tags else None, context=context
|
||||
)
|
||||
|
||||
except ParseError:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to parse observation: {e}")
|
||||
raise ParseError(f"Failed to parse observation: {str(e)}")
|
||||
|
||||
async def parse_relation(self, line: str) -> Optional[Relation]:
|
||||
"""Parse a single relation line."""
|
||||
if not line or line.isspace():
|
||||
return None
|
||||
|
||||
try:
|
||||
# Remove leading/trailing whitespace and bullet
|
||||
line = line.strip()
|
||||
if not line.startswith("-"):
|
||||
return None
|
||||
line = line[1:].strip()
|
||||
|
||||
# Extract context if present (context)
|
||||
context = None
|
||||
if line.endswith(")"):
|
||||
context_start = line.rfind("(")
|
||||
if context_start != -1:
|
||||
context = line[context_start + 1 : -1].strip()
|
||||
line = line[:context_start].strip()
|
||||
|
||||
# Extract relation type and target [[Entity]]
|
||||
if "[[" not in line or "]]" not in line:
|
||||
raise ParseError("Invalid relation format - missing [[entity]]")
|
||||
|
||||
# Split into relation type and target
|
||||
parts = line.split("[[", 1)
|
||||
rel_type = parts[0].strip()
|
||||
if not rel_type:
|
||||
raise ParseError("Missing relation type")
|
||||
|
||||
target_part = parts[1]
|
||||
close_pos = target_part.find("]]")
|
||||
if close_pos == -1:
|
||||
raise ParseError("Unclosed [[ ]] in target")
|
||||
|
||||
target = target_part[:close_pos].strip()
|
||||
if not target:
|
||||
raise ParseError("Empty target entity")
|
||||
|
||||
return Relation(type=rel_type, target=target, context=context)
|
||||
|
||||
except ParseError:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to parse relation: {e}")
|
||||
raise ParseError(f"Failed to parse relation: {str(e)}")
|
||||
|
||||
async def create_document(
|
||||
self, frontmatter: EntityFrontmatter, content: EntityContent, metadata: EntityMetadata
|
||||
) -> EntityMarkdown:
|
||||
"""Create entity from parsed sections."""
|
||||
return EntityMarkdown(frontmatter=frontmatter, content=content, entity_metadata=metadata)
|
||||
@@ -0,0 +1,230 @@
|
||||
"""Markdown-it plugins for Basic Memory markdown parsing."""
|
||||
|
||||
from typing import List, Any, Dict
|
||||
from markdown_it import MarkdownIt
|
||||
from markdown_it.token import Token
|
||||
|
||||
|
||||
# Observation handling functions
|
||||
def is_observation(token: Token) -> bool:
|
||||
"""Check if token looks like our observation format."""
|
||||
if token.type != 'inline':
|
||||
return False
|
||||
|
||||
content = token.content.strip()
|
||||
return ((content.startswith('[') and ']' in content) or # Has category
|
||||
'#' in content) # Has tags
|
||||
|
||||
|
||||
def parse_observation(token: Token) -> Dict[str, Any]:
|
||||
"""Extract observation parts from token."""
|
||||
# Strip bullet point if present
|
||||
content = token.content.strip()
|
||||
if content.startswith('- '):
|
||||
content = content[2:].strip()
|
||||
elif content.startswith('-'):
|
||||
content = content[1:].strip()
|
||||
|
||||
# Parse [category]
|
||||
category = None
|
||||
if content.startswith('['):
|
||||
end = content.find(']')
|
||||
if end != -1:
|
||||
category = content[1:end].strip()
|
||||
content = content[end + 1:].strip()
|
||||
|
||||
# Parse (context)
|
||||
context = None
|
||||
if content.endswith(')'):
|
||||
start = content.rfind('(')
|
||||
if start != -1:
|
||||
context = content[start + 1:-1].strip()
|
||||
content = content[:start].strip()
|
||||
|
||||
# Parse #tags and content
|
||||
parts = content.split()
|
||||
content_parts = []
|
||||
tags = set() # Use set to avoid duplicates
|
||||
|
||||
for part in parts:
|
||||
if part.startswith('#'):
|
||||
# Handle multiple #tags stuck together
|
||||
if '#' in part[1:]:
|
||||
# Split on # but keep non-empty tags
|
||||
subtags = [t for t in part.split('#') if t]
|
||||
tags.update(subtags)
|
||||
else:
|
||||
tags.add(part[1:])
|
||||
else:
|
||||
content_parts.append(part)
|
||||
|
||||
return {
|
||||
'category': category,
|
||||
'content': ' '.join(content_parts).strip(),
|
||||
'tags': list(tags) if tags else None,
|
||||
'context': context
|
||||
}
|
||||
|
||||
|
||||
# Relation handling functions
|
||||
def is_explicit_relation(token: Token) -> bool:
|
||||
"""Check if token looks like our relation format."""
|
||||
if token.type != 'inline':
|
||||
return False
|
||||
|
||||
content = token.content.strip()
|
||||
return ('[[' in content and
|
||||
']]' in content and
|
||||
content.index('[[') < content.index(']]'))
|
||||
|
||||
|
||||
def parse_relation(token: Token) -> Dict[str, Any]:
|
||||
"""Extract relation parts from token."""
|
||||
# Remove bullet point if present
|
||||
content = token.content.strip()
|
||||
if content.startswith('- '):
|
||||
content = content[2:].strip()
|
||||
elif content.startswith('-'):
|
||||
content = content[1:].strip()
|
||||
|
||||
# Extract [[target]]
|
||||
target = None
|
||||
rel_type = 'relates_to' # default
|
||||
context = None
|
||||
|
||||
start = content.find('[[')
|
||||
end = content.find(']]')
|
||||
|
||||
if start != -1 and end != -1:
|
||||
# Get text before link as relation type
|
||||
before = content[:start].strip()
|
||||
if before:
|
||||
rel_type = before
|
||||
|
||||
# Get target
|
||||
target = content[start + 2:end].strip()
|
||||
|
||||
# Look for context after
|
||||
after = content[end + 2:].strip()
|
||||
if after.startswith('(') and after.endswith(')'):
|
||||
context = after[1:-1].strip()
|
||||
|
||||
if not target:
|
||||
return None
|
||||
|
||||
return {
|
||||
'type': rel_type,
|
||||
'target': target,
|
||||
'context': context
|
||||
}
|
||||
|
||||
|
||||
def parse_inline_relations(content: str) -> List[Dict[str, Any]]:
|
||||
"""Find wiki-style links in regular content."""
|
||||
relations = []
|
||||
|
||||
import re
|
||||
pattern = r'\[\[([^\]]+)\]\]'
|
||||
|
||||
for match in re.finditer(pattern, content):
|
||||
target = match.group(1).strip()
|
||||
if target and not target.startswith('[['): # Avoid nested matches
|
||||
relations.append({
|
||||
'type': 'mentions',
|
||||
'target': target,
|
||||
'context': None
|
||||
})
|
||||
|
||||
return relations
|
||||
|
||||
|
||||
def observation_plugin(md: MarkdownIt) -> None:
|
||||
"""Plugin for parsing observation format:
|
||||
- [category] Content text #tag1 #tag2 (context)
|
||||
- Content text #tag1 (context) # No category is also valid
|
||||
"""
|
||||
|
||||
def observation_rule(state: Any) -> None:
|
||||
"""Process observations in token stream."""
|
||||
tokens = state.tokens
|
||||
current_section = None
|
||||
in_list_item = False
|
||||
|
||||
for idx in range(len(tokens)):
|
||||
token = tokens[idx]
|
||||
|
||||
# Track current section by headings
|
||||
if token.type == 'heading_open':
|
||||
next_token = tokens[idx + 1] if idx + 1 < len(tokens) else None
|
||||
if next_token and next_token.type == 'inline':
|
||||
current_section = next_token.content.lower()
|
||||
|
||||
# Track list nesting
|
||||
elif token.type == 'list_item_open':
|
||||
in_list_item = True
|
||||
elif token.type == 'list_item_close':
|
||||
in_list_item = False
|
||||
|
||||
# Initialize meta for all tokens
|
||||
token.meta = token.meta or {}
|
||||
|
||||
# Parse observations in list items
|
||||
if token.type == 'inline' and in_list_item and is_observation(token):
|
||||
obs = parse_observation(token)
|
||||
if obs['content']: # Only store if we have content
|
||||
token.meta['observation'] = obs
|
||||
|
||||
# Add the rule after inline processing
|
||||
md.core.ruler.after('inline', 'observations', observation_rule)
|
||||
|
||||
|
||||
def relation_plugin(md: MarkdownIt) -> None:
|
||||
"""Plugin for parsing relation formats:
|
||||
|
||||
Explicit relations:
|
||||
- relation_type [[target]] (context)
|
||||
|
||||
Implicit relations (links in content):
|
||||
Some text with [[target]] reference
|
||||
"""
|
||||
|
||||
def relation_rule(state: Any) -> None:
|
||||
"""Process relations in token stream."""
|
||||
tokens = state.tokens
|
||||
current_section = None
|
||||
in_list_item = False
|
||||
|
||||
for idx in range(len(tokens)):
|
||||
token = tokens[idx]
|
||||
|
||||
# Track current section by headings
|
||||
if token.type == 'heading_open':
|
||||
next_token = tokens[idx + 1] if idx + 1 < len(tokens) else None
|
||||
if next_token and next_token.type == 'inline':
|
||||
current_section = next_token.content.lower()
|
||||
|
||||
# Track list nesting
|
||||
elif token.type == 'list_item_open':
|
||||
in_list_item = True
|
||||
elif token.type == 'list_item_close':
|
||||
in_list_item = False
|
||||
|
||||
# Initialize meta for all tokens
|
||||
token.meta = token.meta or {}
|
||||
|
||||
# Only process inline tokens
|
||||
if token.type == 'inline':
|
||||
# Check for explicit relations in list items
|
||||
if in_list_item and is_explicit_relation(token):
|
||||
rel = parse_relation(token)
|
||||
if rel:
|
||||
token.meta['relations'] = [rel]
|
||||
|
||||
# Always check for inline relations in any text
|
||||
elif '[[' in token.content:
|
||||
rels = parse_inline_relations(token.content)
|
||||
if rels:
|
||||
token.meta['relations'] = token.meta.get('relations', []) + rels
|
||||
|
||||
# Add the rule after inline processing
|
||||
md.core.ruler.after('inline', 'relations', relation_rule)
|
||||
@@ -26,6 +26,7 @@ class Relation(BaseModel):
|
||||
class EntityFrontmatter(BaseModel):
|
||||
"""Required frontmatter fields for an entity."""
|
||||
|
||||
title: str
|
||||
type: str
|
||||
id: str
|
||||
created: datetime
|
||||
@@ -36,18 +37,9 @@ class EntityFrontmatter(BaseModel):
|
||||
class EntityContent(BaseModel):
|
||||
"""Content sections of an entity markdown file."""
|
||||
|
||||
title: str
|
||||
summary: Optional[str] = None
|
||||
content: Optional[str] = None
|
||||
observations: List[Observation] = []
|
||||
relations: List[Relation] = []
|
||||
context: Optional[str] = None
|
||||
|
||||
|
||||
class EntityMetadata(BaseModel):
|
||||
"""Optional metadata for an entity."""
|
||||
|
||||
# Changed from 'metadata' to 'data' to avoid pydantic special field name
|
||||
data: Dict[str, Any] = {}
|
||||
|
||||
|
||||
class EntityMarkdown(BaseModel):
|
||||
@@ -55,4 +47,3 @@ class EntityMarkdown(BaseModel):
|
||||
|
||||
frontmatter: EntityFrontmatter
|
||||
content: EntityContent
|
||||
entity_metadata: EntityMetadata = EntityMetadata()
|
||||
|
||||
@@ -5,7 +5,7 @@ from pathlib import Path
|
||||
from loguru import logger
|
||||
|
||||
from basic_memory.config import ProjectConfig
|
||||
from basic_memory.markdown import KnowledgeParser
|
||||
from basic_memory.markdown import EntityParser
|
||||
from basic_memory.services.search_service import SearchService
|
||||
from basic_memory.sync import FileChangeScanner
|
||||
from basic_memory.sync.knowledge_sync_service import KnowledgeSyncService
|
||||
@@ -24,12 +24,12 @@ class SyncService:
|
||||
self,
|
||||
scanner: FileChangeScanner,
|
||||
knowledge_sync_service: KnowledgeSyncService,
|
||||
knowledge_parser: KnowledgeParser,
|
||||
entity_parser: EntityParser,
|
||||
search_service: SearchService,
|
||||
):
|
||||
self.scanner = scanner
|
||||
self.knowledge_sync_service = knowledge_sync_service
|
||||
self.knowledge_parser = knowledge_parser
|
||||
self.knowledge_parser = entity_parser
|
||||
self.search_service = search_service
|
||||
|
||||
|
||||
|
||||
+5
-5
@@ -10,7 +10,7 @@ from sqlalchemy.ext.asyncio import AsyncSession, AsyncEngine, async_sessionmaker
|
||||
from basic_memory import db
|
||||
from basic_memory.config import ProjectConfig
|
||||
from basic_memory.db import DatabaseType
|
||||
from basic_memory.markdown.knowledge_parser import KnowledgeParser
|
||||
from basic_memory.markdown import EntityParser
|
||||
from basic_memory.markdown.knowledge_writer import KnowledgeWriter
|
||||
from basic_memory.models import Base
|
||||
from basic_memory.models.knowledge import Entity
|
||||
@@ -130,9 +130,9 @@ def knowledge_writer():
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def knowledge_parser():
|
||||
def entity_parser(test_config):
|
||||
"""Create parser instance."""
|
||||
return KnowledgeParser()
|
||||
return EntityParser(test_config.home)
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
@@ -181,14 +181,14 @@ async def knowledge_sync_service(
|
||||
async def sync_service(
|
||||
knowledge_sync_service: KnowledgeSyncService,
|
||||
file_change_scanner: FileChangeScanner,
|
||||
knowledge_parser: KnowledgeParser,
|
||||
entity_parser: EntityParser,
|
||||
search_service: SearchService,
|
||||
) -> SyncService:
|
||||
"""Create sync service for testing."""
|
||||
return SyncService(
|
||||
scanner=file_change_scanner,
|
||||
knowledge_sync_service=knowledge_sync_service,
|
||||
knowledge_parser=knowledge_parser,
|
||||
knowledge_parser=entity_parser,
|
||||
search_service=search_service,
|
||||
)
|
||||
|
||||
|
||||
@@ -1,25 +1,34 @@
|
||||
"""Test to explore markdown-it token structure."""
|
||||
from textwrap import dedent
|
||||
"""Debug markdown-it token structure."""
|
||||
|
||||
from markdown_it import MarkdownIt
|
||||
from basic_memory.markdown.plugins import observation_plugin, parse_observation, is_observation
|
||||
|
||||
def test_token_structure():
|
||||
"""Analyze markdown-it token structure."""
|
||||
content = dedent('''
|
||||
# Title
|
||||
|
||||
## Observations
|
||||
- [test] First line #tag
|
||||
- [another] Second line #tag2
|
||||
''')
|
||||
|
||||
md = MarkdownIt()
|
||||
def test_debug_observations():
|
||||
"""Debug observation parsing."""
|
||||
md = MarkdownIt().use(observation_plugin)
|
||||
|
||||
content = "- [design] Core feature #important #mvp"
|
||||
tokens = md.parse(content)
|
||||
|
||||
# Print full token structure
|
||||
print("\nToken structure:")
|
||||
|
||||
# Print token info
|
||||
print("\nTokens:")
|
||||
for i, token in enumerate(tokens):
|
||||
attrs = {key: getattr(token, key) for key in dir(token)
|
||||
if not key.startswith('_') and not callable(getattr(token, key))}
|
||||
print(f"\nToken {i}:")
|
||||
for key, value in attrs.items():
|
||||
print(f" {key}: {value!r}")
|
||||
print(f" Type: {token.type}")
|
||||
print(f" Tag: {token.tag}")
|
||||
print(f" Content: {token.content}")
|
||||
print(f" Nesting: {token.nesting}")
|
||||
if hasattr(token, 'meta'):
|
||||
print(f" Meta: {token.meta}")
|
||||
|
||||
# Try the functions directly
|
||||
token = next(t for t in tokens if t.type == 'inline')
|
||||
print("\nTesting observation functions:")
|
||||
print(f"Is observation: {is_observation(token)}")
|
||||
obs = parse_observation(token)
|
||||
print(f"Parsed observation: {obs}")
|
||||
|
||||
# Verify meta was set
|
||||
print("\nVerifying meta:")
|
||||
token = next(t for t in tokens if t.meta and 'observation' in t.meta)
|
||||
print(f"Meta observation: {token.meta}")
|
||||
|
||||
@@ -5,7 +5,7 @@ from textwrap import dedent
|
||||
|
||||
import pytest
|
||||
|
||||
from basic_memory.markdown.knowledge_parser import KnowledgeParser
|
||||
from basic_memory.markdown.entity_parser import EntityParser
|
||||
from basic_memory.markdown.schemas import EntityMarkdown, EntityFrontmatter, EntityContent
|
||||
from basic_memory.utils.file_utils import ParseError, FileError
|
||||
|
||||
@@ -15,6 +15,7 @@ def valid_entity_content():
|
||||
"""A complete, valid entity file with all features."""
|
||||
return dedent("""
|
||||
---
|
||||
title: Auth Service
|
||||
type: component
|
||||
id: auth_service
|
||||
created: 2024-12-21T14:00:00Z
|
||||
@@ -22,8 +23,6 @@ def valid_entity_content():
|
||||
tags: authentication, security, core
|
||||
---
|
||||
|
||||
# Auth Service
|
||||
|
||||
Core authentication service that handles user authentication.
|
||||
|
||||
## Observations
|
||||
@@ -35,24 +34,16 @@ def valid_entity_content():
|
||||
- implements [[OAuth Implementation]] (Core auth flows)
|
||||
- uses [[Redis Cache]] (Token caching)
|
||||
- specified_by [[Auth API Spec]] (OpenAPI spec)
|
||||
|
||||
# Metadata
|
||||
<!-- anything below this line is for AI -->
|
||||
|
||||
```yml
|
||||
owner: team-auth
|
||||
priority: high
|
||||
```
|
||||
""")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_parse_complete_file(tmp_path, valid_entity_content):
|
||||
async def test_parse_complete_file(test_config, valid_entity_content):
|
||||
"""Test parsing a complete entity file with all features."""
|
||||
test_file = tmp_path / "test_entity.md"
|
||||
test_file = test_config.home / "test_entity.md"
|
||||
test_file.write_text(valid_entity_content)
|
||||
|
||||
parser = KnowledgeParser()
|
||||
parser = EntityParser(test_config.home)
|
||||
entity = await parser.parse_file(test_file)
|
||||
|
||||
# Verify entity structure
|
||||
@@ -61,16 +52,13 @@ async def test_parse_complete_file(tmp_path, valid_entity_content):
|
||||
assert isinstance(entity.content, EntityContent)
|
||||
|
||||
# Check frontmatter
|
||||
assert entity.frontmatter.title == "Auth Service"
|
||||
assert entity.frontmatter.type == "component"
|
||||
assert entity.frontmatter.id == "auth_service"
|
||||
assert set(entity.frontmatter.tags) == {"authentication", "security", "core"}
|
||||
|
||||
# Check content
|
||||
assert entity.content.title == "Auth Service"
|
||||
assert (
|
||||
entity.content.summary
|
||||
== "Core authentication service that handles user authentication."
|
||||
)
|
||||
assert "Core authentication service that handles user authentication." in entity.content.content
|
||||
|
||||
# Check observations
|
||||
assert len(entity.content.observations) == 3
|
||||
@@ -87,9 +75,6 @@ async def test_parse_complete_file(tmp_path, valid_entity_content):
|
||||
assert rel.target == "OAuth Implementation"
|
||||
assert rel.context == "Core auth flows"
|
||||
|
||||
# Check metadata
|
||||
assert entity.entity_metadata.data["owner"] == "team-auth"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_parse_minimal_file(tmp_path):
|
||||
@@ -115,106 +100,29 @@ async def test_parse_minimal_file(tmp_path):
|
||||
test_file = tmp_path / "minimal.md"
|
||||
test_file.write_text(content)
|
||||
|
||||
parser = KnowledgeParser()
|
||||
parser = EntityParser(tmp_path)
|
||||
entity = await parser.parse_file(test_file)
|
||||
|
||||
assert entity.frontmatter.type == "component"
|
||||
assert entity.frontmatter.id == "minimal"
|
||||
assert len(entity.content.observations) == 1
|
||||
assert len(entity.content.relations) == 1
|
||||
assert not entity.entity_metadata.data # Empty metadata
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_parse_content_str(valid_entity_content):
|
||||
"""Test parsing content string directly."""
|
||||
parser = KnowledgeParser()
|
||||
entity = await parser.parse_content_str(valid_entity_content)
|
||||
|
||||
assert isinstance(entity, EntityMarkdown)
|
||||
assert entity.frontmatter.type == "component"
|
||||
assert entity.content.title == "Auth Service"
|
||||
assert len(entity.content.observations) == 3
|
||||
assert len(entity.content.relations) == 3
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_metadata_handling(tmp_path):
|
||||
"""Test metadata section parsing."""
|
||||
parser = KnowledgeParser()
|
||||
|
||||
# Multiple metadata fields
|
||||
content = dedent("""
|
||||
---
|
||||
type: component
|
||||
id: test
|
||||
created: 2024-12-21T14:00:00Z
|
||||
modified: 2024-12-21T14:00:00Z
|
||||
tags: []
|
||||
---
|
||||
# Test Entity
|
||||
## Observations
|
||||
- [test] Test
|
||||
|
||||
# Metadata
|
||||
<!-- anything below this line is for AI -->
|
||||
|
||||
```yml
|
||||
owner: test-team
|
||||
priority: high
|
||||
nested:
|
||||
key: value
|
||||
list: [1, 2, 3]
|
||||
```
|
||||
""")
|
||||
test_file = tmp_path / "metadata.md"
|
||||
test_file.write_text(content)
|
||||
|
||||
entity = await parser.parse_file(test_file)
|
||||
assert entity.entity_metadata.data["owner"] == "test-team"
|
||||
assert entity.entity_metadata.data["priority"] == "high"
|
||||
assert entity.entity_metadata.data["nested"]["key"] == "value"
|
||||
assert entity.entity_metadata.data["nested"]["list"] == [1, 2, 3]
|
||||
|
||||
# No metadata section
|
||||
content = dedent("""
|
||||
---
|
||||
type: component
|
||||
id: test
|
||||
created: 2024-12-21T14:00:00Z
|
||||
modified: 2024-12-21T14:00:00Z
|
||||
tags: []
|
||||
---
|
||||
# Test Entity
|
||||
## Observations
|
||||
- [test] Test
|
||||
""")
|
||||
test_file = tmp_path / "no_metadata.md"
|
||||
test_file.write_text(content)
|
||||
|
||||
entity = await parser.parse_file(test_file)
|
||||
assert entity.entity_metadata.data == {}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_error_handling(tmp_path):
|
||||
"""Test error handling."""
|
||||
parser = KnowledgeParser()
|
||||
parser = EntityParser(tmp_path)
|
||||
|
||||
# Missing file
|
||||
with pytest.raises(FileError):
|
||||
with pytest.raises(FileNotFoundError):
|
||||
await parser.parse_file(Path("nonexistent.md"))
|
||||
|
||||
# Invalid file encoding
|
||||
test_file = tmp_path / "binary.md"
|
||||
with open(test_file, "wb") as f:
|
||||
f.write(b"\x80\x81") # Invalid UTF-8
|
||||
with pytest.raises(ParseError):
|
||||
await parser.parse_file(test_file)
|
||||
|
||||
# No frontmatter section
|
||||
content = "# Just a title\nNo frontmatter"
|
||||
test_file = tmp_path / "no_frontmatter.md"
|
||||
test_file.write_text(content)
|
||||
with pytest.raises(ParseError):
|
||||
with pytest.raises(UnicodeDecodeError):
|
||||
await parser.parse_file(test_file)
|
||||
@@ -0,0 +1,163 @@
|
||||
"""Tests for markdown-it plugins."""
|
||||
|
||||
import pytest
|
||||
from markdown_it import MarkdownIt
|
||||
|
||||
from basic_memory.markdown.plugins import observation_plugin, relation_plugin
|
||||
|
||||
|
||||
def test_observation_plugin():
|
||||
"""Test observation plugin parsing."""
|
||||
md = MarkdownIt().use(observation_plugin)
|
||||
|
||||
# Basic observation
|
||||
tokens = md.parse("- [design] Core feature #important #mvp")
|
||||
obs_token = next(t for t in tokens if t.meta and 'observation' in t.meta)
|
||||
obs = obs_token.meta['observation']
|
||||
|
||||
assert obs['category'] == 'design'
|
||||
assert obs['content'] == 'Core feature'
|
||||
assert set(obs['tags']) == {'important', 'mvp'}
|
||||
assert obs['context'] is None
|
||||
|
||||
# With context
|
||||
tokens = md.parse("- [feature] Authentication system #security (Required for MVP)")
|
||||
obs_token = next(t for t in tokens if t.meta and 'observation' in t.meta)
|
||||
obs = obs_token.meta['observation']
|
||||
|
||||
assert obs['category'] == 'feature'
|
||||
assert obs['content'] == 'Authentication system'
|
||||
assert set(obs['tags']) == {'security'}
|
||||
assert obs['context'] == 'Required for MVP'
|
||||
|
||||
# Without category
|
||||
tokens = md.parse("- Authentication system #security (Required for MVP)")
|
||||
obs_token = next(t for t in tokens if t.meta and 'observation' in t.meta)
|
||||
obs = obs_token.meta['observation']
|
||||
|
||||
assert obs['category'] is None
|
||||
assert obs['content'] == 'Authentication system'
|
||||
assert set(obs['tags']) == {'security'}
|
||||
assert obs['context'] == 'Required for MVP'
|
||||
|
||||
|
||||
def test_observation_edge_cases():
|
||||
"""Test observation plugin edge cases."""
|
||||
md = MarkdownIt().use(observation_plugin)
|
||||
|
||||
# Multiple word tags
|
||||
tokens = md.parse("- [tech] Database #high-priority #needs-review")
|
||||
obs_token = next(t for t in tokens if t.meta and 'observation' in t.meta)
|
||||
obs = obs_token.meta['observation']
|
||||
assert set(obs['tags']) == {'high-priority', 'needs-review'}
|
||||
|
||||
# Multiple word category
|
||||
tokens = md.parse("- [user experience] Design #ux")
|
||||
obs_token = next(t for t in tokens if t.meta and 'observation' in t.meta)
|
||||
obs = obs_token.meta['observation']
|
||||
assert obs['category'] == 'user experience'
|
||||
|
||||
# Parentheses in content
|
||||
tokens = md.parse("- [code] Function (x) returns y #function")
|
||||
obs_token = next(t for t in tokens if t.meta and 'observation' in t.meta)
|
||||
obs = obs_token.meta['observation']
|
||||
assert obs['content'] == 'Function (x) returns y'
|
||||
assert obs['context'] is None
|
||||
|
||||
# Multiple hashtags together
|
||||
tokens = md.parse("- [test] Feature #important#urgent#now")
|
||||
obs_token = next(t for t in tokens if t.meta and 'observation' in t.meta)
|
||||
obs = obs_token.meta['observation']
|
||||
assert set(obs['tags']) == {'important', 'urgent', 'now'}
|
||||
|
||||
|
||||
def test_relation_plugin():
|
||||
"""Test relation plugin parsing."""
|
||||
md = MarkdownIt().use(relation_plugin)
|
||||
|
||||
# Basic relation
|
||||
tokens = md.parse("- implements [[Auth Service]]")
|
||||
rel_token = next(t for t in tokens if t.meta and 'relations' in t.meta)
|
||||
rel = rel_token.meta['relations'][0]
|
||||
|
||||
assert rel['type'] == 'implements'
|
||||
assert rel['target'] == 'Auth Service'
|
||||
assert rel['context'] is None
|
||||
|
||||
# With context
|
||||
tokens = md.parse("- depends_on [[Database]] (Required for persistence)")
|
||||
rel_token = next(t for t in tokens if t.meta and 'relations' in t.meta)
|
||||
rel = rel_token.meta['relations'][0]
|
||||
|
||||
assert rel['type'] == 'depends_on'
|
||||
assert rel['target'] == 'Database'
|
||||
assert rel['context'] == 'Required for persistence'
|
||||
|
||||
|
||||
def test_relation_edge_cases():
|
||||
"""Test relation plugin edge cases."""
|
||||
md = MarkdownIt().use(relation_plugin)
|
||||
|
||||
# Multiple word type
|
||||
tokens = md.parse("- is used by [[Client App]] (Primary consumer)")
|
||||
rel_token = next(t for t in tokens if t.meta and 'relations' in t.meta)
|
||||
rel = rel_token.meta['relations'][0]
|
||||
assert rel['type'] == 'is used by'
|
||||
|
||||
# Extra spaces
|
||||
tokens = md.parse("- specifies [[Format]] (Documentation)")
|
||||
rel_token = next(t for t in tokens if t.meta and 'relations' in t.meta)
|
||||
rel = rel_token.meta['relations'][0]
|
||||
assert rel['type'] == 'specifies'
|
||||
assert rel['target'] == 'Format'
|
||||
|
||||
|
||||
def test_inline_relations():
|
||||
"""Test finding relations in regular content."""
|
||||
md = MarkdownIt().use(relation_plugin)
|
||||
|
||||
# Single inline link
|
||||
tokens = md.parse("This references [[Another Doc]].")
|
||||
rel_token = next(t for t in tokens if t.meta and 'relations' in t.meta)
|
||||
rel = rel_token.meta['relations'][0]
|
||||
assert rel['type'] == 'mentions'
|
||||
assert rel['target'] == 'Another Doc'
|
||||
|
||||
# Multiple inline links
|
||||
tokens = md.parse("Links to [[Doc1]] and [[Doc2]].")
|
||||
rel_token = next(t for t in tokens if t.meta and 'relations' in t.meta)
|
||||
rels = rel_token.meta['relations']
|
||||
assert len(rels) == 2
|
||||
assert {r['target'] for r in rels} == {'Doc1', 'Doc2'}
|
||||
|
||||
|
||||
def test_combined_plugins():
|
||||
"""Test both plugins working together."""
|
||||
md = (MarkdownIt()
|
||||
.use(observation_plugin)
|
||||
.use(relation_plugin))
|
||||
|
||||
content = """# Document
|
||||
|
||||
Some text with a [[Link]].
|
||||
|
||||
## Observations
|
||||
- [tech] Implements [[Feature]] #important
|
||||
- [design] References [[AnotherDoc]] (For context)
|
||||
"""
|
||||
|
||||
tokens = md.parse(content)
|
||||
|
||||
# Should find both observations and relations
|
||||
observations = [t.meta['observation'] for t in tokens if t.meta and 'observation' in t.meta]
|
||||
relations = [r for t in tokens if t.meta and 'relations' in t.meta for r in t.meta['relations']]
|
||||
|
||||
assert len(observations) == 2
|
||||
assert any(o['category'] == 'tech' and 'important' in o['tags'] for o in observations)
|
||||
assert any(o['category'] == 'design' and o['context'] == 'For context' for o in observations)
|
||||
|
||||
# Should find all relations (Feature, AnotherDoc, and Link)
|
||||
assert len(relations) == 3
|
||||
assert any(r['target'] == 'Link' and r['type'] == 'mentions' for r in relations)
|
||||
assert any(r['target'] == 'Feature' for r in relations)
|
||||
assert any(r['target'] == 'AnotherDoc' for r in relations)
|
||||
@@ -1,95 +1,142 @@
|
||||
"""Tests for edge cases in observation parsing."""
|
||||
|
||||
import pytest
|
||||
from markdown_it import MarkdownIt
|
||||
|
||||
from basic_memory.markdown import ParseError, KnowledgeParser
|
||||
from basic_memory.markdown.plugins import (
|
||||
observation_plugin,
|
||||
is_observation,
|
||||
parse_observation
|
||||
)
|
||||
from basic_memory.markdown.schemas import Observation
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_observation_empty_input():
|
||||
def test_empty_input():
|
||||
"""Test handling of empty input."""
|
||||
parser = KnowledgeParser()
|
||||
assert await parser.parse_observation("") is None
|
||||
assert await parser.parse_observation(" ") is None
|
||||
assert await parser.parse_observation("\n") is None
|
||||
md = MarkdownIt().use(observation_plugin)
|
||||
|
||||
tokens = md.parse("")
|
||||
assert not any(t.meta and 'observation' in t.meta for t in tokens)
|
||||
|
||||
tokens = md.parse(" ")
|
||||
assert not any(t.meta and 'observation' in t.meta for t in tokens)
|
||||
|
||||
tokens = md.parse("\n")
|
||||
assert not any(t.meta and 'observation' in t.meta for t in tokens)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_observation_invalid_context():
|
||||
def test_invalid_context():
|
||||
"""Test handling of invalid context format."""
|
||||
parser = KnowledgeParser()
|
||||
obs = await parser.parse_observation("- [test] Content (unclosed")
|
||||
assert obs is not None
|
||||
assert obs.content == "Content (unclosed"
|
||||
assert obs.context is None
|
||||
|
||||
obs = await parser.parse_observation("- [test] Content (with) extra) parens)")
|
||||
assert obs is not None
|
||||
assert obs.content == "Content"
|
||||
assert obs.context == "with) extra) parens"
|
||||
md = MarkdownIt().use(observation_plugin)
|
||||
|
||||
# Unclosed context
|
||||
tokens = md.parse("- [test] Content (unclosed")
|
||||
token = next(t for t in tokens if t.type == 'inline')
|
||||
obs = parse_observation(token)
|
||||
assert obs['content'] == "Content (unclosed"
|
||||
assert obs['context'] is None
|
||||
|
||||
# Multiple parens
|
||||
tokens = md.parse("- [test] Content (with) extra) parens)")
|
||||
token = next(t for t in tokens if t.type == 'inline')
|
||||
obs = parse_observation(token)
|
||||
assert obs['content'] == "Content"
|
||||
assert obs['context'] == "with) extra) parens"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_observation_complex_format():
|
||||
def test_complex_format():
|
||||
"""Test parsing complex observation formats."""
|
||||
# Test multiple nested tags and spaces
|
||||
parser = KnowledgeParser()
|
||||
obs = await parser.parse_observation("- [complex test] This is #tag1#tag2 with #tag3 content")
|
||||
assert obs is not None
|
||||
assert obs.category == "complex test"
|
||||
assert set(obs.tags) == {"tag1", "tag2", "tag3"} # pyright: ignore [reportArgumentType]
|
||||
assert obs.content == "This is with content"
|
||||
md = MarkdownIt().use(observation_plugin)
|
||||
|
||||
# Multiple hashtags together
|
||||
tokens = md.parse("- [complex test] This is #tag1#tag2 with #tag3 content")
|
||||
token = next(t for t in tokens if t.type == 'inline')
|
||||
|
||||
obs = parse_observation(token)
|
||||
assert obs['category'] == "complex test"
|
||||
assert set(obs['tags']) == {"tag1", "tag2", "tag3"}
|
||||
assert obs['content'] == "This is with content"
|
||||
|
||||
# Pydantic model validation
|
||||
observation = Observation.model_validate(obs)
|
||||
assert observation.category == "complex test"
|
||||
assert set(observation.tags) == {"tag1", "tag2", "tag3"}
|
||||
assert observation.content == "This is with content"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_observation_exception_handling():
|
||||
"""Test general error handling in observation parsing."""
|
||||
# Test with a problematic regex pattern that could cause catastrophic backtracking
|
||||
long_input = "[test] " + "a" * 1000000 # Very long input
|
||||
parser = KnowledgeParser()
|
||||
|
||||
assert await parser.parse_observation(long_input) is None
|
||||
|
||||
# Test with invalid types
|
||||
assert await parser.parse_observation(None) is None # type: ignore
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_observation_malformed_category():
|
||||
def test_malformed_category():
|
||||
"""Test handling of malformed category brackets."""
|
||||
parser = KnowledgeParser()
|
||||
with pytest.raises(ParseError, match="unclosed category"):
|
||||
await parser.parse_observation("- [test Content")
|
||||
md = MarkdownIt().use(observation_plugin)
|
||||
|
||||
# Empty category
|
||||
tokens = md.parse("- [] Empty category")
|
||||
token = next(t for t in tokens if t.type == 'inline')
|
||||
obs = parse_observation(token)
|
||||
assert obs['category'] is None
|
||||
assert obs['content'] == "Empty category"
|
||||
|
||||
# Missing close bracket
|
||||
tokens = md.parse("- [test Content")
|
||||
token = next(t for t in tokens if t.type == 'inline')
|
||||
obs = parse_observation(token)
|
||||
# Should treat whole thing as content
|
||||
assert obs['category'] is None
|
||||
assert "test Content" in obs['content']
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_observation_empty_category():
|
||||
parser = KnowledgeParser()
|
||||
obs = await parser.parse_observation("- [] Empty category")
|
||||
|
||||
assert obs is not None
|
||||
assert obs.category is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_observation_whitespace():
|
||||
"""Test handling of whitespace."""
|
||||
# Valid whitespace cases
|
||||
parser = KnowledgeParser()
|
||||
obs = await parser.parse_observation("- [test] Content")
|
||||
assert obs is not None
|
||||
assert obs.content == "Content"
|
||||
|
||||
# Test individual whitespace chars
|
||||
test_chars = {
|
||||
" ": "space",
|
||||
"\t": "tab",
|
||||
#'\n': 'newline', # newline should be end of content
|
||||
"\r": "return",
|
||||
}
|
||||
|
||||
def test_whitespace_handling():
|
||||
"""Test handling of various whitespace."""
|
||||
md = MarkdownIt().use(observation_plugin)
|
||||
|
||||
# Various whitespace in content
|
||||
test_chars = {" ": "space", "\t": "tab", "\r": "return"}
|
||||
for char, name in test_chars.items():
|
||||
content = f"- [test] Content{char}with{char}{name}"
|
||||
obs = await parser.parse_observation(content)
|
||||
assert obs is not None
|
||||
assert obs.content == f"Content with {name}"
|
||||
tokens = md.parse(content)
|
||||
token = next(t for t in tokens if t.type == 'inline')
|
||||
obs = parse_observation(token)
|
||||
assert obs['content'] == f"Content with {name}"
|
||||
|
||||
# Indented observation
|
||||
tokens = md.parse(" - [test] Indented")
|
||||
assert any(t.meta and 'observation' in t.meta for t in tokens)
|
||||
|
||||
# Multiple blank lines
|
||||
text = """
|
||||
|
||||
- [test] After blanks
|
||||
|
||||
"""
|
||||
tokens = md.parse(text)
|
||||
assert any(t.meta and 'observation' in t.meta for t in tokens)
|
||||
|
||||
|
||||
def test_unicode_content():
|
||||
"""Test handling of Unicode content."""
|
||||
md = MarkdownIt().use(observation_plugin)
|
||||
|
||||
# Emoji
|
||||
tokens = md.parse("- [test] Emoji test 👍 #emoji #test (Testing emoji)")
|
||||
token = next(t for t in tokens if t.type == 'inline')
|
||||
obs = parse_observation(token)
|
||||
assert "👍" in obs['content']
|
||||
assert "emoji" in obs['tags']
|
||||
|
||||
# Non-Latin scripts
|
||||
tokens = md.parse("- [中文] Chinese text 测试 #language (Script test)")
|
||||
token = next(t for t in tokens if t.type == 'inline')
|
||||
obs = parse_observation(token)
|
||||
assert obs['category'] == "中文"
|
||||
assert "测试" in obs['content']
|
||||
|
||||
# Mixed scripts and emoji
|
||||
tokens = md.parse("- [test] Mixed 中文 and 👍 #mixed")
|
||||
token = next(t for t in tokens if t.type == 'inline')
|
||||
obs = parse_observation(token)
|
||||
assert "中文" in obs['content']
|
||||
assert "👍" in obs['content']
|
||||
|
||||
# Model validation with Unicode
|
||||
observation = Observation.model_validate(obs)
|
||||
assert "中文" in observation.content
|
||||
assert "👍" in observation.content
|
||||
@@ -1,88 +0,0 @@
|
||||
"""Tests for the markdown entity parser."""
|
||||
|
||||
import pytest
|
||||
|
||||
from basic_memory.markdown.knowledge_parser import KnowledgeParser, ParseError
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_parse_observation_basic():
|
||||
"""Test basic observation parsing with category and tags."""
|
||||
parser = KnowledgeParser()
|
||||
|
||||
obs = await parser.parse_observation("- [design] Core feature #important #mvp")
|
||||
|
||||
assert obs is not None
|
||||
assert obs.category == "design"
|
||||
assert obs.content == "Core feature"
|
||||
assert set(obs.tags) == {"important", "mvp"} # pyright: ignore [reportArgumentType]
|
||||
assert obs.context is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_parse_observation_with_context():
|
||||
"""Test observation parsing with context in parentheses."""
|
||||
parser = KnowledgeParser()
|
||||
|
||||
obs = await parser.parse_observation(
|
||||
"- [feature] Authentication system #security #auth (Required for MVP)"
|
||||
)
|
||||
assert obs is not None
|
||||
assert obs.category == "feature"
|
||||
assert obs.content == "Authentication system"
|
||||
assert set(obs.tags) == {"security", "auth"} # pyright: ignore [reportArgumentType]
|
||||
assert obs.context == "Required for MVP"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_parse_observation_without_category():
|
||||
"""Test observation parsing with context in parentheses."""
|
||||
parser = KnowledgeParser()
|
||||
|
||||
obs = await parser.parse_observation(
|
||||
"- Authentication system #security #auth (Required for MVP)"
|
||||
)
|
||||
assert obs is not None
|
||||
assert obs.category is None
|
||||
assert obs.content == "Authentication system"
|
||||
assert set(obs.tags) == {"security", "auth"} # pyright: ignore [reportArgumentType]
|
||||
assert obs.context == "Required for MVP"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_parse_observation_edge_cases():
|
||||
"""Test observation parsing edge cases."""
|
||||
parser = KnowledgeParser()
|
||||
|
||||
# Multiple word tags
|
||||
obs = await parser.parse_observation("- [tech] Database #high-priority #needs-review")
|
||||
|
||||
assert obs is not None
|
||||
|
||||
assert set(obs.tags) == {"high-priority", "needs-review"} # pyright: ignore [reportArgumentType]
|
||||
|
||||
# Multiple word category
|
||||
obs = await parser.parse_observation("- [user experience] Design #ux")
|
||||
assert obs is not None
|
||||
assert obs.category == "user experience"
|
||||
|
||||
# Parentheses in content shouldn't be treated as context
|
||||
obs = await parser.parse_observation("- [code] Function (x) returns y #function")
|
||||
assert obs is not None
|
||||
assert obs.content == "Function (x) returns y"
|
||||
assert obs.context is None
|
||||
|
||||
# Multiple hashtags together
|
||||
obs = await parser.parse_observation("- [test] Feature #important#urgent#now")
|
||||
assert obs is not None
|
||||
assert set(obs.tags) == {"important", "urgent", "now"} # pyright: ignore [reportArgumentType]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_parse_observation_errors():
|
||||
"""Test error handling in observation parsing."""
|
||||
parser = KnowledgeParser()
|
||||
|
||||
# Unclosed category
|
||||
with pytest.raises(ParseError, match="unclosed category"):
|
||||
await parser.parse_observation("- [design Core feature #test")
|
||||
@@ -1,65 +0,0 @@
|
||||
"""Tests for the markdown entity parser."""
|
||||
|
||||
import pytest
|
||||
|
||||
from basic_memory.markdown.knowledge_parser import KnowledgeParser, ParseError
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_parse_relation_basic():
|
||||
"""Test basic relation parsing."""
|
||||
parser = KnowledgeParser()
|
||||
|
||||
rel = await parser.parse_relation("- implements [[Auth Service]]")
|
||||
assert rel is not None
|
||||
assert rel.type == "implements"
|
||||
assert rel.target == "Auth Service"
|
||||
assert rel.context is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_parse_relation_with_context():
|
||||
"""Test relation parsing with context."""
|
||||
parser = KnowledgeParser()
|
||||
|
||||
rel = await parser.parse_relation("- depends_on [[Database]] (Required for persistence)")
|
||||
assert rel is not None
|
||||
assert rel.type == "depends_on"
|
||||
assert rel.target == "Database"
|
||||
assert rel.context == "Required for persistence"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_parse_relation_edge_cases():
|
||||
"""Test relation parsing edge cases."""
|
||||
parser = KnowledgeParser()
|
||||
|
||||
# Multiple word type
|
||||
rel = await parser.parse_relation("- is used by [[Client App]] (Primary consumer)")
|
||||
assert rel is not None
|
||||
assert rel.type == "is used by"
|
||||
|
||||
# Brackets in context
|
||||
rel = await parser.parse_relation("- implements [[API]] (Follows [OpenAPI] spec)")
|
||||
assert rel is not None
|
||||
assert rel.context == "Follows [OpenAPI] spec"
|
||||
|
||||
# Extra spaces
|
||||
rel = await parser.parse_relation("- specifies [[Format]] (Documentation)")
|
||||
assert rel is not None
|
||||
assert rel.type == "specifies"
|
||||
assert rel.target == "Format"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_parse_relation_errors():
|
||||
"""Test error handling in relation parsing."""
|
||||
parser = KnowledgeParser()
|
||||
|
||||
# Missing target brackets
|
||||
with pytest.raises(ParseError, match="missing \\[\\["):
|
||||
await parser.parse_relation("- implements Auth Service")
|
||||
|
||||
# Unclosed target
|
||||
with pytest.raises(ParseError, match="Invalid relation format - missing \\[\\[entity\\]\\]"):
|
||||
await parser.parse_relation("- implements [[Auth Service")
|
||||
@@ -4,9 +4,10 @@ from pathlib import Path
|
||||
from textwrap import dedent
|
||||
|
||||
import pytest
|
||||
from markdown_it import MarkdownIt
|
||||
|
||||
from basic_memory.markdown.knowledge_parser import KnowledgeParser
|
||||
from basic_memory.utils.file_utils import FileError, ParseError
|
||||
from basic_memory.markdown.entity_parser import EntityParser
|
||||
from basic_memory.utils.file_utils import FileError
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -20,164 +21,154 @@ async def test_unicode_content(tmp_path):
|
||||
modified: 2024-12-21T14:00:00Z
|
||||
tags: [unicode, 测试]
|
||||
---
|
||||
|
||||
|
||||
# Unicode Test 🧪
|
||||
|
||||
|
||||
## Observations
|
||||
- [test] Emoji test 👍 #emoji #test (Testing emoji)
|
||||
- [中文] Chinese text 测试 #language (Script test)
|
||||
- [русский] Russian привет #language (More scripts)
|
||||
- [note] Emoji in text 😀 #meta (Category test)
|
||||
|
||||
|
||||
## Relations
|
||||
- tested_by [[测试组件]] (Unicode test)
|
||||
- depends_on [[компонент]] (Another test)
|
||||
|
||||
## Metadata
|
||||
```yml
|
||||
category: test
|
||||
status: active
|
||||
```
|
||||
""")
|
||||
|
||||
|
||||
test_file = tmp_path / "unicode.md"
|
||||
test_file.write_text(content, encoding="utf-8")
|
||||
|
||||
parser = KnowledgeParser()
|
||||
|
||||
parser = EntityParser(tmp_path)
|
||||
entity = await parser.parse_file(test_file)
|
||||
|
||||
|
||||
assert "测试" in entity.frontmatter.tags
|
||||
assert "китайский" not in entity.frontmatter.tags
|
||||
assert entity.content.title == "Unicode Test 🧪"
|
||||
assert "chinese" not in entity.frontmatter.tags
|
||||
assert "🧪" in entity.content.content
|
||||
|
||||
# Verify Unicode in observations
|
||||
assert any(o.content == "Emoji test 👍" for o in entity.content.observations)
|
||||
assert any(o.category == "中文" for o in entity.content.observations)
|
||||
assert any(o.category == "русский" for o in entity.content.observations)
|
||||
|
||||
# Verify Unicode in relations
|
||||
assert any(r.target == "测试组件" for r in entity.content.relations)
|
||||
assert any(r.target == "компонент" for r in entity.content.relations)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fallback_encoding(tmp_path):
|
||||
"""Test UTF-16 fallback when UTF-8 fails."""
|
||||
async def test_empty_file(tmp_path):
|
||||
"""Test handling of empty files."""
|
||||
empty_file = tmp_path / "empty.md"
|
||||
empty_file.write_text("")
|
||||
|
||||
parser = EntityParser(tmp_path)
|
||||
entity = await parser.parse_file(empty_file)
|
||||
assert entity.content.observations == []
|
||||
assert entity.content.relations == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_missing_sections(tmp_path):
|
||||
"""Test handling of files with missing sections."""
|
||||
content = dedent("""
|
||||
Hello 世界
|
||||
No proper sections here
|
||||
---
|
||||
type: test
|
||||
id: test/missing
|
||||
created: 2024-01-09
|
||||
modified: 2024-01-09
|
||||
tags: []
|
||||
---
|
||||
|
||||
Just some content
|
||||
with [[links]] but no sections
|
||||
""")
|
||||
test_file = tmp_path / "unicode_file.md"
|
||||
test_file.write_text(content, encoding="utf-16")
|
||||
|
||||
parser = KnowledgeParser()
|
||||
with pytest.raises(ParseError):
|
||||
await parser.parse_file(test_file)
|
||||
|
||||
test_file = tmp_path / "missing.md"
|
||||
test_file.write_text(content)
|
||||
|
||||
parser = EntityParser(tmp_path)
|
||||
entity = await parser.parse_file(test_file)
|
||||
assert len(entity.content.relations) == 1
|
||||
assert entity.content.relations[0].target == "links"
|
||||
assert entity.content.relations[0].type == "mentions"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_encoding_errors(tmp_path):
|
||||
"""Test handling of encoding errors."""
|
||||
# Create a file with invalid UTF-8 bytes
|
||||
test_file = tmp_path / "invalid.md"
|
||||
with open(test_file, "wb") as f:
|
||||
f.write(b"\xff\xfe\x00\x00") # Invalid UTF-8
|
||||
async def test_nested_content(tmp_path):
|
||||
"""Test handling of deeply nested content."""
|
||||
content = dedent("""
|
||||
---
|
||||
type: test
|
||||
id: test/nested
|
||||
created: 2024-01-09
|
||||
modified: 2024-01-09
|
||||
tags: []
|
||||
---
|
||||
|
||||
# Test
|
||||
|
||||
## Level 1
|
||||
- [test] Level 1 #test (First level)
|
||||
- implements [[One]]
|
||||
|
||||
### Level 2
|
||||
- [test] Level 2 #test (Second level)
|
||||
- uses [[Two]]
|
||||
|
||||
#### Level 3
|
||||
- [test] Level 3 #test (Third level)
|
||||
- needs [[Three]]
|
||||
""")
|
||||
|
||||
test_file = tmp_path / "nested.md"
|
||||
test_file.write_text(content)
|
||||
|
||||
parser = EntityParser(tmp_path)
|
||||
entity = await parser.parse_file(test_file)
|
||||
|
||||
# Should find all observations and relations regardless of nesting
|
||||
assert len(entity.content.observations) == 3
|
||||
assert len(entity.content.relations) == 3
|
||||
assert {r.target for r in entity.content.relations} == {"One", "Two", "Three"}
|
||||
|
||||
parser = KnowledgeParser()
|
||||
with pytest.raises(ParseError):
|
||||
await parser.parse_file(test_file, encoding="ascii")
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_malformed_frontmatter(tmp_path):
|
||||
"""Test handling of malformed frontmatter."""
|
||||
# Missing fields
|
||||
content = dedent("""
|
||||
---
|
||||
type: test
|
||||
---
|
||||
|
||||
# Test
|
||||
""")
|
||||
|
||||
test_file = tmp_path / "malformed.md"
|
||||
test_file.write_text(content)
|
||||
|
||||
parser = EntityParser(tmp_path)
|
||||
entity = await parser.parse_file(test_file)
|
||||
assert entity.frontmatter.id == "malformed" # Generated from filename
|
||||
|
||||
# Invalid YAML
|
||||
content = dedent("""
|
||||
---
|
||||
type: test
|
||||
tags: [unclosed
|
||||
---
|
||||
|
||||
# Test
|
||||
""")
|
||||
|
||||
test_file.write_text(content)
|
||||
entity = await parser.parse_file(test_file)
|
||||
assert entity.frontmatter.tags == [] # Default to empty list
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_file_not_found():
|
||||
"""Test handling of non-existent files."""
|
||||
parser = KnowledgeParser()
|
||||
parser = EntityParser(Path("/tmp"))
|
||||
with pytest.raises(FileError):
|
||||
await parser.parse_file(Path("nonexistent.md"))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_nested_structures(tmp_path):
|
||||
"""Test handling of nested markdown structures."""
|
||||
content = dedent("""
|
||||
---
|
||||
type: test
|
||||
id: test/nested
|
||||
created: 2024-12-21T14:00:00Z
|
||||
modified: 2024-12-21T14:00:00Z
|
||||
tags: [test]
|
||||
---
|
||||
|
||||
# Nested Test
|
||||
|
||||
## Observations
|
||||
- [test] Main point #main (Top level)
|
||||
- [test] Subpoint #sub (Should be ignored)
|
||||
- [test] Sub-subpoint #detail (Also ignored)
|
||||
|
||||
## Relations
|
||||
- depends_on [[Sub Entity]] (Top level)
|
||||
- uses [[Another Entity]] (Should be ignored)
|
||||
- implements [[Third Entity]] (Also ignored)
|
||||
""")
|
||||
|
||||
test_file = tmp_path / "nested.md"
|
||||
test_file.write_text(content)
|
||||
|
||||
parser = KnowledgeParser()
|
||||
entity = await parser.parse_file(test_file)
|
||||
|
||||
assert len(entity.content.observations) == 3
|
||||
assert len(entity.content.relations) == 3
|
||||
assert entity.content.observations[0].tags == ["main"]
|
||||
assert entity.content.observations[0].context == "Top level"
|
||||
assert entity.content.relations[0].target == "Sub Entity"
|
||||
assert entity.content.relations[0].type == "depends_on"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_malformed_sections(tmp_path):
|
||||
"""Test various malformed section contents."""
|
||||
content = dedent("""
|
||||
---
|
||||
type: test
|
||||
id: test/malformed
|
||||
created: 2024-12-21T14:00:00Z
|
||||
modified: 2024-12-21T14:00:00Z
|
||||
tags: [test]
|
||||
---
|
||||
|
||||
# Malformed Test
|
||||
|
||||
## Observations
|
||||
- not a valid observation
|
||||
- [unclosed category content
|
||||
- no content]
|
||||
- [] empty category
|
||||
|
||||
## Relations
|
||||
- not a valid relation
|
||||
- missing_brackets Entity
|
||||
- implements incomplete [[
|
||||
- implements ]] backwards
|
||||
""")
|
||||
|
||||
test_file = tmp_path / "malformed.md"
|
||||
test_file.write_text(content)
|
||||
|
||||
parser = KnowledgeParser()
|
||||
with pytest.raises(ParseError):
|
||||
entity = await parser.parse_file(test_file)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_missing_required_sections(tmp_path):
|
||||
"""Test handling of missing required sections."""
|
||||
# Test file with only frontmatter
|
||||
content = dedent("""
|
||||
---
|
||||
type: test
|
||||
id: test/incomplete
|
||||
created: 2024-12-21T14:00:00Z
|
||||
modified: 2024-12-21T14:00:00Z
|
||||
tags: [test]
|
||||
---
|
||||
""")
|
||||
|
||||
test_file = tmp_path / "incomplete.md"
|
||||
test_file.write_text(content)
|
||||
|
||||
parser = KnowledgeParser()
|
||||
with pytest.raises(ParseError):
|
||||
await parser.parse_file(test_file)
|
||||
|
||||
@@ -1,21 +1,157 @@
|
||||
"""Tests for edge cases in relation parsing."""
|
||||
|
||||
import pytest
|
||||
from markdown_it import MarkdownIt
|
||||
|
||||
from basic_memory.markdown import KnowledgeParser
|
||||
from basic_memory.markdown.plugins import (
|
||||
relation_plugin,
|
||||
is_explicit_relation,
|
||||
parse_relation,
|
||||
parse_inline_relations
|
||||
)
|
||||
from basic_memory.markdown.schemas import Relation
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_relation_empty_target():
|
||||
def test_empty_targets():
|
||||
"""Test handling of empty targets."""
|
||||
md = MarkdownIt().use(relation_plugin)
|
||||
|
||||
# Empty brackets
|
||||
parser = KnowledgeParser()
|
||||
|
||||
assert await parser.parse_relation("type [[]]") is None
|
||||
assert await parser.parse_relation("type [[ ]]") is None
|
||||
|
||||
tokens = md.parse("- type [[]]")
|
||||
token = next(t for t in tokens if t.type == 'inline')
|
||||
assert parse_relation(token) is None
|
||||
|
||||
# Only spaces
|
||||
assert await parser.parse_relation("type [[ ]]") is None
|
||||
tokens = md.parse("- type [[ ]]")
|
||||
token = next(t for t in tokens if t.type == 'inline')
|
||||
assert parse_relation(token) is None
|
||||
|
||||
# Whitespace in brackets
|
||||
tokens = md.parse("- type [[ ]]")
|
||||
token = next(t for t in tokens if t.type == 'inline')
|
||||
assert parse_relation(token) is None
|
||||
|
||||
# Only white spaces
|
||||
assert await parser.parse_relation(" ") is None
|
||||
|
||||
def test_malformed_links():
|
||||
"""Test handling of malformed wiki links."""
|
||||
md = MarkdownIt().use(relation_plugin)
|
||||
|
||||
# Missing close brackets
|
||||
tokens = md.parse("- type [[Target")
|
||||
assert not any(t.meta and 'relations' in t.meta for t in tokens)
|
||||
|
||||
# Missing open brackets
|
||||
tokens = md.parse("- type Target]]")
|
||||
assert not any(t.meta and 'relations' in t.meta for t in tokens)
|
||||
|
||||
# Backwards brackets
|
||||
tokens = md.parse("- type ]]Target[[")
|
||||
assert not any(t.meta and 'relations' in t.meta for t in tokens)
|
||||
|
||||
# Nested brackets
|
||||
tokens = md.parse("- type [[Outer [[Inner]] ]]")
|
||||
token = next(t for t in tokens if t.type == 'inline')
|
||||
rel = parse_relation(token)
|
||||
assert rel is not None
|
||||
assert "Outer" in rel['target']
|
||||
|
||||
|
||||
def test_context_handling():
|
||||
"""Test handling of contexts."""
|
||||
md = MarkdownIt().use(relation_plugin)
|
||||
|
||||
# Unclosed context
|
||||
tokens = md.parse("- type [[Target]] (unclosed")
|
||||
token = next(t for t in tokens if t.type == 'inline')
|
||||
rel = parse_relation(token)
|
||||
assert rel['context'] is None
|
||||
|
||||
# Multiple parens
|
||||
tokens = md.parse("- type [[Target]] (with (nested) parens)")
|
||||
token = next(t for t in tokens if t.type == 'inline')
|
||||
rel = parse_relation(token)
|
||||
assert rel['context'] == "with (nested) parens"
|
||||
|
||||
# Empty context
|
||||
tokens = md.parse("- type [[Target]] ()")
|
||||
token = next(t for t in tokens if t.type == 'inline')
|
||||
rel = parse_relation(token)
|
||||
assert rel['context'] is None
|
||||
|
||||
|
||||
def test_inline_relations():
|
||||
"""Test inline relation detection."""
|
||||
md = MarkdownIt().use(relation_plugin)
|
||||
|
||||
# Multiple links in text
|
||||
text = "Text with [[Link1]] and [[Link2]] and [[Link3]]"
|
||||
rels = parse_inline_relations(text)
|
||||
assert len(rels) == 3
|
||||
assert {r['target'] for r in rels} == {'Link1', 'Link2', 'Link3'}
|
||||
|
||||
# Links with surrounding text
|
||||
text = "Before [[Target]] After"
|
||||
rels = parse_inline_relations(text)
|
||||
assert len(rels) == 1
|
||||
assert rels[0]['target'] == 'Target'
|
||||
|
||||
# Multiple links on same line
|
||||
tokens = md.parse("[[One]] [[Two]] [[Three]]")
|
||||
token = next(t for t in tokens if t.type == 'inline')
|
||||
assert len(token.meta['relations']) == 3
|
||||
|
||||
|
||||
def test_unicode_targets():
|
||||
"""Test handling of Unicode in targets."""
|
||||
md = MarkdownIt().use(relation_plugin)
|
||||
|
||||
# Unicode in target
|
||||
tokens = md.parse("- type [[测试]]")
|
||||
token = next(t for t in tokens if t.type == 'inline')
|
||||
rel = parse_relation(token)
|
||||
assert rel['target'] == "测试"
|
||||
|
||||
# Unicode in type
|
||||
tokens = md.parse("- 使用 [[Target]]")
|
||||
token = next(t for t in tokens if t.type == 'inline')
|
||||
rel = parse_relation(token)
|
||||
assert rel['type'] == "使用"
|
||||
|
||||
# Unicode in context
|
||||
tokens = md.parse("- type [[Target]] (测试)")
|
||||
token = next(t for t in tokens if t.type == 'inline')
|
||||
rel = parse_relation(token)
|
||||
assert rel['context'] == "测试"
|
||||
|
||||
# Model validation with Unicode
|
||||
relation = Relation.model_validate(rel)
|
||||
assert relation.type == "type"
|
||||
assert relation.target == "Target"
|
||||
assert relation.context == "测试"
|
||||
|
||||
|
||||
def test_nested_content():
|
||||
"""Test handling of nested content structures."""
|
||||
md = MarkdownIt().use(relation_plugin)
|
||||
|
||||
content = """
|
||||
# Section
|
||||
|
||||
Text with [[Link]]
|
||||
|
||||
## Subsection
|
||||
- implements [[Component]] (In subsection)
|
||||
- uses [[Other]] (Nested, should be separate)
|
||||
|
||||
More text with [[Another Link]]
|
||||
"""
|
||||
|
||||
tokens = md.parse(content)
|
||||
relations = [r for t in tokens if t.meta and 'relations' in t.meta for r in t.meta['relations']]
|
||||
|
||||
# Should find all relations at any nesting level
|
||||
assert len(relations) >= 4 # All links should be found
|
||||
assert any(r['target'] == 'Link' for r in relations)
|
||||
assert any(r['target'] == 'Component' for r in relations)
|
||||
assert any(r['target'] == 'Other' for r in relations)
|
||||
assert any(r['target'] == 'Another Link' for r in relations)
|
||||
Reference in New Issue
Block a user