mirror of
https://github.com/basicmachines-co/basic-memory
synced 2026-06-21 13:47:35 +00:00
enity parsing tests
This commit is contained in:
phernandez
@@ -1,11 +1,9 @@
|
||||
"""Base package for markdown parsing."""
|
||||
|
||||
from basic_memory.markdown.knowledge_parser import KnowledgeParser
|
||||
from basic_memory.markdown.entity_parser import EntityParser
|
||||
from basic_memory.markdown.schemas import (
|
||||
EntityMarkdown,
|
||||
EntityContent,
|
||||
EntityFrontmatter,
|
||||
EntityMetadata,
|
||||
Observation,
|
||||
Relation,
|
||||
)
|
||||
@@ -15,8 +13,7 @@ __all__ = [
|
||||
"EntityMarkdown",
|
||||
"EntityContent",
|
||||
"EntityFrontmatter",
|
||||
"EntityMetadata",
|
||||
"KnowledgeParser",
|
||||
"EntityParser",
|
||||
"Observation",
|
||||
"Relation",
|
||||
"ParseError",
|
||||
|
||||
@@ -1,317 +1,112 @@
|
||||
"""Universal parser for markdown files with optional frontmatter, observations, and relations.
|
||||
"""Parser for markdown files into Entity objects.
|
||||
|
||||
The id field in frontmatter is derived from the filename, converted to snake_case with .md extension removed.
|
||||
For example:
|
||||
'My Project Notes.md' -> 'my_project_notes'
|
||||
'API-Design.md' -> 'api_design'
|
||||
Uses markdown-it with plugins to parse structured data from markdown content.
|
||||
"""
|
||||
|
||||
import re
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Dict, Any, Optional, Tuple, List
|
||||
from datetime import datetime
|
||||
from typing import Dict, Any, Optional
|
||||
|
||||
from loguru import logger
|
||||
from markdown_it import MarkdownIt
|
||||
import frontmatter
|
||||
|
||||
from basic_memory.markdown.base_parser import MarkdownParser, ParseError
|
||||
from basic_memory.markdown.plugins import observation_plugin, relation_plugin
|
||||
from basic_memory.markdown.schemas import (
|
||||
EntityMarkdown,
|
||||
EntityFrontmatter,
|
||||
EntityContent,
|
||||
EntityMetadata,
|
||||
Observation,
|
||||
Relation,
|
||||
EntityContent, Observation, Relation,
|
||||
)
|
||||
from basic_memory.schemas.base import to_snake_case
|
||||
|
||||
|
||||
class EntityParser(MarkdownParser[EntityMarkdown]):
|
||||
"""A forgiving parser that extracts as much structure as it can find.
|
||||
|
||||
Generates entity IDs from filenames by:
|
||||
1. Removing .md extension
|
||||
2. Converting to snake_case
|
||||
3. Removing any invalid characters
|
||||
"""
|
||||
class EntityParser:
|
||||
"""Parser for markdown files into Entity objects."""
|
||||
|
||||
def convert_to_id(self, filename: str) -> str:
|
||||
"""Convert a filename to a valid entity ID.
|
||||
|
||||
Args:
|
||||
filename: Name of the file (with or without .md extension)
|
||||
|
||||
Returns:
|
||||
Snake case version of filename without extension
|
||||
|
||||
Examples:
|
||||
'My Project Notes.md' -> 'my_project_notes'
|
||||
'API-Design.md' -> 'api_design'
|
||||
def __init__(self, base_path: Path):
|
||||
"""Initialize parser with base path for relative path_id generation."""
|
||||
self.base_path = base_path.resolve()
|
||||
self.md = (MarkdownIt()
|
||||
.use(observation_plugin)
|
||||
.use(relation_plugin))
|
||||
|
||||
def get_path_id(self, file_path: Path) -> str:
|
||||
"""Get path_id from file path relative to base_path.
|
||||
|
||||
Example:
|
||||
base_path: /project/root
|
||||
file_path: /project/root/design/models/data.md
|
||||
returns: "design/models/data"
|
||||
"""
|
||||
# Remove .md extension if present
|
||||
if filename.lower().endswith('.md'):
|
||||
filename = filename[:-3]
|
||||
|
||||
return to_snake_case(filename)
|
||||
|
||||
def parse_dates(self, frontmatter: Dict[str, Any], file_path: Path) -> Tuple[datetime, datetime]:
|
||||
"""Parse created and updated dates from frontmatter or file system.
|
||||
|
||||
Args:
|
||||
frontmatter: Dictionary containing frontmatter fields
|
||||
file_path: Path to the source file for fallback dates
|
||||
|
||||
Returns:
|
||||
Tuple of (created_date, updated_date)
|
||||
|
||||
Priority:
|
||||
1. Valid frontmatter dates
|
||||
2. File system dates (created/modified)
|
||||
"""
|
||||
created = None
|
||||
updated = None
|
||||
|
||||
# Try frontmatter first
|
||||
try:
|
||||
if 'created' in frontmatter:
|
||||
created = self.parse_date(frontmatter['created'])
|
||||
if 'updated' in frontmatter or 'modified' in frontmatter:
|
||||
updated = self.parse_date(frontmatter.get('updated') or frontmatter.get('modified'))
|
||||
except Exception as e:
|
||||
logger.warning(f"Error parsing frontmatter dates: {e}")
|
||||
|
||||
# Fall back to file system dates if needed
|
||||
try:
|
||||
stats = file_path.stat()
|
||||
if not created:
|
||||
created = datetime.fromtimestamp(stats.st_ctime)
|
||||
if not updated:
|
||||
updated = datetime.fromtimestamp(stats.st_mtime)
|
||||
except Exception as e:
|
||||
logger.warning(f"Error getting file stats: {e}")
|
||||
# Last resort - use current time
|
||||
now = datetime.now()
|
||||
created = created or now
|
||||
updated = updated or now
|
||||
|
||||
return created, updated
|
||||
# Get relative path and remove .md extension
|
||||
rel_path = file_path.resolve().relative_to(self.base_path)
|
||||
if rel_path.suffix.lower() == '.md':
|
||||
return str(rel_path.with_suffix(''))
|
||||
return str(rel_path)
|
||||
|
||||
def parse_date(self, value: Any) -> Optional[datetime]:
|
||||
"""Convert various date formats to datetime."""
|
||||
"""Parse various date formats into datetime."""
|
||||
if isinstance(value, datetime):
|
||||
return value
|
||||
try:
|
||||
if isinstance(value, str):
|
||||
if isinstance(value, str):
|
||||
try:
|
||||
return datetime.fromisoformat(value.replace("Z", "+00:00"))
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
return None
|
||||
|
||||
def parse_tags(self, tags: Any) -> List[str]:
|
||||
"""Convert various tag formats to list of strings."""
|
||||
async def parse_file(self, file_path: Path) -> EntityMarkdown:
|
||||
"""Parse markdown file into EntityMarkdown."""
|
||||
# Parse frontmatter and content using python-frontmatter
|
||||
post = frontmatter.load(str(file_path))
|
||||
|
||||
# Extract or generate required fields
|
||||
path_id = post.metadata.get("id") or self.get_path_id(file_path)
|
||||
stats = file_path.stat()
|
||||
|
||||
# Parse frontmatter
|
||||
entity_frontmatter = EntityFrontmatter(
|
||||
type=str(post.metadata.get("type", "note")),
|
||||
id=path_id,
|
||||
title=str(post.metadata.get("title", file_path.name)),
|
||||
created=self.parse_date(post.metadata.get("created"))
|
||||
or datetime.fromtimestamp(stats.st_ctime),
|
||||
modified=self.parse_date(post.metadata.get("modified"))
|
||||
or datetime.fromtimestamp(stats.st_mtime),
|
||||
tags=self.parse_tags(post.metadata.get("tags", [])),
|
||||
)
|
||||
|
||||
# Parse content for observations and relations using markdown-it
|
||||
tokens = self.md.parse(post.content)
|
||||
|
||||
# Extract observations and relations from token meta
|
||||
observations = []
|
||||
relations = []
|
||||
|
||||
for token in tokens:
|
||||
if token.meta: # Token might not have meta
|
||||
if 'observation' in token.meta:
|
||||
obs = token.meta['observation']
|
||||
observation = Observation.model_validate(obs)
|
||||
observations.append(observation)
|
||||
if 'relations' in token.meta:
|
||||
rels = token.meta['relations']
|
||||
relations.extend([Relation.model_validate(r) for r in rels])
|
||||
|
||||
# Create EntityContent
|
||||
entity_content = EntityContent(
|
||||
content=post.content,
|
||||
observations=observations,
|
||||
relations=relations,
|
||||
)
|
||||
|
||||
return EntityMarkdown(
|
||||
frontmatter=entity_frontmatter,
|
||||
content=entity_content,
|
||||
)
|
||||
|
||||
def parse_tags(self, tags: Any) -> list[str]:
|
||||
"""Parse tags into list of strings."""
|
||||
if isinstance(tags, str):
|
||||
return [t.strip() for t in tags.split(",") if t.strip()]
|
||||
if isinstance(tags, (list, tuple)):
|
||||
return [str(t).strip() for t in tags if str(t).strip()]
|
||||
return []
|
||||
|
||||
async def parse_frontmatter(self, frontmatter: Dict[str, Any], file_path: Optional[Path] = None) -> EntityFrontmatter:
|
||||
"""Parse frontmatter with sensible defaults for missing fields.
|
||||
|
||||
Args:
|
||||
frontmatter: Dictionary of frontmatter fields
|
||||
file_path: Optional path to source file, used for id and dates
|
||||
"""
|
||||
try:
|
||||
# Get or generate ID from filename
|
||||
entity_name = None
|
||||
if file_path:
|
||||
entity_name = self.convert_to_id(file_path.name)
|
||||
|
||||
# Get dates from frontmatter or file
|
||||
created, updated = self.parse_dates(frontmatter, file_path) if file_path else (datetime.now(), datetime.now())
|
||||
|
||||
# Ensure we have minimum required fields
|
||||
processed = {
|
||||
"type": str(frontmatter.get("type", "document")).strip(),
|
||||
"id": str(frontmatter.get("id", entity_name or "document")).strip(),
|
||||
"created": created,
|
||||
"modified": updated,
|
||||
"tags": self.parse_tags(frontmatter.get("tags", []))
|
||||
}
|
||||
|
||||
return EntityFrontmatter(**processed)
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"Error parsing frontmatter, using defaults: {e}")
|
||||
return EntityFrontmatter(
|
||||
type="document",
|
||||
id=entity_name or "document",
|
||||
created=created,
|
||||
modified=updated,
|
||||
tags=[]
|
||||
)
|
||||
|
||||
async def parse_content(self, title: str, sections: Dict[str, str]) -> EntityContent:
|
||||
"""Parse content sections without requiring any particular structure."""
|
||||
try:
|
||||
# Get content from content section if it exists
|
||||
content = sections.get("content", "").strip() or None
|
||||
|
||||
# Try to parse observations if they exist
|
||||
observations = []
|
||||
if "observations" in sections:
|
||||
for line in sections["observations"].splitlines():
|
||||
try:
|
||||
obs = await self.parse_observation(line)
|
||||
if obs:
|
||||
observations.append(obs)
|
||||
except ParseError as e:
|
||||
logger.warning(f"Skipping invalid observation: {e}")
|
||||
|
||||
# Try to parse relations if they exist
|
||||
relations = []
|
||||
if "relations" in sections:
|
||||
for line in sections["relations"].splitlines():
|
||||
try:
|
||||
rel = await self.parse_relation(line)
|
||||
if rel:
|
||||
relations.append(rel)
|
||||
except ParseError as e:
|
||||
logger.warning(f"Skipping invalid relation: {e}")
|
||||
|
||||
# Also look for wiki-links in content as implicit relations
|
||||
try:
|
||||
content_relations = await self.parse_content_relations(sections.get("content", ""))
|
||||
relations.extend(content_relations)
|
||||
except Exception as e:
|
||||
logger.warning(f"Error parsing content relations: {e}")
|
||||
|
||||
return EntityContent(
|
||||
title=title or "Untitled",
|
||||
summary=content,
|
||||
observations=observations,
|
||||
relations=relations
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error parsing content, using minimal structure: {e}")
|
||||
return EntityContent(
|
||||
title=title or "Untitled",
|
||||
summary=None,
|
||||
observations=[],
|
||||
relations=[]
|
||||
)
|
||||
|
||||
async def parse_observation(self, line: str) -> Optional[Observation]:
|
||||
"""Parse a single observation line."""
|
||||
if not line or not line.strip().startswith("-"):
|
||||
return None
|
||||
|
||||
line = line.strip()[1:].strip() # Remove leading "-" and whitespace
|
||||
|
||||
# Extract category if present
|
||||
category = None
|
||||
if line.startswith("["):
|
||||
end = line.find("]")
|
||||
if end != -1:
|
||||
category = line[1:end].strip()
|
||||
line = line[end + 1:].strip()
|
||||
|
||||
# Extract context if present
|
||||
context = None
|
||||
if line.endswith(")"):
|
||||
start = line.rfind("(")
|
||||
if start != -1:
|
||||
context = line[start + 1:-1].strip()
|
||||
line = line[:start].strip()
|
||||
|
||||
# Extract tags and content
|
||||
parts = line.split()
|
||||
content_parts = []
|
||||
tags = []
|
||||
|
||||
for part in parts:
|
||||
if part.startswith("#"):
|
||||
tags.append(part[1:])
|
||||
else:
|
||||
content_parts.append(part)
|
||||
|
||||
content = " ".join(content_parts).strip()
|
||||
if not content:
|
||||
return None
|
||||
|
||||
return Observation(
|
||||
category=category,
|
||||
content=content,
|
||||
tags=tags if tags else None,
|
||||
context=context
|
||||
)
|
||||
|
||||
async def parse_relation(self, line: str) -> Optional[Relation]:
|
||||
"""Parse a single relation line."""
|
||||
if not line or not line.strip().startswith("-"):
|
||||
return None
|
||||
|
||||
line = line.strip()[1:].strip()
|
||||
|
||||
# Look for [[target]]
|
||||
start = line.find("[[")
|
||||
end = line.find("]]")
|
||||
if start == -1 or end == -1:
|
||||
return None
|
||||
|
||||
# Extract parts
|
||||
rel_type = line[:start].strip() or "relates_to" # Default type if none specified
|
||||
target = line[start + 2:end].strip()
|
||||
|
||||
# Extract context if present
|
||||
context = None
|
||||
remaining = line[end + 2:].strip()
|
||||
if remaining.startswith("(") and remaining.endswith(")"):
|
||||
context = remaining[1:-1].strip()
|
||||
|
||||
if not target:
|
||||
return None
|
||||
|
||||
return Relation(
|
||||
type=rel_type,
|
||||
target=target,
|
||||
context=context
|
||||
)
|
||||
|
||||
async def parse_content_relations(self, content: str) -> List[Relation]:
|
||||
"""Extract wiki-style links from content as relations."""
|
||||
relations = []
|
||||
if not content:
|
||||
return relations
|
||||
|
||||
import re
|
||||
pattern = r'\[\[([^\]]+)\]\]'
|
||||
|
||||
for match in re.finditer(pattern, content):
|
||||
target = match.group(1).strip()
|
||||
if target:
|
||||
relations.append(Relation(
|
||||
type="mentions",
|
||||
target=target,
|
||||
context=None
|
||||
))
|
||||
|
||||
return relations
|
||||
|
||||
async def parse_metadata(self, metadata_section: Optional[str]) -> EntityMetadata:
|
||||
"""Metadata section is no longer used."""
|
||||
return EntityMetadata()
|
||||
|
||||
async def create_document(
|
||||
self,
|
||||
frontmatter: EntityFrontmatter,
|
||||
content: EntityContent,
|
||||
metadata: EntityMetadata
|
||||
) -> EntityMarkdown:
|
||||
"""Create the final EntityMarkdown document."""
|
||||
return EntityMarkdown(
|
||||
frontmatter=frontmatter,
|
||||
content=content,
|
||||
entity_metadata=metadata
|
||||
)
|
||||
@@ -1,282 +0,0 @@
|
||||
"""Parser for Basic Memory entity markdown files."""
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Dict, Any, Optional
|
||||
|
||||
import yaml
|
||||
from loguru import logger
|
||||
|
||||
from basic_memory.markdown.base_parser import MarkdownParser, ParseError
|
||||
from basic_memory.markdown.schemas import (
|
||||
EntityMarkdown,
|
||||
EntityFrontmatter,
|
||||
EntityContent,
|
||||
EntityMetadata,
|
||||
Observation,
|
||||
Relation,
|
||||
)
|
||||
|
||||
|
||||
class KnowledgeParser(MarkdownParser[EntityMarkdown]):
|
||||
"""Parser for entity markdown files.
|
||||
|
||||
Entity files must have:
|
||||
- YAML frontmatter (type, id, created, modified, tags)
|
||||
- Title (# Title)
|
||||
- Optional description
|
||||
- Observations section (## Observations)
|
||||
- Relations section (## Relations)
|
||||
- Optional # Metadata section with YAML in code block
|
||||
|
||||
Example Metadata:
|
||||
```yml
|
||||
field: value
|
||||
other: other value
|
||||
```
|
||||
"""
|
||||
|
||||
async def parse_metadata(self, metadata_section: Optional[str]) -> EntityMetadata:
|
||||
"""Parse metadata section."""
|
||||
try:
|
||||
if not metadata_section:
|
||||
logger.debug("No metadata section found")
|
||||
return EntityMetadata()
|
||||
|
||||
logger.debug(f"Raw metadata section:\n{metadata_section}")
|
||||
lines = metadata_section.strip().splitlines()
|
||||
yaml_lines = []
|
||||
in_yaml = False
|
||||
|
||||
# Look for ```yml or ```yaml starter
|
||||
for line in lines:
|
||||
stripped = line.strip().lower()
|
||||
if in_yaml:
|
||||
if stripped == "```":
|
||||
logger.debug("Found end of YAML block")
|
||||
break
|
||||
yaml_lines.append(line)
|
||||
logger.debug(f"Added YAML line: {line}")
|
||||
elif stripped in ["```yml", "```yaml"]:
|
||||
logger.debug("Found start of YAML block")
|
||||
in_yaml = True
|
||||
|
||||
if not yaml_lines:
|
||||
logger.debug("No YAML lines found in metadata section")
|
||||
return EntityMetadata()
|
||||
|
||||
yaml_content = "\n".join(yaml_lines)
|
||||
logger.debug(f"YAML content to parse:\n{yaml_content}")
|
||||
|
||||
# Parse the YAML content
|
||||
try:
|
||||
parsed = yaml.safe_load(yaml_content)
|
||||
if not isinstance(parsed, dict):
|
||||
logger.warning(f"Metadata YAML is not a dictionary: {parsed}")
|
||||
return EntityMetadata()
|
||||
logger.debug(f"Successfully parsed metadata: {parsed}")
|
||||
return EntityMetadata(data=parsed)
|
||||
except yaml.YAMLError as e:
|
||||
logger.warning(f"Failed to parse metadata YAML: {e}")
|
||||
return EntityMetadata()
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to parse metadata: {e}")
|
||||
return EntityMetadata()
|
||||
|
||||
async def parse_frontmatter(self, frontmatter: Dict[str, Any]) -> EntityFrontmatter:
|
||||
"""Parse entity frontmatter."""
|
||||
try:
|
||||
# Check required fields
|
||||
required_fields = {"type", "id", "created", "modified"}
|
||||
missing = required_fields - set(frontmatter.keys())
|
||||
if missing:
|
||||
raise ParseError(f"Missing required frontmatter fields: {', '.join(missing)}")
|
||||
|
||||
# Validate date fields
|
||||
for date_field in ["created", "modified"]:
|
||||
try:
|
||||
if not isinstance(frontmatter[date_field], datetime):
|
||||
datetime.fromisoformat(str(frontmatter[date_field]).replace("Z", "+00:00"))
|
||||
except (ValueError, TypeError):
|
||||
raise ParseError(
|
||||
f"Invalid date format for {date_field}: {frontmatter[date_field]}"
|
||||
)
|
||||
|
||||
# Prepare fields
|
||||
processed = {
|
||||
"type": frontmatter["type"].strip(),
|
||||
"id": str(frontmatter["id"]).strip(),
|
||||
"created": frontmatter["created"],
|
||||
"modified": frontmatter["modified"],
|
||||
"tags": (
|
||||
[tag.strip() for tag in frontmatter.get("tags", "").split(",")]
|
||||
if isinstance(frontmatter.get("tags"), str)
|
||||
else [t.strip() for t in frontmatter.get("tags", [])]
|
||||
),
|
||||
}
|
||||
|
||||
return EntityFrontmatter(**processed)
|
||||
|
||||
except Exception as e:
|
||||
if isinstance(e, ParseError):
|
||||
raise
|
||||
logger.error(f"Invalid entity frontmatter: {e}")
|
||||
raise ParseError(f"Invalid entity frontmatter: {str(e)}")
|
||||
|
||||
async def parse_content(self, title: str, sections: Dict[str, str]) -> EntityContent:
|
||||
"""Parse entity content section."""
|
||||
try:
|
||||
# Get description (if any)
|
||||
description = None
|
||||
if "content" in sections:
|
||||
description = sections["content"]
|
||||
|
||||
# Parse observations (required)
|
||||
observations = []
|
||||
if "observations" not in sections:
|
||||
raise ParseError("Missing required observations section")
|
||||
|
||||
for line in sections["observations"].splitlines():
|
||||
if line and not line.isspace():
|
||||
observation = await self.parse_observation(line)
|
||||
if observation:
|
||||
observations.append(observation)
|
||||
|
||||
# Parse relations (optional)
|
||||
relations = []
|
||||
if "relations" in sections:
|
||||
for line in sections["relations"].splitlines():
|
||||
if line and not line.isspace():
|
||||
relation = await self.parse_relation(line)
|
||||
if relation:
|
||||
relations.append(relation)
|
||||
|
||||
return EntityContent(
|
||||
title=title, summary=description, observations=observations, relations=relations
|
||||
)
|
||||
|
||||
except ParseError:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error(f"Invalid entity content: {e}")
|
||||
raise ParseError(f"Invalid entity content: {str(e)}")
|
||||
|
||||
async def parse_observation(self, line: str) -> Optional[Observation]:
|
||||
"""Parse a single observation line."""
|
||||
if not line or line.isspace():
|
||||
return None
|
||||
|
||||
try:
|
||||
# Remove leading/trailing whitespace and bullet
|
||||
line = line.strip()
|
||||
if not line.startswith("-"):
|
||||
return None
|
||||
line = line[1:].strip()
|
||||
|
||||
category = None
|
||||
# Handle malformed category brackets first
|
||||
if line.startswith("]"):
|
||||
raise ParseError("missing category")
|
||||
elif line.startswith("["):
|
||||
# category
|
||||
close_bracket = line.find("]")
|
||||
if close_bracket == -1:
|
||||
raise ParseError("unclosed category")
|
||||
|
||||
# Extract category - return None for empty category
|
||||
category_string = line[1:close_bracket].strip()
|
||||
category = category_string if category_string else None
|
||||
|
||||
line = line[close_bracket + 1 :].strip()
|
||||
|
||||
# Extract context if present (context)
|
||||
context = None
|
||||
# check if line ends with ")"
|
||||
if line.endswith(")"):
|
||||
# find "(" starting from end
|
||||
last_open = line.rfind("(")
|
||||
if last_open != -1:
|
||||
context = line[last_open + 1 : -1].strip()
|
||||
# remove context from the line
|
||||
line = line[:last_open].strip()
|
||||
|
||||
# Extract tags and clean content
|
||||
tags = []
|
||||
content_parts = []
|
||||
for part in line.split():
|
||||
if part.startswith("#"):
|
||||
# Handle multiple hashtags stuck together
|
||||
if "#" in part[1:]:
|
||||
multi_tags = part.split("#")
|
||||
tags.extend(t for t in multi_tags if t)
|
||||
else:
|
||||
tags.append(part[1:])
|
||||
else:
|
||||
content_parts.append(part)
|
||||
|
||||
content = " ".join(content_parts).strip()
|
||||
if not content:
|
||||
raise ParseError("Empty content")
|
||||
|
||||
return Observation(
|
||||
category=category, content=content, tags=tags if tags else None, context=context
|
||||
)
|
||||
|
||||
except ParseError:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to parse observation: {e}")
|
||||
raise ParseError(f"Failed to parse observation: {str(e)}")
|
||||
|
||||
async def parse_relation(self, line: str) -> Optional[Relation]:
|
||||
"""Parse a single relation line."""
|
||||
if not line or line.isspace():
|
||||
return None
|
||||
|
||||
try:
|
||||
# Remove leading/trailing whitespace and bullet
|
||||
line = line.strip()
|
||||
if not line.startswith("-"):
|
||||
return None
|
||||
line = line[1:].strip()
|
||||
|
||||
# Extract context if present (context)
|
||||
context = None
|
||||
if line.endswith(")"):
|
||||
context_start = line.rfind("(")
|
||||
if context_start != -1:
|
||||
context = line[context_start + 1 : -1].strip()
|
||||
line = line[:context_start].strip()
|
||||
|
||||
# Extract relation type and target [[Entity]]
|
||||
if "[[" not in line or "]]" not in line:
|
||||
raise ParseError("Invalid relation format - missing [[entity]]")
|
||||
|
||||
# Split into relation type and target
|
||||
parts = line.split("[[", 1)
|
||||
rel_type = parts[0].strip()
|
||||
if not rel_type:
|
||||
raise ParseError("Missing relation type")
|
||||
|
||||
target_part = parts[1]
|
||||
close_pos = target_part.find("]]")
|
||||
if close_pos == -1:
|
||||
raise ParseError("Unclosed [[ ]] in target")
|
||||
|
||||
target = target_part[:close_pos].strip()
|
||||
if not target:
|
||||
raise ParseError("Empty target entity")
|
||||
|
||||
return Relation(type=rel_type, target=target, context=context)
|
||||
|
||||
except ParseError:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to parse relation: {e}")
|
||||
raise ParseError(f"Failed to parse relation: {str(e)}")
|
||||
|
||||
async def create_document(
|
||||
self, frontmatter: EntityFrontmatter, content: EntityContent, metadata: EntityMetadata
|
||||
) -> EntityMarkdown:
|
||||
"""Create entity from parsed sections."""
|
||||
return EntityMarkdown(frontmatter=frontmatter, content=content, entity_metadata=metadata)
|
||||
@@ -0,0 +1,230 @@
|
||||
"""Markdown-it plugins for Basic Memory markdown parsing."""
|
||||
|
||||
from typing import List, Any, Dict
|
||||
from markdown_it import MarkdownIt
|
||||
from markdown_it.token import Token
|
||||
|
||||
|
||||
# Observation handling functions
|
||||
def is_observation(token: Token) -> bool:
|
||||
"""Check if token looks like our observation format."""
|
||||
if token.type != 'inline':
|
||||
return False
|
||||
|
||||
content = token.content.strip()
|
||||
return ((content.startswith('[') and ']' in content) or # Has category
|
||||
'#' in content) # Has tags
|
||||
|
||||
|
||||
def parse_observation(token: Token) -> Dict[str, Any]:
|
||||
"""Extract observation parts from token."""
|
||||
# Strip bullet point if present
|
||||
content = token.content.strip()
|
||||
if content.startswith('- '):
|
||||
content = content[2:].strip()
|
||||
elif content.startswith('-'):
|
||||
content = content[1:].strip()
|
||||
|
||||
# Parse [category]
|
||||
category = None
|
||||
if content.startswith('['):
|
||||
end = content.find(']')
|
||||
if end != -1:
|
||||
category = content[1:end].strip()
|
||||
content = content[end + 1:].strip()
|
||||
|
||||
# Parse (context)
|
||||
context = None
|
||||
if content.endswith(')'):
|
||||
start = content.rfind('(')
|
||||
if start != -1:
|
||||
context = content[start + 1:-1].strip()
|
||||
content = content[:start].strip()
|
||||
|
||||
# Parse #tags and content
|
||||
parts = content.split()
|
||||
content_parts = []
|
||||
tags = set() # Use set to avoid duplicates
|
||||
|
||||
for part in parts:
|
||||
if part.startswith('#'):
|
||||
# Handle multiple #tags stuck together
|
||||
if '#' in part[1:]:
|
||||
# Split on # but keep non-empty tags
|
||||
subtags = [t for t in part.split('#') if t]
|
||||
tags.update(subtags)
|
||||
else:
|
||||
tags.add(part[1:])
|
||||
else:
|
||||
content_parts.append(part)
|
||||
|
||||
return {
|
||||
'category': category,
|
||||
'content': ' '.join(content_parts).strip(),
|
||||
'tags': list(tags) if tags else None,
|
||||
'context': context
|
||||
}
|
||||
|
||||
|
||||
# Relation handling functions
|
||||
def is_explicit_relation(token: Token) -> bool:
|
||||
"""Check if token looks like our relation format."""
|
||||
if token.type != 'inline':
|
||||
return False
|
||||
|
||||
content = token.content.strip()
|
||||
return ('[[' in content and
|
||||
']]' in content and
|
||||
content.index('[[') < content.index(']]'))
|
||||
|
||||
|
||||
def parse_relation(token: Token) -> Dict[str, Any]:
|
||||
"""Extract relation parts from token."""
|
||||
# Remove bullet point if present
|
||||
content = token.content.strip()
|
||||
if content.startswith('- '):
|
||||
content = content[2:].strip()
|
||||
elif content.startswith('-'):
|
||||
content = content[1:].strip()
|
||||
|
||||
# Extract [[target]]
|
||||
target = None
|
||||
rel_type = 'relates_to' # default
|
||||
context = None
|
||||
|
||||
start = content.find('[[')
|
||||
end = content.find(']]')
|
||||
|
||||
if start != -1 and end != -1:
|
||||
# Get text before link as relation type
|
||||
before = content[:start].strip()
|
||||
if before:
|
||||
rel_type = before
|
||||
|
||||
# Get target
|
||||
target = content[start + 2:end].strip()
|
||||
|
||||
# Look for context after
|
||||
after = content[end + 2:].strip()
|
||||
if after.startswith('(') and after.endswith(')'):
|
||||
context = after[1:-1].strip()
|
||||
|
||||
if not target:
|
||||
return None
|
||||
|
||||
return {
|
||||
'type': rel_type,
|
||||
'target': target,
|
||||
'context': context
|
||||
}
|
||||
|
||||
|
||||
def parse_inline_relations(content: str) -> List[Dict[str, Any]]:
|
||||
"""Find wiki-style links in regular content."""
|
||||
relations = []
|
||||
|
||||
import re
|
||||
pattern = r'\[\[([^\]]+)\]\]'
|
||||
|
||||
for match in re.finditer(pattern, content):
|
||||
target = match.group(1).strip()
|
||||
if target and not target.startswith('[['): # Avoid nested matches
|
||||
relations.append({
|
||||
'type': 'mentions',
|
||||
'target': target,
|
||||
'context': None
|
||||
})
|
||||
|
||||
return relations
|
||||
|
||||
|
||||
def observation_plugin(md: MarkdownIt) -> None:
|
||||
"""Plugin for parsing observation format:
|
||||
- [category] Content text #tag1 #tag2 (context)
|
||||
- Content text #tag1 (context) # No category is also valid
|
||||
"""
|
||||
|
||||
def observation_rule(state: Any) -> None:
|
||||
"""Process observations in token stream."""
|
||||
tokens = state.tokens
|
||||
current_section = None
|
||||
in_list_item = False
|
||||
|
||||
for idx in range(len(tokens)):
|
||||
token = tokens[idx]
|
||||
|
||||
# Track current section by headings
|
||||
if token.type == 'heading_open':
|
||||
next_token = tokens[idx + 1] if idx + 1 < len(tokens) else None
|
||||
if next_token and next_token.type == 'inline':
|
||||
current_section = next_token.content.lower()
|
||||
|
||||
# Track list nesting
|
||||
elif token.type == 'list_item_open':
|
||||
in_list_item = True
|
||||
elif token.type == 'list_item_close':
|
||||
in_list_item = False
|
||||
|
||||
# Initialize meta for all tokens
|
||||
token.meta = token.meta or {}
|
||||
|
||||
# Parse observations in list items
|
||||
if token.type == 'inline' and in_list_item and is_observation(token):
|
||||
obs = parse_observation(token)
|
||||
if obs['content']: # Only store if we have content
|
||||
token.meta['observation'] = obs
|
||||
|
||||
# Add the rule after inline processing
|
||||
md.core.ruler.after('inline', 'observations', observation_rule)
|
||||
|
||||
|
||||
def relation_plugin(md: MarkdownIt) -> None:
|
||||
"""Plugin for parsing relation formats:
|
||||
|
||||
Explicit relations:
|
||||
- relation_type [[target]] (context)
|
||||
|
||||
Implicit relations (links in content):
|
||||
Some text with [[target]] reference
|
||||
"""
|
||||
|
||||
def relation_rule(state: Any) -> None:
|
||||
"""Process relations in token stream."""
|
||||
tokens = state.tokens
|
||||
current_section = None
|
||||
in_list_item = False
|
||||
|
||||
for idx in range(len(tokens)):
|
||||
token = tokens[idx]
|
||||
|
||||
# Track current section by headings
|
||||
if token.type == 'heading_open':
|
||||
next_token = tokens[idx + 1] if idx + 1 < len(tokens) else None
|
||||
if next_token and next_token.type == 'inline':
|
||||
current_section = next_token.content.lower()
|
||||
|
||||
# Track list nesting
|
||||
elif token.type == 'list_item_open':
|
||||
in_list_item = True
|
||||
elif token.type == 'list_item_close':
|
||||
in_list_item = False
|
||||
|
||||
# Initialize meta for all tokens
|
||||
token.meta = token.meta or {}
|
||||
|
||||
# Only process inline tokens
|
||||
if token.type == 'inline':
|
||||
# Check for explicit relations in list items
|
||||
if in_list_item and is_explicit_relation(token):
|
||||
rel = parse_relation(token)
|
||||
if rel:
|
||||
token.meta['relations'] = [rel]
|
||||
|
||||
# Always check for inline relations in any text
|
||||
elif '[[' in token.content:
|
||||
rels = parse_inline_relations(token.content)
|
||||
if rels:
|
||||
token.meta['relations'] = token.meta.get('relations', []) + rels
|
||||
|
||||
# Add the rule after inline processing
|
||||
md.core.ruler.after('inline', 'relations', relation_rule)
|
||||
@@ -26,6 +26,7 @@ class Relation(BaseModel):
|
||||
class EntityFrontmatter(BaseModel):
|
||||
"""Required frontmatter fields for an entity."""
|
||||
|
||||
title: str
|
||||
type: str
|
||||
id: str
|
||||
created: datetime
|
||||
@@ -36,18 +37,9 @@ class EntityFrontmatter(BaseModel):
|
||||
class EntityContent(BaseModel):
|
||||
"""Content sections of an entity markdown file."""
|
||||
|
||||
title: str
|
||||
summary: Optional[str] = None
|
||||
content: Optional[str] = None
|
||||
observations: List[Observation] = []
|
||||
relations: List[Relation] = []
|
||||
context: Optional[str] = None
|
||||
|
||||
|
||||
class EntityMetadata(BaseModel):
|
||||
"""Optional metadata for an entity."""
|
||||
|
||||
# Changed from 'metadata' to 'data' to avoid pydantic special field name
|
||||
data: Dict[str, Any] = {}
|
||||
|
||||
|
||||
class EntityMarkdown(BaseModel):
|
||||
@@ -55,4 +47,3 @@ class EntityMarkdown(BaseModel):
|
||||
|
||||
frontmatter: EntityFrontmatter
|
||||
content: EntityContent
|
||||
entity_metadata: EntityMetadata = EntityMetadata()
|
||||
|
||||
@@ -5,7 +5,7 @@ from pathlib import Path
|
||||
from loguru import logger
|
||||
|
||||
from basic_memory.config import ProjectConfig
|
||||
from basic_memory.markdown import KnowledgeParser
|
||||
from basic_memory.markdown import EntityParser
|
||||
from basic_memory.services.search_service import SearchService
|
||||
from basic_memory.sync import FileChangeScanner
|
||||
from basic_memory.sync.knowledge_sync_service import KnowledgeSyncService
|
||||
@@ -24,12 +24,12 @@ class SyncService:
|
||||
self,
|
||||
scanner: FileChangeScanner,
|
||||
knowledge_sync_service: KnowledgeSyncService,
|
||||
knowledge_parser: KnowledgeParser,
|
||||
entity_parser: EntityParser,
|
||||
search_service: SearchService,
|
||||
):
|
||||
self.scanner = scanner
|
||||
self.knowledge_sync_service = knowledge_sync_service
|
||||
self.knowledge_parser = knowledge_parser
|
||||
self.knowledge_parser = entity_parser
|
||||
self.search_service = search_service
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user