mirror of
https://github.com/basicmachines-co/basic-memory
synced 2026-06-21 13:47:35 +00:00
phernandez
117 lines
4.2 KiB
Python
117 lines
4.2 KiB
Python
"""Parser for markdown files into Entity objects.
|
|
|
|
Uses markdown-it with plugins to parse structured data from markdown content.
|
|
"""
|
|
|
|
from pathlib import Path
|
|
from datetime import datetime
|
|
from typing import Any, Optional
|
|
|
|
from markdown_it import MarkdownIt
|
|
import frontmatter
|
|
|
|
from basic_memory.markdown.plugins import observation_plugin, relation_plugin
|
|
from basic_memory.markdown.schemas import (
|
|
EntityMarkdown,
|
|
EntityFrontmatter,
|
|
EntityContent,
|
|
Observation,
|
|
Relation,
|
|
)
|
|
|
|
|
|
class EntityParser:
|
|
"""Parser for markdown files into Entity objects."""
|
|
|
|
def __init__(self, base_path: Path):
|
|
"""Initialize parser with base path for relative permalink generation."""
|
|
self.base_path = base_path.resolve()
|
|
self.md = MarkdownIt().use(observation_plugin).use(relation_plugin)
|
|
|
|
def get_permalink(self, file_path: Path) -> str:
|
|
"""Get permalink from file path relative to base_path.
|
|
|
|
Example:
|
|
base_path: /project/root
|
|
file_path: /project/root/design/models/data.md
|
|
returns: "design/models/data"
|
|
"""
|
|
# Get relative path and remove .md extension
|
|
rel_path = file_path.resolve().relative_to(self.base_path)
|
|
if rel_path.suffix.lower() == ".md":
|
|
return str(rel_path.with_suffix(""))
|
|
return str(rel_path)
|
|
|
|
def parse_date(self, value: Any) -> Optional[datetime]:
|
|
"""Parse various date formats into datetime."""
|
|
if isinstance(value, datetime):
|
|
return value
|
|
if isinstance(value, str):
|
|
try:
|
|
return datetime.fromisoformat(value.replace("Z", "+00:00"))
|
|
except (ValueError, TypeError):
|
|
pass
|
|
return None
|
|
|
|
async def parse_file(self, file_path: Path) -> EntityMarkdown:
|
|
"""Parse markdown file into EntityMarkdown."""
|
|
# Parse frontmatter and content using python-frontmatter
|
|
post = frontmatter.load(str(file_path))
|
|
|
|
# Extract or generate required fields
|
|
permalink = post.metadata.get("id") or self.get_permalink(file_path)
|
|
stats = file_path.stat()
|
|
|
|
# Parse frontmatter
|
|
entity_frontmatter = EntityFrontmatter(
|
|
type=str(post.metadata.get("type", "note")),
|
|
id=permalink,
|
|
title=str(post.metadata.get("title", file_path.name)),
|
|
created=self.parse_date(post.metadata.get("created"))
|
|
or datetime.fromtimestamp(stats.st_ctime),
|
|
modified=self.parse_date(post.metadata.get("modified"))
|
|
or datetime.fromtimestamp(stats.st_mtime),
|
|
tags=self.parse_tags(post.metadata.get("tags", [])),
|
|
)
|
|
|
|
# Parse content for observations and relations using markdown-it
|
|
observations, relations = await self.parse_observations_and_relations(post.content)
|
|
|
|
# Create EntityContent
|
|
entity_content = EntityContent(
|
|
content=post.content,
|
|
observations=observations,
|
|
relations=relations,
|
|
)
|
|
|
|
return EntityMarkdown(
|
|
frontmatter=entity_frontmatter,
|
|
content=entity_content,
|
|
)
|
|
|
|
async def parse_observations_and_relations(
|
|
self, content: str
|
|
) -> tuple[list[Observation], list[Relation]]:
|
|
tokens = self.md.parse(content)
|
|
# Extract observations and relations from token meta
|
|
observations = []
|
|
relations = []
|
|
for token in tokens:
|
|
if token.meta: # Token might not have meta
|
|
if "observation" in token.meta:
|
|
obs = token.meta["observation"]
|
|
observation = Observation.model_validate(obs)
|
|
observations.append(observation)
|
|
if "relations" in token.meta:
|
|
rels = token.meta["relations"]
|
|
relations.extend([Relation.model_validate(r) for r in rels])
|
|
return observations, relations
|
|
|
|
def parse_tags(self, tags: Any) -> list[str]:
|
|
"""Parse tags into list of strings."""
|
|
if isinstance(tags, str):
|
|
return [t.strip() for t in tags.split(",") if t.strip()]
|
|
if isinstance(tags, (list, tuple)):
|
|
return [str(t).strip() for t in tags if str(t).strip()]
|
|
return []
|