admin/basicmachines-co-basic-memory
1
0
Fork 0
mirror of https://github.com/basicmachines-co/basic-memory synced 2026-06-21 13:47:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix updates and save content. remove entity.summary

Browse Source
This commit is contained in:
phernandez
2025-01-26 00:00:37 -06:00
parent 9019ffedb7
commit 18bd850019
36 changed files with 409 additions and 979 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/basic_memory/api/routers/knowledge_router.py
+1 -1
View File
@@ -49,7 +49,7 @@ async def create_or_update_entity(
entity, created = await entity_service.create_or_update_entity(data)
response.status_code = 201 if created else 200
# Always reindex since content has changed
# reindex
await search_service.index_entity(entity, background_tasks=background_tasks)
return EntityResponse.model_validate(entity)
src/basic_memory/deps.py
+34 -16
View File
@@ -11,7 +11,8 @@ from sqlalchemy.ext.asyncio import (
from basic_memory import db
from basic_memory.config import ProjectConfig, config
from basic_memory.markdown.knowledge_writer import KnowledgeWriter
from basic_memory.markdown import EntityParser
from basic_memory.markdown.markdown_processor import MarkdownProcessor
from basic_memory.repository.entity_repository import EntityRepository
from basic_memory.repository.observation_repository import ObservationRepository
from basic_memory.repository.relation_repository import RelationRepository
@@ -106,18 +107,39 @@ SearchRepositoryDep = Annotated[SearchRepository, Depends(get_search_repository)
## services
async def get_file_service(project_config: ProjectConfigDep) -> FileService:
return FileService(project_config.home, KnowledgeWriter())
async def get_entity_parser(project_config: ProjectConfigDep) -> EntityParser:
return EntityParser(project_config.home)
EntityParserDep = Annotated["EntityParser", Depends(get_entity_parser)]
async def get_markdown_processor(entity_parser: EntityParserDep) -> MarkdownProcessor:
return MarkdownProcessor(entity_parser)
MarkdownProcessorDep = Annotated[MarkdownProcessor, Depends(get_markdown_processor)]
async def get_file_service(
project_config: ProjectConfigDep, markdown_processor: MarkdownProcessorDep
) -> FileService:
return FileService(project_config.home, markdown_processor)
FileServiceDep = Annotated[FileService, Depends(get_file_service)]
async def get_entity_service(
entity_repository: EntityRepositoryDep, file_service: FileServiceDep, link_resolver: "LinkResolverDep"
entity_repository: EntityRepositoryDep,
file_service: FileServiceDep,
link_resolver: "LinkResolverDep",
) -> EntityService:
"""Create EntityService with repository."""
return EntityService(entity_repository=entity_repository, file_service=file_service, link_resolver=link_resolver)
return EntityService(
entity_repository=entity_repository, file_service=file_service, link_resolver=link_resolver
)
EntityServiceDep = Annotated[EntityService, Depends(get_entity_service)]
@@ -158,28 +180,24 @@ RelationServiceDep = Annotated[RelationService, Depends(get_relation_service)]
async def get_search_service(
search_repository: SearchRepositoryDep, entity_repository: EntityRepositoryDep
search_repository: SearchRepositoryDep, entity_repository: EntityRepositoryDep, file_service: FileServiceDep,
) -> SearchService:
"""Create SearchService with dependencies."""
return SearchService(search_repository, entity_repository)
return SearchService(search_repository, entity_repository, file_service)
SearchServiceDep = Annotated[SearchService, Depends(get_search_service)]
async def get_knowledge_writer() -> KnowledgeWriter:
return KnowledgeWriter()
async def get_link_resolver(
entity_repository: EntityRepositoryDep, search_service: SearchServiceDep
) -> LinkResolver:
return LinkResolver(entity_repository=entity_repository, search_service=search_service)
KnowledgeWriterDep = Annotated[KnowledgeWriter, Depends(get_knowledge_writer)]
async def get_link_resolver(entity_repository: EntityRepositoryDep,
search_service: SearchServiceDep) -> LinkResolver:
return LinkResolver(entity_repository=entity_repository,
search_service=search_service)
LinkResolverDep = Annotated[LinkResolver, Depends(get_link_resolver)]
async def get_context_service(
search_repository: SearchRepositoryDep, entity_repository: EntityRepositoryDep
) -> ContextService:
src/basic_memory/markdown/entity_parser.py
+13 -10
View File
@@ -45,7 +45,7 @@ class EntityParser:
def parse_date(self, value: Any) -> Optional[datetime]:
"""Parse date strings using dateparser for maximum flexibility.
Supports human friendly formats like:
- 2024-01-15
- Jan 15, 2024
@@ -70,19 +70,22 @@ class EntityParser:
post = frontmatter.load(str(file_path))
# Extract or generate required fields
permalink = post.metadata.get("permalink")
file_stats = file_path.stat()
metadata = post.metadata
metadata["title"] = post.metadata.get("title", file_path.name)
metadata["type"] = metadata.get("type", "note")
metadata["created"] = self.parse_date(
post.metadata.get("created")
) or datetime.fromtimestamp(file_stats.st_ctime)
metadata["modified"] = self.parse_date(
post.metadata.get("modified")
) or datetime.fromtimestamp(file_stats.st_mtime)
metadata["tags"] = self.parse_tags(post.metadata.get("tags", []))
# Parse frontmatter
entity_frontmatter = EntityFrontmatter(
type=str(post.metadata.get("type", "note")),
permalink=permalink,
title=str(post.metadata.get("title", file_path.name)),
created=self.parse_date(post.metadata.get("created"))
or datetime.fromtimestamp(file_stats.st_ctime),
modified=self.parse_date(post.metadata.get("modified"))
or datetime.fromtimestamp(file_stats.st_mtime),
tags=self.parse_tags(post.metadata.get("tags", [])),
metadata=post.metadata,
)
# Parse content for observations and relations using markdown-it
src/basic_memory/markdown/knowledge_writer.py
-110
View File
@@ -1,110 +0,0 @@
"""Writer for knowledge entity markdown files."""
from typing import Optional
from loguru import logger
from basic_memory.markdown import EntityFrontmatter
from basic_memory.models import Entity as EntityModel, Observation
class KnowledgeWriter:
"""Formats entities into markdown files.
Content handling:
1. If raw content is provided, use it directly
2. If structured data exists (observations/relations), generate structured content
3. If neither, create basic content from name/summary
"""
async def format_frontmatter(self, entity: EntityModel) -> dict:
"""Generate frontmatter metadata for entity."""
frontmatter = {
"permalink": entity.permalink,
"type": entity.entity_type,
"created": entity.created_at.isoformat(),
"modified": entity.updated_at.isoformat(),
}
if entity.entity_metadata:
frontmatter.update(entity.entity_metadata)
return frontmatter
async def format_observation(self, obs: Observation) -> str:
"""Format a single observation with category, content, tags and context."""
line = f"- [{obs.category}] {obs.content}"
# Add tags if present
if obs.tags:
line += " " + " ".join(f"#{tag}" for tag in sorted(obs.tags))
# Add context if present
if obs.context:
line += f" ({obs.context})"
return line
async def format_content(self, entity: EntityModel, content: Optional[str] = None) -> str:
"""Format entity content as markdown.
Args:
entity: Entity to format
content: Optional raw content to use instead of generating structured content
Returns:
Formatted markdown content
"""
# If raw content provided, use it directly
if content is not None:
logger.debug("Content supplied to entity writer, using it directly")
return content
# Otherwise, build structured content from entity data
sections = []
# Only add entity title if we don't have structured content
# This prevents duplicate titles when raw content already has a title
if not (entity.observations or entity.outgoing_relations):
sections.extend(
[
f"# {entity.title}",
"", # Empty line after title
]
)
if entity.summary:
sections.extend([entity.summary, ""])
# Add observations if present
if entity.observations:
sections.extend(
[
"## Observations",
"<!-- Format: - [category] Content text #tag1 #tag2 (optional context) -->",
"",
]
)
for obs in entity.observations:
sections.append(await self.format_observation(obs))
sections.append("")
# Add relations if present
if entity.outgoing_relations:
sections.extend(
[
"## Relations",
"<!-- Format: - relation_type [[Entity]] (context) -->",
"", # Empty line after format comment
]
)
for rel in entity.outgoing_relations:
line = f"- {rel.relation_type} [[{rel.to_entity.title}]]"
if rel.context:
line += f" ({rel.context})"
sections.append(line)
sections.append("") # Empty line after relations
# Return joined sections, ensure content isn't empty
content = "\n".join(sections).strip()
return content if content else f"# {entity.title}"
src/basic_memory/markdown/markdown_processor.py
+29 -35
View File
@@ -8,7 +8,7 @@ This module follows a Read -> Modify -> Write pattern for all file operations:
No in-place updates are performed. Each write reconstructs the entire file from the schema.
The file format has two distinct types of content:
1. User content - Free form text that is preserved exactly as written
2. Structured sections - Observations and Relations that are always formatted
2. Structured sections - Observations and Relations that are always formatted
in a standard way and can be overwritten since they're tracked in our schema
"""
@@ -26,33 +26,33 @@ from basic_memory.markdown.schemas import EntityMarkdown, Observation, Relation
class DirtyFileError(Exception):
"""Raised when attempting to write to a file that has been modified."""
pass
class MarkdownProcessor:
"""Process markdown files while preserving content and structure.
This class handles the file I/O aspects of our markdown processing. It:
1. Uses EntityParser for reading/parsing files into our schema
2. Handles writing files with proper frontmatter
3. Formats structured sections (observations/relations) consistently
4. Preserves user content exactly as written
5. Performs atomic writes using temp files
It does NOT:
1. Modify the schema directly (that's done by services)
2. Handle in-place updates (everything is read->modify->write)
3. Track schema changes (that's done by the database)
"""
def __init__(self, base_path: Path, entity_parser: EntityParser):
def __init__(self, entity_parser: EntityParser):
"""Initialize processor with base path and parser."""
self.base_path = base_path.resolve()
self.entity_parser = entity_parser
async def read_file(self, path: Path) -> EntityMarkdown:
"""Read and parse file into EntityMarkdown schema.
This is step 1 of our read->modify->write pattern.
We use EntityParser to handle all the markdown parsing.
"""
@@ -65,7 +65,7 @@ class MarkdownProcessor:
expected_checksum: Optional[str] = None,
) -> str:
"""Write EntityMarkdown schema back to file.
This is step 3 of our read->modify->write pattern.
The entire file is rewritten atomically on each update.
@@ -74,10 +74,10 @@ class MarkdownProcessor:
frontmatter fields
---
user content area (preserved exactly)
## Observations (if any)
formatted observations
## Relations (if any)
formatted relations
@@ -98,59 +98,53 @@ class MarkdownProcessor:
current_checksum = await file_utils.compute_checksum(current_content)
if current_checksum != expected_checksum:
raise DirtyFileError(f"File {path} has been modified")
# Convert frontmatter to dict, dropping None values
metadata = markdown.frontmatter.metadata or {}
frontmatter_dict = {
"type": markdown.frontmatter.type,
"permalink": markdown.frontmatter.permalink,
"created": markdown.frontmatter.created.isoformat() if markdown.frontmatter.created else None,
"modified": markdown.frontmatter.modified.isoformat() if markdown.frontmatter.modified else None,
"tags": markdown.frontmatter.tags,
**metadata
}
frontmatter_dict = {k: v for k, v in frontmatter_dict.items() if v is not None}
# Start with user content (or minimal title for new files)
content = markdown.content.content or f"# {markdown.frontmatter.title}\n"
# Add structured sections if present
if markdown.content.observations:
content += "\n## Observations\n" + self.format_observations(markdown.content.observations)
content += (
"\n\n## Observations\n\n"
+ self.format_observations(markdown.content.observations)
)
if markdown.content.relations:
content += "\n## Relations\n" + self.format_relations(markdown.content.relations)
content += "\n## Relations\n\n" + self.format_relations(markdown.content.relations)
# Create Post object for frontmatter
post = Post(content, **frontmatter_dict)
final_content = frontmatter.dumps(post)
# Write atomically and return checksum
logger.debug(f"writing file {path} with content:\n{final_content}")
# Write atomically and return checksum of updated file
path.parent.mkdir(parents=True, exist_ok=True)
await file_utils.write_file_atomic(path, final_content)
return await file_utils.compute_checksum(final_content)
def format_observations(self, observations: list[Observation]) -> str:
"""Format observations section in standard way.
Format: - [category] content #tag1 #tag2 (context)
"""
lines = []
for obs in observations:
line = f"- [{obs.category}] {obs.content}"
if obs.tags:
line += " " + " ".join(f"#{tag}" for tag in sorted(obs.tags))
if obs.context:
line += f" ({obs.context})"
lines.append(line)
lines = [f"- {obs}" for obs in observations]
return "\n".join(lines) + "\n"
def format_relations(self, relations: list[Relation]) -> str:
"""Format relations section in standard way.
Format: - relation_type [[target]] (context)
"""
lines = []
for rel in relations:
line = f"- {rel.type} [[{rel.target}]]"
if rel.context:
line += f" ({rel.context})"
lines.append(line)
return "\n".join(lines) + "\n"
lines = [f"- {rel}" for rel in relations]
return "\n".join(lines) + "\n"
src/basic_memory/markdown/schemas.py
+38 -6
View File
@@ -13,6 +13,14 @@ class Observation(BaseModel):
content: str
tags: Optional[List[str]] = None
context: Optional[str] = None
def __str__(self) -> str:
obs_string = f"[{self.category}] {self.content}"
if self.tags:
obs_string += " " + " ".join(f"#{tag}" for tag in sorted(self.tags))
if self.context:
obs_string += f" ({self.context})"
return obs_string
class Relation(BaseModel):
@@ -21,18 +29,42 @@ class Relation(BaseModel):
type: str
target: str
context: Optional[str] = None
def __str__(self) -> str:
rel_string = f"{self.type} [[{self.target}]]"
if self.context:
rel_string += f" ({self.context})"
return rel_string
class EntityFrontmatter(BaseModel):
"""Required frontmatter fields for an entity."""
title: str
type: str
permalink: Optional[str] = None
created: datetime
modified: datetime
tags: Optional[List[str]] = None
metadata: Optional[dict] = None
@property
def tags(self) -> List[str]:
return self.metadata.get("tags") if self.metadata else []
@property
def title(self) -> str:
return self.metadata.get("title") if self.metadata else None
@property
def type(self) -> str:
return self.metadata.get("type", "note") if self.metadata else "note"
@property
def permalink(self) -> str:
return self.metadata.get("permalink") if self.metadata else None
@property
def created(self) -> datetime:
return self.metadata.get("created") if self.metadata else None
@property
def modified(self) -> datetime:
return self.metadata.get("modified") if self.metadata else None
class EntityContent(BaseModel):
"""Content sections of an entity markdown file."""
src/basic_memory/markdown/utils.py
+47
View File
@@ -0,0 +1,47 @@
from typing import Optional
from basic_memory.markdown import EntityMarkdown, EntityFrontmatter, EntityContent, Observation, Relation
class EntityModel:
pass
def entity_model_to_markdown(entity: EntityModel, content: Optional[str] = None) -> EntityMarkdown:
"""Convert entity model to markdown schema.
Args:
entity: Entity model to convert
content: Optional content to use (falls back to title)
Returns:
EntityMarkdown schema
"""
metadata=entity.entity_metadata or {}
metadata["permalink"] = entity.permalink
metadata["type"] = entity.entity_type or "note"
metadata["title"] = entity.title
metadata["created"] = entity.created_at
metadata["modified"] = entity.updated_at
entity_frontmatter = EntityFrontmatter(
metadata=metadata
)
entity_content = EntityContent(
content=content, # Use provided content
observations=[
Observation(
category=obs.category, content=obs.content, tags=obs.tags, context=obs.context
)
for obs in entity.observations
],
relations=[
Relation(type=r.relation_type, target=r.to_entity.title, context=r.context) for r in entity.outgoing_relations
],
)
return EntityMarkdown(
frontmatter=entity_frontmatter,
content=entity_content,
)
src/basic_memory/models/knowledge.py
+3 -6
View File
@@ -56,9 +56,6 @@ class Entity(Base):
# checksum of file
checksum: Mapped[Optional[str]] = mapped_column(String, nullable=True)
# Content summary
summary: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
# Metadata and tracking
created_at: Mapped[datetime] = mapped_column(DateTime)
updated_at: Mapped[datetime] = mapped_column(DateTime)
@@ -105,7 +102,7 @@ class Entity(Base):
return value
def __repr__(self) -> str:
return f"Entity(id={self.id}, name='{self.title}', type='{self.entity_type}', summary='{self.summary}')"
return f"Entity(id={self.id}, name='{self.title}', type='{self.entity_type}'"
class ObservationCategory(str, Enum):
@@ -209,8 +206,8 @@ class Relation(Base):
return generate_permalink(
f"{self.from_entity.permalink}/{self.relation_type}/{self.to_entity.permalink}"
if self.to_entity
else f"{self.from_entity.permalink}/{self.relation_type}"
else f"{self.from_entity.permalink}/{self.relation_type}/{self.to_name}"
)
def __repr__(self) -> str:
return f"Relation(id={self.id}, from_id={self.from_id}, to_id={self.to_id}, type='{self.relation_type}')"
return f"Relation(id={self.id}, from_id={self.from_id}, to_id={self.to_id}, to_name={self.to_name}, type='{self.relation_type}')"
src/basic_memory/schemas/base.py
-1
View File
@@ -179,7 +179,6 @@ class Entity(BaseModel):
entity_type: EntityType
entity_metadata: Optional[Dict] = Field(default=None, description="Optional metadata")
content: Optional[str] = None
summary: Optional[str] = None
content_type: ContentType = Field(
description="MIME type of the content (e.g. text/markdown, image/jpeg)",
examples=["text/markdown", "image/jpeg"],
src/basic_memory/schemas/request.py
-1
View File
@@ -103,6 +103,5 @@ class UpdateEntityRequest(BaseModel):
title: Optional[str] = None
entity_type: Optional[EntityType] = None
summary: Optional[str] = None
content: Optional[str] = None
entity_metadata: Optional[Dict[str, Any]] = None
src/basic_memory/services/entity_service.py
-2
View File
@@ -20,7 +20,6 @@ def entity_model(entity: EntitySchema):
entity_metadata=entity.entity_metadata,
permalink=entity.permalink,
file_path=entity.file_path,
summary=entity.summary,
content_type=entity.content_type,
observations=[Observation(content=observation) for observation in entity.observations],
)
@@ -58,7 +57,6 @@ class EntityService(BaseService[EntityModel]):
"entity_type": schema.entity_type,
"entity_metadata": schema.entity_metadata,
"content_type": schema.content_type,
"summary": schema.summary,
}
return await self.update_entity(
src/basic_memory/services/file_service.py
+57 -180
View File
@@ -1,16 +1,20 @@
"""Service for file operations with checksum tracking."""
from pathlib import Path
from typing import Optional, Dict, Any, Tuple
from typing import Optional, Tuple
from loguru import logger
from basic_memory import file_utils
from basic_memory.markdown.knowledge_writer import KnowledgeWriter
from basic_memory.markdown import EntityFrontmatter, EntityContent, EntityMarkdown, Observation, Relation
from basic_memory.markdown.markdown_processor import MarkdownProcessor
from basic_memory.markdown.utils import entity_model_to_markdown
from basic_memory.services.exceptions import FileOperationError
from basic_memory.models import Entity as EntityModel
class FileService:
"""
Service for handling file operations.
@@ -25,10 +29,10 @@ class FileService:
def __init__(
self,
base_path: Path,
knowledge_writer: KnowledgeWriter,
markdown_processor: MarkdownProcessor,
):
self.base_path = base_path
self.knowledge_writer = knowledge_writer
self.markdown_processor = markdown_processor
def get_entity_path(self, entity: EntityModel) -> Path:
"""Generate filesystem path for entity."""
@@ -40,54 +44,76 @@ class FileService:
self,
entity: EntityModel,
content: Optional[str] = None,
expected_checksum: Optional[str] = None,
) -> Tuple[Path, str]:
"""Write entity to filesystem and return path and checksum.
If content is not provided, tries to preserve existing file content.
Uses read->modify->write pattern:
1. Read existing file if it exists
2. Update with new content if provided
3. Write back atomically
Args:
entity: Entity model to write
content: Optional new content (preserves existing if None)
expected_checksum: Optional checksum to verify file hasn't changed
Returns:
Tuple of (file path, new checksum)
Raises:
FileOperationError: If write fails
"""
try:
# Try to preserve existing content if not provided
content = await self.read_entity_content(entity) if content is None else content
# Get frontmatter and content
frontmatter = await self.knowledge_writer.format_frontmatter(entity)
file_content = await self.knowledge_writer.format_content(
entity=entity, content=content
)
# Add frontmatter and write
content_with_frontmatter = await self.add_frontmatter(
frontmatter=frontmatter, content=file_content
)
path = self.get_entity_path(entity)
return path, await self.write_file(path, content_with_frontmatter)
# Read current state if file exists
if path.exists():
# read the existing file
existing_markdown = await self.markdown_processor.read_file(path)
# merge content with entity
# if content is supplied use it or existing content
markdown = entity_model_to_markdown(
entity, content=content or existing_markdown.content.content
)
else:
# Create new file structure with provided content
markdown = entity_model_to_markdown(entity, content=content)
# Write back atomically
checksum = await self.markdown_processor.write_file(
path=path, markdown=markdown, expected_checksum=expected_checksum
)
return path, checksum
except Exception as e:
logger.error(f"Failed to write entity file: {e}")
raise FileOperationError(f"Failed to write entity file: {e}")
async def read_entity_content(self, entity: EntityModel) -> str:
"""Get entity's content if it's a note.
"""Get entity's content without frontmatter or structured sections.
Args:
permalink: Entity's path ID
entity: Entity to read content for
Returns:
content without frontmatter
Raw content without frontmatter, observations, or relations
Raises:
FileOperationError: If entity file doesn't exist
"""
logger.debug(f"Reading entity with permalink: {entity.permalink}")
# For notes, read the actual file content
file_path = self.get_entity_path(entity)
content, _ = await self.read_file(file_path)
if "---" in content:
# Strip frontmatter from content
_, _, content = content.split("---", 2)
content = content.strip()
return content
try:
file_path = self.get_entity_path(entity)
markdown = await self.markdown_processor.read_file(file_path)
return markdown.content.content or ""
except Exception as e:
logger.error(f"Failed to read entity content: {e}")
raise FileOperationError(f"Failed to read entity content: {e}")
async def delete_entity_file(self, entity: EntityModel) -> None:
"""Delete entity file from filesystem."""
@@ -184,152 +210,3 @@ class FileService:
except Exception as e:
logger.error(f"Failed to delete file {path}: {e}")
raise FileOperationError(f"Failed to delete file: {e}")
@staticmethod
async def has_frontmatter(content: str) -> bool:
"""
Check if content has frontmatter markers.
Args:
content: Content to check
Returns:
True if content appears to have frontmatter
"""
try:
return file_utils.has_frontmatter(content)
except Exception as e:
logger.error(f"Failed to check frontmatter: {e}")
return False
@staticmethod
async def parse_frontmatter(content: str) -> Dict[str, Any]:
"""
Parse frontmatter from content.
Args:
content: Content containing frontmatter
Returns:
Parsed frontmatter as dict
Raises:
FileOperationError: If parsing fails
"""
try:
return file_utils.parse_frontmatter(content)
except Exception as e:
logger.error(f"Failed to parse frontmatter: {e}")
raise FileOperationError(f"Failed to parse frontmatter: {e}")
@staticmethod
async def remove_frontmatter(content: str) -> str:
"""
Remove frontmatter from content.
Args:
content: Content with frontmatter
Returns:
Content with frontmatter removed
Raises:
FileOperationError: If removal fails
"""
try:
return file_utils.remove_frontmatter(content)
except Exception as e:
logger.error(f"Failed to remove frontmatter: {e}")
raise FileOperationError(f"Failed to remove frontmatter: {e}")
async def remove_frontmatter_lenient(self, content: str) -> str:
"""
Remove frontmatter without validation.
Args:
content: Content that may contain frontmatter
Returns:
Content with potential frontmatter removed
"""
try:
return file_utils.remove_frontmatter_lenient(content)
except Exception as e:
logger.error(f"Failed to remove frontmatter leniently: {e}")
raise FileOperationError(f"Failed to remove frontmatter: {e}")
async def add_frontmatter(
self,
content: str,
frontmatter: Dict[str, Any],
metadata: Optional[Dict[str, Any]] = None,
) -> str:
"""
Add YAML frontmatter to content.
Args:
content: Content to add frontmatter to
frontmatter: Frontmatter to add
metadata: Optional additional metadata
Returns:
Content with frontmatter added
Raises:
FileOperationError: If frontmatter creation fails
"""
try:
if metadata:
frontmatter.update(metadata)
return await file_utils.add_frontmatter(content, frontmatter)
except Exception as e:
logger.error(f"Failed to add frontmatter: {e}")
raise FileOperationError(f"Failed to add frontmatter: {e}")
async def write_with_frontmatter(
self,
path: Path,
content: str,
frontmatter: Dict[str, Any],
) -> str:
"""
Write content to file with frontmatter, properly handling existing frontmatter.
If content already has frontmatter, it will be updated with new values.
If not, frontmatter will be added.
Args:
path: Path where to write
content: Content to write
frontmatter: Frontmatter to add/update
Returns:
Checksum of written content
Raises:
FileOperationError: If operation fails
"""
try:
final_content: str
if await self.has_frontmatter(content):
try:
# Try to parse and merge existing frontmatter
existing_frontmatter = await self.parse_frontmatter(content)
content_only = await self.remove_frontmatter(content)
merged_frontmatter = {**existing_frontmatter, **frontmatter}
final_content = await self.add_frontmatter(content_only, merged_frontmatter)
except FileOperationError:
# If parsing fails, just strip any frontmatter-like content and start fresh
content_only = await self.remove_frontmatter_lenient(content)
final_content = await self.add_frontmatter(content_only, frontmatter)
else:
# No existing frontmatter, just add new
final_content = await self.add_frontmatter(content, frontmatter)
# Write and return checksum
return await self.write_file(path, final_content)
except Exception as e:
logger.error(f"Failed to write file with frontmatter {path}: {e}")
raise FileOperationError(f"Failed to write file with frontmatter: {e}")
src/basic_memory/services/search_service.py
+31 -19
View File
@@ -9,7 +9,8 @@ from basic_memory.models import Entity
from basic_memory.repository import EntityRepository
from basic_memory.repository.search_repository import SearchRepository, SearchIndexRow
from basic_memory.schemas.search import SearchQuery, SearchResult, SearchItemType
from basic_memory.utils import generate_permalink
from basic_memory.services import FileService
from basic_memory.services.exceptions import FileOperationError
class SearchService:
@@ -25,9 +26,11 @@ class SearchService:
self,
search_repository: SearchRepository,
entity_repository: EntityRepository,
file_service: FileService,
):
self.repository = search_repository
self.entity_repository = entity_repository
self.file_service = file_service
async def init_search_index(self):
"""Create FTS5 virtual table if it doesn't exist."""
@@ -100,7 +103,9 @@ class SearchService:
return variants
async def index_entity(
self, entity: Entity, background_tasks: Optional[BackgroundTasks] = None
self,
entity: Entity,
background_tasks: Optional[BackgroundTasks] = None,
) -> None:
"""Index an entity and all its observations and relations.
@@ -119,12 +124,25 @@ class SearchService:
Each type gets its own row in the search index with appropriate metadata.
"""
if background_tasks:
background_tasks.add_task(self.index_entity_data, entity)
else:
await self.index_entity_data(entity)
async def index_entity_data(
self,
entity: Entity,
) -> None:
"""Actually perform the indexing."""
content_parts = []
title_variants = self._generate_variants(entity.title)
content_parts.extend(title_variants)
if entity.summary:
content_parts.append(entity.summary)
# TODO should we do something to content on indexing?
content = await self.file_service.read_entity_content(entity)
if content:
content_parts.append(content)
content_parts.extend(self._generate_variants(entity.permalink))
content_parts.extend(self._generate_variants(entity.file_path))
@@ -132,7 +150,7 @@ class SearchService:
entity_content = "\n".join(p for p in content_parts if p and p.strip())
# Index entity
await self._do_index(
await self.repository.index_item(
SearchIndexRow(
id=entity.id,
type=SearchItemType.ENTITY.value,
@@ -150,11 +168,10 @@ class SearchService:
)
)
# Index each observation with synthetic permalink
# Index each observation with permalink
for obs in entity.observations:
# Index with parent entity's file path since that's where it's defined
await self._do_index(
await self.repository.index_item(
SearchIndexRow(
id=obs.id,
type=SearchItemType.OBSERVATION.value,
@@ -177,9 +194,13 @@ class SearchService:
# Only index outgoing relations (ones defined in this file)
for rel in entity.outgoing_relations:
# Create descriptive title showing the relationship
relation_title = f"{rel.from_entity.title} → {rel.to_entity.title}" if rel.to_entity else f"{rel.from_entity.title}"
relation_title = (
f"{rel.from_entity.title} → {rel.to_entity.title}"
if rel.to_entity
else f"{rel.from_entity.title}"
)
await self._do_index(
await self.repository.index_item(
SearchIndexRow(
id=rel.id,
title=relation_title,
@@ -199,15 +220,6 @@ class SearchService:
)
)
async def _do_index(
self, index_row: SearchIndexRow, background_tasks: Optional[BackgroundTasks] = None
) -> None:
"""Actually perform the indexing."""
if background_tasks:
background_tasks.add_task(self.repository.index_item, index_row)
else:
await self.repository.index_item(index_row)
async def delete_by_permalink(self, path_id: str):
"""Delete an item from the search index."""
await self.repository.delete_by_permalink(path_id)
src/basic_memory/sync/entity_sync_service.py
+2 -2
View File
@@ -1,4 +1,5 @@
"""Service for managing entities in the database."""
from pathlib import Path
from loguru import logger
from sqlalchemy.exc import IntegrityError
@@ -28,12 +29,11 @@ def entity_model_from_markdown(file_path: str, markdown: EntityMarkdown) -> Enti
# TODO handle permalink conflicts
permalink = markdown.frontmatter.permalink or generate_permalink(file_path)
model = EntityModel(
title=markdown.frontmatter.title,
title=markdown.frontmatter.title or Path(file_path).stem,
entity_type=markdown.frontmatter.type,
permalink=permalink,
file_path=file_path,
content_type="text/markdown",
summary=markdown.content.content,
created_at=markdown.frontmatter.created,
updated_at=markdown.frontmatter.modified,
observations=[