admin/basicmachines-co-basic-memory
1
0
Fork 0
mirror of https://github.com/basicmachines-co/basic-memory synced 2026-06-21 13:47:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(sync): instrument single markdown indexing

Browse Source 
Signed-off-by: phernandez <paul@basicmachines.co>
This commit is contained in:
phernandez
2026-04-16 18:04:17 -05:00
parent e2e65575d6
commit c50d97e548
2 changed files with 79 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/basic_memory/indexing/batch_indexer.py
+26 -12
View File
@@ -11,6 +11,7 @@ from typing import Awaitable, Callable, Mapping, TypeVar
from loguru import logger
from sqlalchemy.exc import IntegrityError
from basic_memory import telemetry
from basic_memory.config import BasicMemoryConfig
from basic_memory.file_utils import compute_checksum, has_frontmatter, remove_frontmatter
from basic_memory.markdown.schemas import EntityMarkdown
@@ -190,35 +191,48 @@ class BatchIndexer:
if not self._is_markdown(file):
raise ValueError(f"index_markdown_file requires markdown input: {file.path}")
prepared = await self._prepare_markdown_file(file)
with telemetry.span("index.markdown_file.prepare", path=file.path):
prepared = await self._prepare_markdown_file(file)
if existing_permalink_by_path is None:
existing_permalink_by_path = {
path: permalink
for path, permalink in (
await self.entity_repository.get_file_path_to_permalink_map()
).items()
}
with telemetry.span("index.markdown_file.load_permalink_map", path=file.path):
existing_permalink_by_path = {
path: permalink
for path, permalink in (
await self.entity_repository.get_file_path_to_permalink_map()
).items()
}
reserved_permalinks = {
permalink
for path, permalink in existing_permalink_by_path.items()
if path != file.path and permalink
}
prepared = await self._normalize_markdown_file(prepared, reserved_permalinks)
with telemetry.span("index.markdown_file.normalize", path=file.path):
prepared = await self._normalize_markdown_file(prepared, reserved_permalinks)
existing_permalink_by_path[file.path] = prepared.markdown.frontmatter.permalink
persisted = await self._persist_markdown_file(prepared, is_new=new)
with telemetry.span("index.markdown_file.persist", path=file.path, is_new=new):
persisted = await self._persist_markdown_file(prepared, is_new=new)
existing_permalink_by_path[file.path] = persisted.entity.permalink
await self._resolve_batch_relations([persisted.entity.id], max_concurrent=1)
refreshed = await self.entity_repository.find_by_ids([persisted.entity.id])
with telemetry.span(
"index.markdown_file.reload_entity",
path=file.path,
entity_id=persisted.entity.id,
):
refreshed = await self.entity_repository.find_by_ids([persisted.entity.id])
if len(refreshed) != 1: # pragma: no cover
raise ValueError(f"Failed to reload indexed entity for {file.path}")
entity = refreshed[0]
prepared_entity = self._build_prepared_entity(persisted.prepared, entity)
if index_search:
return await self._refresh_search_index(prepared_entity, entity)
with telemetry.span(
"index.markdown_file.refresh_search_index",
path=file.path,
entity_id=entity.id,
):
return await self._refresh_search_index(prepared_entity, entity)
return IndexedEntity(
path=prepared_entity.path,