admin/basicmachines-co-basic-memory
1
0
Fork 0
mirror of https://github.com/basicmachines-co/basic-memory synced 2026-06-21 13:47:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(core): exclude stale entity rows from embedding coverage stats (#675)

Browse Source 
Signed-off-by: phernandez <paul@basicmachines.co>
Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Paul Hernandez
2026-03-15 18:45:36 -05:00
committed by GitHub
parent 013864ebf0
commit c8b00449d2
3 changed files with 119 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/basic_memory/services/project_service.py
+19 -7
View File
@@ -997,9 +997,18 @@ class ProjectService:
)
# --- Count queries (tables exist) ---
# Filter by entity existence to exclude stale rows from deleted entities
# that remain in derived search tables (search_index, search_vector_chunks)
entity_exists = "AND entity_id IN (SELECT id FROM entity WHERE project_id = :project_id)"
# Same filter for aliased chunks table (used in JOIN queries below)
chunk_entity_exists = (
"AND c.entity_id IN (SELECT id FROM entity WHERE project_id = :project_id)"
)
si_result = await self.repository.execute_query(
text(
"SELECT COUNT(DISTINCT entity_id) FROM search_index WHERE project_id = :project_id"
"SELECT COUNT(DISTINCT entity_id) FROM search_index "
f"WHERE project_id = :project_id {entity_exists}"
),
{"project_id": project_id},
)
@@ -1007,7 +1016,10 @@ class ProjectService:
try:
chunks_result = await self.repository.execute_query(
text("SELECT COUNT(*) FROM search_vector_chunks WHERE project_id = :project_id"),
text(
"SELECT COUNT(*) FROM search_vector_chunks "
f"WHERE project_id = :project_id {entity_exists}"
),
{"project_id": project_id},
)
total_chunks = chunks_result.scalar() or 0
@@ -1015,7 +1027,7 @@ class ProjectService:
entities_with_chunks_result = await self.repository.execute_query(
text(
"SELECT COUNT(DISTINCT entity_id) FROM search_vector_chunks "
"WHERE project_id = :project_id"
f"WHERE project_id = :project_id {entity_exists}"
),
{"project_id": project_id},
)
@@ -1026,13 +1038,13 @@ class ProjectService:
embeddings_sql = text(
"SELECT COUNT(*) FROM search_vector_chunks c "
"JOIN search_vector_embeddings e ON e.chunk_id = c.id "
"WHERE c.project_id = :project_id"
f"WHERE c.project_id = :project_id {chunk_entity_exists}"
)
else:
embeddings_sql = text(
"SELECT COUNT(*) FROM search_vector_chunks c "
"JOIN search_vector_embeddings e ON e.rowid = c.id "
"WHERE c.project_id = :project_id"
f"WHERE c.project_id = :project_id {chunk_entity_exists}"
)
embeddings_result = await self.repository.execute_query(
@@ -1045,13 +1057,13 @@ class ProjectService:
orphan_sql = text(
"SELECT COUNT(*) FROM search_vector_chunks c "
"LEFT JOIN search_vector_embeddings e ON e.chunk_id = c.id "
"WHERE c.project_id = :project_id AND e.chunk_id IS NULL"
f"WHERE c.project_id = :project_id AND e.chunk_id IS NULL {chunk_entity_exists}"
)
else:
orphan_sql = text(
"SELECT COUNT(*) FROM search_vector_chunks c "
"LEFT JOIN search_vector_embeddings e ON e.rowid = c.id "
"WHERE c.project_id = :project_id AND e.rowid IS NULL"
f"WHERE c.project_id = :project_id AND e.rowid IS NULL {chunk_entity_exists}"
)
orphan_result = await self.repository.execute_query(
src/basic_memory/services/search_service.py
+51
View File
@@ -403,6 +403,11 @@ class SearchService:
"""
entities = await self.entity_repository.find_all()
entity_ids = [entity.id for entity in entities]
# Clean up stale rows in search_index and search_vector_chunks
# that reference entity_ids no longer in the entity table
await self._purge_stale_search_rows()
batch_result = await self.repository.sync_entity_vectors_batch(
entity_ids,
progress_callback=progress_callback,
@@ -419,6 +424,52 @@ class SearchService:
return stats
async def _purge_stale_search_rows(self) -> None:
"""Remove rows from search_index and search_vector_chunks for deleted entities.
Trigger: entities are deleted but their derived search rows remain
Why: stale rows inflate embedding coverage stats in project info
Outcome: search tables only contain rows for entities that still exist
"""
from basic_memory.repository.sqlite_search_repository import SQLiteSearchRepository
from sqlalchemy import text
project_id = self.repository.project_id
stale_entity_filter = (
"entity_id NOT IN (SELECT id FROM entity WHERE project_id = :project_id)"
)
params = {"project_id": project_id}
# Delete stale search_index rows
await self.repository.execute_query(
text(
f"DELETE FROM search_index WHERE project_id = :project_id AND {stale_entity_filter}"
),
params,
)
# SQLite vec has no CASCADE — must delete embeddings before chunks
if isinstance(self.repository, SQLiteSearchRepository):
await self.repository.execute_query(
text(
"DELETE FROM search_vector_embeddings WHERE rowid IN ("
"SELECT id FROM search_vector_chunks "
f"WHERE project_id = :project_id AND {stale_entity_filter})"
),
params,
)
# Postgres CASCADE handles embedding deletion automatically
await self.repository.execute_query(
text(
f"DELETE FROM search_vector_chunks "
f"WHERE project_id = :project_id AND {stale_entity_filter}"
),
params,
)
logger.info("Purged stale search rows for deleted entities", project_id=project_id)
async def index_entity_file(
self,
entity: Entity,