admin/basicmachines-co-basic-memory
1
0
Fork 0
mirror of https://github.com/basicmachines-co/basic-memory synced 2026-06-21 13:47:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: min_similarity override, cloud promo improvements (#570)

Browse Source 
Signed-off-by: phernandez <paul@basicmachines.co>
Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Paul Hernandez
2026-02-16 16:44:02 -06:00
committed by GitHub
parent 6afe4fd0cc
commit 55d675e278
106 changed files with 7500 additions and 852 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/basic_memory/repository/postgres_search_repository.py
+14 -9
View File
@@ -51,6 +51,7 @@ class PostgresSearchRepository(SearchRepositoryBase):
self._app_config = app_config or ConfigManager().config
self._semantic_enabled = self._app_config.semantic_search_enabled
self._semantic_vector_k = self._app_config.semantic_vector_k
self._semantic_min_similarity = self._app_config.semantic_min_similarity
self._embedding_provider = embedding_provider
self._vector_dimensions = 384
self._vector_tables_initialized = False
@@ -64,17 +65,16 @@ class PostgresSearchRepository(SearchRepositoryBase):
async def init_search_index(self):
"""Create Postgres table with tsvector column and GIN indexes.
Note: This is handled by Alembic migrations. This method is a no-op
for Postgres as the schema is created via migrations.
Note: FTS schema is handled by Alembic migrations. Vector tables are
created here at startup so missing pgvector or provider errors surface
immediately.
"""
logger.info("PostgreSQL search index initialization handled by migrations")
# Table creation is done via Alembic migrations
# This includes:
# - CREATE TABLE search_index (...)
# - ADD COLUMN textsearchable_index_col tsvector GENERATED ALWAYS AS (...)
# - CREATE INDEX USING GIN on textsearchable_index_col
# - CREATE INDEX USING GIN on metadata jsonb_path_ops
pass
# Fail fast: create vector tables at startup so missing pgvector
# or embedding provider errors surface immediately
if self._semantic_enabled:
await self._ensure_vector_tables()
async def index_item(self, search_index_row: SearchIndexRow) -> None:
"""Index or update a single item using UPSERT.
@@ -260,6 +260,8 @@ class PostgresSearchRepository(SearchRepositoryBase):
if self._vector_tables_initialized:
return
logger.info("Ensuring Postgres vector tables exist for semantic search")
async with self._vector_tables_lock:
if self._vector_tables_initialized:
return
@@ -349,6 +351,7 @@ class PostgresSearchRepository(SearchRepositoryBase):
)
await session.commit()
logger.info(f"Postgres vector tables ready (dimensions={self._vector_dimensions})")
self._vector_tables_initialized = True
async def _get_existing_embedding_dims(self, session: AsyncSession) -> int | None:
@@ -587,6 +590,7 @@ class PostgresSearchRepository(SearchRepositoryBase):
search_item_types: Optional[List[SearchItemType]] = None,
metadata_filters: Optional[dict] = None,
retrieval_mode: SearchRetrievalMode = SearchRetrievalMode.FTS,
min_similarity: Optional[float] = None,
limit: int = 10,
offset: int = 0,
) -> List[SearchIndexRow]:
@@ -602,6 +606,7 @@ class PostgresSearchRepository(SearchRepositoryBase):
search_item_types=search_item_types,
metadata_filters=metadata_filters,
retrieval_mode=retrieval_mode,
min_similarity=min_similarity,
limit=limit,
offset=offset,
)
src/basic_memory/repository/search_repository.py
+1
View File
@@ -42,6 +42,7 @@ class SearchRepository(Protocol):
search_item_types: Optional[List[SearchItemType]] = None,
metadata_filters: Optional[dict] = None,
retrieval_mode: SearchRetrievalMode = SearchRetrievalMode.FTS,
min_similarity: Optional[float] = None,
limit: int = 10,
offset: int = 0,
) -> List[SearchIndexRow]:
src/basic_memory/repository/search_repository_base.py
+38 -6
View File
@@ -49,6 +49,7 @@ class SearchRepositoryBase(ABC):
# --- Subclass-populated attributes ---
_semantic_enabled: bool
_semantic_vector_k: int
_semantic_min_similarity: float
_embedding_provider: Optional[EmbeddingProvider]
_vector_dimensions: int
_vector_tables_initialized: bool
@@ -112,6 +113,7 @@ class SearchRepositoryBase(ABC):
search_item_types: Optional[List[SearchItemType]] = None,
metadata_filters: Optional[Dict[str, Any]] = None,
retrieval_mode: SearchRetrievalMode = SearchRetrievalMode.FTS,
min_similarity: Optional[float] = None,
limit: int = 10,
offset: int = 0,
) -> List[SearchIndexRow]:
@@ -753,6 +755,7 @@ class SearchRepositoryBase(ABC):
search_item_types: Optional[List[SearchItemType]],
metadata_filters: Optional[dict],
retrieval_mode: SearchRetrievalMode,
min_similarity: Optional[float] = None,
limit: int,
offset: int,
) -> Optional[List[SearchIndexRow]]:
@@ -784,6 +787,7 @@ class SearchRepositoryBase(ABC):
after_date=after_date,
search_item_types=search_item_types,
metadata_filters=metadata_filters,
min_similarity=min_similarity,
limit=limit,
offset=offset,
)
@@ -802,6 +806,7 @@ class SearchRepositoryBase(ABC):
after_date=after_date,
search_item_types=search_item_types,
metadata_filters=metadata_filters,
min_similarity=min_similarity,
limit=limit,
offset=offset,
)
@@ -830,6 +835,7 @@ class SearchRepositoryBase(ABC):
after_date: Optional[datetime],
search_item_types: Optional[List[SearchItemType]],
metadata_filters: Optional[dict],
min_similarity: Optional[float] = None,
limit: int,
offset: int,
) -> List[SearchIndexRow]:
@@ -843,7 +849,7 @@ class SearchRepositoryBase(ABC):
await self._ensure_vector_tables()
assert self._embedding_provider is not None
query_embedding = await self._embedding_provider.embed_query(search_text.strip())
candidate_limit = max(self._semantic_vector_k, (limit + offset) * 5)
candidate_limit = max(self._semantic_vector_k, (limit + offset) * 10)
async with db.scoped_session(self.session_maker) as session:
await self._prepare_vector_session(session)
@@ -872,6 +878,18 @@ class SearchRepositoryBase(ABC):
if not similarity_by_si_id:
return []
# Filter out results below the minimum similarity threshold.
# Per-query min_similarity overrides the instance-level default.
effective_min_similarity = (
min_similarity if min_similarity is not None else self._semantic_min_similarity
)
if effective_min_similarity > 0.0:
similarity_by_si_id = {
k: v for k, v in similarity_by_si_id.items() if v >= effective_min_similarity
}
if not similarity_by_si_id:
return []
# Fetch the actual search_index rows
si_ids = list(similarity_by_si_id.keys())
search_index_rows = await self._fetch_search_index_rows_by_ids(si_ids)
@@ -1029,6 +1047,7 @@ class SearchRepositoryBase(ABC):
after_date: Optional[datetime],
search_item_types: Optional[List[SearchItemType]],
metadata_filters: Optional[dict],
min_similarity: Optional[float] = None,
limit: int,
offset: int,
) -> List[SearchIndexRow]:
@@ -1061,26 +1080,39 @@ class SearchRepositoryBase(ABC):
after_date=after_date,
search_item_types=search_item_types,
metadata_filters=metadata_filters,
min_similarity=min_similarity,
limit=candidate_limit,
offset=0,
)
# RRF fusion keyed on search_index row id for granular results.
# This allows observations and relations to surface as individual results,
# not collapsed into their parent entity.
# Score-weighted RRF fusion keyed on search_index row id.
# Multiplies the standard 1/(k+rank) score by the normalized original score
# so that high-confidence matches contribute more than weak ones at the same rank.
fused_scores: dict[int, float] = {}
rows_by_id: dict[int, SearchIndexRow] = {}
# Normalize FTS scores to [0, 1] — handles both SQLite (negative bm25)
# and Postgres (positive ts_rank) by using absolute values
fts_abs = [abs(row.score or 0.0) for row in fts_results]
fts_max = max(fts_abs) if fts_abs else 1.0
for rank, row in enumerate(fts_results, start=1):
if row.id is None:
continue
fused_scores[row.id] = fused_scores.get(row.id, 0.0) + (1.0 / (RRF_K + rank))
norm = abs(row.score or 0.0) / fts_max if fts_max > 0 else 0.0
weight = max(norm, 0.1) # floor preserves RRF stability
fused_scores[row.id] = fused_scores.get(row.id, 0.0) + weight * (1.0 / (RRF_K + rank))
rows_by_id[row.id] = row
# Vector scores already in [0, 1] from the similarity formula
vec_max = max((row.score or 0.0) for row in vector_results) if vector_results else 1.0
for rank, row in enumerate(vector_results, start=1):
if row.id is None:
continue
fused_scores[row.id] = fused_scores.get(row.id, 0.0) + (1.0 / (RRF_K + rank))
norm = (row.score or 0.0) / vec_max if vec_max > 0 else 0.0
weight = max(norm, 0.1) # floor preserves RRF stability
fused_scores[row.id] = fused_scores.get(row.id, 0.0) + weight * (1.0 / (RRF_K + rank))
rows_by_id[row.id] = row
ranked = sorted(fused_scores.items(), key=lambda item: item[1], reverse=True)
src/basic_memory/repository/sqlite_search_repository.py
+18 -1
View File
@@ -51,6 +51,7 @@ class SQLiteSearchRepository(SearchRepositoryBase):
self._app_config = app_config or ConfigManager().config
self._semantic_enabled = self._app_config.semantic_search_enabled
self._semantic_vector_k = self._app_config.semantic_vector_k
self._semantic_min_similarity = self._app_config.semantic_min_similarity
self._embedding_provider = embedding_provider
self._sqlite_vec_lock = asyncio.Lock()
self._vector_tables_initialized = False
@@ -72,7 +73,8 @@ class SQLiteSearchRepository(SearchRepositoryBase):
"""Create FTS5 virtual table for search if it doesn't exist.
Uses CREATE VIRTUAL TABLE IF NOT EXISTS to preserve existing indexed data
across server restarts.
across server restarts. Also creates vector tables when semantic search
is enabled so missing dependencies are caught at startup, not first query.
"""
logger.info("Initializing SQLite FTS5 search index")
try:
@@ -84,6 +86,11 @@ class SQLiteSearchRepository(SearchRepositoryBase):
logger.error(f"Error initializing search index: {e}")
raise e
# Fail fast: create vector tables at startup so missing sqlite-vec
# or embedding provider errors surface immediately
if self._semantic_enabled:
await self._ensure_vector_tables()
# ------------------------------------------------------------------
# FTS5 query preparation (backend-specific)
# ------------------------------------------------------------------
@@ -367,6 +374,8 @@ class SQLiteSearchRepository(SearchRepositoryBase):
if self._vector_tables_initialized:
return
logger.info("Ensuring SQLite vector tables exist for semantic search")
async with db.scoped_session(self.session_maker) as session:
await self._ensure_sqlite_vec_loaded(session)
@@ -386,6 +395,7 @@ class SQLiteSearchRepository(SearchRepositoryBase):
}
schema_mismatch = bool(chunks_columns) and set(chunks_columns) != expected_columns
if schema_mismatch:
logger.warning("search_vector_chunks schema mismatch, recreating vector tables")
await session.execute(text("DROP TABLE IF EXISTS search_vector_embeddings"))
await session.execute(text("DROP TABLE IF EXISTS search_vector_chunks"))
@@ -408,11 +418,16 @@ class SQLiteSearchRepository(SearchRepositoryBase):
expected_dimension_sql = f"float[{self._vector_dimensions}]"
if vector_sql and expected_dimension_sql not in vector_sql:
logger.warning(
f"Embedding dimension mismatch (expected {self._vector_dimensions}), "
"recreating search_vector_embeddings"
)
await session.execute(text("DROP TABLE IF EXISTS search_vector_embeddings"))
await session.execute(create_sqlite_search_vector_embeddings(self._vector_dimensions))
await session.commit()
logger.info(f"SQLite vector tables ready (dimensions={self._vector_dimensions})")
self._vector_tables_initialized = True
async def _prepare_vector_session(self, session: AsyncSession) -> None:
@@ -566,6 +581,7 @@ class SQLiteSearchRepository(SearchRepositoryBase):
search_item_types: Optional[List[SearchItemType]] = None,
metadata_filters: Optional[dict] = None,
retrieval_mode: SearchRetrievalMode = SearchRetrievalMode.FTS,
min_similarity: Optional[float] = None,
limit: int = 10,
offset: int = 0,
) -> List[SearchIndexRow]:
@@ -581,6 +597,7 @@ class SQLiteSearchRepository(SearchRepositoryBase):
search_item_types=search_item_types,
metadata_filters=metadata_filters,
retrieval_mode=retrieval_mode,
min_similarity=min_similarity,
limit=limit,
offset=offset,
)