admin/basicmachines-co-basic-memory
1
0
Fork 0
mirror of https://github.com/basicmachines-co/basic-memory synced 2026-06-21 13:47:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(core): add observation category filter to search (#908)

Browse Source 
Signed-off-by: phernandez <paul@basicmachines.co>
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Paul Hernandez
2026-06-07 19:57:19 -05:00
committed by GitHub
parent 8acdb49a41
commit 4fe6fe09c8
18 changed files with 395 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/basic_memory/repository/postgres_search_repository.py
+21
View File
@@ -705,6 +705,7 @@ class PostgresSearchRepository(SearchRepositoryBase):
note_types: Optional[List[str]] = None,
after_date: Optional[datetime] = None,
search_item_types: Optional[List[SearchItemType]] = None,
categories: Optional[List[str]] = None,
metadata_filters: Optional[dict] = None,
) -> tuple[str, str, dict, str, str]:
"""Build Postgres FTS FROM/WHERE params shared by search and count."""
@@ -762,6 +763,20 @@ class PostgresSearchRepository(SearchRepositoryBase):
type_placeholders.append(f":{param_name}")
conditions.append(f"search_index.type IN ({', '.join(type_placeholders)})")
# Handle observation category filter (parameterized for defense-in-depth).
# Trigger: caller passed `categories` to scope observation results.
# Why: `entity_types=["observation"]` only narrows to the observation row type;
# callers expect exact-category matching, not incidental text matches.
# Outcome: only rows whose indexed category exactly equals a requested value
# survive (entities/relations have NULL category and are excluded).
if categories:
category_placeholders = []
for idx, category in enumerate(categories):
param_name = f"category_{idx}"
params[param_name] = category
category_placeholders.append(f":{param_name}")
conditions.append(f"search_index.category IN ({', '.join(category_placeholders)})")
# Handle note type filter using JSONB containment (parameterized)
if note_types:
type_conditions = []
@@ -879,6 +894,7 @@ class PostgresSearchRepository(SearchRepositoryBase):
note_types: Optional[List[str]] = None,
after_date: Optional[datetime] = None,
search_item_types: Optional[List[SearchItemType]] = None,
categories: Optional[List[str]] = None,
metadata_filters: Optional[dict] = None,
retrieval_mode: SearchRetrievalMode = SearchRetrievalMode.FTS,
min_similarity: Optional[float] = None,
@@ -895,6 +911,7 @@ class PostgresSearchRepository(SearchRepositoryBase):
note_types=note_types,
after_date=after_date,
search_item_types=search_item_types,
categories=categories,
metadata_filters=metadata_filters,
retrieval_mode=retrieval_mode,
min_similarity=min_similarity,
@@ -919,6 +936,7 @@ class PostgresSearchRepository(SearchRepositoryBase):
note_types=note_types,
after_date=after_date,
search_item_types=search_item_types,
categories=categories,
metadata_filters=metadata_filters,
)
@@ -1008,6 +1026,7 @@ class PostgresSearchRepository(SearchRepositoryBase):
note_types: Optional[List[str]] = None,
after_date: Optional[datetime] = None,
search_item_types: Optional[List[SearchItemType]] = None,
categories: Optional[List[str]] = None,
metadata_filters: Optional[dict] = None,
retrieval_mode: SearchRetrievalMode = SearchRetrievalMode.FTS,
min_similarity: Optional[float] = None,
@@ -1022,6 +1041,7 @@ class PostgresSearchRepository(SearchRepositoryBase):
note_types=note_types,
after_date=after_date,
search_item_types=search_item_types,
categories=categories,
metadata_filters=metadata_filters,
retrieval_mode=retrieval_mode,
min_similarity=min_similarity,
@@ -1041,6 +1061,7 @@ class PostgresSearchRepository(SearchRepositoryBase):
note_types=note_types,
after_date=after_date,
search_item_types=search_item_types,
categories=categories,
metadata_filters=metadata_filters,
)
sql = f"SELECT COUNT(*) FROM {from_clause} WHERE {where_clause}"
src/basic_memory/repository/search_repository.py
+2
View File
@@ -42,6 +42,7 @@ class SearchRepository(Protocol):
note_types: Optional[List[str]] = None,
after_date: Optional[datetime] = None,
search_item_types: Optional[List[SearchItemType]] = None,
categories: Optional[List[str]] = None,
metadata_filters: Optional[dict] = None,
retrieval_mode: SearchRetrievalMode = SearchRetrievalMode.FTS,
min_similarity: Optional[float] = None,
@@ -60,6 +61,7 @@ class SearchRepository(Protocol):
note_types: Optional[List[str]] = None,
after_date: Optional[datetime] = None,
search_item_types: Optional[List[SearchItemType]] = None,
categories: Optional[List[str]] = None,
metadata_filters: Optional[dict] = None,
retrieval_mode: SearchRetrievalMode = SearchRetrievalMode.FTS,
min_similarity: Optional[float] = None,
src/basic_memory/repository/search_repository_base.py
+12
View File
@@ -218,6 +218,7 @@ class SearchRepositoryBase(ABC):
note_types: Optional[List[str]] = None,
after_date: Optional[datetime] = None,
search_item_types: Optional[List[SearchItemType]] = None,
categories: Optional[List[str]] = None,
metadata_filters: Optional[Dict[str, Any]] = None,
retrieval_mode: SearchRetrievalMode = SearchRetrievalMode.FTS,
min_similarity: Optional[float] = None,
@@ -234,6 +235,7 @@ class SearchRepositoryBase(ABC):
note_types: Filter by note types (from metadata.note_type)
after_date: Filter by created_at > after_date
search_item_types: Filter by SearchItemType (ENTITY, OBSERVATION, RELATION)
categories: Filter observations by exact category (e.g. "requirement")
metadata_filters: Structured frontmatter metadata filters
limit: Maximum results to return
offset: Number of results to skip
@@ -256,6 +258,7 @@ class SearchRepositoryBase(ABC):
note_types: Optional[List[str]] = None,
after_date: Optional[datetime] = None,
search_item_types: Optional[List[SearchItemType]] = None,
categories: Optional[List[str]] = None,
metadata_filters: Optional[Dict[str, Any]] = None,
retrieval_mode: SearchRetrievalMode = SearchRetrievalMode.FTS,
min_similarity: Optional[float] = None,
@@ -1785,6 +1788,7 @@ class SearchRepositoryBase(ABC):
note_types: Optional[List[str]],
after_date: Optional[datetime],
search_item_types: Optional[List[SearchItemType]],
categories: Optional[List[str]],
metadata_filters: Optional[dict],
retrieval_mode: SearchRetrievalMode,
min_similarity: Optional[float] = None,
@@ -1818,6 +1822,7 @@ class SearchRepositoryBase(ABC):
note_types=note_types,
after_date=after_date,
search_item_types=search_item_types,
categories=categories,
metadata_filters=metadata_filters,
min_similarity=min_similarity,
limit=limit,
@@ -1837,6 +1842,7 @@ class SearchRepositoryBase(ABC):
note_types=note_types,
after_date=after_date,
search_item_types=search_item_types,
categories=categories,
metadata_filters=metadata_filters,
min_similarity=min_similarity,
limit=limit,
@@ -1866,6 +1872,7 @@ class SearchRepositoryBase(ABC):
note_types: Optional[List[str]],
after_date: Optional[datetime],
search_item_types: Optional[List[SearchItemType]],
categories: Optional[List[str]],
metadata_filters: Optional[dict],
min_similarity: Optional[float] = None,
limit: int,
@@ -1976,6 +1983,7 @@ class SearchRepositoryBase(ABC):
note_types,
after_date,
search_item_types,
categories,
metadata_filters,
]
)
@@ -1989,6 +1997,7 @@ class SearchRepositoryBase(ABC):
note_types=note_types,
after_date=after_date,
search_item_types=search_item_types,
categories=categories,
metadata_filters=metadata_filters,
retrieval_mode=SearchRetrievalMode.FTS,
limit=VECTOR_FILTER_SCAN_LIMIT,
@@ -2139,6 +2148,7 @@ class SearchRepositoryBase(ABC):
note_types: Optional[List[str]],
after_date: Optional[datetime],
search_item_types: Optional[List[SearchItemType]],
categories: Optional[List[str]],
metadata_filters: Optional[dict],
min_similarity: Optional[float] = None,
limit: int,
@@ -2163,6 +2173,7 @@ class SearchRepositoryBase(ABC):
note_types=note_types,
after_date=after_date,
search_item_types=search_item_types,
categories=categories,
metadata_filters=metadata_filters,
retrieval_mode=SearchRetrievalMode.FTS,
limit=candidate_limit,
@@ -2178,6 +2189,7 @@ class SearchRepositoryBase(ABC):
note_types=note_types,
after_date=after_date,
search_item_types=search_item_types,
categories=categories,
metadata_filters=metadata_filters,
min_similarity=min_similarity,
limit=candidate_limit,
src/basic_memory/repository/sqlite_search_repository.py
+21
View File
@@ -733,6 +733,7 @@ class SQLiteSearchRepository(SearchRepositoryBase):
note_types: Optional[List[str]] = None,
after_date: Optional[datetime] = None,
search_item_types: Optional[List[SearchItemType]] = None,
categories: Optional[List[str]] = None,
metadata_filters: Optional[dict] = None,
) -> tuple[str, str, dict, str]:
"""Build SQLite FTS FROM/WHERE params shared by search and count."""
@@ -794,6 +795,20 @@ class SQLiteSearchRepository(SearchRepositoryBase):
type_placeholders.append(f":{param_name}")
conditions.append(f"search_index.type IN ({', '.join(type_placeholders)})")
# Handle observation category filter (parameterized for defense-in-depth).
# Trigger: caller passed `categories` to scope observation results.
# Why: `entity_types=["observation"]` only narrows to the observation row type;
# callers expect exact-category matching, not incidental text matches.
# Outcome: only rows whose indexed category exactly equals a requested value
# survive (entities/relations have NULL category and are excluded).
if categories:
category_placeholders = []
for idx, category in enumerate(categories):
param_name = f"category_{idx}"
params[param_name] = category
category_placeholders.append(f":{param_name}")
conditions.append(f"search_index.category IN ({', '.join(category_placeholders)})")
# Handle note type filter (frontmatter type field, parameterized)
if note_types:
type_placeholders = []
@@ -925,6 +940,7 @@ class SQLiteSearchRepository(SearchRepositoryBase):
note_types: Optional[List[str]] = None,
after_date: Optional[datetime] = None,
search_item_types: Optional[List[SearchItemType]] = None,
categories: Optional[List[str]] = None,
metadata_filters: Optional[dict] = None,
retrieval_mode: SearchRetrievalMode = SearchRetrievalMode.FTS,
min_similarity: Optional[float] = None,
@@ -941,6 +957,7 @@ class SQLiteSearchRepository(SearchRepositoryBase):
note_types=note_types,
after_date=after_date,
search_item_types=search_item_types,
categories=categories,
metadata_filters=metadata_filters,
retrieval_mode=retrieval_mode,
min_similarity=min_similarity,
@@ -959,6 +976,7 @@ class SQLiteSearchRepository(SearchRepositoryBase):
note_types=note_types,
after_date=after_date,
search_item_types=search_item_types,
categories=categories,
metadata_filters=metadata_filters,
)
@@ -1046,6 +1064,7 @@ class SQLiteSearchRepository(SearchRepositoryBase):
note_types: Optional[List[str]] = None,
after_date: Optional[datetime] = None,
search_item_types: Optional[List[SearchItemType]] = None,
categories: Optional[List[str]] = None,
metadata_filters: Optional[dict] = None,
retrieval_mode: SearchRetrievalMode = SearchRetrievalMode.FTS,
min_similarity: Optional[float] = None,
@@ -1060,6 +1079,7 @@ class SQLiteSearchRepository(SearchRepositoryBase):
note_types=note_types,
after_date=after_date,
search_item_types=search_item_types,
categories=categories,
metadata_filters=metadata_filters,
retrieval_mode=retrieval_mode,
min_similarity=min_similarity,
@@ -1073,6 +1093,7 @@ class SQLiteSearchRepository(SearchRepositoryBase):
note_types=note_types,
after_date=after_date,
search_item_types=search_item_types,
categories=categories,
metadata_filters=metadata_filters,
)
sql = f"SELECT COUNT(*) FROM {from_clause} WHERE {where_clause}"