feat(core): add observation category filter to search (#908)

Signed-off-by: phernandez <paul@basicmachines.co>
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

src/basic_memory/api/v2/routers/search_router.py

@@ -64,7 +64,9 @@ async def search(
             or query.permalink
             or query.permalink_match
         ),
-        has_filters=bool(query.note_types or query.entity_types or query.metadata_filters),
+        has_filters=bool(
+            query.note_types or query.entity_types or query.categories or query.metadata_filters
+        ),
     ):
         offset = (page - 1) * page_size
         exact_count_available = query.retrieval_mode == SearchRetrievalMode.FTS

src/basic_memory/mcp/tools/search.py

@@ -155,7 +155,7 @@ def _format_search_error_response(
             - Boolean NOT: `project NOT archived`
             - Grouped: `(project OR planning) AND notes`
             - Exact phrases: `"weekly standup meeting"`
-            - Content-specific: `tag:example` or `category:observation`
+            - Content-specific: `tag:example`
 
             ## Try again with:
             ```
@@ -222,7 +222,7 @@ def _format_search_error_response(
 
             6. **Try advanced search patterns**:
                - Tag search: `search_notes("{project}","tag:your-tag")`
-               - Category search: `search_notes("{project}","category:observation")`
+               - Observation category: `search_notes("{project}","{query}", entity_types=["observation"], categories=["requirement"])`
                - Pattern matching: `search_notes("{project}","*{query}*", search_type="permalink")`
 
             ## Explore what content exists:
@@ -299,7 +299,8 @@ Error searching for '{query}': {error_message}
 - **Boolean**: `term1 AND term2`, `term1 OR term2`, `term1 NOT term2`
 - **Phrases**: `"exact phrase"`
 - **Grouping**: `(term1 OR term2) AND term3`
-- **Patterns**: `tag:example`, `category:observation`"""
+- **Tags**: `tag:example`
+- **Observation categories**: `entity_types=["observation"], categories=["requirement"]`"""
 
 
 def _format_search_markdown(result: SearchResponse, project: str, query: str | None) -> str:
@@ -497,6 +498,7 @@ async def _search_all_projects(
     output_format: Literal["text", "json"],
     note_types: list[str],
     entity_types: list[str],
+    categories: list[str],
     after_date: str | None,
     metadata_filters: dict[str, Any] | None,
     tags: list[str] | None,
@@ -555,6 +557,7 @@ async def _search_all_projects(
                 output_format="json",
                 note_types=note_types or None,
                 entity_types=entity_types or None,
+                categories=categories or None,
                 after_date=after_date,
                 metadata_filters=metadata_filters,
                 tags=tags,
@@ -653,6 +656,14 @@ async def search_notes(
         "'relation'. Defaults to 'entity'. Do NOT pass schema/frontmatter types like "
         "'Chapter' here — use note_types instead.",
     ] = None,
+    categories: Annotated[
+        List[str] | None,
+        BeforeValidator(coerce_list),
+        Field(default=None, validation_alias=AliasChoices("categories", "category")),
+        "Filter observation results to these exact categories (e.g. ['requirement']). "
+        "Pair with entity_types=['observation'] to return only observations whose "
+        "category matches exactly — not every row mentioning the word.",
+    ] = None,
     # Time-filter naming varies wildly across APIs.
     after_date: Annotated[
         Optional[str],
@@ -707,7 +718,8 @@ async def search_notes(
 
     ### Content-Specific Searches
     - `search_notes("research", "tag:example")` - Search within specific tags (if supported by content)
-    - `search_notes("work-project", "category:observation")` - Filter by observation categories
+    - `search_notes("work-project", "req", entity_types=["observation"], categories=["requirement"])`
+      - Return only observations whose category is exactly "requirement"
     - `search_notes("team-docs", "author:username")` - Find content by author (if metadata available)
 
     **Note:** `tag:` shorthand is automatically converted to a `tags` filter, so it works
@@ -725,6 +737,8 @@ async def search_notes(
     - `search_notes("my-project", "query", note_types=["note"])` - Search only notes
     - `search_notes("work-docs", "query", note_types=["note", "person"])` - Multiple note types
     - `search_notes("research", "query", entity_types=["observation"])` - Filter by entity type
+    - `search_notes("research", "query", entity_types=["observation"], categories=["requirement"])`
+      - Filter observations to an exact category
     - `search_notes("team-docs", "query", after_date="2024-01-01")` - Recent content only
     - `search_notes("my-project", "query", after_date="1 week")` - Relative date filtering
     - `search_notes("my-project", "query", tags=["security"])` - Filter by frontmatter tags
@@ -776,6 +790,9 @@ async def search_notes(
             "json" returns a machine-readable dictionary payload.
         note_types: Optional list of note types to search (e.g., ["note", "person"])
         entity_types: Optional list of entity types to filter by (e.g., ["entity", "observation"])
+        categories: Optional list of observation categories for exact matching (e.g.,
+                   ["requirement"]). Pair with entity_types=["observation"] to return only
+                   observations whose category matches exactly.
         after_date: Optional date filter for recent content (e.g., "1 week", "2d", "2024-01-01")
         metadata_filters: Optional structured frontmatter filters (e.g., {"status": "in-progress"})
         tags: Optional tag filter (frontmatter tags); shorthand for metadata_filters["tags"]
@@ -853,6 +870,9 @@ async def search_notes(
     # Lowercase note_types so "Chapter" matches the stored "chapter".
     note_types = [t.lower() for t in note_types] if note_types else []
     entity_types = entity_types or []
+    # Categories are matched exactly against the indexed observation category,
+    # so preserve their original casing (unlike the lowercased note_types).
+    categories = categories or []
 
     # Parse tag:<value> shorthand at tool level so it works with all search modes.
     # Handles "tag:security", "tag:coffee tag:brewing", "tag:coffee AND tag:brewing".
@@ -894,6 +914,7 @@ async def search_notes(
             output_format=output_format,
             note_types=note_types,
             entity_types=entity_types,
+            categories=categories,
             after_date=after_date,
             metadata_filters=metadata_filters,
             tags=tags,
@@ -917,8 +938,15 @@ async def search_notes(
         has_query=bool(query and query.strip()),
         note_type_filter_count=len(note_types),
         entity_type_filter_count=len(entity_types),
+        category_filter_count=len(categories),
         has_filters=bool(
-            metadata_filters or tags or status or note_types or entity_types or after_date
+            metadata_filters
+            or tags
+            or status
+            or note_types
+            or entity_types
+            or categories
+            or after_date
         ),
         has_tags_filter=bool(tags),
         has_status_filter=bool(status),
@@ -981,6 +1009,8 @@ async def search_notes(
                 # Add optional filters if provided (empty lists are treated as no filter)
                 if entity_types:
                     search_query.entity_types = [SearchItemType(t) for t in entity_types]
+                if categories:
+                    search_query.categories = categories
                 if note_types:
                     search_query.note_types = note_types
                 if after_date:
@@ -1006,7 +1036,8 @@ async def search_notes(
                     return (
                         "# No Search Criteria\n\n"
                         "Please provide at least one of: `query`, `metadata_filters`, "
-                        "`tags`, `status`, `note_types`, `entity_types`, or `after_date`."
+                        "`tags`, `status`, `note_types`, `entity_types`, `categories`, "
+                        "or `after_date`."
                     )
 
                 # Default to entity-level results to avoid returning individual
@@ -1014,7 +1045,17 @@ async def search_notes(
                 # Applied after no_criteria() so that the implicit default doesn't
                 # mask a truly empty search request.
                 if not search_query.entity_types:
-                    search_query.entity_types = [SearchItemType("entity")]
+                    # Trigger: a category filter was supplied without an explicit
+                    #          entity_types.
+                    # Why: categories only exist on observations — defaulting to "entity"
+                    #      (whose rows have NULL category) would AND a category filter against
+                    #      entity rows and return nothing, defeating a category-only search.
+                    # Outcome: scope the implicit default to observations so
+                    #          search_notes(categories=[...]) returns the matching bullets.
+                    if search_query.categories:
+                        search_query.entity_types = [SearchItemType("observation")]
+                    else:
+                        search_query.entity_types = [SearchItemType("entity")]
 
                 logger.debug(
                     f"Search request: project={active_project.name} "

src/basic_memory/repository/postgres_search_repository.py

@@ -705,6 +705,7 @@ class PostgresSearchRepository(SearchRepositoryBase):
         note_types: Optional[List[str]] = None,
         after_date: Optional[datetime] = None,
         search_item_types: Optional[List[SearchItemType]] = None,
+        categories: Optional[List[str]] = None,
         metadata_filters: Optional[dict] = None,
     ) -> tuple[str, str, dict, str, str]:
         """Build Postgres FTS FROM/WHERE params shared by search and count."""
@@ -762,6 +763,20 @@ class PostgresSearchRepository(SearchRepositoryBase):
                 type_placeholders.append(f":{param_name}")
             conditions.append(f"search_index.type IN ({', '.join(type_placeholders)})")
 
+        # Handle observation category filter (parameterized for defense-in-depth).
+        # Trigger: caller passed `categories` to scope observation results.
+        # Why: `entity_types=["observation"]` only narrows to the observation row type;
+        #      callers expect exact-category matching, not incidental text matches.
+        # Outcome: only rows whose indexed category exactly equals a requested value
+        #          survive (entities/relations have NULL category and are excluded).
+        if categories:
+            category_placeholders = []
+            for idx, category in enumerate(categories):
+                param_name = f"category_{idx}"
+                params[param_name] = category
+                category_placeholders.append(f":{param_name}")
+            conditions.append(f"search_index.category IN ({', '.join(category_placeholders)})")
+
         # Handle note type filter using JSONB containment (parameterized)
         if note_types:
             type_conditions = []
@@ -879,6 +894,7 @@ class PostgresSearchRepository(SearchRepositoryBase):
         note_types: Optional[List[str]] = None,
         after_date: Optional[datetime] = None,
         search_item_types: Optional[List[SearchItemType]] = None,
+        categories: Optional[List[str]] = None,
         metadata_filters: Optional[dict] = None,
         retrieval_mode: SearchRetrievalMode = SearchRetrievalMode.FTS,
         min_similarity: Optional[float] = None,
@@ -895,6 +911,7 @@ class PostgresSearchRepository(SearchRepositoryBase):
             note_types=note_types,
             after_date=after_date,
             search_item_types=search_item_types,
+            categories=categories,
             metadata_filters=metadata_filters,
             retrieval_mode=retrieval_mode,
             min_similarity=min_similarity,
@@ -919,6 +936,7 @@ class PostgresSearchRepository(SearchRepositoryBase):
             note_types=note_types,
             after_date=after_date,
             search_item_types=search_item_types,
+            categories=categories,
             metadata_filters=metadata_filters,
         )
 
@@ -1008,6 +1026,7 @@ class PostgresSearchRepository(SearchRepositoryBase):
         note_types: Optional[List[str]] = None,
         after_date: Optional[datetime] = None,
         search_item_types: Optional[List[SearchItemType]] = None,
+        categories: Optional[List[str]] = None,
         metadata_filters: Optional[dict] = None,
         retrieval_mode: SearchRetrievalMode = SearchRetrievalMode.FTS,
         min_similarity: Optional[float] = None,
@@ -1022,6 +1041,7 @@ class PostgresSearchRepository(SearchRepositoryBase):
                 note_types=note_types,
                 after_date=after_date,
                 search_item_types=search_item_types,
+                categories=categories,
                 metadata_filters=metadata_filters,
                 retrieval_mode=retrieval_mode,
                 min_similarity=min_similarity,
@@ -1041,6 +1061,7 @@ class PostgresSearchRepository(SearchRepositoryBase):
             note_types=note_types,
             after_date=after_date,
             search_item_types=search_item_types,
+            categories=categories,
             metadata_filters=metadata_filters,
         )
         sql = f"SELECT COUNT(*) FROM {from_clause} WHERE {where_clause}"

src/basic_memory/repository/search_repository.py

@@ -42,6 +42,7 @@ class SearchRepository(Protocol):
         note_types: Optional[List[str]] = None,
         after_date: Optional[datetime] = None,
         search_item_types: Optional[List[SearchItemType]] = None,
+        categories: Optional[List[str]] = None,
         metadata_filters: Optional[dict] = None,
         retrieval_mode: SearchRetrievalMode = SearchRetrievalMode.FTS,
         min_similarity: Optional[float] = None,
@@ -60,6 +61,7 @@ class SearchRepository(Protocol):
         note_types: Optional[List[str]] = None,
         after_date: Optional[datetime] = None,
         search_item_types: Optional[List[SearchItemType]] = None,
+        categories: Optional[List[str]] = None,
         metadata_filters: Optional[dict] = None,
         retrieval_mode: SearchRetrievalMode = SearchRetrievalMode.FTS,
         min_similarity: Optional[float] = None,

src/basic_memory/repository/search_repository_base.py

@@ -218,6 +218,7 @@ class SearchRepositoryBase(ABC):
         note_types: Optional[List[str]] = None,
         after_date: Optional[datetime] = None,
         search_item_types: Optional[List[SearchItemType]] = None,
+        categories: Optional[List[str]] = None,
         metadata_filters: Optional[Dict[str, Any]] = None,
         retrieval_mode: SearchRetrievalMode = SearchRetrievalMode.FTS,
         min_similarity: Optional[float] = None,
@@ -234,6 +235,7 @@ class SearchRepositoryBase(ABC):
             note_types: Filter by note types (from metadata.note_type)
             after_date: Filter by created_at > after_date
             search_item_types: Filter by SearchItemType (ENTITY, OBSERVATION, RELATION)
+            categories: Filter observations by exact category (e.g. "requirement")
             metadata_filters: Structured frontmatter metadata filters
             limit: Maximum results to return
             offset: Number of results to skip
@@ -256,6 +258,7 @@ class SearchRepositoryBase(ABC):
         note_types: Optional[List[str]] = None,
         after_date: Optional[datetime] = None,
         search_item_types: Optional[List[SearchItemType]] = None,
+        categories: Optional[List[str]] = None,
         metadata_filters: Optional[Dict[str, Any]] = None,
         retrieval_mode: SearchRetrievalMode = SearchRetrievalMode.FTS,
         min_similarity: Optional[float] = None,
@@ -1785,6 +1788,7 @@ class SearchRepositoryBase(ABC):
         note_types: Optional[List[str]],
         after_date: Optional[datetime],
         search_item_types: Optional[List[SearchItemType]],
+        categories: Optional[List[str]],
         metadata_filters: Optional[dict],
         retrieval_mode: SearchRetrievalMode,
         min_similarity: Optional[float] = None,
@@ -1818,6 +1822,7 @@ class SearchRepositoryBase(ABC):
                 note_types=note_types,
                 after_date=after_date,
                 search_item_types=search_item_types,
+                categories=categories,
                 metadata_filters=metadata_filters,
                 min_similarity=min_similarity,
                 limit=limit,
@@ -1837,6 +1842,7 @@ class SearchRepositoryBase(ABC):
                 note_types=note_types,
                 after_date=after_date,
                 search_item_types=search_item_types,
+                categories=categories,
                 metadata_filters=metadata_filters,
                 min_similarity=min_similarity,
                 limit=limit,
@@ -1866,6 +1872,7 @@ class SearchRepositoryBase(ABC):
         note_types: Optional[List[str]],
         after_date: Optional[datetime],
         search_item_types: Optional[List[SearchItemType]],
+        categories: Optional[List[str]],
         metadata_filters: Optional[dict],
         min_similarity: Optional[float] = None,
         limit: int,
@@ -1976,6 +1983,7 @@ class SearchRepositoryBase(ABC):
                 note_types,
                 after_date,
                 search_item_types,
+                categories,
                 metadata_filters,
             ]
         )
@@ -1989,6 +1997,7 @@ class SearchRepositoryBase(ABC):
                 note_types=note_types,
                 after_date=after_date,
                 search_item_types=search_item_types,
+                categories=categories,
                 metadata_filters=metadata_filters,
                 retrieval_mode=SearchRetrievalMode.FTS,
                 limit=VECTOR_FILTER_SCAN_LIMIT,
@@ -2139,6 +2148,7 @@ class SearchRepositoryBase(ABC):
         note_types: Optional[List[str]],
         after_date: Optional[datetime],
         search_item_types: Optional[List[SearchItemType]],
+        categories: Optional[List[str]],
         metadata_filters: Optional[dict],
         min_similarity: Optional[float] = None,
         limit: int,
@@ -2163,6 +2173,7 @@ class SearchRepositoryBase(ABC):
             note_types=note_types,
             after_date=after_date,
             search_item_types=search_item_types,
+            categories=categories,
             metadata_filters=metadata_filters,
             retrieval_mode=SearchRetrievalMode.FTS,
             limit=candidate_limit,
@@ -2178,6 +2189,7 @@ class SearchRepositoryBase(ABC):
             note_types=note_types,
             after_date=after_date,
             search_item_types=search_item_types,
+            categories=categories,
             metadata_filters=metadata_filters,
             min_similarity=min_similarity,
             limit=candidate_limit,

src/basic_memory/repository/sqlite_search_repository.py

@@ -733,6 +733,7 @@ class SQLiteSearchRepository(SearchRepositoryBase):
         note_types: Optional[List[str]] = None,
         after_date: Optional[datetime] = None,
         search_item_types: Optional[List[SearchItemType]] = None,
+        categories: Optional[List[str]] = None,
         metadata_filters: Optional[dict] = None,
     ) -> tuple[str, str, dict, str]:
         """Build SQLite FTS FROM/WHERE params shared by search and count."""
@@ -794,6 +795,20 @@ class SQLiteSearchRepository(SearchRepositoryBase):
                 type_placeholders.append(f":{param_name}")
             conditions.append(f"search_index.type IN ({', '.join(type_placeholders)})")
 
+        # Handle observation category filter (parameterized for defense-in-depth).
+        # Trigger: caller passed `categories` to scope observation results.
+        # Why: `entity_types=["observation"]` only narrows to the observation row type;
+        #      callers expect exact-category matching, not incidental text matches.
+        # Outcome: only rows whose indexed category exactly equals a requested value
+        #          survive (entities/relations have NULL category and are excluded).
+        if categories:
+            category_placeholders = []
+            for idx, category in enumerate(categories):
+                param_name = f"category_{idx}"
+                params[param_name] = category
+                category_placeholders.append(f":{param_name}")
+            conditions.append(f"search_index.category IN ({', '.join(category_placeholders)})")
+
         # Handle note type filter (frontmatter type field, parameterized)
         if note_types:
             type_placeholders = []
@@ -925,6 +940,7 @@ class SQLiteSearchRepository(SearchRepositoryBase):
         note_types: Optional[List[str]] = None,
         after_date: Optional[datetime] = None,
         search_item_types: Optional[List[SearchItemType]] = None,
+        categories: Optional[List[str]] = None,
         metadata_filters: Optional[dict] = None,
         retrieval_mode: SearchRetrievalMode = SearchRetrievalMode.FTS,
         min_similarity: Optional[float] = None,
@@ -941,6 +957,7 @@ class SQLiteSearchRepository(SearchRepositoryBase):
             note_types=note_types,
             after_date=after_date,
             search_item_types=search_item_types,
+            categories=categories,
             metadata_filters=metadata_filters,
             retrieval_mode=retrieval_mode,
             min_similarity=min_similarity,
@@ -959,6 +976,7 @@ class SQLiteSearchRepository(SearchRepositoryBase):
             note_types=note_types,
             after_date=after_date,
             search_item_types=search_item_types,
+            categories=categories,
             metadata_filters=metadata_filters,
         )
 
@@ -1046,6 +1064,7 @@ class SQLiteSearchRepository(SearchRepositoryBase):
         note_types: Optional[List[str]] = None,
         after_date: Optional[datetime] = None,
         search_item_types: Optional[List[SearchItemType]] = None,
+        categories: Optional[List[str]] = None,
         metadata_filters: Optional[dict] = None,
         retrieval_mode: SearchRetrievalMode = SearchRetrievalMode.FTS,
         min_similarity: Optional[float] = None,
@@ -1060,6 +1079,7 @@ class SQLiteSearchRepository(SearchRepositoryBase):
                 note_types=note_types,
                 after_date=after_date,
                 search_item_types=search_item_types,
+                categories=categories,
                 metadata_filters=metadata_filters,
                 retrieval_mode=retrieval_mode,
                 min_similarity=min_similarity,
@@ -1073,6 +1093,7 @@ class SQLiteSearchRepository(SearchRepositoryBase):
             note_types=note_types,
             after_date=after_date,
             search_item_types=search_item_types,
+            categories=categories,
             metadata_filters=metadata_filters,
         )
         sql = f"SELECT COUNT(*) FROM {from_clause} WHERE {where_clause}"

src/basic_memory/schemas/search.py

@@ -42,6 +42,7 @@ class SearchQuery(BaseModel):
     Optionally filter results by:
     - note_types: Limit to specific note types (frontmatter "type")
     - entity_types: Limit to search item types (entity/observation/relation)
+    - categories: Limit observation results to exact category matches (e.g. "requirement")
     - after_date: Only items after date
     - metadata_filters: Structured frontmatter filters (field -> value)
     - tags: Convenience frontmatter tag filter
@@ -63,6 +64,7 @@ class SearchQuery(BaseModel):
     # Optional filters
     note_types: Optional[List[str]] = None  # Filter by note type (frontmatter "type")
     entity_types: Optional[List[SearchItemType]] = None  # Filter by entity type
+    categories: Optional[List[str]] = None  # Filter observations by exact category
     after_date: Optional[Union[datetime, str]] = None  # Time-based filter
     metadata_filters: Optional[dict[str, Any]] = None  # Structured frontmatter filters
     tags: Optional[List[str]] = None  # Convenience tag filter
@@ -85,6 +87,7 @@ class SearchQuery(BaseModel):
         status_is_empty = self.status is None or (isinstance(self.status, str) and not self.status)
         note_types_is_empty = not self.note_types
         entity_types_is_empty = not self.entity_types
+        categories_is_empty = not self.categories
         return (
             self.permalink is None
             and self.permalink_match is None
@@ -93,6 +96,7 @@ class SearchQuery(BaseModel):
             and self.after_date is None
             and note_types_is_empty
             and entity_types_is_empty
+            and categories_is_empty
             and metadata_is_empty
             and tags_is_empty
             and status_is_empty

src/basic_memory/services/search_service.py

@@ -75,6 +75,7 @@ class _PreparedSearchQuery:
     title: str | None
     note_types: list[str] | None
     search_item_types: list[SearchItemType] | None
+    categories: list[str] | None
     after_date: datetime | None
     metadata_filters: dict[str, Any] | None
     retrieval_mode: SearchRetrievalMode
@@ -194,6 +195,7 @@ class SearchService:
             title=query.title,
             note_types=query.note_types,
             search_item_types=query.entity_types,
+            categories=query.categories,
             after_date=after_date,
             metadata_filters=metadata_filters,
             retrieval_mode=query.retrieval_mode or SearchRetrievalMode.FTS,
@@ -207,6 +209,7 @@ class SearchService:
             or prepared.title
             or prepared.note_types
             or prepared.search_item_types
+            or prepared.categories
             or prepared.after_date
             or prepared.metadata_filters
         )
@@ -221,6 +224,7 @@ class SearchService:
             prepared.metadata_filters
             or prepared.note_types
             or prepared.search_item_types
+            or prepared.categories
             or prepared.after_date
         )
 
@@ -239,6 +243,7 @@ class SearchService:
             title=prepared.title,
             note_types=prepared.note_types,
             search_item_types=prepared.search_item_types,
+            categories=prepared.categories,
             after_date=prepared.after_date,
             metadata_filters=prepared.metadata_filters,
             retrieval_mode=prepared.retrieval_mode,
@@ -260,6 +265,7 @@ class SearchService:
             title=prepared.title,
             note_types=prepared.note_types,
             search_item_types=prepared.search_item_types,
+            categories=prepared.categories,
             after_date=prepared.after_date,
             metadata_filters=prepared.metadata_filters,
             retrieval_mode=prepared.retrieval_mode,