feat(core): add observation category filter to search (#908)

Signed-off-by: phernandez <paul@basicmachines.co>
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Paul Hernandez
2026-06-07 19:57:19 -05:00
committed by GitHub
parent 8acdb49a41
commit 4fe6fe09c8
18 changed files with 395 additions and 8 deletions
@@ -64,7 +64,9 @@ async def search(
or query.permalink
or query.permalink_match
),
has_filters=bool(query.note_types or query.entity_types or query.metadata_filters),
has_filters=bool(
query.note_types or query.entity_types or query.categories or query.metadata_filters
),
):
offset = (page - 1) * page_size
exact_count_available = query.retrieval_mode == SearchRetrievalMode.FTS
+48 -7
View File
@@ -155,7 +155,7 @@ def _format_search_error_response(
- Boolean NOT: `project NOT archived`
- Grouped: `(project OR planning) AND notes`
- Exact phrases: `"weekly standup meeting"`
- Content-specific: `tag:example` or `category:observation`
- Content-specific: `tag:example`
## Try again with:
```
@@ -222,7 +222,7 @@ def _format_search_error_response(
6. **Try advanced search patterns**:
- Tag search: `search_notes("{project}","tag:your-tag")`
- Category search: `search_notes("{project}","category:observation")`
- Observation category: `search_notes("{project}","{query}", entity_types=["observation"], categories=["requirement"])`
- Pattern matching: `search_notes("{project}","*{query}*", search_type="permalink")`
## Explore what content exists:
@@ -299,7 +299,8 @@ Error searching for '{query}': {error_message}
- **Boolean**: `term1 AND term2`, `term1 OR term2`, `term1 NOT term2`
- **Phrases**: `"exact phrase"`
- **Grouping**: `(term1 OR term2) AND term3`
- **Patterns**: `tag:example`, `category:observation`"""
- **Tags**: `tag:example`
- **Observation categories**: `entity_types=["observation"], categories=["requirement"]`"""
def _format_search_markdown(result: SearchResponse, project: str, query: str | None) -> str:
@@ -497,6 +498,7 @@ async def _search_all_projects(
output_format: Literal["text", "json"],
note_types: list[str],
entity_types: list[str],
categories: list[str],
after_date: str | None,
metadata_filters: dict[str, Any] | None,
tags: list[str] | None,
@@ -555,6 +557,7 @@ async def _search_all_projects(
output_format="json",
note_types=note_types or None,
entity_types=entity_types or None,
categories=categories or None,
after_date=after_date,
metadata_filters=metadata_filters,
tags=tags,
@@ -653,6 +656,14 @@ async def search_notes(
"'relation'. Defaults to 'entity'. Do NOT pass schema/frontmatter types like "
"'Chapter' here — use note_types instead.",
] = None,
categories: Annotated[
List[str] | None,
BeforeValidator(coerce_list),
Field(default=None, validation_alias=AliasChoices("categories", "category")),
"Filter observation results to these exact categories (e.g. ['requirement']). "
"Pair with entity_types=['observation'] to return only observations whose "
"category matches exactly — not every row mentioning the word.",
] = None,
# Time-filter naming varies wildly across APIs.
after_date: Annotated[
Optional[str],
@@ -707,7 +718,8 @@ async def search_notes(
### Content-Specific Searches
- `search_notes("research", "tag:example")` - Search within specific tags (if supported by content)
- `search_notes("work-project", "category:observation")` - Filter by observation categories
- `search_notes("work-project", "req", entity_types=["observation"], categories=["requirement"])`
- Return only observations whose category is exactly "requirement"
- `search_notes("team-docs", "author:username")` - Find content by author (if metadata available)
**Note:** `tag:` shorthand is automatically converted to a `tags` filter, so it works
@@ -725,6 +737,8 @@ async def search_notes(
- `search_notes("my-project", "query", note_types=["note"])` - Search only notes
- `search_notes("work-docs", "query", note_types=["note", "person"])` - Multiple note types
- `search_notes("research", "query", entity_types=["observation"])` - Filter by entity type
- `search_notes("research", "query", entity_types=["observation"], categories=["requirement"])`
- Filter observations to an exact category
- `search_notes("team-docs", "query", after_date="2024-01-01")` - Recent content only
- `search_notes("my-project", "query", after_date="1 week")` - Relative date filtering
- `search_notes("my-project", "query", tags=["security"])` - Filter by frontmatter tags
@@ -776,6 +790,9 @@ async def search_notes(
"json" returns a machine-readable dictionary payload.
note_types: Optional list of note types to search (e.g., ["note", "person"])
entity_types: Optional list of entity types to filter by (e.g., ["entity", "observation"])
categories: Optional list of observation categories for exact matching (e.g.,
["requirement"]). Pair with entity_types=["observation"] to return only
observations whose category matches exactly.
after_date: Optional date filter for recent content (e.g., "1 week", "2d", "2024-01-01")
metadata_filters: Optional structured frontmatter filters (e.g., {"status": "in-progress"})
tags: Optional tag filter (frontmatter tags); shorthand for metadata_filters["tags"]
@@ -853,6 +870,9 @@ async def search_notes(
# Lowercase note_types so "Chapter" matches the stored "chapter".
note_types = [t.lower() for t in note_types] if note_types else []
entity_types = entity_types or []
# Categories are matched exactly against the indexed observation category,
# so preserve their original casing (unlike the lowercased note_types).
categories = categories or []
# Parse tag:<value> shorthand at tool level so it works with all search modes.
# Handles "tag:security", "tag:coffee tag:brewing", "tag:coffee AND tag:brewing".
@@ -894,6 +914,7 @@ async def search_notes(
output_format=output_format,
note_types=note_types,
entity_types=entity_types,
categories=categories,
after_date=after_date,
metadata_filters=metadata_filters,
tags=tags,
@@ -917,8 +938,15 @@ async def search_notes(
has_query=bool(query and query.strip()),
note_type_filter_count=len(note_types),
entity_type_filter_count=len(entity_types),
category_filter_count=len(categories),
has_filters=bool(
metadata_filters or tags or status or note_types or entity_types or after_date
metadata_filters
or tags
or status
or note_types
or entity_types
or categories
or after_date
),
has_tags_filter=bool(tags),
has_status_filter=bool(status),
@@ -981,6 +1009,8 @@ async def search_notes(
# Add optional filters if provided (empty lists are treated as no filter)
if entity_types:
search_query.entity_types = [SearchItemType(t) for t in entity_types]
if categories:
search_query.categories = categories
if note_types:
search_query.note_types = note_types
if after_date:
@@ -1006,7 +1036,8 @@ async def search_notes(
return (
"# No Search Criteria\n\n"
"Please provide at least one of: `query`, `metadata_filters`, "
"`tags`, `status`, `note_types`, `entity_types`, or `after_date`."
"`tags`, `status`, `note_types`, `entity_types`, `categories`, "
"or `after_date`."
)
# Default to entity-level results to avoid returning individual
@@ -1014,7 +1045,17 @@ async def search_notes(
# Applied after no_criteria() so that the implicit default doesn't
# mask a truly empty search request.
if not search_query.entity_types:
search_query.entity_types = [SearchItemType("entity")]
# Trigger: a category filter was supplied without an explicit
# entity_types.
# Why: categories only exist on observations — defaulting to "entity"
# (whose rows have NULL category) would AND a category filter against
# entity rows and return nothing, defeating a category-only search.
# Outcome: scope the implicit default to observations so
# search_notes(categories=[...]) returns the matching bullets.
if search_query.categories:
search_query.entity_types = [SearchItemType("observation")]
else:
search_query.entity_types = [SearchItemType("entity")]
logger.debug(
f"Search request: project={active_project.name} "
@@ -705,6 +705,7 @@ class PostgresSearchRepository(SearchRepositoryBase):
note_types: Optional[List[str]] = None,
after_date: Optional[datetime] = None,
search_item_types: Optional[List[SearchItemType]] = None,
categories: Optional[List[str]] = None,
metadata_filters: Optional[dict] = None,
) -> tuple[str, str, dict, str, str]:
"""Build Postgres FTS FROM/WHERE params shared by search and count."""
@@ -762,6 +763,20 @@ class PostgresSearchRepository(SearchRepositoryBase):
type_placeholders.append(f":{param_name}")
conditions.append(f"search_index.type IN ({', '.join(type_placeholders)})")
# Handle observation category filter (parameterized for defense-in-depth).
# Trigger: caller passed `categories` to scope observation results.
# Why: `entity_types=["observation"]` only narrows to the observation row type;
# callers expect exact-category matching, not incidental text matches.
# Outcome: only rows whose indexed category exactly equals a requested value
# survive (entities/relations have NULL category and are excluded).
if categories:
category_placeholders = []
for idx, category in enumerate(categories):
param_name = f"category_{idx}"
params[param_name] = category
category_placeholders.append(f":{param_name}")
conditions.append(f"search_index.category IN ({', '.join(category_placeholders)})")
# Handle note type filter using JSONB containment (parameterized)
if note_types:
type_conditions = []
@@ -879,6 +894,7 @@ class PostgresSearchRepository(SearchRepositoryBase):
note_types: Optional[List[str]] = None,
after_date: Optional[datetime] = None,
search_item_types: Optional[List[SearchItemType]] = None,
categories: Optional[List[str]] = None,
metadata_filters: Optional[dict] = None,
retrieval_mode: SearchRetrievalMode = SearchRetrievalMode.FTS,
min_similarity: Optional[float] = None,
@@ -895,6 +911,7 @@ class PostgresSearchRepository(SearchRepositoryBase):
note_types=note_types,
after_date=after_date,
search_item_types=search_item_types,
categories=categories,
metadata_filters=metadata_filters,
retrieval_mode=retrieval_mode,
min_similarity=min_similarity,
@@ -919,6 +936,7 @@ class PostgresSearchRepository(SearchRepositoryBase):
note_types=note_types,
after_date=after_date,
search_item_types=search_item_types,
categories=categories,
metadata_filters=metadata_filters,
)
@@ -1008,6 +1026,7 @@ class PostgresSearchRepository(SearchRepositoryBase):
note_types: Optional[List[str]] = None,
after_date: Optional[datetime] = None,
search_item_types: Optional[List[SearchItemType]] = None,
categories: Optional[List[str]] = None,
metadata_filters: Optional[dict] = None,
retrieval_mode: SearchRetrievalMode = SearchRetrievalMode.FTS,
min_similarity: Optional[float] = None,
@@ -1022,6 +1041,7 @@ class PostgresSearchRepository(SearchRepositoryBase):
note_types=note_types,
after_date=after_date,
search_item_types=search_item_types,
categories=categories,
metadata_filters=metadata_filters,
retrieval_mode=retrieval_mode,
min_similarity=min_similarity,
@@ -1041,6 +1061,7 @@ class PostgresSearchRepository(SearchRepositoryBase):
note_types=note_types,
after_date=after_date,
search_item_types=search_item_types,
categories=categories,
metadata_filters=metadata_filters,
)
sql = f"SELECT COUNT(*) FROM {from_clause} WHERE {where_clause}"
@@ -42,6 +42,7 @@ class SearchRepository(Protocol):
note_types: Optional[List[str]] = None,
after_date: Optional[datetime] = None,
search_item_types: Optional[List[SearchItemType]] = None,
categories: Optional[List[str]] = None,
metadata_filters: Optional[dict] = None,
retrieval_mode: SearchRetrievalMode = SearchRetrievalMode.FTS,
min_similarity: Optional[float] = None,
@@ -60,6 +61,7 @@ class SearchRepository(Protocol):
note_types: Optional[List[str]] = None,
after_date: Optional[datetime] = None,
search_item_types: Optional[List[SearchItemType]] = None,
categories: Optional[List[str]] = None,
metadata_filters: Optional[dict] = None,
retrieval_mode: SearchRetrievalMode = SearchRetrievalMode.FTS,
min_similarity: Optional[float] = None,
@@ -218,6 +218,7 @@ class SearchRepositoryBase(ABC):
note_types: Optional[List[str]] = None,
after_date: Optional[datetime] = None,
search_item_types: Optional[List[SearchItemType]] = None,
categories: Optional[List[str]] = None,
metadata_filters: Optional[Dict[str, Any]] = None,
retrieval_mode: SearchRetrievalMode = SearchRetrievalMode.FTS,
min_similarity: Optional[float] = None,
@@ -234,6 +235,7 @@ class SearchRepositoryBase(ABC):
note_types: Filter by note types (from metadata.note_type)
after_date: Filter by created_at > after_date
search_item_types: Filter by SearchItemType (ENTITY, OBSERVATION, RELATION)
categories: Filter observations by exact category (e.g. "requirement")
metadata_filters: Structured frontmatter metadata filters
limit: Maximum results to return
offset: Number of results to skip
@@ -256,6 +258,7 @@ class SearchRepositoryBase(ABC):
note_types: Optional[List[str]] = None,
after_date: Optional[datetime] = None,
search_item_types: Optional[List[SearchItemType]] = None,
categories: Optional[List[str]] = None,
metadata_filters: Optional[Dict[str, Any]] = None,
retrieval_mode: SearchRetrievalMode = SearchRetrievalMode.FTS,
min_similarity: Optional[float] = None,
@@ -1785,6 +1788,7 @@ class SearchRepositoryBase(ABC):
note_types: Optional[List[str]],
after_date: Optional[datetime],
search_item_types: Optional[List[SearchItemType]],
categories: Optional[List[str]],
metadata_filters: Optional[dict],
retrieval_mode: SearchRetrievalMode,
min_similarity: Optional[float] = None,
@@ -1818,6 +1822,7 @@ class SearchRepositoryBase(ABC):
note_types=note_types,
after_date=after_date,
search_item_types=search_item_types,
categories=categories,
metadata_filters=metadata_filters,
min_similarity=min_similarity,
limit=limit,
@@ -1837,6 +1842,7 @@ class SearchRepositoryBase(ABC):
note_types=note_types,
after_date=after_date,
search_item_types=search_item_types,
categories=categories,
metadata_filters=metadata_filters,
min_similarity=min_similarity,
limit=limit,
@@ -1866,6 +1872,7 @@ class SearchRepositoryBase(ABC):
note_types: Optional[List[str]],
after_date: Optional[datetime],
search_item_types: Optional[List[SearchItemType]],
categories: Optional[List[str]],
metadata_filters: Optional[dict],
min_similarity: Optional[float] = None,
limit: int,
@@ -1976,6 +1983,7 @@ class SearchRepositoryBase(ABC):
note_types,
after_date,
search_item_types,
categories,
metadata_filters,
]
)
@@ -1989,6 +1997,7 @@ class SearchRepositoryBase(ABC):
note_types=note_types,
after_date=after_date,
search_item_types=search_item_types,
categories=categories,
metadata_filters=metadata_filters,
retrieval_mode=SearchRetrievalMode.FTS,
limit=VECTOR_FILTER_SCAN_LIMIT,
@@ -2139,6 +2148,7 @@ class SearchRepositoryBase(ABC):
note_types: Optional[List[str]],
after_date: Optional[datetime],
search_item_types: Optional[List[SearchItemType]],
categories: Optional[List[str]],
metadata_filters: Optional[dict],
min_similarity: Optional[float] = None,
limit: int,
@@ -2163,6 +2173,7 @@ class SearchRepositoryBase(ABC):
note_types=note_types,
after_date=after_date,
search_item_types=search_item_types,
categories=categories,
metadata_filters=metadata_filters,
retrieval_mode=SearchRetrievalMode.FTS,
limit=candidate_limit,
@@ -2178,6 +2189,7 @@ class SearchRepositoryBase(ABC):
note_types=note_types,
after_date=after_date,
search_item_types=search_item_types,
categories=categories,
metadata_filters=metadata_filters,
min_similarity=min_similarity,
limit=candidate_limit,
@@ -733,6 +733,7 @@ class SQLiteSearchRepository(SearchRepositoryBase):
note_types: Optional[List[str]] = None,
after_date: Optional[datetime] = None,
search_item_types: Optional[List[SearchItemType]] = None,
categories: Optional[List[str]] = None,
metadata_filters: Optional[dict] = None,
) -> tuple[str, str, dict, str]:
"""Build SQLite FTS FROM/WHERE params shared by search and count."""
@@ -794,6 +795,20 @@ class SQLiteSearchRepository(SearchRepositoryBase):
type_placeholders.append(f":{param_name}")
conditions.append(f"search_index.type IN ({', '.join(type_placeholders)})")
# Handle observation category filter (parameterized for defense-in-depth).
# Trigger: caller passed `categories` to scope observation results.
# Why: `entity_types=["observation"]` only narrows to the observation row type;
# callers expect exact-category matching, not incidental text matches.
# Outcome: only rows whose indexed category exactly equals a requested value
# survive (entities/relations have NULL category and are excluded).
if categories:
category_placeholders = []
for idx, category in enumerate(categories):
param_name = f"category_{idx}"
params[param_name] = category
category_placeholders.append(f":{param_name}")
conditions.append(f"search_index.category IN ({', '.join(category_placeholders)})")
# Handle note type filter (frontmatter type field, parameterized)
if note_types:
type_placeholders = []
@@ -925,6 +940,7 @@ class SQLiteSearchRepository(SearchRepositoryBase):
note_types: Optional[List[str]] = None,
after_date: Optional[datetime] = None,
search_item_types: Optional[List[SearchItemType]] = None,
categories: Optional[List[str]] = None,
metadata_filters: Optional[dict] = None,
retrieval_mode: SearchRetrievalMode = SearchRetrievalMode.FTS,
min_similarity: Optional[float] = None,
@@ -941,6 +957,7 @@ class SQLiteSearchRepository(SearchRepositoryBase):
note_types=note_types,
after_date=after_date,
search_item_types=search_item_types,
categories=categories,
metadata_filters=metadata_filters,
retrieval_mode=retrieval_mode,
min_similarity=min_similarity,
@@ -959,6 +976,7 @@ class SQLiteSearchRepository(SearchRepositoryBase):
note_types=note_types,
after_date=after_date,
search_item_types=search_item_types,
categories=categories,
metadata_filters=metadata_filters,
)
@@ -1046,6 +1064,7 @@ class SQLiteSearchRepository(SearchRepositoryBase):
note_types: Optional[List[str]] = None,
after_date: Optional[datetime] = None,
search_item_types: Optional[List[SearchItemType]] = None,
categories: Optional[List[str]] = None,
metadata_filters: Optional[dict] = None,
retrieval_mode: SearchRetrievalMode = SearchRetrievalMode.FTS,
min_similarity: Optional[float] = None,
@@ -1060,6 +1079,7 @@ class SQLiteSearchRepository(SearchRepositoryBase):
note_types=note_types,
after_date=after_date,
search_item_types=search_item_types,
categories=categories,
metadata_filters=metadata_filters,
retrieval_mode=retrieval_mode,
min_similarity=min_similarity,
@@ -1073,6 +1093,7 @@ class SQLiteSearchRepository(SearchRepositoryBase):
note_types=note_types,
after_date=after_date,
search_item_types=search_item_types,
categories=categories,
metadata_filters=metadata_filters,
)
sql = f"SELECT COUNT(*) FROM {from_clause} WHERE {where_clause}"
+4
View File
@@ -42,6 +42,7 @@ class SearchQuery(BaseModel):
Optionally filter results by:
- note_types: Limit to specific note types (frontmatter "type")
- entity_types: Limit to search item types (entity/observation/relation)
- categories: Limit observation results to exact category matches (e.g. "requirement")
- after_date: Only items after date
- metadata_filters: Structured frontmatter filters (field -> value)
- tags: Convenience frontmatter tag filter
@@ -63,6 +64,7 @@ class SearchQuery(BaseModel):
# Optional filters
note_types: Optional[List[str]] = None # Filter by note type (frontmatter "type")
entity_types: Optional[List[SearchItemType]] = None # Filter by entity type
categories: Optional[List[str]] = None # Filter observations by exact category
after_date: Optional[Union[datetime, str]] = None # Time-based filter
metadata_filters: Optional[dict[str, Any]] = None # Structured frontmatter filters
tags: Optional[List[str]] = None # Convenience tag filter
@@ -85,6 +87,7 @@ class SearchQuery(BaseModel):
status_is_empty = self.status is None or (isinstance(self.status, str) and not self.status)
note_types_is_empty = not self.note_types
entity_types_is_empty = not self.entity_types
categories_is_empty = not self.categories
return (
self.permalink is None
and self.permalink_match is None
@@ -93,6 +96,7 @@ class SearchQuery(BaseModel):
and self.after_date is None
and note_types_is_empty
and entity_types_is_empty
and categories_is_empty
and metadata_is_empty
and tags_is_empty
and status_is_empty
@@ -75,6 +75,7 @@ class _PreparedSearchQuery:
title: str | None
note_types: list[str] | None
search_item_types: list[SearchItemType] | None
categories: list[str] | None
after_date: datetime | None
metadata_filters: dict[str, Any] | None
retrieval_mode: SearchRetrievalMode
@@ -194,6 +195,7 @@ class SearchService:
title=query.title,
note_types=query.note_types,
search_item_types=query.entity_types,
categories=query.categories,
after_date=after_date,
metadata_filters=metadata_filters,
retrieval_mode=query.retrieval_mode or SearchRetrievalMode.FTS,
@@ -207,6 +209,7 @@ class SearchService:
or prepared.title
or prepared.note_types
or prepared.search_item_types
or prepared.categories
or prepared.after_date
or prepared.metadata_filters
)
@@ -221,6 +224,7 @@ class SearchService:
prepared.metadata_filters
or prepared.note_types
or prepared.search_item_types
or prepared.categories
or prepared.after_date
)
@@ -239,6 +243,7 @@ class SearchService:
title=prepared.title,
note_types=prepared.note_types,
search_item_types=prepared.search_item_types,
categories=prepared.categories,
after_date=prepared.after_date,
metadata_filters=prepared.metadata_filters,
retrieval_mode=prepared.retrieval_mode,
@@ -260,6 +265,7 @@ class SearchService:
title=prepared.title,
note_types=prepared.note_types,
search_item_types=prepared.search_item_types,
categories=prepared.categories,
after_date=prepared.after_date,
metadata_filters=prepared.metadata_filters,
retrieval_mode=prepared.retrieval_mode,
+1
View File
@@ -91,6 +91,7 @@ EXPECTED_TOOL_SIGNATURES: dict[str, list[str]] = {
"output_format",
"note_types",
"entity_types",
"categories",
"after_date",
"metadata_filters",
"tags",
+87
View File
@@ -370,6 +370,93 @@ async def test_search_with_entity_type_filter(client, test_project):
pytest.fail(f"Search failed with error: {response}")
@pytest.mark.asyncio
async def test_search_with_categories_filter(client, test_project):
"""Observation category filter returns only the exact category (#430).
Writes a note whose body has a [requirement] observation and a [decision]
observation that also mentions the word "requirement". The categories filter
must return only the requirement observation.
"""
await write_note(
project=test_project.name,
title="Category Filter Note",
directory="test",
content=(
"# Category Filter Note\n"
"- [requirement] The system must enforce auth on every request\n"
"- [decision] We deferred the auth requirement to next sprint\n"
),
)
response = await search_notes(
project=test_project.name,
query="requirement",
search_type="text",
entity_types=["observation"],
categories=["requirement"],
output_format="json",
)
assert isinstance(response, dict), f"Search failed with error: {response}"
results = response["results"]
assert len(results) > 0
# Every result is a requirement observation; the [decision] row is excluded
# even though its text contains the word "requirement".
assert all(r["type"] == "observation" for r in results)
assert all(r["category"] == "requirement" for r in results)
# A non-matching category yields no results for the same text query.
decision = await search_notes(
project=test_project.name,
query="requirement",
search_type="text",
entity_types=["observation"],
categories=["decision"],
output_format="json",
)
assert isinstance(decision, dict), f"Search failed with error: {decision}"
assert all(r["category"] == "decision" for r in decision["results"])
# The requirement observation must not leak into a decision-scoped search.
assert all("requirement" != r.get("category") for r in decision["results"])
@pytest.mark.asyncio
async def test_search_categories_without_entity_types_returns_observations(client, test_project):
"""categories=[...] WITHOUT entity_types must return the matching observations (#908).
search_notes defaults entity_types to "entity" when unset, but categories only exist on
observations — so a category filter without an explicit entity_types would AND the
category against entity rows (which have NULL category) and return nothing. The implicit
default must scope to observations when categories is supplied.
"""
await write_note(
project=test_project.name,
title="Category Default Note",
directory="test",
content=(
"# Category Default Note\n"
"- [requirement] Auth tokens must rotate every 24 hours\n"
"- [decision] We chose JWT for the auth token format\n"
),
)
# Note: no entity_types passed — exercises the implicit default.
response = await search_notes(
project=test_project.name,
query="auth",
search_type="text",
categories=["requirement"],
output_format="json",
)
assert isinstance(response, dict), f"Search failed with error: {response}"
results = response["results"]
assert len(results) > 0, "category-only search must return matching observations"
assert all(r["type"] == "observation" for r in results)
assert all(r["category"] == "requirement" for r in results)
@pytest.mark.asyncio
async def test_search_with_date_filter(client, test_project):
"""Test search with date filter."""
+1
View File
@@ -165,6 +165,7 @@ async def test_search_notes_emits_root_operation_and_project_context(
"has_query": True,
"note_type_filter_count": 0,
"entity_type_filter_count": 0,
"category_filter_count": 0,
"has_filters": True,
"has_tags_filter": True,
"has_status_filter": False,
+2
View File
@@ -71,6 +71,7 @@ class ConcreteSearchRepo(SearchRepositoryBase):
note_types: Optional[list[str]] = None,
after_date: Optional[datetime] = None,
search_item_types: Optional[list[SearchItemType]] = None,
categories: Optional[list[str]] = None,
metadata_filters: Optional[dict[str, Any]] = None,
retrieval_mode: SearchRetrievalMode = SearchRetrievalMode.FTS,
min_similarity: Optional[float] = None,
@@ -126,6 +127,7 @@ HYBRID_KWARGS: dict[str, Any] = dict(
note_types=None,
after_date=None,
search_item_types=None,
categories=None,
metadata_filters=None,
limit=10,
offset=0,
@@ -932,3 +932,72 @@ async def test_postgres_metadata_filters_path_parameterized(session_maker, test_
# Nested path should work without SQL injection risk
results = await repo.search(metadata_filters={"schema.confidence": {"$gt": 0.5}})
assert isinstance(results, list)
@pytest.mark.asyncio
async def test_postgres_search_categories_exact_match(session_maker, test_project):
"""categories filter matches the observation category exactly (mirror of #430).
A [decision] observation that merely mentions "requirement" must be excluded
when categories=["requirement"] is requested.
"""
repo = PostgresSearchRepository(session_maker, project_id=test_project.id)
now = datetime.now(timezone.utc)
await repo.bulk_index_items(
[
SearchIndexRow(
project_id=test_project.id,
id=70101,
type=SearchItemType.OBSERVATION.value,
content_stems="the auth requirement must be enforced on every call",
content_snippet="the auth requirement must be enforced on every call",
permalink="test/obs/requirement/70101",
file_path="test/obs.md",
entity_id=1,
category="requirement",
metadata={"note_type": "note"},
created_at=now,
updated_at=now,
),
SearchIndexRow(
project_id=test_project.id,
id=70102,
type=SearchItemType.OBSERVATION.value,
content_stems="we deferred the auth requirement to next sprint",
content_snippet="we deferred the auth requirement to next sprint",
permalink="test/obs/decision/70102",
file_path="test/obs.md",
entity_id=1,
category="decision",
metadata={"note_type": "note"},
created_at=now,
updated_at=now,
),
]
)
# Without the category filter, a text search for "requirement" matches both.
text_results = await repo.search(
search_text="requirement",
search_item_types=[SearchItemType.OBSERVATION],
)
assert {r.id for r in text_results} == {70101, 70102}
# With categories=["requirement"], only the requirement observation survives.
filtered = await repo.search(
search_text="requirement",
search_item_types=[SearchItemType.OBSERVATION],
categories=["requirement"],
)
assert {r.id for r in filtered} == {70101}
assert filtered[0].category == "requirement"
# Standalone filter and count both honor the exact category.
filtered_only = await repo.search(categories=["requirement"])
assert {r.id for r in filtered_only} == {70101}
assert await repo.count(categories=["requirement"]) == 1
# Multiple categories union.
multi = await repo.search(categories=["requirement", "decision"])
assert {r.id for r in multi} == {70101, 70102}
@@ -1044,3 +1044,83 @@ async def test_search_item_types_parameterized(search_repository):
results = await search_repository.search(search_item_types=[SearchItemType.ENTITY])
# Should not raise — parameterized query handles enum values safely
assert isinstance(results, list)
async def _index_observation(
search_repository,
*,
row_id: int,
entity_id: int,
category: str,
content: str,
) -> None:
"""Index a single observation row with an explicit category for filter tests."""
now = datetime.now(timezone.utc)
search_row = SearchIndexRow(
id=row_id,
type=SearchItemType.OBSERVATION.value,
content_stems=content,
content_snippet=content,
permalink=f"test/obs/{category}/{row_id}",
file_path="test/obs.md",
entity_id=entity_id,
category=category,
metadata={"note_type": "note"},
created_at=now,
updated_at=now,
project_id=search_repository.project_id,
)
await search_repository.index_item(search_row)
@pytest.mark.asyncio
async def test_search_categories_exact_match(search_repository, search_entity):
"""categories must match the observation category exactly, not by text.
Regression for #430: searching observations for "requirement" used to also
return a [decision] observation that merely mentions the word. The categories
filter scopes results to the exact indexed category.
"""
await _index_observation(
search_repository,
row_id=70001,
entity_id=search_entity.id,
category="requirement",
content="The auth requirement must be enforced on every call",
)
# A decision observation whose text mentions "requirement" but whose category
# is NOT requirement — it must be excluded by an exact-category filter.
await _index_observation(
search_repository,
row_id=70002,
entity_id=search_entity.id,
category="decision",
content="We deferred the auth requirement to next sprint",
)
# Without the category filter, a text search for "requirement" matches both.
text_results = await search_repository.search(
search_text="requirement",
search_item_types=[SearchItemType.OBSERVATION],
)
assert {r.id for r in text_results} == {70001, 70002}
# With categories=["requirement"], only the requirement observation survives.
filtered = await search_repository.search(
search_text="requirement",
search_item_types=[SearchItemType.OBSERVATION],
categories=["requirement"],
)
assert {r.id for r in filtered} == {70001}
assert filtered[0].category == "requirement"
# categories also works as a standalone filter (no text query).
filtered_only = await search_repository.search(categories=["requirement"])
assert {r.id for r in filtered_only} == {70001}
# count mirrors the filtered search.
assert await search_repository.count(categories=["requirement"]) == 1
# Multiple categories union: both observations come back.
multi = await search_repository.search(categories=["requirement", "decision"])
assert {r.id for r in multi} == {70001, 70002}
@@ -83,6 +83,7 @@ class _ConcreteRepo(SearchRepositoryBase):
note_types: list[str] | None = None,
after_date: datetime | None = None,
search_item_types: list[SearchItemType] | None = None,
categories: list[str] | None = None,
metadata_filters: dict[str, Any] | None = None,
retrieval_mode: SearchRetrievalMode = SearchRetrievalMode.FTS,
min_similarity: float | None = None,
@@ -56,6 +56,7 @@ class ConcreteSearchRepo(SearchRepositoryBase):
note_types: list[str] | None = None,
after_date: datetime | None = None,
search_item_types: list[SearchItemType] | None = None,
categories: list[str] | None = None,
metadata_filters: dict[str, Any] | None = None,
retrieval_mode: SearchRetrievalMode = SearchRetrievalMode.FTS,
min_similarity: float | None = None,
@@ -158,6 +159,7 @@ async def test_page1_scores_gte_page2_scores():
note_types=None,
after_date=None,
search_item_types=None,
categories=None,
metadata_filters=None,
limit=limit,
offset=offset,
@@ -60,6 +60,7 @@ class ConcreteSearchRepo(SearchRepositoryBase):
note_types: Optional[list[str]] = None,
after_date: Optional[datetime] = None,
search_item_types: Optional[list[SearchItemType]] = None,
categories: Optional[list[str]] = None,
metadata_filters: Optional[dict[str, Any]] = None,
retrieval_mode: SearchRetrievalMode = SearchRetrievalMode.FTS,
min_similarity: Optional[float] = None,
@@ -130,6 +131,7 @@ COMMON_SEARCH_KWARGS: dict[str, Any] = dict(
note_types=None,
after_date=None,
search_item_types=None,
categories=None,
metadata_filters=None,
limit=10,
offset=0,
+33
View File
@@ -277,6 +277,39 @@ async def test_search_entity_type(search_service, test_graph):
assert r.type == SearchItemType.ENTITY
@pytest.mark.asyncio
async def test_search_categories_filter(search_service, test_graph):
"""categories propagates through _prepare_query/has_criteria to scope results.
The test_graph fixture indexes observations with categories "note" and "tech".
A categories filter must return only matching observation categories.
"""
# categories alone is enough criteria to run a query (has_criteria True).
note_results = await search_service.search(SearchQuery(categories=["note"]))
assert len(note_results) > 0
assert all(r.type == SearchItemType.OBSERVATION for r in note_results)
assert all(r.category == "note" for r in note_results)
# A different category yields a disjoint, non-empty result set.
tech_results = await search_service.search(SearchQuery(categories=["tech"]))
assert len(tech_results) > 0
assert all(r.category == "tech" for r in tech_results)
note_ids = {r.id for r in note_results}
tech_ids = {r.id for r in tech_results}
assert note_ids.isdisjoint(tech_ids)
# count() must agree with the filtered search via the same prepared query.
assert await search_service.count(SearchQuery(categories=["note"])) == len(note_results)
@pytest.mark.asyncio
async def test_search_categories_only_is_not_no_criteria():
"""A SearchQuery carrying only categories must not be treated as empty."""
assert SearchQuery(categories=["requirement"]).no_criteria() is False
assert SearchQuery().no_criteria() is True
@pytest.mark.asyncio
async def test_extract_entity_tags_exception_handling(search_service):
"""Test the _extract_entity_tags method exception handling (lines 147-151)."""