feat: fast edit entities, refactors for webui, enhance search (#532)

Signed-off-by: phernandez <paul@basicmachines.co>

src/basic_memory/repository/metadata_filters.py

@@ -0,0 +1,131 @@
+"""Helpers for parsing structured metadata filters for search."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from datetime import date, datetime
+import re
+from typing import Any, Iterable, List
+
+
+_KEY_RE = re.compile(r"^[A-Za-z0-9_-]+(\.[A-Za-z0-9_-]+)*$")
+_NUMERIC_RE = re.compile(r"^-?\d+(\.\d+)?$")
+
+
+@dataclass(frozen=True)
+class ParsedMetadataFilter:
+    """Normalized metadata filter for SQL generation."""
+
+    path_parts: List[str]
+    op: str
+    value: Any
+    comparison: str | None = None  # "numeric" or "text" for comparisons
+
+
+def _is_numeric_value(value: Any) -> bool:
+    if isinstance(value, bool):
+        return False
+    if isinstance(value, (int, float)):
+        return True
+    if isinstance(value, str):
+        return bool(_NUMERIC_RE.match(value.strip()))
+    return False
+
+
+def _is_numeric_collection(values: Iterable[Any]) -> bool:
+    return all(_is_numeric_value(v) for v in values)
+
+
+def _normalize_scalar(value: Any) -> Any:
+    if isinstance(value, datetime):
+        return value.isoformat()
+    if isinstance(value, date):
+        return value.isoformat()
+    if isinstance(value, bool):
+        return str(value)
+    if isinstance(value, (int, float)):
+        return str(value)
+    return value
+
+
+def parse_metadata_filters(filters: dict[str, Any]) -> List[ParsedMetadataFilter]:
+    """Parse metadata filters into normalized clauses.
+
+    Supported forms:
+    - {"status": "in-progress"}
+    - {"tags": ["security", "oauth"]}  # array contains all
+    - {"priority": {"$in": ["high", "critical"]}}
+    - {"schema.confidence": {"$gt": 0.7}}
+    - {"schema.confidence": {"$between": [0.3, 0.6]}}
+    """
+    parsed: List[ParsedMetadataFilter] = []
+
+    for raw_key, raw_value in (filters or {}).items():
+        if not isinstance(raw_key, str) or not raw_key.strip():
+            raise ValueError("metadata filter keys must be non-empty strings")
+        key = raw_key.strip()
+        if not _KEY_RE.match(key):
+            raise ValueError(f"Unsupported metadata filter key: {raw_key}")
+
+        path_parts = key.split(".")
+
+        # Operator form
+        if isinstance(raw_value, dict):
+            if len(raw_value) != 1:
+                raise ValueError(f"Invalid metadata filter for '{raw_key}': {raw_value}")
+            op, value = next(iter(raw_value.items()))
+
+            if op == "$in":
+                if not isinstance(value, list) or not value:
+                    raise ValueError(f"$in requires a non-empty list for '{raw_key}'")
+                parsed.append(
+                    ParsedMetadataFilter(path_parts, "in", [_normalize_scalar(v) for v in value])
+                )
+                continue
+
+            if op in {"$gt", "$gte", "$lt", "$lte"}:
+                normalized = _normalize_scalar(value)
+                comparison = "numeric" if _is_numeric_value(normalized) else "text"
+                parsed.append(
+                    ParsedMetadataFilter(path_parts, op.lstrip("$"), normalized, comparison)
+                )
+                continue
+
+            if op == "$between":
+                if not isinstance(value, list) or len(value) != 2:
+                    raise ValueError(f"$between requires [min, max] for '{raw_key}'")
+                normalized = [_normalize_scalar(v) for v in value]
+                comparison = "numeric" if _is_numeric_collection(normalized) else "text"
+                parsed.append(ParsedMetadataFilter(path_parts, "between", normalized, comparison))
+                continue
+
+            raise ValueError(f"Unsupported operator '{op}' in metadata filter for '{raw_key}'")
+
+        # Array contains (all)
+        if isinstance(raw_value, list):
+            if not raw_value:
+                raise ValueError(f"Empty list not allowed for metadata filter '{raw_key}'")
+            parsed.append(
+                ParsedMetadataFilter(
+                    path_parts, "contains", [_normalize_scalar(v) for v in raw_value]
+                )
+            )
+            continue
+
+        # Simple equality
+        parsed.append(ParsedMetadataFilter(path_parts, "eq", _normalize_scalar(raw_value)))
+
+    return parsed
+
+
+def build_sqlite_json_path(parts: List[str]) -> str:
+    """Build a SQLite JSON path for json_extract/json_each."""
+    path = "$"
+    for part in parts:
+        path += f'."{part}"'
+    return path
+
+
+def build_postgres_json_path(parts: List[str]) -> str:
+    """Build a Postgres JSON path for #>>/#> operators."""
+    return "{" + ",".join(parts) + "}"

src/basic_memory/repository/postgres_search_repository.py

@@ -12,6 +12,10 @@ from sqlalchemy import text
 from basic_memory import db
 from basic_memory.repository.search_index_row import SearchIndexRow
 from basic_memory.repository.search_repository_base import SearchRepositoryBase
+from basic_memory.repository.metadata_filters import (
+    parse_metadata_filters,
+    build_postgres_json_path,
+)
 from basic_memory.schemas.search import SearchItemType
 
 
@@ -215,6 +219,7 @@ class PostgresSearchRepository(SearchRepositoryBase):
         types: Optional[List[str]] = None,
         after_date: Optional[datetime] = None,
         search_item_types: Optional[List[SearchItemType]] = None,
+        metadata_filters: Optional[dict] = None,
         limit: int = 10,
         offset: int = 0,
     ) -> List[SearchIndexRow]:
@@ -222,6 +227,7 @@ class PostgresSearchRepository(SearchRepositoryBase):
         conditions = []
         params = {}
         order_by_clause = ""
+        from_clause = "search_index"
 
         # Handle text search for title and content using tsvector
         if search_text:
@@ -233,18 +239,22 @@ class PostgresSearchRepository(SearchRepositoryBase):
                 processed_text = self._prepare_search_term(search_text.strip())
                 params["text"] = processed_text
                 # Use @@ operator for tsvector matching
-                conditions.append("textsearchable_index_col @@ to_tsquery('english', :text)")
+                conditions.append(
+                    "search_index.textsearchable_index_col @@ to_tsquery('english', :text)"
+                )
 
         # Handle title search
         if title:
             title_text = self._prepare_search_term(title.strip(), is_prefix=False)
             params["title_text"] = title_text
-            conditions.append("to_tsvector('english', title) @@ to_tsquery('english', :title_text)")
+            conditions.append(
+                "to_tsvector('english', search_index.title) @@ to_tsquery('english', :title_text)"
+            )
 
         # Handle permalink exact search
         if permalink:
             params["permalink"] = permalink
-            conditions.append("permalink = :permalink")
+            conditions.append("search_index.permalink = :permalink")
 
         # Handle permalink pattern match
         if permalink_match:
@@ -255,14 +265,14 @@ class PostgresSearchRepository(SearchRepositoryBase):
                 # Convert * to % for SQL LIKE
                 permalink_pattern = permalink_text.replace("*", "%")
                 params["permalink"] = permalink_pattern
-                conditions.append("permalink LIKE :permalink")
+                conditions.append("search_index.permalink LIKE :permalink")
             else:
-                conditions.append("permalink = :permalink")
+                conditions.append("search_index.permalink = :permalink")
 
         # Handle search item type filter
         if search_item_types:
             type_list = ", ".join(f"'{t.value}'" for t in search_item_types)
-            conditions.append(f"type IN ({type_list})")
+            conditions.append(f"search_index.type IN ({type_list})")
 
         # Handle entity type filter using JSONB containment
         if types:
@@ -270,19 +280,91 @@ class PostgresSearchRepository(SearchRepositoryBase):
             type_conditions = []
             for entity_type in types:
                 # Create JSONB containment condition for each type
-                type_conditions.append(f'metadata @> \'{{"entity_type": "{entity_type}"}}\'')
+                type_conditions.append(
+                    f'search_index.metadata @> \'{{"entity_type": "{entity_type}"}}\''
+                )
             conditions.append(f"({' OR '.join(type_conditions)})")
 
         # Handle date filter
         if after_date:
             params["after_date"] = after_date
-            conditions.append("created_at > :after_date")
+            conditions.append("search_index.created_at > :after_date")
             # order by most recent first
-            order_by_clause = ", updated_at DESC"
+            order_by_clause = ", search_index.updated_at DESC"
+
+        # Handle structured metadata filters (frontmatter)
+        if metadata_filters:
+            parsed_filters = parse_metadata_filters(metadata_filters)
+            from_clause = "search_index JOIN entity ON search_index.entity_id = entity.id"
+            metadata_expr = "entity.entity_metadata::jsonb"
+
+            for idx, filt in enumerate(parsed_filters):
+                path = build_postgres_json_path(filt.path_parts)
+                text_expr = f"({metadata_expr} #>> '{path}')"
+                json_expr = f"({metadata_expr} #> '{path}')"
+
+                if filt.op == "eq":
+                    value_param = f"meta_val_{idx}"
+                    params[value_param] = filt.value
+                    conditions.append(f"{text_expr} = :{value_param}")
+                    continue
+
+                if filt.op == "in":
+                    placeholders = []
+                    for j, val in enumerate(filt.value):
+                        value_param = f"meta_val_{idx}_{j}"
+                        params[value_param] = val
+                        placeholders.append(f":{value_param}")
+                    conditions.append(f"{text_expr} IN ({', '.join(placeholders)})")
+                    continue
+
+                if filt.op == "contains":
+                    import json as _json
+
+                    base_param = f"meta_val_{idx}"
+                    tag_conditions = []
+                    # Require all values to be present
+                    for j, val in enumerate(filt.value):
+                        tag_param = f"{base_param}_{j}"
+                        params[tag_param] = _json.dumps([val])
+                        like_param = f"{base_param}_{j}_like"
+                        params[like_param] = f'%"{val}"%'
+                        like_param_single = f"{base_param}_{j}_like_single"
+                        params[like_param_single] = f"%'{val}'%"
+                        tag_conditions.append(
+                            f"({json_expr} @> :{tag_param}::jsonb "
+                            f"OR {text_expr} LIKE :{like_param} "
+                            f"OR {text_expr} LIKE :{like_param_single})"
+                        )
+                    conditions.append(" AND ".join(tag_conditions))
+                    continue
+
+                if filt.op in {"gt", "gte", "lt", "lte", "between"}:
+                    if filt.comparison == "numeric":
+                        numeric_expr = (
+                            f"CASE WHEN ({text_expr}) ~ '^-?\\\\d+(\\\\.\\\\d+)?$' "
+                            f"THEN ({text_expr})::double precision END"
+                        )
+                        compare_expr = numeric_expr
+                    else:
+                        compare_expr = text_expr
+
+                    if filt.op == "between":
+                        min_param = f"meta_val_{idx}_min"
+                        max_param = f"meta_val_{idx}_max"
+                        params[min_param] = filt.value[0]
+                        params[max_param] = filt.value[1]
+                        conditions.append(f"{compare_expr} BETWEEN :{min_param} AND :{max_param}")
+                    else:
+                        value_param = f"meta_val_{idx}"
+                        params[value_param] = filt.value
+                        operator = {"gt": ">", "gte": ">=", "lt": "<", "lte": "<="}[filt.op]
+                        conditions.append(f"{compare_expr} {operator} :{value_param}")
+                    continue
 
         # Always filter by project_id
         params["project_id"] = self.project_id
-        conditions.append("project_id = :project_id")
+        conditions.append("search_index.project_id = :project_id")
 
         # set limit and offset
         params["limit"] = limit
@@ -294,31 +376,33 @@ class PostgresSearchRepository(SearchRepositoryBase):
         # Build SQL with ts_rank() for scoring
         # Note: If no text search, score will be NULL, so we use COALESCE to default to 0
         if search_text and search_text.strip() and search_text.strip() != "*":
-            score_expr = "ts_rank(textsearchable_index_col, to_tsquery('english', :text))"
+            score_expr = (
+                "ts_rank(search_index.textsearchable_index_col, to_tsquery('english', :text))"
+            )
         else:
             score_expr = "0"
 
         sql = f"""
             SELECT
-                project_id,
-                id,
-                title,
-                permalink,
-                file_path,
-                type,
-                metadata,
-                from_id,
-                to_id,
-                relation_type,
-                entity_id,
-                content_snippet,
-                category,
-                created_at,
-                updated_at,
+                search_index.project_id,
+                search_index.id,
+                search_index.title,
+                search_index.permalink,
+                search_index.file_path,
+                search_index.type,
+                search_index.metadata,
+                search_index.from_id,
+                search_index.to_id,
+                search_index.relation_type,
+                search_index.entity_id,
+                search_index.content_snippet,
+                search_index.category,
+                search_index.created_at,
+                search_index.updated_at,
                 {score_expr} as score
-            FROM search_index
+            FROM {from_clause}
             WHERE {where_clause}
-            ORDER BY score DESC, id ASC {order_by_clause}
+            ORDER BY score DESC, search_index.id ASC {order_by_clause}
             LIMIT :limit
             OFFSET :offset
         """

src/basic_memory/repository/search_repository.py

@@ -40,6 +40,7 @@ class SearchRepository(Protocol):
         types: Optional[List[str]] = None,
         after_date: Optional[datetime] = None,
         search_item_types: Optional[List[SearchItemType]] = None,
+        metadata_filters: Optional[dict] = None,
         limit: int = 10,
         offset: int = 0,
     ) -> List[SearchIndexRow]:

src/basic_memory/repository/search_repository_base.py

@@ -78,6 +78,7 @@ class SearchRepositoryBase(ABC):
         types: Optional[List[str]] = None,
         after_date: Optional[datetime] = None,
         search_item_types: Optional[List[SearchItemType]] = None,
+        metadata_filters: Optional[Dict[str, Any]] = None,
         limit: int = 10,
         offset: int = 0,
     ) -> List[SearchIndexRow]:
@@ -91,6 +92,7 @@ class SearchRepositoryBase(ABC):
             types: Filter by entity types (from metadata.entity_type)
             after_date: Filter by created_at > after_date
             search_item_types: Filter by SearchItemType (ENTITY, OBSERVATION, RELATION)
+            metadata_filters: Structured frontmatter metadata filters
             limit: Maximum results to return
             offset: Number of results to skip
 

src/basic_memory/repository/sqlite_search_repository.py

@@ -13,6 +13,7 @@ from basic_memory import db
 from basic_memory.models.search import CREATE_SEARCH_INDEX
 from basic_memory.repository.search_index_row import SearchIndexRow
 from basic_memory.repository.search_repository_base import SearchRepositoryBase
+from basic_memory.repository.metadata_filters import parse_metadata_filters, build_sqlite_json_path
 from basic_memory.schemas.search import SearchItemType
 
 
@@ -26,6 +27,17 @@ class SQLiteSearchRepository(SearchRepositoryBase):
     - Prefix wildcard matching with *
     """
 
+    def __init__(self, session_maker, project_id: int):
+        super().__init__(session_maker, project_id)
+        self._entity_columns: set[str] | None = None
+
+    async def _get_entity_columns(self) -> set[str]:
+        if self._entity_columns is None:
+            async with db.scoped_session(self.session_maker) as session:
+                result = await session.execute(text("PRAGMA table_info(entity)"))
+                self._entity_columns = {row[1] for row in result.fetchall()}
+        return self._entity_columns
+
     async def init_search_index(self):
         """Create FTS5 virtual table for search if it doesn't exist.
 
@@ -287,6 +299,7 @@ class SQLiteSearchRepository(SearchRepositoryBase):
         types: Optional[List[str]] = None,
         after_date: Optional[datetime] = None,
         search_item_types: Optional[List[SearchItemType]] = None,
+        metadata_filters: Optional[dict] = None,
         limit: int = 10,
         offset: int = 0,
     ) -> List[SearchIndexRow]:
@@ -294,6 +307,7 @@ class SQLiteSearchRepository(SearchRepositoryBase):
         conditions = []
         params = {}
         order_by_clause = ""
+        from_clause = "search_index"
 
         # Handle text search for title and content
         if search_text:
@@ -305,18 +319,20 @@ class SQLiteSearchRepository(SearchRepositoryBase):
                 # Use _prepare_search_term to handle both Boolean and non-Boolean queries
                 processed_text = self._prepare_search_term(search_text.strip())
                 params["text"] = processed_text
-                conditions.append("(title MATCH :text OR content_stems MATCH :text)")
+                conditions.append(
+                    "(search_index.title MATCH :text OR search_index.content_stems MATCH :text)"
+                )
 
         # Handle title match search
         if title:
             title_text = self._prepare_search_term(title.strip(), is_prefix=False)
             params["title_text"] = title_text
-            conditions.append("title MATCH :title_text")
+            conditions.append("search_index.title MATCH :title_text")
 
         # Handle permalink exact search
         if permalink:
             params["permalink"] = permalink
-            conditions.append("permalink = :permalink")
+            conditions.append("search_index.permalink = :permalink")
 
         # Handle permalink match search, supports *
         if permalink_match:
@@ -325,38 +341,122 @@ class SQLiteSearchRepository(SearchRepositoryBase):
             permalink_text = permalink_match.lower().strip()
             params["permalink"] = permalink_text
             if "*" in permalink_match:
-                conditions.append("permalink GLOB :permalink")
+                conditions.append("search_index.permalink GLOB :permalink")
             else:
                 # For exact matches without *, we can use FTS5 MATCH
                 # but only prepare the term if it doesn't look like a path
                 if "/" in permalink_text:
-                    conditions.append("permalink = :permalink")
+                    conditions.append("search_index.permalink = :permalink")
                 else:
                     permalink_text = self._prepare_search_term(permalink_text, is_prefix=False)
                     params["permalink"] = permalink_text
-                    conditions.append("permalink MATCH :permalink")
+                    conditions.append("search_index.permalink MATCH :permalink")
 
         # Handle entity type filter
         if search_item_types:
             type_list = ", ".join(f"'{t.value}'" for t in search_item_types)
-            conditions.append(f"type IN ({type_list})")
+            conditions.append(f"search_index.type IN ({type_list})")
 
         # Handle type filter
         if types:
             type_list = ", ".join(f"'{t}'" for t in types)
-            conditions.append(f"json_extract(metadata, '$.entity_type') IN ({type_list})")
+            conditions.append(
+                f"json_extract(search_index.metadata, '$.entity_type') IN ({type_list})"
+            )
 
         # Handle date filter using datetime() for proper comparison
         if after_date:
             params["after_date"] = after_date
-            conditions.append("datetime(created_at) > datetime(:after_date)")
+            conditions.append("datetime(search_index.created_at) > datetime(:after_date)")
 
             # order by most recent first
-            order_by_clause = ", updated_at DESC"
+            order_by_clause = ", search_index.updated_at DESC"
+
+        # Handle structured metadata filters (frontmatter)
+        if metadata_filters:
+            parsed_filters = parse_metadata_filters(metadata_filters)
+            from_clause = "search_index JOIN entity ON search_index.entity_id = entity.id"
+            entity_columns = await self._get_entity_columns()
+
+            for idx, filt in enumerate(parsed_filters):
+                path_param = f"meta_path_{idx}"
+                extract_expr = None
+                use_tags_column = False
+
+                if filt.path_parts == ["status"] and "frontmatter_status" in entity_columns:
+                    extract_expr = "entity.frontmatter_status"
+                elif filt.path_parts == ["type"] and "frontmatter_type" in entity_columns:
+                    extract_expr = "entity.frontmatter_type"
+                elif filt.path_parts == ["tags"] and "tags_json" in entity_columns:
+                    extract_expr = "entity.tags_json"
+                    use_tags_column = True
+
+                if extract_expr is None:
+                    params[path_param] = build_sqlite_json_path(filt.path_parts)
+                    extract_expr = f"json_extract(entity.entity_metadata, :{path_param})"
+
+                if filt.op == "eq":
+                    value_param = f"meta_val_{idx}"
+                    params[value_param] = filt.value
+                    conditions.append(f"{extract_expr} = :{value_param}")
+                    continue
+
+                if filt.op == "in":
+                    placeholders = []
+                    for j, val in enumerate(filt.value):
+                        value_param = f"meta_val_{idx}_{j}"
+                        params[value_param] = val
+                        placeholders.append(f":{value_param}")
+                    conditions.append(f"{extract_expr} IN ({', '.join(placeholders)})")
+                    continue
+
+                if filt.op == "contains":
+                    tag_conditions = []
+                    for j, val in enumerate(filt.value):
+                        value_param = f"meta_val_{idx}_{j}"
+                        params[value_param] = val
+                        like_param = f"{value_param}_like"
+                        params[like_param] = f'%"{val}"%'
+                        like_param_single = f"{value_param}_like_single"
+                        params[like_param_single] = f"%'{val}'%"
+                        json_each_expr = (
+                            "json_each(entity.tags_json)"
+                            if use_tags_column
+                            else f"json_each(entity.entity_metadata, :{path_param})"
+                        )
+                        tag_conditions.append(
+                            "("
+                            f"EXISTS (SELECT 1 FROM {json_each_expr} WHERE value = :{value_param}) "
+                            f"OR {extract_expr} LIKE :{like_param} "
+                            f"OR {extract_expr} LIKE :{like_param_single}"
+                            ")"
+                        )
+                    conditions.append(" AND ".join(tag_conditions))
+                    continue
+
+                if filt.op in {"gt", "gte", "lt", "lte", "between"}:
+                    compare_expr = (
+                        f"CAST({extract_expr} AS REAL)"
+                        if filt.comparison == "numeric"
+                        else extract_expr
+                    )
+
+                    if filt.op == "between":
+                        min_param = f"meta_val_{idx}_min"
+                        max_param = f"meta_val_{idx}_max"
+                        params[min_param] = filt.value[0]
+                        params[max_param] = filt.value[1]
+                        conditions.append(f"{compare_expr} BETWEEN :{min_param} AND :{max_param}")
+                    else:
+                        value_param = f"meta_val_{idx}"
+                        params[value_param] = filt.value
+                        operator = {"gt": ">", "gte": ">=", "lt": "<", "lte": "<="}[filt.op]
+                        conditions.append(f"{compare_expr} {operator} :{value_param}")
+                    continue
 
         # Always filter by project_id
         params["project_id"] = self.project_id
-        conditions.append("project_id = :project_id")
+        conditions.append("search_index.project_id = :project_id")
 
         # set limit on search query
         params["limit"] = limit
@@ -367,23 +467,23 @@ class SQLiteSearchRepository(SearchRepositoryBase):
 
         sql = f"""
             SELECT
-                project_id,
-                id,
-                title,
-                permalink,
-                file_path,
-                type,
-                metadata,
-                from_id,
-                to_id,
-                relation_type,
-                entity_id,
-                content_snippet,
-                category,
-                created_at,
-                updated_at,
+                search_index.project_id,
+                search_index.id,
+                search_index.title,
+                search_index.permalink,
+                search_index.file_path,
+                search_index.type,
+                search_index.metadata,
+                search_index.from_id,
+                search_index.to_id,
+                search_index.relation_type,
+                search_index.entity_id,
+                search_index.content_snippet,
+                search_index.category,
+                search_index.created_at,
+                search_index.updated_at,
                 bm25(search_index) as score
-            FROM search_index
+            FROM {from_clause}
             WHERE {where_clause}
             ORDER BY score ASC {order_by_clause}
             LIMIT :limit