chore: Make semantic deps default, auto-backfill embeddings, and default search to semantic (#586)

Signed-off-by: phernandez <paul@basicmachines.co>

src/basic_memory/alembic/versions/i2c3d4e5f6g7_auto_backfill_semantic_embeddings.py

@@ -0,0 +1,29 @@
+"""Trigger automatic semantic embedding backfill during migration.
+
+Revision ID: i2c3d4e5f6g7
+Revises: h1b2c3d4e5f6
+Create Date: 2026-02-19 00:00:00.000000
+
+"""
+
+from typing import Sequence, Union
+
+# revision identifiers, used by Alembic.
+revision: str = "i2c3d4e5f6g7"
+down_revision: Union[str, None] = "h1b2c3d4e5f6"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    """No schema change.
+
+    Trigger: this revision is newly applied.
+    Why: db.run_migrations() detects this revision transition and runs the existing
+    sync_entity_vectors() pipeline to backfill semantic embeddings automatically.
+    Outcome: users no longer need to run `bm reindex --embeddings` after upgrading.
+    """
+
+
+def downgrade() -> None:
+    """No-op downgrade."""

src/basic_memory/cli/commands/tool.py

@@ -847,8 +847,8 @@ def search_notes(
         if not metadata_filters:
             metadata_filters = None
 
-        # set search type
-        search_type = "text"
+        # set search type (None delegates to MCP tool default selection)
+        search_type: str | None = None
         if permalink:
             search_type = "permalink"
             if query and "*" in query:

src/basic_memory/config.py

@@ -40,8 +40,9 @@ class DatabaseBackend(str, Enum):
 
 
 def _default_semantic_search_enabled() -> bool:
-    """Enable semantic search by default when semantic extras are installed."""
-    return importlib.util.find_spec("fastembed") is not None
+    """Enable semantic search by default when required local semantic dependencies exist."""
+    required_modules = ("fastembed", "sqlite_vec")
+    return all(importlib.util.find_spec(module_name) is not None for module_name in required_modules)
 
 
 @dataclass
@@ -145,7 +146,7 @@ class BasicMemoryConfig(BaseSettings):
     # Semantic search configuration
     semantic_search_enabled: bool = Field(
         default_factory=_default_semantic_search_enabled,
-        description="Enable semantic search (vector/hybrid retrieval). Works on both SQLite and Postgres backends. Requires semantic extras.",
+        description="Enable semantic search (vector/hybrid retrieval). Works on both SQLite and Postgres backends. Requires semantic dependencies (included by default).",
     )
     semantic_embedding_provider: str = Field(
         default="fastembed",

src/basic_memory/db.py

@@ -43,6 +43,99 @@ if sys.platform == "win32":  # pragma: no cover
 _engine: Optional[AsyncEngine] = None
 _session_maker: Optional[async_sessionmaker[AsyncSession]] = None
 
+# Alembic revision that enables one-time automatic embedding backfill.
+SEMANTIC_EMBEDDING_BACKFILL_REVISION = "i2c3d4e5f6g7"
+
+
+async def _load_applied_alembic_revisions(
+    session_maker: async_sessionmaker[AsyncSession],
+) -> set[str]:
+    """Load applied Alembic revisions from alembic_version.
+
+    Returns an empty set when the version table does not exist yet
+    (fresh database before first migration).
+    """
+    try:
+        async with scoped_session(session_maker) as session:
+            result = await session.execute(text("SELECT version_num FROM alembic_version"))
+            return {str(row[0]) for row in result.fetchall() if row[0]}
+    except Exception as exc:
+        error_message = str(exc).lower()
+        if "alembic_version" in error_message and (
+            "no such table" in error_message or "does not exist" in error_message
+        ):
+            return set()
+        raise
+
+
+def _should_run_semantic_embedding_backfill(
+    revisions_before_upgrade: set[str],
+    revisions_after_upgrade: set[str],
+) -> bool:
+    """Check if this migration run newly applied the backfill-trigger revision."""
+    return (
+        SEMANTIC_EMBEDDING_BACKFILL_REVISION in revisions_after_upgrade
+        and SEMANTIC_EMBEDDING_BACKFILL_REVISION not in revisions_before_upgrade
+    )
+
+
+async def _run_semantic_embedding_backfill(
+    app_config: BasicMemoryConfig,
+    session_maker: async_sessionmaker[AsyncSession],
+) -> None:
+    """Backfill semantic embeddings for all active projects/entities."""
+    if not app_config.semantic_search_enabled:
+        logger.info("Skipping automatic semantic embedding backfill: semantic search is disabled.")
+        return
+
+    async with scoped_session(session_maker) as session:
+        project_result = await session.execute(
+            text("SELECT id, name FROM project WHERE is_active = :is_active ORDER BY id"),
+            {"is_active": True},
+        )
+        projects = [(int(row[0]), str(row[1])) for row in project_result.fetchall()]
+
+    if not projects:
+        logger.info("Skipping automatic semantic embedding backfill: no active projects found.")
+        return
+
+    repository_class = (
+        PostgresSearchRepository
+        if app_config.database_backend == DatabaseBackend.POSTGRES
+        else SQLiteSearchRepository
+    )
+
+    total_entities = 0
+    for project_id, project_name in projects:
+        async with scoped_session(session_maker) as session:
+            entity_result = await session.execute(
+                text("SELECT id FROM entity WHERE project_id = :project_id ORDER BY id"),
+                {"project_id": project_id},
+            )
+            entity_ids = [int(row[0]) for row in entity_result.fetchall()]
+
+        if not entity_ids:
+            continue
+
+        total_entities += len(entity_ids)
+        logger.info(
+            "Automatic semantic embedding backfill: "
+            f"project={project_name}, entities={len(entity_ids)}"
+        )
+
+        search_repository = repository_class(
+            session_maker,
+            project_id=project_id,
+            app_config=app_config,
+        )
+        for entity_id in entity_ids:
+            await search_repository.sync_entity_vectors(entity_id)
+
+    logger.info(
+        "Automatic semantic embedding backfill complete: "
+        f"projects={len(projects)}, entities={total_entities}"
+    )
+
 
 class DatabaseType(Enum):
     """Types of supported databases."""
@@ -384,6 +477,23 @@ async def run_migrations(
     """
     logger.info("Running database migrations...")
     try:
+        revisions_before_upgrade: set[str] = set()
+        # Trigger: run_migrations() can be invoked before module-level session maker is set.
+        # Why: we still need reliable before/after revision detection for one-time backfill.
+        # Outcome: create a short-lived session maker when needed, then dispose it immediately.
+        if _session_maker is None:
+            temp_engine, temp_session_maker = _create_engine_and_session(
+                app_config.database_path,
+                database_type,
+                app_config,
+            )
+            try:
+                revisions_before_upgrade = await _load_applied_alembic_revisions(temp_session_maker)
+            finally:
+                await temp_engine.dispose()
+        else:
+            revisions_before_upgrade = await _load_applied_alembic_revisions(_session_maker)
+
         # Get the absolute path to the alembic directory relative to this file
         alembic_dir = Path(__file__).parent / "alembic"
         config = Config()
@@ -422,6 +532,13 @@ async def run_migrations(
             await PostgresSearchRepository(session_maker, 1).init_search_index()
         else:
             await SQLiteSearchRepository(session_maker, 1).init_search_index()
+
+        revisions_after_upgrade = await _load_applied_alembic_revisions(session_maker)
+        if _should_run_semantic_embedding_backfill(
+            revisions_before_upgrade,
+            revisions_after_upgrade,
+        ):
+            await _run_semantic_embedding_backfill(app_config, session_maker)
     except Exception as e:  # pragma: no cover
         logger.error(f"Error running migrations: {e}")
         raise

src/basic_memory/mcp/tools/chatgpt_tools.py

@@ -120,7 +120,6 @@ async def search(
             project=default_project,  # Use default project for ChatGPT
             page=1,
             page_size=10,  # Reasonable default for ChatGPT consumption
-            search_type="text",  # Default to full-text search
             output_format="json",
             context=context,
         )

src/basic_memory/mcp/tools/search.py

@@ -29,6 +29,11 @@ def _semantic_search_enabled_for_text_search() -> bool:
         return ConfigManager().config.semantic_search_enabled
 
 
+def _default_search_type() -> str:
+    """Pick default search mode from semantic-search config."""
+    return "hybrid" if _semantic_search_enabled_for_text_search() else "text"
+
+
 def _format_search_error_response(
     project: str, error_message: str, query: str, search_type: str = "text"
 ) -> str:
@@ -57,7 +62,7 @@ def _format_search_error_response(
             Semantic retrieval is enabled but required packages are not installed.
 
             ## Fix
-            1. Install semantic extras: `pip install 'basic-memory[semantic]'`
+            1. Install/update Basic Memory: `pip install -U basic-memory`
             2. Restart Basic Memory
             3. Retry your query:
                `search_notes("{project}", "{query}", search_type="{search_type}")`
@@ -252,7 +257,7 @@ async def search_notes(
     workspace: Optional[str] = None,
     page: int = 1,
     page_size: int = 10,
-    search_type: str = "text",
+    search_type: str | None = None,
     output_format: Literal["text", "json"] = "text",
     types: List[str] | None = None,
     entity_types: List[str] | None = None,
@@ -295,8 +300,8 @@ async def search_notes(
     ### Search Type Examples
     - `search_notes("my-project", "Meeting", search_type="title")` - Search only in titles
     - `search_notes("work-docs", "docs/meeting-*", search_type="permalink")` - Pattern match permalinks
-    - `search_notes("research", "keyword", search_type="text")` - Text search (default; auto-upgrades
-      to hybrid when semantic search is enabled)
+    - `search_notes("research", "keyword")` - Default search (hybrid when semantic is enabled,
+      text when disabled)
 
     ### Filtering Options
     - `search_notes("my-project", "query", types=["entity"])` - Search only entities
@@ -339,8 +344,8 @@ async def search_notes(
         page: The page number of results to return (default 1)
         page_size: The number of results to return per page (default 10)
         search_type: Type of search to perform, one of:
-                    "text", "title", "permalink", "vector", "semantic", "hybrid" (default: "text";
-                    text mode auto-upgrades to hybrid when semantic search is enabled)
+                    "text", "title", "permalink", "vector", "semantic", "hybrid".
+                    Default is dynamic: "hybrid" when semantic search is enabled, otherwise "text".
         output_format: "text" preserves existing structured search response behavior.
             "json" returns a machine-readable dictionary payload.
         types: Optional list of note types to search (e.g., ["note", "person"])
@@ -426,9 +431,10 @@ async def search_notes(
         _, resolved_query, is_memory_url = await resolve_project_and_path(
             client, query, project, context
         )
+        effective_search_type = search_type or _default_search_type()
         if is_memory_url:
             query = resolved_query
-            search_type = "permalink"
+            effective_search_type = "permalink"
 
         try:
             # Create a SearchQuery object based on the parameters
@@ -436,27 +442,24 @@ async def search_notes(
 
             # Map search_type to the appropriate query field and retrieval mode
             valid_search_types = {"text", "title", "permalink", "vector", "semantic", "hybrid"}
-            if search_type == "text":
+            if effective_search_type == "text":
                 search_query.text = query
-                # Upgrade to hybrid when semantic search is available —
-                # combines FTS keyword matching with vector similarity for better results
-                if _semantic_search_enabled_for_text_search():
-                    search_query.retrieval_mode = SearchRetrievalMode.HYBRID
-            elif search_type in ("vector", "semantic"):
+                search_query.retrieval_mode = SearchRetrievalMode.FTS
+            elif effective_search_type in ("vector", "semantic"):
                 search_query.text = query
                 search_query.retrieval_mode = SearchRetrievalMode.VECTOR
-            elif search_type == "hybrid":
+            elif effective_search_type == "hybrid":
                 search_query.text = query
                 search_query.retrieval_mode = SearchRetrievalMode.HYBRID
-            elif search_type == "title":
+            elif effective_search_type == "title":
                 search_query.title = query
-            elif search_type == "permalink" and "*" in query:
+            elif effective_search_type == "permalink" and "*" in query:
                 search_query.permalink_match = query
-            elif search_type == "permalink":
+            elif effective_search_type == "permalink":
                 search_query.permalink = query
             else:
                 raise ValueError(
-                    f"Invalid search_type '{search_type}'. "
+                    f"Invalid search_type '{effective_search_type}'. "
                     f"Valid options: {', '.join(sorted(valid_search_types))}"
                 )
 
@@ -504,7 +507,9 @@ async def search_notes(
         except Exception as e:
             logger.error(f"Search failed for query '{query}': {e}, project: {active_project.name}")
             # Return formatted error message as string for better user experience
-            return _format_search_error_response(active_project.name, str(e), query, search_type)
+            return _format_search_error_response(
+                active_project.name, str(e), query, effective_search_type
+            )
 
 
 @mcp.tool(

src/basic_memory/mcp/tools/ui_sdk.py

@@ -26,7 +26,7 @@ async def search_notes_ui(
     project: Optional[str] = None,
     page: int = 1,
     page_size: int = 10,
-    search_type: str = "text",
+    search_type: Optional[str] = None,
     types: List[str] | None = None,
     entity_types: List[str] | None = None,
     after_date: Optional[str] = None,

src/basic_memory/repository/fastembed_provider.py

@@ -48,7 +48,8 @@ class FastEmbedEmbeddingProvider(EmbeddingProvider):
                 ) as exc:  # pragma: no cover - exercised via tests with monkeypatch
                     raise SemanticDependenciesMissingError(
                         "fastembed package is missing. "
-                        "Install semantic extras: pip install 'basic-memory[semantic]'"
+                        "Install/update basic-memory to include semantic dependencies: "
+                        "pip install -U basic-memory"
                     ) from exc
                 resolved_model_name = self._MODEL_ALIASES.get(self.model_name, self.model_name)
                 return TextEmbedding(model_name=resolved_model_name)

src/basic_memory/repository/openai_provider.py

@@ -45,7 +45,8 @@ class OpenAIEmbeddingProvider(EmbeddingProvider):
             except ImportError as exc:  # pragma: no cover - covered via monkeypatch tests
                 raise SemanticDependenciesMissingError(
                     "OpenAI dependency is missing. "
-                    "Install semantic extras: pip install 'basic-memory[semantic]'"
+                    "Install/update basic-memory to include semantic dependencies: "
+                    "pip install -U basic-memory"
                 ) from exc
 
             api_key = self._api_key or os.getenv("OPENAI_API_KEY")

src/basic_memory/repository/search_repository_base.py

@@ -355,7 +355,8 @@ class SearchRepositoryBase(ABC):
         if self._embedding_provider is None:
             raise SemanticDependenciesMissingError(
                 "No embedding provider configured. "
-                "Install semantic extras: pip install 'basic-memory[semantic]' "
+                "Install/update basic-memory to include semantic dependencies "
+                "(pip install -U basic-memory) "
                 "and set semantic_search_enabled=true."
             )
 

src/basic_memory/repository/sqlite_search_repository.py

@@ -347,7 +347,8 @@ class SQLiteSearchRepository(SearchRepositoryBase):
         except ImportError as exc:
             raise SemanticDependenciesMissingError(
                 "sqlite-vec package is missing. "
-                "Install semantic extras: pip install 'basic-memory[semantic]'"
+                "Install/update basic-memory to include semantic dependencies: "
+                "pip install -U basic-memory"
             ) from exc
 
         async with self._sqlite_vec_lock: