perf(ci): disable semantic search in default test fixtures (#938)

Disable the silently-on semantic embedding stack in default test fixtures and deselect on-demand benchmarks from CI int jobs. int SQLite 337s -> 110s, Postgres unit ~25min -> ~13min.

Signed-off-by: phernandez <paul@basicmemory.com>

justfile

@@ -46,9 +46,10 @@ test-unit-sqlite: testmon-seed
 test-unit-postgres: testmon-seed
     BASIC_MEMORY_ENV=test BASIC_MEMORY_TEST_POSTGRES=1 uv run pytest -p pytest_mock -v --no-cov {{TESTMON_FLAGS}} --testmon-env=unit-postgres tests
 
-# Run integration tests against SQLite (excludes semantic benchmarks — use just test-semantic)
+# Run integration tests against SQLite (excludes semantic tests and on-demand benchmarks —
+# use just test-semantic / run benchmark files explicitly)
 test-int-sqlite: testmon-seed
-    BASIC_MEMORY_ENV=test uv run pytest -p pytest_mock -v --no-cov {{TESTMON_FLAGS}} --testmon-env=int-sqlite -m "not semantic" test-int
+    BASIC_MEMORY_ENV=test uv run pytest -p pytest_mock -v --no-cov {{TESTMON_FLAGS}} --testmon-env=int-sqlite -m "not semantic and not benchmark" test-int
 
 # Run integration tests against Postgres
 # Note: Uses timeout due to FastMCP Client + asyncpg cleanup hang (tests pass, process hangs on exit)
@@ -59,10 +60,10 @@ test-int-postgres: testmon-seed
     # Use gtimeout (macOS/Homebrew) or timeout (Linux)
     TIMEOUT_CMD=$(command -v gtimeout || command -v timeout || echo "")
     if [[ -n "$TIMEOUT_CMD" ]]; then
-        $TIMEOUT_CMD --signal=KILL 600 bash -c 'BASIC_MEMORY_ENV=test BASIC_MEMORY_TEST_POSTGRES=1 uv run pytest -p pytest_mock -v --no-cov {{TESTMON_FLAGS}} --testmon-env=int-postgres -m "not semantic" test-int' || test $? -eq 137
+        $TIMEOUT_CMD --signal=KILL 600 bash -c 'BASIC_MEMORY_ENV=test BASIC_MEMORY_TEST_POSTGRES=1 uv run pytest -p pytest_mock -v --no-cov {{TESTMON_FLAGS}} --testmon-env=int-postgres -m "not semantic and not benchmark" test-int' || test $? -eq 137
     else
         echo "⚠️  No timeout command found, running without timeout..."
-        BASIC_MEMORY_ENV=test BASIC_MEMORY_TEST_POSTGRES=1 uv run pytest -p pytest_mock -v --no-cov {{TESTMON_FLAGS}} --testmon-env=int-postgres -m "not semantic" test-int
+        BASIC_MEMORY_ENV=test BASIC_MEMORY_TEST_POSTGRES=1 uv run pytest -p pytest_mock -v --no-cov {{TESTMON_FLAGS}} --testmon-env=int-postgres -m "not semantic and not benchmark" test-int
     fi
 
 # Run tests impacted by recent changes (requires pytest-testmon)

test-int/conftest.py

@@ -381,6 +381,13 @@ def app_config(
         sync_changes=False,  # Disable file sync in tests - prevents lifespan from starting blocking task
         database_backend=database_backend,
         database_url=database_url,
+        # Trigger: semantic_search_enabled defaults to True whenever fastembed/sqlite-vec
+        #          are importable, which they are in dev and CI environments.
+        # Why: with it on, every test that syncs pays the ONNX embedding stack (~5-7s per
+        #      sync) — embeddings are covered by test-int/semantic/, which configures
+        #      semantic_search_enabled explicitly in its own conftest.
+        # Outcome: non-semantic integration tests skip embedding work entirely.
+        semantic_search_enabled=False,
     )
     return app_config
 

test-int/test_embedding_status_vec0.py

@@ -17,12 +17,13 @@ this path.
 
 import os
 import sqlite3
+from unittest.mock import patch
 
 import pytest
 from sqlalchemy import text
 
 from basic_memory import db
-from basic_memory.config import BasicMemoryConfig, DatabaseBackend
+from basic_memory.config import BasicMemoryConfig, ConfigManager, DatabaseBackend
 from basic_memory.repository.entity_repository import EntityRepository
 from basic_memory.repository.project_repository import ProjectRepository
 from basic_memory.repository.sqlite_search_repository import SQLiteSearchRepository
@@ -153,7 +154,19 @@ async def test_embedding_status_reads_real_vec0_table(engine_factory, test_proje
     project_repository = ProjectRepository(session_maker)
     project_service = ProjectService(project_repository)
 
-    status = await project_service.get_embedding_status(project_id)
+    # Test fixtures run with semantic search disabled; the status call reads the global
+    # ConfigManager, so patch it to report semantic enabled for this regression path.
+    def _config_manager_semantic_enabled() -> ConfigManager:
+        cm = ConfigManager()
+        cm.config.semantic_search_enabled = True
+        return cm
+
+    with patch.object(
+        type(project_service),
+        "config_manager",
+        new_callable=lambda: property(lambda self: _config_manager_semantic_enabled()),
+    ):
+        status = await project_service.get_embedding_status(project_id)
 
     assert status.semantic_search_enabled is True
     # The vec0 JOIN must succeed, so the table is reported as present and healthy.

tests/conftest.py

@@ -292,6 +292,13 @@ def app_config(config_home, db_backend, postgres_container, monkeypatch) -> Basi
         update_permalinks_on_move=True,
         database_backend=backend,
         database_url=database_url,
+        # Trigger: semantic_search_enabled defaults to True whenever fastembed/sqlite-vec
+        #          are importable, which they are in dev and CI environments.
+        # Why: with it on, every test that syncs pays the ONNX embedding stack (~5-7s per
+        #      sync) — embeddings are covered by the dedicated semantic suites, which
+        #      configure semantic_search_enabled explicitly themselves.
+        # Outcome: non-semantic tests skip embedding work entirely.
+        semantic_search_enabled=False,
     )
 
     return app_config

tests/services/test_project_service_embedding_status.py

@@ -14,6 +14,28 @@ def _is_postgres() -> bool:
     return os.environ.get("BASIC_MEMORY_TEST_POSTGRES", "").lower() in ("1", "true", "yes")
 
 
+async def _create_embeddings_stub(project_service: ProjectService) -> None:
+    """Create a minimal search_vector_embeddings stub so vector_tables_exist is True.
+
+    Test fixtures run with semantic search disabled, so the real vec0/pgvector
+    embeddings table is never created. get_embedding_status only probes table
+    existence and joins on chunk_id (rowid on SQLite), so a plain table suffices.
+    """
+    await project_service.repository.execute_query(
+        text(
+            "CREATE TABLE IF NOT EXISTS search_vector_embeddings (  chunk_id INTEGER PRIMARY KEY)"
+        ),
+        {},
+    )
+
+
+async def _drop_embeddings_stub(project_service: ProjectService) -> None:
+    """Drop the stub table to avoid polluting subsequent tests."""
+    await project_service.repository.execute_query(
+        text("DROP TABLE IF EXISTS search_vector_embeddings"), {}
+    )
+
+
 @pytest.mark.asyncio
 async def test_embedding_status_semantic_disabled(project_service: ProjectService, test_project):
     """When semantic search is disabled, return minimal status with zero counts."""
@@ -69,7 +91,9 @@ async def test_embedding_status_entities_without_chunks(
     project_service: ProjectService, test_graph, test_project
 ):
     """When entities have search_index rows but no chunks, recommend reindex."""
-    # search_vector_chunks table is created by the test fixture (empty)
+    # search_vector_chunks comes from Base.metadata; the embeddings table needs a stub
+    # because fixtures run with semantic search disabled.
+    await _create_embeddings_stub(project_service)
     with patch.object(
         type(project_service),
         "config_manager",
@@ -79,6 +103,8 @@ async def test_embedding_status_entities_without_chunks(
     ):
         status = await project_service.get_embedding_status(test_project.id)
 
+    await _drop_embeddings_stub(project_service)
+
     assert status.semantic_search_enabled is True
     assert status.vector_tables_exist is True
     # test_graph creates entities indexed in search_index, but no vector chunks
@@ -159,6 +185,10 @@ async def test_embedding_status_handles_sqlite_vec_unavailable(
     if _is_postgres():
         pytest.skip("sqlite-vec unavailable handling is SQLite-specific.")
 
+    # Both vector tables must exist so the status check reaches the vec query;
+    # fixtures run with semantic search disabled, so stub the embeddings table.
+    await _create_embeddings_stub(project_service)
+
     # scalar_vec_query returns None when the extension can't be loaded on this
     # Python build (e.g. the python.org macOS interpreter). Simulate that here.
     async def _vec_query_unavailable(query, params=None):
@@ -178,6 +208,8 @@ async def test_embedding_status_handles_sqlite_vec_unavailable(
         ):
             status = await project_service.get_embedding_status(test_project.id)
 
+    await _drop_embeddings_stub(project_service)
+
     assert status.semantic_search_enabled is True
     assert status.total_indexed_entities > 0
     assert status.vector_tables_exist is False
@@ -272,6 +304,9 @@ async def test_embedding_status_excludes_stale_entity_ids(
     # Include 'id' column — required NOT NULL on Postgres (regular table),
     # ignored on SQLite (FTS5 virtual table where id is UNINDEXED).
     stale_entity_id = 999999
+    # Both vector tables must exist to reach the stale-filtered count queries;
+    # fixtures run with semantic search disabled, so stub the embeddings table.
+    await _create_embeddings_stub(project_service)
     await project_service.repository.execute_query(
         text(
             "INSERT INTO search_index "

tests/services/test_search_service.py

@@ -1384,6 +1384,21 @@ async def test_reindex_vectors(search_service, session_maker, test_project, monk
 
     entity_repo = EntityRepository(session_maker, project_id=test_project.id)
 
+    # Test fixtures disable semantic search, and delete_stale_vector_rows is the one call
+    # in this flow that requires the semantic stack — stub it so the test exercises the
+    # reindex wiring (id collection, batch call, stats mapping) without embeddings.
+    # raising=False: the method is SQLite-only; the Postgres purge path never calls it,
+    # so on Postgres this just attaches an unused attribute.
+    async def _noop_delete_stale_vector_rows() -> None:
+        return None
+
+    monkeypatch.setattr(
+        search_service.repository,
+        "delete_stale_vector_rows",
+        _noop_delete_stale_vector_rows,
+        raising=False,
+    )
+
     # Create some entities
     created_entity_ids: list[int] = []
     for i in range(3):
@@ -1454,6 +1469,22 @@ async def test_reindex_vectors_no_callback(
     from datetime import datetime
 
     entity_repo = EntityRepository(session_maker, project_id=test_project.id)
+
+    # Test fixtures disable semantic search, and delete_stale_vector_rows is the one call
+    # in this flow that requires the semantic stack — stub it so the test exercises the
+    # reindex wiring without embeddings.
+    # raising=False: the method is SQLite-only; the Postgres purge path never calls it,
+    # so on Postgres this just attaches an unused attribute.
+    async def _noop_delete_stale_vector_rows() -> None:
+        return None
+
+    monkeypatch.setattr(
+        search_service.repository,
+        "delete_stale_vector_rows",
+        _noop_delete_stale_vector_rows,
+        raising=False,
+    )
+
     entity = await entity_repo.create(
         {
             "title": "No Callback Entity",