diff --git a/src/basic_memory/models/search.py b/src/basic_memory/models/search.py index e0789845..6308737c 100644 --- a/src/basic_memory/models/search.py +++ b/src/basic_memory/models/search.py @@ -5,11 +5,12 @@ from sqlalchemy import DDL # Define FTS5 virtual table creation CREATE_SEARCH_INDEX = DDL(""" CREATE VIRTUAL TABLE IF NOT EXISTS search_index USING fts5( - content, -- Searchable text content - permalink UNINDEXED, -- Link to entity/document (must be unique) + title, -- Title for exact/fuzzy matching + content, -- Additional searchable content + permalink UNINDEXED, -- Link to entity/document file_path UNINDEXED, -- Filesystem path - type UNINDEXED, -- 'entity' or 'document' - metadata UNINDEXED, -- JSON with timestamps, types, etc. + type UNINDEXED, -- entity type + metadata UNINDEXED, -- Additional metadata tokenize='porter unicode61' -- Enable stemming + unicode ); """) diff --git a/src/basic_memory/repository/search_repository.py b/src/basic_memory/repository/search_repository.py index c405520f..d6e70757 100644 --- a/src/basic_memory/repository/search_repository.py +++ b/src/basic_memory/repository/search_repository.py @@ -8,8 +8,8 @@ from sqlalchemy import text from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker from basic_memory import db -from basic_memory.schemas.search import SearchQuery, SearchResult, SearchItemType from basic_memory.models.search import CREATE_SEARCH_INDEX +from basic_memory.schemas.search import SearchQuery, SearchResult, SearchItemType class SearchRepository: @@ -27,46 +27,13 @@ class SearchRepository: async def search( self, query: SearchQuery, context: Optional[List[str]] = None ) -> List[SearchResult]: - """Search across all indexed content using FTS5. - - Uses a three-tier matching strategy: - 1. title:term matches (highest priority) - 2. Exact term matches (medium priority) - 3. Prefix matches (lowest priority) - """ + """Search across all indexed content.""" conditions = [] params = {} - # Handle text search with fuzzy matching + # Handle text search if query.text: - # Prepare search terms - search_terms = query.text.lower().split() - params["search_terms"] = search_terms - - # Build match conditions in priority order - matches = [] - - # 1. Title field matches (highest weight) - title_matches = [f'title:"{term}"' for term in search_terms] - if len(search_terms) > 1: - # Multi-word title match - title_phrase = " ".join(f'title:"{term}"' for term in search_terms) - title_matches.append(f'NEAR({title_phrase}, {len(search_terms)})') - matches.extend(title_matches) - - # 2. Exact word matches - matches.extend([f'"{term}"' for term in search_terms]) - if len(search_terms) > 1: - # Multi-word proximity match - phrase = " ".join(f'"{term}"' for term in search_terms) - matches.append(f'NEAR({phrase}, {len(search_terms) * 2})') - - # 3. Prefix matches - matches.extend([f'{term}*' for term in search_terms]) - - # Complete match expression - match_expr = " OR ".join(matches) - conditions.append(f"content MATCH '{match_expr}'") + conditions.append(f"content MATCH '{query.text}'") # Handle type filter if query.types: @@ -87,26 +54,16 @@ class SearchRepository: # Build WHERE clause where_clause = " AND ".join(conditions) if conditions else "1=1" - # Build SQL query sql = f""" - WITH search_results AS ( - SELECT - permalink, - file_path, - type, - metadata, - rank - FROM search_index - WHERE {where_clause} - ) SELECT permalink, file_path, type, metadata, - rank as score - FROM search_results - ORDER BY rank ASC + bm25(search_index) as score + FROM search_index + WHERE {where_clause} + ORDER BY score DESC """ logger.debug(f"Search query: {sql}") @@ -129,6 +86,7 @@ class SearchRepository: async def index_item( self, + title: str, content: str, permalink: str, file_path: str, @@ -170,4 +128,4 @@ class SearchRepository: text("DELETE FROM search_index WHERE permalink = :permalink"), {"permalink": permalink}, ) - await session.commit() \ No newline at end of file + await session.commit() diff --git a/src/basic_memory/services/search_service.py b/src/basic_memory/services/search_service.py index dd2821f0..8e7071cf 100644 --- a/src/basic_memory/services/search_service.py +++ b/src/basic_memory/services/search_service.py @@ -50,12 +50,11 @@ class SearchService: async def index_entity( self, entity: Entity, background_tasks: Optional[BackgroundTasks] = None ) -> None: - """Index an entity's core identity for fuzzy matching. + """Index an entity's content for search. - Core content: - - Original title (lowercase) - - Title with field marker (for weighted matching) - - Path components as discrete terms + Indexes: + - Title in dedicated field for better match control + - Path components in content field for findability """ # Build searchable content content_parts = [] @@ -64,6 +63,7 @@ class SearchService: title = entity.title.lower() content_parts.extend([ title, # Base title + entity.summary or "", f"title:{title}", # Field marker for targeted matching *title.split(), # Individual words ]) @@ -88,6 +88,7 @@ class SearchService: if background_tasks: background_tasks.add_task( self._do_index, + title=title, content=content, permalink=entity.permalink, file_path=entity.file_path, @@ -96,6 +97,7 @@ class SearchService: ) else: await self._do_index( + title=title, content=content, permalink=entity.permalink, file_path=entity.file_path, @@ -104,10 +106,17 @@ class SearchService: ) async def _do_index( - self, content: str, permalink: str, file_path: str, type: SearchItemType, metadata: dict + self, + title: str, + content: str, + permalink: str, + file_path: str, + type: SearchItemType, + metadata: dict, ) -> None: """Actually perform the indexing.""" await self.repository.index_item( + title=title, content=content, permalink=permalink, file_path=file_path, diff --git a/tests/services/test_search_service.py b/tests/services/test_search_service.py index a46808ac..495ca3af 100644 --- a/tests/services/test_search_service.py +++ b/tests/services/test_search_service.py @@ -1,8 +1,9 @@ """Tests for search service fuzzy matching.""" -import pytest from datetime import datetime, timezone +import pytest +import pytest_asyncio from sqlalchemy import text from basic_memory import db @@ -10,9 +11,10 @@ from basic_memory.schemas import Entity as EntitySchema from basic_memory.schemas.search import SearchQuery, SearchItemType -@pytest.fixture +@pytest_asyncio.fixture def test_entity(): """Create a test entity""" + class Entity: id = 1 title = "TestComponent" @@ -29,130 +31,26 @@ def test_entity(): return Entity() + @pytest.fixture -async def test_entities(search_service): - """Create a set of test entities with variations.""" - entities = [ - # Core services - type("Entity", (), { - "id": 1, - "title": "Core Service", - "entity_type": "component", - "permalink": "components/core-service", - "file_path": "components/core-service.md", - "content_type": "text/markdown", - "created_at": datetime.now(timezone.utc), - "updated_at": datetime.now(timezone.utc), - "observations": [], - "relations": [], - "summary": "The core service implementation", - "entity_metadata": {} - })(), - # Auth service - type("Entity", (), { - "id": 2, - "title": "Auth Service", - "entity_type": "component", - "permalink": "components/auth/service", - "file_path": "components/auth/service.md", - "content_type": "text/markdown", - "created_at": datetime.now(timezone.utc), - "updated_at": datetime.now(timezone.utc), - "observations": [], - "relations": [], - "summary": "Authentication service", - "entity_metadata": {} - })(), - # Config service - type("Entity", (), { - "id": 3, - "title": "Service Config", - "entity_type": "config", - "permalink": "config/service-config", - "file_path": "config/service-config.md", - "content_type": "text/markdown", - "created_at": datetime.now(timezone.utc), - "updated_at": datetime.now(timezone.utc), - "observations": [], - "relations": [], - "summary": "Service configuration", - "entity_metadata": {} - })(), - # Features doc - type("Entity", (), { - "id": 4, - "title": "Core Features", - "entity_type": "specs", - "permalink": "specs/features/core", - "file_path": "specs/features/core.md", - "content_type": "text/markdown", - "created_at": datetime.now(timezone.utc), - "updated_at": datetime.now(timezone.utc), - "observations": [], - "relations": [], - "summary": "Core feature specifications", - "entity_metadata": {} - })() - ] - - # Index all entities - for entity in entities: - await search_service.index_entity(entity) - - return entities +def test_document(): + """Create a test document""" + + class Document: + id = 1 + permalink = "docs/test_doc.md" + file_path = "docs/test_doc.md" + doc_metadata = {"title": "Test Document", "type": "technical"} + created_at = datetime.now(timezone.utc) + updated_at = datetime.now(timezone.utc) + + return Document() -@pytest.mark.asyncio -async def test_exact_match(search_service, test_entities): - """Test exact matching of titles.""" - results = await search_service.search(SearchQuery(text="Core Service")) - assert len(results) == 1 - assert results[0].permalink == "components/core-service" - - -@pytest.mark.asyncio -async def test_partial_match(search_service, test_entities): - """Test partial matching of titles.""" - results = await search_service.search(SearchQuery(text="Auth Serv")) - assert len(results) == 1 - assert results[0].permalink == "components/auth/service" - - -@pytest.mark.asyncio -async def test_misspelling_match(search_service, test_entities): - """Test matching with misspellings.""" - results = await search_service.search(SearchQuery(text="Core Servise")) - assert len(results) == 1 - assert results[0].permalink == "components/core-service" - - -@pytest.mark.asyncio -async def test_multiple_results_ranking(search_service, test_entities): - """Test result ranking with multiple matches.""" - # Search for 'service' - should find multiple but rank appropriately - results = await search_service.search(SearchQuery(text="service")) - assert len(results) > 1 - # Auth service should rank higher than core service - auth_idx = next(i for i, r in enumerate(results) if r.permalink == "components/auth/service") - core_idx = next(i for i, r in enumerate(results) if r.permalink == "components/core-service") - assert auth_idx < core_idx # Lower index = better rank - - -@pytest.mark.asyncio -async def test_directory_context(search_service, test_entities): - """Test matching with directory context.""" - results = await search_service.search( - SearchQuery(text="Core"), - context=["components/"] # Should prefer matches in components/ - ) - assert len(results) > 0 - assert results[0].permalink.startswith("components/") - - -# Original test_init_search_index @pytest.mark.asyncio async def test_init_search_index(search_service, session_maker): """Test search index initialization""" + # Check that table exists async with db.scoped_session(session_maker) as session: result = await session.execute( text("SELECT name FROM sqlite_master WHERE type='table' AND name='search_index';") @@ -160,7 +58,6 @@ async def test_init_search_index(search_service, session_maker): assert result.scalar() == "search_index" -# Original test_index_entity @pytest.mark.asyncio async def test_index_entity(search_service, test_entity): """Test indexing an entity""" @@ -173,7 +70,6 @@ async def test_index_entity(search_service, test_entity): assert results[0].type == SearchItemType.ENTITY -# Original test_search_filtering @pytest.mark.asyncio async def test_search_filtering(search_service, test_entity): """Test search with filters""" @@ -190,7 +86,6 @@ async def test_search_filtering(search_service, test_entity): assert len(results) == 0 -# Original test_update_index @pytest.mark.asyncio async def test_update_index(search_service, test_entity): """Test updating indexed content""" @@ -205,7 +100,6 @@ async def test_update_index(search_service, test_entity): assert len(results) == 1 -# Original test_search_date_filter @pytest.mark.asyncio async def test_search_date_filter(search_service, test_entity): """Test searching with date filter""" @@ -217,7 +111,6 @@ async def test_search_date_filter(search_service, test_entity): assert len(results) == 0 -# Original test_reindex_all @pytest.mark.asyncio async def test_reindex_all(search_service, entity_service, session_maker): """Test reindexing all content.""" @@ -252,7 +145,6 @@ async def test_reindex_all(search_service, entity_service, session_maker): assert entity_results[0].type == SearchItemType.ENTITY -# Original test_reindex_with_background_tasks @pytest.mark.asyncio async def test_reindex_with_background_tasks(search_service, entity_service, session_maker): """Test reindexing with background tasks.""" @@ -284,4 +176,4 @@ async def test_reindex_with_background_tasks(search_service, entity_service, ses # Verify everything was indexed all_results = await search_service.search(SearchQuery(text="test")) - assert len(all_results) == 1 \ No newline at end of file + assert len(all_results) == 1