From a509ef1e2a91e39e6c58debf828465dee414d2a9 Mon Sep 17 00:00:00 2001 From: phernandez Date: Wed, 15 Jan 2025 16:02:53 -0600 Subject: [PATCH] add pattern match to search index, index permalink --- src/basic_memory/models/search.py | 18 +++---- .../repository/search_repository.py | 36 +++++++++++-- tests/services/test_search_service.py | 51 ++++++++++++++++--- 3 files changed, 87 insertions(+), 18 deletions(-) diff --git a/src/basic_memory/models/search.py b/src/basic_memory/models/search.py index 7424552e..fa6c9f70 100644 --- a/src/basic_memory/models/search.py +++ b/src/basic_memory/models/search.py @@ -9,18 +9,18 @@ CREATE VIRTUAL TABLE IF NOT EXISTS search_index USING fts5( id UNINDEXED, -- Row ID title, -- Title for searching content, -- Main searchable content - permalink UNINDEXED, -- Stable identifier - file_path UNINDEXED, -- Physical location - type UNINDEXED, -- entity/relation/observation + permalink, -- Stable identifier (now indexed for path search) + file_path UNINDEXED, -- Physical location + type UNINDEXED, -- entity/relation/observation -- Relation fields - from_id UNINDEXED, -- Source entity - to_id UNINDEXED, -- Target entity + from_id UNINDEXED, -- Source entity + to_id UNINDEXED, -- Target entity relation_type UNINDEXED, -- Type of relation -- Observation fields - entity_id UNINDEXED, -- Parent entity - category UNINDEXED, -- Observation category + entity_id UNINDEXED, -- Parent entity + category UNINDEXED, -- Observation category -- Common fields metadata UNINDEXED, -- JSON metadata @@ -28,7 +28,7 @@ CREATE VIRTUAL TABLE IF NOT EXISTS search_index USING fts5( updated_at UNINDEXED, -- Last update -- Configuration - tokenize='porter unicode61', - prefix='2,3' + tokenize=\"unicode61 separators '/'\", -- Treat / as part of tokens + prefix='1,2,3,4' -- Support longer prefixes for paths ); """) \ No newline at end of file diff --git a/src/basic_memory/repository/search_repository.py b/src/basic_memory/repository/search_repository.py index b116bc66..5c926d47 100644 --- a/src/basic_memory/repository/search_repository.py +++ b/src/basic_memory/repository/search_repository.py @@ -31,6 +31,34 @@ class SearchRepository: return f'"{term}"' return term + def _convert_pattern_to_fts(self, pattern: str) -> str: + """Convert glob pattern to FTS5 query. + + Examples: + specs/search -> "specs/search" + specs/* -> "specs/"* + specs/*/doc -> "specs/" AND "service" + */doc -> "service" + """ + if not pattern: + return "" + + # Split on * to identify wildcard positions + parts = pattern.split('*') + + # No wildcards - do exact phrase match + if len(parts) == 1: + return f'"{pattern}"' + + # Strip empty parts and clean up + parts = [p.strip('/') for p in parts if p.strip('/')] + + # For patterns with wildcards, quote each part + quoted = [f'"{p}"' for p in parts] + + # Join with AND to ensure all parts match + return ' AND '.join(quoted) + async def search(self, query: SearchQuery) -> List[SearchResult]: """Search across all indexed content with fuzzy matching.""" conditions = [] @@ -42,10 +70,12 @@ class SearchRepository: params["text"] = f"{search_text}*" conditions.append("(title MATCH :text OR content MATCH :text)") - # Handle pattern search on permalink if specified + # Handle pattern search on permalink using FTS if query.permalink_pattern: - params["permalink_pattern"] = query.permalink_pattern - conditions.append("permalink LIKE :permalink_pattern") + fts_pattern = self._convert_pattern_to_fts(query.permalink_pattern) + if fts_pattern: + params["permalink_pattern"] = fts_pattern + conditions.append("permalink MATCH :permalink_pattern") # Handle type filter if query.types: diff --git a/tests/services/test_search_service.py b/tests/services/test_search_service.py index 6245cbd7..472f6710 100644 --- a/tests/services/test_search_service.py +++ b/tests/services/test_search_service.py @@ -16,7 +16,7 @@ async def test_entities(entity_repository): Entity( title="Core Service", entity_type="component", - permalink="core-service", + permalink="components/core-service", # Updated to use path-style permalinks summary="The core service implementation", file_path="components/core-service.md", content_type="text/markdown", @@ -24,7 +24,7 @@ async def test_entities(entity_repository): Entity( title="Service Config", entity_type="config", - permalink="service-config", + permalink="config/service-config", summary="Configuration for services", file_path="config/service-config.md", content_type="text/markdown", @@ -32,7 +32,7 @@ async def test_entities(entity_repository): Entity( title="Auth Service", entity_type="component", - permalink="auth-service", + permalink="components/auth/service", # Nested path summary="Authentication service implementation", file_path="components/auth/service.md", content_type="text/markdown", @@ -40,7 +40,7 @@ async def test_entities(entity_repository): Entity( title="Core Features", entity_type="specs", - permalink="core-features", + permalink="specs/features/core", summary="Core feature specifications", file_path="specs/features/core.md", content_type="text/markdown", @@ -48,7 +48,7 @@ async def test_entities(entity_repository): Entity( title="API Documentation", entity_type="docs", - permalink="api-documentation", + permalink="docs/api/documentation", summary="API documentation and examples", file_path="docs/api/documentation.md", content_type="text/markdown", @@ -145,6 +145,45 @@ async def test_search_filters(indexed_search): assert len(results) == 0 +@pytest.mark.asyncio +async def test_path_pattern_search(indexed_search): + """Test path pattern matching in permalinks.""" + # Test exact path match + results = await indexed_search.search( + SearchQuery(permalink_pattern="components/core-service") + ) + assert len(results) == 1 + assert results[0].permalink == "components/core-service" + + # Test prefix matching with * + results = await indexed_search.search( + SearchQuery(permalink_pattern="components/*") + ) + assert len(results) == 2 # Should match both core-service and auth/service + permalinks = {r.permalink for r in results} + assert "components/core-service" in permalinks + assert "components/auth/service" in permalinks + + # Test nested path matching + results = await indexed_search.search( + SearchQuery(permalink_pattern="components/*/service") + ) + permalinks = [r.permalink for r in results] + assert len(permalinks) == 2 + assert "components/auth/service" in permalinks + assert "components/core-service" in permalinks + + # Test top-level pattern + results = await indexed_search.search( + SearchQuery(permalink_pattern="*/service") + ) + permalinks = [r.permalink for r in results] + assert len(permalinks) == 3 + assert "components/auth/service" in permalinks + assert "components/core-service" in permalinks + assert "config/service-config" in permalinks + + @pytest.mark.asyncio async def test_init_search_index(search_service, session_maker): """Test search index initialization.""" @@ -166,4 +205,4 @@ async def test_update_index(search_service, full_entity): # Search for new terms results = await search_service.search(SearchQuery(text="new terms")) - assert len(results) == 1 + assert len(results) == 1 \ No newline at end of file