From 324025f87d6f2bc6e7e0face65ae68ffb31ae680 Mon Sep 17 00:00:00 2001 From: phernandez Date: Sun, 19 Jan 2025 14:44:44 -0600 Subject: [PATCH] dry up search indexing --- src/basic_memory/services/search_service.py | 118 +++++--------------- tests/services/test_search_service.py | 14 ++- 2 files changed, 38 insertions(+), 94 deletions(-) diff --git a/src/basic_memory/services/search_service.py b/src/basic_memory/services/search_service.py index 41371148..c9024fda 100644 --- a/src/basic_memory/services/search_service.py +++ b/src/basic_memory/services/search_service.py @@ -9,6 +9,7 @@ from basic_memory.models import Entity from basic_memory.repository import EntityRepository from basic_memory.repository.search_repository import SearchRepository, SearchIndexRow from basic_memory.schemas.search import SearchQuery, SearchResult, SearchItemType +from basic_memory.utils import generate_permalink class SearchService: @@ -60,7 +61,7 @@ class SearchService: if query.no_criteria(): logger.debug("no criteria passed to query") return [] - + logger.debug(f"Searching with query: {query}") # permalink search @@ -131,69 +132,42 @@ class SearchService: entity_content = "\n".join(p for p in content_parts if p and p.strip()) # Index entity - if background_tasks: - background_tasks.add_task( - self._do_index, + await self._do_index( + SearchIndexRow( id=entity.id, + type=SearchItemType.ENTITY.value, title=entity.title, content=entity_content, permalink=entity.permalink, file_path=entity.file_path, - type=SearchItemType.ENTITY.value, - metadata={ - "entity_type": entity.entity_type, - "created_at": entity.created_at.isoformat(), - "updated_at": entity.updated_at.isoformat(), - }, - ) - else: - await self._do_index( - id=entity.id, - title=entity.title, - content=entity_content, - permalink=entity.permalink, - file_path=entity.file_path, - type=SearchItemType.ENTITY.value, metadata={ "entity_type": entity.entity_type, "created_at": entity.created_at.isoformat(), "updated_at": entity.updated_at.isoformat(), }, + created_at=entity.created_at.isoformat(), + updated_at=entity.updated_at.isoformat(), ) + ) # Index each observation with synthetic permalink for obs in entity.observations: # Create synthetic permalink for the observation # We can construct these because observations are always # defined in and owned by a single entity - observation_permalink = f"{entity.permalink}/observations/{obs.id}" + observation_permalink = ( + f"{entity.permalink}/observations/{obs.category}/{generate_permalink(obs.content)}" + ) # Index with parent entity's file path since that's where it's defined - if background_tasks: - background_tasks.add_task( - self._do_index, + await self._do_index( + SearchIndexRow( id=obs.id, + type=SearchItemType.OBSERVATION.value, title=f"{obs.category}: {obs.content[:50]}...", content=obs.content, permalink=observation_permalink, file_path=entity.file_path, - type=SearchItemType.OBSERVATION.value, - category=obs.category, - entity_id=entity.id, - metadata={ - "created_at": obs.created_at.isoformat(), - "updated_at": obs.updated_at.isoformat(), - "tags": obs.tags, - }, - ) - else: - await self._do_index( - id=obs.id, - title=f"{obs.category}: {obs.content[:50]}...", - content=obs.content, - permalink=observation_permalink, - file_path=entity.file_path, - type=SearchItemType.OBSERVATION.value, category=obs.category, entity_id=entity.id, metadata={ @@ -201,7 +175,10 @@ class SearchService: "updated_at": obs.updated_at.isoformat(), "tags": obs.tags, }, + created_at=obs.created_at.isoformat(), + updated_at=obs.updated_at.isoformat(), ) + ) # Only index outgoing relations (ones defined in this file) for rel in entity.outgoing_relations: @@ -215,25 +192,8 @@ class SearchService: # Create descriptive title showing the relationship relation_title = f"{rel.from_entity.title} → {rel.to_entity.title}" - if background_tasks: - background_tasks.add_task( - self._do_index, - id=rel.id, - title=relation_title, - content=rel.context or "", - permalink=relation_permalink, - file_path=entity.file_path, - type=SearchItemType.RELATION.value, - from_id=rel.from_id, - to_id=rel.to_id, - relation_type=rel.relation_type, - metadata={ - "created_at": rel.created_at.isoformat(), - "updated_at": rel.updated_at.isoformat(), - }, - ) - else: - await self._do_index( + await self._do_index( + SearchIndexRow( id=rel.id, title=relation_title, content=rel.context or "", @@ -247,43 +207,19 @@ class SearchService: "created_at": rel.created_at.isoformat(), "updated_at": rel.updated_at.isoformat(), }, + created_at=rel.created_at.isoformat(), + updated_at=rel.updated_at.isoformat(), ) + ) async def _do_index( - self, - id: int, - title: str, - content: str, - permalink: str, - file_path: str, - type: SearchItemType, - metadata: dict, - from_id: Optional[int] = None, - to_id: Optional[int] = None, - relation_type: Optional[str] = None, - entity_id: Optional[int] = None, - category: Optional[str] = None, + self, index_row: SearchIndexRow, background_tasks: Optional[BackgroundTasks] = None ) -> None: """Actually perform the indexing.""" - - await self.repository.index_item( - SearchIndexRow( - id=id, - title=title, - content=content, - permalink=permalink, - file_path=file_path, - type=type, - metadata=metadata, - from_id=from_id, - to_id=to_id, - relation_type=relation_type, - entity_id=entity_id, - category=category, - created_at=metadata.get("created_at"), - updated_at=metadata.get("updated_at"), - ) - ) + if background_tasks: + background_tasks.add_task(self.repository.index_item, index_row) + else: + await self.repository.index_item(index_row) async def delete_by_permalink(self, path_id: str): """Delete an item from the search index.""" diff --git a/tests/services/test_search_service.py b/tests/services/test_search_service.py index 31332122..553bc6b3 100644 --- a/tests/services/test_search_service.py +++ b/tests/services/test_search_service.py @@ -20,13 +20,21 @@ async def test_search_permalink(search_service, test_graph): @pytest.mark.asyncio -async def test_search_permalink_wildcard(search_service, test_graph): +async def test_search_permalink_observations_wildcard(search_service, test_graph): """Pattern matching""" results = await search_service.search(SearchQuery(permalink_match="test/root/observations/*")) assert len(results) == 2 permalinks = {r.permalink for r in results} - assert "test/root/observations/1" in permalinks - assert "test/root/observations/2" in permalinks + assert "test/root/observations/note/root-note-1" in permalinks + assert "test/root/observations/tech/root-tech-note" in permalinks + +@pytest.mark.asyncio +async def test_search_permalink_relation_wildcard(search_service, test_graph): + """Pattern matching""" + results = await search_service.search(SearchQuery(permalink_match="test/root/connects_to/*")) + assert len(results) == 1 + permalinks = {r.permalink for r in results} + assert "test/root/connects_to/test/connected1" in permalinks @pytest.mark.skip("search prefix see:'https://sqlite.org/fts5.html#FTS5 Prefix Queries'")