From 9dff24746d2658cdc6f6f9dc123184a3081dc2cf Mon Sep 17 00:00:00 2001 From: phernandez Date: Tue, 14 Jan 2025 18:14:24 -0600 Subject: [PATCH] index relations and observations --- src/basic_memory/models/search.py | 43 ++++++-- src/basic_memory/schemas/search.py | 19 +++- src/basic_memory/services/search_service.py | 116 +++++++++++++++----- 3 files changed, 136 insertions(+), 42 deletions(-) diff --git a/src/basic_memory/models/search.py b/src/basic_memory/models/search.py index 74920595..6d3d8daa 100644 --- a/src/basic_memory/models/search.py +++ b/src/basic_memory/models/search.py @@ -5,16 +5,39 @@ from sqlalchemy import DDL # Define FTS5 virtual table creation CREATE_SEARCH_INDEX = DDL(""" CREATE VIRTUAL TABLE IF NOT EXISTS search_index USING fts5( - id UNINDEXED, -- row id of the indexed item - title, -- Title for exact/fuzzy matching - content, -- Additional searchable content - permalink UNINDEXED, -- Link to entity - file_path UNINDEXED, -- Filesystem path - type UNINDEXED, -- entity type - metadata UNINDEXED, -- Additional metadata + -- Core entity fields + id UNINDEXED, -- Row ID + title, -- Title for searching + content, -- Main searchable content + permalink UNINDEXED, -- Stable identifier + file_path UNINDEXED, -- Physical location + type UNINDEXED, -- entity/relation/observation - -- Use unicode61 for basic tokenization with prefix matching - tokenize='unicode61 remove_diacritics 2', - prefix='2,3' -- Enable prefix matching for 2-3 char prefixes + -- Relation fields + from_id UNINDEXED, -- Source entity + to_id UNINDEXED, -- Target entity + relation_type UNINDEXED, -- Type of relation + + -- Observation fields + entity_id UNINDEXED, -- Parent entity + category UNINDEXED, -- Observation category + + -- Common fields + metadata UNINDEXED, -- JSON metadata + created_at UNINDEXED, -- Creation timestamp + updated_at UNINDEXED, -- Last update + + -- Configuration + tokenize='porter unicode61', + prefix='2,3', + content='title content' -- Default searchable columns ); + +-- Add index for relation traversal +CREATE INDEX IF NOT EXISTS idx_search_relations +ON search_index(from_id, to_id) WHERE type = 'relation'; + +-- Add index for type filtering +CREATE INDEX IF NOT EXISTS idx_search_type_perm +ON search_index(type); """) \ No newline at end of file diff --git a/src/basic_memory/schemas/search.py b/src/basic_memory/schemas/search.py index 095e42d7..4502be1c 100644 --- a/src/basic_memory/schemas/search.py +++ b/src/basic_memory/schemas/search.py @@ -9,9 +9,9 @@ from pydantic import BaseModel, field_validator class SearchItemType(str, Enum): """Types of searchable items.""" - DOCUMENT = "document" ENTITY = "entity" - + OBSERVATION = "observation" + RELATION = "relation" class SearchQuery(BaseModel): """Search query parameters.""" @@ -35,11 +35,22 @@ class SearchResult(BaseModel): """Search result item.""" id: int - permalink: str - file_path: str type: SearchItemType score: float + + # Entity Observation-specific fields + permalink: Optional[str] = None + file_path: Optional[str] = None metadata: dict + + # Observation-specific fields + entity_id: Optional[int] = None + category: Optional[str] = None + + # Relation-specific fields + from_id: Optional[int] = None + to_id: Optional[int] = None + relation_type: Optional[str] = None class SearchResponse(BaseModel): diff --git a/src/basic_memory/services/search_service.py b/src/basic_memory/services/search_service.py index ce413cda..d144134b 100644 --- a/src/basic_memory/services/search_service.py +++ b/src/basic_memory/services/search_service.py @@ -75,62 +75,122 @@ class SearchService: ) -> None: """Index an entity's content for search. - Indexes: - - Title and its variations for fuzzy matching - - Path components for better findability + Each type gets its own row in the search index with appropriate metadata + and type-specific fields populated. - Content with context preservation """ - # Generate searchable content with variations content_parts = [] - - # Add title variations title_variants = self._generate_variants(entity.title) content_parts.extend(title_variants) - # Add summary if available if entity.summary: content_parts.append(entity.summary) - # Add permalink variations - permalink_variants = self._generate_variants(entity.permalink) - content_parts.extend(permalink_variants) + content_parts.extend(self._generate_variants(entity.permalink)) + content_parts.extend(self._generate_variants(entity.file_path)) + + entity_content = "\n".join(p for p in content_parts if p and p.strip()) - # Add file path components - path_variants = self._generate_variants(entity.file_path) - content_parts.extend(path_variants) - - # Join all parts and remove empty strings - content = "\n".join(p for p in content_parts if p and p.strip()) - - metadata = { - "entity_type": entity.entity_type, - "created_at": entity.created_at.isoformat(), - "updated_at": entity.updated_at.isoformat(), - } - - # Queue indexing if background_tasks provided + # Index entity if background_tasks: background_tasks.add_task( self._do_index, id=entity.id, title=entity.title, - content=content, + content=entity_content, permalink=entity.permalink, file_path=entity.file_path, type=SearchItemType.ENTITY, - metadata=metadata, + metadata={ + "entity_type": entity.entity_type, + "created_at": entity.created_at.isoformat(), + "updated_at": entity.updated_at.isoformat(), + } ) else: await self._do_index( id=entity.id, title=entity.title, - content=content, + content=entity_content, permalink=entity.permalink, file_path=entity.file_path, type=SearchItemType.ENTITY, - metadata=metadata, + metadata={ + "entity_type": entity.entity_type, + "created_at": entity.created_at.isoformat(), + "updated_at": entity.updated_at.isoformat(), + } ) + # Index each observation + for obs in entity.observations: + if background_tasks: + background_tasks.add_task( + self._do_index, + id=obs.id, + title=f"{obs.category}: {obs.content[:50]}...", + content=obs.content, + permalink=f"{entity.permalink}/observations/{obs.id}", + file_path=entity.file_path, + type=SearchItemType.OBSERVATION, + metadata={ + "entity_id": entity.id, + "category": obs.category, + "created_at": obs.created_at.isoformat(), + "updated_at": obs.updated_at.isoformat(), + "tags": obs.tags + } + ) + else: + await self._do_index( + id=obs.id, + title=f"{obs.category}: {obs.content[:50]}...", + content=obs.content, + permalink=f"{entity.permalink}/observations/{obs.id}", + file_path=entity.file_path, + type=SearchItemType.OBSERVATION, + metadata={ + "entity_id": entity.id, + "category": obs.category, + "created_at": obs.created_at.isoformat(), + "updated_at": obs.updated_at.isoformat(), + "tags": obs.tags + } + ) + + # Index each relation + for rel in entity.relations: + if background_tasks: + background_tasks.add_task( + self._do_index, + id=rel.id, + title=f"{rel.relation_type}", + content=rel.context or "", + permalink=f"{entity.permalink}/relations/{rel.id}", + file_path=entity.file_path, + type=SearchItemType.RELATION, + metadata={ + "from_id": rel.from_id, + "to_id": rel.to_id, + "created_at": rel.created_at.isoformat(), + "updated_at": rel.updated_at.isoformat() + } + ) + else: + await self._do_index( + id=rel.id, + title=f"{rel.relation_type}", + content=rel.context or "", + permalink=f"{entity.permalink}/relations/{rel.id}", + file_path=entity.file_path, + type=SearchItemType.RELATION, + metadata={ + "from_id": rel.from_id, + "to_id": rel.to_id, + "created_at": rel.created_at.isoformat(), + "updated_at": rel.updated_at.isoformat() + } + ) async def _do_index( self, id: int,