mirror of
https://github.com/basicmachines-co/basic-memory
synced 2026-06-21 13:47:35 +00:00
index relations and observations
This commit is contained in:
phernandez
@@ -5,16 +5,39 @@ from sqlalchemy import DDL
|
||||
# Define FTS5 virtual table creation
|
||||
CREATE_SEARCH_INDEX = DDL("""
|
||||
CREATE VIRTUAL TABLE IF NOT EXISTS search_index USING fts5(
|
||||
id UNINDEXED, -- row id of the indexed item
|
||||
title, -- Title for exact/fuzzy matching
|
||||
content, -- Additional searchable content
|
||||
permalink UNINDEXED, -- Link to entity
|
||||
file_path UNINDEXED, -- Filesystem path
|
||||
type UNINDEXED, -- entity type
|
||||
metadata UNINDEXED, -- Additional metadata
|
||||
-- Core entity fields
|
||||
id UNINDEXED, -- Row ID
|
||||
title, -- Title for searching
|
||||
content, -- Main searchable content
|
||||
permalink UNINDEXED, -- Stable identifier
|
||||
file_path UNINDEXED, -- Physical location
|
||||
type UNINDEXED, -- entity/relation/observation
|
||||
|
||||
-- Use unicode61 for basic tokenization with prefix matching
|
||||
tokenize='unicode61 remove_diacritics 2',
|
||||
prefix='2,3' -- Enable prefix matching for 2-3 char prefixes
|
||||
-- Relation fields
|
||||
from_id UNINDEXED, -- Source entity
|
||||
to_id UNINDEXED, -- Target entity
|
||||
relation_type UNINDEXED, -- Type of relation
|
||||
|
||||
-- Observation fields
|
||||
entity_id UNINDEXED, -- Parent entity
|
||||
category UNINDEXED, -- Observation category
|
||||
|
||||
-- Common fields
|
||||
metadata UNINDEXED, -- JSON metadata
|
||||
created_at UNINDEXED, -- Creation timestamp
|
||||
updated_at UNINDEXED, -- Last update
|
||||
|
||||
-- Configuration
|
||||
tokenize='porter unicode61',
|
||||
prefix='2,3',
|
||||
content='title content' -- Default searchable columns
|
||||
);
|
||||
|
||||
-- Add index for relation traversal
|
||||
CREATE INDEX IF NOT EXISTS idx_search_relations
|
||||
ON search_index(from_id, to_id) WHERE type = 'relation';
|
||||
|
||||
-- Add index for type filtering
|
||||
CREATE INDEX IF NOT EXISTS idx_search_type_perm
|
||||
ON search_index(type);
|
||||
""")
|
||||
@@ -9,9 +9,9 @@ from pydantic import BaseModel, field_validator
|
||||
class SearchItemType(str, Enum):
|
||||
"""Types of searchable items."""
|
||||
|
||||
DOCUMENT = "document"
|
||||
ENTITY = "entity"
|
||||
|
||||
OBSERVATION = "observation"
|
||||
RELATION = "relation"
|
||||
|
||||
class SearchQuery(BaseModel):
|
||||
"""Search query parameters."""
|
||||
@@ -35,11 +35,22 @@ class SearchResult(BaseModel):
|
||||
"""Search result item."""
|
||||
|
||||
id: int
|
||||
permalink: str
|
||||
file_path: str
|
||||
type: SearchItemType
|
||||
score: float
|
||||
|
||||
# Entity Observation-specific fields
|
||||
permalink: Optional[str] = None
|
||||
file_path: Optional[str] = None
|
||||
metadata: dict
|
||||
|
||||
# Observation-specific fields
|
||||
entity_id: Optional[int] = None
|
||||
category: Optional[str] = None
|
||||
|
||||
# Relation-specific fields
|
||||
from_id: Optional[int] = None
|
||||
to_id: Optional[int] = None
|
||||
relation_type: Optional[str] = None
|
||||
|
||||
|
||||
class SearchResponse(BaseModel):
|
||||
|
||||
@@ -75,62 +75,122 @@ class SearchService:
|
||||
) -> None:
|
||||
"""Index an entity's content for search.
|
||||
|
||||
Indexes:
|
||||
- Title and its variations for fuzzy matching
|
||||
- Path components for better findability
|
||||
Each type gets its own row in the search index with appropriate metadata
|
||||
and type-specific fields populated.
|
||||
- Content with context preservation
|
||||
"""
|
||||
# Generate searchable content with variations
|
||||
content_parts = []
|
||||
|
||||
# Add title variations
|
||||
title_variants = self._generate_variants(entity.title)
|
||||
content_parts.extend(title_variants)
|
||||
|
||||
# Add summary if available
|
||||
if entity.summary:
|
||||
content_parts.append(entity.summary)
|
||||
|
||||
# Add permalink variations
|
||||
permalink_variants = self._generate_variants(entity.permalink)
|
||||
content_parts.extend(permalink_variants)
|
||||
content_parts.extend(self._generate_variants(entity.permalink))
|
||||
content_parts.extend(self._generate_variants(entity.file_path))
|
||||
|
||||
entity_content = "\n".join(p for p in content_parts if p and p.strip())
|
||||
|
||||
# Add file path components
|
||||
path_variants = self._generate_variants(entity.file_path)
|
||||
content_parts.extend(path_variants)
|
||||
|
||||
# Join all parts and remove empty strings
|
||||
content = "\n".join(p for p in content_parts if p and p.strip())
|
||||
|
||||
metadata = {
|
||||
"entity_type": entity.entity_type,
|
||||
"created_at": entity.created_at.isoformat(),
|
||||
"updated_at": entity.updated_at.isoformat(),
|
||||
}
|
||||
|
||||
# Queue indexing if background_tasks provided
|
||||
# Index entity
|
||||
if background_tasks:
|
||||
background_tasks.add_task(
|
||||
self._do_index,
|
||||
id=entity.id,
|
||||
title=entity.title,
|
||||
content=content,
|
||||
content=entity_content,
|
||||
permalink=entity.permalink,
|
||||
file_path=entity.file_path,
|
||||
type=SearchItemType.ENTITY,
|
||||
metadata=metadata,
|
||||
metadata={
|
||||
"entity_type": entity.entity_type,
|
||||
"created_at": entity.created_at.isoformat(),
|
||||
"updated_at": entity.updated_at.isoformat(),
|
||||
}
|
||||
)
|
||||
else:
|
||||
await self._do_index(
|
||||
id=entity.id,
|
||||
title=entity.title,
|
||||
content=content,
|
||||
content=entity_content,
|
||||
permalink=entity.permalink,
|
||||
file_path=entity.file_path,
|
||||
type=SearchItemType.ENTITY,
|
||||
metadata=metadata,
|
||||
metadata={
|
||||
"entity_type": entity.entity_type,
|
||||
"created_at": entity.created_at.isoformat(),
|
||||
"updated_at": entity.updated_at.isoformat(),
|
||||
}
|
||||
)
|
||||
|
||||
# Index each observation
|
||||
for obs in entity.observations:
|
||||
if background_tasks:
|
||||
background_tasks.add_task(
|
||||
self._do_index,
|
||||
id=obs.id,
|
||||
title=f"{obs.category}: {obs.content[:50]}...",
|
||||
content=obs.content,
|
||||
permalink=f"{entity.permalink}/observations/{obs.id}",
|
||||
file_path=entity.file_path,
|
||||
type=SearchItemType.OBSERVATION,
|
||||
metadata={
|
||||
"entity_id": entity.id,
|
||||
"category": obs.category,
|
||||
"created_at": obs.created_at.isoformat(),
|
||||
"updated_at": obs.updated_at.isoformat(),
|
||||
"tags": obs.tags
|
||||
}
|
||||
)
|
||||
else:
|
||||
await self._do_index(
|
||||
id=obs.id,
|
||||
title=f"{obs.category}: {obs.content[:50]}...",
|
||||
content=obs.content,
|
||||
permalink=f"{entity.permalink}/observations/{obs.id}",
|
||||
file_path=entity.file_path,
|
||||
type=SearchItemType.OBSERVATION,
|
||||
metadata={
|
||||
"entity_id": entity.id,
|
||||
"category": obs.category,
|
||||
"created_at": obs.created_at.isoformat(),
|
||||
"updated_at": obs.updated_at.isoformat(),
|
||||
"tags": obs.tags
|
||||
}
|
||||
)
|
||||
|
||||
# Index each relation
|
||||
for rel in entity.relations:
|
||||
if background_tasks:
|
||||
background_tasks.add_task(
|
||||
self._do_index,
|
||||
id=rel.id,
|
||||
title=f"{rel.relation_type}",
|
||||
content=rel.context or "",
|
||||
permalink=f"{entity.permalink}/relations/{rel.id}",
|
||||
file_path=entity.file_path,
|
||||
type=SearchItemType.RELATION,
|
||||
metadata={
|
||||
"from_id": rel.from_id,
|
||||
"to_id": rel.to_id,
|
||||
"created_at": rel.created_at.isoformat(),
|
||||
"updated_at": rel.updated_at.isoformat()
|
||||
}
|
||||
)
|
||||
else:
|
||||
await self._do_index(
|
||||
id=rel.id,
|
||||
title=f"{rel.relation_type}",
|
||||
content=rel.context or "",
|
||||
permalink=f"{entity.permalink}/relations/{rel.id}",
|
||||
file_path=entity.file_path,
|
||||
type=SearchItemType.RELATION,
|
||||
metadata={
|
||||
"from_id": rel.from_id,
|
||||
"to_id": rel.to_id,
|
||||
"created_at": rel.created_at.isoformat(),
|
||||
"updated_at": rel.updated_at.isoformat()
|
||||
}
|
||||
)
|
||||
async def _do_index(
|
||||
self,
|
||||
id: int,
|
||||
|
||||
Reference in New Issue
Block a user