admin/basicmachines-co-basic-memory
1
0
Fork 0
mirror of https://github.com/basicmachines-co/basic-memory synced 2026-06-21 13:47:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

add relations and observations to search index

Browse Source
This commit is contained in:
phernandez
2025-01-14 19:06:00 -06:00
parent 9dff24746d
commit 73e8c5bee7
8 changed files with 117 additions and 111 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/basic_memory/models/search.py
+1 -10
View File
@@ -29,15 +29,6 @@ CREATE VIRTUAL TABLE IF NOT EXISTS search_index USING fts5(
-- Configuration
tokenize='porter unicode61',
prefix='2,3',
content='title content' -- Default searchable columns
prefix='2,3'
);
-- Add index for relation traversal
CREATE INDEX IF NOT EXISTS idx_search_relations
ON search_index(from_id, to_id) WHERE type = 'relation';
-- Add index for type filtering
CREATE INDEX IF NOT EXISTS idx_search_type_perm
ON search_index(type);
""")
src/basic_memory/repository/search_repository.py
+21 -7
View File
@@ -26,7 +26,7 @@ class SearchRepository:
def _quote_search_term(self, term: str) -> str:
"""Add quotes if term contains special characters."""
if any(c in term for c in '/-'):
if any(c in term for c in "/-"):
return f'"{term}"'
return term
@@ -112,12 +112,18 @@ class SearchRepository:
# Insert new record
await session.execute(
text("""
INSERT INTO search_index (
id, title, content, permalink, file_path, type, metadata
) VALUES (
:id, :title, :content, :permalink, :file_path, :type, :metadata
)
"""),
INSERT INTO search_index (
id, title, content, permalink, file_path, type, metadata,
from_id, to_id, relation_type,
entity_id, category,
created_at, updated_at
) VALUES (
:id, :title, :content, :permalink, :file_path, :type, :metadata,
:from_id, :to_id, :relation_type,
:entity_id, :category,
:created_at, :updated_at
)
"""),
{
"id": id,
"title": title,
@@ -126,6 +132,14 @@ class SearchRepository:
"file_path": file_path,
"type": type.value,
"metadata": json.dumps(metadata),
# Optional fields based on type
"from_id": metadata.get("from_id"),
"to_id": metadata.get("to_id"),
"relation_type": metadata.get("relation_type"),
"entity_id": metadata.get("entity_id"),
"category": metadata.get("category"),
"created_at": metadata.get("created_at"),
"updated_at": metadata.get("updated_at")
},
)
logger.debug(f"indexed {permalink}")
src/basic_memory/schemas/search.py
+5 -6
View File
@@ -8,11 +8,11 @@ from pydantic import BaseModel, field_validator
class SearchItemType(str, Enum):
"""Types of searchable items."""
ENTITY = "entity"
OBSERVATION = "observation"
RELATION = "relation"
class SearchQuery(BaseModel):
"""Search query parameters."""
text: str
@@ -33,15 +33,15 @@ class SearchQuery(BaseModel):
class SearchResult(BaseModel):
"""Search result item."""
id: int
type: SearchItemType
score: float
metadata: dict
# Entity Observation-specific fields
# File-based fields (optional since observations/relations
# don't have their own files)
permalink: Optional[str] = None
file_path: Optional[str] = None
metadata: dict
# Observation-specific fields
entity_id: Optional[int] = None
@@ -55,5 +55,4 @@ class SearchResult(BaseModel):
class SearchResponse(BaseModel):
"""Wrapper for search results list."""
results: List[SearchResult]
results: List[SearchResult]
src/basic_memory/services/search_service.py
+36 -12
View File
@@ -73,11 +73,22 @@ class SearchService:
async def index_entity(
self, entity: Entity, background_tasks: Optional[BackgroundTasks] = None
) -> None:
"""Index an entity's content for search.
"""Index an entity and all its observations and relations.
Indexing structure:
1. Entities
- permalink: direct from entity (e.g., "specs/search")
- file_path: physical file location
2. Observations
- permalink: entity permalink + /observations/id (e.g., "specs/search/observations/123")
- file_path: parent entity's file (where observation is defined)
3. Relations (only index outgoing relations defined in this file)
- permalink: from_entity/relation_type/to_entity (e.g., "specs/search/implements/features/search-ui")
- file_path: source entity's file (where relation is defined)
Each type gets its own row in the search index with appropriate metadata
and type-specific fields populated.
- Content with context preservation
Each type gets its own row in the search index with appropriate metadata.
"""
content_parts = []
title_variants = self._generate_variants(entity.title)
@@ -122,15 +133,21 @@ class SearchService:
}
)
# Index each observation
# Index each observation with synthetic permalink
for obs in entity.observations:
# Create synthetic permalink for the observation
# We can construct these because observations are always
# defined in and owned by a single entity
observation_permalink = f"{entity.permalink}/observations/{obs.id}"
# Index with parent entity's file path since that's where it's defined
if background_tasks:
background_tasks.add_task(
self._do_index,
id=obs.id,
title=f"{obs.category}: {obs.content[:50]}...",
content=obs.content,
permalink=f"{entity.permalink}/observations/{obs.id}",
permalink=observation_permalink,
file_path=entity.file_path,
type=SearchItemType.OBSERVATION,
metadata={
@@ -146,7 +163,7 @@ class SearchService:
id=obs.id,
title=f"{obs.category}: {obs.content[:50]}...",
content=obs.content,
permalink=f"{entity.permalink}/observations/{obs.id}",
permalink=observation_permalink,
file_path=entity.file_path,
type=SearchItemType.OBSERVATION,
metadata={
@@ -158,15 +175,21 @@ class SearchService:
}
)
# Index each relation
for rel in entity.relations:
# Only index outgoing relations (ones defined in this file)
# Relations are indexed from the source entity's perspective since
# that's where they are defined in the markdown
for rel in entity.outgoing_relations:
# Create relation permalink showing the semantic connection:
# source/relation_type/target
# e.g., "specs/search/implements/features/search-ui"
relation_permalink = f"{rel.from_entity.permalink}/{rel.relation_type}/{rel.to_entity.permalink}"
if background_tasks:
background_tasks.add_task(
self._do_index,
id=rel.id,
title=f"{rel.relation_type}",
content=rel.context or "",
permalink=f"{entity.permalink}/relations/{rel.id}",
permalink=relation_permalink,
file_path=entity.file_path,
type=SearchItemType.RELATION,
metadata={
@@ -181,7 +204,7 @@ class SearchService:
id=rel.id,
title=f"{rel.relation_type}",
content=rel.context or "",
permalink=f"{entity.permalink}/relations/{rel.id}",
permalink=relation_permalink,
file_path=entity.file_path,
type=SearchItemType.RELATION,
metadata={
@@ -191,6 +214,7 @@ class SearchService:
"updated_at": rel.updated_at.isoformat()
}
)
async def _do_index(
self,
id: int,
@@ -214,4 +238,4 @@ class SearchService:
async def delete_by_permalink(self, path_id: str):
"""Delete an item from the search index."""
await self.repository.delete_by_permalink(path_id)
await self.repository.delete_by_permalink(path_id)