admin/basicmachines-co-basic-memory
1
0
Fork 0
mirror of https://github.com/basicmachines-co/basic-memory synced 2026-06-21 13:47:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

revert search service changes

Browse Source
This commit is contained in:
phernandez
2025-01-12 19:56:35 -06:00
parent 94ffd25b66
commit abe80f74b2
4 changed files with 49 additions and 189 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/basic_memory/models/search.py
+5 -4
View File
@@ -5,11 +5,12 @@ from sqlalchemy import DDL
# Define FTS5 virtual table creation
CREATE_SEARCH_INDEX = DDL("""
CREATE VIRTUAL TABLE IF NOT EXISTS search_index USING fts5(
content, -- Searchable text content
permalink UNINDEXED, -- Link to entity/document (must be unique)
title, -- Title for exact/fuzzy matching
content, -- Additional searchable content
permalink UNINDEXED, -- Link to entity/document
file_path UNINDEXED, -- Filesystem path
type UNINDEXED, -- 'entity' or 'document'
metadata UNINDEXED, -- JSON with timestamps, types, etc.
type UNINDEXED, -- entity type
metadata UNINDEXED, -- Additional metadata
tokenize='porter unicode61' -- Enable stemming + unicode
);
""")
src/basic_memory/repository/search_repository.py
+10 -52
View File
@@ -8,8 +8,8 @@ from sqlalchemy import text
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from basic_memory import db
from basic_memory.schemas.search import SearchQuery, SearchResult, SearchItemType
from basic_memory.models.search import CREATE_SEARCH_INDEX
from basic_memory.schemas.search import SearchQuery, SearchResult, SearchItemType
class SearchRepository:
@@ -27,46 +27,13 @@ class SearchRepository:
async def search(
self, query: SearchQuery, context: Optional[List[str]] = None
) -> List[SearchResult]:
"""Search across all indexed content using FTS5.
Uses a three-tier matching strategy:
1. title:term matches (highest priority)
2. Exact term matches (medium priority)
3. Prefix matches (lowest priority)
"""
"""Search across all indexed content."""
conditions = []
params = {}
# Handle text search with fuzzy matching
# Handle text search
if query.text:
# Prepare search terms
search_terms = query.text.lower().split()
params["search_terms"] = search_terms
# Build match conditions in priority order
matches = []
# 1. Title field matches (highest weight)
title_matches = [f'title:"{term}"' for term in search_terms]
if len(search_terms) > 1:
# Multi-word title match
title_phrase = " ".join(f'title:"{term}"' for term in search_terms)
title_matches.append(f'NEAR({title_phrase}, {len(search_terms)})')
matches.extend(title_matches)
# 2. Exact word matches
matches.extend([f'"{term}"' for term in search_terms])
if len(search_terms) > 1:
# Multi-word proximity match
phrase = " ".join(f'"{term}"' for term in search_terms)
matches.append(f'NEAR({phrase}, {len(search_terms) * 2})')
# 3. Prefix matches
matches.extend([f'{term}*' for term in search_terms])
# Complete match expression
match_expr = " OR ".join(matches)
conditions.append(f"content MATCH '{match_expr}'")
conditions.append(f"content MATCH '{query.text}'")
# Handle type filter
if query.types:
@@ -87,26 +54,16 @@ class SearchRepository:
# Build WHERE clause
where_clause = " AND ".join(conditions) if conditions else "1=1"
# Build SQL query
sql = f"""
WITH search_results AS (
SELECT
permalink,
file_path,
type,
metadata,
rank
FROM search_index
WHERE {where_clause}
)
SELECT
permalink,
file_path,
type,
metadata,
rank as score
FROM search_results
ORDER BY rank ASC
bm25(search_index) as score
FROM search_index
WHERE {where_clause}
ORDER BY score DESC
"""
logger.debug(f"Search query: {sql}")
@@ -129,6 +86,7 @@ class SearchRepository:
async def index_item(
self,
title: str,
content: str,
permalink: str,
file_path: str,
@@ -170,4 +128,4 @@ class SearchRepository:
text("DELETE FROM search_index WHERE permalink = :permalink"),
{"permalink": permalink},
)
await session.commit()
await session.commit()
src/basic_memory/services/search_service.py
+15 -6
View File
@@ -50,12 +50,11 @@ class SearchService:
async def index_entity(
self, entity: Entity, background_tasks: Optional[BackgroundTasks] = None
) -> None:
"""Index an entity's core identity for fuzzy matching.
"""Index an entity's content for search.
Core content:
- Original title (lowercase)
- Title with field marker (for weighted matching)
- Path components as discrete terms
Indexes:
- Title in dedicated field for better match control
- Path components in content field for findability
"""
# Build searchable content
content_parts = []
@@ -64,6 +63,7 @@ class SearchService:
title = entity.title.lower()
content_parts.extend([
title, # Base title
entity.summary or "",
f"title:{title}", # Field marker for targeted matching
*title.split(), # Individual words
])
@@ -88,6 +88,7 @@ class SearchService:
if background_tasks:
background_tasks.add_task(
self._do_index,
title=title,
content=content,
permalink=entity.permalink,
file_path=entity.file_path,
@@ -96,6 +97,7 @@ class SearchService:
)
else:
await self._do_index(
title=title,
content=content,
permalink=entity.permalink,
file_path=entity.file_path,
@@ -104,10 +106,17 @@ class SearchService:
)
async def _do_index(
self, content: str, permalink: str, file_path: str, type: SearchItemType, metadata: dict
self,
title: str,
content: str,
permalink: str,
file_path: str,
type: SearchItemType,
metadata: dict,
) -> None:
"""Actually perform the indexing."""
await self.repository.index_item(
title=title,
content=content,
permalink=permalink,
file_path=file_path,
tests/services/test_search_service.py
+19 -127
View File
@@ -1,8 +1,9 @@
"""Tests for search service fuzzy matching."""
import pytest
from datetime import datetime, timezone
import pytest
import pytest_asyncio
from sqlalchemy import text
from basic_memory import db
@@ -10,9 +11,10 @@ from basic_memory.schemas import Entity as EntitySchema
from basic_memory.schemas.search import SearchQuery, SearchItemType
@pytest.fixture
@pytest_asyncio.fixture
def test_entity():
"""Create a test entity"""
class Entity:
id = 1
title = "TestComponent"
@@ -29,130 +31,26 @@ def test_entity():
return Entity()
@pytest.fixture
async def test_entities(search_service):
"""Create a set of test entities with variations."""
entities = [
# Core services
type("Entity", (), {
"id": 1,
"title": "Core Service",
"entity_type": "component",
"permalink": "components/core-service",
"file_path": "components/core-service.md",
"content_type": "text/markdown",
"created_at": datetime.now(timezone.utc),
"updated_at": datetime.now(timezone.utc),
"observations": [],
"relations": [],
"summary": "The core service implementation",
"entity_metadata": {}
})(),
# Auth service
type("Entity", (), {
"id": 2,
"title": "Auth Service",
"entity_type": "component",
"permalink": "components/auth/service",
"file_path": "components/auth/service.md",
"content_type": "text/markdown",
"created_at": datetime.now(timezone.utc),
"updated_at": datetime.now(timezone.utc),
"observations": [],
"relations": [],
"summary": "Authentication service",
"entity_metadata": {}
})(),
# Config service
type("Entity", (), {
"id": 3,
"title": "Service Config",
"entity_type": "config",
"permalink": "config/service-config",
"file_path": "config/service-config.md",
"content_type": "text/markdown",
"created_at": datetime.now(timezone.utc),
"updated_at": datetime.now(timezone.utc),
"observations": [],
"relations": [],
"summary": "Service configuration",
"entity_metadata": {}
})(),
# Features doc
type("Entity", (), {
"id": 4,
"title": "Core Features",
"entity_type": "specs",
"permalink": "specs/features/core",
"file_path": "specs/features/core.md",
"content_type": "text/markdown",
"created_at": datetime.now(timezone.utc),
"updated_at": datetime.now(timezone.utc),
"observations": [],
"relations": [],
"summary": "Core feature specifications",
"entity_metadata": {}
})()
]
# Index all entities
for entity in entities:
await search_service.index_entity(entity)
return entities
def test_document():
"""Create a test document"""
class Document:
id = 1
permalink = "docs/test_doc.md"
file_path = "docs/test_doc.md"
doc_metadata = {"title": "Test Document", "type": "technical"}
created_at = datetime.now(timezone.utc)
updated_at = datetime.now(timezone.utc)
return Document()
@pytest.mark.asyncio
async def test_exact_match(search_service, test_entities):
"""Test exact matching of titles."""
results = await search_service.search(SearchQuery(text="Core Service"))
assert len(results) == 1
assert results[0].permalink == "components/core-service"
@pytest.mark.asyncio
async def test_partial_match(search_service, test_entities):
"""Test partial matching of titles."""
results = await search_service.search(SearchQuery(text="Auth Serv"))
assert len(results) == 1
assert results[0].permalink == "components/auth/service"
@pytest.mark.asyncio
async def test_misspelling_match(search_service, test_entities):
"""Test matching with misspellings."""
results = await search_service.search(SearchQuery(text="Core Servise"))
assert len(results) == 1
assert results[0].permalink == "components/core-service"
@pytest.mark.asyncio
async def test_multiple_results_ranking(search_service, test_entities):
"""Test result ranking with multiple matches."""
# Search for 'service' - should find multiple but rank appropriately
results = await search_service.search(SearchQuery(text="service"))
assert len(results) > 1
# Auth service should rank higher than core service
auth_idx = next(i for i, r in enumerate(results) if r.permalink == "components/auth/service")
core_idx = next(i for i, r in enumerate(results) if r.permalink == "components/core-service")
assert auth_idx < core_idx # Lower index = better rank
@pytest.mark.asyncio
async def test_directory_context(search_service, test_entities):
"""Test matching with directory context."""
results = await search_service.search(
SearchQuery(text="Core"),
context=["components/"] # Should prefer matches in components/
)
assert len(results) > 0
assert results[0].permalink.startswith("components/")
# Original test_init_search_index
@pytest.mark.asyncio
async def test_init_search_index(search_service, session_maker):
"""Test search index initialization"""
# Check that table exists
async with db.scoped_session(session_maker) as session:
result = await session.execute(
text("SELECT name FROM sqlite_master WHERE type='table' AND name='search_index';")
@@ -160,7 +58,6 @@ async def test_init_search_index(search_service, session_maker):
assert result.scalar() == "search_index"
# Original test_index_entity
@pytest.mark.asyncio
async def test_index_entity(search_service, test_entity):
"""Test indexing an entity"""
@@ -173,7 +70,6 @@ async def test_index_entity(search_service, test_entity):
assert results[0].type == SearchItemType.ENTITY
# Original test_search_filtering
@pytest.mark.asyncio
async def test_search_filtering(search_service, test_entity):
"""Test search with filters"""
@@ -190,7 +86,6 @@ async def test_search_filtering(search_service, test_entity):
assert len(results) == 0
# Original test_update_index
@pytest.mark.asyncio
async def test_update_index(search_service, test_entity):
"""Test updating indexed content"""
@@ -205,7 +100,6 @@ async def test_update_index(search_service, test_entity):
assert len(results) == 1
# Original test_search_date_filter
@pytest.mark.asyncio
async def test_search_date_filter(search_service, test_entity):
"""Test searching with date filter"""
@@ -217,7 +111,6 @@ async def test_search_date_filter(search_service, test_entity):
assert len(results) == 0
# Original test_reindex_all
@pytest.mark.asyncio
async def test_reindex_all(search_service, entity_service, session_maker):
"""Test reindexing all content."""
@@ -252,7 +145,6 @@ async def test_reindex_all(search_service, entity_service, session_maker):
assert entity_results[0].type == SearchItemType.ENTITY
# Original test_reindex_with_background_tasks
@pytest.mark.asyncio
async def test_reindex_with_background_tasks(search_service, entity_service, session_maker):
"""Test reindexing with background tasks."""
@@ -284,4 +176,4 @@ async def test_reindex_with_background_tasks(search_service, entity_service, ses
# Verify everything was indexed
all_results = await search_service.search(SearchQuery(text="test"))
assert len(all_results) == 1
assert len(all_results) == 1