fix fuzzy search on note create

This commit is contained in:
phernandez
2025-01-31 16:38:06 -06:00
parent dec1cc0827
commit f0fd1ee942
10 changed files with 104 additions and 68 deletions
@@ -3,6 +3,7 @@
from typing import Annotated
from fastapi import APIRouter, HTTPException, BackgroundTasks, Depends, Query, Response
from loguru import logger
from basic_memory.deps import (
EntityServiceDep,
@@ -26,29 +27,36 @@ router = APIRouter(prefix="/knowledge", tags=["knowledge"])
@router.post("/entities", response_model=EntityResponse)
async def create_entity(
data: Entity,
background_tasks: BackgroundTasks,
entity_service: EntityServiceDep,
search_service: SearchServiceDep,
data: Entity,
background_tasks: BackgroundTasks,
entity_service: EntityServiceDep,
search_service: SearchServiceDep,
) -> EntityResponse:
"""Create an entity."""
logger.info(f"request: create_entity with data={data}")
entity = await entity_service.create_entity(data)
# reindex
await search_service.index_entity(entity, background_tasks=background_tasks)
return EntityResponse.model_validate(entity)
result = EntityResponse.model_validate(entity)
logger.info(f"response: create_entity with result={result}")
return result
@router.put("/entities/{permalink:path}", response_model=EntityResponse)
async def create_or_update_entity(
permalink: PathId,
data: Entity,
response: Response,
background_tasks: BackgroundTasks,
entity_service: EntityServiceDep,
search_service: SearchServiceDep,
permalink: PathId,
data: Entity,
response: Response,
background_tasks: BackgroundTasks,
entity_service: EntityServiceDep,
search_service: SearchServiceDep,
) -> EntityResponse:
"""Create or update an entity. If entity exists, it will be updated, otherwise created."""
logger.info(f"request: create_or_update_entity with permalink={permalink}, data={data}")
# Validate permalink matches
if data.permalink != permalink:
raise HTTPException(status_code=400, detail="Entity permalink must match URL path")
@@ -59,8 +67,10 @@ async def create_or_update_entity(
# reindex
await search_service.index_entity(entity, background_tasks=background_tasks)
result = EntityResponse.model_validate(entity)
return EntityResponse.model_validate(entity)
logger.info(f"response: create_or_update_entity with result={result}, status_code={response.status_code}")
return result
## Read endpoints
@@ -68,8 +78,8 @@ async def create_or_update_entity(
@router.get("/entities/{permalink:path}", response_model=EntityResponse)
async def get_entity(
entity_service: EntityServiceDep,
permalink: str,
entity_service: EntityServiceDep,
permalink: str,
) -> EntityResponse:
"""Get a specific entity by ID.
@@ -78,42 +88,52 @@ async def get_entity(
content: If True, include full file content
:param entity_service: EntityService
"""
logger.info(f"request: get_entity with permalink={permalink}")
try:
entity = await entity_service.get_by_permalink(permalink)
entity_response = EntityResponse.model_validate(entity)
return entity_response
result = EntityResponse.model_validate(entity)
logger.info(f"response: get_entity with result={result}")
return result
except EntityNotFoundError:
logger.error(f"Error: Entity with {permalink} not found")
raise HTTPException(status_code=404, detail=f"Entity with {permalink} not found")
@router.get("/entities", response_model=EntityListResponse)
async def get_entities(
entity_service: EntityServiceDep,
permalink: Annotated[list[str] | None, Query()] = None,
entity_service: EntityServiceDep,
permalink: Annotated[list[str] | None, Query()] = None,
) -> EntityListResponse:
"""Open specific entities"""
# permalink is a list of parameters on the request ?permalink=foo
logger.info(f"request: get_entities with permalinks={permalink}")
entities = await entity_service.get_entities_by_permalinks(permalink)
return EntityListResponse(
result = EntityListResponse(
entities=[EntityResponse.model_validate(entity) for entity in entities]
)
logger.info(f"response: get_entities with result={result}")
return result
## Delete endpoints
@router.delete("/entities/{identifier:path}", response_model=DeleteEntitiesResponse)
async def delete_entity(
identifier: str,
background_tasks: BackgroundTasks,
entity_service: EntityServiceDep,
link_resolver: LinkResolverDep,
search_service=Depends(get_search_service),
identifier: str,
background_tasks: BackgroundTasks,
entity_service: EntityServiceDep,
link_resolver: LinkResolverDep,
search_service=Depends(get_search_service),
) -> DeleteEntitiesResponse:
"""Delete a single entity and remove from search index."""
logger.info(f"request: delete_entity with identifier={identifier}")
entity = await link_resolver.resolve_link(identifier)
if entity is None:
logger.info("response: delete_entity with result=DeleteEntitiesResponse(deleted=False)")
return DeleteEntitiesResponse(deleted=False)
# Delete the entity
@@ -122,17 +142,20 @@ async def delete_entity(
# Remove from search index
background_tasks.add_task(search_service.delete_by_permalink, entity.permalink)
return DeleteEntitiesResponse(deleted=deleted)
result = DeleteEntitiesResponse(deleted=deleted)
logger.info(f"response: delete_entity with result={result}")
return result
@router.post("/entities/delete", response_model=DeleteEntitiesResponse)
async def delete_entities(
data: DeleteEntitiesRequest,
background_tasks: BackgroundTasks,
entity_service: EntityServiceDep,
search_service=Depends(get_search_service),
data: DeleteEntitiesRequest,
background_tasks: BackgroundTasks,
entity_service: EntityServiceDep,
search_service=Depends(get_search_service),
) -> DeleteEntitiesResponse:
"""Delete entities and remove from search index."""
logger.info(f"request: delete_entities with data={data}")
deleted = False
# Remove each deleted entity from search index
@@ -140,4 +163,6 @@ async def delete_entities(
deleted = await entity_service.delete_entity(permalink)
background_tasks.add_task(search_service.delete_by_permalink, permalink)
return DeleteEntitiesResponse(deleted=deleted)
result = DeleteEntitiesResponse(deleted=deleted)
logger.info(f"response: delete_entities with result={result}")
return result
+2 -2
View File
@@ -36,13 +36,13 @@ async def write_note(
Examples:
# Create a simple note
write_note(
title="Meeting Notes: Project Planning",
file_path="Meeting Notes: Project Planning",
content="# Key Points\\n\\n- Discussed timeline\\n- Set priorities"
)
# Create note with tags
write_note(
title="Security Review",
file_path="Security Review",
content="# Findings\\n\\n1. Updated auth flow\\n2. Added rate limiting",
tags=["security", "development"]
)
@@ -93,6 +93,7 @@ class SearchRepository:
search_text: Optional[str] = None,
permalink: Optional[str] = None,
permalink_match: Optional[str] = None,
title: Optional[str] = None,
types: List[SearchItemType] = None,
after_date: datetime = None,
entity_types: List[str] = None,
@@ -108,6 +109,12 @@ class SearchRepository:
params["text"] = f"{search_text}*"
conditions.append("(title MATCH :text OR content MATCH :text)")
# Handle title match search
if title:
title_text = self._quote_search_term(title.lower().strip())
params["text"] = f"{title_text}*"
conditions.append("title MATCH :text")
# Handle permalink exact search
if permalink:
params["permalink"] = permalink
@@ -162,7 +169,7 @@ class SearchRepository:
LIMIT :limit
"""
#logger.debug(f"Search {sql} params: {params}")
logger.debug(f"Search {sql} params: {params}")
async with db.scoped_session(self.session_maker) as session:
result = await session.execute(text(sql), params)
rows = result.fetchall()
@@ -188,7 +195,8 @@ class SearchRepository:
for row in rows
]
#logger.debug(f"Search results: {results}")
for r in results:
logger.debug(f"Search result: type:{r.type} title: {r.title} permalink: {r.permalink} score: {r.score}")
return results
async def index_item(
+1
View File
@@ -38,6 +38,7 @@ class SearchQuery(BaseModel):
permalink: Optional[str] = None # Exact permalink match
permalink_match: Optional[str] = None # Exact permalink match
text: Optional[str] = None # Full-text search
title: Optional[str] = None # title only search
# Optional filters
types: Optional[List[SearchItemType]] = None # Filter by item type
+4 -4
View File
@@ -52,7 +52,7 @@ class EntityService(BaseService[EntityModel]):
if existing:
logger.debug(f"Found existing entity: {existing.permalink}")
return await self.update_entity(schema), False
return await self.update_entity(existing, schema), False
else:
# Create new entity
return await self.create_entity(schema), True
@@ -90,12 +90,12 @@ class EntityService(BaseService[EntityModel]):
return await self.repository.update(entity.id, {"checksum": checksum})
async def update_entity(self, schema: EntitySchema) -> EntityModel:
async def update_entity(self, entity: EntityModel, schema: EntitySchema) -> EntityModel:
"""Update an entity's content and metadata."""
logger.debug(f"Updating entity with permalink: {schema.permalink}")
logger.debug(f"Updating entity with permalink: {entity.permalink}")
# get file path
file_path = Path(schema.file_path)
file_path = Path(entity.file_path)
post = await schema_to_markdown(schema)
+17 -18
View File
@@ -5,7 +5,6 @@ from typing import Optional, Tuple, List
from loguru import logger
from basic_memory.repository.entity_repository import EntityRepository
from basic_memory.services.exceptions import EntityNotFoundError
from basic_memory.services.search_service import SearchService
from basic_memory.models import Entity
from basic_memory.schemas.search import SearchQuery, SearchResult, SearchItemType
@@ -26,10 +25,7 @@ class LinkResolver:
self.entity_repository = entity_repository
self.search_service = search_service
async def resolve_link(
self,
link_text: str,
) -> Optional[Entity]:
async def resolve_link(self, link_text: str, use_search: bool = True) -> Optional[Entity]:
"""Resolve a markdown link to a permalink."""
logger.debug(f"Resolving link: {link_text}")
@@ -45,21 +41,23 @@ class LinkResolver:
# 2. Try exact title match
entity = await self.entity_repository.get_by_title(clean_text)
if entity:
logger.debug(f"Found title match: {entity.permalink}")
logger.debug(f"Found title match: {entity.title}")
return entity
# 3. Fall back to search for fuzzy matching
results = await self.search_service.search(
query=SearchQuery(text=clean_text, types=[SearchItemType.ENTITY]),
)
if results:
# Look for best match
best_match = self._select_best_match(clean_text, results)
logger.debug(f"Selected best match from {len(results)} results: {best_match.permalink}")
return await self.entity_repository.get_by_permalink(best_match.permalink)
# if we couldn't find anything then return None
if use_search:
# 3. Fall back to search for fuzzy matching on title if specified
results = await self.search_service.search(
query=SearchQuery(title=clean_text, types=[SearchItemType.ENTITY]),
)
if results:
# Look for best match
best_match = self._select_best_match(clean_text, results)
logger.debug(f"Selected best match from {len(results)} results: {best_match.permalink}")
return await self.entity_repository.get_by_permalink(best_match.permalink)
# if we couldn't find anything then return None
return None
def _normalize_link_text(self, link_text: str) -> Tuple[str, Optional[str]]:
@@ -124,4 +122,5 @@ class LinkResolver:
# Sort by score (lowest first) and return best
scored_results.sort(key=lambda x: x[0], reverse=True)
return scored_results[0][1]
@@ -72,6 +72,7 @@ class SearchService:
search_text=query.text,
permalink=query.permalink,
permalink_match=query.permalink_match,
title=query.title,
types=query.types,
entity_types=query.entity_types,
after_date=query.after_date,
+2 -2
View File
@@ -133,8 +133,8 @@ class SyncService:
"to_name": target_entity.title # Update to actual title
})
# update search index
await self.search_service.index_entity(target_entity)
# update search index
await self.search_service.index_entity(target_entity)
return changes