Compare commits

..

13 Commits

Author SHA1 Message Date
phernandez 4e10e21a6c perf(core): streamline entity write upserts
Signed-off-by: phernandez <paul@basicmachines.co>
2026-04-04 00:59:19 -05:00
Paul Hernandez 69808b23ca perf(core): reuse written note content after writes (#717)
Signed-off-by: phernandez <paul@basicmachines.co>
2026-04-04 00:16:22 -05:00
Paul Hernandez 6f207c20c0 test(api): add recent activity hydration regression coverage (#716)
Signed-off-by: phernandez <paul@basicmachines.co>
2026-04-03 17:30:44 -05:00
Paul Hernandez cff31c5797 feat(cli): support cloud project visibility on add (#715)
Signed-off-by: phernandez <paul@basicmachines.co>
2026-04-03 17:30:09 -05:00
dependabot[bot] 2d1ccfa36c chore(deps): bump the uv group across 1 directory with 2 updates (#697)
Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2026-04-03 17:29:46 -05:00
dependabot[bot] a2e0f935d6 chore(deps): bump picomatch from 4.0.3 to 4.0.4 in /ui/tool-ui-react in the npm_and_yarn group across 1 directory (#696)
Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2026-04-03 17:28:46 -05:00
Paul Hernandez e6b98a15c7 fix(cli): propagate cloud workspace routing and incremental sync (#712)
Signed-off-by: phernandez <paul@basicmachines.co>
2026-04-03 14:25:02 -05:00
Drew Cain 733c4f7514 fix: eliminate N+1 query in search hydrate_results (#713)
Signed-off-by: Drew Cain <groksrc@gmail.com>
Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
2026-04-03 12:54:24 -05:00
Paul Hernandez cfa70004be fix: concurrent delete race conditions in delete_entity (#702)
Signed-off-by: phernandez <paul@basicmachines.co>
Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
2026-03-30 19:57:30 -05:00
phernandez 7696fca826 fix: restore MCP telemetry compatibility and outcomes
Signed-off-by: phernandez <paul@basicmachines.co>
2026-03-28 15:31:13 -05:00
phernandez 98a2a3cbaf Unify MCP telemetry spans across routers and services
Signed-off-by: phernandez <paul@basicmachines.co>
2026-03-28 14:42:18 -05:00
phernandez 01cbad1dbe Allow long relation_type values in responses
Signed-off-by: phernandez <paul@basicmachines.co>
2026-03-28 09:56:22 -05:00
phernandez a4e0422926 perf fixes
Signed-off-by: phernandez <paul@basicmachines.co>
2026-03-27 22:12:44 -05:00
56 changed files with 5389 additions and 1581 deletions
+32 -10
View File
@@ -6,7 +6,6 @@ concurrency:
on:
push:
branches: [ "main" ]
pull_request:
branches: [ "main" ]
@@ -52,7 +51,6 @@ jobs:
test-sqlite-unit:
name: Test SQLite Unit (${{ matrix.os }}, Python ${{ matrix.python-version }})
timeout-minutes: 30
needs: [static-checks]
strategy:
fail-fast: false
matrix:
@@ -99,7 +97,6 @@ jobs:
test-sqlite-integration:
name: Test SQLite Integration (${{ matrix.os }}, Python ${{ matrix.python-version }})
timeout-minutes: 45
needs: [static-checks]
strategy:
fail-fast: false
matrix:
@@ -146,7 +143,6 @@ jobs:
test-postgres-unit:
name: Test Postgres Unit (Python ${{ matrix.python-version }})
timeout-minutes: 30
needs: [static-checks]
strategy:
fail-fast: false
matrix:
@@ -155,8 +151,22 @@ jobs:
- python-version: "3.13"
- python-version: "3.14"
runs-on: ubuntu-latest
# Note: No services section needed - testcontainers handles Postgres in Docker
services:
postgres:
image: pgvector/pgvector:pg16
env:
POSTGRES_USER: basic_memory_user
POSTGRES_PASSWORD: dev_password
POSTGRES_DB: basic_memory_test
ports:
- 5432:5432
options: >-
--health-cmd "pg_isready -U basic_memory_user -d basic_memory_test"
--health-interval 10s
--health-timeout 5s
--health-retries 5
env:
BASIC_MEMORY_TEST_POSTGRES_URL: postgresql://basic_memory_user:dev_password@127.0.0.1:5432/basic_memory_test
steps:
- uses: actions/checkout@v4
@@ -190,7 +200,6 @@ jobs:
test-postgres-integration:
name: Test Postgres Integration (Python ${{ matrix.python-version }})
timeout-minutes: 45
needs: [static-checks]
strategy:
fail-fast: false
matrix:
@@ -199,8 +208,22 @@ jobs:
- python-version: "3.13"
- python-version: "3.14"
runs-on: ubuntu-latest
# Note: No services section needed - testcontainers handles Postgres in Docker
services:
postgres:
image: pgvector/pgvector:pg16
env:
POSTGRES_USER: basic_memory_user
POSTGRES_PASSWORD: dev_password
POSTGRES_DB: basic_memory_test
ports:
- 5432:5432
options: >-
--health-cmd "pg_isready -U basic_memory_user -d basic_memory_test"
--health-interval 10s
--health-timeout 5s
--health-retries 5
env:
BASIC_MEMORY_TEST_POSTGRES_URL: postgresql://basic_memory_user:dev_password@127.0.0.1:5432/basic_memory_test
steps:
- uses: actions/checkout@v4
@@ -234,7 +257,6 @@ jobs:
test-semantic:
name: Test Semantic (Python 3.12)
timeout-minutes: 45
needs: [static-checks]
runs-on: ubuntu-latest
steps:
+4
View File
@@ -442,5 +442,9 @@ With GitHub integration, the development workflow includes:
3. **Branch management** - Claude can create feature branches for implementations
4. **Documentation maintenance** - Claude can keep documentation updated as the code evolves
5. **Code Commits**: ALWAYS sign off commits with `git commit -s`
6. **Pull Request Titles**: PR titles must follow the semantic format enforced by `.github/workflows/pr-title.yml`: `type(scope): summary`
- Allowed types: `feat`, `fix`, `chore`, `docs`, `style`, `refactor`, `perf`, `test`, `build`, `ci`
- Allowed scopes: `core`, `cli`, `api`, `mcp`, `sync`, `ui`, `deps`, `installer`
- Example: `fix(cli): propagate cloud workspace routing`
This level of integration represents a new paradigm in AI-human collaboration, where the AI assistant becomes a full-fledged team member rather than just a tool for generating code snippets.
@@ -13,6 +13,7 @@ Key improvements:
from fastapi import APIRouter, HTTPException, BackgroundTasks, Depends, Response, Path, Query
from loguru import logger
from basic_memory import telemetry
from basic_memory.deps import (
EntityServiceV2ExternalDep,
SearchServiceV2ExternalDep,
@@ -142,47 +143,66 @@ async def resolve_identifier(
"resolution_method": "permalink"
}
"""
logger.info(f"API v2 request: resolve_identifier for '{data.identifier}'")
with telemetry.operation(
"api.request.knowledge.resolve_entity",
entrypoint="api",
domain="knowledge",
action="resolve_entity",
):
logger.info(f"API v2 request: resolve_identifier for '{data.identifier}'")
# Try to resolve by external_id first
entity = await entity_repository.get_by_external_id(data.identifier)
resolution_method = "external_id" if entity else "search"
with telemetry.scope(
"api.knowledge.resolve_entity.lookup_entity",
domain="knowledge",
action="resolve_entity",
phase="lookup_entity",
):
entity = await entity_repository.get_by_external_id(data.identifier)
resolution_method = "external_id" if entity else "search"
# If not found by external_id, try other resolution methods
# Pass source_path for context-aware resolution (prefers notes closer to source)
# Pass strict to control fuzzy search fallback (default False allows fuzzy matching)
if not entity:
entity = await link_resolver.resolve_link(
data.identifier, source_path=data.source_path, strict=data.strict
if not entity:
with telemetry.scope(
"api.knowledge.resolve_entity.resolve_link",
domain="knowledge",
action="resolve_entity",
phase="resolve_link",
):
entity = await link_resolver.resolve_link(
data.identifier, source_path=data.source_path, strict=data.strict
)
if entity:
if entity.permalink == data.identifier:
resolution_method = "permalink"
elif entity.title == data.identifier:
resolution_method = "title"
elif entity.file_path == data.identifier:
resolution_method = "path"
else:
resolution_method = "search"
if not entity:
raise HTTPException(status_code=404, detail=f"Entity not found: '{data.identifier}'")
with telemetry.scope(
"api.knowledge.resolve_entity.shape_response",
domain="knowledge",
action="resolve_entity",
phase="shape_response",
):
result = EntityResolveResponse(
external_id=entity.external_id,
entity_id=entity.id,
permalink=entity.permalink,
file_path=entity.file_path,
title=entity.title,
resolution_method=resolution_method,
)
logger.debug(
f"API v2 response: resolved '{data.identifier}' to external_id={result.external_id} via {resolution_method}"
)
if entity:
# Determine resolution method
if entity.permalink == data.identifier:
resolution_method = "permalink"
elif entity.title == data.identifier:
resolution_method = "title"
elif entity.file_path == data.identifier:
resolution_method = "path"
else:
resolution_method = "search"
if not entity:
raise HTTPException(status_code=404, detail=f"Entity not found: '{data.identifier}'")
result = EntityResolveResponse(
external_id=entity.external_id,
entity_id=entity.id,
permalink=entity.permalink,
file_path=entity.file_path,
title=entity.title,
resolution_method=resolution_method,
)
logger.debug(
f"API v2 response: resolved '{data.identifier}' to external_id={result.external_id} via {resolution_method}"
)
return result
return result
## Read endpoints
@@ -208,18 +228,36 @@ async def get_entity_by_id(
Raises:
HTTPException: 404 if entity not found
"""
logger.info(f"API v2 request: get_entity_by_id entity_id={entity_id}")
with telemetry.operation(
"api.request.knowledge.get_entity",
entrypoint="api",
domain="knowledge",
action="get_entity",
):
logger.info(f"API v2 request: get_entity_by_id entity_id={entity_id}")
entity = await entity_repository.get_by_external_id(entity_id)
if not entity:
raise HTTPException(
status_code=404, detail=f"Entity with external_id '{entity_id}' not found"
)
with telemetry.scope(
"api.knowledge.get_entity.load_entity",
domain="knowledge",
action="get_entity",
phase="load_entity",
):
entity = await entity_repository.get_by_external_id(entity_id)
if not entity:
raise HTTPException(
status_code=404, detail=f"Entity with external_id '{entity_id}' not found"
)
result = EntityResponseV2.model_validate(entity)
logger.info(f"API v2 response: external_id={entity_id}, title='{result.title}'")
with telemetry.scope(
"api.knowledge.get_entity.shape_response",
domain="knowledge",
action="get_entity",
phase="shape_response",
):
result = EntityResponseV2.model_validate(entity)
logger.info(f"API v2 response: external_id={entity_id}, title='{result.title}'")
return result
return result
## Create endpoints
@@ -248,39 +286,92 @@ async def create_entity(
Returns:
Created entity with generated external_id (UUID) and file content
"""
logger.info(
"API v2 request", endpoint="create_entity", note_type=data.note_type, title=data.title
)
if fast:
entity = await entity_service.fast_write_entity(data)
task_scheduler.schedule(
"reindex_entity",
entity_id=entity.id,
project_id=project_id,
)
else:
entity = await entity_service.create_entity(data)
await search_service.index_entity(entity)
_schedule_vector_sync_if_enabled(
task_scheduler=task_scheduler,
app_config=app_config,
entity_id=entity.id,
project_id=project_id,
with telemetry.operation(
"api.request.knowledge.create_entity",
entrypoint="api",
domain="knowledge",
action="create_entity",
fast=fast,
):
logger.info(
"API v2 request", endpoint="create_entity", note_type=data.note_type, title=data.title
)
result = EntityResponseV2.model_validate(entity)
if fast:
result = result.model_copy(update={"observations": [], "relations": []})
with telemetry.scope(
"api.knowledge.create_entity.write_entity",
domain="knowledge",
action="create_entity",
phase="write_entity",
fast=fast,
):
if fast:
entity = await entity_service.fast_write_entity(data)
written_content = None
search_content = None
else:
write_result = await entity_service.create_entity_with_content(data)
entity = write_result.entity
written_content = write_result.content
search_content = write_result.search_content
# Always read and return file content
content = await file_service.read_file_content(entity.file_path)
result = result.model_copy(update={"content": content})
if fast:
with telemetry.scope(
"api.knowledge.create_entity.enqueue_reindex",
domain="knowledge",
action="create_entity",
phase="enqueue_reindex",
fast=fast,
):
task_scheduler.schedule(
"reindex_entity",
entity_id=entity.id,
project_id=project_id,
)
else:
with telemetry.scope(
"api.knowledge.create_entity.search_index",
domain="knowledge",
action="create_entity",
phase="search_index",
):
await search_service.index_entity(entity, content=search_content)
with telemetry.scope(
"api.knowledge.create_entity.vector_sync",
domain="knowledge",
action="create_entity",
phase="vector_sync",
):
_schedule_vector_sync_if_enabled(
task_scheduler=task_scheduler,
app_config=app_config,
entity_id=entity.id,
project_id=project_id,
)
logger.info(
f"API v2 response: endpoint='create_entity' external_id={entity.external_id}, title={result.title}, permalink={result.permalink}, status_code=201"
)
return result
result = EntityResponseV2.model_validate(entity)
if fast:
result = result.model_copy(update={"observations": [], "relations": []})
with telemetry.scope(
"api.knowledge.create_entity.read_content",
domain="knowledge",
action="create_entity",
phase="read_content",
source="file" if fast else "memory",
):
if fast:
content = await file_service.read_file_content(entity.file_path)
else:
# Non-fast writes already captured the markdown in memory. Reuse it here
# instead of re-reading the file; format_on_save is the one config that can
# still make the persisted file diverge because write_file only returns a checksum.
content = written_content
result = result.model_copy(update={"content": content})
logger.info(
f"API v2 response: endpoint='create_entity' external_id={entity.external_id}, title={result.title}, permalink={result.permalink}, status_code=201"
)
return result
## Update endpoints
@@ -315,61 +406,121 @@ async def update_entity_by_id(
Returns:
Updated entity with file content
"""
logger.info(f"API v2 request: update_entity_by_id entity_id={entity_id}")
with telemetry.operation(
"api.request.knowledge.update_entity",
entrypoint="api",
domain="knowledge",
action="update_entity",
fast=fast,
):
logger.info(f"API v2 request: update_entity_by_id entity_id={entity_id}")
# Check if entity exists (external_id is the source of truth for v2)
existing = await entity_repository.get_by_external_id(entity_id)
created = existing is None
with telemetry.scope(
"api.knowledge.update_entity.load_entity",
domain="knowledge",
action="update_entity",
phase="load_entity",
):
existing = await entity_repository.get_by_external_id(entity_id)
created = existing is None
if fast:
entity = await entity_service.fast_write_entity(data, external_id=entity_id)
response.status_code = 200 if existing else 201
task_scheduler.schedule(
"reindex_entity",
entity_id=entity.id,
project_id=project_id,
resolve_relations=created,
)
else:
if existing:
# Update the existing entity in-place to avoid path-based duplication
entity = await entity_service.update_entity(existing, data)
response.status_code = 200
else:
# Create new entity, then bind external_id to the requested UUID
entity = await entity_service.create_entity(data)
if entity.external_id != entity_id:
entity = await entity_repository.update(
entity.id,
{"external_id": entity_id},
with telemetry.scope(
"api.knowledge.update_entity.write_entity",
domain="knowledge",
action="update_entity",
phase="write_entity",
fast=fast,
):
if fast:
entity = await entity_service.fast_write_entity(data, external_id=entity_id)
written_content = None
search_content = None
response.status_code = 200 if existing else 201
else:
if existing:
write_result = await entity_service.update_entity_with_content(existing, data)
entity = write_result.entity
written_content = write_result.content
search_content = write_result.search_content
response.status_code = 200
else:
write_result = await entity_service.create_entity_with_content(data)
entity = write_result.entity
written_content = write_result.content
search_content = write_result.search_content
if entity.external_id != entity_id:
entity = await entity_repository.update(
entity.id,
{"external_id": entity_id},
)
# external_id fixup only changes the DB row. The file content is unchanged,
# so the markdown captured during the write remains valid downstream.
if not entity:
raise HTTPException(
status_code=404,
detail=f"Entity with external_id '{entity_id}' not found",
)
response.status_code = 201
if fast:
with telemetry.scope(
"api.knowledge.update_entity.enqueue_reindex",
domain="knowledge",
action="update_entity",
phase="enqueue_reindex",
fast=fast,
):
task_scheduler.schedule(
"reindex_entity",
entity_id=entity.id,
project_id=project_id,
resolve_relations=created,
)
else:
with telemetry.scope(
"api.knowledge.update_entity.search_index",
domain="knowledge",
action="update_entity",
phase="search_index",
):
await search_service.index_entity(entity, content=search_content)
with telemetry.scope(
"api.knowledge.update_entity.vector_sync",
domain="knowledge",
action="update_entity",
phase="vector_sync",
):
_schedule_vector_sync_if_enabled(
task_scheduler=task_scheduler,
app_config=app_config,
entity_id=entity.id,
project_id=project_id,
)
if not entity:
raise HTTPException(
status_code=404,
detail=f"Entity with external_id '{entity_id}' not found",
)
response.status_code = 201
await search_service.index_entity(entity)
_schedule_vector_sync_if_enabled(
task_scheduler=task_scheduler,
app_config=app_config,
entity_id=entity.id,
project_id=project_id,
result = EntityResponseV2.model_validate(entity)
if fast:
result = result.model_copy(update={"observations": [], "relations": []})
with telemetry.scope(
"api.knowledge.update_entity.read_content",
domain="knowledge",
action="update_entity",
phase="read_content",
source="file" if fast else "memory",
):
if fast:
content = await file_service.read_file_content(entity.file_path)
else:
# Non-fast writes already captured the markdown in memory. Reuse it here
# instead of re-reading the file; format_on_save is the one config that can
# still make the persisted file diverge because write_file only returns a checksum.
content = written_content
result = result.model_copy(update={"content": content})
logger.info(
f"API v2 response: external_id={entity_id}, created={created}, status_code={response.status_code}"
)
result = EntityResponseV2.model_validate(entity)
if fast:
result = result.model_copy(update={"observations": [], "relations": []})
# Always read and return file content
content = await file_service.read_file_content(entity.file_path)
result = result.model_copy(update={"content": content})
logger.info(
f"API v2 response: external_id={entity_id}, created={created}, status_code={response.status_code}"
)
return result
return result
@router.patch("/entities/{entity_id}", response_model=EntityResponseV2)
@@ -401,69 +552,125 @@ async def edit_entity_by_id(
Raises:
HTTPException: 404 if entity not found, 400 if edit fails
"""
logger.info(
f"API v2 request: edit_entity_by_id entity_id={entity_id}, operation='{data.operation}'"
)
# Verify entity exists
entity = await entity_repository.get_by_external_id(entity_id)
if not entity: # pragma: no cover
raise HTTPException(
status_code=404, detail=f"Entity with external_id '{entity_id}' not found"
)
try:
if fast:
updated_entity = await entity_service.fast_edit_entity(
entity=entity,
operation=data.operation,
content=data.content,
section=data.section,
find_text=data.find_text,
expected_replacements=data.expected_replacements,
)
task_scheduler.schedule(
"reindex_entity",
entity_id=updated_entity.id,
project_id=project_id,
)
else:
# Edit using the entity's permalink or path
identifier = entity.permalink or entity.file_path
updated_entity = await entity_service.edit_entity(
identifier=identifier,
operation=data.operation,
content=data.content,
section=data.section,
find_text=data.find_text,
expected_replacements=data.expected_replacements,
)
await search_service.index_entity(updated_entity)
_schedule_vector_sync_if_enabled(
task_scheduler=task_scheduler,
app_config=app_config,
entity_id=updated_entity.id,
project_id=project_id,
)
result = EntityResponseV2.model_validate(updated_entity)
if fast:
result = result.model_copy(update={"observations": [], "relations": []})
# Always read and return file content
content = await file_service.read_file_content(updated_entity.file_path)
result = result.model_copy(update={"content": content})
with telemetry.operation(
"api.request.knowledge.edit_entity",
entrypoint="api",
domain="knowledge",
action="edit_entity",
fast=fast,
):
logger.info(
f"API v2 response: external_id={entity_id}, operation='{data.operation}', status_code=200"
f"API v2 request: edit_entity_by_id entity_id={entity_id}, operation='{data.operation}'"
)
return result
with telemetry.scope(
"api.knowledge.edit_entity.load_entity",
domain="knowledge",
action="edit_entity",
phase="load_entity",
):
entity = await entity_repository.get_by_external_id(entity_id)
if not entity: # pragma: no cover
raise HTTPException(
status_code=404, detail=f"Entity with external_id '{entity_id}' not found"
)
except Exception as e:
logger.error(f"Error editing entity {entity_id}: {e}")
raise HTTPException(status_code=400, detail=str(e))
try:
with telemetry.scope(
"api.knowledge.edit_entity.write_entity",
domain="knowledge",
action="edit_entity",
phase="write_entity",
fast=fast,
):
if fast:
updated_entity = await entity_service.fast_edit_entity(
entity=entity,
operation=data.operation,
content=data.content,
section=data.section,
find_text=data.find_text,
expected_replacements=data.expected_replacements,
)
written_content = None
search_content = None
else:
identifier = entity.permalink or entity.file_path
write_result = await entity_service.edit_entity_with_content(
identifier=identifier,
operation=data.operation,
content=data.content,
section=data.section,
find_text=data.find_text,
expected_replacements=data.expected_replacements,
)
updated_entity = write_result.entity
written_content = write_result.content
search_content = write_result.search_content
if fast:
with telemetry.scope(
"api.knowledge.edit_entity.enqueue_reindex",
domain="knowledge",
action="edit_entity",
phase="enqueue_reindex",
fast=fast,
):
task_scheduler.schedule(
"reindex_entity",
entity_id=updated_entity.id,
project_id=project_id,
)
else:
with telemetry.scope(
"api.knowledge.edit_entity.search_index",
domain="knowledge",
action="edit_entity",
phase="search_index",
):
await search_service.index_entity(updated_entity, content=search_content)
with telemetry.scope(
"api.knowledge.edit_entity.vector_sync",
domain="knowledge",
action="edit_entity",
phase="vector_sync",
):
_schedule_vector_sync_if_enabled(
task_scheduler=task_scheduler,
app_config=app_config,
entity_id=updated_entity.id,
project_id=project_id,
)
result = EntityResponseV2.model_validate(updated_entity)
if fast:
result = result.model_copy(update={"observations": [], "relations": []})
with telemetry.scope(
"api.knowledge.edit_entity.read_content",
domain="knowledge",
action="edit_entity",
phase="read_content",
source="file" if fast else "memory",
):
if fast:
content = await file_service.read_file_content(updated_entity.file_path)
else:
# Non-fast writes already captured the markdown in memory. Reuse it here
# instead of re-reading the file; format_on_save is the one config that can
# still make the persisted file diverge because write_file only returns a checksum.
content = written_content
result = result.model_copy(update={"content": content})
logger.info(
f"API v2 response: external_id={entity_id}, operation='{data.operation}', status_code=200"
)
return result
except Exception as e:
logger.error(f"Error editing entity {entity_id}: {e}")
raise HTTPException(status_code=400, detail=str(e))
## Delete endpoints
@@ -9,6 +9,7 @@ from typing import Annotated, Optional
from fastapi import APIRouter, Query, Path
from loguru import logger
from basic_memory import telemetry
from basic_memory.deps import ContextServiceV2ExternalDep, EntityRepositoryV2ExternalDep
from basic_memory.schemas.base import TimeFrame, parse_timeframe
from basic_memory.schemas.memory import (
@@ -50,30 +51,55 @@ async def recent(
Returns:
GraphContext with recent activity and related entities
"""
# return all types by default
types = (
[SearchItemType.ENTITY, SearchItemType.RELATION, SearchItemType.OBSERVATION]
if not type
else type
)
with telemetry.operation(
"api.request.memory.recent_activity",
entrypoint="api",
domain="memory",
action="recent_activity",
page=page,
page_size=page_size,
):
types = (
[SearchItemType.ENTITY, SearchItemType.RELATION, SearchItemType.OBSERVATION]
if not type
else type
)
logger.debug(
f"V2 Getting recent context for project {project_id}: `{types}` depth: `{depth}` timeframe: `{timeframe}` page: `{page}` page_size: `{page_size}` max_related: `{max_related}`"
)
# Parse timeframe
since = parse_timeframe(timeframe)
limit = page_size
offset = (page - 1) * page_size
logger.debug(
f"V2 Getting recent context for project {project_id}: `{types}` depth: `{depth}` timeframe: `{timeframe}` page: `{page}` page_size: `{page_size}` max_related: `{max_related}`"
)
since = parse_timeframe(timeframe)
limit = page_size
offset = (page - 1) * page_size
# Build context
context = await context_service.build_context(
types=types, depth=depth, since=since, limit=limit, offset=offset, max_related=max_related
)
recent_context = await to_graph_context(
context, entity_repository=entity_repository, page=page, page_size=page_size
)
logger.debug(f"V2 Recent context: {recent_context.model_dump_json()}")
return recent_context
with telemetry.scope(
"api.memory.recent_activity.build_context",
domain="memory",
action="recent_activity",
phase="build_context",
page=page,
page_size=page_size,
):
context = await context_service.build_context(
types=types,
depth=depth,
since=since,
limit=limit,
offset=offset,
max_related=max_related,
)
with telemetry.scope(
"api.memory.recent_activity.shape_response",
domain="memory",
action="recent_activity",
phase="shape_response",
result_count=len(context.results),
):
recent_context = await to_graph_context(
context, entity_repository=entity_repository, page=page, page_size=page_size
)
logger.debug(f"V2 Recent context: {recent_context.model_dump_json()}")
return recent_context
# get_memory_context needs to be declared last so other paths can match
@@ -111,20 +137,46 @@ async def get_memory_context(
Returns:
GraphContext with the entity and its related context
"""
logger.debug(
f"V2 Getting context for project {project_id}, URI: `{uri}` depth: `{depth}` timeframe: `{timeframe}` page: `{page}` page_size: `{page_size}` max_related: `{max_related}`"
)
memory_url = normalize_memory_url(uri)
with telemetry.operation(
"api.request.memory.build_context",
entrypoint="api",
domain="memory",
action="build_context",
page=page,
page_size=page_size,
):
logger.debug(
f"V2 Getting context for project {project_id}, URI: `{uri}` depth: `{depth}` timeframe: `{timeframe}` page: `{page}` page_size: `{page_size}` max_related: `{max_related}`"
)
memory_url = normalize_memory_url(uri)
# Parse timeframe
since = parse_timeframe(timeframe) if timeframe else None
limit = page_size
offset = (page - 1) * page_size
since = parse_timeframe(timeframe) if timeframe else None
limit = page_size
offset = (page - 1) * page_size
# Build context
context = await context_service.build_context(
memory_url, depth=depth, since=since, limit=limit, offset=offset, max_related=max_related
)
return await to_graph_context(
context, entity_repository=entity_repository, page=page, page_size=page_size
)
with telemetry.scope(
"api.memory.build_context.build_context",
domain="memory",
action="build_context",
phase="build_context",
page=page,
page_size=page_size,
):
context = await context_service.build_context(
memory_url,
depth=depth,
since=since,
limit=limit,
offset=offset,
max_related=max_related,
)
with telemetry.scope(
"api.memory.build_context.shape_response",
domain="memory",
action="build_context",
phase="shape_response",
result_count=len(context.results),
):
return await to_graph_context(
context, entity_repository=entity_repository, page=page, page_size=page_size
)
+226 -164
View File
@@ -15,6 +15,7 @@ from pathlib import Path as PathLib
from fastapi import APIRouter, HTTPException, Response, Path
from loguru import logger
from basic_memory import telemetry
from basic_memory.deps import (
ProjectConfigV2ExternalDep,
FileServiceV2ExternalDep,
@@ -55,36 +56,62 @@ async def get_resource_content(
Raises:
HTTPException: 404 if entity or file not found
"""
logger.debug(f"V2 Getting content for project {project_id}, entity_id: {entity_id}")
with telemetry.operation(
"api.request.resource.get_content",
entrypoint="api",
domain="resource",
action="get_content",
):
logger.debug(f"V2 Getting content for project {project_id}, entity_id: {entity_id}")
# Get entity by external_id
entity = await entity_repository.get_by_external_id(entity_id)
if not entity:
raise HTTPException(status_code=404, detail=f"Entity {entity_id} not found")
with telemetry.scope(
"api.resource.get_content.load_entity",
domain="resource",
action="get_content",
phase="load_entity",
):
entity = await entity_repository.get_by_external_id(entity_id)
if not entity:
raise HTTPException(status_code=404, detail=f"Entity {entity_id} not found")
# Validate entity file path to prevent path traversal
project_path = PathLib(config.home)
if not validate_project_path(entity.file_path, project_path):
logger.error( # pragma: no cover
f"Invalid file path in entity {entity.id}: {entity.file_path}"
)
raise HTTPException( # pragma: no cover
status_code=500,
detail="Entity contains invalid file path",
)
with telemetry.scope(
"api.resource.get_content.validate_path",
domain="resource",
action="get_content",
phase="validate_path",
):
project_path = PathLib(config.home)
if not validate_project_path(entity.file_path, project_path):
logger.error( # pragma: no cover
f"Invalid file path in entity {entity.id}: {entity.file_path}"
)
raise HTTPException( # pragma: no cover
status_code=500,
detail="Entity contains invalid file path",
)
# Check file exists via file_service (for cloud compatibility)
if not await file_service.exists(entity.file_path):
raise HTTPException( # pragma: no cover
status_code=404,
detail=f"File not found: {entity.file_path}",
)
with telemetry.scope(
"api.resource.get_content.ensure_exists",
domain="resource",
action="get_content",
phase="ensure_exists",
):
if not await file_service.exists(entity.file_path):
raise HTTPException( # pragma: no cover
status_code=404,
detail=f"File not found: {entity.file_path}",
)
# Read content via file_service as bytes (works with both local and S3)
content = await file_service.read_file_bytes(entity.file_path)
content_type = file_service.content_type(entity.file_path)
with telemetry.scope(
"api.resource.get_content.read_content",
domain="resource",
action="get_content",
phase="read_content",
):
content = await file_service.read_file_bytes(entity.file_path)
content_type = file_service.content_type(entity.file_path)
return Response(content=content, media_type=content_type)
return Response(content=content, media_type=content_type)
@router.post("", response_model=ResourceResponse)
@@ -112,74 +139,94 @@ async def create_resource(
Raises:
HTTPException: 400 for invalid file paths, 409 if file already exists
"""
try:
# Validate path to prevent path traversal attacks
project_path = PathLib(config.home)
if not validate_project_path(data.file_path, project_path):
logger.warning(
f"Invalid file path attempted: {data.file_path} in project {config.name}"
with telemetry.operation(
"api.request.resource.create",
entrypoint="api",
domain="resource",
action="create",
):
try:
# Validate path to prevent path traversal attacks
project_path = PathLib(config.home)
if not validate_project_path(data.file_path, project_path):
logger.warning(
f"Invalid file path attempted: {data.file_path} in project {config.name}"
)
raise HTTPException(
status_code=400,
detail=f"Invalid file path: {data.file_path}. "
"Path must be relative and stay within project boundaries.",
)
existing_entity = await entity_repository.get_by_file_path(data.file_path)
if existing_entity:
raise HTTPException(
status_code=409,
detail=f"Resource already exists at {data.file_path} with entity_id {existing_entity.external_id}. "
f"Use PUT /resource/{existing_entity.external_id} to update it.",
)
with telemetry.scope(
"api.resource.create.write_file",
domain="resource",
action="create",
phase="write_file",
):
await file_service.ensure_directory(PathLib(data.file_path).parent)
checksum = await file_service.write_file(data.file_path, data.content)
with telemetry.scope(
"api.resource.create.read_metadata",
domain="resource",
action="create",
phase="read_metadata",
):
file_metadata = await file_service.get_file_metadata(data.file_path)
file_name = PathLib(data.file_path).name
content_type = file_service.content_type(data.file_path)
note_type = "canvas" if data.file_path.endswith(".canvas") else "file"
entity = EntityModel(
external_id=str(uuid.uuid4()),
title=file_name,
note_type=note_type,
content_type=content_type,
file_path=data.file_path,
checksum=checksum,
created_at=file_metadata.created_at,
updated_at=file_metadata.modified_at,
)
raise HTTPException(
status_code=400,
detail=f"Invalid file path: {data.file_path}. "
"Path must be relative and stay within project boundaries.",
with telemetry.scope(
"api.resource.create.upsert_entity",
domain="resource",
action="create",
phase="upsert_entity",
):
entity = await entity_repository.add(entity)
with telemetry.scope(
"api.resource.create.search_index",
domain="resource",
action="create",
phase="search_index",
):
await search_service.index_entity(entity) # pyright: ignore
return ResourceResponse(
entity_id=entity.id,
external_id=entity.external_id,
file_path=data.file_path,
checksum=checksum,
size=file_metadata.size,
created_at=file_metadata.created_at.timestamp(),
modified_at=file_metadata.modified_at.timestamp(),
)
# Check if entity already exists
existing_entity = await entity_repository.get_by_file_path(data.file_path)
if existing_entity:
raise HTTPException(
status_code=409,
detail=f"Resource already exists at {data.file_path} with entity_id {existing_entity.external_id}. "
f"Use PUT /resource/{existing_entity.external_id} to update it.",
)
# Cloud compatibility: avoid assuming a local filesystem path.
# Delegate directory creation + writes to FileService (local or S3).
await file_service.ensure_directory(PathLib(data.file_path).parent)
checksum = await file_service.write_file(data.file_path, data.content)
# Get file info
file_metadata = await file_service.get_file_metadata(data.file_path)
# Determine file details
file_name = PathLib(data.file_path).name
content_type = file_service.content_type(data.file_path)
note_type = "canvas" if data.file_path.endswith(".canvas") else "file"
# Create a new entity model
# Explicitly set external_id to ensure NOT NULL constraint is satisfied (fixes #512)
entity = EntityModel(
external_id=str(uuid.uuid4()),
title=file_name,
note_type=note_type,
content_type=content_type,
file_path=data.file_path,
checksum=checksum,
created_at=file_metadata.created_at,
updated_at=file_metadata.modified_at,
)
entity = await entity_repository.add(entity)
# Index the file for search
await search_service.index_entity(entity) # pyright: ignore
# Return success response
return ResourceResponse(
entity_id=entity.id,
external_id=entity.external_id,
file_path=data.file_path,
checksum=checksum,
size=file_metadata.size,
created_at=file_metadata.created_at.timestamp(),
modified_at=file_metadata.modified_at.timestamp(),
)
except HTTPException:
# Re-raise HTTP exceptions without wrapping
raise
except Exception as e: # pragma: no cover
logger.error(f"Error creating resource {data.file_path}: {e}")
raise HTTPException(status_code=500, detail=f"Failed to create resource: {str(e)}")
except HTTPException:
raise
except Exception as e: # pragma: no cover
logger.error(f"Error creating resource {data.file_path}: {e}")
raise HTTPException(status_code=500, detail=f"Failed to create resource: {str(e)}")
@router.put("/{entity_id}", response_model=ResourceResponse)
@@ -211,79 +258,94 @@ async def update_resource(
Raises:
HTTPException: 404 if entity not found, 400 for invalid paths
"""
try:
# Get existing entity by external_id
entity = await entity_repository.get_by_external_id(entity_id)
if not entity:
raise HTTPException(status_code=404, detail=f"Entity {entity_id} not found")
with telemetry.operation(
"api.request.resource.update",
entrypoint="api",
domain="resource",
action="update",
):
try:
entity = await entity_repository.get_by_external_id(entity_id)
if not entity:
raise HTTPException(status_code=404, detail=f"Entity {entity_id} not found")
# Determine target file path
target_file_path = data.file_path if data.file_path else entity.file_path
target_file_path = data.file_path if data.file_path else entity.file_path
# Validate path to prevent path traversal attacks
project_path = PathLib(config.home)
if not validate_project_path(target_file_path, project_path):
logger.warning(
f"Invalid file path attempted: {target_file_path} in project {config.name}"
project_path = PathLib(config.home)
if not validate_project_path(target_file_path, project_path):
logger.warning(
f"Invalid file path attempted: {target_file_path} in project {config.name}"
)
raise HTTPException(
status_code=400,
detail=f"Invalid file path: {target_file_path}. "
"Path must be relative and stay within project boundaries.",
)
with telemetry.scope(
"api.resource.update.write_file",
domain="resource",
action="update",
phase="write_file",
):
if data.file_path and data.file_path != entity.file_path:
await file_service.ensure_directory(PathLib(target_file_path).parent)
if await file_service.exists(entity.file_path):
await file_service.delete_file(entity.file_path)
else:
await file_service.ensure_directory(PathLib(target_file_path).parent)
checksum = await file_service.write_file(target_file_path, data.content)
with telemetry.scope(
"api.resource.update.read_metadata",
domain="resource",
action="update",
phase="read_metadata",
):
file_metadata = await file_service.get_file_metadata(target_file_path)
file_name = PathLib(target_file_path).name
content_type = file_service.content_type(target_file_path)
note_type = "canvas" if target_file_path.endswith(".canvas") else "file"
with telemetry.scope(
"api.resource.update.update_entity",
domain="resource",
action="update",
phase="update_entity",
):
updated_entity = await entity_repository.update(
entity.id,
{
"title": file_name,
"note_type": note_type,
"content_type": content_type,
"file_path": target_file_path,
"checksum": checksum,
"updated_at": file_metadata.modified_at,
},
)
with telemetry.scope(
"api.resource.update.search_index",
domain="resource",
action="update",
phase="search_index",
):
await search_service.index_entity(updated_entity) # pyright: ignore
return ResourceResponse(
entity_id=entity.id,
external_id=entity.external_id,
file_path=target_file_path,
checksum=checksum,
size=file_metadata.size,
created_at=file_metadata.created_at.timestamp(),
modified_at=file_metadata.modified_at.timestamp(),
)
raise HTTPException(
status_code=400,
detail=f"Invalid file path: {target_file_path}. "
"Path must be relative and stay within project boundaries.",
)
# If moving file, handle the move
if data.file_path and data.file_path != entity.file_path:
# Ensure new parent directory exists (no-op for S3)
await file_service.ensure_directory(PathLib(target_file_path).parent)
# If old file exists, remove it via file_service (for cloud compatibility)
if await file_service.exists(entity.file_path):
await file_service.delete_file(entity.file_path)
else:
# Ensure directory exists for in-place update
await file_service.ensure_directory(PathLib(target_file_path).parent)
# Write content to target file
checksum = await file_service.write_file(target_file_path, data.content)
# Get file info
file_metadata = await file_service.get_file_metadata(target_file_path)
# Determine file details
file_name = PathLib(target_file_path).name
content_type = file_service.content_type(target_file_path)
note_type = "canvas" if target_file_path.endswith(".canvas") else "file"
# Update entity using internal ID
updated_entity = await entity_repository.update(
entity.id,
{
"title": file_name,
"note_type": note_type,
"content_type": content_type,
"file_path": target_file_path,
"checksum": checksum,
"updated_at": file_metadata.modified_at,
},
)
# Index the updated file for search
await search_service.index_entity(updated_entity) # pyright: ignore
# Return success response
return ResourceResponse(
entity_id=entity.id,
external_id=entity.external_id,
file_path=target_file_path,
checksum=checksum,
size=file_metadata.size,
created_at=file_metadata.created_at.timestamp(),
modified_at=file_metadata.modified_at.timestamp(),
)
except HTTPException:
# Re-raise HTTP exceptions without wrapping
raise
except Exception as e: # pragma: no cover
logger.error(f"Error updating resource {entity_id}: {e}")
raise HTTPException(status_code=500, detail=f"Failed to update resource: {str(e)}")
except HTTPException:
raise
except Exception as e: # pragma: no cover
logger.error(f"Error updating resource {entity_id}: {e}")
raise HTTPException(status_code=500, detail=f"Failed to update resource: {str(e)}")
@@ -51,18 +51,31 @@ async def search(
with telemetry.operation(
"api.request.search",
entrypoint="api",
domain="search",
action="search",
page=page,
page_size=page_size,
retrieval_mode=query.retrieval_mode.value,
has_text_query=bool(query.text and query.text.strip()),
has_title_query=bool(query.title),
has_permalink_query=bool(query.permalink or query.permalink_match),
has_query=bool(
(query.text and query.text.strip())
or query.title
or query.permalink
or query.permalink_match
),
has_filters=bool(query.note_types or query.entity_types or query.metadata_filters),
):
offset = (page - 1) * page_size
# Fetch one extra item to detect whether more pages exist (N+1 trick)
fetch_limit = page_size + 1
try:
results = await search_service.search(query, limit=fetch_limit, offset=offset)
with telemetry.scope(
"api.search.search.execute_query",
domain="search",
action="search",
phase="execute_query",
page=page,
page_size=page_size,
):
results = await search_service.search(query, limit=fetch_limit, offset=offset)
except SemanticSearchDisabledError as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc
except SemanticDependenciesMissingError as exc:
@@ -70,17 +83,38 @@ async def search(
except ValueError as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc
has_more = len(results) > page_size
if has_more:
results = results[:page_size]
with telemetry.scope(
"api.search.search.paginate_results",
domain="search",
action="search",
phase="paginate_results",
result_count=len(results),
):
has_more = len(results) > page_size
if has_more:
results = results[:page_size]
search_results = await to_search_results(entity_service, results)
return SearchResponse(
results=search_results,
current_page=page,
page_size=page_size,
has_more=has_more,
)
with telemetry.scope(
"api.search.search.hydrate_results",
domain="search",
action="search",
phase="hydrate_results",
result_count=len(results),
):
search_results = await to_search_results(entity_service, results)
with telemetry.scope(
"api.search.search.build_response",
domain="search",
action="search",
phase="build_response",
result_count=len(search_results),
):
return SearchResponse(
results=search_results,
current_page=page,
page_size=page_size,
has_more=has_more,
)
@router.post("/search/reindex")
+215 -158
View File
@@ -1,5 +1,7 @@
from typing import Optional, List
from basic_memory import telemetry
from basic_memory.models import Entity as EntityModel
from basic_memory.repository import EntityRepository
from basic_memory.repository.search_repository import SearchIndexRow
from basic_memory.schemas.memory import (
@@ -24,169 +26,224 @@ async def to_graph_context(
page: Optional[int] = None,
page_size: Optional[int] = None,
):
# First pass: collect all entity IDs needed for external_id lookup
# This includes: entity primary results, observation parent entities, relation from/to entities
entity_ids_needed: set[int] = set()
for context_item in context_result.results:
for item in (
[context_item.primary_result] + context_item.observations + context_item.related_results
):
if item.type == SearchItemType.ENTITY:
# Entity's own ID for its external_id
entity_ids_needed.add(item.id)
elif item.type == SearchItemType.OBSERVATION:
# Parent entity ID for entity_external_id
if item.entity_id: # pyright: ignore
entity_ids_needed.add(item.entity_id) # pyright: ignore
elif item.type == SearchItemType.RELATION:
# Source and target entity IDs for external_ids
if item.from_id: # pyright: ignore
entity_ids_needed.add(item.from_id) # pyright: ignore
if item.to_id:
entity_ids_needed.add(item.to_id)
# Batch fetch all entities at once - get both title and external_id
entity_title_lookup: dict[int, str] = {}
entity_external_id_lookup: dict[int, str] = {}
if entity_ids_needed:
entities = await entity_repository.find_by_ids(list(entity_ids_needed))
for e in entities:
entity_title_lookup[e.id] = e.title
entity_external_id_lookup[e.id] = e.external_id
# Helper function to convert items to summaries
def to_summary(item: SearchIndexRow | ContextResultRow):
match item.type:
case SearchItemType.ENTITY:
return EntitySummary(
external_id=entity_external_id_lookup.get(item.id, ""),
entity_id=item.id,
title=item.title, # pyright: ignore
permalink=item.permalink,
content=item.content,
file_path=item.file_path,
created_at=item.created_at,
)
case SearchItemType.OBSERVATION:
entity_ext_id = None
if item.entity_id: # pyright: ignore
entity_ext_id = entity_external_id_lookup.get(item.entity_id) # pyright: ignore
return ObservationSummary(
observation_id=item.id,
entity_id=item.entity_id, # pyright: ignore
entity_external_id=entity_ext_id,
title=entity_title_lookup.get(item.entity_id), # pyright: ignore
file_path=item.file_path,
category=item.category, # pyright: ignore
content=item.content, # pyright: ignore
permalink=item.permalink, # pyright: ignore
created_at=item.created_at,
)
case SearchItemType.RELATION:
from_title = entity_title_lookup.get(item.from_id) if item.from_id else None # pyright: ignore
to_title = entity_title_lookup.get(item.to_id) if item.to_id else None
from_ext_id = entity_external_id_lookup.get(item.from_id) if item.from_id else None # pyright: ignore
to_ext_id = entity_external_id_lookup.get(item.to_id) if item.to_id else None
return RelationSummary(
relation_id=item.id,
entity_id=item.entity_id, # pyright: ignore
title=item.title, # pyright: ignore
file_path=item.file_path,
permalink=item.permalink, # pyright: ignore
relation_type=item.relation_type, # pyright: ignore
from_entity=from_title,
from_entity_id=item.from_id, # pyright: ignore
from_entity_external_id=from_ext_id,
to_entity=to_title,
to_entity_id=item.to_id,
to_entity_external_id=to_ext_id,
created_at=item.created_at,
)
case _: # pragma: no cover
raise ValueError(f"Unexpected type: {item.type}")
# Process the hierarchical results
hierarchical_results = []
for context_item in context_result.results:
# Process primary result
primary_result = to_summary(context_item.primary_result)
# Process observations (always ObservationSummary, validated by context_service)
observations = [to_summary(obs) for obs in context_item.observations]
# Process related results
related = [to_summary(rel) for rel in context_item.related_results]
# Add to hierarchical results
hierarchical_results.append(
ContextResult(
primary_result=primary_result,
observations=observations, # pyright: ignore[reportArgumentType]
related_results=related,
)
)
# Create schema metadata from service metadata
metadata = MemoryMetadata(
uri=context_result.metadata.uri,
types=context_result.metadata.types,
depth=context_result.metadata.depth,
timeframe=context_result.metadata.timeframe,
generated_at=context_result.metadata.generated_at,
primary_count=context_result.metadata.primary_count,
related_count=context_result.metadata.related_count,
total_results=context_result.metadata.primary_count + context_result.metadata.related_count,
total_relations=context_result.metadata.total_relations,
total_observations=context_result.metadata.total_observations,
)
# Return new GraphContext with just hierarchical results
return GraphContext(
results=hierarchical_results,
metadata=metadata,
with telemetry.scope(
"memory.hydrate_context",
domain="memory",
action="build_context",
phase="hydrate_context",
page=page,
page_size=page_size,
has_more=context_result.metadata.has_more,
)
result_count=len(context_result.results),
):
# First pass: collect all entity IDs needed for external_id lookup
# This includes: entity primary results, observation parent entities, relation from/to entities
entity_ids_needed: set[int] = set()
for context_item in context_result.results:
for item in (
[context_item.primary_result]
+ context_item.observations
+ context_item.related_results
):
if item.type == SearchItemType.ENTITY:
# Entity's own ID for its external_id
entity_ids_needed.add(item.id)
elif item.type == SearchItemType.OBSERVATION:
# Parent entity ID for entity_external_id
if item.entity_id: # pyright: ignore
entity_ids_needed.add(item.entity_id) # pyright: ignore
elif item.type == SearchItemType.RELATION:
# Source and target entity IDs for external_ids
if item.from_id: # pyright: ignore
entity_ids_needed.add(item.from_id) # pyright: ignore
if item.to_id:
entity_ids_needed.add(item.to_id)
# Batch fetch all entities at once - get both title and external_id
entity_title_lookup: dict[int, str] = {}
entity_external_id_lookup: dict[int, str] = {}
if entity_ids_needed:
with telemetry.scope(
"memory.hydrate_context.lookup_entities",
domain="memory",
action="build_context",
phase="lookup_entities",
result_count=len(entity_ids_needed),
):
entities = await entity_repository.find_by_ids(list(entity_ids_needed))
for e in entities:
entity_title_lookup[e.id] = e.title
entity_external_id_lookup[e.id] = e.external_id
# Helper function to convert items to summaries
def to_summary(item: SearchIndexRow | ContextResultRow):
match item.type:
case SearchItemType.ENTITY:
return EntitySummary(
external_id=entity_external_id_lookup.get(item.id, ""),
entity_id=item.id,
title=item.title, # pyright: ignore
permalink=item.permalink,
content=item.content,
file_path=item.file_path,
created_at=item.created_at,
)
case SearchItemType.OBSERVATION:
entity_ext_id = None
if item.entity_id: # pyright: ignore
entity_ext_id = entity_external_id_lookup.get(item.entity_id) # pyright: ignore
return ObservationSummary(
observation_id=item.id,
entity_id=item.entity_id, # pyright: ignore
entity_external_id=entity_ext_id,
title=entity_title_lookup.get(item.entity_id), # pyright: ignore
file_path=item.file_path,
category=item.category, # pyright: ignore
content=item.content, # pyright: ignore
permalink=item.permalink, # pyright: ignore
created_at=item.created_at,
)
case SearchItemType.RELATION:
from_title = entity_title_lookup.get(item.from_id) if item.from_id else None # pyright: ignore
to_title = entity_title_lookup.get(item.to_id) if item.to_id else None
from_ext_id = (
entity_external_id_lookup.get(item.from_id) if item.from_id else None
) # pyright: ignore
to_ext_id = entity_external_id_lookup.get(item.to_id) if item.to_id else None
return RelationSummary(
relation_id=item.id,
entity_id=item.entity_id, # pyright: ignore
title=item.title, # pyright: ignore
file_path=item.file_path,
permalink=item.permalink, # pyright: ignore
relation_type=item.relation_type, # pyright: ignore
from_entity=from_title,
from_entity_id=item.from_id, # pyright: ignore
from_entity_external_id=from_ext_id,
to_entity=to_title,
to_entity_id=item.to_id,
to_entity_external_id=to_ext_id,
created_at=item.created_at,
)
case _: # pragma: no cover
raise ValueError(f"Unexpected type: {item.type}")
with telemetry.scope(
"memory.hydrate_context.shape_results",
domain="memory",
action="build_context",
phase="shape_results",
result_count=len(context_result.results),
):
hierarchical_results = []
for context_item in context_result.results:
primary_result = to_summary(context_item.primary_result)
observations = [to_summary(obs) for obs in context_item.observations]
related = [to_summary(rel) for rel in context_item.related_results]
hierarchical_results.append(
ContextResult(
primary_result=primary_result,
observations=observations, # pyright: ignore[reportArgumentType]
related_results=related,
)
)
metadata = MemoryMetadata(
uri=context_result.metadata.uri,
types=context_result.metadata.types,
depth=context_result.metadata.depth,
timeframe=context_result.metadata.timeframe,
generated_at=context_result.metadata.generated_at,
primary_count=context_result.metadata.primary_count,
related_count=context_result.metadata.related_count,
total_results=context_result.metadata.primary_count
+ context_result.metadata.related_count,
total_relations=context_result.metadata.total_relations,
total_observations=context_result.metadata.total_observations,
)
return GraphContext(
results=hierarchical_results,
metadata=metadata,
page=page,
page_size=page_size,
has_more=context_result.metadata.has_more,
)
async def to_search_results(entity_service: EntityService, results: List[SearchIndexRow]):
search_results = []
for r in results:
entities = await entity_service.get_entities_by_id([r.entity_id, r.from_id, r.to_id]) # pyright: ignore
with telemetry.scope(
"search.hydrate_results",
domain="search",
action="search",
phase="hydrate_results",
result_count=len(results),
):
# Collect all unique entity IDs across all results in a single pass
# This avoids N+1 queries — one batch fetch instead of one per result
all_entity_ids: set[int] = set()
for result in results:
for eid in (result.entity_id, result.from_id, result.to_id):
if eid is not None:
all_entity_ids.add(eid)
# Determine which IDs to set based on type
entity_id = None
observation_id = None
relation_id = None
# Single batch fetch for all entities
entities_by_id: dict[int, EntityModel] = {}
with telemetry.scope(
"search.hydrate_results.fetch_entities",
domain="search",
action="search",
phase="fetch_entities",
result_count=len(all_entity_ids),
):
if all_entity_ids:
entities = await entity_service.get_entities_by_id(list(all_entity_ids))
entities_by_id = {e.id: e for e in entities}
if r.type == SearchItemType.ENTITY:
entity_id = r.id
elif r.type == SearchItemType.OBSERVATION:
observation_id = r.id
entity_id = r.entity_id # Parent entity
elif r.type == SearchItemType.RELATION:
relation_id = r.id
entity_id = r.entity_id # Parent entity
search_results = []
with telemetry.scope(
"search.hydrate_results.shape_results",
domain="search",
action="search",
phase="shape_results",
result_count=len(results),
):
for result in results:
entity_id = None
observation_id = None
relation_id = None
search_results.append(
SearchResult(
title=r.title, # pyright: ignore
type=r.type, # pyright: ignore
permalink=r.permalink,
score=r.score, # pyright: ignore
entity=entities[0].permalink if entities else None,
content=r.content,
matched_chunk=r.matched_chunk_text,
file_path=r.file_path,
metadata=r.metadata,
entity_id=entity_id,
observation_id=observation_id,
relation_id=relation_id,
category=r.category,
from_entity=entities[0].permalink if entities else None,
to_entity=entities[1].permalink if len(entities) > 1 else None,
relation_type=r.relation_type,
)
)
return search_results
if result.type == SearchItemType.ENTITY:
entity_id = result.id
elif result.type == SearchItemType.OBSERVATION:
observation_id = result.id
entity_id = result.entity_id
elif result.type == SearchItemType.RELATION:
relation_id = result.id
entity_id = result.entity_id
# Look up entities by their specific IDs
parent_entity = entities_by_id.get(result.entity_id) if result.entity_id else None # pyright: ignore
from_entity = entities_by_id.get(result.from_id) if result.from_id else None # pyright: ignore
to_entity = entities_by_id.get(result.to_id) if result.to_id else None
search_results.append(
SearchResult(
title=result.title, # pyright: ignore
type=result.type, # pyright: ignore
permalink=result.permalink,
score=result.score, # pyright: ignore
entity=parent_entity.permalink if parent_entity else None,
content=result.content,
matched_chunk=result.matched_chunk_text,
file_path=result.file_path,
metadata=result.metadata,
entity_id=entity_id,
observation_id=observation_id,
relation_id=relation_id,
category=result.category,
from_entity=from_entity.permalink if from_entity else None,
to_entity=to_entity.permalink if to_entity else None,
relation_type=result.relation_type,
)
)
return search_results
@@ -2,10 +2,12 @@
from basic_memory.cli.commands.cloud.api_client import make_api_request
from basic_memory.config import ConfigManager
from basic_memory.mcp.async_client import resolve_configured_workspace
from basic_memory.schemas.cloud import (
CloudProjectList,
CloudProjectCreateRequest,
CloudProjectCreateResponse,
ProjectVisibility,
)
from basic_memory.utils import generate_permalink
@@ -16,8 +18,25 @@ class CloudUtilsError(Exception):
pass
def _workspace_headers(
*,
project_name: str | None = None,
workspace: str | None = None,
) -> dict[str, str]:
"""Build optional workspace headers using the CLI config resolution chain."""
resolved_workspace = resolve_configured_workspace(
project_name=project_name,
workspace=workspace,
)
if resolved_workspace is None:
return {}
return {"X-Workspace-ID": resolved_workspace}
async def fetch_cloud_projects(
*,
project_name: str | None = None,
workspace: str | None = None,
api_request=make_api_request,
) -> CloudProjectList:
"""Fetch list of projects from cloud API.
@@ -30,7 +49,11 @@ async def fetch_cloud_projects(
config = config_manager.config
host_url = config.cloud_host.rstrip("/")
response = await api_request(method="GET", url=f"{host_url}/proxy/v2/projects/")
response = await api_request(
method="GET",
url=f"{host_url}/proxy/v2/projects/",
headers=_workspace_headers(project_name=project_name, workspace=workspace),
)
return CloudProjectList.model_validate(response.json())
except Exception as e:
@@ -40,12 +63,16 @@ async def fetch_cloud_projects(
async def create_cloud_project(
project_name: str,
*,
workspace: str | None = None,
visibility: ProjectVisibility = "workspace",
api_request=make_api_request,
) -> CloudProjectCreateResponse:
"""Create a new project on cloud.
Args:
project_name: Name of project to create
workspace: Optional workspace override for tenant-scoped project creation
visibility: Visibility for the created cloud project
Returns:
CloudProjectCreateResponse with project details from API
@@ -62,12 +89,16 @@ async def create_cloud_project(
name=project_name,
path=project_path,
set_default=False,
visibility=visibility,
)
response = await api_request(
method="POST",
url=f"{host_url}/proxy/v2/projects/",
headers={"Content-Type": "application/json"},
headers={
"Content-Type": "application/json",
**_workspace_headers(project_name=project_name, workspace=workspace),
},
json_data=project_data.model_dump(),
)
@@ -91,18 +122,28 @@ async def sync_project(project_name: str, force_full: bool = False) -> None:
raise CloudUtilsError(f"Failed to sync project '{project_name}': {e}") from e
async def project_exists(project_name: str, *, api_request=make_api_request) -> bool:
async def project_exists(
project_name: str,
*,
workspace: str | None = None,
api_request=make_api_request,
) -> bool:
"""Check if a project exists on cloud.
Args:
project_name: Name of project to check
workspace: Optional workspace override for tenant-scoped project lookup
Returns:
True if project exists, False otherwise
Raises:
CloudUtilsError: If the project list cannot be fetched from cloud
"""
try:
projects = await fetch_cloud_projects(api_request=api_request)
project_names = {p.name for p in projects.projects}
return project_name in project_names
except Exception:
return False
projects = await fetch_cloud_projects(
project_name=project_name,
workspace=workspace,
api_request=api_request,
)
project_names = {p.name for p in projects.projects}
return project_name in project_names
@@ -54,7 +54,7 @@ def _require_cloud_credentials(config) -> None:
async def _get_cloud_project(name: str) -> ProjectItem | None:
"""Fetch a project by name from the cloud API."""
async with get_client() as client:
async with get_client(project_name=name) as client:
projects_list = await ProjectClient(client).list_projects()
for proj in projects_list.projects:
if generate_permalink(proj.name) == generate_permalink(name):
@@ -129,9 +129,9 @@ def sync_project_command(
if not dry_run:
async def _trigger_db_sync():
async with get_client() as client:
async with get_client(project_name=name) as client:
return await ProjectClient(client).sync(
project_data.external_id, force_full=True
project_data.external_id, force_full=False
)
try:
@@ -195,7 +195,10 @@ def bisync_project_command(
# Update config — sync_entry is guaranteed non-None because
# _get_sync_project validated local_sync_path (which comes from sync_entry)
sync_entry = config.projects.get(name)
assert sync_entry is not None
if sync_entry is None:
raise RuntimeError(
f"Sync entry for project '{name}' unexpectedly missing after validation"
)
sync_entry.last_sync = datetime.now()
sync_entry.bisync_initialized = True
ConfigManager().save_config(config)
@@ -204,9 +207,9 @@ def bisync_project_command(
if not dry_run:
async def _trigger_db_sync():
async with get_client() as client:
async with get_client(project_name=name) as client:
return await ProjectClient(client).sync(
project_data.external_id, force_full=True
project_data.external_id, force_full=False
)
try:
@@ -320,7 +323,7 @@ def setup_project_sync(
async def _verify_project_exists():
"""Verify the project exists on cloud by listing all projects."""
async with get_client() as client:
async with get_client(project_name=name) as client:
projects_list = await ProjectClient(client).list_projects()
project_names = [p.name for p in projects_list.projects]
if name not in project_names:
@@ -1,5 +1,6 @@
"""Upload CLI commands for basic-memory projects."""
from functools import partial
from pathlib import Path
import typer
@@ -8,12 +9,16 @@ from rich.console import Console
from basic_memory.cli.app import cloud_app
from basic_memory.cli.commands.command_utils import run_with_cleanup
from basic_memory.cli.commands.cloud.cloud_utils import (
CloudUtilsError,
create_cloud_project,
project_exists,
sync_project,
)
from basic_memory.cli.commands.cloud.upload import upload_path
from basic_memory.mcp.async_client import get_cloud_control_plane_client
from basic_memory.mcp.async_client import (
get_cloud_control_plane_client,
resolve_configured_workspace,
)
console = Console()
@@ -73,12 +78,20 @@ def upload(
"""
async def _upload():
resolved_workspace = resolve_configured_workspace(project_name=project)
try:
project_already_exists = await project_exists(project, workspace=resolved_workspace)
except CloudUtilsError as e:
console.print(f"[red]Failed to check cloud project '{project}': {e}[/red]")
raise typer.Exit(1)
# Check if project exists
if not await project_exists(project):
if not project_already_exists:
if create_project:
console.print(f"[blue]Creating cloud project '{project}'...[/blue]")
try:
await create_cloud_project(project)
await create_cloud_project(project, workspace=resolved_workspace)
console.print(f"[green]Created project '{project}'[/green]")
except Exception as e:
console.print(f"[red]Failed to create project: {e}[/red]")
@@ -106,7 +119,10 @@ def upload(
verbose=verbose,
use_gitignore=not no_gitignore,
dry_run=dry_run,
client_cm_factory=get_cloud_control_plane_client,
client_cm_factory=partial(
get_cloud_control_plane_client,
workspace=resolved_workspace,
),
)
if not success:
console.print("[red]Upload failed[/red]")
@@ -117,8 +133,10 @@ def upload(
else:
console.print(f"[green]Successfully uploaded to '{project}'[/green]")
# Sync project if requested (skip on dry run)
# Force full scan after bisync to ensure database is up-to-date with synced files
# Sync project if requested (skip on dry run).
# Trigger: upload adds new files the watcher has not observed locally.
# Why: force_full ensures those freshly uploaded files are indexed immediately.
# Outcome: upload keeps its eager reindex while sync/bisync stay incremental.
if sync and not dry_run:
console.print(f"[blue]Syncing project '{project}'...[/blue]")
try:
+103 -53
View File
@@ -4,6 +4,7 @@ import json
import os
from datetime import datetime
from pathlib import Path
from typing import cast
import typer
from rich.console import Console, Group
@@ -27,6 +28,7 @@ from basic_memory.cli.commands.routing import force_routing, validate_routing_fl
from basic_memory.config import ConfigManager, ProjectEntry, ProjectMode
from basic_memory.mcp.async_client import get_client
from basic_memory.mcp.clients import ProjectClient
from basic_memory.schemas.cloud import ProjectVisibility
from basic_memory.schemas.project_info import ProjectItem, ProjectList
from basic_memory.utils import generate_permalink, normalize_project_path
@@ -56,6 +58,57 @@ def make_bar(value: int, max_value: int, width: int = 40) -> Text:
return bar
def _normalize_project_visibility(visibility: str | None) -> ProjectVisibility:
"""Normalize CLI visibility input to the cloud API contract."""
if visibility is None:
return "workspace"
normalized = visibility.strip().lower()
if normalized in {"workspace", "shared", "private"}:
return cast(ProjectVisibility, normalized)
raise ValueError("Invalid visibility. Expected one of: workspace, shared, private.")
def _resolve_workspace_id(config, workspace: str | None) -> str | None:
"""Resolve a workspace name or tenant_id to a tenant_id."""
from basic_memory.mcp.project_context import (
_workspace_choices,
_workspace_matches_identifier,
get_available_workspaces,
)
if workspace is not None:
workspaces = run_with_cleanup(get_available_workspaces())
matches = [ws for ws in workspaces if _workspace_matches_identifier(ws, workspace)]
if not matches:
console.print(f"[red]Error: Workspace '{workspace}' not found[/red]")
if workspaces:
console.print(f"[dim]Available:\n{_workspace_choices(workspaces)}[/dim]")
raise typer.Exit(1)
if len(matches) > 1:
console.print(
f"[red]Error: Workspace name '{workspace}' matches multiple workspaces. "
f"Use tenant_id instead.[/red]"
)
console.print(f"[dim]Available:\n{_workspace_choices(workspaces)}[/dim]")
raise typer.Exit(1)
return matches[0].tenant_id
if config.default_workspace:
return config.default_workspace
try:
workspaces = run_with_cleanup(get_available_workspaces())
if len(workspaces) == 1:
return workspaces[0].tenant_id
except Exception:
# Workspace resolution is optional until a command needs a specific tenant.
pass
return None
@project_app.command("list")
def list_projects(
local: bool = typer.Option(False, "--local", help="Force local routing for this command"),
@@ -257,6 +310,16 @@ def add_project(
local_path: str = typer.Option(
None, "--local-path", help="Local sync path for cloud mode (optional)"
),
workspace: str = typer.Option(
None,
"--workspace",
help="Cloud workspace name or tenant_id (cloud mode only)",
),
visibility: str = typer.Option(
None,
"--visibility",
help="Cloud project visibility: workspace, shared, or private",
),
set_default: bool = typer.Option(False, "--default", help="Set as default project"),
local: bool = typer.Option(
False, "--local", help="Force local API routing (ignore cloud mode)"
@@ -271,6 +334,8 @@ def add_project(
Cloud mode examples:\n
bm project add research # No local sync\n
bm project add research --local-path ~/docs # With local sync\n
bm project add research --cloud --visibility shared\n
bm project add research --cloud --workspace Personal --visibility shared\n
Local mode example:\n
bm project add research ~/Documents/research
@@ -285,6 +350,7 @@ def add_project(
# Determine effective mode: default local, cloud only when explicitly requested.
effective_cloud_mode = cloud and not local
resolved_workspace_id: str | None = None
# Resolve local sync path early (needed for both cloud and local mode)
local_sync_path: str | None = None
@@ -293,18 +359,31 @@ def add_project(
if effective_cloud_mode:
_require_cloud_credentials(config)
try:
resolved_visibility = _normalize_project_visibility(visibility)
except ValueError as e:
console.print(f"[red]Error: {e}[/red]")
raise typer.Exit(1)
resolved_workspace_id = _resolve_workspace_id(config, workspace)
# Cloud mode: path auto-generated from name, local sync is optional
async def _add_project():
async with get_client() as client:
async with get_client(workspace=resolved_workspace_id) as client:
data = {
"name": name,
"path": generate_permalink(name),
"local_sync_path": local_sync_path,
"set_default": set_default,
"visibility": resolved_visibility,
}
return await ProjectClient(client).create_project(data)
else:
if workspace is not None:
console.print("[red]Error: --workspace is only supported in cloud mode[/red]")
raise typer.Exit(1)
if visibility is not None:
console.print("[red]Error: --visibility is only supported in cloud mode[/red]")
raise typer.Exit(1)
# Local mode: path is required
if path is None:
console.print("[red]Error: path argument is required in local mode[/red]")
@@ -323,25 +402,34 @@ def add_project(
result = run_with_cleanup(_add_project())
console.print(f"[green]{result.message}[/green]")
# Trigger: local config needs enough metadata to route future commands back to cloud.
# Why: explicit workspace selection and local sync state should persist across CLI sessions.
# Outcome: cloud-backed projects keep cloud mode, workspace_id, and optional local sync path.
if effective_cloud_mode and (local_sync_path or resolved_workspace_id):
entry = config.projects.get(name)
if entry:
entry.mode = ProjectMode.CLOUD
if local_sync_path:
entry.path = local_sync_path
entry.local_sync_path = local_sync_path
if resolved_workspace_id:
entry.workspace_id = resolved_workspace_id
else:
# Project may not be in local config yet (cloud-only add)
config.projects[name] = ProjectEntry(
path=local_sync_path or "",
mode=ProjectMode.CLOUD,
local_sync_path=local_sync_path,
workspace_id=resolved_workspace_id,
)
ConfigManager().save_config(config)
# Save local sync path to config if in cloud mode
if effective_cloud_mode and local_sync_path:
# Create local directory if it doesn't exist
local_dir = Path(local_sync_path)
local_dir.mkdir(parents=True, exist_ok=True)
# Update project entry — path is always the local directory
entry = config.projects.get(name)
if entry:
entry.path = local_sync_path
entry.local_sync_path = local_sync_path
else:
# Project may not be in local config yet (cloud-only add)
config.projects[name] = ProjectEntry(
path=local_sync_path,
local_sync_path=local_sync_path,
)
ConfigManager().save_config(config)
console.print(f"\n[green]Local sync path configured: {local_sync_path}[/green]")
console.print("\nNext steps:")
console.print(f" 1. Preview: bm cloud bisync --name {name} --resync --dry-run")
@@ -575,45 +663,7 @@ def set_cloud(
console.print("[dim]Run 'bm cloud api-key save <key>' or 'bm cloud login' first[/dim]")
raise typer.Exit(1)
# --- Resolve workspace to tenant_id ---
resolved_workspace_id: str | None = None
if workspace is not None:
# Explicit --workspace: resolve to tenant_id via cloud lookup
from basic_memory.mcp.project_context import (
get_available_workspaces,
_workspace_matches_identifier,
_workspace_choices,
)
workspaces = run_with_cleanup(get_available_workspaces())
matches = [ws for ws in workspaces if _workspace_matches_identifier(ws, workspace)]
if not matches:
console.print(f"[red]Error: Workspace '{workspace}' not found[/red]")
if workspaces:
console.print(f"[dim]Available:\n{_workspace_choices(workspaces)}[/dim]")
raise typer.Exit(1)
if len(matches) > 1:
console.print(
f"[red]Error: Workspace name '{workspace}' matches multiple workspaces. "
f"Use tenant_id instead.[/red]"
)
console.print(f"[dim]Available:\n{_workspace_choices(workspaces)}[/dim]")
raise typer.Exit(1)
resolved_workspace_id = matches[0].tenant_id
elif config.default_workspace:
# Fall back to global default
resolved_workspace_id = config.default_workspace
else:
# Try auto-select if single workspace
try:
from basic_memory.mcp.project_context import get_available_workspaces
workspaces = run_with_cleanup(get_available_workspaces())
if len(workspaces) == 1:
resolved_workspace_id = workspaces[0].tenant_id
except Exception:
pass # Workspace resolution is optional at set-cloud time
resolved_workspace_id = _resolve_workspace_id(config, workspace)
config.set_project_mode(name, ProjectMode.CLOUD)
if resolved_workspace_id:
+40 -4
View File
@@ -66,6 +66,27 @@ async def _resolve_cloud_token(config) -> str:
)
def resolve_configured_workspace(
*,
config=None,
project_name: Optional[str] = None,
workspace: Optional[str] = None,
) -> Optional[str]:
"""Resolve workspace from explicit input, per-project config, then global default."""
if workspace is not None:
return workspace
if config is None:
config = ConfigManager().config
if project_name is not None:
project_entry = config.projects.get(project_name)
if project_entry and project_entry.workspace_id:
return project_entry.workspace_id
return config.default_workspace
@asynccontextmanager
async def _cloud_client(
config,
@@ -88,15 +109,20 @@ async def _cloud_client(
@asynccontextmanager
async def get_cloud_control_plane_client() -> AsyncIterator[AsyncClient]:
async def get_cloud_control_plane_client(
workspace: Optional[str] = None,
) -> AsyncIterator[AsyncClient]:
"""Create a control-plane cloud client for endpoints outside /proxy."""
config = ConfigManager().config
timeout = _build_timeout()
token = await _resolve_cloud_token(config)
headers = {"Authorization": f"Bearer {token}"}
if workspace:
headers["X-Workspace-ID"] = workspace
logger.info(f"Creating HTTP client for cloud control plane at: {config.cloud_host}")
async with AsyncClient(
base_url=config.cloud_host,
headers={"Authorization": f"Bearer {token}"},
headers=headers,
timeout=timeout,
) as client:
yield client
@@ -167,7 +193,12 @@ async def get_client(
if _force_cloud_mode():
logger.debug("Explicit cloud routing enabled - using cloud proxy client")
async with _cloud_client(config, timeout, workspace=workspace) as client:
effective_workspace = resolve_configured_workspace(
config=config,
project_name=project_name,
workspace=workspace,
)
async with _cloud_client(config, timeout, workspace=effective_workspace) as client:
yield client
return
@@ -179,8 +210,13 @@ async def get_client(
project_mode = config.get_project_mode(project_name)
if project_mode == ProjectMode.CLOUD:
logger.debug(f"Project '{project_name}' is cloud mode - using cloud proxy client")
effective_workspace = resolve_configured_workspace(
config=config,
project_name=project_name,
workspace=workspace,
)
try:
async with _cloud_client(config, timeout, workspace=workspace) as client:
async with _cloud_client(config, timeout, workspace=effective_workspace) as client:
yield client
except RuntimeError as exc:
raise RuntimeError(
+125 -49
View File
@@ -7,6 +7,7 @@ from typing import Any
from httpx import AsyncClient
from basic_memory import telemetry
from basic_memory.mcp.tools.utils import call_get, call_post, call_put, call_patch, call_delete
from basic_memory.schemas.response import (
EntityResponse,
@@ -58,12 +59,21 @@ class KnowledgeClient:
ToolError: If the request fails
"""
params = {"fast": fast} if fast is not None else None
response = await call_post(
self.http_client,
f"{self._base_path}/entities",
json=entity_data,
params=params,
)
with telemetry.scope(
"mcp.client.knowledge.create_entity",
client_name="knowledge",
operation="create_entity",
fast=fast,
):
response = await call_post(
self.http_client,
f"{self._base_path}/entities",
json=entity_data,
params=params,
client_name="knowledge",
operation="create_entity",
path_template="/v2/projects/{project_id}/knowledge/entities",
)
return EntityResponse.model_validate(response.json())
async def update_entity(
@@ -86,12 +96,21 @@ class KnowledgeClient:
ToolError: If the request fails
"""
params = {"fast": fast} if fast is not None else None
response = await call_put(
self.http_client,
f"{self._base_path}/entities/{entity_id}",
json=entity_data,
params=params,
)
with telemetry.scope(
"mcp.client.knowledge.update_entity",
client_name="knowledge",
operation="update_entity",
fast=fast,
):
response = await call_put(
self.http_client,
f"{self._base_path}/entities/{entity_id}",
json=entity_data,
params=params,
client_name="knowledge",
operation="update_entity",
path_template="/v2/projects/{project_id}/knowledge/entities/{entity_id}",
)
return EntityResponse.model_validate(response.json())
async def get_entity(self, entity_id: str) -> EntityResponse:
@@ -106,10 +125,18 @@ class KnowledgeClient:
Raises:
ToolError: If the entity is not found or request fails
"""
response = await call_get(
self.http_client,
f"{self._base_path}/entities/{entity_id}",
)
with telemetry.scope(
"mcp.client.knowledge.get_entity",
client_name="knowledge",
operation="get_entity",
):
response = await call_get(
self.http_client,
f"{self._base_path}/entities/{entity_id}",
client_name="knowledge",
operation="get_entity",
path_template="/v2/projects/{project_id}/knowledge/entities/{entity_id}",
)
return EntityResponse.model_validate(response.json())
async def patch_entity(
@@ -132,12 +159,21 @@ class KnowledgeClient:
ToolError: If the request fails
"""
params = {"fast": fast} if fast is not None else None
response = await call_patch(
self.http_client,
f"{self._base_path}/entities/{entity_id}",
json=patch_data,
params=params,
)
with telemetry.scope(
"mcp.client.knowledge.patch_entity",
client_name="knowledge",
operation="patch_entity",
fast=fast,
):
response = await call_patch(
self.http_client,
f"{self._base_path}/entities/{entity_id}",
json=patch_data,
params=params,
client_name="knowledge",
operation="patch_entity",
path_template="/v2/projects/{project_id}/knowledge/entities/{entity_id}",
)
return EntityResponse.model_validate(response.json())
async def delete_entity(self, entity_id: str) -> DeleteEntitiesResponse:
@@ -152,10 +188,18 @@ class KnowledgeClient:
Raises:
ToolError: If the entity is not found or request fails
"""
response = await call_delete(
self.http_client,
f"{self._base_path}/entities/{entity_id}",
)
with telemetry.scope(
"mcp.client.knowledge.delete_entity",
client_name="knowledge",
operation="delete_entity",
):
response = await call_delete(
self.http_client,
f"{self._base_path}/entities/{entity_id}",
client_name="knowledge",
operation="delete_entity",
path_template="/v2/projects/{project_id}/knowledge/entities/{entity_id}",
)
return DeleteEntitiesResponse.model_validate(response.json())
async def move_entity(self, entity_id: str, destination_path: str) -> EntityResponse:
@@ -171,11 +215,19 @@ class KnowledgeClient:
Raises:
ToolError: If the request fails
"""
response = await call_put(
self.http_client,
f"{self._base_path}/entities/{entity_id}/move",
json={"destination_path": destination_path},
)
with telemetry.scope(
"mcp.client.knowledge.move_entity",
client_name="knowledge",
operation="move_entity",
):
response = await call_put(
self.http_client,
f"{self._base_path}/entities/{entity_id}/move",
json={"destination_path": destination_path},
client_name="knowledge",
operation="move_entity",
path_template="/v2/projects/{project_id}/knowledge/entities/{entity_id}/move",
)
return EntityResponse.model_validate(response.json())
async def move_directory(
@@ -193,14 +245,22 @@ class KnowledgeClient:
Raises:
ToolError: If the request fails
"""
response = await call_post(
self.http_client,
f"{self._base_path}/move-directory",
json={
"source_directory": source_directory,
"destination_directory": destination_directory,
},
)
with telemetry.scope(
"mcp.client.knowledge.move_directory",
client_name="knowledge",
operation="move_directory",
):
response = await call_post(
self.http_client,
f"{self._base_path}/move-directory",
json={
"source_directory": source_directory,
"destination_directory": destination_directory,
},
client_name="knowledge",
operation="move_directory",
path_template="/v2/projects/{project_id}/knowledge/move-directory",
)
return DirectoryMoveResult.model_validate(response.json())
async def delete_directory(self, directory: str) -> DirectoryDeleteResult:
@@ -215,11 +275,19 @@ class KnowledgeClient:
Raises:
ToolError: If the request fails
"""
response = await call_post(
self.http_client,
f"{self._base_path}/delete-directory",
json={"directory": directory},
)
with telemetry.scope(
"mcp.client.knowledge.delete_directory",
client_name="knowledge",
operation="delete_directory",
):
response = await call_post(
self.http_client,
f"{self._base_path}/delete-directory",
json={"directory": directory},
client_name="knowledge",
operation="delete_directory",
path_template="/v2/projects/{project_id}/knowledge/delete-directory",
)
return DirectoryDeleteResult.model_validate(response.json())
# --- Resolution ---
@@ -237,10 +305,18 @@ class KnowledgeClient:
Raises:
ToolError: If the identifier cannot be resolved
"""
response = await call_post(
self.http_client,
f"{self._base_path}/resolve",
json={"identifier": identifier, "strict": strict},
)
with telemetry.scope(
"mcp.client.knowledge.resolve_entity",
client_name="knowledge",
operation="resolve_entity",
):
response = await call_post(
self.http_client,
f"{self._base_path}/resolve",
json={"identifier": identifier, "strict": strict},
client_name="knowledge",
operation="resolve_entity",
path_template="/v2/projects/{project_id}/knowledge/resolve",
)
data = response.json()
return data["external_id"]
+31 -10
View File
@@ -7,6 +7,7 @@ from typing import Optional
from httpx import AsyncClient
from basic_memory import telemetry
from basic_memory.mcp.tools.utils import call_get
from basic_memory.schemas.memory import GraphContext
@@ -71,11 +72,21 @@ class MemoryClient:
if timeframe:
params["timeframe"] = timeframe
response = await call_get(
self.http_client,
f"{self._base_path}/{path}",
params=params,
)
with telemetry.scope(
"mcp.client.memory.build_context",
client_name="memory",
operation="build_context",
page=page,
page_size=page_size,
):
response = await call_get(
self.http_client,
f"{self._base_path}/{path}",
params=params,
client_name="memory",
operation="build_context",
path_template="/v2/projects/{project_id}/memory/{path}",
)
return GraphContext.model_validate(response.json())
async def recent(
@@ -112,9 +123,19 @@ class MemoryClient:
# Join types as comma-separated string if provided
params["type"] = ",".join(types) if isinstance(types, list) else types
response = await call_get(
self.http_client,
f"{self._base_path}/recent",
params=params,
)
with telemetry.scope(
"mcp.client.memory.recent_activity",
client_name="memory",
operation="recent_activity",
page=page,
page_size=page_size,
):
response = await call_get(
self.http_client,
f"{self._base_path}/recent",
params=params,
client_name="memory",
operation="recent_activity",
path_template="/v2/projects/{project_id}/memory/recent",
)
return GraphContext.model_validate(response.json())
+16 -5
View File
@@ -7,6 +7,7 @@ from typing import Optional
from httpx import AsyncClient, Response
from basic_memory import telemetry
from basic_memory.mcp.tools.utils import call_get
@@ -64,8 +65,18 @@ class ResourceClient:
if page_size is not None:
params["page_size"] = page_size
return await call_get(
self.http_client,
f"{self._base_path}/{entity_id}",
params=params if params else None,
)
with telemetry.scope(
"mcp.client.resource.read",
client_name="resource",
operation="read",
page=page,
page_size=page_size,
):
return await call_get(
self.http_client,
f"{self._base_path}/{entity_id}",
params=params if params else None,
client_name="resource",
operation="read",
path_template="/v2/projects/{project_id}/resource/{entity_id}",
)
+17 -6
View File
@@ -7,6 +7,7 @@ from typing import Any
from httpx import AsyncClient
from basic_memory import telemetry
from basic_memory.mcp.tools.utils import call_post
from basic_memory.schemas.search import SearchResponse
@@ -56,10 +57,20 @@ class SearchClient:
Raises:
ToolError: If the request fails
"""
response = await call_post(
self.http_client,
f"{self._base_path}/",
json=query,
params={"page": page, "page_size": page_size},
)
with telemetry.scope(
"mcp.client.search.search",
client_name="search",
operation="search",
page=page,
page_size=page_size,
):
response = await call_post(
self.http_client,
f"{self._base_path}/",
json=query,
params={"page": page, "page_size": page_size},
client_name="search",
operation="search",
path_template="/v2/projects/{project_id}/search/",
)
return SearchResponse.model_validate(response.json())
+99 -20
View File
@@ -64,6 +64,41 @@ async def _resolve_default_project_from_api() -> Optional[str]:
return None
async def _get_cached_active_project(context: Optional[Context]) -> Optional[ProjectItem]:
"""Return the cached active project from context when available."""
if not context:
return None
cached_raw = await context.get_state("active_project")
if isinstance(cached_raw, dict):
return ProjectItem.model_validate(cached_raw)
return None
async def _set_cached_active_project(
context: Optional[Context],
active_project: ProjectItem,
) -> None:
"""Persist the active project and known default-project metadata in context."""
if not context:
return
await context.set_state("active_project", active_project.model_dump())
if active_project.is_default:
await context.set_state("default_project_name", active_project.name)
async def _get_cached_default_project(context: Optional[Context]) -> Optional[str]:
"""Return the cached default project name from context when available."""
if not context:
return None
cached_default = await context.get_state("default_project_name")
if isinstance(cached_default, str):
return cached_default
return None
def _canonicalize_project_name(
project_name: Optional[str],
config: BasicMemoryConfig,
@@ -85,11 +120,23 @@ def _canonicalize_project_name(
return project_name
def _project_matches_identifier(project_item: ProjectItem, identifier: Optional[str]) -> bool:
"""Return True when the identifier refers to the cached project."""
if identifier is None:
return True
normalized_identifier = generate_permalink(identifier)
return normalized_identifier in {
generate_permalink(project_item.name),
project_item.permalink,
}
async def resolve_project_parameter(
project: Optional[str] = None,
allow_discovery: bool = False,
default_project: Optional[str] = None,
context: Optional[Context] = None,
) -> Optional[str]:
"""Resolve project parameter using unified linear priority chain.
@@ -119,14 +166,32 @@ async def resolve_project_parameter(
):
config = ConfigManager().config
# Load config for any values not explicitly provided.
# ConfigManager reads from the local config file, which doesn't exist in cloud mode.
# When it returns None, fall back to querying the projects API for the is_default flag.
if default_project is None:
# Trigger: project already resolved earlier in the same MCP request
# Why: the active project is request-constant, so re-discovering the
# default project via /v2/projects/ just repeats work
# Outcome: reuse the cached project name as the explicit candidate
if project is None:
cached_project = await _get_cached_active_project(context)
if cached_project is not None:
project = cached_project.name
# Trigger: there is no explicit project after env/context normalization
# Why: default-project discovery is only needed as a fallback; doing it
# for explicit requests adds an avoidable /v2/projects/ round-trip
# Outcome: skip default lookup when the active project is already known
if default_project is None and project is None:
# Load config for any values not explicitly provided.
# ConfigManager reads from the local config file, which doesn't exist in cloud mode.
# When it returns None, fall back to querying the projects API for the is_default flag.
default_project = config.default_project
if default_project is None:
default_project = await _resolve_default_project_from_api()
if default_project is None:
default_project = await _get_cached_default_project(context)
if default_project is None:
default_project = await _resolve_default_project_from_api()
if default_project and context:
await context.set_state("default_project_name", default_project)
# Create resolver with configuration and resolve
resolver = ProjectResolver.from_env(
@@ -290,7 +355,12 @@ async def get_active_project(
# Deferred import to avoid circular dependency with tools
from basic_memory.mcp.tools.utils import call_post
resolved_project = await resolve_project_parameter(project)
cached_project = await _get_cached_active_project(context)
if cached_project and _project_matches_identifier(cached_project, project):
logger.debug(f"Using cached project from context: {cached_project.name}")
return cached_project
resolved_project = await resolve_project_parameter(project, context=context)
if not resolved_project:
project_names = await get_project_names(client, headers)
raise ValueError(
@@ -301,14 +371,9 @@ async def get_active_project(
project = resolved_project
# Check if already cached in context
if context:
cached_raw = await context.get_state("active_project")
if isinstance(cached_raw, dict):
cached_project = ProjectItem.model_validate(cached_raw)
if cached_project.name == project:
logger.debug(f"Using cached project from context: {project}")
return cached_project
if cached_project and _project_matches_identifier(cached_project, project):
logger.debug(f"Using cached project from context: {cached_project.name}")
return cached_project
# Validate project exists by calling API
logger.debug(f"Validating project: {project}")
@@ -328,8 +393,8 @@ async def get_active_project(
)
# Cache in context if available
await _set_cached_active_project(context, active_project)
if context:
await context.set_state("active_project", active_project.model_dump())
logger.debug(f"Cached project in context: {project}")
logger.debug(f"Validated project: {active_project.name}")
@@ -383,6 +448,21 @@ async def resolve_project_and_path(
# Why: allow project-scoped memory URLs without requiring a separate project parameter
# Outcome: attempt to resolve the prefix as a project and route to it
if project_prefix:
cached_project = await _get_cached_active_project(context)
if cached_project and _project_matches_identifier(cached_project, project_prefix):
resolved_project = await resolve_project_parameter(project_prefix, context=context)
if resolved_project and generate_permalink(resolved_project) != generate_permalink(
project_prefix
):
raise ValueError(
f"Project is constrained to '{resolved_project}', cannot use '{project_prefix}'."
)
resolved_path = (
f"{cached_project.permalink}/{remainder}" if include_project else remainder
)
return cached_project, resolved_path, True
try:
from basic_memory.mcp.tools.utils import call_post
@@ -397,7 +477,7 @@ async def resolve_project_and_path(
if "project not found" not in str(exc).lower():
raise
else:
resolved_project = await resolve_project_parameter(project_prefix)
resolved_project = await resolve_project_parameter(project_prefix, context=context)
if resolved_project and generate_permalink(resolved_project) != generate_permalink(
project_prefix
):
@@ -412,8 +492,7 @@ async def resolve_project_and_path(
path=resolved.path,
is_default=resolved.is_default,
)
if context:
await context.set_state("active_project", active_project.model_dump())
await _set_cached_active_project(context, active_project)
resolved_path = (
f"{resolved.permalink}/{remainder}" if include_project else remainder
@@ -530,7 +609,7 @@ async def get_project_client(
)
# Step 1: Resolve project name from config (no network call)
resolved_project = await resolve_project_parameter(project)
resolved_project = await resolve_project_parameter(project, context=context)
if not resolved_project:
# Fall back to local client to discover projects and raise helpful error
async with get_client() as client:
+39 -27
View File
@@ -206,17 +206,25 @@ async def read_note(
resource_client = ResourceClient(client, active_project.external_id)
async def _read_json_payload(entity_id: str) -> dict:
entity = await knowledge_client.get_entity(entity_id)
response = await resource_client.read(entity_id, page=page, page_size=page_size)
content_text = response.text
body_content, parsed_frontmatter = _parse_opening_frontmatter(content_text)
return {
"title": entity.title,
"permalink": entity.permalink,
"file_path": entity.file_path,
"content": content_text if include_frontmatter else body_content,
"frontmatter": parsed_frontmatter,
}
with telemetry.scope(
"mcp.read_note.shape_response",
domain="mcp",
action="read_note",
phase="shape_response",
):
entity = await knowledge_client.get_entity(entity_id)
response = await resource_client.read(
entity_id, page=page, page_size=page_size
)
content_text = response.text
body_content, parsed_frontmatter = _parse_opening_frontmatter(content_text)
return {
"title": entity.title,
"permalink": entity.permalink,
"file_path": entity.file_path,
"content": content_text if include_frontmatter else body_content,
"frontmatter": parsed_frontmatter,
}
def _empty_json_payload() -> dict:
return {
@@ -233,6 +241,24 @@ async def read_note(
results = payload.get("results")
return results if isinstance(results, list) else []
async def _search_candidates(identifier_text: str, *, title_only: bool) -> dict:
# Trigger: direct entity resolution failed for the caller's identifier.
# Why: search_notes applies the same memory:// normalization and tool-level
# query handling as the rest of MCP routing, which raw client calls skip.
# Outcome: unresolved memory URLs still fall back through normalized search.
search_type = "title" if title_only else "text"
response = await search_notes(
project=active_project.name,
workspace=workspace,
query=identifier_text,
search_type=search_type,
page=page,
page_size=page_size,
output_format="json",
context=context,
)
return response if isinstance(response, dict) else {}
def _result_title(item: dict) -> str:
return str(item.get("title") or "")
@@ -265,14 +291,7 @@ async def read_note(
# Fallback 1: Try title search via API
logger.info(f"Search title for: {identifier}")
title_results = await search_notes(
query=identifier,
search_type="title",
project=active_project.name,
workspace=workspace,
output_format="json",
context=context,
)
title_results = await _search_candidates(identifier, title_only=True)
title_candidates = _search_results(title_results)
if title_candidates:
@@ -319,14 +338,7 @@ async def read_note(
# Fallback 2: Text search as a last resort
logger.info(f"Title search failed, trying text search for: {identifier}")
text_results = await search_notes(
query=identifier,
search_type="text",
project=active_project.name,
workspace=workspace,
output_format="json",
context=context,
)
text_results = await _search_candidates(identifier, title_only=False)
# We didn't find a direct match, construct a helpful error message
text_candidates = _search_results(text_results)
+3 -1
View File
@@ -537,7 +537,9 @@ async def search_notes(
has_query=bool(query and query.strip()),
note_type_filter_count=len(note_types),
entity_type_filter_count=len(entity_types),
has_metadata_filters=bool(metadata_filters),
has_filters=bool(
metadata_filters or tags or status or note_types or entity_types or after_date
),
has_tags_filter=bool(tags),
has_status_filter=bool(status),
):
+220 -58
View File
@@ -5,6 +5,7 @@ to the Basic Memory API, with improved error handling and logging.
"""
import typing
from contextlib import contextmanager
from typing import Optional
from httpx import Response, URL, AsyncClient, HTTPStatusError
@@ -23,9 +24,62 @@ from httpx._types import (
from loguru import logger
from mcp.server.fastmcp.exceptions import ToolError
from basic_memory import telemetry
from basic_memory.config import ConfigManager
def _classify_http_outcome(status_code: int) -> str:
"""Map HTTP status codes to a low-cardinality outcome label."""
if 200 <= status_code < 300:
return "success"
if 300 <= status_code < 400: # pragma: no cover
return "redirect"
if 400 <= status_code < 500:
return "client_error"
if 500 <= status_code < 600:
return "server_error"
return "unknown" # pragma: no cover
class _RequestSpan:
"""Small adapter for attaching outcome metadata to a live request span."""
def __init__(self, active_span: typing.Any | None):
self._active_span = active_span
def record_response(self, response: Response) -> None:
self._set_attributes(
{
"status_code": response.status_code,
"is_success": response.is_success,
"outcome": _classify_http_outcome(response.status_code),
}
)
def record_transport_error(self, exc: Exception) -> None:
self._set_attributes(
{
"is_success": False,
"outcome": "transport_error",
"error_type": type(exc).__name__,
}
)
def _set_attributes(self, attrs: dict[str, typing.Any]) -> None:
if self._active_span is None:
return
set_attributes = getattr(self._active_span, "set_attributes", None)
if callable(set_attributes):
set_attributes(attrs)
return
set_attribute = getattr(self._active_span, "set_attribute", None)
if callable(set_attribute):
for key, value in attrs.items():
set_attribute(key, value)
def get_error_message(
status_code: int, url: URL | str, method: str, msg: Optional[str] = None
) -> str:
@@ -135,10 +189,38 @@ def _resolve_error_message(
return get_error_message(status_code, url, method)
@contextmanager
def _request_scope(
method: str,
*,
client_name: str | None,
operation: str | None,
path_template: str | None,
params: QueryParamTypes | None = None,
has_body: bool = False,
):
"""Create the shared MCP transport span used by all HTTP helpers."""
attrs = {
"method": method,
"client_name": client_name,
"operation": operation,
"path_template": path_template,
"phase": "request",
"has_query": bool(params),
"has_body": has_body,
}
with telemetry.contextualize(**attrs):
with telemetry.started_span("mcp.http.request", **attrs) as active_span:
yield _RequestSpan(active_span)
async def call_get(
client: AsyncClient,
url: URL | str,
*,
client_name: str | None = None,
operation: str | None = None,
path_template: str | None = None,
params: QueryParamTypes | None = None,
headers: HeaderTypes | None = None,
cookies: CookieTypes | None = None,
@@ -168,18 +250,27 @@ async def call_get(
"""
logger.debug(f"Calling GET '{url}' params: '{params}'")
error_message = None
request_span: _RequestSpan | None = None
try:
response = await client.get(
url,
with _request_scope(
"GET",
client_name=client_name,
operation=operation,
path_template=path_template,
params=params,
headers=headers,
cookies=cookies,
auth=auth,
follow_redirects=follow_redirects,
timeout=timeout,
extensions=extensions,
)
) as request_span:
response = await client.get(
url,
params=params,
headers=headers,
cookies=cookies,
auth=auth,
follow_redirects=follow_redirects,
timeout=timeout,
extensions=extensions,
)
request_span.record_response(response)
if response.is_success:
return response
@@ -206,12 +297,19 @@ async def call_get(
except HTTPStatusError as e:
raise ToolError(error_message) from e
except Exception as e:
if request_span is not None:
request_span.record_transport_error(e)
raise
async def call_put(
client: AsyncClient,
url: URL | str,
*,
client_name: str | None = None,
operation: str | None = None,
path_template: str | None = None,
content: RequestContent | None = None,
data: RequestData | None = None,
files: RequestFiles | None = None,
@@ -249,22 +347,32 @@ async def call_put(
"""
logger.debug(f"Calling PUT '{url}'")
error_message = None
request_span: _RequestSpan | None = None
try:
response = await client.put(
url,
content=content,
data=data,
files=files,
json=json,
with _request_scope(
"PUT",
client_name=client_name,
operation=operation,
path_template=path_template,
params=params,
headers=headers,
cookies=cookies,
auth=auth,
follow_redirects=follow_redirects,
timeout=timeout,
extensions=extensions,
)
has_body=any(value is not None for value in (content, data, files, json)),
) as request_span:
response = await client.put(
url,
content=content,
data=data,
files=files,
json=json,
params=params,
headers=headers,
cookies=cookies,
auth=auth,
follow_redirects=follow_redirects,
timeout=timeout,
extensions=extensions,
)
request_span.record_response(response)
if response.is_success:
return response
@@ -292,12 +400,19 @@ async def call_put(
except HTTPStatusError as e:
raise ToolError(error_message) from e
except Exception as e:
if request_span is not None:
request_span.record_transport_error(e)
raise
async def call_patch(
client: AsyncClient,
url: URL | str,
*,
client_name: str | None = None,
operation: str | None = None,
path_template: str | None = None,
content: RequestContent | None = None,
data: RequestData | None = None,
files: RequestFiles | None = None,
@@ -334,22 +449,32 @@ async def call_patch(
ToolError: If the request fails with an appropriate error message
"""
logger.debug(f"Calling PATCH '{url}'")
request_span: _RequestSpan | None = None
try:
response = await client.patch(
url,
content=content,
data=data,
files=files,
json=json,
with _request_scope(
"PATCH",
client_name=client_name,
operation=operation,
path_template=path_template,
params=params,
headers=headers,
cookies=cookies,
auth=auth,
follow_redirects=follow_redirects,
timeout=timeout,
extensions=extensions,
)
has_body=any(value is not None for value in (content, data, files, json)),
) as request_span:
response = await client.patch(
url,
content=content,
data=data,
files=files,
json=json,
params=params,
headers=headers,
cookies=cookies,
auth=auth,
follow_redirects=follow_redirects,
timeout=timeout,
extensions=extensions,
)
request_span.record_response(response)
if response.is_success:
return response
@@ -382,12 +507,19 @@ async def call_patch(
error_message = _resolve_error_message(status_code, url, "PATCH", response_data)
raise ToolError(error_message) from e
except Exception as e:
if request_span is not None:
request_span.record_transport_error(e)
raise
async def call_post(
client: AsyncClient,
url: URL | str,
*,
client_name: str | None = None,
operation: str | None = None,
path_template: str | None = None,
content: RequestContent | None = None,
data: RequestData | None = None,
files: RequestFiles | None = None,
@@ -425,23 +557,33 @@ async def call_post(
"""
logger.debug(f"Calling POST '{url}'")
error_message = None
request_span: _RequestSpan | None = None
try:
response = await client.post(
url=url,
content=content,
data=data,
files=files,
json=json,
with _request_scope(
"POST",
client_name=client_name,
operation=operation,
path_template=path_template,
params=params,
headers=headers,
cookies=cookies,
auth=auth,
follow_redirects=follow_redirects,
timeout=timeout,
extensions=extensions,
)
logger.debug(f"response: {response.json()}")
has_body=any(value is not None for value in (content, data, files, json)),
) as request_span:
response = await client.post(
url=url,
content=content,
data=data,
files=files,
json=json,
params=params,
headers=headers,
cookies=cookies,
auth=auth,
follow_redirects=follow_redirects,
timeout=timeout,
extensions=extensions,
)
request_span.record_response(response)
logger.debug(f"response: {_extract_response_data(response)}")
if response.is_success:
return response
@@ -468,6 +610,10 @@ async def call_post(
except HTTPStatusError as e:
raise ToolError(error_message) from e
except Exception as e:
if request_span is not None:
request_span.record_transport_error(e)
raise
async def resolve_entity_id(client: AsyncClient, project_external_id: str, identifier: str) -> str:
@@ -506,6 +652,9 @@ async def call_delete(
client: AsyncClient,
url: URL | str,
*,
client_name: str | None = None,
operation: str | None = None,
path_template: str | None = None,
params: QueryParamTypes | None = None,
headers: HeaderTypes | None = None,
cookies: CookieTypes | None = None,
@@ -535,18 +684,27 @@ async def call_delete(
"""
logger.debug(f"Calling DELETE '{url}'")
error_message = None
request_span: _RequestSpan | None = None
try:
response = await client.delete(
url=url,
with _request_scope(
"DELETE",
client_name=client_name,
operation=operation,
path_template=path_template,
params=params,
headers=headers,
cookies=cookies,
auth=auth,
follow_redirects=follow_redirects,
timeout=timeout,
extensions=extensions,
)
) as request_span:
response = await client.delete(
url=url,
params=params,
headers=headers,
cookies=cookies,
auth=auth,
follow_redirects=follow_redirects,
timeout=timeout,
extensions=extensions,
)
request_span.record_response(response)
if response.is_success:
return response
@@ -573,3 +731,7 @@ async def call_delete(
except HTTPStatusError as e:
raise ToolError(error_message) from e
except Exception as e:
if request_span is not None:
request_span.record_transport_error(e)
raise
@@ -59,16 +59,27 @@ class EntityRepository(Repository[Entity]):
)
return await self.find_one(query)
async def get_by_permalink(self, permalink: str) -> Optional[Entity]:
async def _find_one_by_query(self, query, *, load_relations: bool) -> Optional[Entity]:
"""Return one entity row with optional eager loading."""
if load_relations:
query = query.options(*self.get_load_options())
return await self.find_one(query)
result = await self.execute_query(query, use_query_options=False)
return result.scalars().one_or_none()
async def get_by_permalink(
self, permalink: str, *, load_relations: bool = True
) -> Optional[Entity]:
"""Get entity by permalink.
Args:
permalink: Unique identifier for the entity
"""
query = self.select().where(Entity.permalink == permalink).options(*self.get_load_options())
return await self.find_one(query)
query = self.select().where(Entity.permalink == permalink)
return await self._find_one_by_query(query, load_relations=load_relations)
async def get_by_title(self, title: str) -> Sequence[Entity]:
async def get_by_title(self, title: str, *, load_relations: bool = True) -> Sequence[Entity]:
"""Get entities by title, ordered by shortest path first.
When multiple entities share the same title (in different folders),
@@ -82,23 +93,20 @@ class EntityRepository(Repository[Entity]):
self.select()
.where(Entity.title == title)
.order_by(func.length(Entity.file_path), Entity.file_path)
.options(*self.get_load_options())
)
result = await self.execute_query(query)
result = await self.execute_query(query, use_query_options=load_relations)
return list(result.scalars().all())
async def get_by_file_path(self, file_path: Union[Path, str]) -> Optional[Entity]:
async def get_by_file_path(
self, file_path: Union[Path, str], *, load_relations: bool = True
) -> Optional[Entity]:
"""Get entity by file_path.
Args:
file_path: Path to the entity file (will be converted to string internally)
"""
query = (
self.select()
.where(Entity.file_path == Path(file_path).as_posix())
.options(*self.get_load_options())
)
return await self.find_one(query)
query = self.select().where(Entity.file_path == Path(file_path).as_posix())
return await self._find_one_by_query(query, load_relations=load_relations)
# -------------------------------------------------------------------------
# Lightweight methods for permalink resolution (no eager loading)
@@ -306,7 +314,7 @@ class EntityRepository(Repository[Entity]):
result = await self.execute_query(query)
return list(result.scalars().all())
async def upsert_entity(self, entity: Entity) -> Entity:
async def upsert_entity(self, entity: Entity, *, reload: bool = True) -> Entity:
"""Insert or update entity using simple try/catch with database-level conflict resolution.
Handles file_path race conditions by checking for existing entity on IntegrityError.
@@ -327,6 +335,9 @@ class EntityRepository(Repository[Entity]):
session.add(entity)
await session.flush()
if not reload:
return entity
# Return with relationships loaded
query = (
self.select()
@@ -363,13 +374,12 @@ class EntityRepository(Repository[Entity]):
await session.rollback()
# Re-query after rollback to get a fresh, attached entity
existing_result = await session.execute(
select(Entity)
.where(
Entity.file_path == entity.file_path, Entity.project_id == entity.project_id
)
.options(*self.get_load_options())
existing_query = select(Entity).where(
Entity.file_path == entity.file_path, Entity.project_id == entity.project_id
)
if reload:
existing_query = existing_query.options(*self.get_load_options())
existing_result = await session.execute(existing_query)
existing_entity = existing_result.scalar_one_or_none()
if existing_entity:
@@ -393,6 +403,9 @@ class EntityRepository(Repository[Entity]):
await session.commit()
if not reload:
return merged_entity
# Re-query to get proper relationships loaded
final_result = await session.execute(
select(Entity)
+17 -2
View File
@@ -268,8 +268,21 @@ class Repository[T: Base]:
return await self.select_by_ids(session, [model.id for model in model_list]) # pyright: ignore [reportAttributeAccessIssue]
async def update(self, entity_id: int, entity_data: dict | T) -> Optional[T]:
"""Update an entity with the given data."""
async def update(
self,
entity_id: int,
entity_data: dict | T,
*,
reload: bool = True,
) -> Optional[T]:
"""Update an entity with the given data.
Args:
entity_id: Primary key to update
entity_data: Column values or a model instance to copy from
reload: When True, re-select the entity with repository load options.
When False, return the attached row after flush/refresh.
"""
logger.debug(f"Updating {self.Model.__name__} {entity_id} with data: {entity_data}")
async with db.scoped_session(self.session_maker) as session:
try:
@@ -291,6 +304,8 @@ class Repository[T: Base]:
await session.refresh(entity) # Refresh
logger.debug(f"Updated {self.Model.__name__}: {entity_id}")
if not reload:
return entity
return await self.select_by_id(session, entity.id) # pyright: ignore [reportAttributeAccessIssue]
except NoResultFound:
+7 -2
View File
@@ -178,8 +178,13 @@ ContentType = Annotated[
]
RelationType = Annotated[str, MinLen(1), MaxLen(200)]
"""Type of relationship between entities. Always use active voice present tense."""
RelationType = Annotated[str, MinLen(1)]
"""Type of relationship between entities. Always use active voice present tense.
The database stores relation_type as an unrestricted string, and response models
need to tolerate existing long-form values written by LLMs. Keeping an API-only
200-character cap here causes reads to fail for valid stored data.
"""
ObservationStr = Annotated[
str,
+8
View File
@@ -1,7 +1,11 @@
"""Schemas for cloud-related API responses."""
from typing import Literal
from pydantic import BaseModel, Field
type ProjectVisibility = Literal["workspace", "shared", "private"]
class TenantMountInfo(BaseModel):
"""Response from /tenant/mount/info endpoint."""
@@ -36,6 +40,10 @@ class CloudProjectCreateRequest(BaseModel):
name: str = Field(..., description="Project name")
path: str = Field(..., description="Project path (permalink)")
set_default: bool = Field(default=False, description="Set as default project")
visibility: ProjectVisibility = Field(
default="workspace",
description="Project visibility for team workspaces",
)
class CloudProjectCreateResponse(BaseModel):
+146 -129
View File
@@ -10,6 +10,7 @@ from typing import List, Optional, Tuple, TYPE_CHECKING
from loguru import logger
from sqlalchemy import text
from basic_memory import telemetry
from basic_memory.repository.entity_repository import EntityRepository
from basic_memory.repository.observation_repository import ObservationRepository
from basic_memory.repository.postgres_search_repository import PostgresSearchRepository
@@ -110,146 +111,162 @@ class ContextService:
f"Building context for URI: '{memory_url}' depth: '{depth}' since: '{since}' limit: '{limit}' offset: '{offset}' max_related: '{max_related}'"
)
# Fetch one extra item to detect whether more pages exist (N+1 trick)
fetch_limit = limit + 1
with telemetry.scope(
"memory.build_context",
domain="memory",
action="build_context",
phase="build_context",
limit=limit,
offset=offset,
):
fetch_limit = limit + 1
normalized_path: Optional[str] = None
if memory_url:
path = memory_url_path(memory_url)
# Check for wildcards before normalization
has_wildcard = "*" in path
normalized_path: Optional[str] = None
with telemetry.scope(
"memory.build_context.resolve_primary",
domain="memory",
action="build_context",
phase="resolve_primary",
):
if memory_url:
path = memory_url_path(memory_url)
has_wildcard = "*" in path
if has_wildcard:
# For wildcard patterns, normalize each segment separately to preserve the *
parts = path.split("*")
normalized_parts = [
generate_permalink(part, split_extension=False) if part else ""
for part in parts
]
normalized_path = "*".join(normalized_parts)
logger.debug(f"Pattern search for '{normalized_path}'")
primary = await self.search_repository.search(
permalink_match=normalized_path, limit=fetch_limit, offset=offset
)
else:
# For exact paths, normalize the whole thing
normalized_path = generate_permalink(path, split_extension=False)
logger.debug(f"Direct lookup for '{normalized_path}'")
primary = await self.search_repository.search(
permalink=normalized_path, limit=fetch_limit, offset=offset
)
# Trigger: exact permalink lookup returned no results
# Why: the identifier may be valid but not an exact permalink match
# (e.g., missing project prefix, title instead of permalink)
# Outcome: use LinkResolver's multi-strategy resolution to find the entity,
# then retry search with its actual permalink
if not primary and self.link_resolver:
entity = await self.link_resolver.resolve_link(
path, use_search=True, strict=False
)
if entity:
logger.debug(
f"LinkResolver resolved '{path}' to permalink '{entity.permalink}'"
)
normalized_path = entity.permalink
if has_wildcard:
parts = path.split("*")
normalized_parts = [
generate_permalink(part, split_extension=False) if part else ""
for part in parts
]
normalized_path = "*".join(normalized_parts)
logger.debug(f"Pattern search for '{normalized_path}'")
primary = await self.search_repository.search(
permalink=entity.permalink, limit=fetch_limit, offset=offset
permalink_match=normalized_path, limit=fetch_limit, offset=offset
)
else:
logger.debug(f"Build context for '{types}'")
primary = await self.search_repository.search(
search_item_types=types, after_date=since, limit=fetch_limit, offset=offset
else:
normalized_path = generate_permalink(path, split_extension=False)
logger.debug(f"Direct lookup for '{normalized_path}'")
primary = await self.search_repository.search(
permalink=normalized_path, limit=fetch_limit, offset=offset
)
if not primary and self.link_resolver:
entity = await self.link_resolver.resolve_link(
path, use_search=True, strict=False
)
if entity:
logger.debug(
f"LinkResolver resolved '{path}' to permalink '{entity.permalink}'"
)
normalized_path = entity.permalink
primary = await self.search_repository.search(
permalink=entity.permalink,
limit=fetch_limit,
offset=offset,
)
else:
logger.debug(f"Build context for '{types}'")
primary = await self.search_repository.search(
search_item_types=types,
after_date=since,
limit=fetch_limit,
offset=offset,
)
has_more = len(primary) > limit
if has_more:
primary = primary[:limit]
type_id_pairs = [(r.type, r.id) for r in primary] if primary else []
logger.debug(f"found primary type_id_pairs: {len(type_id_pairs)}")
with telemetry.scope(
"memory.build_context.find_related",
domain="memory",
action="build_context",
phase="find_related",
):
related = await self.find_related(
type_id_pairs, max_depth=depth, since=since, max_results=max_related
)
logger.debug(f"Found {len(related)} related results")
entity_ids = []
for result in primary:
if result.type == SearchItemType.ENTITY.value:
entity_ids.append(result.id)
for result in related:
if result.type == SearchItemType.ENTITY.value:
entity_ids.append(result.id)
observations_by_entity = {}
if include_observations and entity_ids:
with telemetry.scope(
"memory.build_context.load_observations",
domain="memory",
action="build_context",
phase="load_observations",
result_count=len(entity_ids),
):
observations_by_entity = await self.observation_repository.find_by_entities(
entity_ids
)
logger.debug(f"Found observations for {len(observations_by_entity)} entities")
metadata = ContextMetadata(
uri=normalized_path if memory_url else None,
types=types,
depth=depth,
timeframe=since.isoformat() if since else None,
primary_count=len(primary),
related_count=len(related),
total_observations=sum(len(obs) for obs in observations_by_entity.values()),
total_relations=sum(1 for r in related if r.type == SearchItemType.RELATION),
has_more=has_more,
)
# Trim to requested limit and set has_more flag
has_more = len(primary) > limit
if has_more:
primary = primary[:limit]
with telemetry.scope(
"memory.build_context.shape_results",
domain="memory",
action="build_context",
phase="shape_results",
result_count=len(primary),
):
context_results = []
for primary_item in primary:
related_to_primary = [r for r in related if r.root_id == primary_item.id]
# Get type_id pairs for traversal
item_observations = []
if primary_item.type == SearchItemType.ENTITY.value and include_observations:
for obs in observations_by_entity.get(primary_item.id, []):
item_observations.append(
ContextResultRow(
type="observation",
id=obs.id,
title=f"{obs.category}: {obs.content[:50]}...",
permalink=generate_permalink(
f"{primary_item.permalink}/observations/{obs.category}/{obs.content}"
),
file_path=primary_item.file_path,
content=obs.content,
category=obs.category,
entity_id=primary_item.id,
depth=0,
root_id=primary_item.id,
created_at=primary_item.created_at,
)
)
type_id_pairs = [(r.type, r.id) for r in primary] if primary else []
logger.debug(f"found primary type_id_pairs: {len(type_id_pairs)}")
# Find related content
related = await self.find_related(
type_id_pairs, max_depth=depth, since=since, max_results=max_related
)
logger.debug(f"Found {len(related)} related results")
# Collect entity IDs from primary and related results
entity_ids = []
for result in primary:
if result.type == SearchItemType.ENTITY.value:
entity_ids.append(result.id)
for result in related:
if result.type == SearchItemType.ENTITY.value:
entity_ids.append(result.id)
# Fetch observations for all entities if requested
observations_by_entity = {}
if include_observations and entity_ids:
# Use our observation repository to get observations for all entities at once
observations_by_entity = await self.observation_repository.find_by_entities(entity_ids)
logger.debug(f"Found observations for {len(observations_by_entity)} entities")
# Create metadata dataclass
metadata = ContextMetadata(
uri=normalized_path if memory_url else None,
types=types,
depth=depth,
timeframe=since.isoformat() if since else None,
primary_count=len(primary),
related_count=len(related),
total_observations=sum(len(obs) for obs in observations_by_entity.values()),
total_relations=sum(1 for r in related if r.type == SearchItemType.RELATION),
has_more=has_more,
)
# Build context results list directly with ContextResultItem objects
context_results = []
# For each primary result
for primary_item in primary:
# Find all related items with this primary item as root
related_to_primary = [r for r in related if r.root_id == primary_item.id]
# Get observations for this item if it's an entity
item_observations = []
if primary_item.type == SearchItemType.ENTITY.value and include_observations:
# Convert Observation models to ContextResultRows
for obs in observations_by_entity.get(primary_item.id, []):
item_observations.append(
ContextResultRow(
type="observation",
id=obs.id,
title=f"{obs.category}: {obs.content[:50]}...",
permalink=generate_permalink(
f"{primary_item.permalink}/observations/{obs.category}/{obs.content}"
),
file_path=primary_item.file_path,
content=obs.content,
category=obs.category,
entity_id=primary_item.id,
depth=0,
root_id=primary_item.id,
created_at=primary_item.created_at, # created_at time from entity
context_results.append(
ContextResultItem(
primary_result=primary_item,
observations=item_observations,
related_results=related_to_primary,
)
)
# Create ContextResultItem directly
context_item = ContextResultItem(
primary_result=primary_item,
observations=item_observations,
related_results=related_to_primary,
)
context_results.append(context_item)
# Return the structured ContextResult
return ContextResult(results=context_results, metadata=metadata)
return ContextResult(results=context_results, metadata=metadata)
async def find_related(
self,
+504 -153
View File
@@ -1,6 +1,7 @@
"""Service for managing entities in the database."""
from collections.abc import Callable
from dataclasses import dataclass
from datetime import datetime
from pathlib import Path
from typing import List, Optional, Sequence, Tuple, Union
@@ -10,7 +11,7 @@ import yaml
from loguru import logger
from sqlalchemy.exc import IntegrityError
from basic_memory import telemetry
from basic_memory.config import ProjectConfig, BasicMemoryConfig
from basic_memory.file_utils import (
has_frontmatter,
@@ -50,6 +51,15 @@ from basic_memory.services.search_service import SearchService
from basic_memory.utils import build_canonical_permalink
@dataclass(frozen=True)
class EntityWriteResult:
"""Persisted entity plus the response/search content produced during this call."""
entity: EntityModel
content: str
search_content: str
class EntityService(BaseService[EntityModel]):
"""Service for managing entities in the database."""
@@ -79,7 +89,7 @@ class EntityService(BaseService[EntityModel]):
async def detect_file_path_conflicts(
self, file_path: str, skip_check: bool = False
) -> List[Entity]:
) -> List[str]:
"""Detect potential file path conflicts for a given file path.
This checks for entities with similar file paths that might cause conflicts:
@@ -93,28 +103,19 @@ class EntityService(BaseService[EntityModel]):
skip_check: If True, skip the check and return empty list (optimization for bulk operations)
Returns:
List of entities that might conflict with the given file path
List of file paths that might conflict with the given file path
"""
if skip_check:
return []
from basic_memory.utils import detect_potential_file_conflicts
conflicts = []
# Get all existing file paths
all_entities = await self.repository.find_all()
existing_paths = [entity.file_path for entity in all_entities]
# Load only file paths. Conflict detection is on the hot write path and
# does not need observations or relations.
existing_paths = await self.repository.get_all_file_paths()
# Use the enhanced conflict detection utility
conflicting_paths = detect_potential_file_conflicts(file_path, existing_paths)
# Find the entities corresponding to conflicting paths
for entity in all_entities:
if entity.file_path in conflicting_paths:
conflicts.append(entity)
return conflicts
return detect_potential_file_conflicts(file_path, existing_paths)
async def resolve_permalink(
self,
@@ -143,8 +144,7 @@ class EntityService(BaseService[EntityModel]):
)
if conflicts:
logger.warning(
f"Detected potential file path conflicts for '{file_path_str}': "
f"{[entity.file_path for entity in conflicts]}"
f"Detected potential file path conflicts for '{file_path_str}': {conflicts}"
)
# If markdown has explicit permalink, try to validate it
@@ -242,9 +242,17 @@ class EntityService(BaseService[EntityModel]):
# Try to find existing entity using strict resolution (no fuzzy search)
# This prevents incorrectly matching similar file paths like "Node A.md" and "Node C.md"
existing = await self.link_resolver.resolve_link(schema.file_path, strict=True)
existing = await self.link_resolver.resolve_link(
schema.file_path,
strict=True,
load_relations=False,
)
if not existing and schema.permalink:
existing = await self.link_resolver.resolve_link(schema.permalink, strict=True)
existing = await self.link_resolver.resolve_link(
schema.permalink,
strict=True,
load_relations=False,
)
if existing:
logger.debug(f"Found existing entity: {existing.file_path}")
@@ -255,6 +263,10 @@ class EntityService(BaseService[EntityModel]):
async def create_entity(self, schema: EntitySchema) -> EntityModel:
"""Create a new entity and write to filesystem."""
return (await self.create_entity_with_content(schema)).entity
async def create_entity_with_content(self, schema: EntitySchema) -> EntityWriteResult:
"""Create a new entity and return both the entity row and written markdown."""
logger.debug(f"Creating entity: {schema.title}")
# Get file path and ensure it's a Path object
@@ -281,34 +293,67 @@ class EntityService(BaseService[EntityModel]):
# Get unique permalink (prioritizing content frontmatter) unless disabled
if self.app_config and self.app_config.disable_permalinks:
# Use empty string as sentinel to indicate permalinks are disabled
# The permalink property will return None when it sees empty string
schema._permalink = ""
else:
# Generate and set permalink
permalink = await self.resolve_permalink(file_path, content_markdown)
with telemetry.scope(
"entity_service.create.resolve_permalink",
domain="entity_service",
action="create",
phase="resolve_permalink",
):
permalink = await self.resolve_permalink(file_path, content_markdown)
schema._permalink = permalink
post = await schema_to_markdown(schema)
# write file
final_content = dump_frontmatter(post)
checksum = await self.file_service.write_file(file_path, final_content)
with telemetry.scope(
"entity_service.create.write_file",
domain="entity_service",
action="create",
phase="write_file",
):
checksum = await self.file_service.write_file(file_path, final_content)
# parse entity from content we just wrote (avoids re-reading file for cloud compatibility)
entity_markdown = await self.entity_parser.parse_markdown_content(
file_path=file_path,
with telemetry.scope(
"entity_service.create.parse_markdown",
domain="entity_service",
action="create",
phase="parse_markdown",
):
entity_markdown = await self.entity_parser.parse_markdown_content(
file_path=file_path,
content=final_content,
)
with telemetry.scope(
"entity_service.create.upsert_entity",
domain="entity_service",
action="create",
phase="upsert_entity",
):
updated = await self.upsert_entity_from_markdown(
file_path,
entity_markdown,
is_new=True,
checksum=checksum,
)
if not updated: # pragma: no cover
raise ValueError(f"Failed to persist entity after create: {file_path}")
return EntityWriteResult(
entity=updated,
content=final_content,
search_content=remove_frontmatter(final_content),
)
# create entity and relations
entity = await self.upsert_entity_from_markdown(file_path, entity_markdown, is_new=True)
# Set final checksum to mark complete
return await self.repository.update(entity.id, {"checksum": checksum})
async def update_entity(self, entity: EntityModel, schema: EntitySchema) -> EntityModel:
"""Update an entity's content and metadata."""
return (await self.update_entity_with_content(entity, schema)).entity
async def update_entity_with_content(
self, entity: EntityModel, schema: EntitySchema
) -> EntityWriteResult:
"""Update an entity and return both the entity row and written markdown."""
logger.debug(
f"Updating entity with permalink: {entity.permalink} content-type: {schema.content_type}"
)
@@ -316,12 +361,23 @@ class EntityService(BaseService[EntityModel]):
# Convert file path string to Path
file_path = Path(entity.file_path)
# Read existing content via file_service (for cloud compatibility)
existing_content = await self.file_service.read_file_content(file_path)
existing_markdown = await self.entity_parser.parse_markdown_content(
file_path=file_path,
content=existing_content,
)
with telemetry.scope(
"entity_service.update.read_file",
domain="entity_service",
action="update",
phase="read_file",
):
existing_content = await self.file_service.read_file_content(file_path)
with telemetry.scope(
"entity_service.update.parse_markdown",
domain="entity_service",
action="update",
phase="parse_markdown",
):
existing_markdown = await self.entity_parser.parse_markdown_content(
file_path=file_path,
content=existing_content,
)
# Parse content frontmatter to check for user-specified permalink and note_type
content_markdown = None
@@ -342,7 +398,13 @@ class EntityService(BaseService[EntityModel]):
if self.app_config and not self.app_config.disable_permalinks:
if content_markdown and content_markdown.frontmatter.permalink:
# Resolve permalink with the new content frontmatter
resolved_permalink = await self.resolve_permalink(file_path, content_markdown)
with telemetry.scope(
"entity_service.update.resolve_permalink",
domain="entity_service",
action="update",
phase="resolve_permalink",
):
resolved_permalink = await self.resolve_permalink(file_path, content_markdown)
if resolved_permalink != entity.permalink:
new_permalink = resolved_permalink
# Update the schema to use the new permalink
@@ -367,24 +429,47 @@ class EntityService(BaseService[EntityModel]):
merged_post = frontmatter.Post(post.content)
merged_post.metadata.update(existing_markdown.frontmatter.metadata)
# write file
final_content = dump_frontmatter(merged_post)
checksum = await self.file_service.write_file(file_path, final_content)
with telemetry.scope(
"entity_service.update.write_file",
domain="entity_service",
action="update",
phase="write_file",
):
checksum = await self.file_service.write_file(file_path, final_content)
# parse entity from content we just wrote (avoids re-reading file for cloud compatibility)
entity_markdown = await self.entity_parser.parse_markdown_content(
file_path=file_path,
with telemetry.scope(
"entity_service.update.parse_markdown",
domain="entity_service",
action="update",
phase="parse_markdown",
):
entity_markdown = await self.entity_parser.parse_markdown_content(
file_path=file_path,
content=final_content,
)
with telemetry.scope(
"entity_service.update.upsert_entity",
domain="entity_service",
action="update",
phase="upsert_entity",
):
entity = await self.upsert_entity_from_markdown(
file_path,
entity_markdown,
is_new=False,
checksum=checksum,
)
if not entity: # pragma: no cover
raise ValueError(f"Failed to persist entity after update: {file_path}")
return EntityWriteResult(
entity=entity,
content=final_content,
search_content=remove_frontmatter(final_content),
)
# update entity and relations
entity = await self.upsert_entity_from_markdown(file_path, entity_markdown, is_new=False)
# Set final checksum to match file
entity = await self.repository.update(entity.id, {"checksum": checksum})
return entity
async def fast_write_entity(
self,
schema: EntitySchema,
@@ -399,7 +484,15 @@ class EntityService(BaseService[EntityModel]):
)
# --- Identity & File Path ---
existing = await self.repository.get_by_external_id(external_id) if external_id else None
with telemetry.scope(
"entity_service.fast_write.resolve_entity",
domain="entity_service",
action="fast_write",
phase="resolve_entity",
):
existing = (
await self.repository.get_by_external_id(external_id) if external_id else None
)
# Trigger: external_id already exists
# Why: avoid duplicate entities when title-derived paths change
@@ -429,18 +522,35 @@ class EntityService(BaseService[EntityModel]):
schema._permalink = ""
else:
if existing and not (content_markdown and content_markdown.frontmatter.permalink):
schema._permalink = existing.permalink or await self.resolve_permalink(
file_path, skip_conflict_check=True
)
with telemetry.scope(
"entity_service.fast_write.resolve_permalink",
domain="entity_service",
action="fast_write",
phase="resolve_permalink",
):
schema._permalink = existing.permalink or await self.resolve_permalink(
file_path, skip_conflict_check=True
)
else:
schema._permalink = await self.resolve_permalink(
file_path, content_markdown, skip_conflict_check=True
)
with telemetry.scope(
"entity_service.fast_write.resolve_permalink",
domain="entity_service",
action="fast_write",
phase="resolve_permalink",
):
schema._permalink = await self.resolve_permalink(
file_path, content_markdown, skip_conflict_check=True
)
# --- File Write ---
post = await schema_to_markdown(schema)
final_content = dump_frontmatter(post)
checksum = await self.file_service.write_file(file_path, final_content)
with telemetry.scope(
"entity_service.fast_write.write_file",
domain="entity_service",
action="fast_write",
phase="write_file",
):
checksum = await self.file_service.write_file(file_path, final_content)
# --- Minimal DB Upsert ---
metadata = normalize_frontmatter_metadata(post.metadata or {})
@@ -462,7 +572,13 @@ class EntityService(BaseService[EntityModel]):
# Preserve existing created_by; only update last_updated_by
if user_id is not None:
update_data["last_updated_by"] = user_id
updated = await self.repository.update(existing.id, update_data)
with telemetry.scope(
"entity_service.fast_write.upsert_entity",
domain="entity_service",
action="fast_write",
phase="upsert_entity",
):
updated = await self.repository.update(existing.id, update_data)
if not updated:
raise ValueError(f"Failed to update entity in database: {existing.id}")
return updated
@@ -473,7 +589,13 @@ class EntityService(BaseService[EntityModel]):
if user_id is not None:
create_data["created_by"] = user_id
create_data["last_updated_by"] = user_id
return await self.repository.create(create_data)
with telemetry.scope(
"entity_service.fast_write.upsert_entity",
domain="entity_service",
action="fast_write",
phase="upsert_entity",
):
return await self.repository.create(create_data)
async def fast_edit_entity(
self,
@@ -487,13 +609,30 @@ class EntityService(BaseService[EntityModel]):
"""Edit an entity quickly and defer full indexing to background."""
logger.debug(f"Fast editing entity: {entity.external_id}, operation: {operation}")
# --- File Edit ---
file_path = Path(entity.file_path)
current_content, _ = await self.file_service.read_file(file_path)
new_content = self.apply_edit_operation(
current_content, operation, content, section, find_text, expected_replacements
)
checksum = await self.file_service.write_file(file_path, new_content)
with telemetry.scope(
"entity_service.fast_edit.read_file",
domain="entity_service",
action="fast_edit",
phase="read_file",
):
current_content, _ = await self.file_service.read_file(file_path)
with telemetry.scope(
"entity_service.fast_edit.apply_operation",
domain="entity_service",
action="fast_edit",
phase="apply_operation",
):
new_content = self.apply_edit_operation(
current_content, operation, content, section, find_text, expected_replacements
)
with telemetry.scope(
"entity_service.fast_edit.write_file",
domain="entity_service",
action="fast_edit",
phase="write_file",
):
checksum = await self.file_service.write_file(file_path, new_content)
# --- Frontmatter Overrides ---
update_data = {
@@ -528,39 +667,86 @@ class EntityService(BaseService[EntityModel]):
if self.app_config and self.app_config.disable_permalinks:
update_data["permalink"] = None
elif content_markdown and content_markdown.frontmatter.permalink:
update_data["permalink"] = await self.resolve_permalink(
file_path, content_markdown, skip_conflict_check=True
)
with telemetry.scope(
"entity_service.fast_edit.resolve_permalink",
domain="entity_service",
action="fast_edit",
phase="resolve_permalink",
):
update_data["permalink"] = await self.resolve_permalink(
file_path, content_markdown, skip_conflict_check=True
)
updated = await self.repository.update(entity.id, update_data)
with telemetry.scope(
"entity_service.fast_edit.update_entity",
domain="entity_service",
action="fast_edit",
phase="update_entity",
):
updated = await self.repository.update(entity.id, update_data)
if not updated:
raise ValueError(f"Failed to update entity in database: {entity.id}")
return updated
async def reindex_entity(self, entity_id: int) -> None:
"""Parse file content and rebuild observations/relations/search for an entity."""
entity = await self.repository.find_by_id(entity_id)
with telemetry.scope(
"entity_service.reindex.load_entity",
domain="entity_service",
action="reindex",
phase="load_entity",
):
entity = await self.repository.find_by_id(entity_id)
if not entity:
raise EntityNotFoundError(f"Entity not found: {entity_id}")
# --- Full Parse ---
file_path = Path(entity.file_path)
content = await self.file_service.read_file_content(file_path)
entity_markdown = await self.entity_parser.parse_markdown_content(
file_path=file_path,
content=content,
)
with telemetry.scope(
"entity_service.reindex.read_file",
domain="entity_service",
action="reindex",
phase="read_file",
):
content = await self.file_service.read_file_content(file_path)
with telemetry.scope(
"entity_service.reindex.parse_markdown",
domain="entity_service",
action="reindex",
phase="parse_markdown",
):
entity_markdown = await self.entity_parser.parse_markdown_content(
file_path=file_path,
content=content,
)
# --- DB Reindex ---
updated = await self.upsert_entity_from_markdown(file_path, entity_markdown, is_new=False)
checksum = await self.file_service.compute_checksum(file_path)
updated = await self.repository.update(updated.id, {"checksum": checksum})
with telemetry.scope(
"entity_service.reindex.upsert_entity",
domain="entity_service",
action="reindex",
phase="upsert_entity",
):
updated = await self.upsert_entity_from_markdown(
file_path, entity_markdown, is_new=False
)
with telemetry.scope(
"entity_service.reindex.update_checksum",
domain="entity_service",
action="reindex",
phase="update_checksum",
):
checksum = await self.file_service.compute_checksum(file_path)
updated = await self.repository.update(updated.id, {"checksum": checksum})
if not updated:
raise ValueError(f"Failed to update entity in database: {entity.id}")
# --- Search Reindex ---
if self.search_service:
await self.search_service.index_entity_data(updated, content=content)
with telemetry.scope(
"entity_service.reindex.search_index",
domain="entity_service",
action="reindex",
phase="search_index",
):
await self.search_service.index_entity_data(updated, content=content)
async def delete_entity(self, permalink_or_id: str | int) -> bool:
"""Delete entity and its file."""
@@ -572,6 +758,10 @@ class EntityService(BaseService[EntityModel]):
entity = await self.get_by_permalink(permalink_or_id)
else:
entities = await self.get_entities_by_id([permalink_or_id])
if len(entities) == 0:
# Entity already deleted (concurrent delete or race condition)
logger.info("Entity already deleted", entity_id=permalink_or_id)
return True
if len(entities) != 1: # pragma: no cover
logger.error(
"Entity lookup error", entity_id=permalink_or_id, found_count=len(entities)
@@ -583,13 +773,28 @@ class EntityService(BaseService[EntityModel]):
# Delete from search index first (if search_service is available)
if self.search_service:
await self.search_service.handle_delete(entity)
try:
await self.search_service.handle_delete(entity)
except Exception:
# Search cleanup is best-effort during concurrent deletes.
# Relationships may have been cascade-deleted by a concurrent request.
logger.warning(
"Search cleanup failed for entity (likely concurrent delete)",
permalink_or_id=permalink_or_id,
exc_info=True,
)
# Delete file
await self.file_service.delete_entity_file(entity)
# Delete from DB (this will cascade to observations/relations)
return await self.repository.delete(entity.id)
# Trigger: repository.delete returns False when entity is already gone (NoResultFound)
# Why: concurrent delete_directory requests can race to delete the same entity
# Outcome: treat as success since the entity is deleted either way
deleted = await self.repository.delete(entity.id)
if not deleted:
logger.info("Entity already removed from DB", entity_id=permalink_or_id)
return True
except EntityNotFoundError:
logger.info(f"Entity not found: {permalink_or_id}")
@@ -643,7 +848,7 @@ class EntityService(BaseService[EntityModel]):
# Use UPSERT to handle conflicts cleanly
try:
return await self.repository.upsert_entity(model)
return await self.repository.upsert_entity(model, reload=False)
except Exception as e:
logger.error(f"Failed to upsert entity for {file_path}: {e}")
raise EntityCreationError(f"Failed to create entity: {str(e)}") from e
@@ -658,7 +863,12 @@ class EntityService(BaseService[EntityModel]):
"""
logger.debug(f"Updating entity and observations: {file_path}")
db_entity = await self.repository.get_by_file_path(file_path.as_posix())
db_entity = await self.repository.get_by_file_path(
file_path.as_posix(),
load_relations=False,
)
if not db_entity: # pragma: no cover
raise EntityNotFoundError(f"Entity not found: {file_path}")
# Clear observations for entity
await self.observation_repository.delete_by_fields(entity_id=db_entity.id)
@@ -675,23 +885,37 @@ class EntityService(BaseService[EntityModel]):
)
for obs in markdown.observations
]
await self.observation_repository.add_all(observations)
if observations:
await self.observation_repository.add_all(observations)
# update values from markdown
db_entity = entity_model_from_markdown(file_path, markdown, db_entity)
# Trigger: the lightweight lookup above returns a detached row without loaded collections
# Why: assigning a new observation list onto that detached ORM object would trigger lazy loads
# Outcome: rebuild a fresh model from markdown, then copy over stable identity fields
db_entity_data = entity_model_from_markdown(
file_path,
markdown,
project_id=self.repository.project_id,
)
db_entity_data.id = db_entity.id
db_entity_data.project_id = db_entity.project_id
db_entity_data.external_id = db_entity.external_id
db_entity_data.created_by = db_entity.created_by
# checksum value is None == not finished with sync
db_entity.checksum = None
db_entity_data.checksum = None
# Set last_updated_by for cloud usage (preserve existing created_by)
user_id = self.get_user_id()
if user_id is not None:
db_entity.last_updated_by = user_id
db_entity_data.last_updated_by = user_id
else:
db_entity_data.last_updated_by = db_entity.last_updated_by
# update entity
return await self.repository.update(
db_entity.id,
db_entity,
db_entity_data,
reload=False,
)
async def upsert_entity_from_markdown(
@@ -700,26 +924,76 @@ class EntityService(BaseService[EntityModel]):
markdown: EntityMarkdown,
*,
is_new: bool,
checksum: Optional[str] = None,
) -> EntityModel:
"""Create/update entity and relations from parsed markdown."""
if is_new:
created = await self.create_entity_from_markdown(file_path, markdown)
else:
created = await self.update_entity_and_observations(file_path, markdown)
return await self.update_entity_relations(created.file_path, markdown)
# --- Base Entity Row ---
# Trigger: writes rebuild the entity row before touching relation edges
# Why: relations need a stable source entity ID, but not a fully hydrated graph
# Outcome: create/update the row with a lightweight return value
with telemetry.scope(
"entity_service.upsert.base_entity",
domain="entity_service",
action="upsert",
phase="base_entity",
):
if is_new:
created = await self.create_entity_from_markdown(file_path, markdown)
else:
created = await self.update_entity_and_observations(file_path, markdown)
# --- Relation Edges ---
with telemetry.scope(
"entity_service.upsert.relations",
domain="entity_service",
action="upsert",
phase="relations",
):
await self.update_entity_relations(created, markdown)
# --- Final Entity State ---
# Trigger: create/update/edit already computed the final file checksum
# Why: fold the checksum write into the upsert flow so callers do one hydrated read
# Outcome: the write path returns the final entity state without an extra checksum step
if checksum is not None:
with telemetry.scope(
"entity_service.upsert.persist_checksum",
domain="entity_service",
action="upsert",
phase="persist_checksum",
):
updated = await self.repository.update(created.id, {"checksum": checksum})
if not updated: # pragma: no cover
raise ValueError(f"Failed to update entity checksum after upsert: {file_path}")
return updated
with telemetry.scope(
"entity_service.upsert.hydrate_entity",
domain="entity_service",
action="upsert",
phase="hydrate_entity",
):
hydrated = await self.repository.get_by_file_path(created.file_path)
if not hydrated: # pragma: no cover
raise EntityNotFoundError(f"Entity not found after upsert: {created.file_path}")
return hydrated
async def update_entity_relations(
self,
path: str,
db_entity: EntityModel,
markdown: EntityMarkdown,
) -> EntityModel:
) -> None:
"""Update relations for entity"""
logger.debug(f"Updating relations for entity: {path}")
db_entity = await self.repository.get_by_file_path(path)
logger.debug(f"Updating relations for entity: {db_entity.file_path}")
# Clear existing relations first
await self.relation_repository.delete_outgoing_relations_from_entity(db_entity.id)
with telemetry.scope(
"entity_service.upsert.delete_relations",
domain="entity_service",
action="upsert",
phase="delete_relations",
):
await self.relation_repository.delete_outgoing_relations_from_entity(db_entity.id)
# Batch resolve all relation targets in parallel
if markdown.relations:
@@ -728,13 +1002,23 @@ class EntityService(BaseService[EntityModel]):
# Create tasks for all relation lookups
# Use strict=True to disable fuzzy search - only exact matches should create resolved relations
# This ensures forward references (links to non-existent entities) remain unresolved (to_id=NULL)
lookup_tasks = [
self.link_resolver.resolve_link(rel.target, strict=True)
for rel in markdown.relations
]
with telemetry.scope(
"entity_service.upsert.resolve_relation_targets",
domain="entity_service",
action="upsert",
phase="resolve_relation_targets",
):
lookup_tasks = [
self.link_resolver.resolve_link(
rel.target,
strict=True,
load_relations=False,
)
for rel in markdown.relations
]
# Execute all lookups in parallel
resolved_entities = await asyncio.gather(*lookup_tasks, return_exceptions=True)
# Execute all lookups in parallel
resolved_entities = await asyncio.gather(*lookup_tasks, return_exceptions=True)
# Process results and create relation records
relations_to_add = []
@@ -763,22 +1047,26 @@ class EntityService(BaseService[EntityModel]):
# Batch insert all relations
if relations_to_add:
try:
await self.relation_repository.add_all(relations_to_add)
except IntegrityError:
# Some relations might be duplicates - fall back to individual inserts
logger.debug("Batch relation insert failed, trying individual inserts")
for relation in relations_to_add:
try:
await self.relation_repository.add(relation)
except IntegrityError:
# Unique constraint violation - relation already exists
logger.debug(
f"Skipping duplicate relation {relation.relation_type} from {db_entity.permalink}"
)
continue
return await self.repository.get_by_file_path(path)
with telemetry.scope(
"entity_service.upsert.insert_relations",
domain="entity_service",
action="upsert",
phase="insert_relations",
):
try:
await self.relation_repository.add_all(relations_to_add)
except IntegrityError:
# Some relations might be duplicates - fall back to individual inserts
logger.debug("Batch relation insert failed, trying individual inserts")
for relation in relations_to_add:
try:
await self.relation_repository.add(relation)
except IntegrityError:
# Unique constraint violation - relation already exists
logger.debug(
f"Skipping duplicate relation {relation.relation_type} from {db_entity.permalink}"
)
continue
async def edit_entity(
self,
@@ -806,39 +1094,102 @@ class EntityService(BaseService[EntityModel]):
EntityNotFoundError: If the entity cannot be found
ValueError: If required parameters are missing for the operation or replacement count doesn't match expected
"""
return (
await self.edit_entity_with_content(
identifier=identifier,
operation=operation,
content=content,
section=section,
find_text=find_text,
expected_replacements=expected_replacements,
)
).entity
async def edit_entity_with_content(
self,
identifier: str,
operation: str,
content: str,
section: Optional[str] = None,
find_text: Optional[str] = None,
expected_replacements: int = 1,
) -> EntityWriteResult:
"""Edit an entity and return both the entity row and written markdown."""
logger.debug(f"Editing entity: {identifier}, operation: {operation}")
# Find the entity using the link resolver with strict mode for destructive operations
entity = await self.link_resolver.resolve_link(identifier, strict=True)
with telemetry.scope(
"entity_service.edit.resolve_entity",
domain="entity_service",
action="edit",
phase="resolve_entity",
):
entity = await self.link_resolver.resolve_link(
identifier,
strict=True,
load_relations=False,
)
if not entity:
raise EntityNotFoundError(f"Entity not found: {identifier}")
# Read the current file content
file_path = Path(entity.file_path)
current_content, _ = await self.file_service.read_file(file_path)
with telemetry.scope(
"entity_service.edit.read_file",
domain="entity_service",
action="edit",
phase="read_file",
):
current_content, _ = await self.file_service.read_file(file_path)
# Apply the edit operation
new_content = self.apply_edit_operation(
current_content, operation, content, section, find_text, expected_replacements
)
with telemetry.scope(
"entity_service.edit.apply_operation",
domain="entity_service",
action="edit",
phase="apply_operation",
):
new_content = self.apply_edit_operation(
current_content, operation, content, section, find_text, expected_replacements
)
# Write the updated content back to the file
checksum = await self.file_service.write_file(file_path, new_content)
with telemetry.scope(
"entity_service.edit.write_file",
domain="entity_service",
action="edit",
phase="write_file",
):
checksum = await self.file_service.write_file(file_path, new_content)
# Parse the content we just wrote (avoids re-reading file for cloud compatibility)
entity_markdown = await self.entity_parser.parse_markdown_content(
file_path=file_path,
with telemetry.scope(
"entity_service.edit.parse_markdown",
domain="entity_service",
action="edit",
phase="parse_markdown",
):
entity_markdown = await self.entity_parser.parse_markdown_content(
file_path=file_path,
content=new_content,
)
with telemetry.scope(
"entity_service.edit.upsert_entity",
domain="entity_service",
action="edit",
phase="upsert_entity",
):
entity = await self.upsert_entity_from_markdown(
file_path,
entity_markdown,
is_new=False,
checksum=checksum,
)
if not entity: # pragma: no cover
raise ValueError(f"Failed to persist entity after edit: {file_path}")
return EntityWriteResult(
entity=entity,
content=new_content,
search_content=remove_frontmatter(new_content),
)
# Update entity and its relationships
entity = await self.upsert_entity_from_markdown(file_path, entity_markdown, is_new=False)
# Set final checksum to match file
entity = await self.repository.update(entity.id, {"checksum": checksum})
return entity
def apply_edit_operation(
self,
current_content: str,
+86 -59
View File
@@ -11,6 +11,7 @@ import aiofiles
import yaml
from basic_memory import telemetry
from basic_memory import file_utils
if TYPE_CHECKING: # pragma: no cover
@@ -79,13 +80,18 @@ class FileService:
"""
logger.debug(f"Reading entity content, entity_id={entity.id}, permalink={entity.permalink}")
# markdown_processor is required for entity content reads — fail fast if not configured
if self.markdown_processor is None:
raise ValueError("markdown_processor is required for read_entity_content")
with telemetry.scope(
"file_service.read_content",
domain="file_service",
action="read_content",
phase="read_content",
):
if self.markdown_processor is None:
raise ValueError("markdown_processor is required for read_entity_content")
file_path = self.get_entity_path(entity)
markdown = await self.markdown_processor.read_file(file_path)
return markdown.content or ""
file_path = self.get_entity_path(entity)
markdown = await self.markdown_processor.read_file(file_path)
return markdown.content or ""
async def delete_entity_file(self, entity: EntityModel) -> None:
"""Delete entity file from filesystem.
@@ -176,32 +182,34 @@ class FileService:
full_path = path_obj if path_obj.is_absolute() else self.base_path / path_obj
try:
# Ensure parent directory exists
await self.ensure_directory(full_path.parent)
with telemetry.scope(
"file_service.write",
domain="file_service",
action="write",
phase="write",
):
await self.ensure_directory(full_path.parent)
# Write content atomically
logger.info(
"Writing file: "
f"path={path_obj}, "
f"content_length={len(content)}, "
f"is_markdown={full_path.suffix.lower() == '.md'}"
)
await file_utils.write_file_atomic(full_path, content)
# Format file if configured
final_content = content
if self.app_config:
formatted_content = await file_utils.format_file(
full_path, self.app_config, is_markdown=self.is_markdown(path)
logger.info(
"Writing file: "
f"path={path_obj}, "
f"content_length={len(content)}, "
f"is_markdown={full_path.suffix.lower() == '.md'}"
)
if formatted_content is not None:
final_content = formatted_content # pragma: no cover
# Compute and return checksum of final content
checksum = await file_utils.compute_checksum(final_content)
logger.debug(f"File write completed path={full_path}, {checksum=}")
return checksum
await file_utils.write_file_atomic(full_path, content)
final_content = content
if self.app_config:
formatted_content = await file_utils.format_file(
full_path, self.app_config, is_markdown=self.is_markdown(path)
)
if formatted_content is not None:
final_content = formatted_content # pragma: no cover
checksum = await file_utils.compute_checksum(final_content)
logger.debug(f"File write completed path={full_path}, {checksum=}")
return checksum
except Exception as e:
logger.exception("File write error", path=str(full_path), error=str(e))
@@ -227,16 +235,24 @@ class FileService:
full_path = path_obj if path_obj.is_absolute() else self.base_path / path_obj
try:
logger.debug("Reading file content", operation="read_file_content", path=str(full_path))
async with aiofiles.open(full_path, mode="r", encoding="utf-8") as f:
content = await f.read()
with telemetry.scope(
"file_service.read_content",
domain="file_service",
action="read_content",
phase="read_content",
):
logger.debug(
"Reading file content", operation="read_file_content", path=str(full_path)
)
async with aiofiles.open(full_path, mode="r", encoding="utf-8") as f:
content = await f.read()
logger.debug(
"File read completed",
path=str(full_path),
content_length=len(content),
)
return content
logger.debug(
"File read completed",
path=str(full_path),
content_length=len(content),
)
return content
except FileNotFoundError:
# Preserve FileNotFoundError so callers (e.g. sync) can treat it as deletion.
@@ -266,16 +282,22 @@ class FileService:
full_path = path_obj if path_obj.is_absolute() else self.base_path / path_obj
try:
logger.debug("Reading file bytes", operation="read_file_bytes", path=str(full_path))
async with aiofiles.open(full_path, mode="rb") as f:
content = await f.read()
with telemetry.scope(
"file_service.read_content",
domain="file_service",
action="read_content",
phase="read_content",
):
logger.debug("Reading file bytes", operation="read_file_bytes", path=str(full_path))
async with aiofiles.open(full_path, mode="rb") as f:
content = await f.read()
logger.debug(
"File read completed",
path=str(full_path),
content_length=len(content),
)
return content
logger.debug(
"File read completed",
path=str(full_path),
content_length=len(content),
)
return content
except Exception as e:
logger.exception("File read error", path=str(full_path), error=str(e))
@@ -303,21 +325,26 @@ class FileService:
full_path = path_obj if path_obj.is_absolute() else self.base_path / path_obj
try:
logger.debug("Reading file", operation="read_file", path=str(full_path))
with telemetry.scope(
"file_service.read",
domain="file_service",
action="read",
phase="read",
):
logger.debug("Reading file", operation="read_file", path=str(full_path))
# Use aiofiles for non-blocking read
async with aiofiles.open(full_path, mode="r", encoding="utf-8") as f:
content = await f.read()
async with aiofiles.open(full_path, mode="r", encoding="utf-8") as f:
content = await f.read()
checksum = await file_utils.compute_checksum(content)
checksum = await file_utils.compute_checksum(content)
logger.debug(
"File read completed",
path=str(full_path),
checksum=checksum,
content_length=len(content),
)
return content, checksum
logger.debug(
"File read completed",
path=str(full_path),
checksum=checksum,
content_length=len(content),
)
return content, checksum
except Exception as e:
logger.exception("File read error", path=str(full_path), error=str(e))
+42 -9
View File
@@ -47,6 +47,7 @@ class LinkResolver:
use_search: bool = True,
strict: bool = False,
source_path: Optional[str] = None,
load_relations: bool = True,
) -> Optional[Entity]:
"""Resolve a markdown link to a permalink.
@@ -56,6 +57,7 @@ class LinkResolver:
strict: If True, only exact matches are allowed (no fuzzy search fallback)
source_path: Optional path of the source file containing the link.
Used to prefer notes closer to the source (context-aware resolution).
load_relations: When False, skip eager loading and return a lightweight entity row.
"""
logger.trace(f"Resolving link: {link_text} (source: {source_path})")
@@ -98,6 +100,7 @@ class LinkResolver:
strict=strict,
source_path=None,
project_permalink=project.permalink,
load_relations=load_relations,
)
current_project_permalink = await self._get_current_project_permalink()
@@ -109,6 +112,7 @@ class LinkResolver:
strict=strict,
source_path=source_path,
project_permalink=current_project_permalink,
load_relations=load_relations,
)
if resolved:
return resolved
@@ -136,6 +140,7 @@ class LinkResolver:
strict=strict,
source_path=None,
project_permalink=project.permalink,
load_relations=load_relations,
)
def _normalize_link_text(self, link_text: str) -> Tuple[str, Optional[str]]:
@@ -176,6 +181,7 @@ class LinkResolver:
strict: bool,
source_path: Optional[str],
project_permalink: Optional[str],
load_relations: bool,
) -> Optional[Entity]:
"""Resolve a link within a specific project scope."""
clean_text = link_text
@@ -223,12 +229,18 @@ class LinkResolver:
# Try with .md extension
if not relative_path.endswith(".md"):
relative_path_md = f"{relative_path}.md"
entity = await entity_repository.get_by_file_path(relative_path_md)
entity = await entity_repository.get_by_file_path(
relative_path_md,
load_relations=load_relations,
)
if entity:
return entity
# Try as-is (already has extension or is a permalink)
entity = await entity_repository.get_by_file_path(relative_path)
entity = await entity_repository.get_by_file_path(
relative_path,
load_relations=load_relations,
)
if entity:
return entity
@@ -242,12 +254,18 @@ class LinkResolver:
# Check permalink match
for candidate_permalink in permalink_candidates:
permalink_entity = await entity_repository.get_by_permalink(candidate_permalink)
permalink_entity = await entity_repository.get_by_permalink(
candidate_permalink,
load_relations=load_relations,
)
if permalink_entity and permalink_entity.id not in [c.id for c in candidates]:
candidates.append(permalink_entity)
# Check title matches
title_entities = await entity_repository.get_by_title(clean_text)
title_entities = await entity_repository.get_by_title(
clean_text,
load_relations=load_relations,
)
for entity in title_entities:
# Avoid duplicates (permalink match might also be in title matches)
if entity.id not in [c.id for c in candidates]:
@@ -263,13 +281,19 @@ class LinkResolver:
# Standard resolution (no source context): permalink first, then title
# 1. Try exact permalink match first (most efficient)
for candidate_permalink in permalink_candidates:
entity = await entity_repository.get_by_permalink(candidate_permalink)
entity = await entity_repository.get_by_permalink(
candidate_permalink,
load_relations=load_relations,
)
if entity:
logger.debug(f"Found exact permalink match: {entity.permalink}")
return entity
# 2. Try exact title match
found = await entity_repository.get_by_title(clean_text)
found = await entity_repository.get_by_title(
clean_text,
load_relations=load_relations,
)
if found:
# Return first match (shortest path) if no source context
entity = found[0]
@@ -277,7 +301,10 @@ class LinkResolver:
return entity
# 3. Try file path
found_path = await entity_repository.get_by_file_path(clean_text)
found_path = await entity_repository.get_by_file_path(
clean_text,
load_relations=load_relations,
)
if found_path:
logger.debug(f"Found entity with path: {found_path.file_path}")
return found_path
@@ -285,7 +312,10 @@ class LinkResolver:
# 4. Try file path with .md extension if not already present
if not clean_text.endswith(".md") and "/" in clean_text:
file_path_with_md = f"{clean_text}.md"
found_path_md = await entity_repository.get_by_file_path(file_path_with_md)
found_path_md = await entity_repository.get_by_file_path(
file_path_with_md,
load_relations=load_relations,
)
if found_path_md:
logger.debug(f"Found entity with path (with .md): {found_path_md.file_path}")
return found_path_md
@@ -309,7 +339,10 @@ class LinkResolver:
f"Selected best match from {len(results)} results: {best_match.permalink}"
)
if best_match.permalink:
return await entity_repository.get_by_permalink(best_match.permalink)
return await entity_repository.get_by_permalink(
best_match.permalink,
load_relations=load_relations,
)
# if we couldn't find anything then return None
return None
+219 -167
View File
@@ -173,33 +173,49 @@ class SearchService:
retrieval_mode = query.retrieval_mode or SearchRetrievalMode.FTS
strict_search_text = query.text
has_query = bool(
strict_search_text or query.title or query.permalink or query.permalink_match
)
has_filters = bool(
metadata_filters
or query.note_types
or query.entity_types
or after_date
or query.tags
or query.status
)
with telemetry.scope(
"search.execute",
retrieval_mode=retrieval_mode.value,
has_text_query=bool(strict_search_text),
has_title_query=bool(query.title),
has_permalink_query=bool(query.permalink or query.permalink_match),
has_metadata_filters=bool(metadata_filters),
has_query=has_query,
has_filters=has_filters,
limit=limit,
offset=offset,
):
logger.trace(f"Searching with query: {query}")
# First pass: preserve existing strict search behavior.
results = await self.repository.search(
search_text=strict_search_text,
permalink=query.permalink,
permalink_match=query.permalink_match,
title=query.title,
note_types=query.note_types,
search_item_types=query.entity_types,
after_date=after_date,
metadata_filters=metadata_filters,
retrieval_mode=retrieval_mode,
min_similarity=query.min_similarity,
limit=limit,
offset=offset,
)
with telemetry.scope(
"search.repository_query",
retrieval_mode=retrieval_mode.value,
phase="repository_query",
has_query=has_query,
has_filters=has_filters,
):
# First pass: preserve existing strict search behavior.
results = await self.repository.search(
search_text=strict_search_text,
permalink=query.permalink,
permalink_match=query.permalink_match,
title=query.title,
note_types=query.note_types,
search_item_types=query.entity_types,
after_date=after_date,
metadata_filters=metadata_filters,
retrieval_mode=retrieval_mode,
min_similarity=query.min_similarity,
limit=limit,
offset=offset,
)
# Trigger: strict FTS with plain multi-term text returned no results.
# Why: natural-language queries often include stopwords that over-constrain implicit AND.
@@ -225,20 +241,27 @@ class SearchService:
limit=limit,
offset=offset,
):
return await self.repository.search(
search_text=relaxed_search_text,
permalink=query.permalink,
permalink_match=query.permalink_match,
title=query.title,
note_types=query.note_types,
search_item_types=query.entity_types,
after_date=after_date,
metadata_filters=metadata_filters,
retrieval_mode=retrieval_mode,
min_similarity=query.min_similarity,
limit=limit,
offset=offset,
)
with telemetry.scope(
"search.repository_query",
retrieval_mode=retrieval_mode.value,
phase="repository_query",
has_query=has_query,
has_filters=has_filters,
):
return await self.repository.search(
search_text=relaxed_search_text,
permalink=query.permalink,
permalink_match=query.permalink_match,
title=query.title,
note_types=query.note_types,
search_item_types=query.entity_types,
after_date=after_date,
metadata_filters=metadata_filters,
retrieval_mode=retrieval_mode,
min_similarity=query.min_similarity,
limit=limit,
offset=offset,
)
@staticmethod
def _tokenize_fts_text(search_text: str) -> list[str]:
@@ -372,13 +395,22 @@ class SearchService:
f"permalink={entity.permalink} project_id={entity.project_id}"
)
try:
# delete all search index data associated with entity
await self.repository.delete_by_entity_id(entity_id=entity.id)
with telemetry.scope(
"search.index_entity_data",
phase="index_entity_data",
result_count=1,
):
with telemetry.scope(
"search.index.delete_existing",
phase="delete_existing",
result_count=1,
):
await self.repository.delete_by_entity_id(entity_id=entity.id)
# reindex
await self.index_entity_markdown(
entity, content
) if entity.is_markdown else await self.index_entity_file(entity)
if entity.is_markdown:
await self.index_entity_markdown(entity, content)
else:
await self.index_entity_file(entity)
logger.debug(
f"[BackgroundTask] Completed search index for entity_id={entity.id} "
@@ -490,23 +522,28 @@ class SearchService:
self,
entity: Entity,
) -> None:
# Index entity file with no content
await self.repository.index_item(
SearchIndexRow(
id=entity.id,
entity_id=entity.id,
type=SearchItemType.ENTITY.value,
title=_strip_nul(entity.title),
permalink=entity.permalink, # Required for Postgres NOT NULL constraint
file_path=entity.file_path,
metadata={
"note_type": entity.note_type,
},
created_at=entity.created_at,
updated_at=_mtime_to_datetime(entity),
project_id=entity.project_id,
with telemetry.scope(
"search.index_file",
phase="index_file",
result_count=1,
):
# Index entity file with no content
await self.repository.index_item(
SearchIndexRow(
id=entity.id,
entity_id=entity.id,
type=SearchItemType.ENTITY.value,
title=_strip_nul(entity.title),
permalink=entity.permalink, # Required for Postgres NOT NULL constraint
file_path=entity.file_path,
metadata={
"note_type": entity.note_type,
},
created_at=entity.created_at,
updated_at=_mtime_to_datetime(entity),
project_id=entity.project_id,
)
)
)
async def index_entity_markdown(
self,
@@ -539,129 +576,144 @@ class SearchService:
The project_id is automatically added by the repository when indexing.
"""
# Collect all search index rows to batch insert at the end
rows_to_index = []
with telemetry.scope(
"search.index_markdown",
phase="index_markdown",
result_count=1,
):
rows_to_index = []
content_stems = []
content_snippet = ""
title_variants = self._generate_variants(entity.title)
content_stems.extend(title_variants)
content_stems = []
content_snippet = ""
title_variants = self._generate_variants(entity.title)
content_stems.extend(title_variants)
# Use provided content or read from file
if content is None:
content = await self.file_service.read_entity_content(entity)
if content:
content_stems.append(content)
# Store full content for vector embedding quality.
# The chunker in the vector pipeline splits this into
# appropriately-sized pieces for embedding.
content_snippet = _strip_nul(content)
if content is None:
with telemetry.scope(
"search.index.read_content",
phase="read_content",
result_count=1,
):
content = await self.file_service.read_entity_content(entity)
if content:
content_stems.append(content)
content_snippet = _strip_nul(content)
if entity.permalink:
content_stems.extend(self._generate_variants(entity.permalink))
with telemetry.scope(
"search.index.build_rows",
phase="build_rows",
result_count=1,
):
if entity.permalink:
content_stems.extend(self._generate_variants(entity.permalink))
content_stems.extend(self._generate_variants(entity.file_path))
content_stems.extend(self._generate_variants(entity.file_path))
# Add entity tags from frontmatter to search content
entity_tags = self._extract_entity_tags(entity)
if entity_tags:
content_stems.extend(entity_tags)
entity_tags = self._extract_entity_tags(entity)
if entity_tags:
content_stems.extend(entity_tags)
entity_content_stems = _strip_nul("\n".join(p for p in content_stems if p and p.strip()))
# Truncate to stay under Postgres's 8KB index row limit
if len(entity_content_stems) > MAX_CONTENT_STEMS_SIZE: # pragma: no cover
entity_content_stems = entity_content_stems[:MAX_CONTENT_STEMS_SIZE] # pragma: no cover
# Add entity row
rows_to_index.append(
SearchIndexRow(
id=entity.id,
type=SearchItemType.ENTITY.value,
title=_strip_nul(entity.title),
content_stems=entity_content_stems,
content_snippet=content_snippet,
permalink=entity.permalink,
file_path=entity.file_path,
entity_id=entity.id,
metadata={
"note_type": entity.note_type,
},
created_at=entity.created_at,
updated_at=_mtime_to_datetime(entity),
project_id=entity.project_id,
)
)
# Add observation rows - dedupe by permalink to avoid unique constraint violations
# Two observations with same entity/category/content generate identical permalinks
seen_permalinks: set[str] = {entity.permalink} if entity.permalink else set()
for obs in entity.observations:
obs_permalink = obs.permalink
if obs_permalink in seen_permalinks:
logger.debug(f"Skipping duplicate observation permalink: {obs_permalink}")
continue
seen_permalinks.add(obs_permalink)
# Index with parent entity's file path since that's where it's defined
obs_content_stems = _strip_nul(
"\n".join(p for p in self._generate_variants(obs.content) if p and p.strip())
)
# Truncate to stay under Postgres's 8KB index row limit
if len(obs_content_stems) > MAX_CONTENT_STEMS_SIZE: # pragma: no cover
obs_content_stems = obs_content_stems[:MAX_CONTENT_STEMS_SIZE] # pragma: no cover
rows_to_index.append(
SearchIndexRow(
id=obs.id,
type=SearchItemType.OBSERVATION.value,
title=_strip_nul(f"{obs.category}: {obs.content[:100]}..."),
content_stems=obs_content_stems,
content_snippet=_strip_nul(obs.content),
permalink=obs_permalink,
file_path=entity.file_path,
category=obs.category,
entity_id=entity.id,
metadata={
"tags": obs.tags,
},
created_at=entity.created_at,
updated_at=_mtime_to_datetime(entity),
project_id=entity.project_id,
entity_content_stems = _strip_nul(
"\n".join(p for p in content_stems if p and p.strip())
)
)
# Add relation rows (only outgoing relations defined in this file)
for rel in entity.outgoing_relations:
# Create descriptive title showing the relationship
relation_title = _strip_nul(
f"{rel.from_entity.title}{rel.to_entity.title}"
if rel.to_entity
else f"{rel.from_entity.title}"
)
if len(entity_content_stems) > MAX_CONTENT_STEMS_SIZE: # pragma: no cover
entity_content_stems = entity_content_stems[
:MAX_CONTENT_STEMS_SIZE
] # pragma: no cover
rel_content_stems = _strip_nul(
"\n".join(p for p in self._generate_variants(relation_title) if p and p.strip())
)
rows_to_index.append(
SearchIndexRow(
id=rel.id,
title=relation_title,
permalink=rel.permalink,
content_stems=rel_content_stems,
file_path=entity.file_path,
type=SearchItemType.RELATION.value,
entity_id=entity.id,
from_id=rel.from_id,
to_id=rel.to_id,
relation_type=rel.relation_type,
created_at=entity.created_at,
updated_at=_mtime_to_datetime(entity),
project_id=entity.project_id,
rows_to_index.append(
SearchIndexRow(
id=entity.id,
type=SearchItemType.ENTITY.value,
title=_strip_nul(entity.title),
content_stems=entity_content_stems,
content_snippet=content_snippet,
permalink=entity.permalink,
file_path=entity.file_path,
entity_id=entity.id,
metadata={
"note_type": entity.note_type,
},
created_at=entity.created_at,
updated_at=_mtime_to_datetime(entity),
project_id=entity.project_id,
)
)
)
# Batch insert all rows at once
await self.repository.bulk_index_items(rows_to_index)
seen_permalinks: set[str] = {entity.permalink} if entity.permalink else set()
for obs in entity.observations:
obs_permalink = obs.permalink
if obs_permalink in seen_permalinks:
logger.debug(f"Skipping duplicate observation permalink: {obs_permalink}")
continue
seen_permalinks.add(obs_permalink)
obs_content_stems = _strip_nul(
"\n".join(
p for p in self._generate_variants(obs.content) if p and p.strip()
)
)
if len(obs_content_stems) > MAX_CONTENT_STEMS_SIZE: # pragma: no cover
obs_content_stems = obs_content_stems[
:MAX_CONTENT_STEMS_SIZE
] # pragma: no cover
rows_to_index.append(
SearchIndexRow(
id=obs.id,
type=SearchItemType.OBSERVATION.value,
title=_strip_nul(f"{obs.category}: {obs.content[:100]}..."),
content_stems=obs_content_stems,
content_snippet=_strip_nul(obs.content),
permalink=obs_permalink,
file_path=entity.file_path,
category=obs.category,
entity_id=entity.id,
metadata={
"tags": obs.tags,
},
created_at=entity.created_at,
updated_at=_mtime_to_datetime(entity),
project_id=entity.project_id,
)
)
for rel in entity.outgoing_relations:
relation_title = _strip_nul(
f"{rel.from_entity.title} -> {rel.to_entity.title}"
if rel.to_entity
else f"{rel.from_entity.title}"
)
rel_content_stems = _strip_nul(
"\n".join(
p for p in self._generate_variants(relation_title) if p and p.strip()
)
)
rows_to_index.append(
SearchIndexRow(
id=rel.id,
title=relation_title,
permalink=rel.permalink,
content_stems=rel_content_stems,
file_path=entity.file_path,
type=SearchItemType.RELATION.value,
entity_id=entity.id,
from_id=rel.from_id,
to_id=rel.to_id,
relation_type=rel.relation_type,
created_at=entity.created_at,
updated_at=_mtime_to_datetime(entity),
project_id=entity.project_id,
)
)
with telemetry.scope(
"search.index.bulk_upsert",
phase="bulk_upsert",
result_count=len(rows_to_index),
):
await self.repository.bulk_index_items(rows_to_index)
async def delete_by_permalink(self, permalink: str):
"""Delete an item from the search index."""
+3 -1
View File
@@ -350,7 +350,9 @@ class SyncService:
# Only resolve relations if there were actual changes
# If no files changed, no new unresolved relations could have been created
if report.total > 0:
with telemetry.scope("sync.project.resolve_relations", relation_scope="all_pending"):
with telemetry.scope(
"sync.project.resolve_relations", relation_scope="all_pending"
):
await self.resolve_relations()
else:
logger.info("Skipping relation resolution - no file changes detected")
+12 -32
View File
@@ -8,7 +8,6 @@ helpers for manual spans and logger context binding.
from __future__ import annotations
from contextlib import contextmanager
from contextvars import ContextVar
from dataclasses import dataclass, field
from typing import Any, Iterator
@@ -41,7 +40,6 @@ class TelemetryState:
_STATE = TelemetryState()
_LOGFIRE_HANDLER: dict[str, Any] | None = None
_ACTIVE_LOG_CONTEXT: ContextVar[dict[str, Any]] = ContextVar("basic_memory_log_context", default={})
def reset_telemetry_state() -> None:
@@ -57,7 +55,6 @@ def reset_telemetry_state() -> None:
_STATE.send_to_logfire = False
_STATE.warnings.clear()
_LOGFIRE_HANDLER = None
_ACTIVE_LOG_CONTEXT.set({})
def _filter_attributes(attrs: dict[str, Any]) -> dict[str, Any]:
@@ -65,11 +62,6 @@ def _filter_attributes(attrs: dict[str, Any]) -> dict[str, Any]:
return {key: value for key, value in attrs.items() if value is not None}
def _current_log_context() -> dict[str, Any]:
"""Return the currently active telemetry context for this execution flow."""
return dict(_ACTIVE_LOG_CONTEXT.get())
def configure_telemetry(
service_name: str,
*,
@@ -144,26 +136,11 @@ def pop_telemetry_warnings() -> list[str]:
return warnings
def bind_telemetry_context(**attrs: Any):
"""Bind stable telemetry attributes onto the shared Loguru logger."""
merged_attrs = _current_log_context()
merged_attrs.update(_filter_attributes(attrs))
return logger.bind(**merged_attrs)
@contextmanager
def contextualize(**attrs: Any) -> Iterator[None]:
"""Apply stable telemetry attributes to all Loguru calls in this scope."""
filtered_attrs = _filter_attributes(attrs)
merged_attrs = _current_log_context()
merged_attrs.update(filtered_attrs)
context_token = _ACTIVE_LOG_CONTEXT.set(merged_attrs)
try:
with logger.contextualize(**filtered_attrs):
yield
finally:
_ACTIVE_LOG_CONTEXT.reset(context_token)
"""Apply filtered telemetry attributes to Loguru calls in this scope."""
with logger.contextualize(**_filter_attributes(attrs)):
yield
@contextmanager
@@ -182,21 +159,23 @@ operation = scope
@contextmanager
def span(name: str, **attrs: Any) -> Iterator[None]:
"""Create a manual Logfire span when telemetry is enabled."""
if not telemetry_enabled():
with started_span(name, **attrs):
yield
return
@contextmanager
def started_span(name: str, **attrs: Any) -> Iterator[Any | None]:
"""Create a manual Logfire span and expose the active span handle when available."""
logfire = _load_logfire()
if logfire is None: # pragma: no cover
if logfire is None or not _STATE.configured: # pragma: no cover
yield # pragma: no cover
return # pragma: no cover
with logfire.span(name, **_filter_attributes(attrs)):
yield
with logfire.span(name, **_filter_attributes(attrs)) as active_span:
yield active_span
__all__ = [
"bind_telemetry_context",
"contextualize",
"configure_telemetry",
"get_logfire_handler",
@@ -205,5 +184,6 @@ __all__ = [
"reset_telemetry_state",
"scope",
"span",
"started_span",
"telemetry_enabled",
]
+84 -25
View File
@@ -51,7 +51,7 @@ The `app` fixture ensures FastAPI dependency overrides are active, and
"""
import os
from typing import AsyncGenerator, Literal
from typing import AsyncGenerator, Generator, Literal
import pytest
import pytest_asyncio
@@ -63,7 +63,13 @@ from testcontainers.postgres import PostgresContainer
from httpx import AsyncClient, ASGITransport
from basic_memory.config import BasicMemoryConfig, ProjectConfig, ConfigManager, DatabaseBackend
from basic_memory.config import (
BasicMemoryConfig,
ProjectConfig,
ProjectEntry,
ConfigManager,
DatabaseBackend,
)
from basic_memory.db import engine_session_factory, DatabaseType
from basic_memory.models import Project
from basic_memory.models.base import Base
@@ -103,7 +109,7 @@ def postgres_container(db_backend):
Uses testcontainers to spin up a real Postgres instance.
Only starts if db_backend is "postgres".
"""
if db_backend != "postgres":
if db_backend != "postgres" or _configured_postgres_sync_url():
yield None
return
@@ -112,6 +118,70 @@ def postgres_container(db_backend):
yield postgres
POSTGRES_EPHEMERAL_TABLES = [
"search_vector_embeddings",
"search_vector_chunks",
"search_vector_index",
]
def _configured_postgres_sync_url() -> str | None:
"""Prefer an externally managed Postgres server when CI provides one."""
configured_url = os.environ.get("BASIC_MEMORY_TEST_POSTGRES_URL") or os.environ.get(
"POSTGRES_TEST_URL"
)
if not configured_url:
return None
return (
configured_url.replace("postgresql+asyncpg://", "postgresql+psycopg2://", 1)
.replace("postgresql://", "postgresql+psycopg2://", 1)
.replace("postgres://", "postgresql+psycopg2://", 1)
)
def _postgres_reset_tables() -> list[str]:
"""Resolve the current ORM table set at reset time."""
return [table.name for table in Base.metadata.sorted_tables] + ["search_index"]
def _resolve_postgres_sync_url(postgres_container) -> str:
"""Use CI's shared service when configured, otherwise fall back to testcontainers."""
configured_url = _configured_postgres_sync_url()
if configured_url:
return configured_url
assert postgres_container is not None
return postgres_container.get_connection_url()
async def _reset_postgres_integration_schema(engine) -> None:
"""Restore the shared Postgres integration schema to a clean baseline."""
from basic_memory.models.search import (
CREATE_POSTGRES_SEARCH_INDEX_FTS,
CREATE_POSTGRES_SEARCH_INDEX_METADATA,
CREATE_POSTGRES_SEARCH_INDEX_PERMALINK,
CREATE_POSTGRES_SEARCH_INDEX_TABLE,
)
async with engine.begin() as conn:
# Trigger: integration tests may leave behind temporary search/vector tables while
# exercising full-stack recovery paths.
# Why: recreating only the missing schema is much cheaper than dropping every table.
# Outcome: each integration test gets the same baseline without paying repeated full DDL cost.
await conn.run_sync(Base.metadata.create_all)
await conn.execute(CREATE_POSTGRES_SEARCH_INDEX_TABLE)
await conn.execute(CREATE_POSTGRES_SEARCH_INDEX_FTS)
await conn.execute(CREATE_POSTGRES_SEARCH_INDEX_METADATA)
await conn.execute(CREATE_POSTGRES_SEARCH_INDEX_PERMALINK)
for table_name in POSTGRES_EPHEMERAL_TABLES:
await conn.execute(text(f"DROP TABLE IF EXISTS {table_name} CASCADE"))
await conn.execute(
text(f"TRUNCATE TABLE {', '.join(_postgres_reset_tables())} RESTART IDENTITY CASCADE")
)
@pytest_asyncio.fixture
async def engine_factory(
app_config,
@@ -121,18 +191,12 @@ async def engine_factory(
tmp_path,
) -> AsyncGenerator[tuple, None]:
"""Create engine and session factory for the configured database backend."""
from basic_memory.models.search import (
CREATE_SEARCH_INDEX,
CREATE_POSTGRES_SEARCH_INDEX_TABLE,
CREATE_POSTGRES_SEARCH_INDEX_FTS,
CREATE_POSTGRES_SEARCH_INDEX_METADATA,
CREATE_POSTGRES_SEARCH_INDEX_PERMALINK,
)
from basic_memory.models.search import CREATE_SEARCH_INDEX
from basic_memory import db
if db_backend == "postgres":
# Postgres mode using testcontainers
sync_url = postgres_container.get_connection_url()
sync_url = _resolve_postgres_sync_url(postgres_container)
async_url = sync_url.replace("postgresql+psycopg2", "postgresql+asyncpg")
engine = create_async_engine(
@@ -153,16 +217,7 @@ async def engine_factory(
db._engine = engine
db._session_maker = session_maker
# Drop and recreate all tables for test isolation
async with engine.begin() as conn:
await conn.execute(text("DROP TABLE IF EXISTS search_index CASCADE"))
await conn.run_sync(Base.metadata.drop_all)
await conn.run_sync(Base.metadata.create_all)
# asyncpg requires separate execute calls for each statement
await conn.execute(CREATE_POSTGRES_SEARCH_INDEX_TABLE)
await conn.execute(CREATE_POSTGRES_SEARCH_INDEX_FTS)
await conn.execute(CREATE_POSTGRES_SEARCH_INDEX_METADATA)
await conn.execute(CREATE_POSTGRES_SEARCH_INDEX_PERMALINK)
await _reset_postgres_integration_schema(engine)
yield engine, session_maker
@@ -228,13 +283,15 @@ def app_config(
monkeypatch.setenv("BASIC_MEMORY_CLOUD_MODE", "false")
# Create a basic config with test-project like unit tests do
projects = {"test-project": str(config_home)}
projects = {"test-project": ProjectEntry(path=str(config_home))}
# Configure database backend based on env var
if db_backend == "postgres":
database_backend = DatabaseBackend.POSTGRES
# Get URL from testcontainer and convert to asyncpg driver
sync_url = postgres_container.get_connection_url()
# Trigger: CI jobs can provide a shared Postgres service instead of per-session containers.
# Why: reusing one pgvector-enabled server avoids Docker startup churn on every job.
# Outcome: local runs keep using testcontainers, while CI injects a stable service URL.
sync_url = _resolve_postgres_sync_url(postgres_container)
database_url = sync_url.replace("postgresql+psycopg2", "postgresql+asyncpg")
else:
database_backend = DatabaseBackend.SQLITE
@@ -285,7 +342,9 @@ def project_config(test_project):
@pytest.fixture
def app(app_config, project_config, engine_factory, test_project, config_manager) -> FastAPI:
def app(
app_config, project_config, engine_factory, test_project, config_manager
) -> Generator[FastAPI, None, None]:
"""Create test FastAPI application with single project."""
# Import the FastAPI app AFTER the config_manager has written the test config to disk
+58
View File
@@ -137,6 +137,58 @@ async def test_get_entity_by_id(client: AsyncClient, test_graph, v2_project_url,
assert entity.api_version == "v2"
@pytest.mark.asyncio
async def test_get_entity_by_id_allows_long_relation_type(
client: AsyncClient,
v2_project_url,
relation_repository,
):
"""GET entity should not fail when stored relation_type exceeds 200 characters."""
source_response = await client.post(
f"{v2_project_url}/knowledge/entities",
json={
"title": "Long Relation Source",
"directory": "test",
"content": "Source entity content",
},
)
assert source_response.status_code == 200
source_entity = EntityResponseV2.model_validate(source_response.json())
target_response = await client.post(
f"{v2_project_url}/knowledge/entities",
json={
"title": "Long Relation Target",
"directory": "test",
"content": "Target entity content",
},
)
assert target_response.status_code == 200
target_entity = EntityResponseV2.model_validate(target_response.json())
long_relation_type = (
"**Architecture/efficiency concern:** "
"the orchestration prompt expanded a short edge label into a full descriptive note "
"that is much longer than 200 characters but should still serialize cleanly."
)
await relation_repository.create(
{
"from_id": source_entity.id,
"to_id": target_entity.id,
"to_name": target_entity.title,
"relation_type": long_relation_type,
}
)
response = await client.get(f"{v2_project_url}/knowledge/entities/{source_entity.external_id}")
assert response.status_code == 200
entity = EntityResponseV2.model_validate(response.json())
assert len(entity.relations) == 1
assert entity.relations[0].relation_type == long_relation_type
@pytest.mark.asyncio
async def test_get_entity_by_id_not_found(client: AsyncClient, v2_project_url):
"""Test getting a non-existent entity by external_id returns 404."""
@@ -303,6 +355,7 @@ async def test_update_entity_by_id(
response = await client.put(
f"{v2_project_url}/knowledge/entities/{original_external_id}",
json=update_data,
params={"fast": False},
)
assert response.status_code == 200
@@ -311,6 +364,8 @@ async def test_update_entity_by_id(
# V2 update must return external_id field
assert updated_entity.external_id is not None
assert updated_entity.api_version == "v2"
assert updated_entity.content is not None
assert "Updated content via V2" in updated_entity.content
# Verify file was updated
file_path = file_service.get_entity_path(updated_entity)
@@ -480,6 +535,7 @@ async def test_edit_entity_by_id_append(
response = await client.patch(
f"{v2_project_url}/knowledge/entities/{original_external_id}",
json=edit_data,
params={"fast": False},
)
assert response.status_code == 200
@@ -488,6 +544,8 @@ async def test_edit_entity_by_id_append(
# V2 patch must return external_id field
assert edited_entity.external_id is not None
assert edited_entity.api_version == "v2"
assert edited_entity.content is not None
assert "Appended content" in edited_entity.content
# Verify file has both original and appended content
file_path = file_service.get_entity_path(edited_entity)
@@ -0,0 +1,245 @@
"""Telemetry coverage for the v2 knowledge router."""
from __future__ import annotations
import importlib
from contextlib import contextmanager
from datetime import datetime, timezone
from types import SimpleNamespace
import pytest
from fastapi import BackgroundTasks, Response
from basic_memory.schemas.base import Entity
from basic_memory.schemas.request import EditEntityRequest
knowledge_router_module = importlib.import_module("basic_memory.api.v2.routers.knowledge_router")
def _capture_spans():
spans: list[tuple[str, dict]] = []
@contextmanager
def fake_span(name: str, **attrs):
spans.append((name, attrs))
yield
return spans, fake_span
def _fake_entity(*, external_id: str = "entity-123", file_path: str = "notes/test.md"):
now = datetime.now(timezone.utc)
return SimpleNamespace(
external_id=external_id,
id=1,
title="Telemetry Entity",
note_type="note",
content_type="text/markdown",
permalink="notes/test",
file_path=file_path,
entity_metadata=None,
observations=[],
relations=[],
created_at=now,
updated_at=now,
created_by=None,
last_updated_by=None,
)
def _assert_names_in_order(names: list[str], expected: list[str]) -> None:
cursor = 0
for expected_name in expected:
cursor = names.index(expected_name, cursor) + 1
@pytest.mark.asyncio
async def test_create_entity_emits_root_and_nested_spans(monkeypatch) -> None:
spans, fake_span = _capture_spans()
monkeypatch.setattr(knowledge_router_module.telemetry, "span", fake_span)
entity = _fake_entity()
response_content = (
"---\ntitle: Telemetry Entity\ntype: note\npermalink: notes/test\n---\n\ntelemetry content"
)
class FakeEntityService:
async def create_entity_with_content(self, data):
return SimpleNamespace(
entity=entity,
content=response_content,
search_content="telemetry content",
)
class FakeSearchService:
async def index_entity(self, entity, content=None):
assert content == "telemetry content"
return None
class FakeTaskScheduler:
def schedule(self, *args, **kwargs):
return None
class FakeFileService:
async def read_file_content(self, path):
raise AssertionError("non-fast create should not re-read file content")
result = await knowledge_router_module.create_entity(
project_id="project-123",
data=Entity(
title="Telemetry Entity",
directory="notes",
note_type="note",
content_type="text/markdown",
content="telemetry content",
),
background_tasks=BackgroundTasks(),
entity_service=FakeEntityService(),
search_service=FakeSearchService(),
task_scheduler=FakeTaskScheduler(),
file_service=FakeFileService(),
app_config=SimpleNamespace(semantic_search_enabled=False),
fast=False,
)
assert result.content == response_content
_assert_names_in_order(
[name for name, _ in spans],
[
"api.request.knowledge.create_entity",
"api.knowledge.create_entity.write_entity",
"api.knowledge.create_entity.search_index",
"api.knowledge.create_entity.vector_sync",
"api.knowledge.create_entity.read_content",
],
)
@pytest.mark.asyncio
async def test_update_entity_emits_root_and_nested_spans(monkeypatch) -> None:
spans, fake_span = _capture_spans()
monkeypatch.setattr(knowledge_router_module.telemetry, "span", fake_span)
entity = _fake_entity()
response_content = "---\ntitle: Telemetry Entity\ntype: note\npermalink: notes/test\n---\n\nupdated telemetry content"
class FakeEntityService:
async def update_entity_with_content(self, existing, data):
return SimpleNamespace(
entity=entity,
content=response_content,
search_content="updated telemetry content",
)
class FakeSearchService:
async def index_entity(self, entity, content=None):
assert content == "updated telemetry content"
return None
class FakeEntityRepository:
async def get_by_external_id(self, external_id):
return entity
class FakeTaskScheduler:
def schedule(self, *args, **kwargs):
return None
class FakeFileService:
async def read_file_content(self, path):
raise AssertionError("non-fast update should not re-read file content")
response = Response()
result = await knowledge_router_module.update_entity_by_id(
data=Entity(
title="Telemetry Entity",
directory="notes",
note_type="note",
content_type="text/markdown",
content="updated telemetry content",
),
response=response,
background_tasks=BackgroundTasks(),
project_id="project-123",
entity_service=FakeEntityService(),
search_service=FakeSearchService(),
entity_repository=FakeEntityRepository(),
task_scheduler=FakeTaskScheduler(),
file_service=FakeFileService(),
app_config=SimpleNamespace(semantic_search_enabled=False),
entity_id=entity.external_id,
fast=False,
)
assert result.content == response_content
_assert_names_in_order(
[name for name, _ in spans],
[
"api.request.knowledge.update_entity",
"api.knowledge.update_entity.load_entity",
"api.knowledge.update_entity.write_entity",
"api.knowledge.update_entity.search_index",
"api.knowledge.update_entity.vector_sync",
"api.knowledge.update_entity.read_content",
],
)
@pytest.mark.asyncio
async def test_edit_entity_emits_root_and_nested_spans(monkeypatch) -> None:
spans, fake_span = _capture_spans()
monkeypatch.setattr(knowledge_router_module.telemetry, "span", fake_span)
entity = _fake_entity()
response_content = "---\ntitle: Telemetry Entity\ntype: note\npermalink: notes/test\n---\n\nedited telemetry content"
class FakeEntityService:
async def edit_entity_with_content(self, **kwargs):
return SimpleNamespace(
entity=entity,
content=response_content,
search_content="edited telemetry content",
)
class FakeSearchService:
async def index_entity(self, entity, content=None):
assert content == "edited telemetry content"
return None
class FakeEntityRepository:
async def get_by_external_id(self, external_id):
return entity
class FakeTaskScheduler:
def schedule(self, *args, **kwargs):
return None
class FakeFileService:
async def read_file_content(self, path):
raise AssertionError("non-fast edit should not re-read file content")
result = await knowledge_router_module.edit_entity_by_id(
data=EditEntityRequest(operation="append", content="edited telemetry content"),
background_tasks=BackgroundTasks(),
project_id="project-123",
entity_service=FakeEntityService(),
search_service=FakeSearchService(),
entity_repository=FakeEntityRepository(),
task_scheduler=FakeTaskScheduler(),
file_service=FakeFileService(),
app_config=SimpleNamespace(semantic_search_enabled=False),
entity_id=entity.external_id,
fast=False,
)
assert result.content == response_content
_assert_names_in_order(
[name for name, _ in spans],
[
"api.request.knowledge.edit_entity",
"api.knowledge.edit_entity.load_entity",
"api.knowledge.edit_entity.write_entity",
"api.knowledge.edit_entity.search_index",
"api.knowledge.edit_entity.vector_sync",
"api.knowledge.edit_entity.read_content",
],
)
+187
View File
@@ -0,0 +1,187 @@
"""Tests for graph context hydration in to_graph_context().
Proves that recent-activity/build-context hydration batches entity lookups
for entities, observations, and relations in a single repository call.
"""
from __future__ import annotations
from datetime import datetime, timezone
from types import SimpleNamespace
import pytest
from basic_memory.api.v2.utils import to_graph_context
from basic_memory.schemas.search import SearchItemType
from basic_memory.services.context_service import (
ContextMetadata,
ContextResult as ServiceContextResult,
ContextResultItem,
ContextResultRow,
)
# --- Helpers ---
def _make_entity(id: int, title: str, external_id: str) -> SimpleNamespace:
return SimpleNamespace(id=id, title=title, external_id=external_id)
def _make_row(*, type: str, id: int, root_id: int, **kwargs) -> ContextResultRow:
now = kwargs.pop("created_at", datetime.now(timezone.utc))
defaults = dict(
title=f"Item {id}",
permalink=f"notes/{id}",
file_path=f"notes/{id}.md",
depth=0,
root_id=root_id,
created_at=now,
)
defaults.update(kwargs)
return ContextResultRow(type=type, id=id, **defaults)
class SpyEntityRepository:
"""Tracks batched ID lookups and returns entities from a preset map."""
def __init__(self, entities_by_id: dict[int, SimpleNamespace]):
self.entities_by_id = entities_by_id
self.calls: list[list[int]] = []
async def find_by_ids(self, ids: list[int]):
self.calls.append(ids)
return [self.entities_by_id[i] for i in ids if i in self.entities_by_id]
# --- Single batch fetch (N+1 elimination) ---
@pytest.mark.asyncio
async def test_to_graph_context_batches_entity_hydration_for_recent_activity():
"""Mixed entity, observation, and relation items must hydrate in one lookup."""
repo = SpyEntityRepository(
{
1: _make_entity(1, "Root", "ext-root"),
2: _make_entity(2, "Child", "ext-child"),
3: _make_entity(3, "Peer", "ext-peer"),
}
)
now = datetime.now(timezone.utc)
root_entity = _make_row(
type="entity",
id=1,
root_id=1,
title="Root",
permalink="notes/root",
file_path="notes/root.md",
created_at=now,
)
root_observation = _make_row(
type="observation",
id=10,
root_id=1,
title="fact: observed",
permalink="notes/root/observations/fact/observed",
file_path="notes/root.md",
category="fact",
content="observed",
entity_id=1,
created_at=now,
)
root_relation = _make_row(
type="relation",
id=20,
root_id=1,
title="links_to: Child",
permalink="notes/root",
file_path="notes/root.md",
relation_type="links_to",
from_id=1,
to_id=2,
depth=1,
created_at=now,
)
child_observation = _make_row(
type="observation",
id=11,
root_id=11,
title="note: child update",
permalink="notes/child/observations/note/update",
file_path="notes/child.md",
category="note",
content="child update",
entity_id=2,
created_at=now,
)
peer_entity = _make_row(
type="entity",
id=3,
root_id=11,
title="Peer",
permalink="notes/peer",
file_path="notes/peer.md",
depth=1,
created_at=now,
)
context = ServiceContextResult(
results=[
ContextResultItem(
primary_result=root_entity,
observations=[root_observation],
related_results=[root_relation],
),
ContextResultItem(
primary_result=child_observation,
observations=[],
related_results=[peer_entity],
),
],
metadata=ContextMetadata(
types=[
SearchItemType.ENTITY,
SearchItemType.OBSERVATION,
SearchItemType.RELATION,
],
depth=1,
primary_count=2,
related_count=2,
total_relations=1,
total_observations=1,
),
)
graph = await to_graph_context(context, entity_repository=repo, page=1, page_size=10)
assert len(repo.calls) == 1, f"Expected 1 entity lookup, got {len(repo.calls)}"
assert set(repo.calls[0]) == {1, 2, 3}
first_result = graph.results[0]
assert first_result.primary_result.external_id == "ext-root"
assert first_result.observations[0].entity_external_id == "ext-root"
assert first_result.observations[0].title == "Root"
relation = first_result.related_results[0]
assert relation.from_entity == "Root"
assert relation.from_entity_external_id == "ext-root"
assert relation.to_entity == "Child"
assert relation.to_entity_external_id == "ext-child"
second_result = graph.results[1]
assert second_result.primary_result.entity_external_id == "ext-child"
assert second_result.primary_result.title == "Child"
assert second_result.related_results[0].external_id == "ext-peer"
@pytest.mark.asyncio
async def test_to_graph_context_empty_results_skip_entity_lookup():
"""An empty context result should not perform any entity hydration lookup."""
repo = SpyEntityRepository({})
context = ServiceContextResult(results=[], metadata=ContextMetadata(depth=1))
graph = await to_graph_context(context, entity_repository=repo)
assert repo.calls == []
assert list(graph.results) == []
+305
View File
@@ -0,0 +1,305 @@
"""Tests for search result hydration in to_search_results().
Proves that the batch fetch eliminates N+1 queries and that
entity ID lookups are correct across all result types.
"""
from __future__ import annotations
from datetime import datetime, timezone
from types import SimpleNamespace
import pytest
from basic_memory.api.v2.utils import to_search_results
from basic_memory.repository.search_index_row import SearchIndexRow
# --- Helpers ---
def _make_entity(id: int, permalink: str) -> SimpleNamespace:
return SimpleNamespace(id=id, permalink=permalink)
def _make_row(*, type: str, id: int, **kwargs) -> SearchIndexRow:
now = datetime.now(timezone.utc)
defaults = dict(
project_id=1,
file_path=f"notes/{id}.md",
created_at=now,
updated_at=now,
score=1.0,
title=f"Item {id}",
permalink=f"notes/{id}",
)
defaults.update(kwargs)
return SearchIndexRow(type=type, id=id, **defaults)
class SpyEntityService:
"""Tracks calls to get_entities_by_id and returns from a preset lookup."""
def __init__(self, entities_by_id: dict[int, SimpleNamespace]):
self.entities_by_id = entities_by_id
self.calls: list[list[int]] = []
async def get_entities_by_id(self, ids: list[int]):
self.calls.append(ids)
return [self.entities_by_id[i] for i in ids if i in self.entities_by_id]
# --- Single batch fetch (N+1 elimination) ---
@pytest.mark.asyncio
async def test_single_db_call_for_multiple_results():
"""Multiple search results must trigger exactly one get_entities_by_id call."""
service = SpyEntityService(
{
1: _make_entity(1, "notes/a"),
2: _make_entity(2, "notes/b"),
3: _make_entity(3, "notes/c"),
}
)
results = [
_make_row(type="entity", id=1, entity_id=1),
_make_row(type="entity", id=2, entity_id=2),
_make_row(type="entity", id=3, entity_id=3),
]
await to_search_results(service, results)
assert len(service.calls) == 1, f"Expected 1 DB call, got {len(service.calls)}"
@pytest.mark.asyncio
async def test_no_db_call_for_empty_results():
"""Empty result list should not make any DB call."""
service = SpyEntityService({})
search_results = await to_search_results(service, [])
assert len(service.calls) == 0
assert search_results == []
# --- ID deduplication ---
@pytest.mark.asyncio
async def test_deduplicates_entity_ids():
"""Shared entity IDs across results should be fetched once, not per-result."""
# entity_id=1 appears in all three results, from_id=1 overlaps with entity_id
service = SpyEntityService(
{
1: _make_entity(1, "notes/shared"),
2: _make_entity(2, "notes/target-a"),
3: _make_entity(3, "notes/target-b"),
}
)
results = [
_make_row(type="relation", id=10, entity_id=1, from_id=1, to_id=2, relation_type="links"),
_make_row(type="relation", id=11, entity_id=1, from_id=1, to_id=3, relation_type="links"),
]
await to_search_results(service, results)
# Single call with deduplicated IDs: {1, 2, 3}
assert len(service.calls) == 1
fetched_ids = set(service.calls[0])
assert fetched_ids == {1, 2, 3}
# --- Correct entity-to-field mapping ---
@pytest.mark.asyncio
async def test_entity_result_maps_permalink():
"""Entity results should populate the 'entity' field with the entity's permalink."""
service = SpyEntityService({5: _make_entity(5, "notes/my-entity")})
results = [_make_row(type="entity", id=5, entity_id=5)]
search_results = await to_search_results(service, results)
assert len(search_results) == 1
r = search_results[0]
assert r.entity == "notes/my-entity"
assert r.entity_id == 5
assert r.from_entity is None
assert r.to_entity is None
@pytest.mark.asyncio
async def test_observation_result_maps_parent_entity():
"""Observation results should populate 'entity' with the parent entity's permalink."""
service = SpyEntityService({10: _make_entity(10, "notes/parent")})
results = [_make_row(type="observation", id=20, entity_id=10)]
search_results = await to_search_results(service, results)
r = search_results[0]
assert r.entity == "notes/parent"
assert r.entity_id == 10
assert r.observation_id == 20
assert r.from_entity is None
assert r.to_entity is None
@pytest.mark.asyncio
async def test_relation_result_maps_from_and_to():
"""Relation results should populate entity, from_entity, and to_entity correctly."""
service = SpyEntityService(
{
1: _make_entity(1, "notes/parent"),
2: _make_entity(2, "notes/source"),
3: _make_entity(3, "notes/target"),
}
)
results = [
_make_row(
type="relation",
id=99,
entity_id=1,
from_id=2,
to_id=3,
relation_type="references",
)
]
search_results = await to_search_results(service, results)
r = search_results[0]
assert r.entity == "notes/parent"
assert r.from_entity == "notes/source"
assert r.to_entity == "notes/target"
assert r.relation_id == 99
assert r.relation_type == "references"
@pytest.mark.asyncio
async def test_relation_with_distinct_entity_and_from_ids():
"""When entity_id != from_id, from_entity must use from_id's permalink, not entity_id's.
This was a bug in the old positional-index code: entities[0] was used for both
'entity' and 'from_entity', which was wrong when entity_id != from_id.
"""
service = SpyEntityService(
{
10: _make_entity(10, "notes/parent-entity"),
20: _make_entity(20, "notes/actual-source"),
30: _make_entity(30, "notes/target"),
}
)
results = [
_make_row(
type="relation",
id=50,
entity_id=10,
from_id=20,
to_id=30,
relation_type="derived_from",
)
]
search_results = await to_search_results(service, results)
r = search_results[0]
# entity should be the parent entity (entity_id=10)
assert r.entity == "notes/parent-entity"
# from_entity must be from_id=20, NOT entity_id=10
assert r.from_entity == "notes/actual-source"
assert r.to_entity == "notes/target"
# --- Mixed result types ---
@pytest.mark.asyncio
async def test_mixed_result_types_single_fetch():
"""A mix of entity, observation, and relation results should all hydrate in one fetch."""
service = SpyEntityService(
{
1: _make_entity(1, "notes/entity-one"),
2: _make_entity(2, "notes/entity-two"),
3: _make_entity(3, "notes/entity-three"),
}
)
results = [
_make_row(type="entity", id=1, entity_id=1),
_make_row(type="observation", id=10, entity_id=2, category="fact"),
_make_row(type="relation", id=20, entity_id=1, from_id=1, to_id=3, relation_type="links"),
]
search_results = await to_search_results(service, results)
# Single DB call
assert len(service.calls) == 1
# Entity result
assert search_results[0].entity == "notes/entity-one"
assert search_results[0].entity_id == 1
# Observation result
assert search_results[1].entity == "notes/entity-two"
assert search_results[1].observation_id == 10
# Relation result
assert search_results[2].from_entity == "notes/entity-one"
assert search_results[2].to_entity == "notes/entity-three"
# --- Graceful handling of missing entities ---
@pytest.mark.asyncio
async def test_missing_entity_returns_none_permalink():
"""If an entity ID isn't found in the DB, permalink fields should be None."""
# Only entity 1 exists; entity 99 (to_id) is missing
service = SpyEntityService({1: _make_entity(1, "notes/source")})
results = [
_make_row(type="relation", id=5, entity_id=1, from_id=1, to_id=99, relation_type="links")
]
search_results = await to_search_results(service, results)
r = search_results[0]
assert r.entity == "notes/source"
assert r.from_entity == "notes/source"
assert r.to_entity is None # entity 99 not found
@pytest.mark.asyncio
async def test_null_ids_handled_gracefully():
"""Results with None entity_id/from_id/to_id should not cause errors."""
service = SpyEntityService({})
# Entity result: entity_id is the row id itself, from_id/to_id are None
results = [_make_row(type="entity", id=1)]
search_results = await to_search_results(service, results)
# No entity_id on the row means no fetch needed, all fields None
r = search_results[0]
assert r.entity is None
assert r.from_entity is None
assert r.to_entity is None
# --- Scaling: prove O(1) DB calls ---
@pytest.mark.asyncio
async def test_single_db_call_scales_to_many_results():
"""Even with many results, only one DB call should be made."""
n = 50
entities = {i: _make_entity(i, f"notes/e-{i}") for i in range(1, n + 1)}
service = SpyEntityService(entities)
results = [_make_row(type="entity", id=i, entity_id=i) for i in range(1, n + 1)]
search_results = await to_search_results(service, results)
assert len(service.calls) == 1, f"Expected 1 DB call for {n} results, got {len(service.calls)}"
assert len(search_results) == n
# Every result got its permalink
for i, r in enumerate(search_results, start=1):
assert r.entity == f"notes/e-{i}"
+4 -3
View File
@@ -51,12 +51,13 @@ async def test_search_router_wraps_request_in_manual_operation() -> None:
"api.request.search",
{
"entrypoint": "api",
"domain": "search",
"action": "search",
"page": 2,
"page_size": 5,
"retrieval_mode": "fts",
"has_text_query": True,
"has_title_query": False,
"has_permalink_query": False,
"has_query": True,
"has_filters": False,
},
)
]
+66
View File
@@ -0,0 +1,66 @@
"""Telemetry coverage for API v2 hydration utilities."""
from __future__ import annotations
import importlib
from contextlib import contextmanager
from datetime import datetime, timezone
from types import SimpleNamespace
import pytest
from basic_memory.repository.search_index_row import SearchIndexRow
utils_module = importlib.import_module("basic_memory.api.v2.utils")
def _capture_spans():
spans: list[tuple[str, dict]] = []
@contextmanager
def fake_span(name: str, **attrs):
spans.append((name, attrs))
yield
return spans, fake_span
@pytest.mark.asyncio
async def test_to_search_results_emits_hydration_spans(monkeypatch) -> None:
spans, fake_span = _capture_spans()
monkeypatch.setattr(utils_module.telemetry, "span", fake_span)
class FakeEntityService:
async def get_entities_by_id(self, ids):
return [
SimpleNamespace(id=1, permalink="notes/root"),
SimpleNamespace(id=2, permalink="notes/child"),
]
now = datetime.now(timezone.utc)
results = [
SearchIndexRow(
project_id=1,
id=1,
type="relation",
file_path="notes/root.md",
created_at=now,
updated_at=now,
permalink="notes/root/relates_to/notes/child",
entity_id=1,
from_id=1,
to_id=2,
relation_type="relates_to",
title="Root relates to Child",
score=1.0,
)
]
search_results = await utils_module.to_search_results(FakeEntityService(), results)
assert search_results[0].relation_type == "relates_to"
assert [name for name, _ in spans] == [
"search.hydrate_results",
"search.hydrate_results.fetch_entities",
"search.hydrate_results.shape_results",
]
@@ -10,10 +10,12 @@ from basic_memory.cli.commands.cloud.api_client import (
make_api_request,
)
from basic_memory.cli.commands.cloud.cloud_utils import (
CloudUtilsError,
create_cloud_project,
fetch_cloud_projects,
project_exists,
)
from basic_memory.config import ProjectMode
@pytest.mark.asyncio
@@ -163,6 +165,109 @@ async def test_cloud_utils_fetch_and_exists_and_create_project(
assert created.new_project["name"] == "My Project"
# Path should be permalink-like (kebab)
assert seen["create_payload"]["path"] == "my-project"
assert seen["create_payload"]["visibility"] == "workspace"
@pytest.mark.asyncio
async def test_create_cloud_project_accepts_visibility_override(config_home, config_manager):
"""Shared cloud helper should pass explicit visibility through to the API payload."""
config = config_manager.load_config()
config.cloud_host = "https://cloud.example.test"
config_manager.save_config(config)
seen_payload: dict | None = None
async def api_request(**kwargs):
nonlocal seen_payload
seen_payload = kwargs["json_data"]
return httpx.Response(
200,
json={
"message": "created",
"status": "success",
"default": False,
"old_project": None,
"new_project": {"name": "shared-project", "path": "shared-project"},
},
)
created = await create_cloud_project(
"Shared Project",
visibility="shared",
api_request=api_request,
)
assert created.new_project is not None
assert seen_payload == {
"name": "Shared Project",
"path": "shared-project",
"set_default": False,
"visibility": "shared",
}
@pytest.mark.asyncio
async def test_cloud_utils_use_configured_workspace_headers(config_home, config_manager):
"""Workspace-aware cloud helpers should prefer project workspace over global default."""
config = config_manager.load_config()
config.cloud_host = "https://cloud.example.test"
config.default_workspace = "default-workspace"
config.set_project_mode("alpha", ProjectMode.CLOUD)
config.projects["alpha"].workspace_id = "project-workspace"
config_manager.save_config(config)
seen: list[tuple[str, str | None]] = []
async def api_request(**kwargs):
seen.append(
(
kwargs["method"],
(kwargs.get("headers") or {}).get("X-Workspace-ID"),
)
)
if kwargs["method"] == "GET":
return httpx.Response(
200,
json={
"projects": [{"id": 1, "name": "alpha", "path": "alpha", "is_default": True}]
},
)
return httpx.Response(
200,
json={
"message": "created",
"status": "success",
"default": False,
"old_project": None,
"new_project": {"name": "alpha", "path": "alpha"},
},
)
assert await project_exists("alpha", api_request=api_request) is True
await create_cloud_project("alpha", api_request=api_request)
await fetch_cloud_projects(project_name="missing", api_request=api_request)
assert seen == [
("GET", "project-workspace"),
("POST", "project-workspace"),
("GET", "default-workspace"),
]
@pytest.mark.asyncio
async def test_project_exists_surfaces_cloud_lookup_failures(config_home, config_manager):
"""project_exists should surface lookup failures instead of pretending the project is missing."""
config = config_manager.load_config()
config.cloud_host = "https://cloud.example.test"
config_manager.save_config(config)
async def api_request(**_kwargs):
raise httpx.ConnectError("boom")
with pytest.raises(CloudUtilsError, match="Failed to fetch cloud projects"):
await project_exists("alpha", api_request=api_request)
@pytest.mark.asyncio
@@ -0,0 +1,114 @@
"""Tests for cloud sync and bisync command behavior."""
import importlib
from contextlib import asynccontextmanager
from types import SimpleNamespace
import pytest
from typer.testing import CliRunner
from basic_memory.cli.app import app
from basic_memory.config import ProjectMode
runner = CliRunner()
@pytest.mark.parametrize(
"argv",
[
["cloud", "sync", "--name", "research"],
["cloud", "bisync", "--name", "research"],
],
)
def test_cloud_sync_commands_use_incremental_db_sync(monkeypatch, argv, config_manager):
"""Cloud sync commands should not force a full database re-index after file sync."""
project_sync_command = importlib.import_module("basic_memory.cli.commands.cloud.project_sync")
seen: dict[str, object] = {}
config = config_manager.load_config()
config.set_project_mode("research", ProjectMode.CLOUD)
config_manager.save_config(config)
monkeypatch.setattr(project_sync_command, "_require_cloud_credentials", lambda _config: None)
monkeypatch.setattr(
project_sync_command,
"get_mount_info",
lambda: _async_value(SimpleNamespace(bucket_name="tenant-bucket")),
)
monkeypatch.setattr(
project_sync_command,
"_get_cloud_project",
lambda _name: _async_value(
SimpleNamespace(name="research", external_id="external-project-id", path="research")
),
)
monkeypatch.setattr(
project_sync_command,
"_get_sync_project",
lambda _name, _config, _project_data: (SimpleNamespace(name="research"), "/tmp/research"),
)
monkeypatch.setattr(project_sync_command, "project_sync", lambda *args, **kwargs: True)
monkeypatch.setattr(project_sync_command, "project_bisync", lambda *args, **kwargs: True)
@asynccontextmanager
async def fake_get_client(*, project_name=None, workspace=None):
seen["project_name"] = project_name
seen["workspace"] = workspace
yield object()
class FakeProjectClient:
def __init__(self, _client):
pass
async def sync(self, external_id: str, force_full: bool = False):
seen["external_id"] = external_id
seen["force_full"] = force_full
return {"message": "queued"}
monkeypatch.setattr(project_sync_command, "get_client", fake_get_client)
monkeypatch.setattr(project_sync_command, "ProjectClient", FakeProjectClient)
result = runner.invoke(app, argv)
assert result.exit_code == 0, result.output
assert seen["project_name"] == "research"
assert seen["external_id"] == "external-project-id"
assert seen["force_full"] is False
def test_cloud_bisync_fails_fast_when_sync_entry_disappears(monkeypatch, config_manager):
"""Bisync should raise a runtime error when validated sync config vanishes before persistence."""
project_sync_command = importlib.import_module("basic_memory.cli.commands.cloud.project_sync")
config = config_manager.load_config()
config.projects.pop("research", None)
config_manager.save_config(config)
monkeypatch.setattr(project_sync_command, "_require_cloud_credentials", lambda _config: None)
monkeypatch.setattr(
project_sync_command,
"get_mount_info",
lambda: _async_value(SimpleNamespace(bucket_name="tenant-bucket")),
)
monkeypatch.setattr(
project_sync_command,
"_get_cloud_project",
lambda _name: _async_value(
SimpleNamespace(name="research", external_id="external-project-id", path="research")
),
)
monkeypatch.setattr(
project_sync_command,
"_get_sync_project",
lambda _name, _config, _project_data: (SimpleNamespace(name="research"), "/tmp/research"),
)
monkeypatch.setattr(project_sync_command, "project_bisync", lambda *args, **kwargs: True)
result = runner.invoke(app, ["cloud", "bisync", "--name", "research"])
assert result.exit_code == 1, result.output
assert "unexpectedly missing after validation" in result.output
async def _async_value(value):
return value
+90 -2
View File
@@ -6,6 +6,8 @@ import httpx
from typer.testing import CliRunner
from basic_memory.cli.app import app
from basic_memory.cli.commands.cloud.cloud_utils import CloudUtilsError
from basic_memory.config import ProjectMode
runner = CliRunner()
@@ -20,11 +22,11 @@ def test_cloud_upload_uses_control_plane_client(monkeypatch, tmp_path):
seen: dict[str, str] = {}
async def fake_project_exists(_project_name: str) -> bool:
async def fake_project_exists(_project_name: str, workspace: str | None = None) -> bool:
return True
@asynccontextmanager
async def fake_get_client():
async def fake_get_client(workspace: str | None = None):
async with httpx.AsyncClient(base_url="https://cloud.example.test") as client:
yield client
@@ -53,3 +55,89 @@ def test_cloud_upload_uses_control_plane_client(monkeypatch, tmp_path):
assert result.exit_code == 0, result.output
assert seen["base_url"] == "https://cloud.example.test"
def test_cloud_upload_uses_project_workspace_for_api_and_webdav(
monkeypatch, tmp_path, config_manager
):
"""Upload command should reuse the configured workspace across API and WebDAV calls."""
import basic_memory.cli.commands.cloud.upload_command as upload_command
config = config_manager.load_config()
config.default_workspace = "default-workspace"
config.set_project_mode("routing-test", ProjectMode.CLOUD)
config.projects["routing-test"].workspace_id = "project-workspace"
config_manager.save_config(config)
upload_dir = tmp_path / "upload"
upload_dir.mkdir()
(upload_dir / "note.md").write_text("hello", encoding="utf-8")
seen: dict[str, str | None] = {}
async def fake_project_exists(_project_name: str, workspace: str | None = None) -> bool:
seen["project_exists_workspace"] = workspace
return True
@asynccontextmanager
async def fake_get_client(workspace: str | None = None):
seen["control_plane_workspace"] = workspace
async with httpx.AsyncClient(base_url="https://cloud.example.test") as client:
yield client
async def fake_upload_path(*args, **kwargs):
client_cm_factory = kwargs.get("client_cm_factory")
assert client_cm_factory is not None
async with client_cm_factory() as client:
seen["base_url"] = str(client.base_url).rstrip("/")
return True
monkeypatch.setattr(upload_command, "project_exists", fake_project_exists)
monkeypatch.setattr(upload_command, "get_cloud_control_plane_client", fake_get_client)
monkeypatch.setattr(upload_command, "upload_path", fake_upload_path)
result = runner.invoke(
app,
[
"cloud",
"upload",
str(upload_dir),
"--project",
"routing-test",
"--no-sync",
],
)
assert result.exit_code == 0, result.output
assert seen["project_exists_workspace"] == "project-workspace"
assert seen["control_plane_workspace"] == "project-workspace"
assert seen["base_url"] == "https://cloud.example.test"
def test_cloud_upload_exits_when_project_lookup_fails(monkeypatch, tmp_path):
"""Upload command should fail fast when cloud project lookup cannot reach the API."""
import basic_memory.cli.commands.cloud.upload_command as upload_command
upload_dir = tmp_path / "upload"
upload_dir.mkdir()
(upload_dir / "note.md").write_text("hello", encoding="utf-8")
async def fake_project_exists(_project_name: str, workspace: str | None = None) -> bool:
raise CloudUtilsError("lookup failed")
monkeypatch.setattr(upload_command, "project_exists", fake_project_exists)
result = runner.invoke(
app,
[
"cloud",
"upload",
str(upload_dir),
"--project",
"routing-test",
"--no-sync",
],
)
assert result.exit_code == 1, result.output
assert "Failed to check cloud project 'routing-test'" in result.output
+164 -2
View File
@@ -52,9 +52,11 @@ def mock_config(tmp_path, monkeypatch):
@pytest.fixture
def mock_api_client(monkeypatch):
"""Stub the API client for project add without stdlib mocks."""
seen_workspaces: list[str | None] = []
@asynccontextmanager
async def fake_get_client():
async def fake_get_client(*, workspace=None):
seen_workspaces.append(workspace)
yield object()
_response_data = {
@@ -80,7 +82,7 @@ def mock_api_client(monkeypatch):
monkeypatch.setattr(project_cmd, "get_client", fake_get_client)
monkeypatch.setattr(ProjectClient, "create_project", fake_create_project)
return calls
return {"calls": calls, "workspaces": seen_workspaces}
def test_project_add_with_local_path_saves_to_config(
@@ -113,6 +115,7 @@ def test_project_add_with_local_path_saves_to_config(
assert "test-project" in config_data["projects"]
entry = config_data["projects"]["test-project"]
# Use as_posix() for cross-platform compatibility (Windows uses backslashes)
assert entry["mode"] == "cloud"
assert entry["local_sync_path"] == local_sync_dir.as_posix()
assert entry.get("last_sync") is None
assert entry.get("bisync_initialized", False) is False
@@ -173,3 +176,162 @@ def test_project_add_local_path_creates_nested_directories(
assert result.exit_code == 0
assert nested_path.exists()
assert nested_path.is_dir()
def test_project_add_cloud_visibility_passes_payload(runner, mock_config, mock_api_client):
"""Cloud project creation should forward visibility to the API payload."""
result = runner.invoke(
app,
["project", "add", "test-project", "--cloud", "--visibility", "shared"],
)
assert result.exit_code == 0
assert mock_api_client["workspaces"] == [None]
assert mock_api_client["calls"] == [
{
"name": "test-project",
"path": "test-project",
"local_sync_path": None,
"set_default": False,
"visibility": "shared",
}
]
def test_project_add_cloud_workspace_resolves_and_persists(
runner, mock_config, mock_api_client, monkeypatch, tmp_path
):
"""Cloud project add should resolve workspace names to tenant IDs."""
from basic_memory.schemas.cloud import WorkspaceInfo
local_sync_dir = tmp_path / "sync" / "team-notes"
async def fake_get_available_workspaces():
return [
WorkspaceInfo(
tenant_id="11111111-1111-1111-1111-111111111111",
workspace_type="organization",
name="Basic Memory",
role="owner",
),
]
monkeypatch.setattr(
"basic_memory.mcp.project_context.get_available_workspaces",
fake_get_available_workspaces,
)
result = runner.invoke(
app,
[
"project",
"add",
"team-notes",
"--cloud",
"--workspace",
"Basic Memory",
"--local-path",
str(local_sync_dir),
],
)
assert result.exit_code == 0
assert mock_api_client["workspaces"] == ["11111111-1111-1111-1111-111111111111"]
assert mock_api_client["calls"] == [
{
"name": "team-notes",
"path": "team-notes",
"local_sync_path": local_sync_dir.as_posix(),
"set_default": False,
"visibility": "workspace",
}
]
config_data = json.loads(mock_config.read_text())
entry = config_data["projects"]["team-notes"]
assert entry["mode"] == "cloud"
assert entry["workspace_id"] == "11111111-1111-1111-1111-111111111111"
def test_project_add_cloud_workspace_persists_without_local_path(
runner, mock_config, mock_api_client, monkeypatch
):
"""Cloud project add should persist workspace routing even without local sync."""
from basic_memory.schemas.cloud import WorkspaceInfo
async def fake_get_available_workspaces():
return [
WorkspaceInfo(
tenant_id="11111111-1111-1111-1111-111111111111",
workspace_type="organization",
name="Basic Memory",
role="owner",
),
]
monkeypatch.setattr(
"basic_memory.mcp.project_context.get_available_workspaces",
fake_get_available_workspaces,
)
result = runner.invoke(
app,
[
"project",
"add",
"team-notes",
"--cloud",
"--workspace",
"Basic Memory",
],
)
assert result.exit_code == 0
assert mock_api_client["workspaces"] == ["11111111-1111-1111-1111-111111111111"]
assert mock_api_client["calls"] == [
{
"name": "team-notes",
"path": "team-notes",
"local_sync_path": None,
"set_default": False,
"visibility": "workspace",
}
]
config_data = json.loads(mock_config.read_text())
entry = config_data["projects"]["team-notes"]
assert entry["path"] == ""
assert entry["mode"] == "cloud"
assert entry["workspace_id"] == "11111111-1111-1111-1111-111111111111"
assert entry["local_sync_path"] is None
def test_project_add_visibility_requires_cloud_mode(runner, mock_config, tmp_path):
"""Visibility is a cloud-only option."""
project_path = tmp_path / "local-project"
result = runner.invoke(
app,
[
"project",
"add",
"local-project",
str(project_path),
"--visibility",
"shared",
],
)
assert result.exit_code == 1
assert "--visibility is only supported in cloud mode" in result.stdout
def test_project_add_rejects_invalid_visibility(runner, mock_config):
"""Invalid visibility values should fail fast before the API call."""
result = runner.invoke(
app,
["project", "add", "test-project", "--cloud", "--visibility", "team-only"],
)
assert result.exit_code == 1
assert "Invalid visibility" in result.stdout
+109 -46
View File
@@ -22,7 +22,13 @@ from sqlalchemy.pool import NullPool
from testcontainers.postgres import PostgresContainer
from basic_memory import db
from basic_memory.config import ProjectConfig, BasicMemoryConfig, ConfigManager, DatabaseBackend
from basic_memory.config import (
ProjectConfig,
ProjectEntry,
BasicMemoryConfig,
ConfigManager,
DatabaseBackend,
)
from basic_memory.db import DatabaseType
from basic_memory.markdown import EntityParser
from basic_memory.markdown.markdown_processor import MarkdownProcessor
@@ -74,7 +80,7 @@ def postgres_container(db_backend):
The container is started once per test session and shared across all tests.
Only starts if db_backend is "postgres".
"""
if db_backend != "postgres":
if db_backend != "postgres" or _configured_postgres_sync_url():
yield None
return
@@ -83,6 +89,100 @@ def postgres_container(db_backend):
yield postgres
POSTGRES_EPHEMERAL_TABLES = [
"search_vector_embeddings",
"search_vector_index",
]
def _configured_postgres_sync_url() -> str | None:
"""Prefer an externally managed Postgres server when CI provides one."""
configured_url = os.environ.get("BASIC_MEMORY_TEST_POSTGRES_URL") or os.environ.get(
"POSTGRES_TEST_URL"
)
if not configured_url:
return None
return (
configured_url.replace("postgresql+asyncpg://", "postgresql+psycopg2://", 1)
.replace("postgresql://", "postgresql+psycopg2://", 1)
.replace("postgres://", "postgresql+psycopg2://", 1)
)
def _postgres_alembic_config(async_url: str) -> Config:
"""Build Alembic config for stamping the shared Postgres test schema."""
alembic_dir = Path(db.__file__).parent / "alembic"
cfg = Config()
cfg.set_main_option("script_location", str(alembic_dir))
cfg.set_main_option(
"file_template",
"%%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d-%%(rev)s_%%(slug)s",
)
cfg.set_main_option("timezone", "UTC")
cfg.set_main_option("revision_environment", "false")
cfg.set_main_option("sqlalchemy.url", async_url)
return cfg
def _postgres_reset_tables() -> list[str]:
"""Resolve the current ORM table set at reset time.
Some tests declare models after conftest import, so the list must stay dynamic.
"""
return [table.name for table in Base.metadata.sorted_tables] + [
"search_index",
"search_vector_chunks",
]
def _resolve_postgres_sync_url(postgres_container) -> str:
"""Use CI's shared service when configured, otherwise fall back to testcontainers."""
configured_url = _configured_postgres_sync_url()
if configured_url:
return configured_url
assert postgres_container is not None
return postgres_container.get_connection_url()
async def _reset_postgres_test_schema(engine: AsyncEngine, async_url: str) -> None:
"""Restore the shared Postgres schema to a clean baseline before each test."""
from basic_memory.models.search import (
CREATE_POSTGRES_SEARCH_INDEX_FTS,
CREATE_POSTGRES_SEARCH_INDEX_METADATA,
CREATE_POSTGRES_SEARCH_INDEX_PERMALINK,
CREATE_POSTGRES_SEARCH_INDEX_TABLE,
CREATE_POSTGRES_SEARCH_VECTOR_CHUNKS_INDEX,
CREATE_POSTGRES_SEARCH_VECTOR_CHUNKS_TABLE,
)
async with engine.begin() as conn:
# Trigger: several tests intentionally drop or stub search tables to exercise recovery code.
# Why: TRUNCATE is much cheaper than drop_all/create_all, but it only works when the schema exists.
# Outcome: we recreate any missing core tables once, then clear rows for deterministic test setup.
await conn.run_sync(Base.metadata.create_all)
await conn.execute(CREATE_POSTGRES_SEARCH_INDEX_TABLE)
await conn.execute(CREATE_POSTGRES_SEARCH_INDEX_FTS)
await conn.execute(CREATE_POSTGRES_SEARCH_INDEX_METADATA)
await conn.execute(CREATE_POSTGRES_SEARCH_INDEX_PERMALINK)
await conn.execute(CREATE_POSTGRES_SEARCH_VECTOR_CHUNKS_TABLE)
await conn.execute(CREATE_POSTGRES_SEARCH_VECTOR_CHUNKS_INDEX)
for table_name in POSTGRES_EPHEMERAL_TABLES:
await conn.execute(text(f"DROP TABLE IF EXISTS {table_name} CASCADE"))
await conn.execute(
text(f"TRUNCATE TABLE {', '.join(_postgres_reset_tables())} RESTART IDENTITY CASCADE")
)
alembic_version_exists = (
await conn.execute(text("SELECT to_regclass('public.alembic_version')"))
).scalar() is not None
if not alembic_version_exists:
command.stamp(_postgres_alembic_config(async_url), "head")
@pytest.fixture
def anyio_backend():
return "asyncio"
@@ -114,13 +214,15 @@ def config_home(tmp_path, monkeypatch) -> Path:
@pytest.fixture(scope="function")
def app_config(config_home, db_backend, postgres_container, monkeypatch) -> BasicMemoryConfig:
"""Create test app configuration for the appropriate backend."""
projects = {"test-project": str(config_home)}
projects = {"test-project": ProjectEntry(path=str(config_home))}
# Set backend based on parameterized db_backend fixture
if db_backend == "postgres":
backend = DatabaseBackend.POSTGRES
# Get URL from testcontainer and convert to asyncpg driver
sync_url = postgres_container.get_connection_url()
# Trigger: CI jobs can provide a shared Postgres service instead of per-session containers.
# Why: reusing one pgvector-enabled server avoids Docker startup churn on every job.
# Outcome: local runs keep using testcontainers, while CI injects a stable service URL.
sync_url = _resolve_postgres_sync_url(postgres_container)
database_url = sync_url.replace("postgresql+psycopg2", "postgresql+asyncpg")
else:
backend = DatabaseBackend.SQLITE
@@ -206,7 +308,7 @@ async def engine_factory(
if db_backend == "postgres":
# Postgres mode using testcontainers
# Get async connection URL (asyncpg driver - same as production)
sync_url = postgres_container.get_connection_url()
sync_url = _resolve_postgres_sync_url(postgres_container)
async_url = sync_url.replace("postgresql+psycopg2", "postgresql+asyncpg")
engine = create_async_engine(
@@ -229,46 +331,7 @@ async def engine_factory(
db._engine = engine
db._session_maker = session_maker
from basic_memory.models.search import (
CREATE_POSTGRES_SEARCH_INDEX_TABLE,
CREATE_POSTGRES_SEARCH_INDEX_FTS,
CREATE_POSTGRES_SEARCH_INDEX_METADATA,
CREATE_POSTGRES_SEARCH_INDEX_PERMALINK,
CREATE_POSTGRES_SEARCH_VECTOR_CHUNKS_TABLE,
CREATE_POSTGRES_SEARCH_VECTOR_CHUNKS_INDEX,
)
# Drop and recreate all tables for test isolation
async with engine.begin() as conn:
# Must drop search_index first (has FK to project, blocks drop_all)
await conn.execute(text("DROP TABLE IF EXISTS search_index CASCADE"))
await conn.run_sync(Base.metadata.drop_all)
await conn.run_sync(Base.metadata.create_all)
# Create search_index via DDL (not ORM - uses composite PK + tsvector)
# asyncpg requires separate execute calls for each statement
await conn.execute(CREATE_POSTGRES_SEARCH_INDEX_TABLE)
await conn.execute(CREATE_POSTGRES_SEARCH_INDEX_FTS)
await conn.execute(CREATE_POSTGRES_SEARCH_INDEX_METADATA)
await conn.execute(CREATE_POSTGRES_SEARCH_INDEX_PERMALINK)
await conn.execute(CREATE_POSTGRES_SEARCH_VECTOR_CHUNKS_TABLE)
await conn.execute(CREATE_POSTGRES_SEARCH_VECTOR_CHUNKS_INDEX)
# Mark migrations as already applied for this test-created schema.
#
# Some codepaths (e.g. ensure_initialization()) invoke Alembic migrations.
# If we create tables via ORM directly, alembic_version is missing and migrations
# will try to create tables again, causing DuplicateTableError.
alembic_dir = Path(db.__file__).parent / "alembic"
cfg = Config()
cfg.set_main_option("script_location", str(alembic_dir))
cfg.set_main_option(
"file_template",
"%%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d-%%(rev)s_%%(slug)s",
)
cfg.set_main_option("timezone", "UTC")
cfg.set_main_option("revision_environment", "false")
cfg.set_main_option("sqlalchemy.url", async_url)
command.stamp(cfg, "head")
await _reset_postgres_test_schema(engine, async_url)
yield engine, session_maker
+42
View File
@@ -79,6 +79,35 @@ async def test_get_client_cloud_adds_workspace_header(config_manager):
assert client.headers.get("X-Workspace-ID") == "tenant-123"
@pytest.mark.asyncio
async def test_get_client_cloud_uses_project_workspace_when_not_explicit(config_manager):
cfg = config_manager.load_config()
cfg.cloud_host = "https://cloud.example.test"
cfg.cloud_api_key = "bmc_test_key_123"
cfg.default_workspace = "default-tenant"
cfg.set_project_mode("research", ProjectMode.CLOUD)
cfg.projects["research"].workspace_id = "project-tenant"
config_manager.save_config(cfg)
async with get_client(project_name="research") as client:
assert str(client.base_url).rstrip("/") == "https://cloud.example.test/proxy"
assert client.headers.get("X-Workspace-ID") == "project-tenant"
@pytest.mark.asyncio
async def test_get_client_cloud_uses_default_workspace_when_project_has_none(config_manager):
cfg = config_manager.load_config()
cfg.cloud_host = "https://cloud.example.test"
cfg.cloud_api_key = "bmc_test_key_123"
cfg.default_workspace = "default-tenant"
cfg.set_project_mode("research", ProjectMode.CLOUD)
config_manager.save_config(cfg)
async with get_client(project_name="research") as client:
assert str(client.base_url).rstrip("/") == "https://cloud.example.test/proxy"
assert client.headers.get("X-Workspace-ID") == "default-tenant"
@pytest.mark.asyncio
async def test_get_client_explicit_cloud_raises_without_credentials(config_manager, monkeypatch):
cfg = config_manager.load_config()
@@ -253,6 +282,19 @@ async def test_get_cloud_control_plane_client_uses_api_key_when_available(config
assert client.headers.get("Authorization") == "Bearer bmc_test_key_123"
@pytest.mark.asyncio
async def test_get_cloud_control_plane_client_adds_workspace_header(config_manager):
cfg = config_manager.load_config()
cfg.cloud_host = "https://cloud.example.test"
cfg.cloud_api_key = "bmc_test_key_123"
config_manager.save_config(cfg)
async with get_cloud_control_plane_client(workspace="tenant-123") as client:
assert str(client.base_url).rstrip("/") == "https://cloud.example.test"
assert client.headers.get("Authorization") == "Bearer bmc_test_key_123"
assert client.headers.get("X-Workspace-ID") == "tenant-123"
@pytest.mark.asyncio
async def test_get_cloud_control_plane_client_uses_oauth_token(config_manager):
cfg = config_manager.load_config()
+191
View File
@@ -0,0 +1,191 @@
"""Telemetry coverage for typed MCP clients and shared HTTP helpers."""
from __future__ import annotations
import importlib
from contextlib import contextmanager
import httpx
import pytest
from mcp.server.fastmcp.exceptions import ToolError
knowledge_client_module = importlib.import_module("basic_memory.mcp.clients.knowledge")
search_client_module = importlib.import_module("basic_memory.mcp.clients.search")
utils_module = importlib.import_module("basic_memory.mcp.tools.utils")
def _capture_spans():
spans: list[tuple[str, dict]] = []
@contextmanager
def fake_span(name: str, **attrs):
spans.append((name, attrs))
yield
@contextmanager
def fake_started_span(name: str, **attrs):
spans.append((name, attrs))
class FakeStartedSpan:
def set_attribute(self, key: str, value) -> None:
attrs[key] = value
def set_attributes(self, new_attrs: dict) -> None:
attrs.update(new_attrs)
yield FakeStartedSpan()
return spans, fake_span, fake_started_span
@pytest.mark.asyncio
async def test_knowledge_client_resolve_entity_emits_client_and_http_spans(monkeypatch) -> None:
spans, fake_span, fake_started_span = _capture_spans()
monkeypatch.setattr(knowledge_client_module.telemetry, "span", fake_span)
monkeypatch.setattr(utils_module.telemetry, "started_span", fake_started_span)
async def handler(request: httpx.Request) -> httpx.Response:
assert request.method == "POST"
return httpx.Response(200, json={"external_id": "entity-123"})
transport = httpx.MockTransport(handler)
async with httpx.AsyncClient(transport=transport, base_url="https://example.test") as client:
knowledge_client = knowledge_client_module.KnowledgeClient(client, "project-123")
resolved = await knowledge_client.resolve_entity("notes/root", strict=True)
assert resolved == "entity-123"
assert [name for name, _ in spans] == [
"mcp.client.knowledge.resolve_entity",
"mcp.http.request",
]
assert spans[1][1] == {
"method": "POST",
"client_name": "knowledge",
"operation": "resolve_entity",
"path_template": "/v2/projects/{project_id}/knowledge/resolve",
"phase": "request",
"has_query": False,
"has_body": True,
"status_code": 200,
"is_success": True,
"outcome": "success",
}
@pytest.mark.asyncio
async def test_search_client_emits_client_and_http_spans(monkeypatch) -> None:
spans, fake_span, fake_started_span = _capture_spans()
monkeypatch.setattr(search_client_module.telemetry, "span", fake_span)
monkeypatch.setattr(utils_module.telemetry, "started_span", fake_started_span)
async def handler(request: httpx.Request) -> httpx.Response:
assert request.method == "POST"
return httpx.Response(
200,
json={
"results": [],
"current_page": 2,
"page_size": 5,
"has_more": False,
},
)
transport = httpx.MockTransport(handler)
async with httpx.AsyncClient(transport=transport, base_url="https://example.test") as client:
search_client = search_client_module.SearchClient(client, "project-123")
response = await search_client.search({"text": "telemetry"}, page=2, page_size=5)
assert response.current_page == 2
assert [name for name, _ in spans] == [
"mcp.client.search.search",
"mcp.http.request",
]
assert spans[1][1] == {
"method": "POST",
"client_name": "search",
"operation": "search",
"path_template": "/v2/projects/{project_id}/search/",
"phase": "request",
"has_query": True,
"has_body": True,
"status_code": 200,
"is_success": True,
"outcome": "success",
}
@pytest.mark.asyncio
async def test_call_get_emits_http_outcome_for_client_errors(monkeypatch) -> None:
spans, _, fake_started_span = _capture_spans()
monkeypatch.setattr(utils_module.telemetry, "started_span", fake_started_span)
async def handler(request: httpx.Request) -> httpx.Response:
assert request.method == "GET"
return httpx.Response(404, json={"detail": "missing"})
transport = httpx.MockTransport(handler)
async with httpx.AsyncClient(transport=transport, base_url="https://example.test") as client:
with pytest.raises(ToolError, match="missing"):
await utils_module.call_get(
client,
"/missing",
client_name="knowledge",
operation="get_entity",
path_template="/v2/projects/{project_id}/knowledge/entities/{entity_id}",
)
assert spans == [
(
"mcp.http.request",
{
"method": "GET",
"client_name": "knowledge",
"operation": "get_entity",
"path_template": "/v2/projects/{project_id}/knowledge/entities/{entity_id}",
"phase": "request",
"has_query": False,
"has_body": False,
"status_code": 404,
"is_success": False,
"outcome": "client_error",
},
)
]
@pytest.mark.asyncio
async def test_call_get_emits_transport_error_outcome(monkeypatch) -> None:
spans, _, fake_started_span = _capture_spans()
monkeypatch.setattr(utils_module.telemetry, "started_span", fake_started_span)
async def handler(request: httpx.Request) -> httpx.Response:
raise httpx.ConnectError("boom", request=request)
transport = httpx.MockTransport(handler)
async with httpx.AsyncClient(transport=transport, base_url="https://example.test") as client:
with pytest.raises(httpx.ConnectError, match="boom"):
await utils_module.call_get(
client,
"/boom",
client_name="knowledge",
operation="get_entity",
path_template="/v2/projects/{project_id}/knowledge/entities/{entity_id}",
)
assert spans == [
(
"mcp.http.request",
{
"method": "GET",
"client_name": "knowledge",
"operation": "get_entity",
"path_template": "/v2/projects/{project_id}/knowledge/entities/{entity_id}",
"phase": "request",
"has_query": False,
"has_body": False,
"is_success": False,
"outcome": "transport_error",
"error_type": "ConnectError",
},
)
]
+163 -1
View File
@@ -311,6 +311,169 @@ async def test_workspace_uses_cached_workspace_without_fetch(monkeypatch):
assert resolved.tenant_id == cached_workspace.tenant_id
@pytest.mark.asyncio
async def test_resolve_project_parameter_uses_cached_active_project_before_api_default_lookup(
config_manager, monkeypatch
):
from basic_memory.mcp.project_context import resolve_project_parameter
from basic_memory.schemas.project_info import ProjectItem
config = config_manager.load_config()
config.default_project = None
config_manager.save_config(config)
context = _ContextState()
cached_project = ProjectItem(
id=1,
external_id="11111111-1111-1111-1111-111111111111",
name="Cached Project",
path="/tmp/cached-project",
is_default=True,
)
await context.set_state("active_project", cached_project.model_dump())
async def fail_if_called(): # pragma: no cover
raise AssertionError("Default project API lookup should not run when project is cached")
monkeypatch.setattr(
"basic_memory.mcp.project_context._resolve_default_project_from_api",
fail_if_called,
)
resolved = await resolve_project_parameter(project=None, context=context)
assert resolved == cached_project.name
@pytest.mark.asyncio
async def test_resolve_project_parameter_caches_api_default_project_name(
config_manager, monkeypatch
):
from basic_memory.mcp.project_context import resolve_project_parameter
config = config_manager.load_config()
config.default_project = None
config_manager.save_config(config)
context = _ContextState()
api_calls = {"count": 0}
async def fake_default_lookup():
api_calls["count"] += 1
return "cloud-default"
monkeypatch.setattr(
"basic_memory.mcp.project_context._resolve_default_project_from_api",
fake_default_lookup,
)
first = await resolve_project_parameter(project=None, context=context)
second = await resolve_project_parameter(project=None, context=context)
assert first == "cloud-default"
assert second == "cloud-default"
assert api_calls["count"] == 1
@pytest.mark.asyncio
async def test_get_active_project_uses_cached_project_before_resolution(monkeypatch):
from basic_memory.mcp.project_context import get_active_project
from basic_memory.schemas.project_info import ProjectItem
context = _ContextState()
cached_project = ProjectItem(
id=1,
external_id="11111111-1111-1111-1111-111111111111",
name="Cached Project",
path="/tmp/cached-project",
is_default=True,
)
await context.set_state("active_project", cached_project.model_dump())
async def fail_if_called(*args, **kwargs): # pragma: no cover
raise AssertionError("Project resolution should not run when cache matches")
monkeypatch.setattr(
"basic_memory.mcp.project_context.resolve_project_parameter",
fail_if_called,
)
resolved = await get_active_project(client=None, context=context)
assert resolved == cached_project
@pytest.mark.asyncio
async def test_get_active_project_uses_cached_project_for_explicit_permalink(monkeypatch):
from basic_memory.mcp.project_context import get_active_project
from basic_memory.schemas.project_info import ProjectItem
context = _ContextState()
cached_project = ProjectItem(
id=1,
external_id="11111111-1111-1111-1111-111111111111",
name="My Research",
path="/tmp/my-research",
is_default=False,
)
await context.set_state("active_project", cached_project.model_dump())
async def fail_if_called(*args, **kwargs): # pragma: no cover
raise AssertionError(
"Project resolution should not run when explicit project matches cache"
)
monkeypatch.setattr(
"basic_memory.mcp.project_context.resolve_project_parameter",
fail_if_called,
)
resolved = await get_active_project(client=None, project="my-research", context=context)
assert resolved == cached_project
@pytest.mark.asyncio
async def test_resolve_project_and_path_uses_cached_project_for_memory_url_prefix(
config_manager, monkeypatch
):
from basic_memory.mcp.project_context import resolve_project_and_path
from basic_memory.schemas.project_info import ProjectItem
config = config_manager.load_config()
config.permalinks_include_project = False
config_manager.save_config(config)
context = _ContextState()
cached_project = ProjectItem(
id=1,
external_id="11111111-1111-1111-1111-111111111111",
name="My Research",
path="/tmp/my-research",
is_default=False,
)
await context.set_state("active_project", cached_project.model_dump())
async def fail_if_called(*args, **kwargs): # pragma: no cover
raise AssertionError("Project resolve API should not run when memory URL matches cache")
async def fake_resolve_project_parameter(project=None, **kwargs):
return cached_project.name if project else cached_project.name
monkeypatch.setattr("basic_memory.mcp.tools.utils.call_post", fail_if_called)
monkeypatch.setattr(
"basic_memory.mcp.project_context.resolve_project_parameter",
fake_resolve_project_parameter,
)
active_project, resolved_path, is_memory_url = await resolve_project_and_path(
client=None,
identifier="memory://my-research/notes/roadmap.md",
context=context,
)
assert active_project == cached_project
assert resolved_path == "notes/roadmap.md"
assert is_memory_url is True
@pytest.mark.asyncio
async def test_get_project_client_rejects_workspace_for_local_project(config_manager):
from basic_memory.mcp.project_context import get_project_client
@@ -383,7 +546,6 @@ class TestDetectProjectFromUrlPrefix:
assert result == "My Research"
class TestGetProjectClientRoutingOrder:
"""Test that get_project_client respects explicit routing before workspace resolution."""
+57
View File
@@ -264,6 +264,63 @@ async def test_read_note_memory_url_with_project_prefix(app, test_project):
assert "Testing memory:// URL handling with project prefix" in content
@pytest.mark.asyncio
async def test_read_note_memory_url_fallback_uses_search_tool_normalization(
monkeypatch, app, test_project
):
"""Fallback search should go back through search_notes for memory:// normalization."""
await write_note(
project=test_project.name,
title="Memory URL Fallback Note",
directory="test",
content="Fallback note content",
)
import importlib
read_note_module = importlib.import_module("basic_memory.mcp.tools.read_note")
clients_mod = importlib.import_module("basic_memory.mcp.clients")
OriginalKnowledgeClient = clients_mod.KnowledgeClient
fallback_memory_url = f"memory://{test_project.name}/test/memory-url-fallback-note"
search_calls: list[tuple[str, str, str | None]] = []
class SelectiveKnowledgeClient(OriginalKnowledgeClient):
async def resolve_entity(self, identifier: str, *, strict: bool = False) -> int:
if strict and identifier.endswith("test/memory-url-fallback-note"):
raise RuntimeError("force direct lookup failure")
return await super().resolve_entity(identifier, strict=strict)
async def fake_search_notes_fn(*, query, search_type, project, **kwargs):
search_calls.append((search_type, query, project))
return {
"results": [
{
"title": "Memory URL Fallback Note",
"permalink": "test/memory-url-fallback-note",
"content": "",
"type": "entity",
"score": 1.0,
"file_path": "test/Memory URL Fallback Note.md",
}
],
"current_page": 1,
"page_size": 10,
}
monkeypatch.setattr(clients_mod, "KnowledgeClient", SelectiveKnowledgeClient)
monkeypatch.setattr(read_note_module, "search_notes", fake_search_notes_fn)
result = await read_note(fallback_memory_url)
assert search_calls == [
("title", fallback_memory_url, test_project.name),
("text", fallback_memory_url, test_project.name),
]
assert "I couldn't find an exact match" in result
assert "Memory URL Fallback Note" in result
class TestReadNoteSecurityValidation:
"""Test read_note security validation features."""
+65 -64
View File
@@ -51,20 +51,19 @@ async def test_write_note_emits_root_operation_and_project_context(
output_format="json",
)
assert operations == [
(
"mcp.tool.write_note",
{
"entrypoint": "mcp",
"tool_name": "write_note",
"requested_project": test_project.name,
"workspace_id": None,
"note_type": "note",
"overwrite": False,
"output_format": "json",
},
)
]
assert operations[0] == (
"mcp.tool.write_note",
{
"entrypoint": "mcp",
"tool_name": "write_note",
"requested_project": test_project.name,
"workspace_id": None,
"note_type": "note",
"overwrite": False,
"output_format": "json",
},
)
assert "api.request.knowledge.create_entity" in [name for name, _ in operations]
assert _contains_context(
contexts,
{
@@ -103,21 +102,23 @@ async def test_read_note_emits_root_operation_and_project_context(
include_frontmatter=True,
)
assert operations == [
(
"mcp.tool.read_note",
{
"entrypoint": "mcp",
"tool_name": "read_note",
"requested_project": test_project.name,
"workspace_id": None,
"output_format": "json",
"page": 1,
"page_size": 10,
"include_frontmatter": True,
},
)
]
assert operations[0] == (
"mcp.tool.read_note",
{
"entrypoint": "mcp",
"tool_name": "read_note",
"requested_project": test_project.name,
"workspace_id": None,
"output_format": "json",
"page": 1,
"page_size": 10,
"include_frontmatter": True,
},
)
operation_names = [name for name, _ in operations]
assert "api.request.knowledge.resolve_entity" in operation_names
assert "api.request.resource.get_content" in operation_names
assert "api.request.knowledge.get_entity" in operation_names
assert _contains_context(
contexts,
{
@@ -172,7 +173,7 @@ async def test_search_notes_emits_root_operation_and_project_context(
"has_query": True,
"note_type_filter_count": 0,
"entity_type_filter_count": 0,
"has_metadata_filters": False,
"has_filters": True,
"has_tags_filter": True,
"has_status_filter": False,
},
@@ -217,22 +218,23 @@ async def test_edit_note_emits_root_operation_and_project_context(
output_format="json",
)
assert operations == [
(
"mcp.tool.edit_note",
{
"entrypoint": "mcp",
"tool_name": "edit_note",
"requested_project": test_project.name,
"workspace_id": None,
"edit_operation": "append",
"output_format": "json",
"has_section": False,
"has_find_text": False,
"expected_replacements": 1,
},
)
]
assert operations[0] == (
"mcp.tool.edit_note",
{
"entrypoint": "mcp",
"tool_name": "edit_note",
"requested_project": test_project.name,
"workspace_id": None,
"edit_operation": "append",
"output_format": "json",
"has_section": False,
"has_find_text": False,
"expected_replacements": 1,
},
)
operation_names = [name for name, _ in operations]
assert "api.request.knowledge.resolve_entity" in operation_names
assert "api.request.knowledge.edit_entity" in operation_names
assert _contains_context(
contexts,
{
@@ -275,24 +277,23 @@ async def test_build_context_emits_root_operation_and_project_context(
output_format="json",
)
assert operations == [
(
"mcp.tool.build_context",
{
"entrypoint": "mcp",
"tool_name": "build_context",
"requested_project": test_project.name,
"workspace_id": None,
"depth": 2,
"timeframe": "7d",
"page": 1,
"page_size": 5,
"max_related": 3,
"output_format": "json",
"is_memory_url": True,
},
)
]
assert operations[0] == (
"mcp.tool.build_context",
{
"entrypoint": "mcp",
"tool_name": "build_context",
"requested_project": test_project.name,
"workspace_id": None,
"depth": 2,
"timeframe": "7d",
"page": 1,
"page_size": 5,
"max_related": 3,
"output_format": "json",
"is_memory_url": True,
},
)
assert "api.request.memory.build_context" in [name for name, _ in operations]
assert _contains_context(
contexts,
{
+20
View File
@@ -91,6 +91,26 @@ def test_relation_response():
assert relation.context is None
def test_relation_response_allows_long_relation_type():
"""Long relation labels should round-trip because stored data has no DB length cap."""
long_relation_type = (
"**Architecture/efficiency concern:** "
"the orchestration prompt expanded a short edge label into a full descriptive note "
"that is much longer than 200 characters but still represents the stored relation type."
)
data = {
"permalink": "test/123/long/test/456",
"from_id": "test/123",
"to_id": "test/456",
"relation_type": long_relation_type,
"from_entity": {"permalink": "test/123"},
"to_entity": {"permalink": "test/456"},
}
relation = RelationResponse.model_validate(data)
assert relation.relation_type == long_relation_type
def test_relation_response_with_null_permalink():
"""Test RelationResponse handles null permalinks by falling back to file_path (fixes issue #483).
+113
View File
@@ -2426,3 +2426,116 @@ async def test_fast_write_entity_null_user_id(entity_service: EntityService):
entity = await entity_service.fast_write_entity(schema, external_id=str(uuid.uuid4()))
assert entity.created_by is None
assert entity.last_updated_by is None
# --- Concurrent Delete Resilience ---
@pytest.mark.asyncio
async def test_delete_entity_by_id_already_deleted(entity_service: EntityService):
"""delete_entity returns True when entity was already deleted (concurrent delete)."""
entity_data = EntitySchema(
title="ConcurrentDeleteTarget",
directory="test",
note_type="test",
)
created = await entity_service.create_entity(entity_data)
entity_id = created.id
# Delete once - should succeed
assert await entity_service.delete_entity(entity_id) is True
# Delete again by ID - should return True (already deleted), not raise ValueError
assert await entity_service.delete_entity(entity_id) is True
@pytest.mark.asyncio
async def test_delete_entity_by_permalink_already_deleted(entity_service: EntityService):
"""delete_entity returns True when entity was already deleted by permalink."""
entity_data = EntitySchema(
title="ConcurrentDeleteByPermalink",
directory="test",
note_type="test",
)
created = await entity_service.create_entity(entity_data)
# Delete once
assert await entity_service.delete_entity(created.id) is True
# Delete again by permalink - should return True (EntityNotFoundError caught)
assert await entity_service.delete_entity(entity_data.permalink) is True
@pytest.mark.asyncio
async def test_delete_directory_concurrent_resilience(entity_service: EntityService):
"""delete_directory succeeds even when entities are concurrently deleted."""
# Create entities in a directory
entities = []
for i in range(5):
entity_data = EntitySchema(
title=f"DirEntity{i}",
directory="concurrent-test",
note_type="test",
)
created = await entity_service.create_entity(entity_data)
entities.append(created)
# Delete some entities directly to simulate concurrent delete
await entity_service.delete_entity(entities[0].id)
await entity_service.delete_entity(entities[2].id)
# Now delete the directory - should handle already-deleted entities gracefully
result = await entity_service.delete_directory("concurrent-test")
# Only 3 remain in DB (2 were already deleted before the directory query)
assert result.total_files == 3
assert result.successful_deletes == 3
assert result.failed_deletes == 0
assert len(result.errors) == 0
@pytest.mark.asyncio
async def test_delete_directory_all_already_deleted(entity_service: EntityService):
"""delete_directory handles case where all entities were concurrently deleted."""
# Create entities
created_entities = []
for i in range(3):
entity_data = EntitySchema(
title=f"AllGone{i}",
directory="all-deleted",
note_type="test",
)
created = await entity_service.create_entity(entity_data)
created_entities.append(created)
# Delete all entities directly
for e in created_entities:
await entity_service.delete_entity(e.id)
# Directory delete should report empty (entities no longer found in prefix query)
result = await entity_service.delete_directory("all-deleted")
assert result.total_files == 0
assert result.successful_deletes == 0
assert result.failed_deletes == 0
@pytest.mark.asyncio
async def test_delete_directory_entity_deleted_between_query_and_delete(
entity_service: EntityService,
):
"""Simulates the real race condition: entity exists in prefix query but is deleted
by a concurrent request before delete_entity is called."""
# Create entities
entity_data = EntitySchema(title="RaceTarget", directory="race-dir", note_type="test")
created = await entity_service.create_entity(entity_data)
# Get the entities via prefix query (as delete_directory does)
entities = await entity_service.repository.find_by_directory_prefix("race-dir")
assert len(entities) == 1
# Now delete the entity behind the scenes (simulating a concurrent request)
await entity_service.delete_entity(created.id)
# Call delete_entity with the stale entity ID - should return True, not raise
result = await entity_service.delete_entity(entities[0].id)
assert result is True
@@ -0,0 +1,138 @@
"""Telemetry coverage for entity service write/edit/reindex paths."""
from __future__ import annotations
import importlib
from contextlib import contextmanager
import pytest
from basic_memory.schemas import Entity as EntitySchema
entity_service_module = importlib.import_module("basic_memory.services.entity_service")
def _capture_spans():
spans: list[tuple[str, dict]] = []
@contextmanager
def fake_span(name: str, **attrs):
spans.append((name, attrs))
yield
return spans, fake_span
def _assert_names_in_order(names: list[str], expected: list[str]) -> None:
cursor = 0
for expected_name in expected:
cursor = names.index(expected_name, cursor) + 1
@pytest.mark.asyncio
async def test_create_entity_emits_expected_phase_spans(entity_service, monkeypatch) -> None:
spans, fake_span = _capture_spans()
monkeypatch.setattr(entity_service_module.telemetry, "span", fake_span)
schema = EntitySchema(
title="Telemetry Create",
directory="notes",
note_type="note",
content_type="text/markdown",
content="Create telemetry content",
)
entity = await entity_service.create_entity(schema)
assert entity.title == "Telemetry Create"
span_names = [name for name, _ in spans]
_assert_names_in_order(
span_names,
[
"entity_service.create.resolve_permalink",
"entity_service.create.write_file",
"file_service.write",
"entity_service.create.parse_markdown",
"entity_service.create.upsert_entity",
"entity_service.upsert.base_entity",
"entity_service.upsert.relations",
"entity_service.upsert.persist_checksum",
],
)
@pytest.mark.asyncio
async def test_edit_entity_emits_expected_phase_spans(entity_service, monkeypatch) -> None:
created = await entity_service.create_entity(
EntitySchema(
title="Telemetry Edit",
directory="notes",
note_type="note",
content_type="text/markdown",
content="Before edit",
)
)
spans, fake_span = _capture_spans()
monkeypatch.setattr(entity_service_module.telemetry, "span", fake_span)
updated = await entity_service.edit_entity(
created.file_path,
operation="append",
content="\n\nAfter edit",
)
assert updated.id == created.id
span_names = [name for name, _ in spans]
_assert_names_in_order(
span_names,
[
"entity_service.edit.resolve_entity",
"entity_service.edit.read_file",
"file_service.read",
"entity_service.edit.apply_operation",
"entity_service.edit.write_file",
"file_service.write",
"entity_service.edit.parse_markdown",
"entity_service.edit.upsert_entity",
"entity_service.upsert.base_entity",
"entity_service.upsert.relations",
"entity_service.upsert.persist_checksum",
],
)
@pytest.mark.asyncio
async def test_reindex_entity_emits_expected_phase_spans(entity_service, monkeypatch) -> None:
created = await entity_service.create_entity(
EntitySchema(
title="Telemetry Reindex",
directory="notes",
note_type="note",
content_type="text/markdown",
content="Reindex telemetry content",
)
)
spans, fake_span = _capture_spans()
monkeypatch.setattr(entity_service_module.telemetry, "span", fake_span)
await entity_service.reindex_entity(created.id)
span_names = [name for name, _ in spans]
_assert_names_in_order(
span_names,
[
"entity_service.reindex.load_entity",
"entity_service.reindex.read_file",
"file_service.read_content",
"entity_service.reindex.parse_markdown",
"entity_service.reindex.upsert_entity",
"entity_service.upsert.base_entity",
"entity_service.upsert.relations",
"entity_service.upsert.hydrate_entity",
"entity_service.reindex.update_checksum",
],
)
if entity_service.search_service is not None:
assert "entity_service.reindex.search_index" in span_names
@@ -0,0 +1,86 @@
"""Tests for EntityWriteResult content variants."""
import pytest
from basic_memory.file_utils import remove_frontmatter
from basic_memory.schemas import Entity as EntitySchema
@pytest.mark.asyncio
async def test_create_entity_with_content_returns_full_and_search_content(
entity_service, file_service
) -> None:
result = await entity_service.create_entity_with_content(
EntitySchema(
title="Create Write Result",
directory="notes",
note_type="note",
content="Create body content",
)
)
file_path = file_service.get_entity_path(result.entity)
file_content, _ = await file_service.read_file(file_path)
assert result.content == file_content
assert result.search_content == remove_frontmatter(file_content)
assert result.search_content == "Create body content"
@pytest.mark.asyncio
async def test_update_entity_with_content_returns_full_and_search_content(
entity_service, file_service
) -> None:
created = await entity_service.create_entity(
EntitySchema(
title="Update Write Result",
directory="notes",
note_type="note",
content="Original body content",
)
)
result = await entity_service.update_entity_with_content(
created,
EntitySchema(
title="Update Write Result",
directory="notes",
note_type="note",
content="Updated body content",
),
)
file_path = file_service.get_entity_path(result.entity)
file_content, _ = await file_service.read_file(file_path)
assert result.content == file_content
assert result.search_content == remove_frontmatter(file_content)
assert result.search_content == "Updated body content"
@pytest.mark.asyncio
async def test_edit_entity_with_content_returns_full_and_search_content(
entity_service, file_service
) -> None:
created = await entity_service.create_entity(
EntitySchema(
title="Edit Write Result",
directory="notes",
note_type="note",
content="Original body content",
)
)
result = await entity_service.edit_entity_with_content(
identifier=created.permalink,
operation="find_replace",
content="Edited body content",
find_text="Original body content",
)
file_path = file_service.get_entity_path(result.entity)
file_content, _ = await file_service.read_file(file_path)
assert result.content == file_content
assert result.search_content == remove_frontmatter(file_content)
assert result.search_content == "Edited body content"
@@ -34,14 +34,21 @@ async def test_search_service_wraps_repository_search(search_service, monkeypatc
"search.execute",
{
"retrieval_mode": "fts",
"has_text_query": True,
"has_title_query": False,
"has_permalink_query": False,
"has_metadata_filters": False,
"has_query": True,
"has_filters": False,
"limit": 10,
"offset": 0,
},
)
assert spans[1] == (
"search.repository_query",
{
"retrieval_mode": "fts",
"phase": "repository_query",
"has_query": True,
"has_filters": False,
},
)
@pytest.mark.asyncio
@@ -58,8 +65,13 @@ async def test_search_service_emits_relaxed_retry_span(search_service, monkeypat
await search_service.search(SearchQuery(text="who are our main competitors and partners"))
assert [name for name, _ in spans] == ["search.execute", "search.relaxed_fts_retry"]
assert spans[1] == (
assert [name for name, _ in spans] == [
"search.execute",
"search.repository_query",
"search.relaxed_fts_retry",
"search.repository_query",
]
assert spans[2] == (
"search.relaxed_fts_retry",
{
"retrieval_mode": "fts",
+42 -26
View File
@@ -6,7 +6,7 @@ from contextlib import contextmanager
from contextvars import copy_context
from loguru import logger
from basic_memory import telemetry
from basic_memory import __version__, telemetry
from basic_memory.config import init_api_logging, init_cli_logging, init_mcp_logging
@@ -33,7 +33,15 @@ class FakeLogfire:
@contextmanager
def span(self, name: str, **attrs):
self.span_calls.append((name, attrs))
yield
class FakeStartedSpan:
def set_attribute(self, key: str, value) -> None:
attrs[key] = value
def set_attributes(self, new_attrs: dict) -> None:
attrs.update(new_attrs)
yield FakeStartedSpan()
def test_configure_telemetry_disabled_is_noop() -> None:
@@ -91,24 +99,6 @@ def test_configure_telemetry_retries_without_send_to_logfire(monkeypatch) -> Non
assert "send_to_logfire" not in fake_logfire.configure_calls[1]
def test_bind_telemetry_context_filters_nulls() -> None:
bound = telemetry.bind_telemetry_context(project_name="main", workspace_id=None)
extra = bound._options[-1] # type: ignore[attr-defined]
assert extra == {"project_name": "main"}
def test_bind_telemetry_context_merges_active_context() -> None:
with telemetry.contextualize(project_name="main", route_mode="local_asgi"):
bound = telemetry.bind_telemetry_context(tool_name="write_note", workspace_id=None)
extra = bound._options[-1] # type: ignore[attr-defined]
assert extra == {
"project_name": "main",
"route_mode": "local_asgi",
"tool_name": "write_note",
}
def test_contextualize_adds_filtered_loguru_context() -> None:
records: list[dict] = []
sink_id = logger.add(lambda message: records.append(message.record["extra"].copy()))
@@ -158,6 +148,34 @@ def test_span_uses_logfire_when_enabled(monkeypatch) -> None:
assert fake_logfire.span_calls == [("mcp.tool.write_note", {"project_name": "main"})]
def test_started_span_exposes_mutable_logfire_handle(monkeypatch) -> None:
fake_logfire = FakeLogfire()
telemetry.reset_telemetry_state()
monkeypatch.setattr(telemetry, "_load_logfire", lambda: fake_logfire)
telemetry.configure_telemetry(
"basic-memory-mcp",
environment="dev",
enable_logfire=True,
)
with telemetry.started_span("mcp.http.request", method="GET") as span:
assert span is not None
span.set_attribute("status_code", 200)
span.set_attributes({"is_success": True, "outcome": "success"})
assert fake_logfire.span_calls == [
(
"mcp.http.request",
{
"method": "GET",
"status_code": 200,
"is_success": True,
"outcome": "success",
},
)
]
def test_operation_creates_span_and_log_context(monkeypatch) -> None:
fake_logfire = FakeLogfire()
records: list[dict] = []
@@ -212,9 +230,7 @@ def test_scope_creates_span_and_nested_log_context(monkeypatch) -> None:
finally:
logger.remove(sink_id)
assert fake_logfire.span_calls == [
("routing.client_session", {"route_mode": "local_asgi"})
]
assert fake_logfire.span_calls == [("routing.client_session", {"route_mode": "local_asgi"})]
assert records == [{"project_name": "main", "route_mode": "local_asgi"}]
@@ -270,21 +286,21 @@ def test_init_logging_functions_configure_telemetry_and_logging(monkeypatch) ->
{
"service_name": "basic-memory-cli",
"environment": "staging",
"service_version": "0.20.2",
"service_version": __version__,
"enable_logfire": True,
"send_to_logfire": False,
},
{
"service_name": "basic-memory-mcp",
"environment": "staging",
"service_version": "0.20.2",
"service_version": __version__,
"enable_logfire": True,
"send_to_logfire": False,
},
{
"service_name": "basic-memory-api",
"environment": "staging",
"service_version": "0.20.2",
"service_version": __version__,
"enable_logfire": True,
"send_to_logfire": False,
},
+57 -3
View File
@@ -1769,6 +1769,60 @@
"node": ">=14.0.0"
}
},
"node_modules/@tailwindcss/oxide-wasm32-wasi/node_modules/@emnapi/core": {
"version": "1.7.1",
"inBundle": true,
"license": "MIT",
"optional": true,
"dependencies": {
"@emnapi/wasi-threads": "1.1.0",
"tslib": "^2.4.0"
}
},
"node_modules/@tailwindcss/oxide-wasm32-wasi/node_modules/@emnapi/runtime": {
"version": "1.7.1",
"inBundle": true,
"license": "MIT",
"optional": true,
"dependencies": {
"tslib": "^2.4.0"
}
},
"node_modules/@tailwindcss/oxide-wasm32-wasi/node_modules/@emnapi/wasi-threads": {
"version": "1.1.0",
"inBundle": true,
"license": "MIT",
"optional": true,
"dependencies": {
"tslib": "^2.4.0"
}
},
"node_modules/@tailwindcss/oxide-wasm32-wasi/node_modules/@napi-rs/wasm-runtime": {
"version": "1.1.0",
"inBundle": true,
"license": "MIT",
"optional": true,
"dependencies": {
"@emnapi/core": "^1.7.1",
"@emnapi/runtime": "^1.7.1",
"@tybys/wasm-util": "^0.10.1"
}
},
"node_modules/@tailwindcss/oxide-wasm32-wasi/node_modules/@tybys/wasm-util": {
"version": "0.10.1",
"inBundle": true,
"license": "MIT",
"optional": true,
"dependencies": {
"tslib": "^2.4.0"
}
},
"node_modules/@tailwindcss/oxide-wasm32-wasi/node_modules/tslib": {
"version": "2.8.1",
"inBundle": true,
"license": "0BSD",
"optional": true
},
"node_modules/@tailwindcss/oxide-win32-arm64-msvc": {
"version": "4.1.18",
"resolved": "https://registry.npmjs.org/@tailwindcss/oxide-win32-arm64-msvc/-/oxide-win32-arm64-msvc-4.1.18.tgz",
@@ -2237,9 +2291,9 @@
"license": "ISC"
},
"node_modules/picomatch": {
"version": "4.0.3",
"resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.3.tgz",
"integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==",
"version": "4.0.4",
"resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.4.tgz",
"integrity": "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==",
"license": "MIT",
"engines": {
"node": ">=12"
Generated
+6 -6
View File
@@ -142,14 +142,14 @@ wheels = [
[[package]]
name = "authlib"
version = "1.6.7"
version = "1.6.9"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "cryptography" },
]
sdist = { url = "https://files.pythonhosted.org/packages/49/dc/ed1681bf1339dd6ea1ce56136bad4baabc6f7ad466e375810702b0237047/authlib-1.6.7.tar.gz", hash = "sha256:dbf10100011d1e1b34048c9d120e83f13b35d69a826ae762b93d2fb5aafc337b", size = 164950, upload-time = "2026-02-06T14:04:14.171Z" }
sdist = { url = "https://files.pythonhosted.org/packages/af/98/00d3dd826d46959ad8e32af2dbb2398868fd9fd0683c26e56d0789bd0e68/authlib-1.6.9.tar.gz", hash = "sha256:d8f2421e7e5980cc1ddb4e32d3f5fa659cfaf60d8eaf3281ebed192e4ab74f04", size = 165134, upload-time = "2026-03-02T07:44:01.998Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/f8/00/3ed12264094ec91f534fae429945efbaa9f8c666f3aa7061cc3b2a26a0cd/authlib-1.6.7-py2.py3-none-any.whl", hash = "sha256:c637340d9a02789d2efa1d003a7437d10d3e565237bcb5fcbc6c134c7b95bab0", size = 244115, upload-time = "2026-02-06T14:04:12.141Z" },
{ url = "https://files.pythonhosted.org/packages/53/23/b65f568ed0c22f1efacb744d2db1a33c8068f384b8c9b482b52ebdbc3ef6/authlib-1.6.9-py2.py3-none-any.whl", hash = "sha256:f08b4c14e08f0861dc18a32357b33fbcfd2ea86cfe3fe149484b4d764c4a0ac3", size = 244197, upload-time = "2026-03-02T07:44:00.307Z" },
]
[[package]]
@@ -2744,7 +2744,7 @@ wheels = [
[[package]]
name = "requests"
version = "2.32.5"
version = "2.33.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "certifi" },
@@ -2752,9 +2752,9 @@ dependencies = [
{ name = "idna" },
{ name = "urllib3" },
]
sdist = { url = "https://files.pythonhosted.org/packages/c9/74/b3ff8e6c8446842c3f5c837e9c3dfcfe2018ea6ecef224c710c85ef728f4/requests-2.32.5.tar.gz", hash = "sha256:dbba0bac56e100853db0ea71b82b4dfd5fe2bf6d3754a8893c3af500cec7d7cf", size = 134517, upload-time = "2025-08-18T20:46:02.573Z" }
sdist = { url = "https://files.pythonhosted.org/packages/34/64/8860370b167a9721e8956ae116825caff829224fbca0ca6e7bf8ddef8430/requests-2.33.0.tar.gz", hash = "sha256:c7ebc5e8b0f21837386ad0e1c8fe8b829fa5f544d8df3b2253bff14ef29d7652", size = 134232, upload-time = "2026-03-25T15:10:41.586Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/1e/db/4254e3eabe8020b458f1a747140d32277ec7a271daf1d235b70dc0b4e6e3/requests-2.32.5-py3-none-any.whl", hash = "sha256:2462f94637a34fd532264295e186976db0f5d453d1cdd31473c85a6a161affb6", size = 64738, upload-time = "2025-08-18T20:46:00.542Z" },
{ url = "https://files.pythonhosted.org/packages/56/5d/c814546c2333ceea4ba42262d8c4d55763003e767fa169adc693bd524478/requests-2.33.0-py3-none-any.whl", hash = "sha256:3324635456fa185245e24865e810cecec7b4caf933d7eb133dcde67d48cee69b", size = 65017, upload-time = "2026-03-25T15:10:40.382Z" },
]
[[package]]