Compare commits

...

36 Commits

Author SHA1 Message Date
phernandez 2d65d1b6bb Support typed search facet filters 2026-05-04 15:24:42 -05:00
phernandez e1adf10f1b test(api): cover postgres search count syntax errors
Signed-off-by: phernandez <paul@basicmachines.co>
2026-05-04 14:54:57 -05:00
phernandez fa4845119d fix(api): avoid semantic count searches
Signed-off-by: phernandez <paul@basicmachines.co>
2026-05-04 14:21:27 -05:00
phernandez 3589e21278 fix(api): handle blank search text
Signed-off-by: phernandez <paul@basicmachines.co>
2026-05-04 13:35:13 -05:00
phernandez 7f9fc80e27 Add exact totals to search pagination
Signed-off-by: phernandez <paul@basicmachines.co>
2026-05-04 12:57:39 -05:00
Paul Hernandez a661e924df feat(mcp): create projects by workspace slug (#789)
Signed-off-by: phernandez <paul@basicmachines.co>
2026-05-03 19:31:55 -05:00
Paul Hernandez 05adda1502 fix(mcp): route workspace-qualified memory urls (#790)
Signed-off-by: phernandez <paul@basicmachines.co>
2026-05-03 18:18:10 -05:00
Drew Cain 0a72d81bb3 fix(mcp): cap recent_activity rows with explicit truncation footer (#785)
Signed-off-by: Drew Cain <groksrc@gmail.com>
Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-03 17:43:42 -05:00
Drew Cain 5ccf433cad fix(cli): point bm cloud setup hint at bm cloud sync-setup (#780)
Signed-off-by: Drew Cain <groksrc@gmail.com>
2026-05-02 17:35:24 -05:00
Paul Hernandez b4bf14ebf7 fix(mcp): list factory projects across workspaces (#778)
Signed-off-by: phernandez <paul@basicmachines.co>
2026-05-02 09:17:25 -05:00
Paul Hernandez 0b335476d6 fix(mcp): resolve projects by external_id, remove workspace from MCP tools (#777)
Signed-off-by: phernandez <paul@basicmachines.co>
Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
2026-05-01 05:15:44 -05:00
Paul Hernandez 2fccc74a20 feat(cli): refuse db reset while basic-memory mcp processes run (#776)
Signed-off-by: phernandez <paul@basicmachines.co>
2026-04-29 15:48:19 -05:00
Paul Hernandez c4956cac16 fix(cli): cleanup local DB state on set-cloud/set-local (#775)
Signed-off-by: phernandez <paul@basicmachines.co>
2026-04-29 11:50:14 -05:00
Paul Hernandez 128c2da40c fix(cli): clear default_workspace on cloud logout (#773)
Signed-off-by: phernandez <paul@basicmachines.co>
2026-04-29 09:53:09 -05:00
Paul Hernandez 2bfb9c76df fix(core): degrade gracefully when sqlite-vec cannot load on init (#774)
Signed-off-by: phernandez <paul@basicmachines.co>
2026-04-29 08:57:31 -05:00
Paul Hernandez 3d927b848f fix(installer): mount docker-compose config volume to appuser home (#772)
Signed-off-by: phernandez <paul@basicmachines.co>
2026-04-29 08:57:26 -05:00
Paul Hernandez 953fe20aef test(core): regression guard for vector-row cleanup on entity delete (#764) (#771)
Signed-off-by: phernandez <paul@basicmachines.co>
2026-04-28 23:20:15 -05:00
Paul Hernandez a4282d9f2f fix(core): skip Obsidian callouts in observation parser (#769)
Signed-off-by: phernandez <paul@basicmachines.co>
2026-04-28 23:15:13 -05:00
Paul Hernandez 799dd6c629 fix(mcp): remove no-op pagination params from read_note and view_note (#768)
Signed-off-by: phernandez <paul@basicmachines.co>
2026-04-28 23:05:14 -05:00
Paul Hernandez 26e74ea118 test(core): regression guard for long relation_type values (#721) (#770)
Signed-off-by: phernandez <paul@basicmachines.co>
2026-04-28 22:55:14 -05:00
Paul Hernandez ee1558ea68 feat(mcp): accept training-data-friendly parameter aliases (#766)
Signed-off-by: phernandez <paul@basicmachines.co>
2026-04-28 20:10:36 -05:00
Viktor Szépe 4d62b623db chore(core): fix typos (#761)
Signed-off-by: Viktor Szépe <viktor@szepe.net>
2026-04-23 10:12:46 -05:00
Paul Hernandez 2fe4488eda fix(sync): constrain watch service to --project scope (#759)
Signed-off-by: phernandez <paul@basicmachines.co>
2026-04-20 13:46:13 -05:00
Paul Hernandez f3e46d7984 feat(mcp): discover projects across workspaces (#757)
Signed-off-by: phernandez <paul@basicmachines.co>
2026-04-18 14:04:57 -05:00
jope-bm 56d6f1b4a5 fix(mcp): report cloud projects as source=cloud in factory mode (#752)
Signed-off-by: Joe P <joe@basicmemory.com>
Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
2026-04-18 11:10:20 -06:00
Paul Hernandez 1b39062ecd refactor(core): rip telemetry wrappers, use logfire directly (#754)
Signed-off-by: phernandez <paul@basicmachines.co>
Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-04-17 13:58:19 -05:00
Paul Hernandez c4cf0aff1e perf(sync): speed up single markdown file indexing (#751)
Signed-off-by: phernandez <paul@basicmachines.co>
2026-04-17 07:08:21 -05:00
phernandez 1c343bed66 perf(sync): skip unchanged markdown indexing
Signed-off-by: phernandez <paul@basicmachines.co>
2026-04-16 18:27:52 -05:00
phernandez c50d97e548 fix(sync): instrument single markdown indexing
Signed-off-by: phernandez <paul@basicmachines.co>
2026-04-16 18:04:17 -05:00
Drew Cain e2e65575d6 fix(core): honor BASIC_MEMORY_CONFIG_DIR across remaining call sites (#744)
Signed-off-by: Drew Cain <groksrc@gmail.com>
Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
2026-04-15 22:55:25 -05:00
Drew Cain bf9a6b4a75 fix(core): resolve FastEmbed cache under data dir instead of /tmp (#743)
Signed-off-by: Drew Cain <groksrc@gmail.com>
Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
2026-04-15 22:55:00 -05:00
Paul Hernandez 474100efef ci(core): reduce duplicate CI and normalize Windows assertions (#749)
Signed-off-by: phernandez <paul@basicmachines.co>
2026-04-15 22:53:33 -05:00
Paul Hernandez b3d5448355 fix(sync): preserve canonical markdown in single-file sync (#746)
Signed-off-by: phernandez <paul@basicmachines.co>
2026-04-15 20:04:28 -05:00
Paul Hernandez 4e53bb83fd refactor(core): simplify note write flow (#739)
Signed-off-by: phernandez <paul@basicmachines.co>
2026-04-15 17:56:27 -05:00
phernandez 8f2b25f0e0 test(core): stabilize postgres fixtures
Signed-off-by: phernandez <paul@basicmachines.co>
2026-04-13 14:25:26 -05:00
Paul Hernandez 052545b661 chore(core): make ty the default typechecker (#736)
Signed-off-by: phernandez <paul@basicmachines.co>
2026-04-13 10:34:01 -05:00
206 changed files with 12713 additions and 5151 deletions
-1
View File
@@ -3,7 +3,6 @@
"env": {
"CLAUDE_BASH_MAINTAIN_PROJECT_WORKING_DIR": "1",
"CLAUDE_CODE_DISABLE_FEEDBACK_SURVEY": "1",
"DISABLE_TELEMETRY": "1",
"CLAUDE_CODE_NO_FLICKER": "1",
"CLAUDE_CODE_DISABLE_ADAPTIVE_THINKING": "1"
},
+4 -2
View File
@@ -5,9 +5,11 @@ concurrency:
cancel-in-progress: true
on:
# Trigger: PR branch pushes already publish commit statuses that show up on the PR.
# Why: running the full matrix on both push and pull_request doubles CI time for the
# exact same branch head commit.
# Outcome: each branch push runs the test suite once, including PR updates.
push:
pull_request:
branches: [ "main" ]
jobs:
static-checks:
+7 -5
View File
@@ -22,16 +22,16 @@ See the [README.md](README.md) file for a project overview.
- Run unit tests (Postgres): `just test-unit-postgres`
- Run integration tests (SQLite): `just test-int-sqlite`
- Run integration tests (Postgres): `just test-int-postgres`
- Run impacted tests: `just testmon` (pytest-testmon)
- Run impacted tests: `just testmon` (pytest-testmon; only tests affected by changed code)
- Run MCP smoke test: `just test-smoke`
- Fast local loop: `just fast-check`
- Fast local loop: `just fast-check` (default iteration flow)
- Local consistency check: `just doctor`
- Generate HTML coverage: `just coverage`
- Single test: `pytest tests/path/to/test_file.py::test_function_name`
- Run benchmarks: `pytest test-int/test_sync_performance_benchmark.py -v -m "benchmark and not slow"`
- Lint: `just lint` or `ruff check . --fix`
- Type check: `just typecheck` or `uv run pyright`
- Type check (supplemental): `just typecheck-ty` or `uv run ty check src/`
- Type check: `just typecheck` or `uv run ty check src tests test-int`
- Type check (pyright): `just typecheck-pyright` or `uv run pyright`
- Format: `just format` or `uv run ruff format .`
- Run all code checks: `just check` (runs lint, format, typecheck, test)
- Create db migration: `just migration "Your migration message"`
@@ -48,10 +48,12 @@ See the [README.md](README.md) file for a project overview.
### Code/Test/Verify Loop (fast path)
1) **Code:** make changes.
2) **Test:** `just fast-check` (lint/format/typecheck + impacted tests + MCP smoke).
2) **Test:** `just fast-check` (lint/format/typecheck + pytest-testmon impacted tests for changed code).
3) **Verify:** `just doctor` (end-to-end file ↔ DB loop in a temp project).
4) **Full gate (when needed):** `just test` or `just check` for SQLite + Postgres.
Run `just test-smoke` when you specifically need the MCP smoke flow.
If testmon is “cold,” the first run may be long. Subsequent runs get much faster.
### Test Structure
+5 -5
View File
@@ -2125,12 +2125,12 @@ Signed-off-by: phernandez <paul@basicmachines.co>
- Update CLAUDE.md ([#33](https://github.com/basicmachines-co/basic-memory/pull/33),
[`dfaf0fe`](https://github.com/basicmachines-co/basic-memory/commit/dfaf0fea9cf5b97d169d51a6276ec70162c21a7e))
fix spelling in CLAUDE.md: enviroment -> environment Signed-off-by: Ikko Eltociear Ashimine
fix spelling in CLAUDE.md: environment typo Signed-off-by: Ikko Eltociear Ashimine
<eltociear@gmail.com>
### Refactoring
- Move project stats into projct subcommand
- Move project stats into project subcommand
([`2a881b1`](https://github.com/basicmachines-co/basic-memory/commit/2a881b1425c73947f037fbe7ac5539c015b62526))
Signed-off-by: phernandez <paul@basicmachines.co>
@@ -2559,7 +2559,7 @@ Co-authored-by: phernandez <phernandez@basicmachines.co>
### Bug Fixes
- Refix vitual env in installer build
- Refix virtual env in installer build
([`052f491`](https://github.com/basicmachines-co/basic-memory/commit/052f491fff629e8ead629c9259f8cb46c608d584))
@@ -2578,7 +2578,7 @@ Co-authored-by: phernandez <phernandez@basicmachines.co>
### Bug Fixes
- Fix path to intaller app artifact
- Fix path to installer app artifact
([`53d220d`](https://github.com/basicmachines-co/basic-memory/commit/53d220df585561f9edd0d49a9e88f1d4055059cf))
@@ -2586,7 +2586,7 @@ Co-authored-by: phernandez <phernandez@basicmachines.co>
### Bug Fixes
- Activate vitualenv in installer build
- Activate virtualenv in installer build
([`d4c8293`](https://github.com/basicmachines-co/basic-memory/commit/d4c8293687a52eaf3337fe02e2f7b80e4cc9a1bb))
- Trigger installer build on release
+1 -1
View File
@@ -516,7 +516,7 @@ canvas(nodes, edges, title, folder) - Generate knowledge visualizations
"What have I been working on in the past week?"
```
## Futher info
## Further info
See the [Documentation](https://docs.basicmemory.com?utm_source=github&utm_medium=referral&utm_campaign=readme) for more info, including:
+3 -1
View File
@@ -17,7 +17,9 @@ services:
volumes:
# Persistent storage for configuration and database
- basic-memory-config:/root/.basic-memory:rw
# Container runs as `appuser` (Dockerfile USER directive), so the CLI
# config dir lives under /home/appuser, not /root.
- basic-memory-config:/home/appuser/.basic-memory:rw
# Mount your knowledge directory (required)
# Change './knowledge' to your actual Obsidian vault or knowledge directory
+41 -1
View File
@@ -91,10 +91,11 @@ SQLite Database (Index)
# List all projects
projects = await list_memory_projects()
# Response structure:
# Response structure (each entry includes external_id you can pass as project_id):
# [
# {
# "name": "main",
# "external_id": "550e8400-e29b-41d4-a716-446655440000",
# "path": "/Users/name/notes",
# "is_default": True,
# "note_count": 156,
@@ -102,6 +103,7 @@ projects = await list_memory_projects()
# },
# {
# "name": "work",
# "external_id": "9f86d081-884c-42a3-b5e3-1c0c5b4c8e52",
# "path": "/Users/name/work-notes",
# "is_default": False,
# "note_count": 89,
@@ -164,6 +166,44 @@ active_project = "main"
results = await search_notes(query="topic", project=active_project)
```
### `project` vs `project_id`
Every project has two identifiers:
- **`project`** — human-readable name (e.g., `"main"`). Easy to use, but can collide across cloud workspaces.
- **`project_id`** — stable `external_id` UUID. Always unambiguous; takes precedence over `project` when both are passed.
**When to prefer `project_id`:**
1. **Cloud multi-workspace setups.** If the user belongs to more than one workspace (personal + organization, or several organizations) and the same project name might exist in more than one of them, pass `project_id` to route to the exact project. Without it, name resolution falls back to the default workspace, which may not be the one the user means.
2. **After `list_memory_projects()`.** Once you have the `external_id`, prefer using it — it's the same number of characters in JSON and saves a name-resolution round-trip.
3. **When persisting a project choice across a long session.** UUIDs are stable; names can be renamed.
**When `project` (name) is fine:**
- Local single-workspace setups (no collision risk).
- One-off operations where the name is clearly visible to the user (e.g., quick `search_notes(project="main", ...)`).
- The user explicitly references a project by name in their message.
**Example — cloud multi-workspace pattern:**
```python
# Discover and pick the right project for this user
projects = await list_memory_projects()
target = next(p for p in projects if p["name"] == "research" and p["workspace"]["slug"] == "acme")
# Use the UUID for all subsequent operations — no ambiguity
await write_note(
title="Meeting Notes",
content="...",
folder="meetings",
project_id=target["external_id"],
)
results = await search_notes(query="kickoff", project_id=target["external_id"])
```
**Precedence rule:** When both are passed, `project_id` wins. This lets you safely supply `project="main"` for backward compatibility while still routing precisely with `project_id`.
### Cross-Project Operations
**Some tools work across all projects when project parameter omitted:**
+4 -4
View File
@@ -113,7 +113,7 @@ bm project add research --cloud
bm project add research --cloud --local-path ~/Documents/research
# Or configure sync for existing project
bm project sync-setup research ~/Documents/research
bm cloud sync-setup research ~/Documents/research
```
**What happens under the covers:**
@@ -236,7 +236,7 @@ bm project add research --cloud --local-path ~/Documents/research
```bash
# Project already exists on cloud
bm project sync-setup research ~/Documents/research
bm cloud sync-setup research ~/Documents/research
```
**What this does:**
@@ -739,7 +739,7 @@ bm project sync --name research
**Solution:**
```bash
bm project sync-setup research ~/Documents/research
bm cloud sync-setup research ~/Documents/research
bm project bisync --name research --resync
```
@@ -795,7 +795,7 @@ bm project list --local # Local project list
bm project list --cloud # Cloud project list
bm project add <name> --cloud # Create cloud project (no sync)
bm project add <name> --cloud --local-path <path> # Create with local sync
bm project sync-setup <name> <path> # Add sync to existing project
bm cloud sync-setup <name> <path> # Add sync to existing project
bm project rm <name> # Delete project
```
+9 -5
View File
@@ -62,20 +62,20 @@ test-int-postgres:
fi
# Run tests impacted by recent changes (requires pytest-testmon)
# Pass paths or node ids after `just testmon` to limit the candidate set further.
testmon *args:
BASIC_MEMORY_ENV=test uv run pytest -p pytest_mock -v --no-cov --testmon --testmon-forceselect {{args}}
BASIC_MEMORY_ENV=test uv run pytest -p pytest_mock -v --no-cov --testmon {{args}}
# Run MCP smoke test (fast end-to-end loop)
test-smoke:
BASIC_MEMORY_ENV=test uv run pytest -p pytest_mock -v --no-cov -m smoke test-int/mcp/test_smoke_integration.py
# Fast local loop: lint, format, typecheck, impacted tests
# Fast local loop: lint, format, typecheck, impacted tests via pytest-testmon
fast-check:
just fix
just format
just typecheck
just testmon
just test-smoke
# Reset Postgres test database (drops and recreates schema)
# Useful when Alembic migration state gets out of sync during development
@@ -170,13 +170,17 @@ lint: fix
fix:
uv run ruff check --fix --unsafe-fixes src tests test-int
# Type check code (pyright)
# Type check code (ty)
typecheck:
uv run ty check src tests test-int
# Type check code (pyright)
typecheck-pyright:
uv run pyright
# Type check code (ty)
typecheck-ty:
uv run ty check src/
just typecheck
# Clean build artifacts and cache files
clean:
+9 -4
View File
@@ -33,7 +33,7 @@ dependencies = [
"pyjwt>=2.10.1",
"python-dotenv>=1.1.0",
"pytest-aio>=1.9.0",
"aiofiles>=24.1.0", # Optional observability (disabled by default via config)
"aiofiles>=24.1.0",
"asyncpg>=0.30.0",
"nest-asyncio>=1.6.0", # For Alembic migrations with Postgres
"pytest-asyncio>=1.2.0",
@@ -47,6 +47,8 @@ dependencies = [
"fastembed>=0.7.4",
"sqlite-vec>=0.1.6",
"openai>=1.100.2",
"logfire>=4.19.0",
"psutil>=5.9.0",
]
[project.urls]
@@ -58,9 +60,6 @@ Documentation = "https://github.com/basicmachines-co/basic-memory#readme"
basic-memory = "basic_memory.cli.main:app"
bm = "basic_memory.cli.main:app"
[project.optional-dependencies]
telemetry = ["logfire>=4.19.0"]
[build-system]
requires = ["hatchling", "uv-dynamic-versioning>=0.7.0"]
build-backend = "hatchling.build"
@@ -71,6 +70,12 @@ addopts = "--cov=basic_memory --cov-report term-missing"
testpaths = ["tests", "test-int"]
asyncio_mode = "strict"
asyncio_default_fixture_loop_scope = "function"
filterwarnings = [
"ignore:The @wait_container_is_ready decorator is deprecated.*:DeprecationWarning:testcontainers\\.core\\.waiting_utils",
"ignore:The default datetime adapter is deprecated as of Python 3\\.12.*:DeprecationWarning:aiosqlite\\.core",
"ignore:codecs\\.open\\(\\) is deprecated\\. Use open\\(\\) instead\\.:DeprecationWarning:frontmatter",
"ignore:Parsing dates involving a day of month without a year specified is ambiguous.*:DeprecationWarning:dateparser\\.utils\\.strptime",
]
markers = [
"benchmark: Performance benchmark tests (deselect with '-m \"not benchmark\"')",
"slow: Slow-running tests (deselect with '-m \"not slow\"')",
+40 -4
View File
@@ -4,6 +4,7 @@ from contextlib import asynccontextmanager
from fastapi import FastAPI, HTTPException, Request
from fastapi.exception_handlers import http_exception_handler
from fastapi.responses import JSONResponse
from fastapi.routing import APIRouter
from loguru import logger
@@ -25,10 +26,16 @@ from basic_memory.api.v2.routers.project_router import (
list_projects,
synchronize_projects,
)
from basic_memory import telemetry
import logfire
from basic_memory.config import init_api_logging
from basic_memory.services.exceptions import EntityAlreadyExistsError
from basic_memory.services.initialization import initialize_app
from basic_memory.workspace_context import (
WORKSPACE_SLUG_HEADER,
WORKSPACE_TYPE_HEADER,
workspace_permalink_context_validation_error,
workspace_permalink_context,
)
@asynccontextmanager
@@ -44,7 +51,7 @@ async def lifespan(app: FastAPI): # pragma: no cover
set_container(container)
app.state.container = container
with telemetry.operation(
with logfire.span(
"api.lifecycle.startup",
entrypoint="api",
mode=container.mode.name.lower(),
@@ -69,7 +76,7 @@ async def lifespan(app: FastAPI): # pragma: no cover
yield
# Shutdown - coordinator handles clean task cancellation
with telemetry.operation(
with logfire.span(
"api.lifecycle.shutdown",
entrypoint="api",
mode=container.mode.name.lower(),
@@ -87,6 +94,32 @@ app = FastAPI(
lifespan=lifespan,
)
@app.middleware("http")
async def workspace_permalink_context_middleware(request: Request, call_next):
"""Populate workspace permalink context from request headers."""
workspace_slug = request.headers.get(WORKSPACE_SLUG_HEADER)
workspace_type = request.headers.get(WORKSPACE_TYPE_HEADER)
validation_error = workspace_permalink_context_validation_error(workspace_slug, workspace_type)
if validation_error is not None:
return JSONResponse(
status_code=400,
content={"detail": validation_error},
)
if not workspace_slug:
return await call_next(request)
# ContextVar state remains active across the awaited downstream handler while
# this context manager is open, so entity creation can see request metadata.
with workspace_permalink_context(
workspace_slug=workspace_slug,
workspace_type=workspace_type,
):
return await call_next(request)
# Include v2 routers FIRST (more specific paths must match before /{project} catch-all)
app.include_router(v2_knowledge, prefix="/v2/projects/{project_id}")
app.include_router(v2_memory, prefix="/v2/projects/{project_id}")
@@ -146,4 +179,7 @@ async def exception_handler(request, exc): # pragma: no cover
error_type=type(exc).__name__,
error=str(exc),
)
return await http_exception_handler(request, HTTPException(status_code=500, detail=str(exc)))
return await http_exception_handler(
request,
HTTPException(status_code=500, detail="Internal server error"),
)
@@ -10,10 +10,10 @@ Key improvements:
- Simplified caching strategies
"""
from fastapi import APIRouter, HTTPException, BackgroundTasks, Depends, Response, Path, Query
from fastapi import APIRouter, HTTPException, Response, Path
from loguru import logger
from basic_memory import telemetry
import logfire
from basic_memory.deps import (
EntityServiceV2ExternalDep,
SearchServiceV2ExternalDep,
@@ -24,7 +24,6 @@ from basic_memory.deps import (
RelationRepositoryV2ExternalDep,
ProjectExternalIdPathDep,
TaskSchedulerDep,
FileServiceV2ExternalDep,
)
from basic_memory.schemas import DeleteEntitiesResponse
from basic_memory.schemas.base import Entity
@@ -75,34 +74,40 @@ async def get_graph(
Returns a flat node/edge structure optimized for rendering with graph libraries.
Only includes resolved relations (where to_id is not null).
"""
logger.info("API v2 request: get_graph")
with logfire.span(
"api.request.knowledge.get_graph",
entrypoint="api",
domain="knowledge",
action="get_graph",
):
logger.info("API v2 request: get_graph")
# Fetch all entities for this project
entities = await entity_repository.find_all(use_load_options=False)
nodes = [
GraphNode(
external_id=entity.external_id,
title=entity.title,
note_type=entity.note_type,
file_path=entity.file_path,
)
for entity in entities
]
# Fetch all entities for this project
entities = await entity_repository.find_all(use_load_options=False)
nodes = [
GraphNode(
external_id=entity.external_id,
title=entity.title,
note_type=entity.note_type,
file_path=entity.file_path,
)
for entity in entities
]
# Fetch all resolved relations (to_id is not null) with eager-loaded entities
relations = await relation_repository.find_all()
edges = [
GraphEdge(
from_id=relation.from_entity.external_id,
to_id=relation.to_entity.external_id,
relation_type=relation.relation_type,
)
for relation in relations
if relation.to_entity is not None
]
# Fetch all resolved relations (to_id is not null) with eager-loaded entities
relations = await relation_repository.find_all()
edges = [
GraphEdge(
from_id=relation.from_entity.external_id,
to_id=relation.to_entity.external_id,
relation_type=relation.relation_type,
)
for relation in relations
if relation.to_entity is not None
]
logger.info(f"API v2 response: graph with {len(nodes)} nodes and {len(edges)} edges")
return GraphResponse(nodes=nodes, edges=edges)
logger.info(f"API v2 response: graph with {len(nodes)} nodes and {len(edges)} edges")
return GraphResponse(nodes=nodes, edges=edges)
## Resolution endpoint
@@ -143,7 +148,7 @@ async def resolve_identifier(
"resolution_method": "permalink"
}
"""
with telemetry.operation(
with logfire.span(
"api.request.knowledge.resolve_entity",
entrypoint="api",
domain="knowledge",
@@ -151,25 +156,13 @@ async def resolve_identifier(
):
logger.info(f"API v2 request: resolve_identifier for '{data.identifier}'")
with telemetry.scope(
"api.knowledge.resolve_entity.lookup_entity",
domain="knowledge",
action="resolve_entity",
phase="lookup_entity",
):
entity = await entity_repository.get_by_external_id(data.identifier)
entity = await entity_repository.get_by_external_id(data.identifier)
resolution_method = "external_id" if entity else "search"
if not entity:
with telemetry.scope(
"api.knowledge.resolve_entity.resolve_link",
domain="knowledge",
action="resolve_entity",
phase="resolve_link",
):
entity = await link_resolver.resolve_link(
data.identifier, source_path=data.source_path, strict=data.strict
)
entity = await link_resolver.resolve_link(
data.identifier, source_path=data.source_path, strict=data.strict
)
if entity:
if entity.permalink == data.identifier:
resolution_method = "permalink"
@@ -183,20 +176,14 @@ async def resolve_identifier(
if not entity:
raise HTTPException(status_code=404, detail=f"Entity not found: '{data.identifier}'")
with telemetry.scope(
"api.knowledge.resolve_entity.shape_response",
domain="knowledge",
action="resolve_entity",
phase="shape_response",
):
result = EntityResolveResponse(
external_id=entity.external_id,
entity_id=entity.id,
permalink=entity.permalink,
file_path=entity.file_path,
title=entity.title,
resolution_method=resolution_method,
)
result = EntityResolveResponse(
external_id=entity.external_id,
entity_id=entity.id,
permalink=entity.permalink,
file_path=entity.file_path,
title=entity.title,
resolution_method=resolution_method,
)
logger.debug(
f"API v2 response: resolved '{data.identifier}' to external_id={result.external_id} via {resolution_method}"
@@ -228,7 +215,7 @@ async def get_entity_by_id(
Raises:
HTTPException: 404 if entity not found
"""
with telemetry.operation(
with logfire.span(
"api.request.knowledge.get_entity",
entrypoint="api",
domain="knowledge",
@@ -236,25 +223,13 @@ async def get_entity_by_id(
):
logger.info(f"API v2 request: get_entity_by_id entity_id={entity_id}")
with telemetry.scope(
"api.knowledge.get_entity.load_entity",
domain="knowledge",
action="get_entity",
phase="load_entity",
):
entity = await entity_repository.get_by_external_id(entity_id)
entity = await entity_repository.get_by_external_id(entity_id)
if not entity:
raise HTTPException(
status_code=404, detail=f"Entity with external_id '{entity_id}' not found"
)
with telemetry.scope(
"api.knowledge.get_entity.shape_response",
domain="knowledge",
action="get_entity",
phase="shape_response",
):
result = EntityResponseV2.model_validate(entity)
result = EntityResponseV2.model_validate(entity)
logger.info(f"API v2 response: external_id={entity_id}, title='{result.title}'")
return result
@@ -267,106 +242,44 @@ async def get_entity_by_id(
async def create_entity(
project_id: ProjectExternalIdPathDep,
data: Entity,
background_tasks: BackgroundTasks,
entity_service: EntityServiceV2ExternalDep,
search_service: SearchServiceV2ExternalDep,
task_scheduler: TaskSchedulerDep,
file_service: FileServiceV2ExternalDep,
app_config: AppConfigDep,
fast: bool = Query(
True, description="If true, write quickly and defer indexing to background tasks."
),
) -> EntityResponseV2:
"""Create a new entity.
Args:
data: Entity data to create
fast: If True, defer indexing to background tasks
Returns:
Created entity with generated external_id (UUID) and file content
"""
with telemetry.operation(
with logfire.span(
"api.request.knowledge.create_entity",
entrypoint="api",
domain="knowledge",
action="create_entity",
fast=fast,
):
logger.info(
"API v2 request", endpoint="create_entity", note_type=data.note_type, title=data.title
)
with telemetry.scope(
"api.knowledge.create_entity.write_entity",
domain="knowledge",
action="create_entity",
phase="write_entity",
fast=fast,
):
if fast:
entity = await entity_service.fast_write_entity(data)
written_content = None
search_content = None
else:
write_result = await entity_service.create_entity_with_content(data)
entity = write_result.entity
written_content = write_result.content
search_content = write_result.search_content
if fast:
with telemetry.scope(
"api.knowledge.create_entity.enqueue_reindex",
domain="knowledge",
action="create_entity",
phase="enqueue_reindex",
fast=fast,
):
task_scheduler.schedule(
"reindex_entity",
entity_id=entity.id,
project_id=project_id,
)
else:
with telemetry.scope(
"api.knowledge.create_entity.search_index",
domain="knowledge",
action="create_entity",
phase="search_index",
):
await search_service.index_entity(entity, content=search_content)
with telemetry.scope(
"api.knowledge.create_entity.vector_sync",
domain="knowledge",
action="create_entity",
phase="vector_sync",
):
_schedule_vector_sync_if_enabled(
task_scheduler=task_scheduler,
app_config=app_config,
entity_id=entity.id,
project_id=project_id,
)
# Note writes are now internally consistent before the response returns. We only leave
# truly derived work, like semantic vectors, on the async scheduler.
write_result = await entity_service.create_entity_with_content(data)
entity = write_result.entity
await search_service.index_entity(entity, content=write_result.search_content)
_schedule_vector_sync_if_enabled(
task_scheduler=task_scheduler,
app_config=app_config,
entity_id=entity.id,
project_id=project_id,
)
result = EntityResponseV2.model_validate(entity)
if fast:
result = result.model_copy(update={"observations": [], "relations": []})
with telemetry.scope(
"api.knowledge.create_entity.read_content",
domain="knowledge",
action="create_entity",
phase="read_content",
source="file" if fast else "memory",
):
if fast:
content = await file_service.read_file_content(entity.file_path)
else:
# Non-fast writes already captured the markdown in memory. Reuse it here
# instead of re-reading the file; format_on_save is the one config that can
# still make the persisted file diverge because write_file only returns a checksum.
content = written_content
result = result.model_copy(update={"content": content})
# The write service already returns the canonical markdown accepted for this request.
result = result.model_copy(update={"content": write_result.content})
logger.info(
f"API v2 response: endpoint='create_entity' external_id={entity.external_id}, title={result.title}, permalink={result.permalink}, status_code=201"
@@ -381,18 +294,13 @@ async def create_entity(
async def update_entity_by_id(
data: Entity,
response: Response,
background_tasks: BackgroundTasks,
project_id: ProjectExternalIdPathDep,
entity_service: EntityServiceV2ExternalDep,
search_service: SearchServiceV2ExternalDep,
entity_repository: EntityRepositoryV2ExternalDep,
task_scheduler: TaskSchedulerDep,
file_service: FileServiceV2ExternalDep,
app_config: AppConfigDep,
entity_id: str = Path(..., description="Entity external ID (UUID)"),
fast: bool = Query(
True, description="If true, write quickly and defer indexing to background tasks."
),
) -> EntityResponseV2:
"""Update an entity by external ID.
@@ -401,121 +309,52 @@ async def update_entity_by_id(
Args:
entity_id: External ID (UUID string)
data: Updated entity data
fast: If True, defer indexing to background tasks
Returns:
Updated entity with file content
"""
with telemetry.operation(
with logfire.span(
"api.request.knowledge.update_entity",
entrypoint="api",
domain="knowledge",
action="update_entity",
fast=fast,
):
logger.info(f"API v2 request: update_entity_by_id entity_id={entity_id}")
with telemetry.scope(
"api.knowledge.update_entity.load_entity",
domain="knowledge",
action="update_entity",
phase="load_entity",
):
existing = await entity_repository.get_by_external_id(entity_id)
existing = await entity_repository.get_by_external_id(entity_id)
created = existing is None
with telemetry.scope(
"api.knowledge.update_entity.write_entity",
domain="knowledge",
action="update_entity",
phase="write_entity",
fast=fast,
):
if fast:
entity = await entity_service.fast_write_entity(data, external_id=entity_id)
written_content = None
search_content = None
response.status_code = 200 if existing else 201
else:
if existing:
write_result = await entity_service.update_entity_with_content(existing, data)
entity = write_result.entity
written_content = write_result.content
search_content = write_result.search_content
response.status_code = 200
else:
write_result = await entity_service.create_entity_with_content(data)
entity = write_result.entity
written_content = write_result.content
search_content = write_result.search_content
if entity.external_id != entity_id:
entity = await entity_repository.update(
entity.id,
{"external_id": entity_id},
)
# external_id fixup only changes the DB row. The file content is unchanged,
# so the markdown captured during the write remains valid downstream.
if not entity:
raise HTTPException(
status_code=404,
detail=f"Entity with external_id '{entity_id}' not found",
)
response.status_code = 201
if fast:
with telemetry.scope(
"api.knowledge.update_entity.enqueue_reindex",
domain="knowledge",
action="update_entity",
phase="enqueue_reindex",
fast=fast,
):
task_scheduler.schedule(
"reindex_entity",
entity_id=entity.id,
project_id=project_id,
resolve_relations=created,
)
if existing:
write_result = await entity_service.update_entity_with_content(existing, data)
entity = write_result.entity
response.status_code = 200
else:
with telemetry.scope(
"api.knowledge.update_entity.search_index",
domain="knowledge",
action="update_entity",
phase="search_index",
):
await search_service.index_entity(entity, content=search_content)
with telemetry.scope(
"api.knowledge.update_entity.vector_sync",
domain="knowledge",
action="update_entity",
phase="vector_sync",
):
_schedule_vector_sync_if_enabled(
task_scheduler=task_scheduler,
app_config=app_config,
entity_id=entity.id,
project_id=project_id,
write_result = await entity_service.create_entity_with_content(data)
entity = write_result.entity
if entity.external_id != entity_id:
entity = await entity_repository.update(
entity.id,
{"external_id": entity_id},
)
# external_id fixup only changes the DB row. The file content is unchanged,
# so the markdown captured during the write remains valid downstream.
if not entity:
raise HTTPException(
status_code=404,
detail=f"Entity with external_id '{entity_id}' not found",
)
response.status_code = 201
await search_service.index_entity(entity, content=write_result.search_content)
_schedule_vector_sync_if_enabled(
task_scheduler=task_scheduler,
app_config=app_config,
entity_id=entity.id,
project_id=project_id,
)
result = EntityResponseV2.model_validate(entity)
if fast:
result = result.model_copy(update={"observations": [], "relations": []})
with telemetry.scope(
"api.knowledge.update_entity.read_content",
domain="knowledge",
action="update_entity",
phase="read_content",
source="file" if fast else "memory",
):
if fast:
content = await file_service.read_file_content(entity.file_path)
else:
# Non-fast writes already captured the markdown in memory. Reuse it here
# instead of re-reading the file; format_on_save is the one config that can
# still make the persisted file diverge because write_file only returns a checksum.
content = written_content
result = result.model_copy(update={"content": content})
result = result.model_copy(update={"content": write_result.content})
logger.info(
f"API v2 response: external_id={entity_id}, created={created}, status_code={response.status_code}"
@@ -526,25 +365,19 @@ async def update_entity_by_id(
@router.patch("/entities/{entity_id}", response_model=EntityResponseV2)
async def edit_entity_by_id(
data: EditEntityRequest,
background_tasks: BackgroundTasks,
project_id: ProjectExternalIdPathDep,
entity_service: EntityServiceV2ExternalDep,
search_service: SearchServiceV2ExternalDep,
entity_repository: EntityRepositoryV2ExternalDep,
task_scheduler: TaskSchedulerDep,
file_service: FileServiceV2ExternalDep,
app_config: AppConfigDep,
entity_id: str = Path(..., description="Entity external ID (UUID)"),
fast: bool = Query(
True, description="If true, write quickly and defer indexing to background tasks."
),
) -> EntityResponseV2:
"""Edit an existing entity by external ID using operations like append, prepend, etc.
Args:
entity_id: External ID (UUID string)
data: Edit operation details
fast: If True, defer indexing to background tasks
Returns:
Updated entity with file content
@@ -552,115 +385,43 @@ async def edit_entity_by_id(
Raises:
HTTPException: 404 if entity not found, 400 if edit fails
"""
with telemetry.operation(
with logfire.span(
"api.request.knowledge.edit_entity",
entrypoint="api",
domain="knowledge",
action="edit_entity",
fast=fast,
):
logger.info(
f"API v2 request: edit_entity_by_id entity_id={entity_id}, operation='{data.operation}'"
)
with telemetry.scope(
"api.knowledge.edit_entity.load_entity",
domain="knowledge",
action="edit_entity",
phase="load_entity",
):
entity = await entity_repository.get_by_external_id(entity_id)
entity = await entity_repository.get_by_external_id(entity_id)
if not entity: # pragma: no cover
raise HTTPException(
status_code=404, detail=f"Entity with external_id '{entity_id}' not found"
)
try:
with telemetry.scope(
"api.knowledge.edit_entity.write_entity",
domain="knowledge",
action="edit_entity",
phase="write_entity",
fast=fast,
):
if fast:
updated_entity = await entity_service.fast_edit_entity(
entity=entity,
operation=data.operation,
content=data.content,
section=data.section,
find_text=data.find_text,
expected_replacements=data.expected_replacements,
)
written_content = None
search_content = None
else:
identifier = entity.permalink or entity.file_path
write_result = await entity_service.edit_entity_with_content(
identifier=identifier,
operation=data.operation,
content=data.content,
section=data.section,
find_text=data.find_text,
expected_replacements=data.expected_replacements,
)
updated_entity = write_result.entity
written_content = write_result.content
search_content = write_result.search_content
if fast:
with telemetry.scope(
"api.knowledge.edit_entity.enqueue_reindex",
domain="knowledge",
action="edit_entity",
phase="enqueue_reindex",
fast=fast,
):
task_scheduler.schedule(
"reindex_entity",
entity_id=updated_entity.id,
project_id=project_id,
)
else:
with telemetry.scope(
"api.knowledge.edit_entity.search_index",
domain="knowledge",
action="edit_entity",
phase="search_index",
):
await search_service.index_entity(updated_entity, content=search_content)
with telemetry.scope(
"api.knowledge.edit_entity.vector_sync",
domain="knowledge",
action="edit_entity",
phase="vector_sync",
):
_schedule_vector_sync_if_enabled(
task_scheduler=task_scheduler,
app_config=app_config,
entity_id=updated_entity.id,
project_id=project_id,
)
identifier = entity.permalink or entity.file_path
write_result = await entity_service.edit_entity_with_content(
identifier=identifier,
operation=data.operation,
content=data.content,
section=data.section,
find_text=data.find_text,
expected_replacements=data.expected_replacements,
)
updated_entity = write_result.entity
await search_service.index_entity(updated_entity, content=write_result.search_content)
_schedule_vector_sync_if_enabled(
task_scheduler=task_scheduler,
app_config=app_config,
entity_id=updated_entity.id,
project_id=project_id,
)
result = EntityResponseV2.model_validate(updated_entity)
if fast:
result = result.model_copy(update={"observations": [], "relations": []})
with telemetry.scope(
"api.knowledge.edit_entity.read_content",
domain="knowledge",
action="edit_entity",
phase="read_content",
source="file" if fast else "memory",
):
if fast:
content = await file_service.read_file_content(updated_entity.file_path)
else:
# Non-fast writes already captured the markdown in memory. Reuse it here
# instead of re-reading the file; format_on_save is the one config that can
# still make the persisted file diverge because write_file only returns a checksum.
content = written_content
result = result.model_copy(update={"content": content})
result = result.model_copy(update={"content": write_result.content})
logger.info(
f"API v2 response: external_id={entity_id}, operation='{data.operation}', status_code=200"
@@ -678,12 +439,10 @@ async def edit_entity_by_id(
@router.delete("/entities/{entity_id}", response_model=DeleteEntitiesResponse)
async def delete_entity_by_id(
background_tasks: BackgroundTasks,
project_id: ProjectExternalIdPathDep,
entity_service: EntityServiceV2ExternalDep,
entity_repository: EntityRepositoryV2ExternalDep,
entity_id: str = Path(..., description="Entity external ID (UUID)"),
search_service=Depends(lambda: None), # Optional for now
) -> DeleteEntitiesResponse:
"""Delete an entity by external ID.
@@ -695,23 +454,25 @@ async def delete_entity_by_id(
Note: Returns deleted=False if entity doesn't exist (idempotent)
"""
logger.info(f"API v2 request: delete_entity_by_id entity_id={entity_id}")
with logfire.span(
"api.request.knowledge.delete_entity",
entrypoint="api",
domain="knowledge",
action="delete_entity",
):
logger.info(f"API v2 request: delete_entity_by_id entity_id={entity_id}")
entity = await entity_repository.get_by_external_id(entity_id)
if entity is None:
logger.info(f"API v2 response: external_id={entity_id} not found, deleted=False")
return DeleteEntitiesResponse(deleted=False)
entity = await entity_repository.get_by_external_id(entity_id)
if entity is None:
logger.info(f"API v2 response: external_id={entity_id} not found, deleted=False")
return DeleteEntitiesResponse(deleted=False)
# Delete the entity using internal ID
deleted = await entity_service.delete_entity(entity.id)
# Delete the entity using internal ID
deleted = await entity_service.delete_entity(entity.id)
# Remove from search index if search service available
if search_service:
background_tasks.add_task(search_service.handle_delete, entity) # pragma: no cover
logger.info(f"API v2 response: external_id={entity_id}, deleted={deleted}")
logger.info(f"API v2 response: external_id={entity_id}, deleted={deleted}")
return DeleteEntitiesResponse(deleted=deleted)
return DeleteEntitiesResponse(deleted=deleted)
## Move endpoint
@@ -720,7 +481,6 @@ async def delete_entity_by_id(
@router.put("/entities/{entity_id}/move", response_model=EntityResponseV2)
async def move_entity(
data: MoveEntityRequestV2,
background_tasks: BackgroundTasks,
project_id: ProjectExternalIdPathDep,
entity_service: EntityServiceV2ExternalDep,
entity_repository: EntityRepositoryV2ExternalDep,
@@ -743,48 +503,58 @@ async def move_entity(
Returns:
Updated entity with new file path
"""
logger.info(
f"API v2 request: move_entity entity_id={entity_id}, destination='{data.destination_path}'"
)
try:
# First, get the entity by external_id to verify it exists
entity = await entity_repository.get_by_external_id(entity_id)
if not entity: # pragma: no cover
raise HTTPException(
status_code=404, detail=f"Entity with external_id '{entity_id}' not found"
)
# Move the entity using its current file path as identifier
moved_entity = await entity_service.move_entity(
identifier=entity.file_path, # Use file path for resolution
destination_path=data.destination_path,
project_config=project_config,
app_config=app_config,
with logfire.span(
"api.request.knowledge.move_entity",
entrypoint="api",
domain="knowledge",
action="move_entity",
):
logger.info(
f"API v2 request: move_entity entity_id={entity_id}, destination='{data.destination_path}'"
)
# Reindex at new location
reindexed_entity = await entity_service.link_resolver.resolve_link(data.destination_path)
if reindexed_entity:
await search_service.index_entity(reindexed_entity)
_schedule_vector_sync_if_enabled(
task_scheduler=task_scheduler,
try:
# First, get the entity by external_id to verify it exists
entity = await entity_repository.get_by_external_id(entity_id)
if not entity: # pragma: no cover
raise HTTPException(
status_code=404, detail=f"Entity with external_id '{entity_id}' not found"
)
# Move the entity using its current file path as identifier
moved_entity = await entity_service.move_entity(
identifier=entity.file_path, # Use file path for resolution
destination_path=data.destination_path,
project_config=project_config,
app_config=app_config,
entity_id=reindexed_entity.id,
project_id=project_id,
)
result = EntityResponseV2.model_validate(moved_entity)
# Reindex at new location
reindexed_entity = await entity_service.link_resolver.resolve_link(
data.destination_path
)
if reindexed_entity:
await search_service.index_entity(reindexed_entity)
_schedule_vector_sync_if_enabled(
task_scheduler=task_scheduler,
app_config=app_config,
entity_id=reindexed_entity.id,
project_id=project_id,
)
logger.info(f"API v2 response: moved external_id={entity_id} to '{data.destination_path}'")
result = EntityResponseV2.model_validate(moved_entity)
return result
logger.info(
f"API v2 response: moved external_id={entity_id} to '{data.destination_path}'"
)
except HTTPException: # pragma: no cover
raise # pragma: no cover
except Exception as e:
logger.error(f"Error moving entity: {e}")
raise HTTPException(status_code=400, detail=str(e))
return result
except HTTPException: # pragma: no cover
raise # pragma: no cover
except Exception as e:
logger.error(f"Error moving entity: {e}")
raise HTTPException(status_code=400, detail=str(e))
## Move directory endpoint
@@ -793,7 +563,6 @@ async def move_entity(
@router.post("/move-directory", response_model=DirectoryMoveResult)
async def move_directory(
data: MoveDirectoryRequestV2,
background_tasks: BackgroundTasks,
project_id: ProjectExternalIdPathDep,
entity_service: EntityServiceV2ExternalDep,
project_config: ProjectConfigV2ExternalDep,
@@ -814,40 +583,46 @@ async def move_directory(
Returns:
DirectoryMoveResult with counts and details of moved files
"""
logger.info(
f"API v2 request: move_directory source='{data.source_directory}', destination='{data.destination_directory}'"
)
try:
# Move the directory using the service
result = await entity_service.move_directory(
source_directory=data.source_directory,
destination_directory=data.destination_directory,
project_config=project_config,
app_config=app_config,
)
# Reindex moved entities
for file_path in result.moved_files:
entity = await entity_service.link_resolver.resolve_link(file_path)
if entity:
await search_service.index_entity(entity)
_schedule_vector_sync_if_enabled(
task_scheduler=task_scheduler,
app_config=app_config,
entity_id=entity.id,
project_id=project_id,
)
with logfire.span(
"api.request.knowledge.move_directory",
entrypoint="api",
domain="knowledge",
action="move_directory",
):
logger.info(
f"API v2 response: move_directory "
f"total={result.total_files}, success={result.successful_moves}, failed={result.failed_moves}"
f"API v2 request: move_directory source='{data.source_directory}', destination='{data.destination_directory}'"
)
return result
except Exception as e:
logger.error(f"Error moving directory: {e}")
raise HTTPException(status_code=400, detail=str(e))
try:
# Move the directory using the service
result = await entity_service.move_directory(
source_directory=data.source_directory,
destination_directory=data.destination_directory,
project_config=project_config,
app_config=app_config,
)
# Reindex moved entities
for file_path in result.moved_files:
entity = await entity_service.link_resolver.resolve_link(file_path)
if entity:
await search_service.index_entity(entity)
_schedule_vector_sync_if_enabled(
task_scheduler=task_scheduler,
app_config=app_config,
entity_id=entity.id,
project_id=project_id,
)
logger.info(
f"API v2 response: move_directory "
f"total={result.total_files}, success={result.successful_moves}, failed={result.failed_moves}"
)
return result
except Exception as e:
logger.error(f"Error moving directory: {e}")
raise HTTPException(status_code=400, detail=str(e))
## Delete directory endpoint
@@ -872,20 +647,26 @@ async def delete_directory(
Returns:
DirectoryDeleteResult with counts and details of deleted files
"""
logger.info(f"API v2 request: delete_directory directory='{data.directory}'")
with logfire.span(
"api.request.knowledge.delete_directory",
entrypoint="api",
domain="knowledge",
action="delete_directory",
):
logger.info(f"API v2 request: delete_directory directory='{data.directory}'")
try:
# Delete the directory using the service
result = await entity_service.delete_directory(
directory=data.directory,
)
try:
# Delete the directory using the service
result = await entity_service.delete_directory(
directory=data.directory,
)
logger.info(
f"API v2 response: delete_directory "
f"total={result.total_files}, success={result.successful_deletes}, failed={result.failed_deletes}"
)
return result
logger.info(
f"API v2 response: delete_directory "
f"total={result.total_files}, success={result.successful_deletes}, failed={result.failed_deletes}"
)
return result
except Exception as e:
logger.error(f"Error deleting directory: {e}")
raise HTTPException(status_code=400, detail=str(e))
except Exception as e:
logger.error(f"Error deleting directory: {e}")
raise HTTPException(status_code=400, detail=str(e))
@@ -9,7 +9,7 @@ from typing import Annotated, Optional
from fastapi import APIRouter, Query, Path
from loguru import logger
from basic_memory import telemetry
import logfire
from basic_memory.deps import ContextServiceV2ExternalDep, EntityRepositoryV2ExternalDep
from basic_memory.schemas.base import TimeFrame, parse_timeframe
from basic_memory.schemas.memory import (
@@ -51,7 +51,7 @@ async def recent(
Returns:
GraphContext with recent activity and related entities
"""
with telemetry.operation(
with logfire.span(
"api.request.memory.recent_activity",
entrypoint="api",
domain="memory",
@@ -72,7 +72,7 @@ async def recent(
limit = page_size
offset = (page - 1) * page_size
with telemetry.scope(
with logfire.span(
"api.memory.recent_activity.build_context",
domain="memory",
action="recent_activity",
@@ -88,7 +88,7 @@ async def recent(
offset=offset,
max_related=max_related,
)
with telemetry.scope(
with logfire.span(
"api.memory.recent_activity.shape_response",
domain="memory",
action="recent_activity",
@@ -137,7 +137,7 @@ async def get_memory_context(
Returns:
GraphContext with the entity and its related context
"""
with telemetry.operation(
with logfire.span(
"api.request.memory.build_context",
entrypoint="api",
domain="memory",
@@ -154,7 +154,7 @@ async def get_memory_context(
limit = page_size
offset = (page - 1) * page_size
with telemetry.scope(
with logfire.span(
"api.memory.build_context.build_context",
domain="memory",
action="build_context",
@@ -170,7 +170,7 @@ async def get_memory_context(
offset=offset,
max_related=max_related,
)
with telemetry.scope(
with logfire.span(
"api.memory.build_context.shape_response",
domain="memory",
action="build_context",
@@ -6,6 +6,7 @@ have entity IDs in URLs - they generate formatted prompts from queries.
"""
from datetime import datetime, timezone
from typing import Any
from fastapi import APIRouter, HTTPException, status, Path
from loguru import logger
@@ -59,6 +60,7 @@ async def continue_conversation(
# Initialize search results
search_results = []
hierarchical_results_for_count = []
# Get data needed for template
if request.topic:
@@ -91,7 +93,8 @@ async def continue_conversation(
# Limit to a reasonable number of total results
all_hierarchical_results = all_hierarchical_results[:10]
template_context = {
hierarchical_results_for_count = all_hierarchical_results
template_context: dict[str, Any] = {
"topic": request.topic,
"timeframe": request.timeframe,
"hierarchical_results": all_hierarchical_results,
@@ -110,6 +113,7 @@ async def continue_conversation(
hierarchical_results = recent_context.results[:5] # Limit to top 5 recent items
hierarchical_results_for_count = hierarchical_results
template_context = {
"topic": f"Recent Activity from ({request.timeframe})",
"timeframe": request.timeframe,
@@ -129,9 +133,6 @@ async def continue_conversation(
relation_count = 0
entity_count = 0
# Get the hierarchical results from the template context
hierarchical_results_for_count = template_context.get("hierarchical_results", [])
# For topic-based search
if request.topic:
for item in hierarchical_results_for_count:
@@ -159,29 +160,24 @@ async def continue_conversation(
elif related.type == "entity": # pragma: no cover
entity_count += 1 # pragma: no cover
# Build metadata
metadata = {
"query": request.topic,
"timeframe": request.timeframe,
"search_count": len(search_results)
if request.topic
else 0, # Original search results count
"context_count": len(hierarchical_results_for_count),
"observation_count": observation_count,
"relation_count": relation_count,
"total_items": (
prompt_metadata = PromptMetadata(
query=request.topic,
timeframe=request.timeframe,
search_count=len(search_results) if request.topic else 0,
context_count=len(hierarchical_results_for_count),
observation_count=observation_count,
relation_count=relation_count,
total_items=(
len(hierarchical_results_for_count)
+ observation_count
+ relation_count
+ entity_count
),
"search_limit": request.search_items_limit,
"context_depth": request.depth,
"related_limit": request.related_items_limit,
"generated_at": datetime.now(timezone.utc).isoformat(),
}
prompt_metadata = PromptMetadata(**metadata)
search_limit=request.search_items_limit,
context_depth=request.depth,
related_limit=request.related_items_limit,
generated_at=datetime.now(timezone.utc).isoformat(),
)
return PromptResponse(
prompt=rendered_prompt, context=template_context, metadata=prompt_metadata
@@ -229,7 +225,7 @@ async def search_prompt(
results = await search_service.search(query, limit=limit, offset=offset)
search_results = await to_search_results(entity_service, results)
template_context = {
template_context: dict[str, Any] = {
"query": request.query,
"timeframe": request.timeframe,
"results": search_results,
@@ -241,22 +237,19 @@ async def search_prompt(
# Render template
rendered_prompt = await template_loader.render("prompts/search.hbs", template_context)
# Build metadata
metadata = {
"query": request.query,
"timeframe": request.timeframe,
"search_count": len(search_results),
"context_count": len(search_results),
"observation_count": 0, # Search results don't include observations
"relation_count": 0, # Search results don't include relations
"total_items": len(search_results),
"search_limit": limit,
"context_depth": 0, # No context depth for basic search
"related_limit": 0, # No related items for basic search
"generated_at": datetime.now(timezone.utc).isoformat(),
}
prompt_metadata = PromptMetadata(**metadata)
prompt_metadata = PromptMetadata(
query=request.query,
timeframe=request.timeframe,
search_count=len(search_results),
context_count=len(search_results),
observation_count=0,
relation_count=0,
total_items=len(search_results),
search_limit=limit,
context_depth=0,
related_limit=0,
generated_at=datetime.now(timezone.utc).isoformat(),
)
return PromptResponse(
prompt=rendered_prompt, context=template_context, metadata=prompt_metadata
@@ -15,7 +15,7 @@ from pathlib import Path as PathLib
from fastapi import APIRouter, HTTPException, Response, Path
from loguru import logger
from basic_memory import telemetry
import logfire
from basic_memory.deps import (
ProjectConfigV2ExternalDep,
FileServiceV2ExternalDep,
@@ -56,7 +56,7 @@ async def get_resource_content(
Raises:
HTTPException: 404 if entity or file not found
"""
with telemetry.operation(
with logfire.span(
"api.request.resource.get_content",
entrypoint="api",
domain="resource",
@@ -64,7 +64,7 @@ async def get_resource_content(
):
logger.debug(f"V2 Getting content for project {project_id}, entity_id: {entity_id}")
with telemetry.scope(
with logfire.span(
"api.resource.get_content.load_entity",
domain="resource",
action="get_content",
@@ -74,7 +74,7 @@ async def get_resource_content(
if not entity:
raise HTTPException(status_code=404, detail=f"Entity {entity_id} not found")
with telemetry.scope(
with logfire.span(
"api.resource.get_content.validate_path",
domain="resource",
action="get_content",
@@ -90,7 +90,7 @@ async def get_resource_content(
detail="Entity contains invalid file path",
)
with telemetry.scope(
with logfire.span(
"api.resource.get_content.ensure_exists",
domain="resource",
action="get_content",
@@ -102,7 +102,7 @@ async def get_resource_content(
detail=f"File not found: {entity.file_path}",
)
with telemetry.scope(
with logfire.span(
"api.resource.get_content.read_content",
domain="resource",
action="get_content",
@@ -139,7 +139,7 @@ async def create_resource(
Raises:
HTTPException: 400 for invalid file paths, 409 if file already exists
"""
with telemetry.operation(
with logfire.span(
"api.request.resource.create",
entrypoint="api",
domain="resource",
@@ -166,7 +166,7 @@ async def create_resource(
f"Use PUT /resource/{existing_entity.external_id} to update it.",
)
with telemetry.scope(
with logfire.span(
"api.resource.create.write_file",
domain="resource",
action="create",
@@ -175,7 +175,7 @@ async def create_resource(
await file_service.ensure_directory(PathLib(data.file_path).parent)
checksum = await file_service.write_file(data.file_path, data.content)
with telemetry.scope(
with logfire.span(
"api.resource.create.read_metadata",
domain="resource",
action="create",
@@ -197,7 +197,7 @@ async def create_resource(
created_at=file_metadata.created_at,
updated_at=file_metadata.modified_at,
)
with telemetry.scope(
with logfire.span(
"api.resource.create.upsert_entity",
domain="resource",
action="create",
@@ -205,13 +205,13 @@ async def create_resource(
):
entity = await entity_repository.add(entity)
with telemetry.scope(
with logfire.span(
"api.resource.create.search_index",
domain="resource",
action="create",
phase="search_index",
):
await search_service.index_entity(entity) # pyright: ignore
await search_service.index_entity(entity)
return ResourceResponse(
entity_id=entity.id,
@@ -258,7 +258,7 @@ async def update_resource(
Raises:
HTTPException: 404 if entity not found, 400 for invalid paths
"""
with telemetry.operation(
with logfire.span(
"api.request.resource.update",
entrypoint="api",
domain="resource",
@@ -282,7 +282,7 @@ async def update_resource(
"Path must be relative and stay within project boundaries.",
)
with telemetry.scope(
with logfire.span(
"api.resource.update.write_file",
domain="resource",
action="update",
@@ -297,7 +297,7 @@ async def update_resource(
checksum = await file_service.write_file(target_file_path, data.content)
with telemetry.scope(
with logfire.span(
"api.resource.update.read_metadata",
domain="resource",
action="update",
@@ -309,7 +309,7 @@ async def update_resource(
content_type = file_service.content_type(target_file_path)
note_type = "canvas" if target_file_path.endswith(".canvas") else "file"
with telemetry.scope(
with logfire.span(
"api.resource.update.update_entity",
domain="resource",
action="update",
@@ -326,14 +326,16 @@ async def update_resource(
"updated_at": file_metadata.modified_at,
},
)
if updated_entity is None:
raise HTTPException(status_code=404, detail=f"Entity {entity_id} not found")
with telemetry.scope(
with logfire.span(
"api.resource.update.search_index",
domain="resource",
action="update",
phase="search_index",
):
await search_service.index_entity(updated_entity) # pyright: ignore
await search_service.index_entity(updated_entity)
return ResourceResponse(
entity_id=entity.id,
@@ -4,15 +4,17 @@ This router uses external_id UUIDs for stable, API-friendly routing.
V1 uses string-based project names which are less efficient and less stable.
"""
import asyncio
from fastapi import APIRouter, HTTPException, Path
from basic_memory import telemetry
import logfire
from basic_memory.api.v2.utils import to_search_results
from basic_memory.repository.semantic_errors import (
SemanticDependenciesMissingError,
SemanticSearchDisabledError,
)
from basic_memory.schemas.search import SearchQuery, SearchResponse
from basic_memory.schemas.search import SearchQuery, SearchResponse, SearchRetrievalMode
from basic_memory.deps import (
SearchServiceV2ExternalDep,
EntityServiceV2ExternalDep,
@@ -48,7 +50,7 @@ async def search(
Returns:
SearchResponse with paginated search results
"""
with telemetry.operation(
with logfire.span(
"api.request.search",
entrypoint="api",
domain="search",
@@ -65,9 +67,9 @@ async def search(
has_filters=bool(query.note_types or query.entity_types or query.metadata_filters),
):
offset = (page - 1) * page_size
fetch_limit = page_size + 1
exact_count_available = query.retrieval_mode == SearchRetrievalMode.FTS
try:
with telemetry.scope(
with logfire.span(
"api.search.search.execute_query",
domain="search",
action="search",
@@ -75,7 +77,14 @@ async def search(
page=page,
page_size=page_size,
):
results = await search_service.search(query, limit=fetch_limit, offset=offset)
if exact_count_available:
results, total = await asyncio.gather(
search_service.search(query, limit=page_size, offset=offset),
search_service.count(query),
)
else:
results = await search_service.search(query, limit=page_size + 1, offset=offset)
total = 0
except SemanticSearchDisabledError as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc
except SemanticDependenciesMissingError as exc:
@@ -83,18 +92,24 @@ async def search(
except ValueError as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc
with telemetry.scope(
with logfire.span(
"api.search.search.paginate_results",
domain="search",
action="search",
phase="paginate_results",
result_count=len(results),
):
has_more = len(results) > page_size
if has_more:
results = results[:page_size]
if exact_count_available:
has_more = offset + len(results) < total
else:
# Trigger: semantic modes would need another vector/hybrid retrieval to count.
# Why: search requests should not pay for a second semantic pass.
# Outcome: preserve probe pagination for semantic search and leave total at 0.
has_more = len(results) > page_size
if has_more:
results = results[:page_size]
with telemetry.scope(
with logfire.span(
"api.search.search.hydrate_results",
domain="search",
action="search",
@@ -102,7 +117,7 @@ async def search(
result_count=len(results),
):
search_results = await to_search_results(entity_service, results)
with telemetry.scope(
with logfire.span(
"api.search.search.build_response",
domain="search",
action="search",
@@ -113,6 +128,7 @@ async def search(
results=search_results,
current_page=page,
page_size=page_size,
total=total,
has_more=has_more,
)
+79 -52
View File
@@ -1,8 +1,6 @@
from typing import Optional, List
from typing import Any, Protocol, Optional, List, Sequence
from basic_memory import telemetry
from basic_memory.models import Entity as EntityModel
from basic_memory.repository import EntityRepository
import logfire
from basic_memory.repository.search_repository import SearchIndexRow
from basic_memory.schemas.memory import (
EntitySummary,
@@ -13,20 +11,39 @@ from basic_memory.schemas.memory import (
ContextResult,
)
from basic_memory.schemas.search import SearchItemType, SearchResult
from basic_memory.services import EntityService
from basic_memory.services.context_service import (
ContextResultRow,
ContextResult as ServiceContextResult,
)
class EntityBatchLookup(Protocol):
async def find_by_ids(self, ids: List[int]) -> Sequence[Any]: ...
class EntityServiceBatchLookup(Protocol):
async def get_entities_by_id(self, ids: List[int]) -> Sequence[Any]: ...
def _required_str(value: str | None, field_name: str) -> str:
"""Return a required search field or fail before producing invalid response data."""
if value is None:
raise ValueError(f"Search result is missing required field: {field_name}")
return value
def _search_item_type(value: str | SearchItemType) -> SearchItemType:
"""Normalize repository row type strings into the public search enum."""
return value if isinstance(value, SearchItemType) else SearchItemType(value)
async def to_graph_context(
context_result: ServiceContextResult,
entity_repository: EntityRepository,
entity_repository: EntityBatchLookup,
page: Optional[int] = None,
page_size: Optional[int] = None,
):
with telemetry.scope(
) -> GraphContext:
with logfire.span(
"memory.hydrate_context",
domain="memory",
action="build_context",
@@ -44,17 +61,18 @@ async def to_graph_context(
+ context_item.observations
+ context_item.related_results
):
if item.type == SearchItemType.ENTITY:
item_type = _search_item_type(item.type)
if item_type == SearchItemType.ENTITY:
# Entity's own ID for its external_id
entity_ids_needed.add(item.id)
elif item.type == SearchItemType.OBSERVATION:
elif item_type == SearchItemType.OBSERVATION:
# Parent entity ID for entity_external_id
if item.entity_id: # pyright: ignore
entity_ids_needed.add(item.entity_id) # pyright: ignore
elif item.type == SearchItemType.RELATION:
if item.entity_id:
entity_ids_needed.add(item.entity_id)
elif item_type == SearchItemType.RELATION:
# Source and target entity IDs for external_ids
if item.from_id: # pyright: ignore
entity_ids_needed.add(item.from_id) # pyright: ignore
if item.from_id:
entity_ids_needed.add(item.from_id)
if item.to_id:
entity_ids_needed.add(item.to_id)
@@ -62,7 +80,7 @@ async def to_graph_context(
entity_title_lookup: dict[int, str] = {}
entity_external_id_lookup: dict[int, str] = {}
if entity_ids_needed:
with telemetry.scope(
with logfire.span(
"memory.hydrate_context.lookup_entities",
domain="memory",
action="build_context",
@@ -75,59 +93,62 @@ async def to_graph_context(
entity_external_id_lookup[e.id] = e.external_id
# Helper function to convert items to summaries
def to_summary(item: SearchIndexRow | ContextResultRow):
match item.type:
def to_summary(
item: SearchIndexRow | ContextResultRow,
) -> EntitySummary | ObservationSummary | RelationSummary:
item_type = _search_item_type(item.type)
match item_type:
case SearchItemType.ENTITY:
return EntitySummary(
external_id=entity_external_id_lookup.get(item.id, ""),
entity_id=item.id,
title=item.title, # pyright: ignore
title=_required_str(item.title, "title"),
permalink=item.permalink,
content=item.content,
file_path=item.file_path,
file_path=_required_str(item.file_path, "file_path"),
created_at=item.created_at,
)
case SearchItemType.OBSERVATION:
entity_ext_id = None
if item.entity_id: # pyright: ignore
entity_ext_id = entity_external_id_lookup.get(item.entity_id) # pyright: ignore
entity_title = None
if item.entity_id:
entity_ext_id = entity_external_id_lookup.get(item.entity_id)
entity_title = entity_title_lookup.get(item.entity_id)
return ObservationSummary(
observation_id=item.id,
entity_id=item.entity_id, # pyright: ignore
entity_id=item.entity_id,
entity_external_id=entity_ext_id,
title=entity_title_lookup.get(item.entity_id), # pyright: ignore
file_path=item.file_path,
category=item.category, # pyright: ignore
content=item.content, # pyright: ignore
permalink=item.permalink, # pyright: ignore
title=entity_title,
file_path=_required_str(item.file_path, "file_path"),
category=_required_str(item.category, "category"),
content=_required_str(item.content, "content"),
permalink=_required_str(item.permalink, "permalink"),
created_at=item.created_at,
)
case SearchItemType.RELATION:
from_title = entity_title_lookup.get(item.from_id) if item.from_id else None # pyright: ignore
from_title = entity_title_lookup.get(item.from_id) if item.from_id else None
to_title = entity_title_lookup.get(item.to_id) if item.to_id else None
from_ext_id = (
entity_external_id_lookup.get(item.from_id) if item.from_id else None
) # pyright: ignore
)
to_ext_id = entity_external_id_lookup.get(item.to_id) if item.to_id else None
return RelationSummary(
relation_id=item.id,
entity_id=item.entity_id, # pyright: ignore
title=item.title, # pyright: ignore
file_path=item.file_path,
permalink=item.permalink, # pyright: ignore
relation_type=item.relation_type, # pyright: ignore
entity_id=item.entity_id,
title=_required_str(item.title, "title"),
file_path=_required_str(item.file_path, "file_path"),
permalink=_required_str(item.permalink, "permalink"),
relation_type=_required_str(item.relation_type, "relation_type"),
from_entity=from_title,
from_entity_id=item.from_id, # pyright: ignore
from_entity_id=item.from_id,
from_entity_external_id=from_ext_id,
to_entity=to_title,
to_entity_id=item.to_id,
to_entity_external_id=to_ext_id,
created_at=item.created_at,
)
case _: # pragma: no cover
raise ValueError(f"Unexpected type: {item.type}")
with telemetry.scope(
with logfire.span(
"memory.hydrate_context.shape_results",
domain="memory",
action="build_context",
@@ -137,12 +158,16 @@ async def to_graph_context(
hierarchical_results = []
for context_item in context_result.results:
primary_result = to_summary(context_item.primary_result)
observations = [to_summary(obs) for obs in context_item.observations]
observations = [
summary
for summary in (to_summary(obs) for obs in context_item.observations)
if isinstance(summary, ObservationSummary)
]
related = [to_summary(rel) for rel in context_item.related_results]
hierarchical_results.append(
ContextResult(
primary_result=primary_result,
observations=observations, # pyright: ignore[reportArgumentType]
observations=observations,
related_results=related,
)
)
@@ -170,8 +195,10 @@ async def to_graph_context(
)
async def to_search_results(entity_service: EntityService, results: List[SearchIndexRow]):
with telemetry.scope(
async def to_search_results(
entity_service: EntityServiceBatchLookup, results: List[SearchIndexRow]
) -> list[SearchResult]:
with logfire.span(
"search.hydrate_results",
domain="search",
action="search",
@@ -187,8 +214,8 @@ async def to_search_results(entity_service: EntityService, results: List[SearchI
all_entity_ids.add(eid)
# Single batch fetch for all entities
entities_by_id: dict[int, EntityModel] = {}
with telemetry.scope(
entities_by_id: dict[int, Any] = {}
with logfire.span(
"search.hydrate_results.fetch_entities",
domain="search",
action="search",
@@ -200,7 +227,7 @@ async def to_search_results(entity_service: EntityService, results: List[SearchI
entities_by_id = {e.id: e for e in entities}
search_results = []
with telemetry.scope(
with logfire.span(
"search.hydrate_results.shape_results",
domain="search",
action="search",
@@ -222,20 +249,20 @@ async def to_search_results(entity_service: EntityService, results: List[SearchI
entity_id = result.entity_id
# Look up entities by their specific IDs
parent_entity = entities_by_id.get(result.entity_id) if result.entity_id else None # pyright: ignore
from_entity = entities_by_id.get(result.from_id) if result.from_id else None # pyright: ignore
parent_entity = entities_by_id.get(result.entity_id) if result.entity_id else None
from_entity = entities_by_id.get(result.from_id) if result.from_id else None
to_entity = entities_by_id.get(result.to_id) if result.to_id else None
search_results.append(
SearchResult(
title=result.title, # pyright: ignore
type=result.type, # pyright: ignore
title=_required_str(result.title, "title"),
type=_search_item_type(result.type),
permalink=result.permalink,
score=result.score, # pyright: ignore
score=result.score if result.score is not None else 0.0,
entity=parent_entity.permalink if parent_entity else None,
content=result.content,
matched_chunk=result.matched_chunk_text,
file_path=result.file_path,
file_path=_required_str(result.file_path, "file_path"),
metadata=result.metadata,
entity_id=entity_id,
observation_id=observation_id,
+2 -2
View File
@@ -12,7 +12,7 @@ from basic_memory.cli.auto_update import maybe_run_periodic_auto_update # noqa:
from basic_memory.cli.container import CliContainer, set_container # noqa: E402
from basic_memory.cli.promo import maybe_show_cloud_promo, maybe_show_init_line # noqa: E402
from basic_memory.config import init_cli_logging # noqa: E402
from basic_memory import telemetry # noqa: E402
import logfire # noqa: E402
def version_callback(value: bool) -> None:
@@ -45,7 +45,7 @@ def app_callback(
init_cli_logging()
command_name = ctx.invoked_subcommand or "root"
ctx.with_resource(
telemetry.operation(
logfire.span(
f"cli.command.{command_name}",
entrypoint="cli",
command_name=command_name,
@@ -99,41 +99,39 @@ async def make_api_request(
response = await client.request(method=method, url=url, headers=headers, json=json_data)
response.raise_for_status()
return response
except httpx.HTTPStatusError as e:
response = e.response
# Try to parse error detail from response
error_detail = None
try:
error_detail = response.json()
except Exception:
# If JSON parsing fails, we'll handle it as a generic error
pass
# Check for subscription_required error (403)
if response.status_code == 403 and isinstance(error_detail, dict):
# Handle both FastAPI HTTPException format (nested under "detail")
# and direct format
detail_obj = error_detail.get("detail", error_detail)
if (
isinstance(detail_obj, dict)
and detail_obj.get("error") == "subscription_required"
):
message = detail_obj.get("message", "Active subscription required")
subscribe_url = detail_obj.get(
"subscribe_url", "https://basicmemory.com/subscribe"
)
raise SubscriptionRequiredError(
message=message, subscribe_url=subscribe_url
) from e
# Raise generic CloudAPIError with status code and detail
raise CloudAPIError(
f"API request failed: {e}",
status_code=response.status_code,
detail=error_detail if isinstance(error_detail, dict) else {},
) from e
except httpx.HTTPError as e:
# Check if this is a response error with response details
if hasattr(e, "response") and e.response is not None: # pyright: ignore [reportAttributeAccessIssue]
response = e.response # type: ignore
# Try to parse error detail from response
error_detail = None
try:
error_detail = response.json()
except Exception:
# If JSON parsing fails, we'll handle it as a generic error
pass
# Check for subscription_required error (403)
if response.status_code == 403 and isinstance(error_detail, dict):
# Handle both FastAPI HTTPException format (nested under "detail")
# and direct format
detail_obj = error_detail.get("detail", error_detail)
if (
isinstance(detail_obj, dict)
and detail_obj.get("error") == "subscription_required"
):
message = detail_obj.get("message", "Active subscription required")
subscribe_url = detail_obj.get(
"subscribe_url", "https://basicmemory.com/subscribe"
)
raise SubscriptionRequiredError(
message=message, subscribe_url=subscribe_url
) from e
# Raise generic CloudAPIError with status code and detail
raise CloudAPIError(
f"API request failed: {e}",
status_code=response.status_code,
detail=error_detail if isinstance(error_detail, dict) else {},
) from e
raise CloudAPIError(f"API request failed: {e}") from e
@@ -76,10 +76,22 @@ def login():
@cloud_app.command()
def logout():
"""Remove stored OAuth tokens."""
config = ConfigManager().config
"""Remove stored OAuth tokens and clear cached workspace selection."""
config_manager = ConfigManager()
config = config_manager.config
auth = CLIAuth(client_id=config.cloud_client_id, authkit_domain=config.cloud_domain)
auth.logout()
# Trigger: ending a session must invalidate the cached workspace.
# Why: a follow-up `bm cloud login` (often as a different user, or returning
# from an org workspace to personal) inherits the previous selection
# and silently routes everything through the wrong tenant. See #755.
# Outcome: re-login starts from a clean slate; the user picks again via
# `bm cloud workspace set-default` or per-project --workspace.
if config.default_workspace is not None:
config.default_workspace = None
config_manager.save_config(config)
console.print("[dim]API key (if configured) remains available for cloud project routing.[/dim]")
@@ -167,7 +179,7 @@ def setup() -> None:
console.print("1. Add a project with local sync path:")
console.print(" bm project add research --cloud --local-path ~/Documents/research")
console.print("\n Or configure sync for an existing project:")
console.print(" bm project sync-setup research ~/Documents/research")
console.print(" bm cloud sync-setup research ~/Documents/research")
console.print("\n2. Preview the initial sync (recommended):")
console.print(" bm project bisync --name research --resync --dry-run")
console.print("\n3. If all looks good, run the actual sync:")
@@ -20,6 +20,7 @@ from loguru import logger
from rich.console import Console
from basic_memory.cli.commands.cloud.rclone_installer import is_rclone_installed
from basic_memory.config import resolve_data_dir
from basic_memory.utils import normalize_project_path
console = Console()
@@ -138,13 +139,16 @@ def get_bmignore_filter_path() -> Path:
def get_project_bisync_state(project_name: str) -> Path:
"""Get path to project's bisync state directory.
Honors ``BASIC_MEMORY_CONFIG_DIR`` so isolated instances each keep their
own bisync state alongside their config.
Args:
project_name: Name of the project
Returns:
Path to bisync state directory for this project
"""
return Path.home() / ".basic-memory" / "bisync-state" / project_name
return resolve_data_dir() / "bisync-state" / project_name
def bisync_initialized(project_name: str) -> bool:
+118 -1
View File
@@ -1,8 +1,10 @@
"""Database management commands."""
import os
from dataclasses import dataclass
from pathlib import Path
from pathlib import Path, PurePosixPath, PureWindowsPath
import psutil
import typer
from loguru import logger
from rich.console import Console
@@ -21,6 +23,103 @@ from basic_memory.sync.sync_service import get_sync_service
console = Console()
def _is_basic_memory_mcp(cmdline: list[str]) -> bool:
"""Heuristic: does this argv represent a `basic-memory mcp` server?
The MCP server can be launched any of:
basic-memory mcp
bm mcp # entrypoint alias from pyproject.toml
python -m basic_memory.cli.main mcp # module form
uv run basic-memory mcp / uv run bm mcp # uv wrappers
/abs/path/to/{bm,basic-memory}[.exe] mcp
A reliable match needs both signals:
1. "mcp" appears as an exact argv token (not "mcp-foo").
2. Some argv token names the basic-memory entrypoint either by
hyphen/underscore form, or as a `bm` script (covers `/usr/local/bin/bm`,
`bm.exe`, etc. via Path.stem).
"""
if "mcp" not in cmdline:
return False
for arg in cmdline:
if "basic-memory" in arg or "basic_memory" in arg:
return True
# Try both POSIX and Windows path interpretations so a test on
# macOS still recognizes `C:\\...\\bm.exe`, and a real Windows
# run still recognizes `/usr/local/bin/bm`. Path() alone uses
# the host OS, which gives wrong stems for foreign separators.
if PurePosixPath(arg).stem == "bm" or PureWindowsPath(arg).stem == "bm":
return True
return False
def _find_live_mcp_processes() -> list[tuple[int, str]]:
"""Return (pid, joined_cmdline) for live `basic-memory mcp` processes.
Why this exists (issue #765):
On POSIX, `Path.unlink()` removes the directory entry but the inode
survives as long as any process holds the file open. A `bm reset`
run while Claude Desktop (or another MCP client) is alive will
therefore "succeed" but the still-running MCP keeps reading the
old, now-invisible memory.db inode and returns phantom rows. On
Windows the OS naturally raises PermissionError on `unlink()`, so
the bug is POSIX-specific. We detect proactively to give the same
error experience on every platform before doing damage.
The current process is excluded so this can be called from inside a
`bm reset` invocation. NoSuchProcess / AccessDenied are swallowed
because process tables race with the scan and we don't want a
transient permission error to mask a real zombie.
"""
me = os.getpid()
matches: list[tuple[int, str]] = []
for proc in psutil.process_iter(["pid", "cmdline"]):
try:
pid = proc.info.get("pid")
if pid is None or pid == me:
continue
cmdline = proc.info.get("cmdline") or []
if not cmdline:
continue
if _is_basic_memory_mcp(cmdline):
matches.append((pid, " ".join(cmdline)))
except (psutil.NoSuchProcess, psutil.AccessDenied):
continue
return matches
def _abort_if_mcp_processes_alive() -> None:
"""Refuse `bm reset` while basic-memory MCP processes are still running.
See _find_live_mcp_processes for the underlying POSIX-vs-Windows
rationale. Prints a per-PID list and platform-appropriate cleanup
instructions, then exits non-zero so destructive work never starts.
"""
zombies = _find_live_mcp_processes()
if not zombies:
return
console.print("[red]Refusing to reset:[/red] basic-memory MCP processes are still running.")
console.print(
"[yellow]On macOS/Linux these would keep reading the deleted memory.db inode "
"and return phantom search results (see #765).[/yellow]"
)
for pid, cmd in zombies:
console.print(f" PID {pid}: {cmd}")
console.print("\n[bold]How to clean up:[/bold]")
console.print(" 1. Quit Claude Desktop and any other MCP clients.")
if os.name == "nt":
console.print(
" 2. Verify nothing remains: "
"[green]Get-CimInstance Win32_Process | "
"Where-Object {$_.CommandLine -like '*basic-memory*mcp*'}[/green]"
)
else:
console.print(" 2. Verify nothing remains: [green]pgrep -fa 'basic-memory mcp'[/green]")
console.print(" 3. Re-run [green]bm reset[/green].")
raise typer.Exit(1)
@dataclass(slots=True)
class EmbeddingProgress:
"""Typed CLI progress payload for embedding backfills."""
@@ -86,6 +185,16 @@ async def _reindex_projects(app_config):
@app.command()
def reset(
reindex: bool = typer.Option(False, "--reindex", help="Rebuild db index from filesystem"),
force: bool = typer.Option(
False,
"--force",
help=(
"Skip the pre-flight check that refuses to reset while "
"basic-memory MCP processes are running. Use only in "
"automated workflows where you've already ensured no MCP "
"clients are attached to the database."
),
),
): # pragma: no cover
"""Reset database (drop all tables and recreate)."""
console.print(
@@ -94,6 +203,14 @@ def reset(
"Use [green]bm reset --reindex[/green] to automatically rebuild the index afterward."
)
if typer.confirm("Reset the database index?"):
# Pre-flight: refuse to proceed if MCP processes still hold the DB
# file open. POSIX would silently let us unlink the inode while
# they keep reading it; Windows would error here anyway. See
# _find_live_mcp_processes for the full story. --force is the
# documented escape hatch for scripted/CI runs.
if not force:
_abort_if_mcp_processes_alive()
logger.info("Resetting database...")
config_manager = ConfigManager()
app_config = config_manager.config
+1 -1
View File
@@ -69,7 +69,7 @@ async def run_doctor() -> None:
content=f"# {api_note_title}\n\n- [note] API to file check",
entity_metadata={"tags": ["doctor"]},
)
api_result = await knowledge_client.create_entity(api_note.model_dump(), fast=False)
api_result = await knowledge_client.create_entity(api_note.model_dump())
api_file = project_path / api_result.file_path
if not api_file.exists():
+126 -4
View File
@@ -26,7 +26,7 @@ from basic_memory.cli.commands.cloud.rclone_commands import (
)
from basic_memory.cli.commands.command_utils import get_project_info, run_with_cleanup
from basic_memory.cli.commands.routing import force_routing, validate_routing_flags
from basic_memory.config import ConfigManager, ProjectEntry, ProjectMode
from basic_memory.config import BasicMemoryConfig, ConfigManager, ProjectEntry, ProjectMode
from basic_memory.mcp.async_client import get_client, resolve_configured_workspace
from basic_memory.mcp.clients import ProjectClient
from basic_memory.schemas.cloud import (
@@ -838,6 +838,76 @@ def move_project(
raise typer.Exit(1)
async def _detach_local_project_row(app_config: BasicMemoryConfig, name: str) -> bool:
"""Drop the project's row from the local index DB.
Trigger: `bm project set-cloud` is making a project cloud-only.
Why: the local row is what causes `_merge_projects` to report
`source: "local+cloud"` after the toggle (#680). Removing it
forces the merged listing to honor the user's chosen mode.
Outcome: returns True if a row was deleted, False if there was
nothing to clean up. On-disk note files are not touched.
"""
from basic_memory import db
from basic_memory.repository import ProjectRepository
_, session_maker = await db.get_or_create_db(
db_path=app_config.database_path,
db_type=db.DatabaseType.FILESYSTEM,
)
try:
repo = ProjectRepository(session_maker)
existing = await repo.get_by_name(name)
if existing is None:
return False
await repo.delete(existing.id)
return True
finally:
# CLI-only: safe to tear down the global DB singleton here since
# set-cloud/set-local never run inside a long-lived MCP/API server.
await db.shutdown_db()
async def _attach_local_project_row(app_config: BasicMemoryConfig, name: str, path: str) -> None:
"""Ensure the project has a row in the local index DB at the given path.
Trigger: `bm project set-local` is making a previously cloud-only
project local again.
Why: without a row in the local DB, every local-side tool (`list`,
`info`, sync, indexing) would skip this project.
Outcome: a row is created if missing, or its path is updated to match
the new local home if it already exists. On-disk files are not
touched the caller is responsible for ensuring the directory
exists.
"""
from basic_memory import db
from basic_memory.repository import ProjectRepository
_, session_maker = await db.get_or_create_db(
db_path=app_config.database_path,
db_type=db.DatabaseType.FILESYSTEM,
)
try:
repo = ProjectRepository(session_maker)
existing = await repo.get_by_name(name)
if existing is None:
await repo.create(
{
"name": name,
"path": path,
"permalink": generate_permalink(name),
"is_active": True,
}
)
return
if existing.path != path:
await repo.update_path(existing.id, path)
finally:
# CLI-only: safe to tear down the global DB singleton here since
# set-cloud/set-local never run inside a long-lived MCP/API server.
await db.shutdown_db()
@project_app.command("set-cloud")
def set_cloud(
name: str = typer.Argument(..., help="Name of the project to route through cloud"),
@@ -855,6 +925,13 @@ def set_cloud(
If omitted, uses the default workspace (if set) or auto-selects when
only one workspace is available.
This is a one-way cutover: the project's row in the local index DB is
removed and the local path in config is cleared so the project's
configured state is purely cloud. On-disk note files are preserved
the caller can keep, archive, or delete them as they see fit. To
return to local mode use `bm project set-local <name> --local-path
<path>`.
Examples:
bm project set-cloud research --workspace Personal
bm project set-cloud research --workspace 11111111-...
@@ -883,27 +960,52 @@ def set_cloud(
resolved_workspace_id = _resolve_workspace_id(config, workspace)
# Drop the local DB row first so the user-visible state stays consistent
# even if the config save below raises for some reason. Idempotent: a
# second `set-cloud` simply finds no row and returns False.
previous_path = config.projects[name].path
detached = run_with_cleanup(_detach_local_project_row(config, name))
config.set_project_mode(name, ProjectMode.CLOUD)
if resolved_workspace_id:
config.projects[name].workspace_id = resolved_workspace_id
# Clear local path: source-of-truth for this project is now the cloud
config.projects[name].path = ""
config_manager.save_config(config)
console.print(f"[green]Project '{name}' set to cloud mode[/green]")
if resolved_workspace_id:
console.print(f"[dim]Workspace: {resolved_workspace_id}[/dim]")
if detached and previous_path:
console.print(
f"[dim]Local index entry removed. Files at {previous_path} are preserved on disk.[/dim]"
)
console.print("[dim]MCP tools and CLI commands for this project will route through cloud[/dim]")
@project_app.command("set-local")
def set_local(
name: str = typer.Argument(..., help="Name of the project to revert to local mode"),
local_path: str = typer.Option(
None,
"--local-path",
help=(
"Local filesystem path for this project. Required unless the project "
"was previously local and its prior path is still in config."
),
),
) -> None:
"""Revert a project to local mode (use in-process ASGI transport).
Clears any associated cloud workspace.
Recreates the project's row in the local index DB and clears any
associated cloud workspace. If the project was previously local and
its prior path is still in config (e.g. an older version that didn't
blank `path` on `set-cloud`), `--local-path` may be omitted and that
path will be reused.
Example:
bm project set-local research
Examples:
bm project set-local research --local-path ~/Documents/research
bm project set-local research # reuse prior path
"""
config_manager = ConfigManager()
config = config_manager.config
@@ -913,11 +1015,31 @@ def set_local(
console.print(f"[red]Error: Project '{name}' not found in config[/red]")
raise typer.Exit(1)
entry = config.projects[name]
candidate = local_path or entry.path
if not candidate:
console.print(
f"[red]Error: --local-path is required for '{name}' "
"(no previous local path is recorded)[/red]"
)
raise typer.Exit(1)
resolved_path = Path(os.path.abspath(os.path.expanduser(candidate))).as_posix()
# Recreate the local DB row. Idempotent: if the row exists with the
# same path it's a no-op; if it exists at a stale path the path is
# updated. The directory itself is not auto-created — the user is
# expected to know whether they want to start a fresh project tree
# or point at an existing one.
run_with_cleanup(_attach_local_project_row(config, name, resolved_path))
config.set_project_mode(name, ProjectMode.LOCAL)
config.projects[name].workspace_id = None
config.projects[name].path = resolved_path
config_manager.save_config(config)
console.print(f"[green]Project '{name}' set to local mode[/green]")
console.print(f"[dim]Path: {resolved_path}[/dim]")
console.print("[dim]MCP tools and CLI commands for this project will use local transport[/dim]")
+55 -33
View File
@@ -62,9 +62,12 @@ def write_note(
help="The project to write to. If not provided, the default project will be used."
),
] = None,
workspace: Annotated[
project_id: Annotated[
Optional[str],
typer.Option(help="Cloud workspace tenant ID or unique name to route this request."),
typer.Option(
"--project-id",
help="Project external_id (UUID). Takes precedence over --project; use to disambiguate same-named projects across cloud workspaces.",
),
] = None,
local: bool = typer.Option(
False, "--local", help="Force local API routing (ignore cloud mode)"
@@ -106,7 +109,7 @@ def write_note(
content=content,
directory=folder,
project=project,
workspace=workspace,
project_id=project_id,
tags=tags,
output_format="json",
)
@@ -128,15 +131,16 @@ def read_note(
include_frontmatter: bool = typer.Option(
False, "--include-frontmatter", help="Include YAML frontmatter in output"
),
page: int = typer.Option(1, "--page", help="Page number for pagination"),
page_size: int = typer.Option(10, "--page-size", help="Number of results per page"),
project: Annotated[
Optional[str],
typer.Option(help="The project to use. If not provided, the default project will be used."),
] = None,
workspace: Annotated[
project_id: Annotated[
Optional[str],
typer.Option(help="Cloud workspace tenant ID or unique name to route this request."),
typer.Option(
"--project-id",
help="Project external_id (UUID). Takes precedence over --project; use to disambiguate same-named projects across cloud workspaces.",
),
] = None,
local: bool = typer.Option(
False, "--local", help="Force local API routing (ignore cloud mode)"
@@ -149,7 +153,6 @@ def read_note(
bm tool read-note my-note
bm tool read-note my-note --include-frontmatter
bm tool read-note my-note --page 2 --page-size 5
"""
try:
validate_routing_flags(local, cloud)
@@ -159,9 +162,7 @@ def read_note(
mcp_read_note(
identifier=identifier,
project=project,
workspace=workspace,
page=page,
page_size=page_size,
project_id=project_id,
include_frontmatter=include_frontmatter,
output_format="json",
)
@@ -200,9 +201,12 @@ def edit_note(
help="The project to edit. If not provided, the default project will be used."
),
] = None,
workspace: Annotated[
project_id: Annotated[
Optional[str],
typer.Option(help="Cloud workspace tenant ID or unique name to route this request."),
typer.Option(
"--project-id",
help="Project external_id (UUID). Takes precedence over --project; use to disambiguate same-named projects across cloud workspaces.",
),
] = None,
local: bool = typer.Option(
False, "--local", help="Force local API routing (ignore cloud mode)"
@@ -227,7 +231,7 @@ def edit_note(
operation=operation,
content=content,
project=project,
workspace=workspace,
project_id=project_id,
section=section,
find_text=find_text,
expected_replacements=expected_replacements,
@@ -265,9 +269,12 @@ def build_context(
Optional[str],
typer.Option(help="The project to use. If not provided, the default project will be used."),
] = None,
workspace: Annotated[
project_id: Annotated[
Optional[str],
typer.Option(help="Cloud workspace tenant ID or unique name to route this request."),
typer.Option(
"--project-id",
help="Project external_id (UUID). Takes precedence over --project; use to disambiguate same-named projects across cloud workspaces.",
),
] = None,
local: bool = typer.Option(
False, "--local", help="Force local API routing (ignore cloud mode)"
@@ -289,7 +296,7 @@ def build_context(
mcp_build_context(
url=url,
project=project,
workspace=workspace,
project_id=project_id,
depth=depth,
timeframe=timeframe,
page=page,
@@ -322,9 +329,12 @@ def recent_activity(
Optional[str],
typer.Option(help="The project to use. If not provided, the default project will be used."),
] = None,
workspace: Annotated[
project_id: Annotated[
Optional[str],
typer.Option(help="Cloud workspace tenant ID or unique name to route this request."),
typer.Option(
"--project-id",
help="Project external_id (UUID). Takes precedence over --project; use to disambiguate same-named projects across cloud workspaces.",
),
] = None,
local: bool = typer.Option(
False, "--local", help="Force local API routing (ignore cloud mode)"
@@ -345,13 +355,13 @@ def recent_activity(
with force_routing(local=local, cloud=cloud):
result = run_with_cleanup(
mcp_recent_activity(
type=type, # pyright: ignore[reportArgumentType]
type=type or "",
depth=depth if depth is not None else 1,
timeframe=timeframe if timeframe is not None else "7d",
page=page,
page_size=page_size,
project=project,
workspace=workspace,
project_id=project_id,
output_format="json",
)
)
@@ -413,9 +423,12 @@ def search_notes(
Optional[str],
typer.Option(help="The project to use. If not provided, the default project will be used."),
] = None,
workspace: Annotated[
project_id: Annotated[
Optional[str],
typer.Option(help="Cloud workspace tenant ID or unique name to route this request."),
typer.Option(
"--project-id",
help="Project external_id (UUID). Takes precedence over --project; use to disambiguate same-named projects across cloud workspaces.",
),
] = None,
local: bool = typer.Option(
False, "--local", help="Force local API routing (ignore cloud mode)"
@@ -487,7 +500,7 @@ def search_notes(
mcp_search(
query=query or None,
project=project,
workspace=workspace,
project_id=project_id,
search_type=search_type,
output_format="json",
page=page,
@@ -597,9 +610,12 @@ def schema_validate(
Optional[str],
typer.Option(help="The project to use. If not provided, the default project will be used."),
] = None,
workspace: Annotated[
project_id: Annotated[
Optional[str],
typer.Option(help="Cloud workspace tenant ID or unique name to route this request."),
typer.Option(
"--project-id",
help="Project external_id (UUID). Takes precedence over --project; use to disambiguate same-named projects across cloud workspaces.",
),
] = None,
local: bool = typer.Option(
False, "--local", help="Force local API routing (ignore cloud mode)"
@@ -634,7 +650,7 @@ def schema_validate(
note_type=note_type,
identifier=identifier,
project=project,
workspace=workspace,
project_id=project_id,
output_format="json",
)
)
@@ -665,9 +681,12 @@ def schema_infer(
Optional[str],
typer.Option(help="The project to use. If not provided, the default project will be used."),
] = None,
workspace: Annotated[
project_id: Annotated[
Optional[str],
typer.Option(help="Cloud workspace tenant ID or unique name to route this request."),
typer.Option(
"--project-id",
help="Project external_id (UUID). Takes precedence over --project; use to disambiguate same-named projects across cloud workspaces.",
),
] = None,
local: bool = typer.Option(
False, "--local", help="Force local API routing (ignore cloud mode)"
@@ -691,7 +710,7 @@ def schema_infer(
note_type=note_type,
threshold=threshold,
project=project,
workspace=workspace,
project_id=project_id,
output_format="json",
)
)
@@ -719,9 +738,12 @@ def schema_diff(
Optional[str],
typer.Option(help="The project to use. If not provided, the default project will be used."),
] = None,
workspace: Annotated[
project_id: Annotated[
Optional[str],
typer.Option(help="Cloud workspace tenant ID or unique name to route this request."),
typer.Option(
"--project-id",
help="Project external_id (UUID). Takes precedence over --project; use to disambiguate same-named projects across cloud workspaces.",
),
] = None,
local: bool = typer.Option(
False, "--local", help="Force local API routing (ignore cloud mode)"
@@ -743,7 +765,7 @@ def schema_diff(
mcp_schema_diff(
note_type=note_type,
project=project,
workspace=workspace,
project_id=project_id,
output_format="json",
)
)
+53 -17
View File
@@ -8,7 +8,7 @@ from dataclasses import dataclass
from datetime import datetime
from enum import Enum
from pathlib import Path
from typing import Any, Dict, Literal, Optional, List, Tuple
from typing import TYPE_CHECKING, Any, Dict, Literal, Optional, List, Tuple
from loguru import logger
from pydantic import AliasChoices, BaseModel, Field, model_validator
@@ -50,6 +50,44 @@ def _default_semantic_search_enabled() -> bool:
)
def resolve_data_dir() -> Path:
"""Resolve the Basic Memory data directory.
Single source of truth for the per-user state directory. Honors
``BASIC_MEMORY_CONFIG_DIR`` so each process/worktree can isolate config
and database state; otherwise falls back to ``<user home>/.basic-memory``.
Cross-platform: ``Path.home()`` reads ``$HOME`` on POSIX and
``%USERPROFILE%`` on Windows, so there's no need to check ``$HOME``
explicitly here.
"""
if config_dir := os.getenv("BASIC_MEMORY_CONFIG_DIR"):
return Path(config_dir)
return Path.home() / DATA_DIR_NAME
def default_fastembed_cache_dir() -> str:
"""Return the default cache directory used for FastEmbed model artifacts.
Resolution order:
1. ``FASTEMBED_CACHE_PATH`` env var honors FastEmbed's own convention
so users who already configure it through the environment keep working.
2. ``<basic-memory data dir>/fastembed_cache`` the same stable,
user-writable directory Basic Memory already uses for config and
the default SQLite database. Honors ``BASIC_MEMORY_CONFIG_DIR``.
Why not ``tempfile.gettempdir()``?
FastEmbed's own default is ``<system tmp>/fastembed_cache``, which is
ephemeral in many sandboxed MCP runtimes (e.g. Codex CLI wipes /tmp
between invocations). The model then disappears and every subsequent
ONNX load raises ``NO_SUCHFILE``. Persisting the cache under the
per-user data directory works identically on macOS, Linux, and Windows.
"""
if env_override := os.getenv("FASTEMBED_CACHE_PATH"):
return env_override
return str(resolve_data_dir() / "fastembed_cache")
@dataclass
class ProjectConfig:
"""Configuration for a specific basic-memory project."""
@@ -122,6 +160,11 @@ class ProjectEntry(BaseModel):
class BasicMemoryConfig(BaseSettings):
"""Pydantic model for Basic Memory global configuration."""
if TYPE_CHECKING:
# Pydantic accepts raw constructor data and validates/coerces it at runtime.
# Model attributes remain strongly typed after initialization.
def __init__(self, **data: Any) -> None: ...
env: Environment = Field(default="dev", description="Environment name")
projects: Dict[str, ProjectEntry] = Field(
@@ -217,7 +260,13 @@ class BasicMemoryConfig(BaseSettings):
)
semantic_embedding_cache_dir: str | None = Field(
default=None,
description="Optional cache directory for FastEmbed model artifacts.",
description=(
"Optional override for the FastEmbed model cache directory. "
"When unset, Basic Memory resolves this at runtime to "
"<basic-memory data dir>/fastembed_cache (or FASTEMBED_CACHE_PATH "
"when that env var is set) so the model persists across runs "
"without hardcoding a path into config.json."
),
)
semantic_embedding_threads: int | None = Field(
default=None,
@@ -704,11 +753,7 @@ class BasicMemoryConfig(BaseSettings):
@property
def data_dir_path(self) -> Path:
"""Get app state directory for config and default SQLite database."""
if config_dir := os.getenv("BASIC_MEMORY_CONFIG_DIR"):
return Path(config_dir)
home = os.getenv("HOME", Path.home())
return Path(home) / DATA_DIR_NAME
return resolve_data_dir()
# Module-level cache for configuration
@@ -726,16 +771,7 @@ class ConfigManager:
def __init__(self) -> None:
"""Initialize the configuration manager."""
home = os.getenv("HOME", Path.home())
if isinstance(home, str):
home = Path(home)
# Allow override via environment variable
if config_dir := os.getenv("BASIC_MEMORY_CONFIG_DIR"):
self.config_dir = Path(config_dir)
else:
self.config_dir = home / DATA_DIR_NAME
self.config_dir = resolve_data_dir()
self.config_file = self.config_dir / CONFIG_FILE_NAME
# Ensure config directory exists
-25
View File
@@ -492,7 +492,6 @@ class LocalTaskScheduler:
async def get_task_scheduler(
entity_service: EntityServiceV2ExternalDep,
sync_service: SyncServiceV2ExternalDep,
search_service: SearchServiceV2ExternalDep,
project_config: ProjectConfigV2ExternalDep,
@@ -500,28 +499,6 @@ async def get_task_scheduler(
) -> TaskScheduler:
"""Create a scheduler that maps task specs to coroutines."""
scheduler: LocalTaskScheduler | None = None
async def _reindex_entity(
entity_id: int,
resolve_relations: bool = False,
**_: Any,
) -> None:
await entity_service.reindex_entity(entity_id)
# Trigger: caller requests relation resolution
# Why: resolve forward references created before the entity existed
# Outcome: updates unresolved relations pointing to this entity
if resolve_relations:
await sync_service.resolve_relations(entity_id=entity_id)
# Trigger: semantic search enabled in local config.
# Why: vector chunks are derived and should refresh after canonical reindex completes.
# Outcome: schedules out-of-band vector sync without extending write latency.
if app_config.semantic_search_enabled and scheduler is not None:
scheduler.schedule("sync_entity_vectors", entity_id=entity_id)
async def _resolve_relations(entity_id: int, **_: Any) -> None:
await sync_service.resolve_relations(entity_id=entity_id)
async def _sync_entity_vectors(entity_id: int, **_: Any) -> None:
await search_service.sync_entity_vectors(entity_id)
@@ -537,8 +514,6 @@ async def get_task_scheduler(
scheduler = LocalTaskScheduler(
{
"reindex_entity": _reindex_entity,
"resolve_relations": _resolve_relations,
"sync_entity_vectors": _sync_entity_vectors,
"sync_project": _sync_project,
"reindex_project": _reindex_project,
+8 -3
View File
@@ -4,6 +4,8 @@ import fnmatch
from pathlib import Path
from typing import Set
from basic_memory.config import resolve_data_dir
# Common directories and patterns to ignore by default
# These are used as fallback if .bmignore doesn't exist
@@ -61,9 +63,11 @@ def get_bmignore_path() -> Path:
"""Get path to .bmignore file.
Returns:
Path to ~/.basic-memory/.bmignore
Path to <basic-memory data dir>/.bmignore, honoring
``BASIC_MEMORY_CONFIG_DIR`` so isolated instances each keep their
own ignore file.
"""
return Path.home() / ".basic-memory" / ".bmignore"
return resolve_data_dir() / ".bmignore"
def create_default_bmignore() -> None:
@@ -176,7 +180,8 @@ def load_gitignore_patterns(base_path: Path, use_gitignore: bool = True) -> Set[
"""Load gitignore patterns from .gitignore file and .bmignore.
Combines patterns from:
1. ~/.basic-memory/.bmignore (user's global ignore patterns)
1. <basic-memory data dir>/.bmignore (user's global ignore patterns, honors
BASIC_MEMORY_CONFIG_DIR)
2. {base_path}/.gitignore (project-specific patterns, if use_gitignore=True)
Args:
+7 -6
View File
@@ -39,23 +39,24 @@ def format_timestamp(timestamp: Any) -> str: # pragma: no cover
Returns:
A formatted string representation of the timestamp.
"""
parsed_timestamp = timestamp
if isinstance(timestamp, str):
try:
# Try ISO format
timestamp = datetime.fromisoformat(timestamp.replace("Z", "+00:00"))
parsed_timestamp = datetime.fromisoformat(timestamp.replace("Z", "+00:00"))
except ValueError:
try:
# Try unix timestamp as string
timestamp = datetime.fromtimestamp(float(timestamp)).astimezone()
parsed_timestamp = datetime.fromtimestamp(float(timestamp)).astimezone()
except ValueError:
# Return as is if we can't parse it
return timestamp
elif isinstance(timestamp, (int, float)):
# Unix timestamp
timestamp = datetime.fromtimestamp(timestamp).astimezone()
parsed_timestamp = datetime.fromtimestamp(timestamp).astimezone()
if isinstance(timestamp, datetime):
return timestamp.strftime("%Y-%m-%d %H:%M:%S")
if isinstance(parsed_timestamp, datetime):
return parsed_timestamp.strftime("%Y-%m-%d %H:%M:%S")
# Return as is if we can't format it
return str(timestamp) # pragma: no cover
return str(parsed_timestamp) # pragma: no cover
+2
View File
@@ -12,6 +12,7 @@ from basic_memory.indexing.models import (
IndexingBatchResult,
IndexInputFile,
IndexProgress,
SyncedMarkdownFile,
)
__all__ = [
@@ -25,5 +26,6 @@ __all__ = [
"IndexingBatchResult",
"IndexInputFile",
"IndexProgress",
"SyncedMarkdownFile",
"build_index_batches",
]
+183 -29
View File
@@ -11,8 +11,9 @@ from typing import Awaitable, Callable, Mapping, TypeVar
from loguru import logger
from sqlalchemy.exc import IntegrityError
import logfire
from basic_memory.config import BasicMemoryConfig
from basic_memory.file_utils import compute_checksum, has_frontmatter
from basic_memory.file_utils import compute_checksum, has_frontmatter, remove_frontmatter
from basic_memory.markdown.schemas import EntityMarkdown
from basic_memory.indexing.models import (
IndexedEntity,
@@ -43,12 +44,19 @@ class _PreparedMarkdownFile:
class _PreparedEntity:
path: str
entity_id: int
permalink: str | None
checksum: str
content_type: str | None
search_content: str | None
markdown_content: str | None = None
@dataclass(slots=True)
class _PersistedMarkdownFile:
prepared: _PreparedMarkdownFile
entity: Entity
class BatchIndexer:
"""Index already-loaded files without assuming where they came from."""
@@ -118,6 +126,9 @@ class BatchIndexer:
)
error_by_path.update(markdown_errors)
prepared_entities.update(markdown_upserts)
if existing_permalink_by_path is not None:
for path, prepared_entity in markdown_upserts.items():
existing_permalink_by_path[path] = prepared_entity.permalink
regular_upserts, regular_errors = await self._run_bounded(
regular_paths,
@@ -168,6 +179,76 @@ class BatchIndexer:
search_indexed=search_indexed,
)
async def index_markdown_file(
self,
file: IndexInputFile,
*,
new: bool | None = None,
existing_permalink_by_path: dict[str, str | None] | None = None,
index_search: bool = True,
resolve_relations: bool = True,
) -> IndexedEntity:
"""Index one markdown file using the same normalization and upsert path as batches."""
if not self._is_markdown(file):
raise ValueError(f"index_markdown_file requires markdown input: {file.path}")
with logfire.span("index.markdown_file.prepare", path=file.path):
prepared = await self._prepare_markdown_file(file)
if existing_permalink_by_path is None:
with logfire.span("index.markdown_file.load_permalink_map", path=file.path):
existing_permalink_by_path = {
path: permalink
for path, permalink in (
await self.entity_repository.get_file_path_to_permalink_map()
).items()
}
reserved_permalinks = {
permalink
for path, permalink in existing_permalink_by_path.items()
if path != file.path and permalink
}
with logfire.span("index.markdown_file.normalize", path=file.path):
prepared = await self._normalize_markdown_file(prepared, reserved_permalinks)
existing_permalink_by_path[file.path] = prepared.markdown.frontmatter.permalink
with logfire.span("index.markdown_file.persist", path=file.path, is_new=new):
persisted = await self._persist_markdown_file(
prepared,
is_new=new,
resolve_relations=resolve_relations,
reload_entity=False,
)
existing_permalink_by_path[file.path] = persisted.entity.permalink
with logfire.span(
"index.markdown_file.reload_entity",
path=file.path,
entity_id=persisted.entity.id,
):
refreshed = await self.entity_repository.find_by_ids([persisted.entity.id])
if len(refreshed) != 1: # pragma: no cover
raise ValueError(f"Failed to reload indexed entity for {file.path}")
entity = refreshed[0]
prepared_entity = self._build_prepared_entity(persisted.prepared, entity)
if index_search:
with logfire.span(
"index.markdown_file.refresh_search_index",
path=file.path,
entity_id=entity.id,
):
return await self._refresh_search_index(prepared_entity, entity)
return IndexedEntity(
path=prepared_entity.path,
entity_id=entity.id,
permalink=entity.permalink,
checksum=prepared_entity.checksum,
content_type=prepared_entity.content_type,
markdown_content=prepared_entity.markdown_content,
)
# --- Preparation ---
async def _prepare_markdown_file(self, file: IndexInputFile) -> _PreparedMarkdownFile:
@@ -320,34 +401,8 @@ class BatchIndexer:
# --- Persistence ---
async def _upsert_markdown_file(self, prepared: _PreparedMarkdownFile) -> _PreparedEntity:
existing = await self.entity_repository.get_by_file_path(
prepared.file.path,
load_relations=False,
)
entity = await self.entity_service.upsert_entity_from_markdown(
Path(prepared.file.path),
prepared.markdown,
is_new=existing is None,
)
updated = await self.entity_repository.update(
entity.id,
self._entity_metadata_updates(prepared.file, prepared.final_checksum),
)
if updated is None:
raise ValueError(f"Failed to update markdown entity metadata for {prepared.file.path}")
return _PreparedEntity(
path=prepared.file.path,
entity_id=updated.id,
checksum=prepared.final_checksum,
content_type=prepared.file.content_type,
search_content=(
prepared.markdown.content
if prepared.markdown.content is not None
else prepared.content
),
markdown_content=prepared.content,
)
persisted = await self._persist_markdown_file(prepared)
return self._build_prepared_entity(persisted.prepared, persisted.entity)
async def _upsert_regular_file(self, file: IndexInputFile) -> _PreparedEntity:
checksum = await self._resolve_checksum(file)
@@ -405,6 +460,7 @@ class BatchIndexer:
return _PreparedEntity(
path=file.path,
entity_id=updated.id,
permalink=updated.permalink,
checksum=checksum,
content_type=file.content_type,
search_content=None,
@@ -495,6 +551,99 @@ class BatchIndexer:
# --- Helpers ---
async def _persist_markdown_file(
self,
prepared: _PreparedMarkdownFile,
*,
is_new: bool | None = None,
resolve_relations: bool = True,
reload_entity: bool = True,
) -> _PersistedMarkdownFile:
existing = await self.entity_repository.get_by_file_path(
prepared.file.path,
load_relations=False,
)
if is_new is None:
is_new = existing is None
entity = await self.entity_service.upsert_entity_from_markdown(
Path(prepared.file.path),
prepared.markdown,
is_new=is_new,
existing_entity=existing,
resolve_relations=resolve_relations,
reload_entity=reload_entity,
)
prepared = await self._reconcile_persisted_permalink(prepared, entity)
metadata_updates = self._entity_metadata_updates(prepared.file, prepared.final_checksum)
updated = await self.entity_repository.update_fields(
entity.id,
metadata_updates,
)
if not updated:
raise ValueError(f"Failed to update markdown entity metadata for {prepared.file.path}")
self._apply_entity_metadata_updates(entity, metadata_updates)
return _PersistedMarkdownFile(prepared=prepared, entity=entity)
async def _reconcile_persisted_permalink(
self,
prepared: _PreparedMarkdownFile,
entity: Entity,
) -> _PreparedMarkdownFile:
# Trigger: the source file started without frontmatter and sync is configured
# to leave frontmatterless files alone.
# Why: upsert may still assign a DB permalink even when disk content should stay untouched.
# Outcome: skip reconciliation writes that would silently inject frontmatter.
if (
self.app_config.disable_permalinks
or (
not prepared.file_contains_frontmatter
and not self.app_config.ensure_frontmatter_on_sync
)
or entity.permalink is None
or entity.permalink == prepared.markdown.frontmatter.permalink
):
return prepared
logger.debug(
"Updating permalink after upsert conflict resolution",
path=prepared.file.path,
old_permalink=prepared.markdown.frontmatter.permalink,
new_permalink=entity.permalink,
)
prepared.markdown.frontmatter.metadata["permalink"] = entity.permalink
write_result = await self.file_writer.write_frontmatter(
IndexFrontmatterUpdate(
path=prepared.file.path,
metadata={"permalink": entity.permalink},
)
)
return _PreparedMarkdownFile(
file=prepared.file,
content=write_result.content,
final_checksum=write_result.checksum,
markdown=prepared.markdown,
file_contains_frontmatter=prepared.file_contains_frontmatter,
)
def _build_prepared_entity(
self,
prepared: _PreparedMarkdownFile,
entity: Entity,
) -> _PreparedEntity:
return _PreparedEntity(
path=prepared.file.path,
entity_id=entity.id,
permalink=entity.permalink,
checksum=prepared.final_checksum,
content_type=prepared.file.content_type,
search_content=(
prepared.markdown.content
if prepared.markdown.content is not None
else remove_frontmatter(prepared.content)
),
markdown_content=prepared.content,
)
async def _resolve_checksum(self, file: IndexInputFile) -> str:
if file.checksum is not None:
return file.checksum
@@ -523,6 +672,11 @@ class BatchIndexer:
updates["content_type"] = file.content_type
return updates
def _apply_entity_metadata_updates(self, entity: Entity, updates: dict[str, object]) -> None:
"""Keep the returned entity aligned with metadata written without reload."""
for key, value in updates.items():
setattr(entity, key, value)
def _is_markdown(self, file: IndexInputFile) -> bool:
if file.content_type is not None:
return file.content_type == "text/markdown"
+17 -1
View File
@@ -4,7 +4,10 @@ from __future__ import annotations
from dataclasses import dataclass, field
from datetime import datetime
from typing import Any, Protocol
from typing import Any, Protocol, TYPE_CHECKING
if TYPE_CHECKING: # pragma: no cover
from basic_memory.models import Entity
@dataclass(slots=True)
@@ -75,6 +78,19 @@ class IndexedEntity:
markdown_content: str | None = None
@dataclass(slots=True)
class SyncedMarkdownFile:
"""Canonical result for syncing one markdown file end-to-end."""
entity: Entity
checksum: str
markdown_content: str
file_path: str
content_type: str
updated_at: datetime
size: int
@dataclass(slots=True)
class IndexingBatchResult:
"""Outcome for one batch execution."""
+15
View File
@@ -180,6 +180,9 @@ def observation_plugin(md: MarkdownIt) -> None:
def observation_rule(state: Any) -> None:
"""Process observations in token stream."""
tokens = state.tokens
# Track blockquote nesting so Obsidian callouts (`> [!info] Title`)
# don't get parsed as observations with category `!info`.
blockquote_depth = 0
for idx in range(len(tokens)):
token = tokens[idx]
@@ -187,6 +190,18 @@ def observation_plugin(md: MarkdownIt) -> None:
# Initialize meta for all tokens
token.meta = token.meta or {}
if token.type == "blockquote_open":
blockquote_depth += 1
continue
if token.type == "blockquote_close":
blockquote_depth -= 1
continue
# Skip parsing inside blockquotes — that's Obsidian callout
# territory, not Basic Memory observation syntax.
if blockquote_depth > 0:
continue
# Parse observations in list items
if token.type == "inline" and is_observation(token):
obs = parse_observation(token)
+32 -8
View File
@@ -1,9 +1,9 @@
"""Schema models for entity markdown files."""
from datetime import datetime
from typing import List, Optional
from typing import TYPE_CHECKING, Any, List, Optional
from pydantic import BaseModel
from pydantic import BaseModel, Field, model_validator
class Observation(BaseModel):
@@ -38,23 +38,47 @@ class Relation(BaseModel):
class EntityFrontmatter(BaseModel):
"""Required frontmatter fields for an entity."""
metadata: dict = {}
if TYPE_CHECKING:
# Frontmatter may be built from raw YAML keys. The validator below
# gathers those keys into the metadata mapping used at runtime.
def __init__(self, **data: Any) -> None: ...
metadata: dict[str, Any] = Field(default_factory=dict)
@model_validator(mode="before")
@classmethod
def collect_metadata(cls, data: Any) -> Any:
if not isinstance(data, dict):
return data
if "metadata" not in data:
return {"metadata": data}
metadata = data.get("metadata") or {}
extras = {key: value for key, value in data.items() if key != "metadata"}
if extras:
return {"metadata": {**extras, **metadata}}
return data
@property
def tags(self) -> List[str]:
return self.metadata.get("tags") if self.metadata else None # pyright: ignore
tags = self.metadata.get("tags")
return [str(tag) for tag in tags] if isinstance(tags, list) else []
@property
def title(self) -> str:
return self.metadata.get("title") if self.metadata else None # pyright: ignore
title = self.metadata.get("title")
return title if isinstance(title, str) else ""
@property
def type(self) -> str:
return self.metadata.get("type", "note") if self.metadata else "note" # pyright: ignore
note_type = self.metadata.get("type", "note")
return note_type if isinstance(note_type, str) else "note"
@property
def permalink(self) -> str:
return self.metadata.get("permalink") if self.metadata else None # pyright: ignore
def permalink(self) -> Optional[str]:
permalink = self.metadata.get("permalink")
return permalink if isinstance(permalink, str) else None
class EntityMarkdown(BaseModel):
+5 -2
View File
@@ -5,7 +5,7 @@ from typing import AsyncIterator, Callable, Optional
from httpx import ASGITransport, AsyncClient, Timeout
from loguru import logger
from basic_memory import telemetry
import logfire
from basic_memory.api.app import app as fastapi_app
from basic_memory.config import ConfigManager, ProjectMode
@@ -44,7 +44,7 @@ def _asgi_client(timeout: Timeout) -> AsyncClient:
async def _resolve_cloud_token(config) -> str:
"""Resolve cloud token with API key preferred, OAuth fallback."""
with telemetry.span(
with logfire.span(
"routing.resolve_cloud_credentials",
has_api_key=bool(config.cloud_api_key),
):
@@ -94,9 +94,12 @@ async def _cloud_client(
workspace: Optional[str] = None,
) -> AsyncIterator[AsyncClient]:
"""Create a cloud proxy client with resolved credentials."""
from basic_memory.workspace_context import workspace_permalink_headers
token = await _resolve_cloud_token(config)
proxy_base_url = f"{config.cloud_host}/proxy"
headers = {"Authorization": f"Bearer {token}"}
headers.update(workspace_permalink_headers())
if workspace:
headers["X-Workspace-ID"] = workspace
logger.info(f"Creating HTTP client for cloud proxy at: {proxy_base_url}")
+11 -26
View File
@@ -7,7 +7,7 @@ from typing import Any
from httpx import AsyncClient
from basic_memory import telemetry
import logfire
from basic_memory.mcp.tools.utils import call_get, call_post, call_put, call_patch, call_delete
from basic_memory.schemas.response import (
EntityResponse,
@@ -44,9 +44,7 @@ class KnowledgeClient:
# --- Entity CRUD Operations ---
async def create_entity(
self, entity_data: dict[str, Any], *, fast: bool | None = None
) -> EntityResponse:
async def create_entity(self, entity_data: dict[str, Any]) -> EntityResponse:
"""Create a new entity.
Args:
@@ -58,18 +56,15 @@ class KnowledgeClient:
Raises:
ToolError: If the request fails
"""
params = {"fast": fast} if fast is not None else None
with telemetry.scope(
with logfire.span(
"mcp.client.knowledge.create_entity",
client_name="knowledge",
operation="create_entity",
fast=fast,
):
response = await call_post(
self.http_client,
f"{self._base_path}/entities",
json=entity_data,
params=params,
client_name="knowledge",
operation="create_entity",
path_template="/v2/projects/{project_id}/knowledge/entities",
@@ -80,8 +75,6 @@ class KnowledgeClient:
self,
entity_id: str,
entity_data: dict[str, Any],
*,
fast: bool | None = None,
) -> EntityResponse:
"""Update an existing entity (full replacement).
@@ -95,18 +88,15 @@ class KnowledgeClient:
Raises:
ToolError: If the request fails
"""
params = {"fast": fast} if fast is not None else None
with telemetry.scope(
with logfire.span(
"mcp.client.knowledge.update_entity",
client_name="knowledge",
operation="update_entity",
fast=fast,
):
response = await call_put(
self.http_client,
f"{self._base_path}/entities/{entity_id}",
json=entity_data,
params=params,
client_name="knowledge",
operation="update_entity",
path_template="/v2/projects/{project_id}/knowledge/entities/{entity_id}",
@@ -125,7 +115,7 @@ class KnowledgeClient:
Raises:
ToolError: If the entity is not found or request fails
"""
with telemetry.scope(
with logfire.span(
"mcp.client.knowledge.get_entity",
client_name="knowledge",
operation="get_entity",
@@ -143,8 +133,6 @@ class KnowledgeClient:
self,
entity_id: str,
patch_data: dict[str, Any],
*,
fast: bool | None = None,
) -> EntityResponse:
"""Partially update an entity.
@@ -158,18 +146,15 @@ class KnowledgeClient:
Raises:
ToolError: If the request fails
"""
params = {"fast": fast} if fast is not None else None
with telemetry.scope(
with logfire.span(
"mcp.client.knowledge.patch_entity",
client_name="knowledge",
operation="patch_entity",
fast=fast,
):
response = await call_patch(
self.http_client,
f"{self._base_path}/entities/{entity_id}",
json=patch_data,
params=params,
client_name="knowledge",
operation="patch_entity",
path_template="/v2/projects/{project_id}/knowledge/entities/{entity_id}",
@@ -188,7 +173,7 @@ class KnowledgeClient:
Raises:
ToolError: If the entity is not found or request fails
"""
with telemetry.scope(
with logfire.span(
"mcp.client.knowledge.delete_entity",
client_name="knowledge",
operation="delete_entity",
@@ -215,7 +200,7 @@ class KnowledgeClient:
Raises:
ToolError: If the request fails
"""
with telemetry.scope(
with logfire.span(
"mcp.client.knowledge.move_entity",
client_name="knowledge",
operation="move_entity",
@@ -245,7 +230,7 @@ class KnowledgeClient:
Raises:
ToolError: If the request fails
"""
with telemetry.scope(
with logfire.span(
"mcp.client.knowledge.move_directory",
client_name="knowledge",
operation="move_directory",
@@ -275,7 +260,7 @@ class KnowledgeClient:
Raises:
ToolError: If the request fails
"""
with telemetry.scope(
with logfire.span(
"mcp.client.knowledge.delete_directory",
client_name="knowledge",
operation="delete_directory",
@@ -305,7 +290,7 @@ class KnowledgeClient:
Raises:
ToolError: If the identifier cannot be resolved
"""
with telemetry.scope(
with logfire.span(
"mcp.client.knowledge.resolve_entity",
client_name="knowledge",
operation="resolve_entity",
+3 -3
View File
@@ -7,7 +7,7 @@ from typing import Optional
from httpx import AsyncClient
from basic_memory import telemetry
import logfire
from basic_memory.mcp.tools.utils import call_get
from basic_memory.schemas.memory import GraphContext
@@ -72,7 +72,7 @@ class MemoryClient:
if timeframe:
params["timeframe"] = timeframe
with telemetry.scope(
with logfire.span(
"mcp.client.memory.build_context",
client_name="memory",
operation="build_context",
@@ -123,7 +123,7 @@ class MemoryClient:
# Join types as comma-separated string if provided
params["type"] = ",".join(types) if isinstance(types, list) else types
with telemetry.scope(
with logfire.span(
"mcp.client.memory.recent_activity",
client_name="memory",
operation="recent_activity",
+3 -22
View File
@@ -3,11 +3,9 @@
Encapsulates all /v2/projects/{project_id}/resource/* endpoints.
"""
from typing import Optional
from httpx import AsyncClient, Response
from basic_memory import telemetry
import logfire
from basic_memory.mcp.tools.utils import call_get
@@ -39,19 +37,11 @@ class ResourceClient:
self.project_id = project_id
self._base_path = f"/v2/projects/{project_id}/resource"
async def read(
self,
entity_id: str,
*,
page: Optional[int] = None,
page_size: Optional[int] = None,
) -> Response:
async def read(self, entity_id: str) -> Response:
"""Read a resource by entity ID.
Args:
entity_id: Entity external_id (UUID)
page: Optional page number for paginated content
page_size: Optional page size for paginated content
Returns:
Raw HTTP Response (caller handles text/binary content)
@@ -59,23 +49,14 @@ class ResourceClient:
Raises:
ToolError: If the resource is not found or request fails
"""
params: dict = {}
if page is not None:
params["page"] = page
if page_size is not None:
params["page_size"] = page_size
with telemetry.scope(
with logfire.span(
"mcp.client.resource.read",
client_name="resource",
operation="read",
page=page,
page_size=page_size,
):
return await call_get(
self.http_client,
f"{self._base_path}/{entity_id}",
params=params if params else None,
client_name="resource",
operation="read",
path_template="/v2/projects/{project_id}/resource/{entity_id}",
+2 -2
View File
@@ -7,7 +7,7 @@ from typing import Any
from httpx import AsyncClient
from basic_memory import telemetry
import logfire
from basic_memory.mcp.tools.utils import call_post
from basic_memory.schemas.search import SearchResponse
@@ -57,7 +57,7 @@ class SearchClient:
Raises:
ToolError: If the request fails
"""
with telemetry.scope(
with logfire.span(
"mcp.client.search.search",
client_name="search",
operation="search",
File diff suppressed because it is too large Load Diff
+7 -7
View File
@@ -95,8 +95,8 @@ def format_prompt_context(context: PromptContext) -> str:
sections = []
# Process each context
for context in context.results: # pyright: ignore
for primary in context.primary_results: # pyright: ignore
for context_item in context.results:
for primary in context_item.primary_results:
if primary.permalink not in added_permalinks:
primary_permalink = primary.permalink
@@ -121,8 +121,8 @@ def format_prompt_context(context: PromptContext) -> str:
section += f"- **Created**: {primary.created_at.strftime('%Y-%m-%d %H:%M')}\n"
# Add content snippet
if hasattr(primary, "content") and primary.content: # pyright: ignore
content = primary.content or "" # pyright: ignore # pragma: no cover
if hasattr(primary, "content") and primary.content:
content = primary.content or "" # pragma: no cover
if content: # pragma: no cover
section += f"\n**Excerpt**:\n{content}\n" # pragma: no cover
@@ -132,14 +132,14 @@ def format_prompt_context(context: PromptContext) -> str:
""")
sections.append(section)
if context.related_results: # pyright: ignore
section += dedent( # pyright: ignore
if context_item.related_results:
section += dedent(
"""
## Related Context
"""
)
for related in context.related_results: # pyright: ignore
for related in context_item.related_results:
section_content = dedent(f"""
- type: **{related.type}**
- title: {related.title}
@@ -18,13 +18,15 @@ Basic Memory creates a semantic knowledge graph from markdown files. Focus on bu
**Resolution priority:**
1. CLI constraint: `BASIC_MEMORY_MCP_PROJECT` env var (highest priority)
2. Explicit parameter: `project="name"` in tool calls
2. Explicit parameter: `project_id="<uuid>"` (preferred when known) or `project="name"` in tool calls
3. Default project: `default_project` in config (fallback)
**`project` vs `project_id`:** Every project has a stable `external_id` (UUID) returned by `list_memory_projects()`. Pass it as `project_id=...` to address a project unambiguously — required when the same project name exists in multiple cloud workspaces. For local single-project setups, the `project` name is fine.
### Quick Setup Check
```python
# Discover projects
# Discover projects (each entry includes external_id you can pass as project_id)
projects = await list_memory_projects()
```
@@ -169,6 +171,14 @@ await write_note(
**Multi-project users:**
- Always specify project explicitly in tool calls
**Cloud multi-workspace users:** project names can collide across workspaces. After calling `list_memory_projects()`, prefer the project's `external_id` via `project_id=...` for any subsequent tool calls — it routes to the exact project regardless of name collisions. The `project` name parameter falls back to the default workspace on ambiguity, which may not be what you want.
```python
# Cloud / multi-workspace: prefer project_id (UUID) once you've discovered it
projects = await list_memory_projects()
results = await search_notes(query="auth", project_id=projects[0]["external_id"])
```
**Discovery:**
```python
# Start with discovery
+3 -3
View File
@@ -15,7 +15,7 @@ from basic_memory.cli.auth import CLIAuth
from basic_memory.db import scoped_session
from basic_memory.mcp.container import McpContainer, set_container
from basic_memory.services.initialization import initialize_app
from basic_memory import telemetry
import logfire
async def _log_embedding_status(session_maker: async_sessionmaker[AsyncSession]) -> None:
@@ -63,7 +63,7 @@ async def lifespan(app: FastMCP):
set_container(container)
config = container.config
with telemetry.operation(
with logfire.span(
"mcp.lifecycle.startup",
entrypoint="mcp",
mode=container.mode.name.lower(),
@@ -124,7 +124,7 @@ async def lifespan(app: FastMCP):
yield
finally:
# Shutdown - coordinator handles clean task cancellation
with telemetry.operation(
with logfire.span(
"mcp.lifecycle.shutdown",
entrypoint="mcp",
mode=container.mode.name.lower(),
+80 -53
View File
@@ -1,14 +1,15 @@
"""Build context tool for Basic Memory MCP server."""
from typing import Optional, Literal
from typing import Annotated, Optional, Literal
import logfire
from loguru import logger
from fastmcp import Context
from pydantic import AliasChoices, Field
from basic_memory.config import ConfigManager
from basic_memory import telemetry
from basic_memory.mcp.project_context import (
detect_project_from_url_prefix,
detect_project_from_memory_url_prefix,
get_project_client,
resolve_project_and_path,
)
@@ -133,14 +134,34 @@ def _format_context_markdown(graph: GraphContext, project: str) -> str:
annotations={"readOnlyHint": True, "openWorldHint": False},
)
async def build_context(
url: MemoryUrl,
url: Annotated[
MemoryUrl,
Field(validation_alias=AliasChoices("url", "uri", "memory_url")),
],
project: Optional[str] = None,
workspace: Optional[str] = None,
project_id: Optional[str] = None,
depth: str | int | None = 1,
timeframe: Optional[TimeFrame] = "7d",
page: int = 1,
page_size: int = 10,
max_related: int = 10,
timeframe: Annotated[
Optional[TimeFrame],
Field(
default="7d",
validation_alias=AliasChoices("timeframe", "since", "time_range", "lookback"),
),
] = "7d",
# `offset` is intentionally NOT aliased: it has different semantics
# (item-indexed vs. 1-indexed page-number).
page: Annotated[
int,
Field(default=1, validation_alias=AliasChoices("page", "page_number")),
] = 1,
page_size: Annotated[
int,
Field(default=10, validation_alias=AliasChoices("page_size", "limit", "per_page")),
] = 10,
max_related: Annotated[
int,
Field(default=10, validation_alias=AliasChoices("max_related", "max_results")),
] = 10,
output_format: Literal["json", "text"] = "json",
context: Context | None = None,
) -> dict | str:
@@ -158,6 +179,9 @@ async def build_context(
Args:
project: Project name to build context from. Optional - server will resolve using hierarchy.
If unknown, use list_memory_projects() to discover available projects.
project_id: Project external_id (UUID). Prefer this over `project` when known
it routes to the exact project regardless of name collisions across cloud
workspaces. Takes precedence over `project`. Get from list_memory_projects().
url: memory:// URI pointing to discussion content (e.g. memory://specs/search)
depth: How many relation hops to traverse (1-3 recommended for performance)
timeframe: How far back to look. Supports natural language like "2 days ago", "last week"
@@ -185,9 +209,14 @@ async def build_context(
Raises:
ToolError: If project doesn't exist or depth parameter is invalid
"""
# Detect project from memory URL prefix before routing
if project is None:
detected = detect_project_from_url_prefix(url, ConfigManager().config)
# Detect project from memory URL prefix before routing.
# project_id routes by external UUID, so it bypasses URL discovery entirely.
if project is None and project_id is None:
detected = await detect_project_from_memory_url_prefix(
url,
ConfigManager().config,
context=context,
)
if detected:
project = detected
@@ -202,12 +231,12 @@ async def build_context(
# URL is already validated and normalized by MemoryUrl type annotation
with telemetry.operation(
with logfire.span(
"mcp.tool.build_context",
entrypoint="mcp",
tool_name="build_context",
requested_project=project,
workspace_id=workspace,
requested_project_id=project_id,
depth=depth or 1,
timeframe=timeframe,
page=page,
@@ -216,48 +245,46 @@ async def build_context(
output_format=output_format,
is_memory_url=str(url).startswith("memory://"),
):
async with get_project_client(project, workspace, context) as (client, active_project):
with telemetry.contextualize(
project_name=active_project.name,
workspace_id=workspace,
tool_name="build_context",
):
logger.info(
f"MCP tool call tool=build_context project={active_project.name} "
f"url={url} depth={depth} timeframe={timeframe} output_format={output_format}"
)
async with get_project_client(project, context=context, project_id=project_id) as (
client,
active_project,
):
logger.info(
f"MCP tool call tool=build_context project={active_project.name} "
f"url={url} depth={depth} timeframe={timeframe} output_format={output_format}"
)
# Resolve memory:// identifier with project-prefix awareness
_, resolved_path, _ = await resolve_project_and_path(
client,
url,
active_project.name,
context,
)
# Resolve memory:// identifier with project-prefix awareness
_, resolved_path, _ = await resolve_project_and_path(
client,
url,
active_project.name,
context,
)
# Import here to avoid circular import
from basic_memory.mcp.clients import MemoryClient
# Import here to avoid circular import
from basic_memory.mcp.clients import MemoryClient
# Use typed MemoryClient for API calls
memory_client = MemoryClient(client, active_project.external_id)
graph = await memory_client.build_context(
resolved_path,
depth=depth or 1,
timeframe=timeframe,
page=page,
page_size=page_size,
max_related=max_related,
)
# Use typed MemoryClient for API calls
memory_client = MemoryClient(client, active_project.external_id)
graph = await memory_client.build_context(
resolved_path,
depth=depth or 1,
timeframe=timeframe,
page=page,
page_size=page_size,
max_related=max_related,
)
logger.info(
f"MCP tool response: tool=build_context project={active_project.name} "
f"uri={graph.metadata.uri or resolved_path} "
f"primary_count={graph.metadata.primary_count or 0} "
f"related_count={graph.metadata.related_count or 0} "
f"output_format={output_format}"
)
logger.info(
f"MCP tool response: tool=build_context project={active_project.name} "
f"uri={graph.metadata.uri or resolved_path} "
f"primary_count={graph.metadata.primary_count or 0} "
f"related_count={graph.metadata.related_count or 0} "
f"output_format={output_format}"
)
if output_format == "text":
return _format_context_markdown(graph, active_project.name)
if output_format == "text":
return _format_context_markdown(graph, active_project.name)
return graph.model_dump()
return graph.model_dump()
+13 -4
View File
@@ -8,7 +8,7 @@ from typing import Annotated, Dict, List, Any, Optional
from loguru import logger
from fastmcp import Context
from pydantic import BeforeValidator
from pydantic import AliasChoices, BeforeValidator, Field
from basic_memory.mcp.project_context import get_project_client
from basic_memory.utils import coerce_list
@@ -24,9 +24,12 @@ async def canvas(
nodes: Annotated[List[Dict[str, Any]], BeforeValidator(coerce_list)],
edges: Annotated[List[Dict[str, Any]], BeforeValidator(coerce_list)],
title: str,
directory: str,
directory: Annotated[
str,
Field(validation_alias=AliasChoices("directory", "folder", "dir", "path")),
],
project: Optional[str] = None,
workspace: Optional[str] = None,
project_id: Optional[str] = None,
context: Context | None = None,
) -> str:
"""Create an Obsidian canvas file with the provided nodes and edges.
@@ -43,6 +46,9 @@ async def canvas(
Args:
project: Project name to create canvas in. Optional - server will resolve using hierarchy.
If unknown, use list_memory_projects() to discover available projects.
project_id: Project external_id (UUID). Prefer this over `project` when known
it routes to the exact project regardless of name collisions across cloud
workspaces. Takes precedence over `project`. Get from list_memory_projects().
nodes: List of node objects following JSON Canvas 1.0 spec
edges: List of edge objects following JSON Canvas 1.0 spec
title: The title of the canvas (will be saved as title.canvas)
@@ -97,7 +103,10 @@ async def canvas(
Raises:
ToolError: If project doesn't exist or directory path is invalid
"""
async with get_project_client(project, workspace, context) as (client, active_project):
async with get_project_client(project, context=context, project_id=project_id) as (
client,
active_project,
):
# Ensure path has .canvas extension
file_title = title if title.endswith(".canvas") else f"{title}.canvas"
file_path = f"{directory}/{file_title}"
@@ -181,8 +181,6 @@ async def fetch(
content = str(
await read_note(
identifier=id,
page=1,
page_size=10,
context=context,
)
)
+76 -10
View File
@@ -1,13 +1,21 @@
from textwrap import dedent
from typing import Optional, Literal
from typing import Annotated, Optional, Literal
from loguru import logger
from fastmcp import Context
from mcp.server.fastmcp.exceptions import ToolError
from pydantic import AliasChoices, Field
from basic_memory.config import ConfigManager
from basic_memory.mcp.project_context import detect_project_from_url_prefix, get_project_client
from basic_memory.mcp.project_context import (
detect_project_from_memory_url_prefix,
get_project_client,
resolve_project_and_path,
)
from basic_memory.mcp.server import mcp
from basic_memory.schemas.project_info import ProjectItem
from basic_memory.utils import generate_permalink, normalize_project_reference
from basic_memory.workspace_context import current_workspace_permalink_context
def _format_delete_error_response(project: str, error_message: str, identifier: str) -> str:
@@ -147,15 +155,44 @@ delete_note("{project}", "correct-identifier-from-search")
If the note should be deleted but the operation keeps failing, send a message to support@basicmemory.com."""
def _directory_path_for_delete(
target_identifier: str,
active_project: ProjectItem,
*,
include_project_prefix: bool,
) -> str:
"""Return the project-relative directory path expected by the delete API."""
directory = normalize_project_reference(target_identifier).strip("/")
project_permalink = active_project.permalink
route_prefixes: list[str] = []
workspace_context = current_workspace_permalink_context()
if workspace_context and workspace_context.should_prefix_permalinks:
route_prefixes.append(
f"{generate_permalink(workspace_context.workspace_slug)}/{project_permalink}"
)
if include_project_prefix:
route_prefixes.append(project_permalink)
for route_prefix in route_prefixes:
if directory.startswith(f"{route_prefix}/"):
return directory.removeprefix(f"{route_prefix}/")
return directory
@mcp.tool(
description="Delete a note or directory by title, permalink, or path",
annotations={"destructiveHint": True, "openWorldHint": False},
)
async def delete_note(
identifier: str,
is_directory: bool = False,
is_directory: Annotated[
bool,
Field(default=False, validation_alias=AliasChoices("is_directory", "is_dir")),
] = False,
project: Optional[str] = None,
workspace: Optional[str] = None,
project_id: Optional[str] = None,
output_format: Literal["text", "json"] = "text",
context: Context | None = None,
) -> bool | str | dict:
@@ -179,6 +216,9 @@ async def delete_note(
(without file extensions). Defaults to False.
project: Project name to delete from. Optional - server will resolve using hierarchy.
If unknown, use list_memory_projects() to discover available projects.
project_id: Project external_id (UUID). Prefer this over `project` when known
it routes to the exact project regardless of name collisions across cloud
workspaces. Takes precedence over `project`. Get from list_memory_projects().
output_format: "text" preserves existing behavior (bool/string). "json"
returns machine-readable deletion metadata.
context: Optional FastMCP context for performance caching.
@@ -224,16 +264,23 @@ async def delete_note(
commands and alternative formats to try.
"""
# Detect project from memory URL prefix before routing
# Trigger: identifier starts with memory:// and no explicit project was provided
# Trigger: identifier starts with memory:// and no explicit project/project_id was provided
# Why: only gate on memory:// to avoid misrouting plain paths like "research/note"
# where "research" is a directory, not a project name
# Outcome: project is set from the URL prefix, routing goes to the correct project
if project is None and identifier.strip().startswith("memory://"):
detected = detect_project_from_url_prefix(identifier, ConfigManager().config)
if project is None and project_id is None and identifier.strip().startswith("memory://"):
detected = await detect_project_from_memory_url_prefix(
identifier,
ConfigManager().config,
context=context,
)
if detected:
project = detected
async with get_project_client(project, workspace, context) as (client, active_project):
async with get_project_client(project, context=context, project_id=project_id) as (
client,
active_project,
):
logger.debug(
f"Deleting {'directory' if is_directory else 'note'}: {identifier} in project: {active_project.name}"
)
@@ -243,11 +290,30 @@ async def delete_note(
# Use typed KnowledgeClient for API calls
knowledge_client = KnowledgeClient(client, active_project.external_id)
_, target_identifier, is_memory_url = await resolve_project_and_path(
client,
identifier,
active_project.name,
context,
)
# Handle directory deletes
if is_directory:
try:
result = await knowledge_client.delete_directory(identifier)
# Trigger: directory input was routed from a memory:// URL.
# Why: resolve_project_and_path returns canonical permalinks, while
# delete_directory filters by project-relative file_path prefixes.
# Outcome: strip only the route prefix before calling the delete API.
directory_identifier = (
_directory_path_for_delete(
target_identifier,
active_project,
include_project_prefix=ConfigManager().config.permalinks_include_project,
)
if is_memory_url
else target_identifier
)
result = await knowledge_client.delete_directory(directory_identifier)
if output_format == "json":
return {
"deleted": result.failed_deletes == 0,
@@ -329,7 +395,7 @@ delete_note("path/to/file.md")
note_file_path = None
try:
# Resolve identifier to entity ID
entity_id = await knowledge_client.resolve_entity(identifier, strict=True)
entity_id = await knowledge_client.resolve_entity(target_identifier, strict=True)
if output_format == "json":
entity = await knowledge_client.get_entity(entity_id)
note_title = entity.title
+248 -218
View File
@@ -1,16 +1,18 @@
"""Edit note tool for Basic Memory MCP server."""
from typing import Optional, Literal
from typing import Annotated, Optional, Literal
import logfire
from loguru import logger
from fastmcp import Context
from pydantic import AliasChoices, Field
from basic_memory.config import ConfigManager
from basic_memory import telemetry
from basic_memory.mcp.project_context import (
detect_project_from_url_prefix,
detect_project_from_memory_url_prefix,
get_project_client,
add_project_metadata,
resolve_project_and_path,
)
from basic_memory.mcp.server import mcp
from basic_memory.schemas.base import Entity
@@ -170,11 +172,33 @@ Error editing note '{identifier}': {error_message}
async def edit_note(
identifier: str,
operation: str,
content: str,
# Accept common replacement-content aliases. Models trained on diff/patch
# APIs reach for new_content/replacement/replace_with on first try.
content: Annotated[
str,
Field(
validation_alias=AliasChoices("content", "new_content", "replacement", "replace_with")
),
],
project: Optional[str] = None,
workspace: Optional[str] = None,
section: Optional[str] = None,
find_text: Optional[str] = None,
project_id: Optional[str] = None,
# Section/heading naming varies across tools; accept the descriptive forms.
section: Annotated[
Optional[str],
Field(
default=None,
validation_alias=AliasChoices("section", "section_heading", "heading"),
),
] = None,
# find_text is the highest-frequency miss per the issue: models reach for
# find/old_text/old_content/search before find_text every time.
find_text: Annotated[
Optional[str],
Field(
default=None,
validation_alias=AliasChoices("find_text", "find", "old_text", "old_content", "search"),
),
] = None,
expected_replacements: Optional[int] = None,
output_format: Literal["text", "json"] = "text",
context: Context | None = None,
@@ -201,6 +225,9 @@ async def edit_note(
content: The content to add or use for replacement
project: Project name to edit in. Optional - server will resolve using hierarchy.
If unknown, use list_memory_projects() to discover available projects.
project_id: Project external_id (UUID). Prefer this over `project` when known
it routes to the exact project regardless of name collisions across cloud
workspaces. Takes precedence over `project`. Get from list_memory_projects().
section: For replace_section operation - the markdown header to replace content under (e.g., "## Notes", "### Implementation")
find_text: For find_replace operation - the text to find and replace
expected_replacements: For find_replace operation - the expected number of replacements (validation will fail if actual doesn't match)
@@ -262,252 +289,255 @@ async def edit_note(
effective_replacements = expected_replacements if expected_replacements is not None else 1
# Detect project from memory URL prefix before routing
# Trigger: identifier starts with memory:// and no explicit project was provided
# Trigger: identifier starts with memory:// and no explicit project/project_id was provided
# Why: only gate on memory:// to avoid misrouting plain paths like "research/note"
# where "research" is a directory, not a project name
# Outcome: project is set from the URL prefix, routing goes to the correct project
if project is None and identifier.strip().startswith("memory://"):
detected = detect_project_from_url_prefix(identifier, ConfigManager().config)
if project is None and project_id is None and identifier.strip().startswith("memory://"):
detected = await detect_project_from_memory_url_prefix(
identifier,
ConfigManager().config,
context=context,
)
if detected:
project = detected
with telemetry.operation(
with logfire.span(
"mcp.tool.edit_note",
entrypoint="mcp",
tool_name="edit_note",
requested_project=project,
workspace_id=workspace,
requested_project_id=project_id,
edit_operation=operation,
output_format=output_format,
has_section=bool(section),
has_find_text=bool(find_text),
expected_replacements=effective_replacements,
):
async with get_project_client(project, workspace, context) as (client, active_project):
with telemetry.contextualize(
project_name=active_project.name,
workspace_id=workspace,
tool_name="edit_note",
):
logger.info(
f"MCP tool call tool=edit_note project={active_project.name} "
f"identifier={identifier} operation={operation} output_format={output_format}"
async with get_project_client(project, context=context, project_id=project_id) as (
client,
active_project,
):
logger.info(
f"MCP tool call tool=edit_note project={active_project.name} "
f"identifier={identifier} operation={operation} output_format={output_format}"
)
# Validate operation
valid_operations = [
"append",
"prepend",
"find_replace",
"replace_section",
"insert_before_section",
"insert_after_section",
]
if operation not in valid_operations:
raise ValueError(
f"Invalid operation '{operation}'. Must be one of: {', '.join(valid_operations)}"
)
# Validate operation
valid_operations = [
"append",
"prepend",
"find_replace",
"replace_section",
"insert_before_section",
"insert_after_section",
]
if operation not in valid_operations:
raise ValueError(
f"Invalid operation '{operation}'. Must be one of: {', '.join(valid_operations)}"
)
# Validate required parameters for specific operations
if operation == "find_replace" and not find_text:
raise ValueError("find_text parameter is required for find_replace operation")
section_ops = ("replace_section", "insert_before_section", "insert_after_section")
if operation in section_ops and not section:
raise ValueError("section parameter is required for section-based operations")
# Validate required parameters for specific operations
if operation == "find_replace" and not find_text:
raise ValueError("find_text parameter is required for find_replace operation")
section_ops = ("replace_section", "insert_before_section", "insert_after_section")
if operation in section_ops and not section:
raise ValueError("section parameter is required for section-based operations")
# Use the PATCH endpoint to edit the entity
try:
# Import here to avoid circular import
from basic_memory.mcp.clients import KnowledgeClient
# Use the PATCH endpoint to edit the entity
# Use typed KnowledgeClient for API calls
knowledge_client = KnowledgeClient(client, active_project.external_id)
_, entity_identifier, _ = await resolve_project_and_path(
client,
identifier,
active_project.name,
context,
)
file_created = False
entity_id = ""
result: EntityResponse | None = None
# Try to resolve the entity; for append/prepend, create it if not found
try:
# Import here to avoid circular import
from basic_memory.mcp.clients import KnowledgeClient
entity_id = await knowledge_client.resolve_entity(
entity_identifier,
strict=True,
)
except Exception as resolve_error:
# Trigger: entity does not exist yet
# Why: append/prepend can meaningfully create a new note from the content,
# while find_replace/replace_section require existing content to modify
# Outcome: note is created via the same path as write_note
error_msg = str(resolve_error).lower()
is_not_found = "entity not found" in error_msg or "not found" in error_msg
# Use typed KnowledgeClient for API calls
knowledge_client = KnowledgeClient(client, active_project.external_id)
if is_not_found and operation in ("append", "prepend"):
title, directory = _parse_identifier_to_title_and_directory(identifier)
file_created = False
entity_id = ""
result: EntityResponse | None = None
# Try to resolve the entity; for append/prepend, create it if not found
try:
entity_id = await knowledge_client.resolve_entity(identifier, strict=True)
except Exception as resolve_error:
# Trigger: entity does not exist yet
# Why: append/prepend can meaningfully create a new note from the content,
# while find_replace/replace_section require existing content to modify
# Outcome: note is created via the same path as write_note
error_msg = str(resolve_error).lower()
is_not_found = "entity not found" in error_msg or "not found" in error_msg
if is_not_found and operation in ("append", "prepend"):
title, directory = _parse_identifier_to_title_and_directory(identifier)
# Validate directory path (same security check as write_note)
project_path = active_project.home
if directory and not validate_project_path(directory, project_path):
logger.warning(
"Attempted path traversal attack blocked",
directory=directory,
project=active_project.name,
)
if output_format == "json":
return {
"title": title,
"permalink": None,
"file_path": None,
"checksum": None,
"operation": operation,
"fileCreated": False,
"error": "SECURITY_VALIDATION_ERROR",
}
return f"# Error\n\nDirectory path '{directory}' is not allowed - paths must stay within project boundaries"
entity = Entity(
title=title,
# Validate directory path (same security check as write_note)
project_path = active_project.home
if directory and not validate_project_path(directory, project_path):
logger.warning(
"Attempted path traversal attack blocked",
directory=directory,
content_type="text/markdown",
content=content,
project=active_project.name,
)
if output_format == "json":
return {
"title": title,
"permalink": None,
"file_path": None,
"checksum": None,
"operation": operation,
"fileCreated": False,
"error": "SECURITY_VALIDATION_ERROR",
}
return f"# Error\n\nDirectory path '{directory}' is not allowed - paths must stay within project boundaries"
logger.info(
"Creating note via edit_note auto-create",
title=title,
directory=directory,
operation=operation,
)
result = await knowledge_client.create_entity(
entity.model_dump(), fast=False
)
file_created = True
else:
# find_replace/replace_section require existing content — re-raise
raise resolve_error
# --- Standard edit path (entity already existed) ---
if not file_created:
# Prepare the edit request data
edit_data = {
"operation": operation,
"content": content,
}
# Add optional parameters
if section:
edit_data["section"] = section
if find_text:
edit_data["find_text"] = find_text
if effective_replacements != 1: # Only send if different from default
edit_data["expected_replacements"] = str(effective_replacements)
# Call the PATCH endpoint
result = await knowledge_client.patch_entity(
entity_id, edit_data, fast=False
entity = Entity(
title=title,
directory=directory,
content_type="text/markdown",
content=content,
)
# --- Format response ---
# result is always set: either by create_entity (auto-create) or patch_entity (edit)
assert result is not None
if file_created:
summary = [
f"# Created note ({operation})",
f"project: {active_project.name}",
f"file_path: {result.file_path}",
f"permalink: {result.permalink}",
f"checksum: {result.checksum[:8] if result.checksum else 'unknown'}",
"fileCreated: true",
]
lines_added = len(content.split("\n"))
summary.append(f"operation: Created note with {lines_added} lines")
logger.info(
"Creating note via edit_note auto-create",
title=title,
directory=directory,
operation=operation,
)
result = await knowledge_client.create_entity(entity.model_dump())
file_created = True
else:
summary = [
f"# Edited note ({operation})",
f"project: {active_project.name}",
f"file_path: {result.file_path}",
f"permalink: {result.permalink}",
f"checksum: {result.checksum[:8] if result.checksum else 'unknown'}",
]
# find_replace/replace_section require existing content — re-raise
raise resolve_error
# Add operation-specific details
if operation == "append":
lines_added = len(content.split("\n"))
summary.append(f"operation: Added {lines_added} lines to end of note")
elif operation == "prepend":
lines_added = len(content.split("\n"))
summary.append(
f"operation: Added {lines_added} lines to beginning of note"
)
elif operation == "find_replace":
# For find_replace, we can't easily count replacements from here
# since we don't have the original content, but the server handled it
summary.append("operation: Find and replace operation completed")
elif operation == "replace_section":
summary.append(f"operation: Replaced content under section '{section}'")
elif operation == "insert_before_section":
summary.append(
f"operation: Inserted content before section '{section}'"
)
elif operation == "insert_after_section":
summary.append(f"operation: Inserted content after section '{section}'")
# --- Standard edit path (entity already existed) ---
if not file_created:
# Prepare the edit request data
edit_data = {
"operation": operation,
"content": content,
}
# Count observations by category (reuse logic from write_note)
categories = {}
if result.observations:
for obs in result.observations:
categories[obs.category] = categories.get(obs.category, 0) + 1
# Add optional parameters
if section:
edit_data["section"] = section
if find_text:
edit_data["find_text"] = find_text
if effective_replacements != 1: # Only send if different from default
edit_data["expected_replacements"] = str(effective_replacements)
summary.append("\n## Observations")
for category, count in sorted(categories.items()):
summary.append(f"- {category}: {count}")
# Call the PATCH endpoint
result = await knowledge_client.patch_entity(entity_id, edit_data)
# Count resolved/unresolved relations
unresolved = 0
resolved = 0
if result.relations:
unresolved = sum(1 for r in result.relations if not r.to_id)
resolved = len(result.relations) - unresolved
# --- Format response ---
# result is always set: either by create_entity (auto-create) or patch_entity (edit)
assert result is not None
if file_created:
summary = [
f"# Created note ({operation})",
f"project: {active_project.name}",
f"file_path: {result.file_path}",
f"permalink: {result.permalink}",
f"checksum: {result.checksum[:8] if result.checksum else 'unknown'}",
"fileCreated: true",
]
lines_added = len(content.split("\n"))
summary.append(f"operation: Created note with {lines_added} lines")
else:
summary = [
f"# Edited note ({operation})",
f"project: {active_project.name}",
f"file_path: {result.file_path}",
f"permalink: {result.permalink}",
f"checksum: {result.checksum[:8] if result.checksum else 'unknown'}",
]
summary.append("\n## Relations")
summary.append(f"- Resolved: {resolved}")
if unresolved:
summary.append(f"- Unresolved: {unresolved}")
# Add operation-specific details
if operation == "append":
lines_added = len(content.split("\n"))
summary.append(f"operation: Added {lines_added} lines to end of note")
elif operation == "prepend":
lines_added = len(content.split("\n"))
summary.append(f"operation: Added {lines_added} lines to beginning of note")
elif operation == "find_replace":
# For find_replace, we can't easily count replacements from here
# since we don't have the original content, but the server handled it
summary.append("operation: Find and replace operation completed")
elif operation == "replace_section":
summary.append(f"operation: Replaced content under section '{section}'")
elif operation == "insert_before_section":
summary.append(f"operation: Inserted content before section '{section}'")
elif operation == "insert_after_section":
summary.append(f"operation: Inserted content after section '{section}'")
logger.info(
f"MCP tool response: tool=edit_note project={active_project.name} "
f"operation={operation} permalink={result.permalink} "
f"observations_count={len(result.observations)} "
f"relations_count={len(result.relations)} "
f"file_created={str(file_created).lower()}"
)
# Count observations by category (reuse logic from write_note)
categories = {}
if result.observations:
for obs in result.observations:
categories[obs.category] = categories.get(obs.category, 0) + 1
if output_format == "json":
return {
"title": result.title,
"permalink": result.permalink,
"file_path": result.file_path,
"checksum": result.checksum,
"operation": operation,
"fileCreated": file_created,
}
summary.append("\n## Observations")
for category, count in sorted(categories.items()):
summary.append(f"- {category}: {count}")
summary_result = "\n".join(summary)
return add_project_metadata(summary_result, active_project.name)
# Count resolved/unresolved relations
unresolved = 0
resolved = 0
if result.relations:
unresolved = sum(1 for r in result.relations if not r.to_id)
resolved = len(result.relations) - unresolved
except Exception as e:
logger.error(f"Error editing note: {e}")
if output_format == "json":
return {
"title": None,
"permalink": None,
"file_path": None,
"checksum": None,
"operation": operation,
"fileCreated": False,
"error": str(e),
}
return _format_error_response(
str(e),
operation,
identifier,
find_text,
effective_replacements,
active_project.name,
)
summary.append("\n## Relations")
summary.append(f"- Resolved: {resolved}")
if unresolved:
summary.append(f"- Unresolved: {unresolved}")
logger.info(
f"MCP tool response: tool=edit_note project={active_project.name} "
f"operation={operation} permalink={result.permalink} "
f"observations_count={len(result.observations)} "
f"relations_count={len(result.relations)} "
f"file_created={str(file_created).lower()}"
)
if output_format == "json":
return {
"title": result.title,
"permalink": result.permalink,
"file_path": result.file_path,
"checksum": result.checksum,
"operation": operation,
"fileCreated": file_created,
}
summary_result = "\n".join(summary)
return add_project_metadata(summary_result, active_project.name)
except Exception as e:
logger.error(f"Error editing note: {e}")
if output_format == "json":
return {
"title": None,
"permalink": None,
"file_path": None,
"checksum": None,
"operation": operation,
"fileCreated": False,
"error": str(e),
}
return _format_error_response(
str(e),
operation,
identifier,
find_text,
effective_replacements,
active_project.name,
)
+25 -5
View File
@@ -1,9 +1,10 @@
"""List directory tool for Basic Memory MCP server."""
from typing import Optional
from typing import Annotated, Optional
from loguru import logger
from fastmcp import Context
from pydantic import AliasChoices, Field
from basic_memory.mcp.project_context import get_project_client
from basic_memory.mcp.server import mcp
@@ -14,11 +15,24 @@ from basic_memory.mcp.server import mcp
annotations={"readOnlyHint": True, "openWorldHint": False},
)
async def list_directory(
dir_name: str = "/",
# `dir_name` is unusual; models reach for directory/folder/path/dir.
dir_name: Annotated[
str,
Field(
default="/",
validation_alias=AliasChoices("dir_name", "directory", "folder", "path", "dir"),
),
] = "/",
depth: int = 1,
file_name_glob: Optional[str] = None,
file_name_glob: Annotated[
Optional[str],
Field(
default=None,
validation_alias=AliasChoices("file_name_glob", "glob", "pattern", "filter"),
),
] = None,
project: Optional[str] = None,
workspace: Optional[str] = None,
project_id: Optional[str] = None,
context: Context | None = None,
) -> str:
"""List directory contents from the knowledge base with optional filtering.
@@ -36,6 +50,9 @@ async def list_directory(
Examples: "*.md", "*meeting*", "project_*"
project: Project name to list directory from. Optional - server will resolve using hierarchy.
If unknown, use list_memory_projects() to discover available projects.
project_id: Project external_id (UUID). Prefer this over `project` when known
it routes to the exact project regardless of name collisions across cloud
workspaces. Takes precedence over `project`. Get from list_memory_projects().
context: Optional FastMCP context for performance caching.
Returns:
@@ -63,7 +80,10 @@ async def list_directory(
Raises:
ToolError: If project doesn't exist or directory path is invalid
"""
async with get_project_client(project, workspace, context) as (client, active_project):
async with get_project_client(project, context=context, project_id=project_id) as (
client,
active_project,
):
logger.debug(
f"Listing directory '{dir_name}' in project {project} with depth={depth}, glob='{file_name_glob}'"
)
+31 -6
View File
@@ -2,11 +2,12 @@
from pathlib import Path, PureWindowsPath
from textwrap import dedent
from typing import Optional, Literal
from typing import Annotated, Optional, Literal
from loguru import logger
from fastmcp import Context
from mcp.server.fastmcp.exceptions import ToolError
from pydantic import AliasChoices, Field
from basic_memory.mcp.server import mcp
from basic_memory.mcp.project_context import get_project_client
@@ -348,11 +349,29 @@ delete_note("{identifier}")
)
async def move_note(
identifier: str,
destination_path: str = "",
destination_folder: Optional[str] = None,
is_directory: bool = False,
# Move/rename APIs across the ecosystem use `to`/`destination`/`new_path`.
destination_path: Annotated[
str,
Field(
default="",
validation_alias=AliasChoices(
"destination_path", "dest_path", "new_path", "to", "destination"
),
),
] = "",
destination_folder: Annotated[
Optional[str],
Field(
default=None,
validation_alias=AliasChoices("destination_folder", "dest_folder", "to_folder"),
),
] = None,
is_directory: Annotated[
bool,
Field(default=False, validation_alias=AliasChoices("is_directory", "is_dir")),
] = False,
project: Optional[str] = None,
workspace: Optional[str] = None,
project_id: Optional[str] = None,
output_format: Literal["text", "json"] = "text",
context: Context | None = None,
) -> str | dict:
@@ -377,6 +396,9 @@ async def move_note(
(without file extensions). Defaults to False.
project: Project name to move within. Optional - server will resolve using hierarchy.
If unknown, use list_memory_projects() to discover available projects.
project_id: Project external_id (UUID). Prefer this over `project` when known
it routes to the exact project regardless of name collisions across cloud
workspaces. Takes precedence over `project`. Get from list_memory_projects().
output_format: "text" returns existing markdown guidance/success text. "json"
returns machine-readable move metadata.
context: Optional FastMCP context for performance caching.
@@ -476,7 +498,10 @@ async def move_note(
"error": "DESTINATION_FOLDER_NOT_FOR_DIRECTORIES",
}
return f"# Move Failed - Invalid Parameters\n\n{error_msg}"
async with get_project_client(project, workspace, context) as (client, active_project):
async with get_project_client(project, context=context, project_id=project_id) as (
client,
active_project,
):
destination_target = destination_folder or destination_path
logger.info(
f"MCP tool call tool=move_note project={active_project.name} "
+211 -66
View File
@@ -11,7 +11,17 @@ from fastmcp import Context
from loguru import logger
from basic_memory.config import ConfigManager, has_cloud_credentials
from basic_memory.mcp.async_client import get_client, get_cloud_proxy_client, is_factory_mode
from basic_memory.mcp.async_client import (
_explicit_routing,
_force_local_mode,
get_client,
is_factory_mode,
)
from basic_memory.mcp.project_context import (
WorkspaceProjectEntry,
ensure_workspace_project_index,
resolve_workspace_parameter,
)
from basic_memory.mcp.server import mcp
from basic_memory.schemas.project_info import ProjectInfoRequest, ProjectItem, ProjectList
from basic_memory.utils import generate_permalink
@@ -20,30 +30,6 @@ from basic_memory.utils import generate_permalink
# --- Helpers for dual-fetch + merge ---
async def _fetch_cloud_projects(
workspace: str | None = None,
context: Context | None = None,
) -> ProjectList | None:
"""Fetch projects from the cloud API, returning None on failure.
Logs warnings on failure so the caller can fall back to local-only results.
"""
try:
from basic_memory.mcp.clients import ProjectClient
async with get_cloud_proxy_client(workspace=workspace) as cloud_client:
cloud_project_client = ProjectClient(cloud_client)
cloud_list = await cloud_project_client.list_projects()
if context: # pragma: no cover
await context.info(f"Discovered {len(cloud_list.projects)} cloud projects")
return cloud_list
except Exception as exc:
logger.warning(f"Cloud project discovery failed: {exc}")
if context: # pragma: no cover
await context.info("Cloud project discovery failed, showing local projects only")
return None
def _merge_projects(
local_list: ProjectList | None,
cloud_list: ProjectList | None,
@@ -51,6 +37,8 @@ def _merge_projects(
cloud_workspace_name: str | None = None,
cloud_workspace_type: str | None = None,
cloud_workspace_tenant_id: str | None = None,
cloud_workspace_slug: str | None = None,
cloud_workspace_is_default: bool = False,
) -> list[dict]:
"""Merge local and cloud project lists by permalink.
@@ -113,9 +101,13 @@ def _merge_projects(
ws_type = cloud_workspace_type if cloud_proj else None
ws_tenant_id = cloud_workspace_tenant_id if cloud_proj else None
proj = cloud_proj or local_proj
external_id = proj.external_id if proj else None
merged.append(
{
"name": name,
"external_id": external_id,
"path": path,
"local_path": local_path,
"cloud_path": cloud_path,
@@ -126,21 +118,120 @@ def _merge_projects(
"workspace_name": ws_name,
"workspace_type": ws_type,
"workspace_tenant_id": ws_tenant_id,
"workspace_slug": cloud_workspace_slug if cloud_proj else None,
"workspace_is_default": cloud_workspace_is_default if cloud_proj else False,
"qualified_name": (
f"{cloud_workspace_slug}/{permalink}"
if cloud_proj and cloud_workspace_slug
else None
),
}
)
return merged
def _merge_workspace_projects(
local_list: ProjectList | None,
cloud_entries: tuple[WorkspaceProjectEntry, ...],
) -> list[dict]:
"""Merge local projects with cloud projects from every accessible workspace."""
local_by_permalink: dict[str, ProjectItem] = {}
if local_list:
for project in local_list.projects:
local_by_permalink[project.permalink] = project
cloud_permalinks = {entry.project.permalink for entry in cloud_entries}
merged: list[dict] = []
for entry in sorted(
cloud_entries,
key=lambda item: (
not item.workspace.is_default,
item.workspace.workspace_type != "personal",
item.workspace.name.casefold(),
item.project.permalink,
),
):
permalink = entry.project.permalink
local_proj = local_by_permalink.get(permalink)
cloud_proj = entry.project
source = "local+cloud" if local_proj else "cloud"
local_path = local_proj.path if local_proj else None
cloud_path = cloud_proj.path
merged.append(
{
"name": cloud_proj.name,
"external_id": cloud_proj.external_id,
"path": local_path or cloud_path,
"local_path": local_path,
"cloud_path": cloud_path,
"source": source,
"is_default": bool((local_proj and local_proj.is_default) or cloud_proj.is_default),
"is_private": cloud_proj.is_private,
"display_name": cloud_proj.display_name,
"workspace_name": entry.workspace.name,
"workspace_type": entry.workspace.workspace_type,
"workspace_tenant_id": entry.workspace.tenant_id,
"workspace_slug": entry.workspace.slug,
"workspace_is_default": entry.workspace.is_default,
"qualified_name": entry.qualified_name,
}
)
if local_list:
for project in sorted(local_list.projects, key=lambda item: item.permalink):
if project.permalink in cloud_permalinks:
continue
merged.append(
{
"name": project.name,
"external_id": project.external_id,
"path": project.path,
"local_path": project.path,
"cloud_path": None,
"source": "local",
"is_default": project.is_default,
"is_private": project.is_private,
"display_name": project.display_name,
"workspace_name": None,
"workspace_type": None,
"workspace_tenant_id": None,
"workspace_slug": None,
"workspace_is_default": False,
"qualified_name": None,
}
)
return merged
def _format_project_list_text(merged: list[dict]) -> str:
"""Format merged project list as human-readable text."""
result = "Available projects:\n"
current_workspace: tuple[str | None, str | None] | None = None
for project in merged:
workspace_slug = project.get("workspace_slug")
workspace_name = project.get("workspace_name")
if workspace_slug:
workspace_key = (workspace_slug, workspace_name)
if workspace_key != current_workspace:
default_label = " default" if project.get("workspace_is_default") else ""
result += f"\nWorkspace: {workspace_name} ({workspace_slug}{default_label})\n"
current_workspace = workspace_key
elif current_workspace is not None:
result += "\nLocal projects:\n"
current_workspace = None
display_name = project["display_name"]
name = project["name"]
label = f"{display_name} ({name})" if display_name else name
source = project["source"]
result += f"{label} ({source})\n"
external_id = project.get("external_id", "")
id_suffix = f" [{external_id}]" if external_id else ""
result += f"- {label} ({source}){id_suffix}\n"
result += "\n" + "" * 40 + "\n"
result += "Next: Ask which project to use for this session.\n"
@@ -172,7 +263,6 @@ def _format_project_list_json(
)
async def list_memory_projects(
output_format: Literal["text", "json"] = "text",
workspace: str | None = None,
context: Context | None = None,
) -> str | dict:
"""List all available projects with their status.
@@ -180,11 +270,14 @@ async def list_memory_projects(
Shows projects from both local and cloud sources when cloud credentials
are available, merging by permalink to give a unified view.
Each project entry includes an `external_id` (UUID). Pass that value as the
`project_id` parameter on other tools to address a specific project
unambiguously across cloud workspaces useful when the same project name
exists in more than one workspace.
Args:
output_format: "text" returns the existing human-readable project list.
"json" returns structured project metadata.
workspace: Cloud workspace name or tenant_id. Falls back to
config.default_workspace when not specified.
context: Optional FastMCP context for progress/status logging.
"""
if context: # pragma: no cover
@@ -196,18 +289,21 @@ async def list_memory_projects(
# --- Factory mode (cloud app) ---
# Trigger: set_client_factory() was called (e.g., basic-memory-cloud)
# Why: there is no local ASGI server; the factory IS the only source
# Outcome: single fetch, no merge needed
# Why: there is no local ASGI server; the factory IS the cloud source
# Outcome: fetch every accessible workspace so callers can discover cross-workspace IDs
if is_factory_mode():
async with get_client() as client:
project_client = ProjectClient(client)
project_list = await project_client.list_projects()
merged = _merge_projects(project_list, None)
workspace_index = await ensure_workspace_project_index(context=context)
merged = _merge_workspace_projects(None, workspace_index.entries)
default_project = next(
(
entry.project.name
for entry in workspace_index.entries
if entry.workspace.is_default and entry.project.is_default
),
None,
)
if output_format == "json":
return _format_project_list_json(
merged, project_list.default_project, constrained_project
)
return _format_project_list_json(merged, default_project, constrained_project)
if constrained_project:
return _format_constrained_text(constrained_project)
return _format_project_list_text(merged)
@@ -220,39 +316,40 @@ async def list_memory_projects(
# Fetch cloud projects when credentials are available
cloud_list: ProjectList | None = None
cloud_entries: tuple[WorkspaceProjectEntry, ...] = ()
cloud_ws_name: str | None = None
cloud_ws_type: str | None = None
cloud_ws_tenant_id: str | None = None
cloud_ws_slug: str | None = None
cloud_ws_is_default = False
config = ConfigManager().config
if has_cloud_credentials(config):
# Use explicit workspace, fall back to config default
effective_workspace = workspace or config.default_workspace
cloud_list = await _fetch_cloud_projects(effective_workspace, context)
# Resolve workspace metadata so each cloud project carries its workspace info
if cloud_list:
cloud_ws_tenant_id = effective_workspace
try:
from basic_memory.mcp.project_context import get_available_workspaces
workspaces = await get_available_workspaces(context)
matched = next(
(ws for ws in workspaces if ws.tenant_id == effective_workspace),
None,
try:
workspace_index = await ensure_workspace_project_index(context=context)
cloud_entries = workspace_index.entries
except Exception as exc:
logger.warning(
f"Cloud workspace project index discovery failed while listing projects; "
f"showing local-only project list: {exc}"
)
if context: # pragma: no cover
await context.info(
"Cloud workspace project discovery failed while listing projects; "
"showing local projects only"
)
if matched:
cloud_ws_name = matched.name
cloud_ws_type = matched.workspace_type
except Exception:
pass # workspace lookup is best-effort
merged = _merge_projects(
local_list,
cloud_list,
cloud_workspace_name=cloud_ws_name,
cloud_workspace_type=cloud_ws_type,
cloud_workspace_tenant_id=cloud_ws_tenant_id,
)
if cloud_entries:
merged = _merge_workspace_projects(local_list, cloud_entries)
else:
merged = _merge_projects(
local_list,
cloud_list,
cloud_workspace_name=cloud_ws_name,
cloud_workspace_type=cloud_ws_type,
cloud_workspace_tenant_id=cloud_ws_tenant_id,
cloud_workspace_slug=cloud_ws_slug,
cloud_workspace_is_default=cloud_ws_is_default,
)
default_project = local_list.default_project
if output_format == "json":
@@ -272,6 +369,31 @@ def _format_constrained_text(constrained_project: str) -> str:
return result
async def _resolve_create_project_workspace(
workspace: str | None,
context: Context | None,
) -> str | None:
"""Resolve the create-project workspace selector to the routing tenant id."""
if workspace is None:
return None
explicit_cloud_routing = _explicit_routing() and not _force_local_mode()
config = ConfigManager().config
should_resolve_workspace = is_factory_mode() or (
explicit_cloud_routing and has_cloud_credentials(config)
)
if not should_resolve_workspace:
return workspace
# Trigger: cloud routing can use workspace discovery and the caller supplied
# a friendly selector such as a slug, name, or tenant id.
# Why: MCP callers should not need to paste UUIDs, but the transport still
# uses X-Workspace-ID with the tenant id as its routing authority.
# Outcome: resolve once at create time and pass only the tenant id downstream.
resolved_workspace = await resolve_workspace_parameter(workspace=workspace, context=context)
return resolved_workspace.tenant_id
@mcp.tool(
"create_memory_project",
annotations={"destructiveHint": False, "openWorldHint": False},
@@ -280,6 +402,7 @@ async def create_memory_project(
project_name: str,
project_path: str,
set_default: bool = False,
workspace: str | None = None,
output_format: Literal["text", "json"] = "text",
context: Context | None = None,
) -> str | dict:
@@ -292,6 +415,11 @@ async def create_memory_project(
project_name: Name for the new project (must be unique)
project_path: File system path where the project will be stored
set_default: Whether to set this project as the default (optional, defaults to False)
workspace: Optional cloud workspace selector to create the project in. Slug is
preferred for AI callers, but tenant_id and unique name are also accepted.
When omitted, the connection's default workspace is used. Discover values
via `list_workspaces`. Only meaningful in cloud mode; ignored for local
projects.
output_format: "text" returns the existing human-readable result text.
"json" returns structured project creation metadata.
context: Optional FastMCP context for progress/status logging.
@@ -302,8 +430,15 @@ async def create_memory_project(
Example:
create_memory_project("my-research", "~/Documents/research")
create_memory_project("work-notes", "/home/user/work", set_default=True)
create_memory_project("team-notes", "/team/notes", workspace="team-paul")
"""
async with get_client() as client:
workspace_id = await _resolve_create_project_workspace(workspace, context)
# workspace targets a non-default cloud workspace at create time.
# Trigger: caller passed workspace (e.g. a slug discovered via list_workspaces).
# Why: there is no project_id yet for per-project routing — the project doesn't exist.
# Outcome: cloud factory routes the create request to the resolved workspace tenant id.
async with get_client(workspace=workspace_id) as client:
# Check if server is constrained to a specific project
constrained_project = os.environ.get("BASIC_MEMORY_MCP_PROJECT")
if constrained_project:
@@ -347,6 +482,7 @@ async def create_memory_project(
if output_format == "json":
return {
"name": existing_match.name,
"external_id": existing_match.external_id,
"path": existing_match.path,
"is_default": is_default,
"created": False,
@@ -356,17 +492,22 @@ async def create_memory_project(
f"✓ Project already exists: {existing_match.name}\n\n"
f"Project Details:\n"
f"• Name: {existing_match.name}\n"
f"• External ID: {existing_match.external_id}\n"
f"• Path: {existing_match.path}\n"
f"{'• Set as default project\n' if is_default else ''}"
"\nProject is already available for use in tool calls.\n"
)
status_response = await project_client.create_project(project_request.model_dump())
from basic_memory.mcp.project_context import invalidate_workspace_project_index
await invalidate_workspace_project_index(context)
if output_format == "json":
new_project = status_response.new_project
return {
"name": new_project.name if new_project else project_name,
"external_id": new_project.external_id if new_project else None,
"path": new_project.path if new_project else project_path,
"is_default": bool(
(new_project.is_default if new_project else False) or set_default
@@ -380,6 +521,7 @@ async def create_memory_project(
if status_response.new_project:
result += "Project Details:\n"
result += f"• Name: {status_response.new_project.name}\n"
result += f"• External ID: {status_response.new_project.external_id}\n"
result += f"• Path: {status_response.new_project.path}\n"
if set_default:
@@ -451,6 +593,9 @@ async def delete_project(project_name: str, context: Context | None = None) -> s
# Delete project using project external_id
status_response = await project_client.delete_project(target_project.external_id)
from basic_memory.mcp.project_context import invalidate_workspace_project_index
await invalidate_workspace_project_index(context)
result = f"{status_response.message}\n\n"
+27 -10
View File
@@ -8,16 +8,17 @@ Files are read directly without any knowledge graph processing.
import base64
import io
from typing import Optional
from typing import Annotated, Optional
from loguru import logger
from PIL import Image as PILImage
from fastmcp import Context
from pydantic import AliasChoices, Field
from mcp.server.fastmcp.exceptions import ToolError
from basic_memory.config import ConfigManager
from basic_memory.mcp.project_context import (
detect_project_from_url_prefix,
detect_project_from_memory_url_prefix,
get_project_client,
resolve_project_and_path,
)
@@ -158,9 +159,12 @@ def optimize_image(img, content_length, max_output_bytes=350000):
annotations={"readOnlyHint": True, "openWorldHint": False},
)
async def read_content(
path: str,
path: Annotated[
str,
Field(validation_alias=AliasChoices("path", "file_path", "filepath", "file")),
],
project: Optional[str] = None,
workspace: Optional[str] = None,
project_id: Optional[str] = None,
context: Context | None = None,
) -> dict:
"""Read a file's raw content by path or permalink.
@@ -181,6 +185,9 @@ async def read_content(
- A permalink (docs/example)
project: Project name to read from. Optional - server will resolve using hierarchy.
If unknown, use list_memory_projects() to discover available projects.
project_id: Project external_id (UUID). Prefer this over `project` when known
it routes to the exact project regardless of name collisions across cloud
workspaces. Takes precedence over `project`. Get from list_memory_projects().
context: Optional FastMCP context for performance caching.
Returns:
@@ -210,17 +217,27 @@ async def read_content(
HTTPError: If project doesn't exist or is inaccessible
SecurityError: If path attempts path traversal
"""
# Detect project from memory URL prefix before routing
if project is None:
detected = detect_project_from_url_prefix(path, ConfigManager().config)
# Detect project from memory URL prefix before routing.
# project_id routes by external UUID, so it bypasses URL discovery entirely.
if project is None and project_id is None:
detected = await detect_project_from_memory_url_prefix(
path,
ConfigManager().config,
context=context,
)
if detected:
project = detected
logger.info(f"MCP tool call tool=read_content project={project} path={path}")
async with get_project_client(project, workspace, context) as (client, active_project):
# Resolve path with project-prefix awareness for memory:// URLs
_, url, _ = await resolve_project_and_path(client, path, project, context)
async with get_project_client(project, context=context, project_id=project_id) as (
client,
active_project,
):
# Resolve path with project-prefix awareness for memory:// URLs.
# Use active_project.name so resolution stays consistent when project_id
# was used or `project` was wrong/ambiguous (matches the cached resolution).
_, url, _ = await resolve_project_and_path(client, path, active_project.name, context)
# Validate path to prevent path traversal attacks
# For memory:// URLs, validate the extracted path (not the raw URL which
+210 -205
View File
@@ -1,17 +1,17 @@
"""Read note tool for Basic Memory MCP server."""
from textwrap import dedent
from typing import Optional, Literal
from typing import Optional, Literal, cast
import logfire
import yaml
from loguru import logger
from fastmcp import Context
from basic_memory import telemetry
from basic_memory.config import ConfigManager
from basic_memory.mcp.project_context import (
detect_project_from_url_prefix,
detect_project_from_memory_url_prefix,
get_project_client,
resolve_project_and_path,
)
@@ -33,10 +33,10 @@ def _parse_opening_frontmatter(content: str) -> tuple[str, dict | None]:
If parsing fails or frontmatter is not a mapping, returns body unchanged and None.
"""
original_content = content
if not content.startswith("---\n"):
lines = content.splitlines(keepends=True)
if not lines or lines[0].strip() != "---":
return original_content, None
lines = content.splitlines(keepends=True)
closing_index = None
for i in range(1, len(lines)):
if lines[i].strip() == "---":
@@ -70,9 +70,7 @@ def _parse_opening_frontmatter(content: str) -> tuple[str, dict | None]:
async def read_note(
identifier: str,
project: Optional[str] = None,
workspace: Optional[str] = None,
page: int = 1,
page_size: int = 10,
project_id: Optional[str] = None,
output_format: Literal["text", "json"] = "text",
include_frontmatter: bool = False,
context: Context | None = None,
@@ -96,10 +94,11 @@ async def read_note(
project: Project name to read from. Optional - server will resolve using the
hierarchy above. If unknown, use list_memory_projects() to discover
available projects.
project_id: Project external_id (UUID). Prefer this over `project` when known
it routes to the exact project regardless of name collisions across cloud
workspaces. Takes precedence over `project`. Get from list_memory_projects().
identifier: The title or permalink of the note to read
Can be a full memory:// URL, a permalink, a title, or search text
page: Page number for paginated results (default: 1)
page_size: Number of items per page (default: 10)
output_format: "text" returns markdown content or guidance text.
"json" returns a structured object with title/permalink/file_path/content/frontmatter.
include_frontmatter: When output_format="json", whether content should include the
@@ -120,9 +119,6 @@ async def read_note(
# Read with memory URL
read_note("my-research", "memory://specs/search-spec")
# Read with pagination
read_note("work-project", "Project Updates", page=2, page_size=5)
# Read recent meeting notes
read_note("team-docs", "Weekly Standup")
@@ -134,230 +130,239 @@ async def read_note(
If the exact note isn't found, this tool provides helpful suggestions
including related notes, search commands, and note creation templates.
"""
# Detect project from memory URL prefix before routing
if project is None:
detected = detect_project_from_url_prefix(identifier, ConfigManager().config)
# Detect project from memory URL prefix before routing.
# project_id routes by external UUID, so it bypasses URL discovery entirely.
if project is None and project_id is None:
detected = await detect_project_from_memory_url_prefix(
identifier,
ConfigManager().config,
context=context,
)
if detected:
project = detected
with telemetry.operation(
with logfire.span(
"mcp.tool.read_note",
entrypoint="mcp",
tool_name="read_note",
requested_project=project,
workspace_id=workspace,
requested_project_id=project_id,
output_format=output_format,
page=page,
page_size=page_size,
include_frontmatter=include_frontmatter,
):
async with get_project_client(project, workspace, context) as (client, active_project):
with telemetry.contextualize(
project_name=active_project.name,
workspace_id=workspace,
tool_name="read_note",
async with get_project_client(project, context=context, project_id=project_id) as (
client,
active_project,
):
# Resolve identifier with project-prefix awareness for memory:// URLs.
# Pass active_project.name (the canonical resolved name) rather than the
# original `project` arg so the inner get_active_project cache hits even
# when project_id was used or `project` was wrong/ambiguous.
_, entity_path, _ = await resolve_project_and_path(
client, identifier, active_project.name, context
)
# Validate identifier to prevent path traversal attacks
# For memory:// URLs, validate the extracted path (not the raw URL which
# has a scheme prefix that confuses path validation)
raw_path = (
memory_url_path(identifier) if identifier.startswith("memory://") else identifier
)
processed_path = entity_path
project_path = active_project.home
if not validate_project_path(raw_path, project_path) or not validate_project_path(
processed_path, project_path
):
# Resolve identifier with project-prefix awareness for memory:// URLs
_, entity_path, _ = await resolve_project_and_path(
client, identifier, project, context
logger.warning(
"Attempted path traversal attack blocked",
identifier=identifier,
processed_path=processed_path,
project=active_project.name,
)
# Validate identifier to prevent path traversal attacks
# For memory:// URLs, validate the extracted path (not the raw URL which
# has a scheme prefix that confuses path validation)
raw_path = (
memory_url_path(identifier)
if identifier.startswith("memory://")
else identifier
)
processed_path = entity_path
project_path = active_project.home
if not validate_project_path(raw_path, project_path) or not validate_project_path(
processed_path, project_path
):
logger.warning(
"Attempted path traversal attack blocked",
identifier=identifier,
processed_path=processed_path,
project=active_project.name,
)
if output_format == "json":
return {
"title": None,
"permalink": None,
"file_path": None,
"content": None,
"frontmatter": None,
"error": "SECURITY_VALIDATION_ERROR",
}
return f"# Error\n\nIdentifier '{identifier}' is not allowed - paths must stay within project boundaries"
# Get the file via REST API - first try direct identifier resolution
logger.info(
f"Attempting to read note from Project: {active_project.name} identifier: {entity_path}"
)
# Import here to avoid circular import
from basic_memory.mcp.clients import KnowledgeClient, ResourceClient
# Use typed clients for API calls
knowledge_client = KnowledgeClient(client, active_project.external_id)
resource_client = ResourceClient(client, active_project.external_id)
async def _read_json_payload(entity_id: str) -> dict:
with telemetry.scope(
"mcp.read_note.shape_response",
domain="mcp",
action="read_note",
phase="shape_response",
):
entity = await knowledge_client.get_entity(entity_id)
response = await resource_client.read(
entity_id, page=page, page_size=page_size
)
content_text = response.text
body_content, parsed_frontmatter = _parse_opening_frontmatter(content_text)
return {
"title": entity.title,
"permalink": entity.permalink,
"file_path": entity.file_path,
"content": content_text if include_frontmatter else body_content,
"frontmatter": parsed_frontmatter,
}
def _empty_json_payload() -> dict:
if output_format == "json":
return {
"title": None,
"permalink": None,
"file_path": None,
"content": None,
"frontmatter": None,
"error": "SECURITY_VALIDATION_ERROR",
}
return f"# Error\n\nIdentifier '{identifier}' is not allowed - paths must stay within project boundaries"
# Get the file via REST API - first try direct identifier resolution
logger.info(
f"Attempting to read note from Project: {active_project.name} identifier: {entity_path}"
)
# Import here to avoid circular import
from basic_memory.mcp.clients import KnowledgeClient, ResourceClient
# Use typed clients for API calls
knowledge_client = KnowledgeClient(client, active_project.external_id)
resource_client = ResourceClient(client, active_project.external_id)
async def _read_json_payload(entity_id: str) -> dict:
with logfire.span(
"mcp.read_note.shape_response",
domain="mcp",
action="read_note",
phase="shape_response",
):
entity = await knowledge_client.get_entity(entity_id)
response = await resource_client.read(entity_id)
content_text = response.text
body_content, parsed_frontmatter = _parse_opening_frontmatter(content_text)
return {
"title": entity.title,
"permalink": entity.permalink,
"file_path": entity.file_path,
"content": content_text if include_frontmatter else body_content,
"frontmatter": parsed_frontmatter,
}
def _search_results(payload: object) -> list[dict]:
if not isinstance(payload, dict):
return []
results = payload.get("results")
return results if isinstance(results, list) else []
def _empty_json_payload() -> dict:
return {
"title": None,
"permalink": None,
"file_path": None,
"content": None,
"frontmatter": None,
}
async def _search_candidates(identifier_text: str, *, title_only: bool) -> dict:
# Trigger: direct entity resolution failed for the caller's identifier.
# Why: search_notes applies the same memory:// normalization and tool-level
# query handling as the rest of MCP routing, which raw client calls skip.
# Outcome: unresolved memory URLs still fall back through normalized search.
search_type = "title" if title_only else "text"
response = await search_notes(
project=active_project.name,
workspace=workspace,
query=identifier_text,
search_type=search_type,
page=page,
page_size=page_size,
output_format="json",
context=context,
)
return response if isinstance(response, dict) else {}
def _search_results(payload: object) -> list[dict[str, object]]:
if not isinstance(payload, dict):
return []
payload_dict = cast(dict[str, object], payload)
results = payload_dict.get("results")
if not isinstance(results, list):
return []
return [
cast(dict[str, object], result)
for result in results
if isinstance(result, dict)
]
def _result_title(item: dict) -> str:
return str(item.get("title") or "")
async def _search_candidates(
identifier_text: str, *, title_only: bool
) -> dict[str, object]:
# Trigger: direct entity resolution failed for the caller's identifier.
# Why: search_notes applies the same memory:// normalization and tool-level
# query handling as the rest of MCP routing, which raw client calls skip.
# Outcome: unresolved memory URLs still fall back through normalized search.
# Pass project_id (external_id UUID) so the workspace selection from the
# outer get_project_client() is preserved across the inner re-resolution.
# Without this, project names that collide across workspaces could re-resolve
# to a different tenant via the default-workspace fallback (CLI/context=None).
search_type = "title" if title_only else "text"
response = await search_notes(
project=active_project.name,
project_id=active_project.external_id,
query=identifier_text,
search_type=search_type,
output_format="json",
context=context,
)
return cast(dict[str, object], response) if isinstance(response, dict) else {}
def _result_permalink(item: dict) -> Optional[str]:
value = item.get("permalink")
return str(value) if value else None
def _result_title(item: dict[str, object]) -> str:
return str(item.get("title") or "")
def _result_file_path(item: dict) -> Optional[str]:
value = item.get("file_path")
return str(value) if value else None
def _result_permalink(item: dict[str, object]) -> Optional[str]:
value = item.get("permalink")
return str(value) if value else None
try:
# Try to resolve identifier to entity ID
entity_id = await knowledge_client.resolve_entity(entity_path, strict=True)
def _result_file_path(item: dict[str, object]) -> Optional[str]:
value = item.get("file_path")
return str(value) if value else None
# Fetch content using entity ID
response = await resource_client.read(entity_id, page=page, page_size=page_size)
try:
# Try to resolve identifier to entity ID
entity_id = await knowledge_client.resolve_entity(entity_path, strict=True)
# If successful, return the content
if response.status_code == 200:
logger.info(
"Returning read_note result from resource: {path}", path=entity_path
)
if output_format == "json":
return await _read_json_payload(entity_id)
return response.text
except Exception as e: # pragma: no cover
logger.info(f"Direct lookup failed for '{entity_path}': {e}")
# Continue to fallback methods
# Fetch content using entity ID
response = await resource_client.read(entity_id)
# Fallback 1: Try title search via API
logger.info(f"Search title for: {identifier}")
title_results = await _search_candidates(identifier, title_only=True)
title_candidates = _search_results(title_results)
if title_candidates:
# Trigger: direct resolution failed and title search returned candidates.
# Why: avoid returning unrelated notes when search yields only fuzzy matches.
# Outcome: fetch content only when a true exact title match exists.
result = next(
(
candidate
for candidate in title_candidates
if _is_exact_title_match(identifier, _result_title(candidate))
),
None,
)
if not result:
logger.info(f"No exact title match found for: {identifier}")
elif _result_permalink(result):
try:
# Resolve the permalink to entity ID
entity_id = await knowledge_client.resolve_entity(
_result_permalink(result) or "", strict=True
)
# Fetch content using the entity ID
response = await resource_client.read(
entity_id, page=page, page_size=page_size
)
if response.status_code == 200:
logger.info(
f"Found note by exact title search: {_result_permalink(result)}"
)
if output_format == "json":
return await _read_json_payload(entity_id)
return response.text
except Exception as e: # pragma: no cover
logger.info(
f"Failed to fetch content for found title match {_result_permalink(result)}: {e}"
)
else:
# If successful, return the content
if response.status_code == 200:
logger.info(
f"No results in title search for: {identifier} in project {active_project.name}"
"Returning read_note result from resource: {path}", path=entity_path
)
# Fallback 2: Text search as a last resort
logger.info(f"Title search failed, trying text search for: {identifier}")
text_results = await _search_candidates(identifier, title_only=False)
# We didn't find a direct match, construct a helpful error message
text_candidates = _search_results(text_results)
if not text_candidates:
if output_format == "json":
return _empty_json_payload()
return format_not_found_message(active_project.name, identifier)
return await _read_json_payload(entity_id)
return response.text
except Exception as e: # pragma: no cover
logger.info(f"Direct lookup failed for '{entity_path}': {e}")
# Continue to fallback methods
# Fallback 1: Try title search via API
logger.info(f"Search title for: {identifier}")
title_results = await _search_candidates(identifier, title_only=True)
title_candidates = _search_results(title_results)
if title_candidates:
# Trigger: direct resolution failed and title search returned candidates.
# Why: avoid returning unrelated notes when search yields only fuzzy matches.
# Outcome: fetch content only when a true exact title match exists.
result = next(
(
candidate
for candidate in title_candidates
if _is_exact_title_match(identifier, _result_title(candidate))
),
None,
)
if not result:
logger.info(f"No exact title match found for: {identifier}")
elif _result_permalink(result):
try:
# Resolve the permalink to entity ID
entity_id = await knowledge_client.resolve_entity(
_result_permalink(result) or "", strict=True
)
# Fetch content using the entity ID
response = await resource_client.read(entity_id)
if response.status_code == 200:
logger.info(
f"Found note by exact title search: {_result_permalink(result)}"
)
if output_format == "json":
return await _read_json_payload(entity_id)
return response.text
except Exception as e: # pragma: no cover
logger.info(
f"Failed to fetch content for found title match {_result_permalink(result)}: {e}"
)
else:
logger.info(
f"No results in title search for: {identifier} in project {active_project.name}"
)
# Fallback 2: Text search as a last resort
logger.info(f"Title search failed, trying text search for: {identifier}")
text_results = await _search_candidates(identifier, title_only=False)
# We didn't find a direct match, construct a helpful error message
text_candidates = _search_results(text_results)
if not text_candidates:
if output_format == "json":
payload = _empty_json_payload()
payload["related_results"] = [
{
"title": _result_title(result),
"permalink": _result_permalink(result),
"file_path": _result_file_path(result),
}
for result in text_candidates[:5]
]
return payload
return format_related_results(active_project.name, identifier, text_candidates[:5])
return _empty_json_payload()
return format_not_found_message(active_project.name, identifier)
if output_format == "json":
payload = _empty_json_payload()
payload["related_results"] = [
{
"title": _result_title(result),
"permalink": _result_permalink(result),
"file_path": _result_file_path(result),
}
for result in text_candidates[:5]
]
return payload
return format_related_results(active_project.name, identifier, text_candidates[:5])
def format_not_found_message(project: str | None, identifier: str) -> str:
+39 -13
View File
@@ -2,10 +2,11 @@
from datetime import timezone
from pathlib import PurePosixPath
from typing import List, Union, Optional, Literal
from typing import Annotated, List, Union, Optional, Literal
from loguru import logger
from fastmcp import Context
from pydantic import AliasChoices, Field
from basic_memory.mcp.async_client import get_client
from basic_memory.mcp.project_context import (
@@ -38,13 +39,30 @@ from basic_memory.schemas.search import SearchItemType
annotations={"readOnlyHint": True, "openWorldHint": False},
)
async def recent_activity(
type: Union[str, List[str]] = "",
type: Annotated[
Union[str, List[str]],
Field(default="", validation_alias=AliasChoices("type", "types", "kind")),
] = "",
depth: int = 1,
timeframe: TimeFrame = "7d",
page: int = 1,
page_size: int = 10,
timeframe: Annotated[
TimeFrame,
Field(
default="7d",
validation_alias=AliasChoices("timeframe", "since", "time_range", "lookback"),
),
] = "7d",
# `offset` is intentionally NOT aliased: it has different semantics
# (item-indexed vs. 1-indexed page-number).
page: Annotated[
int,
Field(default=1, validation_alias=AliasChoices("page", "page_number")),
] = 1,
page_size: Annotated[
int,
Field(default=10, validation_alias=AliasChoices("page_size", "limit", "per_page")),
] = 10,
project: Optional[str] = None,
workspace: Optional[str] = None,
project_id: Optional[str] = None,
output_format: Literal["text", "json"] = "text",
context: Context | None = None,
) -> str | list[dict]:
@@ -86,6 +104,9 @@ async def recent_activity(
project: Project name to query. Optional - server will resolve using the
hierarchy above. If unknown, use list_memory_projects() to discover
available projects.
project_id: Project external_id (UUID). Prefer this over `project` when known
it routes to the exact project regardless of name collisions across cloud
workspaces. Takes precedence over `project`. Get from list_memory_projects().
output_format: "text" returns human-readable summary text. "json" returns
a flat list of recent items.
context: Optional FastMCP context for performance caching.
@@ -161,9 +182,12 @@ async def recent_activity(
if "type" not in params:
params["type"] = [SearchItemType.ENTITY.value]
# Resolve project parameter using the three-tier hierarchy
# allow_discovery=True enables Discovery Mode, so a project is not required
resolved_project = await resolve_project_parameter(project, allow_discovery=True)
# Resolve project parameter using the three-tier hierarchy.
# allow_discovery=True enables Discovery Mode, so a project is not required.
# project_id (UUID) takes precedence over project name — without this fallback,
# callers passing only project_id would fall into Discovery Mode.
effective_identifier = project_id if project_id else project
resolved_project = await resolve_project_parameter(effective_identifier, allow_discovery=True)
if resolved_project is None:
# Discovery Mode: Get activity across all projects
@@ -278,7 +302,7 @@ async def recent_activity(
f"Getting recent activity from project {resolved_project}: type={type}, depth={depth}, timeframe={timeframe}"
)
async with get_project_client(resolved_project, workspace, context) as (
async with get_project_client(resolved_project, context=context, project_id=project_id) as (
client,
active_project,
):
@@ -464,10 +488,12 @@ def _format_project_output(
elif result.primary_result.type == "observation":
observations.append(result.primary_result)
# Show entities (notes/documents)
# Show entities (notes/documents). Render every row the API returned —
# `page_size` is the single knob for how much comes back, so heading count
# and body row count always agree (regression: #784 silent truncation).
if entities:
lines.append(f"\n**📄 Recent Notes & Documents ({len(entities)}):**")
for entity in entities[:5]: # Show top 5
for entity in entities:
title = entity.title or "Untitled"
# Get folder from file_path
folder = ""
@@ -500,7 +526,7 @@ def _format_project_output(
# Show relations (connections)
if relations:
lines.append(f"\n**🔗 Recent Connections ({len(relations)}):**")
for rel in relations[:5]: # Show top 5
for rel in relations:
rel_type = rel.relation_type
from_entity = rel.from_entity or "Unknown"
to_entity = rel.to_entity
+24 -6
View File
@@ -211,7 +211,7 @@ async def schema_validate(
note_type: Optional[str] = None,
identifier: Optional[str] = None,
project: Optional[str] = None,
workspace: Optional[str] = None,
project_id: Optional[str] = None,
output_format: Literal["text", "json"] = "text",
context: Context | None = None,
) -> ValidationReport | str | dict:
@@ -236,6 +236,9 @@ async def schema_validate(
identifier: Specific note to validate (permalink, title, or path).
If provided, validates only this note.
project: Project name. Optional -- server will resolve.
project_id: Project external_id (UUID). Prefer this over `project` when known
it routes to the exact project regardless of name collisions across cloud
workspaces. Takes precedence over `project`. Get from list_memory_projects().
context: Optional FastMCP context for performance caching.
Returns:
@@ -251,7 +254,10 @@ async def schema_validate(
# Validate in a specific project
schema_validate(note_type="person", project="my-research")
"""
async with get_project_client(project, workspace, context) as (client, active_project):
async with get_project_client(project, context=context, project_id=project_id) as (
client,
active_project,
):
logger.info(
f"MCP tool call tool=schema_validate project={active_project.name} "
f"note_type={note_type} identifier={identifier}"
@@ -318,7 +324,7 @@ async def schema_infer(
note_type: str,
threshold: float = 0.25,
project: Optional[str] = None,
workspace: Optional[str] = None,
project_id: Optional[str] = None,
output_format: Literal["text", "json"] = "text",
context: Context | None = None,
) -> str | dict:
@@ -342,6 +348,9 @@ async def schema_infer(
threshold: Minimum frequency (0-1) for a field to be suggested as optional.
Default 0.25 (25%). Fields above 95% become required.
project: Project name. Optional -- server will resolve.
project_id: Project external_id (UUID). Prefer this over `project` when known
it routes to the exact project regardless of name collisions across cloud
workspaces. Takes precedence over `project`. Get from list_memory_projects().
context: Optional FastMCP context for performance caching.
Returns:
@@ -357,7 +366,10 @@ async def schema_infer(
# Infer in a specific project
schema_infer("person", project="my-research")
"""
async with get_project_client(project, workspace, context) as (client, active_project):
async with get_project_client(project, context=context, project_id=project_id) as (
client,
active_project,
):
logger.info(
f"MCP tool call tool=schema_infer project={active_project.name} "
f"note_type={note_type} threshold={threshold}"
@@ -432,7 +444,7 @@ async def schema_infer(
async def schema_diff(
note_type: str,
project: Optional[str] = None,
workspace: Optional[str] = None,
project_id: Optional[str] = None,
output_format: Literal["text", "json"] = "text",
context: Context | None = None,
) -> str | dict:
@@ -453,6 +465,9 @@ async def schema_diff(
Args:
note_type: The note type to check for drift (e.g., "person").
project: Project name. Optional -- server will resolve.
project_id: Project external_id (UUID). Prefer this over `project` when known
it routes to the exact project regardless of name collisions across cloud
workspaces. Takes precedence over `project`. Get from list_memory_projects().
context: Optional FastMCP context for performance caching.
Returns:
@@ -466,7 +481,10 @@ async def schema_diff(
# Check drift in a specific project
schema_diff("person", project="my-research")
"""
async with get_project_client(project, workspace, context) as (client, active_project):
async with get_project_client(project, context=context, project_id=project_id) as (
client,
active_project,
):
logger.info(
f"MCP tool call tool=schema_diff project={active_project.name} note_type={note_type}"
)
+178 -140
View File
@@ -4,16 +4,16 @@ import re
from textwrap import dedent
from typing import Annotated, List, Optional, Dict, Any, Literal
import logfire
from loguru import logger
from fastmcp import Context
from pydantic import BeforeValidator
from pydantic import AliasChoices, BeforeValidator, Field
from basic_memory import telemetry
from basic_memory.config import ConfigManager
from basic_memory.utils import coerce_dict, coerce_list
from basic_memory.mcp.container import get_container
from basic_memory.mcp.project_context import (
detect_project_from_url_prefix,
detect_project_from_memory_url_prefix,
get_project_client,
resolve_project_and_path,
)
@@ -301,27 +301,51 @@ def _format_search_markdown(result: SearchResponse, project: str, query: str | N
annotations={"readOnlyHint": True, "openWorldHint": False},
)
async def search_notes(
query: Optional[str] = None,
# Accept common search-query aliases models reach for from training data.
# `q` is the universal HTTP convention; `search`/`text` are common in NL APIs.
query: Annotated[
Optional[str],
Field(default=None, validation_alias=AliasChoices("query", "q", "search", "text")),
] = None,
project: Optional[str] = None,
workspace: Optional[str] = None,
page: int = 1,
page_size: int = 10,
project_id: Optional[str] = None,
# `offset` is intentionally NOT aliased to `page`: offset is item-indexed
# (skip N items) while page is 1-indexed page-number. Direct aliasing would
# silently return the wrong slice.
page: Annotated[
int,
Field(default=1, validation_alias=AliasChoices("page", "page_number")),
] = 1,
page_size: Annotated[
int,
Field(default=10, validation_alias=AliasChoices("page_size", "limit", "per_page")),
] = 10,
search_type: str | None = None,
output_format: Literal["text", "json"] = "text",
# Plural-vs-singular trips models constantly. Accept the singular too.
note_types: Annotated[
List[str] | None,
BeforeValidator(coerce_list),
Field(default=None, validation_alias=AliasChoices("note_types", "note_type", "types")),
"Filter by the 'type' field in note frontmatter (e.g. 'note', 'chapter', 'person'). "
"Case-insensitive.",
] = None,
entity_types: Annotated[
List[str] | None,
BeforeValidator(coerce_list),
Field(default=None, validation_alias=AliasChoices("entity_types", "entity_type")),
"Filter by knowledge graph item type: 'entity' (whole notes), 'observation', or "
"'relation'. Defaults to 'entity'. Do NOT pass schema/frontmatter types like "
"'Chapter' here — use note_types instead.",
] = None,
after_date: Optional[str] = None,
# Time-filter naming varies wildly across APIs.
after_date: Annotated[
Optional[str],
Field(
default=None,
validation_alias=AliasChoices("after_date", "since", "after", "from_date"),
),
] = None,
metadata_filters: Annotated[
Dict[str, Any] | None,
BeforeValidator(coerce_dict),
@@ -331,7 +355,13 @@ async def search_notes(
BeforeValidator(coerce_list),
] = None,
status: Optional[str] = None,
min_similarity: Optional[float] = None,
min_similarity: Annotated[
Optional[float],
Field(
default=None,
validation_alias=AliasChoices("min_similarity", "threshold", "similarity_threshold"),
),
] = None,
context: Context | None = None,
) -> dict | str:
"""Search across all content in the knowledge base with comprehensive syntax support.
@@ -415,6 +445,9 @@ async def search_notes(
Omit or pass None for filter-only searches using metadata_filters, tags, or status.
project: Project name to search in. Optional - server will resolve using hierarchy.
If unknown, use list_memory_projects() to discover available projects.
project_id: Project external_id (UUID). Prefer this over `project` when known
it routes to the exact project regardless of name collisions across cloud
workspaces. Takes precedence over `project`. Get from list_memory_projects().
page: The page number of results to return (default 1)
page_size: The number of results to return per page (default 10)
search_type: Type of search to perform, one of:
@@ -518,18 +551,23 @@ async def search_notes(
remainder = re.sub(r"\b(AND|OR|NOT)\b", "", remainder).strip()
query = remainder or None
# Detect project from memory URL prefix before routing
if project is None and query is not None:
detected = detect_project_from_url_prefix(query, ConfigManager().config)
# Detect project from memory URL prefix before routing.
# project_id routes by external UUID, so it bypasses URL discovery entirely.
if project is None and project_id is None and query is not None:
detected = await detect_project_from_memory_url_prefix(
query,
ConfigManager().config,
context=context,
)
if detected:
project = detected
with telemetry.operation(
with logfire.span(
"mcp.tool.search_notes",
entrypoint="mcp",
tool_name="search_notes",
requested_project=project,
workspace_id=workspace,
requested_project_id=project_id,
search_type=search_type or "default",
output_format=output_format,
page=page,
@@ -543,140 +581,140 @@ async def search_notes(
has_tags_filter=bool(tags),
has_status_filter=bool(status),
):
async with get_project_client(project, workspace, context) as (client, active_project):
with telemetry.contextualize(
project_name=active_project.name,
workspace_id=workspace,
tool_name="search_notes",
):
# Handle memory:// URLs by resolving to permalink search
is_memory_url = False
if query is not None:
_, resolved_query, is_memory_url = await resolve_project_and_path(
client, query, project, context
)
if is_memory_url:
query = resolved_query
effective_search_type = search_type or _default_search_type()
async with get_project_client(project, context=context, project_id=project_id) as (
client,
active_project,
):
# Handle memory:// URLs by resolving to permalink search.
# Use active_project.name so resolution hits the cached active project
# when project_id was used or `project` was wrong/ambiguous.
is_memory_url = False
if query is not None:
_, resolved_query, is_memory_url = await resolve_project_and_path(
client, query, active_project.name, context
)
if is_memory_url:
effective_search_type = "permalink"
query = resolved_query
effective_search_type = search_type or _default_search_type()
if is_memory_url:
effective_search_type = "permalink"
try:
# Create a SearchQuery object based on the parameters
search_query = SearchQuery()
try:
# Create a SearchQuery object based on the parameters
search_query = SearchQuery()
# Only map search_type to query fields when there is an actual query string.
# When query is None/empty, skip the search mode block — filters-only path.
effective_query = (query or "").strip()
if effective_query:
valid_search_types = {
"text",
"title",
"permalink",
"vector",
"semantic",
"hybrid",
}
if effective_search_type == "text":
search_query.text = effective_query
search_query.retrieval_mode = SearchRetrievalMode.FTS
elif effective_search_type in ("vector", "semantic"):
search_query.text = effective_query
search_query.retrieval_mode = SearchRetrievalMode.VECTOR
elif effective_search_type == "hybrid":
search_query.text = effective_query
search_query.retrieval_mode = SearchRetrievalMode.HYBRID
elif effective_search_type == "title":
search_query.title = effective_query
elif effective_search_type == "permalink" and "*" in effective_query:
search_query.permalink_match = effective_query
elif effective_search_type == "permalink":
search_query.permalink = effective_query
else:
raise ValueError(
f"Invalid search_type '{effective_search_type}'. "
f"Valid options: {', '.join(sorted(valid_search_types))}"
)
# Add optional filters if provided (empty lists are treated as no filter)
if entity_types:
search_query.entity_types = [SearchItemType(t) for t in entity_types]
if note_types:
search_query.note_types = note_types
if after_date:
search_query.after_date = after_date
if metadata_filters:
# Alias common column/model names to their frontmatter key equivalents.
# Users often pass "note_type" (the entity model column) when the
# frontmatter field is actually "type".
_METADATA_KEY_ALIASES = {"note_type": "type"}
metadata_filters = {
_METADATA_KEY_ALIASES.get(k, k): v for k, v in metadata_filters.items()
}
search_query.metadata_filters = metadata_filters
if tags:
search_query.tags = tags
if status:
search_query.status = status
if min_similarity is not None:
search_query.min_similarity = min_similarity
# Reject searches with no criteria at all
if search_query.no_criteria():
return (
"# No Search Criteria\n\n"
"Please provide at least one of: `query`, `metadata_filters`, "
"`tags`, `status`, `note_types`, `entity_types`, or `after_date`."
# Only map search_type to query fields when there is an actual query string.
# When query is None/empty, skip the search mode block — filters-only path.
effective_query = (query or "").strip()
if effective_query:
valid_search_types = {
"text",
"title",
"permalink",
"vector",
"semantic",
"hybrid",
}
if effective_search_type == "text":
search_query.text = effective_query
search_query.retrieval_mode = SearchRetrievalMode.FTS
elif effective_search_type in ("vector", "semantic"):
search_query.text = effective_query
search_query.retrieval_mode = SearchRetrievalMode.VECTOR
elif effective_search_type == "hybrid":
search_query.text = effective_query
search_query.retrieval_mode = SearchRetrievalMode.HYBRID
elif effective_search_type == "title":
search_query.title = effective_query
elif effective_search_type == "permalink" and "*" in effective_query:
search_query.permalink_match = effective_query
elif effective_search_type == "permalink":
search_query.permalink = effective_query
else:
raise ValueError(
f"Invalid search_type '{effective_search_type}'. "
f"Valid options: {', '.join(sorted(valid_search_types))}"
)
# Default to entity-level results to avoid returning individual
# observations/relations as separate search results (see issue #31).
# Applied after no_criteria() so that the implicit default doesn't
# mask a truly empty search request.
if not search_query.entity_types:
search_query.entity_types = [SearchItemType("entity")]
# Add optional filters if provided (empty lists are treated as no filter)
if entity_types:
search_query.entity_types = [SearchItemType(t) for t in entity_types]
if note_types:
search_query.note_types = note_types
if after_date:
search_query.after_date = after_date
if metadata_filters:
# Alias common column/model names to their frontmatter key equivalents.
# Users often pass "note_type" (the entity model column) when the
# frontmatter field is actually "type".
_METADATA_KEY_ALIASES = {"note_type": "type"}
metadata_filters = {
_METADATA_KEY_ALIASES.get(k, k): v for k, v in metadata_filters.items()
}
search_query.metadata_filters = metadata_filters
if tags:
search_query.tags = tags
if status:
search_query.status = status
if min_similarity is not None:
search_query.min_similarity = min_similarity
# Reject searches with no criteria at all
if search_query.no_criteria():
return (
"# No Search Criteria\n\n"
"Please provide at least one of: `query`, `metadata_filters`, "
"`tags`, `status`, `note_types`, `entity_types`, or `after_date`."
)
# Default to entity-level results to avoid returning individual
# observations/relations as separate search results (see issue #31).
# Applied after no_criteria() so that the implicit default doesn't
# mask a truly empty search request.
if not search_query.entity_types:
search_query.entity_types = [SearchItemType("entity")]
logger.debug(
f"Search request: project={active_project.name} "
f"search_type={effective_search_type} "
f"query={effective_query or '<filters-only>'} "
f"note_types={len(note_types)} entity_types={len(search_query.entity_types or [])} "
f"page={page} page_size={page_size}"
)
# Import here to avoid circular import (tools → clients → utils → tools)
from basic_memory.mcp.clients import SearchClient
# Use typed SearchClient for API calls
search_client = SearchClient(client, active_project.external_id)
result = await search_client.search(
search_query.model_dump(),
page=page,
page_size=page_size,
)
logger.debug(
f"Search response: project={active_project.name} "
f"results={len(result.results)} has_more={str(result.has_more).lower()} "
f"page={result.current_page} page_size={result.page_size}"
)
# Check if we got no results and provide helpful guidance
if not result.results:
logger.debug(
f"Search request: project={active_project.name} "
f"search_type={effective_search_type} "
f"query={effective_query or '<filters-only>'} "
f"note_types={len(note_types)} entity_types={len(search_query.entity_types or [])} "
f"page={page} page_size={page_size}"
f"Search returned no results for query: {query} in project {active_project.name}"
)
# Import here to avoid circular import (tools → clients → utils → tools)
from basic_memory.mcp.clients import SearchClient
# Don't treat this as an error, but the user might want guidance
# We return the empty result as normal - the user can decide if they need help
# Use typed SearchClient for API calls
search_client = SearchClient(client, active_project.external_id)
result = await search_client.search(
search_query.model_dump(),
page=page,
page_size=page_size,
)
logger.debug(
f"Search response: project={active_project.name} "
f"results={len(result.results)} has_more={str(result.has_more).lower()} "
f"page={result.current_page} page_size={result.page_size}"
)
if output_format == "json":
return result.model_dump(mode="json", exclude_none=True)
# Check if we got no results and provide helpful guidance
if not result.results:
logger.debug(
f"Search returned no results for query: {query} in project {active_project.name}"
)
# Don't treat this as an error, but the user might want guidance
# We return the empty result as normal - the user can decide if they need help
return _format_search_markdown(result, active_project.name, query)
if output_format == "json":
return result.model_dump(mode="json", exclude_none=True)
return _format_search_markdown(result, active_project.name, query)
except Exception as e:
logger.error(
f"Search failed for query '{query or ''}': {e}, project: {active_project.name}"
)
# Return formatted error message as string for better user experience
return _format_search_error_response(
active_project.name, str(e), query or "", effective_search_type
)
except Exception as e:
logger.error(
f"Search failed for query '{query or ''}': {e}, project: {active_project.name}"
)
# Return formatted error message as string for better user experience
return _format_search_error_response(
active_project.name, str(e), query or "", effective_search_type
)
+4 -6
View File
@@ -25,6 +25,7 @@ def _text_block(message: str) -> List[ContentBlock]:
async def search_notes_ui(
query: str,
project: Optional[str] = None,
project_id: Optional[str] = None,
page: int = 1,
page_size: int = 10,
search_type: Optional[str] = None,
@@ -49,6 +50,7 @@ async def search_notes_ui(
result = await search_notes(
query=query,
project=project,
project_id=project_id,
page=page,
page_size=page_size,
search_type=search_type,
@@ -97,16 +99,14 @@ async def search_notes_ui(
async def read_note_ui(
identifier: str,
project: Optional[str] = None,
page: int = 1,
page_size: int = 10,
project_id: Optional[str] = None,
context: Context | None = None,
) -> List[ContentBlock]:
"""Return a note preview UI as an embedded MCP-UI resource."""
content = await read_note(
identifier=identifier,
project=project,
page=page,
page_size=page_size,
project_id=project_id,
output_format="text",
context=context,
)
@@ -114,8 +114,6 @@ async def read_note_ui(
render_data = {
"toolInput": {
"identifier": identifier,
"page": page,
"page_size": page_size,
},
"toolOutput": content,
}
+69 -99
View File
@@ -5,9 +5,9 @@ to the Basic Memory API, with improved error handling and logging.
"""
import typing
from contextlib import contextmanager
from typing import Optional
from typing import Any, Optional
import logfire
from httpx import Response, URL, AsyncClient, HTTPStatusError
from httpx._client import UseClientDefault, USE_CLIENT_DEFAULT
from httpx._types import (
@@ -24,7 +24,6 @@ from httpx._types import (
from loguru import logger
from mcp.server.fastmcp.exceptions import ToolError
from basic_memory import telemetry
from basic_memory.config import ConfigManager
@@ -41,43 +40,22 @@ def _classify_http_outcome(status_code: int) -> str:
return "unknown" # pragma: no cover
class _RequestSpan:
"""Small adapter for attaching outcome metadata to a live request span."""
def _response_span_attrs(response: Response) -> dict[str, Any]:
"""Attributes to attach to a request span after a response lands."""
return {
"status_code": response.status_code,
"is_success": response.is_success,
"outcome": _classify_http_outcome(response.status_code),
}
def __init__(self, active_span: typing.Any | None):
self._active_span = active_span
def record_response(self, response: Response) -> None:
self._set_attributes(
{
"status_code": response.status_code,
"is_success": response.is_success,
"outcome": _classify_http_outcome(response.status_code),
}
)
def record_transport_error(self, exc: Exception) -> None:
self._set_attributes(
{
"is_success": False,
"outcome": "transport_error",
"error_type": type(exc).__name__,
}
)
def _set_attributes(self, attrs: dict[str, typing.Any]) -> None:
if self._active_span is None:
return
set_attributes = getattr(self._active_span, "set_attributes", None)
if callable(set_attributes):
set_attributes(attrs)
return
set_attribute = getattr(self._active_span, "set_attribute", None)
if callable(set_attribute):
for key, value in attrs.items():
set_attribute(key, value)
def _transport_error_span_attrs(exc: Exception) -> dict[str, Any]:
"""Attributes to attach when the transport layer fails before any response."""
return {
"is_success": False,
"outcome": "transport_error",
"error_type": type(exc).__name__,
}
def get_error_message(
@@ -130,15 +108,20 @@ def get_error_message(
return f"HTTP error {status_code}: {method} request to '{path}' failed"
def _extract_response_data(response: Response) -> typing.Any:
"""Safely decode response payload for error reporting."""
try:
return response.json()
except Exception:
def _extract_response_data(response: Response) -> Any:
"""Decode the JSON payload of an API response for error reporting.
Upstream gateways (Fly, Cloudflare, load balancers) can return HTML
error pages before the request reaches our FastAPI app; those have no
structured `detail` to surface, so we skip them. A malformed body with
a JSON content-type is a server bug and we let it raise.
"""
if "application/json" not in response.headers.get("content-type", ""):
return None
return response.json()
def _response_detail_text(response_data: typing.Any) -> str | None:
def _response_detail_text(response_data: Any) -> str | None:
"""Extract textual error detail from API payloads."""
if isinstance(response_data, dict):
detail = response_data.get("detail")
@@ -189,31 +172,6 @@ def _resolve_error_message(
return get_error_message(status_code, url, method)
@contextmanager
def _request_scope(
method: str,
*,
client_name: str | None,
operation: str | None,
path_template: str | None,
params: QueryParamTypes | None = None,
has_body: bool = False,
):
"""Create the shared MCP transport span used by all HTTP helpers."""
attrs = {
"method": method,
"client_name": client_name,
"operation": operation,
"path_template": path_template,
"phase": "request",
"has_query": bool(params),
"has_body": has_body,
}
with telemetry.contextualize(**attrs):
with telemetry.started_span("mcp.http.request", **attrs) as active_span:
yield _RequestSpan(active_span)
async def call_get(
client: AsyncClient,
url: URL | str,
@@ -250,15 +208,18 @@ async def call_get(
"""
logger.debug(f"Calling GET '{url}' params: '{params}'")
error_message = None
request_span: _RequestSpan | None = None
request_span: logfire.LogfireSpan | None = None
try:
with _request_scope(
"GET",
with logfire.span(
"mcp.http.request",
method="GET",
client_name=client_name,
operation=operation,
path_template=path_template,
params=params,
phase="request",
has_query=bool(params),
has_body=False,
) as request_span:
response = await client.get(
url,
@@ -270,7 +231,7 @@ async def call_get(
timeout=timeout,
extensions=extensions,
)
request_span.record_response(response)
request_span.set_attributes(_response_span_attrs(response))
if response.is_success:
return response
@@ -299,7 +260,7 @@ async def call_get(
raise ToolError(error_message) from e
except Exception as e:
if request_span is not None:
request_span.record_transport_error(e)
request_span.set_attributes(_transport_error_span_attrs(e))
raise
@@ -347,15 +308,17 @@ async def call_put(
"""
logger.debug(f"Calling PUT '{url}'")
error_message = None
request_span: _RequestSpan | None = None
request_span: logfire.LogfireSpan | None = None
try:
with _request_scope(
"PUT",
with logfire.span(
"mcp.http.request",
method="PUT",
client_name=client_name,
operation=operation,
path_template=path_template,
params=params,
phase="request",
has_query=bool(params),
has_body=any(value is not None for value in (content, data, files, json)),
) as request_span:
response = await client.put(
@@ -372,7 +335,7 @@ async def call_put(
timeout=timeout,
extensions=extensions,
)
request_span.record_response(response)
request_span.set_attributes(_response_span_attrs(response))
if response.is_success:
return response
@@ -402,7 +365,7 @@ async def call_put(
raise ToolError(error_message) from e
except Exception as e:
if request_span is not None:
request_span.record_transport_error(e)
request_span.set_attributes(_transport_error_span_attrs(e))
raise
@@ -449,15 +412,17 @@ async def call_patch(
ToolError: If the request fails with an appropriate error message
"""
logger.debug(f"Calling PATCH '{url}'")
request_span: _RequestSpan | None = None
request_span: logfire.LogfireSpan | None = None
try:
with _request_scope(
"PATCH",
with logfire.span(
"mcp.http.request",
method="PATCH",
client_name=client_name,
operation=operation,
path_template=path_template,
params=params,
phase="request",
has_query=bool(params),
has_body=any(value is not None for value in (content, data, files, json)),
) as request_span:
response = await client.patch(
@@ -474,7 +439,7 @@ async def call_patch(
timeout=timeout,
extensions=extensions,
)
request_span.record_response(response)
request_span.set_attributes(_response_span_attrs(response))
if response.is_success:
return response
@@ -509,7 +474,7 @@ async def call_patch(
raise ToolError(error_message) from e
except Exception as e:
if request_span is not None:
request_span.record_transport_error(e)
request_span.set_attributes(_transport_error_span_attrs(e))
raise
@@ -557,15 +522,17 @@ async def call_post(
"""
logger.debug(f"Calling POST '{url}'")
error_message = None
request_span: _RequestSpan | None = None
request_span: logfire.LogfireSpan | None = None
try:
with _request_scope(
"POST",
with logfire.span(
"mcp.http.request",
method="POST",
client_name=client_name,
operation=operation,
path_template=path_template,
params=params,
phase="request",
has_query=bool(params),
has_body=any(value is not None for value in (content, data, files, json)),
) as request_span:
response = await client.post(
@@ -582,7 +549,7 @@ async def call_post(
timeout=timeout,
extensions=extensions,
)
request_span.record_response(response)
request_span.set_attributes(_response_span_attrs(response))
logger.debug(f"response: {_extract_response_data(response)}")
if response.is_success:
@@ -612,7 +579,7 @@ async def call_post(
raise ToolError(error_message) from e
except Exception as e:
if request_span is not None:
request_span.record_transport_error(e)
request_span.set_attributes(_transport_error_span_attrs(e))
raise
@@ -684,15 +651,18 @@ async def call_delete(
"""
logger.debug(f"Calling DELETE '{url}'")
error_message = None
request_span: _RequestSpan | None = None
request_span: logfire.LogfireSpan | None = None
try:
with _request_scope(
"DELETE",
with logfire.span(
"mcp.http.request",
method="DELETE",
client_name=client_name,
operation=operation,
path_template=path_template,
params=params,
phase="request",
has_query=bool(params),
has_body=False,
) as request_span:
response = await client.delete(
url=url,
@@ -704,7 +674,7 @@ async def call_delete(
timeout=timeout,
extensions=extensions,
)
request_span.record_response(response)
request_span.set_attributes(_response_span_attrs(response))
if response.is_success:
return response
@@ -733,5 +703,5 @@ async def call_delete(
raise ToolError(error_message) from e
except Exception as e:
if request_span is not None:
request_span.record_transport_error(e)
request_span.set_attributes(_transport_error_span_attrs(e))
raise
+5 -11
View File
@@ -17,9 +17,7 @@ from basic_memory.mcp.tools.read_note import read_note
async def view_note(
identifier: str,
project: Optional[str] = None,
workspace: Optional[str] = None,
page: int = 1,
page_size: int = 10,
project_id: Optional[str] = None,
context: Context | None = None,
) -> str:
"""View a markdown note as a formatted artifact.
@@ -32,8 +30,9 @@ async def view_note(
identifier: The title or permalink of the note to view
project: Project name to read from. Optional - server will resolve using hierarchy.
If unknown, use list_memory_projects() to discover available projects.
page: Page number for paginated results (default: 1)
page_size: Number of items per page (default: 10)
project_id: Project external_id (UUID). Prefer this over `project` when known
it routes to the exact project regardless of name collisions across cloud
workspaces. Takes precedence over `project`. Get from list_memory_projects().
context: Optional FastMCP context for performance caching.
Returns:
@@ -46,9 +45,6 @@ async def view_note(
# View a note by permalink
view_note("meetings/weekly-standup")
# View with pagination
view_note("large-document", page=2, page_size=5)
# Explicit project specification
view_note("Meeting Notes", project="my-project")
@@ -63,9 +59,7 @@ async def view_note(
await read_note(
identifier=identifier,
project=project,
workspace=workspace,
page=page,
page_size=page_size,
project_id=project_id,
context=context,
)
)
+43 -29
View File
@@ -6,6 +6,42 @@ from fastmcp import Context
from basic_memory.mcp.project_context import get_available_workspaces
from basic_memory.mcp.server import mcp
from basic_memory.schemas.cloud import WorkspaceInfo, WorkspaceListResponse
def _personal_workspace() -> WorkspaceInfo:
"""Return a display-only personal workspace when discovery has no rows.
This keeps list_workspaces friendly for non-teams or local-only users. It is not
the cloud routing source of truth; project-scoped routing still depends on real
workspace discovery and the workspace project index in project_context.
"""
return WorkspaceInfo(
tenant_id="personal",
workspace_type="personal",
slug="personal",
name="Personal",
role="owner",
is_default=True,
has_active_subscription=True,
)
def _workspace_list_response(workspaces: list[WorkspaceInfo]) -> WorkspaceListResponse:
"""Build the structured MCP response from the shared cloud workspace schema."""
if not workspaces:
workspaces = [_personal_workspace()]
default_workspace_id = next(
(workspace.tenant_id for workspace in workspaces if workspace.is_default),
None,
)
return WorkspaceListResponse(
workspaces=workspaces,
count=len(workspaces),
default_workspace_id=default_workspace_id,
current_workspace_id=None,
)
@mcp.tool(
@@ -24,40 +60,18 @@ async def list_workspaces(
context: Optional FastMCP context for progress/status logging.
"""
workspaces = await get_available_workspaces(context=context)
response = _workspace_list_response(workspaces)
if output_format == "json":
return {
"workspaces": [
{
"tenant_id": ws.tenant_id,
"name": ws.name,
"workspace_type": ws.workspace_type,
"role": ws.role,
"organization_id": ws.organization_id,
"has_active_subscription": ws.has_active_subscription,
}
for ws in workspaces
],
"count": len(workspaces),
}
return response.model_dump(mode="json")
if not workspaces:
return (
"# No Workspaces Available\n\n"
"No accessible workspaces were found for this account. "
"Ensure the account has an active subscription and tenant access."
)
lines = [
f"# Available Workspaces ({len(workspaces)})",
"",
"Use `workspace` as either the `tenant_id` or unique `name` in project-scoped tool calls.",
"",
]
for workspace in workspaces:
lines = [f"# Available Workspaces ({response.count})", ""]
for workspace in response.workspaces:
default_label = ", default" if workspace.is_default else ""
lines.append(
f"- {workspace.name} "
f"(type={workspace.workspace_type}, role={workspace.role}, tenant_id={workspace.tenant_id})"
f"(slug={workspace.slug}, type={workspace.workspace_type}, "
f"role={workspace.role}{default_label}, tenant_id={workspace.tenant_id})"
)
return "\n".join(lines)
+168 -163
View File
@@ -3,10 +3,10 @@
import textwrap
from typing import Annotated, List, Union, Optional, Literal
import logfire
from loguru import logger
from pydantic import BeforeValidator
from pydantic import AliasChoices, BeforeValidator, Field
from basic_memory import telemetry
from basic_memory.config import ConfigManager
from basic_memory.mcp.project_context import get_project_client, add_project_metadata
from basic_memory.mcp.server import mcp
@@ -25,13 +25,21 @@ TagType = Union[List[str], str, None]
async def write_note(
title: str,
content: str,
directory: str,
# Folder/dir/path are interchangeable in models' training data.
directory: Annotated[
str,
Field(validation_alias=AliasChoices("directory", "folder", "dir", "path")),
],
project: Optional[str] = None,
workspace: Optional[str] = None,
project_id: Optional[str] = None,
tags: list[str] | str | None = None,
note_type: str = "note",
metadata: Annotated[dict | None, BeforeValidator(coerce_dict)] = None,
overwrite: bool | None = None,
# Force/replace are the file-write idioms models default to.
overwrite: Annotated[
bool | None,
Field(default=None, validation_alias=AliasChoices("overwrite", "force", "replace")),
] = None,
output_format: Literal["text", "json"] = "text",
context: Context | None = None,
) -> str | dict:
@@ -74,6 +82,9 @@ async def write_note(
project: Project name to write to. Optional - server will resolve using the
hierarchy above. If unknown, use list_memory_projects() to discover
available projects.
project_id: Project external_id (UUID). Prefer this over `project` when known
it routes to the exact project regardless of name collisions across cloud
workspaces. Takes precedence over `project`. Get from list_memory_projects().
tags: Tags to categorize the note. Can be a list of strings, a comma-separated string, or None.
Note: If passing from external MCP clients, use a string format (e.g. "tag1,tag2,tag3")
note_type: Type of note to create (stored in frontmatter). Defaults to "note".
@@ -149,180 +160,174 @@ async def write_note(
overwrite if overwrite is not None else ConfigManager().config.write_note_overwrite_default
)
with telemetry.operation(
with logfire.span(
"mcp.tool.write_note",
entrypoint="mcp",
tool_name="write_note",
requested_project=project,
workspace_id=workspace,
requested_project_id=project_id,
note_type=note_type,
overwrite=effective_overwrite,
output_format=output_format,
):
async with get_project_client(project, workspace, context) as (client, active_project):
with telemetry.contextualize(
project_name=active_project.name,
workspace_id=workspace,
tool_name="write_note",
):
logger.info(
f"MCP tool call tool=write_note project={active_project.name} directory={directory}, title={title}, tags={tags}"
)
async with get_project_client(project, context=context, project_id=project_id) as (
client,
active_project,
):
logger.info(
f"MCP tool call tool=write_note project={active_project.name} directory={directory}, title={title}, tags={tags}"
)
# Normalize "/" to empty string for root directory (must happen before validation)
if directory == "/":
directory = ""
# Normalize "/" to empty string for root directory (must happen before validation)
if directory == "/":
directory = ""
# Validate directory path to prevent path traversal attacks
project_path = active_project.home
if directory and not validate_project_path(directory, project_path):
logger.warning(
"Attempted path traversal attack blocked",
directory=directory,
project=active_project.name,
)
if output_format == "json":
return {
"title": title,
"permalink": None,
"file_path": None,
"checksum": None,
"action": "created",
"error": "SECURITY_VALIDATION_ERROR",
}
return f"# Error\n\nDirectory path '{directory}' is not allowed - paths must stay within project boundaries"
# Process tags using the helper function
tag_list = parse_tags(tags)
# Build entity_metadata from optional metadata, then explicit tags on top
# Order matters: explicit tags parameter takes precedence over metadata["tags"]
entity_metadata = {}
if metadata:
entity_metadata.update(metadata)
if tag_list:
entity_metadata["tags"] = tag_list
entity = Entity(
title=title,
# Validate directory path to prevent path traversal attacks
project_path = active_project.home
if directory and not validate_project_path(directory, project_path):
logger.warning(
"Attempted path traversal attack blocked",
directory=directory,
note_type=note_type,
content_type="text/markdown",
content=content,
entity_metadata=entity_metadata or None,
)
# Import here to avoid circular import
from basic_memory.mcp.clients import KnowledgeClient
# Use typed KnowledgeClient for API calls
knowledge_client = KnowledgeClient(client, active_project.external_id)
# Try to create the entity first (optimistic create)
logger.debug(f"Attempting to create entity permalink={entity.permalink}")
action = "Created" # Default to created
try:
result = await knowledge_client.create_entity(entity.model_dump(), fast=False)
action = "Created"
except Exception as e:
# If creation failed due to conflict (already exists), try to update
if (
"409" in str(e)
or "conflict" in str(e).lower()
or "already exists" in str(e).lower()
):
# Guard: block overwrite unless explicitly enabled
if not effective_overwrite:
logger.warning(
f"write_note blocked: note already exists (overwrite not enabled) "
f"permalink={entity.permalink}"
)
if output_format == "json":
return {
"title": title,
"permalink": entity.permalink,
"file_path": None,
"checksum": None,
"action": "conflict",
"error": "NOTE_ALREADY_EXISTS",
}
return _format_overwrite_error(
title, entity.permalink, active_project.name
)
logger.debug(
f"Entity exists, updating instead permalink={entity.permalink}"
)
try:
if not entity.permalink:
raise ValueError(
"Entity permalink is required for updates"
) # pragma: no cover
entity_id = await knowledge_client.resolve_entity(entity.permalink)
result = await knowledge_client.update_entity(
entity_id, entity.model_dump(), fast=False
)
action = "Updated"
except Exception as update_error: # pragma: no cover
# Re-raise the original error if update also fails
raise e from update_error # pragma: no cover
else:
# Re-raise if it's not a conflict error
raise # pragma: no cover
summary = [
f"# {action} note",
f"project: {active_project.name}",
f"file_path: {result.file_path}",
f"permalink: {result.permalink}",
f"checksum: {result.checksum[:8] if result.checksum else 'unknown'}",
]
# Count observations by category
categories = {}
if result.observations:
for obs in result.observations:
categories[obs.category] = categories.get(obs.category, 0) + 1
summary.append("\n## Observations")
for category, count in sorted(categories.items()):
summary.append(f"- {category}: {count}")
# Count resolved/unresolved relations
unresolved = 0
resolved = 0
if result.relations:
unresolved = sum(1 for r in result.relations if not r.to_id)
resolved = len(result.relations) - unresolved
summary.append("\n## Relations")
summary.append(f"- Resolved: {resolved}")
if unresolved:
summary.append(f"- Unresolved: {unresolved}")
summary.append(
"\nNote: Unresolved relations point to entities that don't exist yet."
)
summary.append(
"They will be automatically resolved when target entities are created or during sync operations."
)
if tag_list:
summary.append(f"\n## Tags\n- {', '.join(tag_list)}")
# Log the response with structured data
logger.info(
f"MCP tool response: tool=write_note project={active_project.name} action={action} permalink={result.permalink} observations_count={len(result.observations)} relations_count={len(result.relations)} resolved_relations={resolved} unresolved_relations={unresolved}"
project=active_project.name,
)
if output_format == "json":
return {
"title": result.title,
"permalink": result.permalink,
"file_path": result.file_path,
"checksum": result.checksum,
"action": action.lower(),
"title": title,
"permalink": None,
"file_path": None,
"checksum": None,
"action": "created",
"error": "SECURITY_VALIDATION_ERROR",
}
return f"# Error\n\nDirectory path '{directory}' is not allowed - paths must stay within project boundaries"
summary_result = "\n".join(summary)
return add_project_metadata(summary_result, active_project.name)
# Process tags using the helper function
tag_list = parse_tags(tags)
# Build entity_metadata from optional metadata, then explicit tags on top
# Order matters: explicit tags parameter takes precedence over metadata["tags"]
entity_metadata = {}
if metadata:
entity_metadata.update(metadata)
if tag_list:
entity_metadata["tags"] = tag_list
entity = Entity(
title=title,
directory=directory,
note_type=note_type,
content_type="text/markdown",
content=content,
entity_metadata=entity_metadata or None,
)
# Import here to avoid circular import
from basic_memory.mcp.clients import KnowledgeClient
# Use typed KnowledgeClient for API calls
knowledge_client = KnowledgeClient(client, active_project.external_id)
# Try to create the entity first (optimistic create)
logger.debug(f"Attempting to create entity permalink={entity.permalink}")
action = "Created" # Default to created
try:
result = await knowledge_client.create_entity(entity.model_dump())
action = "Created"
except Exception as e:
# If creation failed due to conflict (already exists), try to update
if (
"409" in str(e)
or "conflict" in str(e).lower()
or "already exists" in str(e).lower()
):
# Guard: block overwrite unless explicitly enabled
if not effective_overwrite:
logger.warning(
f"write_note blocked: note already exists (overwrite not enabled) "
f"permalink={entity.permalink}"
)
if output_format == "json":
return {
"title": title,
"permalink": entity.permalink,
"file_path": None,
"checksum": None,
"action": "conflict",
"error": "NOTE_ALREADY_EXISTS",
}
return _format_overwrite_error(title, entity.permalink, active_project.name)
logger.debug(f"Entity exists, updating instead permalink={entity.permalink}")
try:
if not entity.permalink:
raise ValueError(
"Entity permalink is required for updates"
) # pragma: no cover
entity_id = await knowledge_client.resolve_entity(entity.permalink)
result = await knowledge_client.update_entity(
entity_id, entity.model_dump()
)
action = "Updated"
except Exception as update_error: # pragma: no cover
# Re-raise the original error if update also fails
raise e from update_error # pragma: no cover
else:
# Re-raise if it's not a conflict error
raise # pragma: no cover
summary = [
f"# {action} note",
f"project: {active_project.name}",
f"file_path: {result.file_path}",
f"permalink: {result.permalink}",
f"checksum: {result.checksum[:8] if result.checksum else 'unknown'}",
]
# Count observations by category
categories = {}
if result.observations:
for obs in result.observations:
categories[obs.category] = categories.get(obs.category, 0) + 1
summary.append("\n## Observations")
for category, count in sorted(categories.items()):
summary.append(f"- {category}: {count}")
# Count resolved/unresolved relations
unresolved = 0
resolved = 0
if result.relations:
unresolved = sum(1 for r in result.relations if not r.to_id)
resolved = len(result.relations) - unresolved
summary.append("\n## Relations")
summary.append(f"- Resolved: {resolved}")
if unresolved:
summary.append(f"- Unresolved: {unresolved}")
summary.append(
"\nNote: Unresolved relations point to entities that don't exist yet."
)
summary.append(
"They will be automatically resolved when target entities are created or during sync operations."
)
if tag_list:
summary.append(f"\n## Tags\n- {', '.join(tag_list)}")
# Log the response with structured data
logger.info(
f"MCP tool response: tool=write_note project={active_project.name} action={action} permalink={result.permalink} observations_count={len(result.observations)} relations_count={len(result.relations)} resolved_relations={resolved} unresolved_relations={unresolved}"
)
if output_format == "json":
return {
"title": result.title,
"permalink": result.permalink,
"file_path": result.file_path,
"checksum": result.checksum,
"action": action.lower(),
}
summary_result = "\n".join(summary)
return add_project_metadata(summary_result, active_project.name)
def _format_overwrite_error(title: str, permalink: str | None, project_name: str) -> str:
+4 -1
View File
@@ -1,5 +1,7 @@
"""Base model class for SQLAlchemy models."""
from typing import TYPE_CHECKING
from sqlalchemy.ext.asyncio import AsyncAttrs
from sqlalchemy.orm import DeclarativeBase
@@ -7,4 +9,5 @@ from sqlalchemy.orm import DeclarativeBase
class Base(AsyncAttrs, DeclarativeBase):
"""Base class for all models"""
pass
if TYPE_CHECKING:
id: int
+3 -3
View File
@@ -62,7 +62,7 @@ class Entity(Base):
)
# Core identity
id: Mapped[int] = mapped_column(Integer, primary_key=True)
id: Mapped[int] = mapped_column(Integer, primary_key=True) # pyright: ignore [reportIncompatibleVariableOverride]
# External UUID for API references - stable identifier that won't change
external_id: Mapped[str] = mapped_column(String, unique=True, default=lambda: str(uuid.uuid4()))
title: Mapped[str] = mapped_column(String)
@@ -229,7 +229,7 @@ class Observation(Base):
Index("ix_observation_category", "category"), # Add category index
)
id: Mapped[int] = mapped_column(Integer, primary_key=True)
id: Mapped[int] = mapped_column(Integer, primary_key=True) # pyright: ignore [reportIncompatibleVariableOverride]
project_id: Mapped[int] = mapped_column(Integer, ForeignKey("project.id"), index=True)
entity_id: Mapped[int] = mapped_column(Integer, ForeignKey("entity.id", ondelete="CASCADE"))
content: Mapped[str] = mapped_column(Text)
@@ -276,7 +276,7 @@ class Relation(Base):
Index("ix_relation_to_id", "to_id"),
)
id: Mapped[int] = mapped_column(Integer, primary_key=True)
id: Mapped[int] = mapped_column(Integer, primary_key=True) # pyright: ignore [reportIncompatibleVariableOverride]
project_id: Mapped[int] = mapped_column(Integer, ForeignKey("project.id"), index=True)
from_id: Mapped[int] = mapped_column(Integer, ForeignKey("entity.id", ondelete="CASCADE"))
to_id: Mapped[Optional[int]] = mapped_column(
@@ -3,7 +3,7 @@
import os
from threading import Lock
from basic_memory.config import BasicMemoryConfig
from basic_memory.config import BasicMemoryConfig, default_fastembed_cache_dir
from basic_memory.repository.embedding_provider import EmbeddingProvider
type ProviderCacheKey = tuple[
@@ -12,7 +12,7 @@ type ProviderCacheKey = tuple[
int | None,
int,
int,
str | None,
str,
int | None,
int | None,
]
@@ -22,6 +22,20 @@ _EMBEDDING_PROVIDER_CACHE_LOCK = Lock()
_FASTEMBED_MAX_THREADS = 8
def _resolve_cache_dir(app_config: BasicMemoryConfig) -> str:
"""Resolve the effective FastEmbed cache dir for this config.
Uses an explicit ``is not None`` check an empty string override from
config or ``BASIC_MEMORY_SEMANTIC_EMBEDDING_CACHE_DIR`` is an invalid
path, not a request to fall back to the default, and FastEmbed's error
message is clearer than silently swapping in a different directory.
"""
configured = app_config.semantic_embedding_cache_dir
if configured is not None:
return configured
return default_fastembed_cache_dir()
def _available_cpu_count() -> int | None:
"""Return the CPU budget available to this process when the runtime exposes it."""
process_cpu_count = getattr(os, "process_cpu_count", None)
@@ -61,7 +75,12 @@ def _resolve_fastembed_runtime_knobs(
def _provider_cache_key(app_config: BasicMemoryConfig) -> ProviderCacheKey:
"""Build a stable cache key from provider-relevant semantic embedding config."""
"""Build a stable cache key from provider-relevant semantic embedding config.
Uses the *resolved* cache dir not the raw config field so different
FASTEMBED_CACHE_PATH values produce distinct cache keys even when the
config field itself is unset.
"""
resolved_threads, resolved_parallel = _resolve_fastembed_runtime_knobs(app_config)
return (
app_config.semantic_embedding_provider.strip().lower(),
@@ -69,7 +88,7 @@ def _provider_cache_key(app_config: BasicMemoryConfig) -> ProviderCacheKey:
app_config.semantic_embedding_dimensions,
app_config.semantic_embedding_batch_size,
app_config.semantic_embedding_request_concurrency,
app_config.semantic_embedding_cache_dir,
_resolve_cache_dir(app_config),
resolved_threads,
resolved_parallel,
)
@@ -103,8 +122,12 @@ def create_embedding_provider(app_config: BasicMemoryConfig) -> EmbeddingProvide
from basic_memory.repository.fastembed_provider import FastEmbedEmbeddingProvider
resolved_threads, resolved_parallel = _resolve_fastembed_runtime_knobs(app_config)
if app_config.semantic_embedding_cache_dir is not None:
extra_kwargs["cache_dir"] = app_config.semantic_embedding_cache_dir
# Trigger: cache_dir is resolved rather than passed through directly.
# Why: FastEmbed's own default caches to <system tmp>/fastembed_cache,
# which disappears in sandboxed MCP runtimes (e.g. Codex CLI). See #741.
# Outcome: always pass an explicit, user-writable cache dir so the ONNX
# model persists across runs.
extra_kwargs["cache_dir"] = _resolve_cache_dir(app_config)
if resolved_threads is not None:
extra_kwargs["threads"] = resolved_threads
if resolved_parallel is not None:
@@ -33,7 +33,7 @@ class EntityRepository(Repository[Entity]):
"""
super().__init__(session_maker, Entity, project_id=project_id)
async def get_by_id(self, entity_id: int) -> Optional[Entity]: # pragma: no cover
async def get_by_id(self, entity_id: int, *, load_relations: bool = True) -> Optional[Entity]:
"""Get entity by numeric ID.
Args:
@@ -43,6 +43,10 @@ class EntityRepository(Repository[Entity]):
Entity if found, None otherwise
"""
async with db.scoped_session(self.session_maker) as session:
if not load_relations:
result = await session.execute(self.select().where(Entity.id == entity_id))
return result.scalars().one_or_none()
return await self.select_by_id(session, entity_id)
async def _find_one_by_query(self, query, *, load_relations: bool) -> Optional[Entity]:
@@ -11,7 +11,7 @@ from basic_memory.repository.embedding_provider import EmbeddingProvider
from basic_memory.repository.semantic_errors import SemanticDependenciesMissingError
if TYPE_CHECKING:
from fastembed import TextEmbedding # type: ignore[import-not-found] # pragma: no cover
from fastembed import TextEmbedding # pragma: no cover
class FastEmbedEmbeddingProvider(EmbeddingProvider):
@@ -62,7 +62,7 @@ class FastEmbedEmbeddingProvider(EmbeddingProvider):
def _create_model() -> "TextEmbedding":
try:
from fastembed import TextEmbedding # type: ignore[import-not-found]
from fastembed import TextEmbedding
except (
ImportError
) as exc: # pragma: no cover - exercised via tests with monkeypatch
@@ -50,7 +50,7 @@ class OpenAIEmbeddingProvider(EmbeddingProvider):
return self._client
try:
from openai import AsyncOpenAI # type: ignore[import-not-found]
from openai import AsyncOpenAI
except ImportError as exc: # pragma: no cover - covered via monkeypatch tests
raise SemanticDependenciesMissingError(
"OpenAI dependency is missing. "
@@ -686,7 +686,17 @@ class PostgresSearchRepository(SearchRepositoryBase):
# FTS search (Postgres-specific)
# ------------------------------------------------------------------
async def search(
@staticmethod
def _is_tsquery_syntax_error(exc: Exception) -> bool:
msg = str(exc).lower()
return (
"syntax error in tsquery" in msg
or "invalid input syntax for type tsquery" in msg
or "no operand in tsquery" in msg
or "no operator in tsquery" in msg
)
async def _build_fts_query_parts(
self,
search_text: Optional[str] = None,
permalink: Optional[str] = None,
@@ -695,32 +705,11 @@ class PostgresSearchRepository(SearchRepositoryBase):
note_types: Optional[List[str]] = None,
after_date: Optional[datetime] = None,
search_item_types: Optional[List[SearchItemType]] = None,
observation_categories: Optional[List[str]] = None,
relation_types: Optional[List[str]] = None,
metadata_filters: Optional[dict] = None,
retrieval_mode: SearchRetrievalMode = SearchRetrievalMode.FTS,
min_similarity: Optional[float] = None,
limit: int = 10,
offset: int = 0,
) -> List[SearchIndexRow]:
"""Search across all indexed content using PostgreSQL tsvector."""
# --- Dispatch vector / hybrid modes (shared logic) ---
dispatched = await self._dispatch_retrieval_mode(
search_text=search_text,
permalink=permalink,
permalink_match=permalink_match,
title=title,
note_types=note_types,
after_date=after_date,
search_item_types=search_item_types,
metadata_filters=metadata_filters,
retrieval_mode=retrieval_mode,
min_similarity=min_similarity,
limit=limit,
offset=offset,
)
if dispatched is not None:
return dispatched
# --- FTS mode (Postgres-specific) ---
) -> tuple[str, str, dict, str, str]:
"""Build Postgres FTS FROM/WHERE params shared by search and count."""
conditions = []
params = {}
order_by_clause = ""
@@ -766,14 +755,28 @@ class PostgresSearchRepository(SearchRepositoryBase):
else:
conditions.append("search_index.permalink = :permalink")
# Handle search item type filter (parameterized for defense-in-depth)
if search_item_types:
type_placeholders = []
for idx, t in enumerate(search_item_types):
param_name = f"search_type_{idx}"
params[param_name] = t.value
type_placeholders.append(f":{param_name}")
conditions.append(f"search_index.type IN ({', '.join(type_placeholders)})")
# Handle typed search row filters (parameterized for defense-in-depth)
self._append_in_filter(
conditions,
params,
column="search_index.type",
values=search_item_types,
param_prefix="search_type",
)
self._append_in_filter(
conditions,
params,
column="search_index.category",
values=observation_categories,
param_prefix="observation_category",
)
self._append_in_filter(
conditions,
params,
column="search_index.relation_type",
values=relation_types,
param_prefix="relation_type",
)
# Handle note type filter using JSONB containment (parameterized)
if note_types:
@@ -868,10 +871,6 @@ class PostgresSearchRepository(SearchRepositoryBase):
params["project_id"] = self.project_id
conditions.append("search_index.project_id = :project_id")
# set limit and offset
params["limit"] = limit
params["offset"] = offset
# Build WHERE clause
where_clause = " AND ".join(conditions) if conditions else "1=1"
@@ -884,6 +883,70 @@ class PostgresSearchRepository(SearchRepositoryBase):
else:
score_expr = "0"
return from_clause, where_clause, params, order_by_clause, score_expr
async def search(
self,
search_text: Optional[str] = None,
permalink: Optional[str] = None,
permalink_match: Optional[str] = None,
title: Optional[str] = None,
note_types: Optional[List[str]] = None,
after_date: Optional[datetime] = None,
search_item_types: Optional[List[SearchItemType]] = None,
observation_categories: Optional[List[str]] = None,
relation_types: Optional[List[str]] = None,
metadata_filters: Optional[dict] = None,
retrieval_mode: SearchRetrievalMode = SearchRetrievalMode.FTS,
min_similarity: Optional[float] = None,
limit: int = 10,
offset: int = 0,
) -> List[SearchIndexRow]:
"""Search across all indexed content using PostgreSQL tsvector."""
# --- Dispatch vector / hybrid modes (shared logic) ---
dispatched = await self._dispatch_retrieval_mode(
search_text=search_text,
permalink=permalink,
permalink_match=permalink_match,
title=title,
note_types=note_types,
after_date=after_date,
search_item_types=search_item_types,
observation_categories=observation_categories,
relation_types=relation_types,
metadata_filters=metadata_filters,
retrieval_mode=retrieval_mode,
min_similarity=min_similarity,
limit=limit,
offset=offset,
)
if dispatched is not None:
return dispatched
# --- FTS mode (Postgres-specific) ---
(
from_clause,
where_clause,
params,
order_by_clause,
score_expr,
) = await self._build_fts_query_parts(
search_text=search_text,
permalink=permalink,
permalink_match=permalink_match,
title=title,
note_types=note_types,
after_date=after_date,
search_item_types=search_item_types,
observation_categories=observation_categories,
relation_types=relation_types,
metadata_filters=metadata_filters,
)
# set limit and offset
params["limit"] = limit
params["offset"] = offset
sql = f"""
SELECT
search_index.project_id,
@@ -915,17 +978,7 @@ class PostgresSearchRepository(SearchRepositoryBase):
result = await session.execute(text(sql), params)
rows = result.fetchall()
except Exception as e:
# Handle tsquery syntax errors (and only those).
#
# Important: Postgres errors for other failures (e.g. missing table) will still mention
# `to_tsquery(...)` in the SQL text, so checking for the substring "tsquery" is too broad.
msg = str(e).lower()
if (
"syntax error in tsquery" in msg
or "invalid input syntax for type tsquery" in msg
or "no operand in tsquery" in msg
or "no operator in tsquery" in msg
):
if self._is_tsquery_syntax_error(e):
logger.warning(f"tsquery syntax error for search term: {search_text}, error: {e}")
return []
@@ -966,3 +1019,66 @@ class PostgresSearchRepository(SearchRepositoryBase):
)
return results
async def count(
self,
search_text: Optional[str] = None,
permalink: Optional[str] = None,
permalink_match: Optional[str] = None,
title: Optional[str] = None,
note_types: Optional[List[str]] = None,
after_date: Optional[datetime] = None,
search_item_types: Optional[List[SearchItemType]] = None,
observation_categories: Optional[List[str]] = None,
relation_types: Optional[List[str]] = None,
metadata_filters: Optional[dict] = None,
retrieval_mode: SearchRetrievalMode = SearchRetrievalMode.FTS,
min_similarity: Optional[float] = None,
) -> int:
"""Count indexed content matching the Postgres FTS query."""
if retrieval_mode != SearchRetrievalMode.FTS:
return await super().count(
search_text=search_text,
permalink=permalink,
permalink_match=permalink_match,
title=title,
note_types=note_types,
after_date=after_date,
search_item_types=search_item_types,
observation_categories=observation_categories,
relation_types=relation_types,
metadata_filters=metadata_filters,
retrieval_mode=retrieval_mode,
min_similarity=min_similarity,
)
(
from_clause,
where_clause,
params,
_order_by_clause,
_score_expr,
) = await self._build_fts_query_parts(
search_text=search_text,
permalink=permalink,
permalink_match=permalink_match,
title=title,
note_types=note_types,
after_date=after_date,
search_item_types=search_item_types,
observation_categories=observation_categories,
relation_types=relation_types,
metadata_filters=metadata_filters,
)
sql = f"SELECT COUNT(*) FROM {from_clause} WHERE {where_clause}"
logger.trace(f"Count {sql} params: {params}")
try:
async with db.scoped_session(self.session_maker) as session:
result = await session.execute(text(sql), params)
return int(result.scalar_one())
except Exception as e:
if self._is_tsquery_syntax_error(e):
logger.warning(f"tsquery syntax error for search term: {search_text}, error: {e}")
return 0
logger.error(f"Database error during search count: {e}")
raise
+40 -5
View File
@@ -13,6 +13,7 @@ from sqlalchemy import (
Result,
and_,
delete,
update as sqlalchemy_update,
)
from sqlalchemy.engine import CursorResult
from sqlalchemy.exc import NoResultFound
@@ -140,6 +141,20 @@ class Repository[T: Base]:
# Query within same session
return await self.select_by_ids(session, [m.id for m in models]) # pyright: ignore [reportAttributeAccessIssue]
async def add_all_no_return(self, models: List[T]) -> int:
"""Insert models without reloading them afterward."""
if not models:
return 0
async with db.scoped_session(self.session_maker) as session:
for model in models:
self._set_project_id_if_needed(model)
session.add_all(models)
await session.flush()
logger.debug(f"Added {len(models)} {self.Model.__name__} records")
return len(models)
def select(self, *entities: Any) -> Select:
"""Create a new SELECT statement.
@@ -268,7 +283,7 @@ class Repository[T: Base]:
return await self.select_by_ids(session, [model.id for model in model_list]) # pyright: ignore [reportAttributeAccessIssue]
async def update(self, entity_id: int, entity_data: dict | T) -> Optional[T]:
async def update(self, entity_id: int, entity_data: dict[str, Any] | T) -> Optional[T]:
"""Update an entity with the given data."""
logger.debug(f"Updating {self.Model.__name__} {entity_id} with data: {entity_data}")
async with db.scoped_session(self.session_maker) as session:
@@ -279,12 +294,13 @@ class Repository[T: Base]:
entity = result.scalars().one()
if isinstance(entity_data, dict):
for key, value in entity_data.items():
if key in self.valid_columns:
setattr(entity, key, value)
update_data = cast(dict[str, Any], entity_data)
for key in self.valid_columns:
if key in update_data:
setattr(entity, key, update_data[key])
elif isinstance(entity_data, self.Model):
for column in self.Model.__table__.columns.keys():
for column in self.valid_columns:
setattr(entity, column, getattr(entity_data, column))
await session.flush() # Make sure changes are flushed
@@ -297,6 +313,25 @@ class Repository[T: Base]:
logger.debug(f"No {self.Model.__name__} found to update: {entity_id}")
return None
async def update_fields(self, entity_id: Any, entity_data: dict[str, Any]) -> bool:
"""Update columns without reloading the model graph afterward."""
update_data = {k: v for k, v in entity_data.items() if k in self.valid_columns}
if not update_data:
return True
async with db.scoped_session(self.session_maker) as session:
conditions = [self.primary_key == entity_id]
if self.has_project_id and self.project_id is not None:
conditions.append(getattr(self.Model, "project_id") == self.project_id)
result = cast(
CursorResult[Any],
await session.execute(
sqlalchemy_update(self.Model).where(and_(*conditions)).values(**update_data)
),
)
return result.rowcount > 0
async def delete(self, entity_id: int) -> bool:
"""Delete an entity from the database."""
logger.debug(f"Deleting {self.Model.__name__}: {entity_id}")
@@ -41,6 +41,8 @@ class SearchRepository(Protocol):
note_types: Optional[List[str]] = None,
after_date: Optional[datetime] = None,
search_item_types: Optional[List[SearchItemType]] = None,
observation_categories: Optional[List[str]] = None,
relation_types: Optional[List[str]] = None,
metadata_filters: Optional[dict] = None,
retrieval_mode: SearchRetrievalMode = SearchRetrievalMode.FTS,
min_similarity: Optional[float] = None,
@@ -50,6 +52,24 @@ class SearchRepository(Protocol):
"""Search across indexed content."""
...
async def count(
self,
search_text: Optional[str] = None,
permalink: Optional[str] = None,
permalink_match: Optional[str] = None,
title: Optional[str] = None,
note_types: Optional[List[str]] = None,
after_date: Optional[datetime] = None,
search_item_types: Optional[List[SearchItemType]] = None,
observation_categories: Optional[List[str]] = None,
relation_types: Optional[List[str]] = None,
metadata_filters: Optional[dict] = None,
retrieval_mode: SearchRetrievalMode = SearchRetrievalMode.FTS,
min_similarity: Optional[float] = None,
) -> int:
"""Count indexed content matching the same filters as search."""
...
async def index_item(self, search_index_row: SearchIndexRow) -> None:
"""Index a single item."""
...
@@ -12,11 +12,12 @@ from dataclasses import dataclass, field, replace
from datetime import datetime
from typing import Any, Callable, Dict, List, Optional
import logfire
from loguru import logger
from sqlalchemy import Executable, Result, text
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from basic_memory import db, telemetry
from basic_memory import db
from basic_memory.repository.embedding_provider import EmbeddingProvider
from basic_memory.repository.search_index_row import SearchIndexRow
from basic_memory.repository.semantic_errors import (
@@ -217,6 +218,8 @@ class SearchRepositoryBase(ABC):
note_types: Optional[List[str]] = None,
after_date: Optional[datetime] = None,
search_item_types: Optional[List[SearchItemType]] = None,
observation_categories: Optional[List[str]] = None,
relation_types: Optional[List[str]] = None,
metadata_filters: Optional[Dict[str, Any]] = None,
retrieval_mode: SearchRetrievalMode = SearchRetrievalMode.FTS,
min_similarity: Optional[float] = None,
@@ -233,6 +236,8 @@ class SearchRepositoryBase(ABC):
note_types: Filter by note types (from metadata.note_type)
after_date: Filter by created_at > after_date
search_item_types: Filter by SearchItemType (ENTITY, OBSERVATION, RELATION)
observation_categories: Filter observation rows by category
relation_types: Filter relation rows by relation_type
metadata_filters: Structured frontmatter metadata filters
limit: Maximum results to return
offset: Number of results to skip
@@ -246,6 +251,26 @@ class SearchRepositoryBase(ABC):
"""
pass
async def count(
self,
search_text: Optional[str] = None,
permalink: Optional[str] = None,
permalink_match: Optional[str] = None,
title: Optional[str] = None,
note_types: Optional[List[str]] = None,
after_date: Optional[datetime] = None,
search_item_types: Optional[List[SearchItemType]] = None,
observation_categories: Optional[List[str]] = None,
relation_types: Optional[List[str]] = None,
metadata_filters: Optional[Dict[str, Any]] = None,
retrieval_mode: SearchRetrievalMode = SearchRetrievalMode.FTS,
min_similarity: Optional[float] = None,
) -> int:
"""Count results when a backend-specific COUNT query is available."""
if retrieval_mode != SearchRetrievalMode.FTS:
raise ValueError("Exact counts are only supported for full-text search retrieval.")
raise NotImplementedError("Backend search repositories must implement full-text counts.")
# ------------------------------------------------------------------
# Abstract methods — semantic search (backend-specific DB operations)
# ------------------------------------------------------------------
@@ -454,6 +479,26 @@ class SearchRepositoryBase(ABC):
logger.debug(f"Query executed successfully in {elapsed_time:.2f}s.")
return result
@staticmethod
def _append_in_filter(
conditions: list[str],
params: dict[str, Any],
*,
column: str,
values: list[Any] | None,
param_prefix: str,
) -> None:
"""Append a parameterized SQL IN clause for controlled column names."""
if not values:
return
placeholders: list[str] = []
for idx, value in enumerate(values):
param_name = f"{param_prefix}_{idx}"
params[param_name] = value.value if isinstance(value, SearchItemType) else value
placeholders.append(f":{param_name}")
conditions.append(f"{column} IN ({', '.join(placeholders)})")
async def delete_entity_vector_rows(self, entity_id: int) -> None:
"""Delete one entity's derived vector rows using the backend's cleanup path."""
await self._ensure_vector_tables()
@@ -845,7 +890,7 @@ class SearchRepositoryBase(ABC):
progress_callback(entity_id, completed_entities, total_entities)
prepare_window_size = self._vector_prepare_window_size()
with telemetry.started_span(
with logfire.span(
"basic_memory.vector_sync.batch",
project_id=self.project_id,
backend=backend_name,
@@ -1068,39 +1113,42 @@ class SearchRepositoryBase(ABC):
write_seconds_total=result.write_seconds_total,
)
batch_total_seconds = time.perf_counter() - batch_start
metric_attrs = {
batch_attrs = {
"backend": backend_name,
"skip_only_batch": result.embedding_jobs_total == 0,
}
telemetry.record_histogram(
"vector_sync_batch_total_seconds",
batch_total_seconds,
unit="s",
**metric_attrs,
logfire.metric_histogram("vector_sync_batch_total_seconds", unit="s").record(
batch_total_seconds, attributes=batch_attrs
)
telemetry.add_counter(
"vector_sync_entities_total", result.entities_total, **metric_attrs
logfire.metric_histogram("vector_sync_prepare_seconds", unit="s").record(
result.prepare_seconds_total, attributes=batch_attrs
)
telemetry.add_counter(
"vector_sync_entities_skipped",
result.entities_skipped,
**metric_attrs,
logfire.metric_histogram("vector_sync_queue_wait_seconds", unit="s").record(
result.queue_wait_seconds_total, attributes=batch_attrs
)
telemetry.add_counter(
"vector_sync_entities_deferred",
result.entities_deferred,
**metric_attrs,
logfire.metric_histogram("vector_sync_embed_seconds", unit="s").record(
result.embed_seconds_total, attributes=batch_attrs
)
telemetry.add_counter(
"vector_sync_embedding_jobs_total",
result.embedding_jobs_total,
**metric_attrs,
logfire.metric_histogram("vector_sync_write_seconds", unit="s").record(
result.write_seconds_total, attributes=batch_attrs
)
telemetry.add_counter("vector_sync_chunks_total", result.chunks_total, **metric_attrs)
telemetry.add_counter(
"vector_sync_chunks_skipped",
result.chunks_skipped,
**metric_attrs,
logfire.metric_counter("vector_sync_entities_total").add(
result.entities_total, attributes=batch_attrs
)
logfire.metric_counter("vector_sync_entities_skipped").add(
result.entities_skipped, attributes=batch_attrs
)
logfire.metric_counter("vector_sync_entities_deferred").add(
result.entities_deferred, attributes=batch_attrs
)
logfire.metric_counter("vector_sync_embedding_jobs_total").add(
result.embedding_jobs_total, attributes=batch_attrs
)
logfire.metric_counter("vector_sync_chunks_total").add(
result.chunks_total, attributes=batch_attrs
)
logfire.metric_counter("vector_sync_chunks_skipped").add(
result.chunks_skipped, attributes=batch_attrs
)
if batch_span is not None:
batch_span.set_attributes(
@@ -1673,36 +1721,12 @@ class SearchRepositoryBase(ABC):
shard_count: int,
remaining_jobs_after_shard: int,
) -> None:
"""Log completion and slow-entity warnings with a consistent format."""
backend_name = type(self).__name__.removesuffix("SearchRepository").lower()
metric_attrs = {
"backend": backend_name,
"skip_only_entity": entity_skipped and embedding_jobs_count == 0,
}
telemetry.record_histogram(
"vector_sync_prepare_seconds",
prepare_seconds,
unit="s",
**metric_attrs,
)
telemetry.record_histogram(
"vector_sync_queue_wait_seconds",
queue_wait_seconds,
unit="s",
**metric_attrs,
)
telemetry.record_histogram(
"vector_sync_embed_seconds",
embed_seconds,
unit="s",
**metric_attrs,
)
telemetry.record_histogram(
"vector_sync_write_seconds",
write_seconds,
unit="s",
**metric_attrs,
)
"""Log completion and slow-entity warnings with a consistent format.
Per-entity timings are aggregated into `VectorSyncBatchResult` and
recorded as batch-level histograms once the batch completes this
function stays on the per-entity hot path so it only emits logs.
"""
if total_seconds > 10:
logger.warning(
"Vector sync slow entity: project_id={project_id} entity_id={entity_id} "
@@ -1779,6 +1803,8 @@ class SearchRepositoryBase(ABC):
note_types: Optional[List[str]],
after_date: Optional[datetime],
search_item_types: Optional[List[SearchItemType]],
observation_categories: Optional[List[str]],
relation_types: Optional[List[str]],
metadata_filters: Optional[dict],
retrieval_mode: SearchRetrievalMode,
min_similarity: Optional[float] = None,
@@ -1812,6 +1838,8 @@ class SearchRepositoryBase(ABC):
note_types=note_types,
after_date=after_date,
search_item_types=search_item_types,
observation_categories=observation_categories,
relation_types=relation_types,
metadata_filters=metadata_filters,
min_similarity=min_similarity,
limit=limit,
@@ -1831,6 +1859,8 @@ class SearchRepositoryBase(ABC):
note_types=note_types,
after_date=after_date,
search_item_types=search_item_types,
observation_categories=observation_categories,
relation_types=relation_types,
metadata_filters=metadata_filters,
min_similarity=min_similarity,
limit=limit,
@@ -1860,6 +1890,8 @@ class SearchRepositoryBase(ABC):
note_types: Optional[List[str]],
after_date: Optional[datetime],
search_item_types: Optional[List[SearchItemType]],
observation_categories: Optional[List[str]],
relation_types: Optional[List[str]],
metadata_filters: Optional[dict],
min_similarity: Optional[float] = None,
limit: int,
@@ -1970,6 +2002,8 @@ class SearchRepositoryBase(ABC):
note_types,
after_date,
search_item_types,
observation_categories,
relation_types,
metadata_filters,
]
)
@@ -1983,6 +2017,8 @@ class SearchRepositoryBase(ABC):
note_types=note_types,
after_date=after_date,
search_item_types=search_item_types,
observation_categories=observation_categories,
relation_types=relation_types,
metadata_filters=metadata_filters,
retrieval_mode=SearchRetrievalMode.FTS,
limit=VECTOR_FILTER_SCAN_LIMIT,
@@ -2133,6 +2169,8 @@ class SearchRepositoryBase(ABC):
note_types: Optional[List[str]],
after_date: Optional[datetime],
search_item_types: Optional[List[SearchItemType]],
observation_categories: Optional[List[str]],
relation_types: Optional[List[str]],
metadata_filters: Optional[dict],
min_similarity: Optional[float] = None,
limit: int,
@@ -2157,6 +2195,8 @@ class SearchRepositoryBase(ABC):
note_types=note_types,
after_date=after_date,
search_item_types=search_item_types,
observation_categories=observation_categories,
relation_types=relation_types,
metadata_filters=metadata_filters,
retrieval_mode=SearchRetrievalMode.FTS,
limit=candidate_limit,
@@ -2172,6 +2212,8 @@ class SearchRepositoryBase(ABC):
note_types=note_types,
after_date=after_date,
search_item_types=search_item_types,
observation_categories=observation_categories,
relation_types=relation_types,
metadata_filters=metadata_filters,
min_similarity=min_similarity,
limit=candidate_limit,
@@ -94,9 +94,22 @@ class SQLiteSearchRepository(SearchRepositoryBase):
raise e
# Fail fast: create vector tables at startup so missing sqlite-vec
# or embedding provider errors surface immediately
# or embedding provider errors surface immediately.
# Trigger: the runtime semantic stack (sqlite-vec extension or embedding
# provider) is unavailable at startup.
# Why: failing the whole MCP boot for a search-only feature blocks
# Claude Desktop's handshake (#711). Keyword-only search is a
# reasonable fallback while the user resolves the dependency.
# Outcome: log the cause, mark this repository as semantic-disabled so
# downstream calls short-circuit cleanly, and let init complete.
if self._semantic_enabled:
await self._ensure_vector_tables()
try:
await self._ensure_vector_tables()
except SemanticDependenciesMissingError as exc:
logger.warning(
f"Semantic search disabled: {exc}. Falling back to keyword-only search."
)
self._semantic_enabled = False
# ------------------------------------------------------------------
# FTS5 query preparation (backend-specific)
@@ -350,7 +363,7 @@ class SQLiteSearchRepository(SearchRepositoryBase):
pass
try:
import sqlite_vec # type: ignore[import-not-found]
import sqlite_vec
except ImportError as exc:
raise SemanticDependenciesMissingError(
"sqlite-vec package is missing. "
@@ -374,6 +387,25 @@ class SQLiteSearchRepository(SearchRepositoryBase):
async_connection = await session.connection()
raw_connection = await async_connection.get_raw_connection()
driver_connection = raw_connection.driver_connection
# Trigger: the underlying CPython was built without sqlite extension support.
# Why: python.org's macOS installer ships a stripped sqlite3 module with no
# enable_load_extension; when uvx happens to pick that interpreter (#711),
# the AttributeError surfaces here and previously crashed startup before
# Claude Desktop could complete its MCP handshake.
# Outcome: convert to SemanticDependenciesMissingError so the init-time
# handler can degrade gracefully to keyword search instead of dying.
if not hasattr(driver_connection, "enable_load_extension"):
raise SemanticDependenciesMissingError(
"This Python build does not support SQLite extension loading "
"(no enable_load_extension on sqlite3.Connection). "
"Common cause: python.org Python on macOS. "
"Reinstall basic-memory under a Python that ships extension "
"support (uv-managed CPython, Homebrew Python, or the official "
"Docker image), or set semantic_search_enabled=false in config "
"to silence this and use keyword-only search."
)
await driver_connection.enable_load_extension(True)
await driver_connection.load_extension(sqlite_vec.loadable_path())
await driver_connection.enable_load_extension(False)
@@ -669,7 +701,11 @@ class SQLiteSearchRepository(SearchRepositoryBase):
# FTS search (backend-specific)
# ------------------------------------------------------------------
async def search(
@staticmethod
def _is_fts5_syntax_error(exc: Exception) -> bool:
return "fts5: syntax error" in str(exc).lower()
async def _build_fts_query_parts(
self,
search_text: Optional[str] = None,
permalink: Optional[str] = None,
@@ -678,32 +714,11 @@ class SQLiteSearchRepository(SearchRepositoryBase):
note_types: Optional[List[str]] = None,
after_date: Optional[datetime] = None,
search_item_types: Optional[List[SearchItemType]] = None,
observation_categories: Optional[List[str]] = None,
relation_types: Optional[List[str]] = None,
metadata_filters: Optional[dict] = None,
retrieval_mode: SearchRetrievalMode = SearchRetrievalMode.FTS,
min_similarity: Optional[float] = None,
limit: int = 10,
offset: int = 0,
) -> List[SearchIndexRow]:
"""Search across all indexed content using SQLite FTS5."""
# --- Dispatch vector / hybrid modes (shared logic) ---
dispatched = await self._dispatch_retrieval_mode(
search_text=search_text,
permalink=permalink,
permalink_match=permalink_match,
title=title,
note_types=note_types,
after_date=after_date,
search_item_types=search_item_types,
metadata_filters=metadata_filters,
retrieval_mode=retrieval_mode,
min_similarity=min_similarity,
limit=limit,
offset=offset,
)
if dispatched is not None:
return dispatched
# --- FTS mode (SQLite-specific) ---
) -> tuple[str, str, dict, str]:
"""Build SQLite FTS FROM/WHERE params shared by search and count."""
conditions = []
match_conditions = []
params = {}
@@ -753,14 +768,28 @@ class SQLiteSearchRepository(SearchRepositoryBase):
params["permalink"] = permalink_text
match_conditions.append("search_index.permalink MATCH :permalink")
# Handle entity type filter (parameterized for defense-in-depth)
if search_item_types:
type_placeholders = []
for idx, t in enumerate(search_item_types):
param_name = f"search_type_{idx}"
params[param_name] = t.value
type_placeholders.append(f":{param_name}")
conditions.append(f"search_index.type IN ({', '.join(type_placeholders)})")
# Handle typed search row filters (parameterized for defense-in-depth)
self._append_in_filter(
conditions,
params,
column="search_index.type",
values=search_item_types,
param_prefix="search_type",
)
self._append_in_filter(
conditions,
params,
column="search_index.category",
values=observation_categories,
param_prefix="observation_category",
)
self._append_in_filter(
conditions,
params,
column="search_index.relation_type",
values=relation_types,
param_prefix="relation_type",
)
# Handle note type filter (frontmatter type field, parameterized)
if note_types:
@@ -879,13 +908,66 @@ class SQLiteSearchRepository(SearchRepositoryBase):
params["project_id"] = self.project_id
conditions.append("search_index.project_id = :project_id")
# Build WHERE clause
where_clause = " AND ".join(conditions) if conditions else "1=1"
return from_clause, where_clause, params, order_by_clause
async def search(
self,
search_text: Optional[str] = None,
permalink: Optional[str] = None,
permalink_match: Optional[str] = None,
title: Optional[str] = None,
note_types: Optional[List[str]] = None,
after_date: Optional[datetime] = None,
search_item_types: Optional[List[SearchItemType]] = None,
observation_categories: Optional[List[str]] = None,
relation_types: Optional[List[str]] = None,
metadata_filters: Optional[dict] = None,
retrieval_mode: SearchRetrievalMode = SearchRetrievalMode.FTS,
min_similarity: Optional[float] = None,
limit: int = 10,
offset: int = 0,
) -> List[SearchIndexRow]:
"""Search across all indexed content using SQLite FTS5."""
# --- Dispatch vector / hybrid modes (shared logic) ---
dispatched = await self._dispatch_retrieval_mode(
search_text=search_text,
permalink=permalink,
permalink_match=permalink_match,
title=title,
note_types=note_types,
after_date=after_date,
search_item_types=search_item_types,
observation_categories=observation_categories,
relation_types=relation_types,
metadata_filters=metadata_filters,
retrieval_mode=retrieval_mode,
min_similarity=min_similarity,
limit=limit,
offset=offset,
)
if dispatched is not None:
return dispatched
# --- FTS mode (SQLite-specific) ---
from_clause, where_clause, params, order_by_clause = await self._build_fts_query_parts(
search_text=search_text,
permalink=permalink,
permalink_match=permalink_match,
title=title,
note_types=note_types,
after_date=after_date,
search_item_types=search_item_types,
observation_categories=observation_categories,
relation_types=relation_types,
metadata_filters=metadata_filters,
)
# set limit on search query
params["limit"] = limit
params["offset"] = offset
# Build WHERE clause
where_clause = " AND ".join(conditions) if conditions else "1=1"
sql = f"""
SELECT
search_index.project_id,
@@ -918,7 +1000,7 @@ class SQLiteSearchRepository(SearchRepositoryBase):
rows = result.fetchall()
except Exception as e:
# Handle FTS5 syntax errors and provide user-friendly feedback
if "fts5: syntax error" in str(e).lower(): # pragma: no cover
if self._is_fts5_syntax_error(e): # pragma: no cover
logger.warning(f"FTS5 syntax error for search term: {search_text}, error: {e}")
# Return empty results rather than crashing
return []
@@ -956,3 +1038,60 @@ class SQLiteSearchRepository(SearchRepositoryBase):
)
return results
async def count(
self,
search_text: Optional[str] = None,
permalink: Optional[str] = None,
permalink_match: Optional[str] = None,
title: Optional[str] = None,
note_types: Optional[List[str]] = None,
after_date: Optional[datetime] = None,
search_item_types: Optional[List[SearchItemType]] = None,
observation_categories: Optional[List[str]] = None,
relation_types: Optional[List[str]] = None,
metadata_filters: Optional[dict] = None,
retrieval_mode: SearchRetrievalMode = SearchRetrievalMode.FTS,
min_similarity: Optional[float] = None,
) -> int:
"""Count indexed content matching the SQLite FTS query."""
if retrieval_mode != SearchRetrievalMode.FTS:
return await super().count(
search_text=search_text,
permalink=permalink,
permalink_match=permalink_match,
title=title,
note_types=note_types,
after_date=after_date,
search_item_types=search_item_types,
observation_categories=observation_categories,
relation_types=relation_types,
metadata_filters=metadata_filters,
retrieval_mode=retrieval_mode,
min_similarity=min_similarity,
)
from_clause, where_clause, params, _order_by_clause = await self._build_fts_query_parts(
search_text=search_text,
permalink=permalink,
permalink_match=permalink_match,
title=title,
note_types=note_types,
after_date=after_date,
search_item_types=search_item_types,
observation_categories=observation_categories,
relation_types=relation_types,
metadata_filters=metadata_filters,
)
sql = f"SELECT COUNT(*) FROM {from_clause} WHERE {where_clause}"
logger.trace(f"Count {sql} params: {params}")
try:
async with db.scoped_session(self.session_maker) as session:
result = await session.execute(text(sql), params)
return int(result.scalar_one())
except Exception as e:
if self._is_fts5_syntax_error(e): # pragma: no cover
logger.warning(f"FTS5 syntax error for search term: {search_text}, error: {e}")
return 0
logger.error(f"Database error during search count: {e}")
raise
+5
View File
@@ -63,8 +63,10 @@ class WorkspaceInfo(BaseModel):
tenant_id: str = Field(..., description="Workspace tenant identifier")
workspace_type: str = Field(..., description="Workspace type (personal or organization)")
slug: str = Field(..., description="Stable workspace slug for qualified project routing")
name: str = Field(..., description="Workspace display name")
role: str = Field(..., description="Current user's role in the workspace")
is_default: bool = Field(..., description="Whether this is the default cloud workspace")
organization_id: str | None = Field(None, description="Organization ID for org workspaces")
has_active_subscription: bool = Field(
default=False, description="Whether the workspace has an active subscription"
@@ -78,6 +80,9 @@ class WorkspaceListResponse(BaseModel):
default_factory=list, description="Available workspaces"
)
count: int = Field(default=0, description="Number of available workspaces")
default_workspace_id: str | None = Field(
default=None, description="Default workspace tenant ID when available"
)
current_workspace_id: str | None = Field(
default=None, description="Current workspace tenant ID when available"
)
+1 -1
View File
@@ -103,7 +103,7 @@ MemoryUrl = Annotated[
memory_url = TypeAdapter(MemoryUrl)
def memory_url_path(url: memory_url) -> str: # pyright: ignore
def memory_url_path(url: str) -> str:
"""
Returns the uri for a url value by removing the prefix "memory://" from a given MemoryUrl.
+1 -1
View File
@@ -194,7 +194,7 @@ class EntityResponse(SQLAlchemyModel):
note_type: NoteType
# COMPAT(v0.18): old clients expect entity_type; remove when no longer needed
@computed_field # type: ignore[prop-decorator]
@computed_field
@property
def entity_type(self) -> str:
return self.note_type
+9
View File
@@ -46,6 +46,8 @@ class SearchQuery(BaseModel):
- metadata_filters: Structured frontmatter filters (field -> value)
- tags: Convenience frontmatter tag filter
- status: Convenience frontmatter status filter
- observation_categories: Limit observation results to categories
- relation_types: Limit relation results to relationship types
Boolean search examples:
- "python AND flask" - Find items with both terms
@@ -67,6 +69,8 @@ class SearchQuery(BaseModel):
metadata_filters: Optional[dict[str, Any]] = None # Structured frontmatter filters
tags: Optional[List[str]] = None # Convenience tag filter
status: Optional[str] = None # Convenience status filter
observation_categories: Optional[List[str]] = None # Filter observations by category
relation_types: Optional[List[str]] = None # Filter relations by relation_type
retrieval_mode: SearchRetrievalMode = SearchRetrievalMode.FTS
min_similarity: Optional[float] = None # Per-query override for semantic_min_similarity
@@ -85,6 +89,8 @@ class SearchQuery(BaseModel):
status_is_empty = self.status is None or (isinstance(self.status, str) and not self.status)
note_types_is_empty = not self.note_types
entity_types_is_empty = not self.entity_types
observation_categories_is_empty = not self.observation_categories
relation_types_is_empty = not self.relation_types
return (
self.permalink is None
and self.permalink_match is None
@@ -96,6 +102,8 @@ class SearchQuery(BaseModel):
and metadata_is_empty
and tags_is_empty
and status_is_empty
and observation_categories_is_empty
and relation_types_is_empty
)
def has_boolean_operators(self) -> bool:
@@ -142,4 +150,5 @@ class SearchResponse(BaseModel):
results: List[SearchResult]
current_page: int
page_size: int
total: int = 0
has_more: bool = False
+1 -1
View File
@@ -5,7 +5,7 @@ from typing import TYPE_CHECKING, Dict, List, Set
from pydantic import BaseModel, Field
# avoid cirular imports
# avoid circular imports
if TYPE_CHECKING: # pragma: no cover
from basic_memory.sync.sync_service import SyncReport
+6 -6
View File
@@ -10,7 +10,7 @@ from typing import List, Optional, Tuple, TYPE_CHECKING
from loguru import logger
from sqlalchemy import text
from basic_memory import telemetry
import logfire
from basic_memory.repository.entity_repository import EntityRepository
from basic_memory.repository.observation_repository import ObservationRepository
from basic_memory.repository.postgres_search_repository import PostgresSearchRepository
@@ -111,7 +111,7 @@ class ContextService:
f"Building context for URI: '{memory_url}' depth: '{depth}' since: '{since}' limit: '{limit}' offset: '{offset}' max_related: '{max_related}'"
)
with telemetry.scope(
with logfire.span(
"memory.build_context",
domain="memory",
action="build_context",
@@ -122,7 +122,7 @@ class ContextService:
fetch_limit = limit + 1
normalized_path: Optional[str] = None
with telemetry.scope(
with logfire.span(
"memory.build_context.resolve_primary",
domain="memory",
action="build_context",
@@ -180,7 +180,7 @@ class ContextService:
type_id_pairs = [(r.type, r.id) for r in primary] if primary else []
logger.debug(f"found primary type_id_pairs: {len(type_id_pairs)}")
with telemetry.scope(
with logfire.span(
"memory.build_context.find_related",
domain="memory",
action="build_context",
@@ -202,7 +202,7 @@ class ContextService:
observations_by_entity = {}
if include_observations and entity_ids:
with telemetry.scope(
with logfire.span(
"memory.build_context.load_observations",
domain="memory",
action="build_context",
@@ -226,7 +226,7 @@ class ContextService:
has_more=has_more,
)
with telemetry.scope(
with logfire.span(
"memory.build_context.shape_results",
domain="memory",
action="build_context",
File diff suppressed because it is too large Load Diff
+12 -6
View File
@@ -12,7 +12,7 @@ import aiofiles
import yaml
from basic_memory import telemetry
import logfire
from basic_memory import file_utils
if TYPE_CHECKING: # pragma: no cover
@@ -89,7 +89,7 @@ class FileService:
"""
logger.debug(f"Reading entity content, entity_id={entity.id}, permalink={entity.permalink}")
with telemetry.scope(
with logfire.span(
"file_service.read_content",
domain="file_service",
action="read_content",
@@ -191,7 +191,7 @@ class FileService:
full_path = path_obj if path_obj.is_absolute() else self.base_path / path_obj
try:
with telemetry.scope(
with logfire.span(
"file_service.write",
domain="file_service",
action="write",
@@ -249,7 +249,7 @@ class FileService:
full_path = path_obj if path_obj.is_absolute() else self.base_path / path_obj
try:
with telemetry.scope(
with logfire.span(
"file_service.read_content",
domain="file_service",
action="read_content",
@@ -273,6 +273,9 @@ class FileService:
logger.warning("File not found", operation="read_file_content", path=str(full_path))
raise
except Exception as e:
if isinstance(e, FileNotFoundError):
logger.warning("File not found", operation="read_file", path=str(full_path))
raise
logger.exception("File read error", path=str(full_path), error=str(e))
raise FileOperationError(f"Failed to read file: {e}")
@@ -296,7 +299,7 @@ class FileService:
full_path = path_obj if path_obj.is_absolute() else self.base_path / path_obj
try:
with telemetry.scope(
with logfire.span(
"file_service.read_content",
domain="file_service",
action="read_content",
@@ -339,7 +342,7 @@ class FileService:
full_path = path_obj if path_obj.is_absolute() else self.base_path / path_obj
try:
with telemetry.scope(
with logfire.span(
"file_service.read",
domain="file_service",
action="read",
@@ -366,6 +369,9 @@ class FileService:
)
return content, checksum
except FileNotFoundError as e:
logger.warning("File not found", operation="read_file", path=str(full_path))
raise FileOperationError(f"Failed to read file: {e}") from e
except Exception as e:
logger.exception("File read error", path=str(full_path), error=str(e))
raise FileOperationError(f"Failed to read file: {e}")
+11 -3
View File
@@ -99,18 +99,23 @@ async def initialize_file_sync(
)
project_repository = ProjectRepository(session_maker)
# Filter to constrained project if MCP server was started with --project.
# Applied to both the initial background sync and the watch service so that
# running multiple `basic-memory mcp --project X` processes does not produce
# duplicate watchers fighting over the same files.
constrained_project = os.environ.get("BASIC_MEMORY_MCP_PROJECT")
# Initialize watch service
watch_service = WatchService(
app_config=app_config,
project_repository=project_repository,
quiet=quiet,
constrained_project=constrained_project,
)
# Get active projects
active_projects = await project_repository.get_active_projects()
# Filter to constrained project if MCP server was started with --project
constrained_project = os.environ.get("BASIC_MEMORY_MCP_PROJECT")
if constrained_project:
active_projects = [p for p in active_projects if p.name == constrained_project]
logger.info(f"Background sync constrained to project: {constrained_project}")
@@ -154,7 +159,10 @@ async def initialize_file_sync(
# Don't await the tasks - let them run in background while we continue
# Then start the watch service in the background
logger.info("Starting watch service for all projects")
if constrained_project:
logger.info(f"Starting watch service constrained to project: {constrained_project}")
else:
logger.info("Starting watch service for all projects")
# run the watch service
await watch_service.run()
+3 -5
View File
@@ -1137,12 +1137,10 @@ class ProjectService:
# Get watch service status if available
watch_status = None
watch_status_path = Path.home() / ".basic-memory" / WATCH_STATUS_JSON
watch_status_path = self.config_manager.config.data_dir_path / WATCH_STATUS_JSON
if watch_status_path.exists():
try: # pragma: no cover
watch_status = json.loads( # pragma: no cover
watch_status_path.read_text(encoding="utf-8")
)
try:
watch_status = json.loads(watch_status_path.read_text(encoding="utf-8"))
except Exception: # pragma: no cover
pass
+300 -221
View File
@@ -3,6 +3,7 @@
import asyncio
import ast
import re
from dataclasses import dataclass
from datetime import datetime
from typing import List, Optional, Set, Dict, Any
@@ -11,7 +12,8 @@ from fastapi import BackgroundTasks
from loguru import logger
from sqlalchemy import text
from basic_memory import telemetry
import logfire
from basic_memory.models import Entity
from basic_memory.repository import EntityRepository
from basic_memory.repository.search_repository import (
@@ -63,6 +65,24 @@ FTS_RELAXED_STOPWORDS = {
}
@dataclass(frozen=True)
class _PreparedSearchQuery:
"""Normalized query inputs shared by search and count."""
search_text: str | None
permalink: str | None
permalink_match: str | None
title: str | None
note_types: list[str] | None
search_item_types: list[SearchItemType] | None
observation_categories: list[str] | None
relation_types: list[str] | None
after_date: datetime | None
metadata_filters: dict[str, Any] | None
retrieval_mode: SearchRetrievalMode
min_similarity: float | None
def _strip_nul(value: str) -> str:
"""Strip NUL bytes that PostgreSQL text columns cannot store.
@@ -131,27 +151,20 @@ class SearchService:
logger.info("Reindex complete")
async def search(self, query: SearchQuery, limit=10, offset=0) -> List[SearchIndexRow]:
"""Search across all indexed content.
def _prepare_query(self, query: SearchQuery) -> _PreparedSearchQuery | None:
"""Normalize a SearchQuery into repository arguments."""
search_text = query.text
tags = query.tags
Supports three modes:
1. Exact permalink: finds direct matches for a specific path
2. Pattern match: handles * wildcards in paths
3. Text search: full-text search across title/content
"""
# Support tag:<tag> shorthand by mapping to tags filter
if query.text:
text = query.text.strip()
if text.lower().startswith("tag:"):
tag_values = re.split(r"[,\s]+", text[4:].strip())
tags = [t for t in tag_values if t]
if tags:
query.tags = tags
query.text = None
if query.no_criteria():
logger.debug("no criteria passed to query")
return []
# Support tag:<tag> shorthand by mapping to tags filter.
if search_text is not None:
search_text = search_text.strip() or None
if search_text and search_text.lower().startswith("tag:"):
tag_values = re.split(r"[,\s]+", search_text[4:].strip())
parsed_tags = [t for t in tag_values if t]
if parsed_tags:
tags = parsed_tags
search_text = None
after_date = (
(
@@ -163,67 +176,146 @@ class SearchService:
else None
)
# Merge structured metadata filters (explicit + convenience fields)
# Merge structured metadata filters (explicit + convenience fields).
metadata_filters: Optional[Dict[str, Any]] = None
if query.metadata_filters or query.tags or query.status:
if query.metadata_filters or tags or query.status:
metadata_filters = dict(query.metadata_filters or {})
if query.tags:
metadata_filters.setdefault("tags", query.tags)
if tags:
metadata_filters.setdefault("tags", tags)
if query.status:
metadata_filters.setdefault("status", query.status)
retrieval_mode = query.retrieval_mode or SearchRetrievalMode.FTS
strict_search_text = query.text
has_query = bool(
strict_search_text or query.title or query.permalink or query.permalink_match
)
has_filters = bool(
metadata_filters
or query.note_types
or query.entity_types
or after_date
or query.tags
or query.status
prepared = _PreparedSearchQuery(
search_text=search_text,
permalink=query.permalink,
permalink_match=query.permalink_match,
title=query.title,
note_types=query.note_types,
search_item_types=query.entity_types,
observation_categories=query.observation_categories,
relation_types=query.relation_types,
after_date=after_date,
metadata_filters=metadata_filters,
retrieval_mode=query.retrieval_mode or SearchRetrievalMode.FTS,
min_similarity=query.min_similarity,
)
with telemetry.scope(
has_criteria = bool(
prepared.search_text
or prepared.permalink
or prepared.permalink_match
or prepared.title
or prepared.note_types
or prepared.search_item_types
or prepared.observation_categories
or prepared.relation_types
or prepared.after_date
or prepared.metadata_filters
)
if not has_criteria:
logger.debug("no criteria passed to query")
return None
return prepared
@staticmethod
def _prepared_has_filters(prepared: _PreparedSearchQuery) -> bool:
return bool(
prepared.metadata_filters
or prepared.note_types
or prepared.search_item_types
or prepared.observation_categories
or prepared.relation_types
or prepared.after_date
)
async def _search_repository(
self,
prepared: _PreparedSearchQuery,
*,
search_text: str | None,
limit: int,
offset: int,
) -> List[SearchIndexRow]:
return await self.repository.search(
search_text=search_text,
permalink=prepared.permalink,
permalink_match=prepared.permalink_match,
title=prepared.title,
note_types=prepared.note_types,
search_item_types=prepared.search_item_types,
observation_categories=prepared.observation_categories,
relation_types=prepared.relation_types,
after_date=prepared.after_date,
metadata_filters=prepared.metadata_filters,
retrieval_mode=prepared.retrieval_mode,
min_similarity=prepared.min_similarity,
limit=limit,
offset=offset,
)
async def _count_repository(
self,
prepared: _PreparedSearchQuery,
*,
search_text: str | None,
) -> int:
return await self.repository.count(
search_text=search_text,
permalink=prepared.permalink,
permalink_match=prepared.permalink_match,
title=prepared.title,
note_types=prepared.note_types,
search_item_types=prepared.search_item_types,
observation_categories=prepared.observation_categories,
relation_types=prepared.relation_types,
after_date=prepared.after_date,
metadata_filters=prepared.metadata_filters,
retrieval_mode=prepared.retrieval_mode,
min_similarity=prepared.min_similarity,
)
async def search(self, query: SearchQuery, limit=10, offset=0) -> List[SearchIndexRow]:
"""Search across all indexed content.
Supports three modes:
1. Exact permalink: finds direct matches for a specific path
2. Pattern match: handles * wildcards in paths
3. Text search: full-text search across title/content
"""
prepared = self._prepare_query(query)
if prepared is None:
return []
strict_search_text = prepared.search_text
has_query = bool(
strict_search_text or prepared.title or prepared.permalink or prepared.permalink_match
)
has_filters = self._prepared_has_filters(prepared)
with logfire.span(
"search.execute",
retrieval_mode=retrieval_mode.value,
retrieval_mode=prepared.retrieval_mode.value,
has_query=has_query,
has_filters=has_filters,
limit=limit,
offset=offset,
):
logger.trace(f"Searching with query: {query}")
with telemetry.scope(
"search.repository_query",
retrieval_mode=retrieval_mode.value,
phase="repository_query",
has_query=has_query,
has_filters=has_filters,
):
# First pass: preserve existing strict search behavior.
results = await self.repository.search(
search_text=strict_search_text,
permalink=query.permalink,
permalink_match=query.permalink_match,
title=query.title,
note_types=query.note_types,
search_item_types=query.entity_types,
after_date=after_date,
metadata_filters=metadata_filters,
retrieval_mode=retrieval_mode,
min_similarity=query.min_similarity,
limit=limit,
offset=offset,
)
results = await self._search_repository(
prepared,
search_text=strict_search_text,
limit=limit,
offset=offset,
)
# Trigger: strict FTS with plain multi-term text returned no results.
# Why: natural-language queries often include stopwords that over-constrain implicit AND.
# Outcome: retry once with relaxed OR terms while preserving explicit boolean intent.
if results:
return results
if not self._is_relaxed_fts_fallback_eligible(query, strict_search_text, retrieval_mode):
if not self._is_relaxed_fts_fallback_eligible(
query, strict_search_text, prepared.retrieval_mode
):
return results
assert strict_search_text is not None
@@ -235,34 +327,58 @@ class SearchService:
"Strict FTS returned 0 results; retrying relaxed FTS query "
f"strict='{strict_search_text}' relaxed='{relaxed_search_text}'"
)
with telemetry.scope(
with logfire.span(
"search.relaxed_fts_retry",
retrieval_mode=retrieval_mode.value,
retrieval_mode=prepared.retrieval_mode.value,
token_count=len(self._tokenize_fts_text(strict_search_text)),
limit=limit,
offset=offset,
):
with telemetry.scope(
"search.repository_query",
retrieval_mode=retrieval_mode.value,
phase="repository_query",
has_query=has_query,
has_filters=has_filters,
):
return await self.repository.search(
search_text=relaxed_search_text,
permalink=query.permalink,
permalink_match=query.permalink_match,
title=query.title,
note_types=query.note_types,
search_item_types=query.entity_types,
after_date=after_date,
metadata_filters=metadata_filters,
retrieval_mode=retrieval_mode,
min_similarity=query.min_similarity,
limit=limit,
offset=offset,
)
return await self._search_repository(
prepared,
search_text=relaxed_search_text,
limit=limit,
offset=offset,
)
async def count(self, query: SearchQuery) -> int:
"""Count all indexed rows matching a query."""
prepared = self._prepare_query(query)
if prepared is None:
return 0
strict_search_text = prepared.search_text
has_query = bool(
strict_search_text or prepared.title or prepared.permalink or prepared.permalink_match
)
has_filters = self._prepared_has_filters(prepared)
with logfire.span(
"search.count",
retrieval_mode=prepared.retrieval_mode.value,
has_query=has_query,
has_filters=has_filters,
):
total = await self._count_repository(prepared, search_text=strict_search_text)
if total > 0:
return total
if not self._is_relaxed_fts_fallback_eligible(
query, strict_search_text, prepared.retrieval_mode
):
return total
assert strict_search_text is not None
relaxed_search_text = self._build_relaxed_fts_query(strict_search_text)
if relaxed_search_text == strict_search_text:
return total
with logfire.span(
"search.count.relaxed_fts_retry",
retrieval_mode=prepared.retrieval_mode.value,
token_count=len(self._tokenize_fts_text(strict_search_text)),
):
return await self._count_repository(prepared, search_text=relaxed_search_text)
@staticmethod
def _tokenize_fts_text(search_text: str) -> list[str]:
@@ -396,17 +512,8 @@ class SearchService:
f"permalink={entity.permalink} project_id={entity.project_id}"
)
try:
with telemetry.scope(
"search.index_entity_data",
phase="index_entity_data",
result_count=1,
):
with telemetry.scope(
"search.index.delete_existing",
phase="delete_existing",
result_count=1,
):
await self.repository.delete_by_entity_id(entity_id=entity.id)
with logfire.span("search.index_entity_data", entity_id=entity.id):
await self.repository.delete_by_entity_id(entity_id=entity.id)
if entity.is_markdown:
await self.index_entity_markdown(entity, content)
@@ -676,28 +783,23 @@ class SearchService:
self,
entity: Entity,
) -> None:
with telemetry.scope(
"search.index_file",
phase="index_file",
result_count=1,
):
# Index entity file with no content
await self.repository.index_item(
SearchIndexRow(
id=entity.id,
entity_id=entity.id,
type=SearchItemType.ENTITY.value,
title=_strip_nul(entity.title),
permalink=entity.permalink, # Required for Postgres NOT NULL constraint
file_path=entity.file_path,
metadata={
"note_type": entity.note_type,
},
created_at=entity.created_at,
updated_at=_mtime_to_datetime(entity),
project_id=entity.project_id,
)
# Index entity file with no content
await self.repository.index_item(
SearchIndexRow(
id=entity.id,
entity_id=entity.id,
type=SearchItemType.ENTITY.value,
title=_strip_nul(entity.title),
permalink=entity.permalink, # Required for Postgres NOT NULL constraint
file_path=entity.file_path,
metadata={
"note_type": entity.note_type,
},
created_at=entity.created_at,
updated_at=_mtime_to_datetime(entity),
project_id=entity.project_id,
)
)
async def index_entity_markdown(
self,
@@ -730,11 +832,7 @@ class SearchService:
The project_id is automatically added by the repository when indexing.
"""
with telemetry.scope(
"search.index_markdown",
phase="index_markdown",
result_count=1,
):
with logfire.span("search.index_markdown", entity_id=entity.id):
rows_to_index = []
content_stems = []
@@ -743,51 +841,76 @@ class SearchService:
content_stems.extend(title_variants)
if content is None:
with telemetry.scope(
"search.index.read_content",
phase="read_content",
result_count=1,
):
content = await self.file_service.read_entity_content(entity)
content = await self.file_service.read_entity_content(entity)
if content:
content_stems.append(content)
content_snippet = _strip_nul(content)
with telemetry.scope(
"search.index.build_rows",
phase="build_rows",
result_count=1,
):
if entity.permalink:
content_stems.extend(self._generate_variants(entity.permalink))
if entity.permalink:
content_stems.extend(self._generate_variants(entity.permalink))
content_stems.extend(self._generate_variants(entity.file_path))
content_stems.extend(self._generate_variants(entity.file_path))
entity_tags = self._extract_entity_tags(entity)
if entity_tags:
content_stems.extend(entity_tags)
entity_tags = self._extract_entity_tags(entity)
if entity_tags:
content_stems.extend(entity_tags)
entity_content_stems = _strip_nul(
"\n".join(p for p in content_stems if p and p.strip())
entity_content_stems = _strip_nul(
"\n".join(p for p in content_stems if p and p.strip())
)
if len(entity_content_stems) > MAX_CONTENT_STEMS_SIZE: # pragma: no cover
entity_content_stems = entity_content_stems[
:MAX_CONTENT_STEMS_SIZE
] # pragma: no cover
rows_to_index.append(
SearchIndexRow(
id=entity.id,
type=SearchItemType.ENTITY.value,
title=_strip_nul(entity.title),
content_stems=entity_content_stems,
content_snippet=content_snippet,
permalink=entity.permalink,
file_path=entity.file_path,
entity_id=entity.id,
metadata={
"note_type": entity.note_type,
},
created_at=entity.created_at,
updated_at=_mtime_to_datetime(entity),
project_id=entity.project_id,
)
)
if len(entity_content_stems) > MAX_CONTENT_STEMS_SIZE: # pragma: no cover
entity_content_stems = entity_content_stems[
seen_permalinks: set[str] = {entity.permalink} if entity.permalink else set()
for obs in entity.observations:
obs_permalink = obs.permalink
if obs_permalink in seen_permalinks:
logger.debug(f"Skipping duplicate observation permalink: {obs_permalink}")
continue
seen_permalinks.add(obs_permalink)
obs_content_stems = _strip_nul(
"\n".join(p for p in self._generate_variants(obs.content) if p and p.strip())
)
if len(obs_content_stems) > MAX_CONTENT_STEMS_SIZE: # pragma: no cover
obs_content_stems = obs_content_stems[
:MAX_CONTENT_STEMS_SIZE
] # pragma: no cover
rows_to_index.append(
SearchIndexRow(
id=entity.id,
type=SearchItemType.ENTITY.value,
title=_strip_nul(entity.title),
content_stems=entity_content_stems,
content_snippet=content_snippet,
permalink=entity.permalink,
id=obs.id,
type=SearchItemType.OBSERVATION.value,
title=_strip_nul(f"{obs.category}: {obs.content[:100]}..."),
content_stems=obs_content_stems,
content_snippet=_strip_nul(obs.content),
permalink=obs_permalink,
file_path=entity.file_path,
category=obs.category,
entity_id=entity.id,
metadata={
"note_type": entity.note_type,
"tags": obs.tags,
},
created_at=entity.created_at,
updated_at=_mtime_to_datetime(entity),
@@ -795,79 +918,35 @@ class SearchService:
)
)
seen_permalinks: set[str] = {entity.permalink} if entity.permalink else set()
for obs in entity.observations:
obs_permalink = obs.permalink
if obs_permalink in seen_permalinks:
logger.debug(f"Skipping duplicate observation permalink: {obs_permalink}")
continue
seen_permalinks.add(obs_permalink)
for rel in entity.outgoing_relations:
relation_title = _strip_nul(
f"{rel.from_entity.title} -> {rel.to_entity.title}"
if rel.to_entity
else f"{rel.from_entity.title}"
)
obs_content_stems = _strip_nul(
"\n".join(
p for p in self._generate_variants(obs.content) if p and p.strip()
)
)
if len(obs_content_stems) > MAX_CONTENT_STEMS_SIZE: # pragma: no cover
obs_content_stems = obs_content_stems[
:MAX_CONTENT_STEMS_SIZE
] # pragma: no cover
rows_to_index.append(
SearchIndexRow(
id=obs.id,
type=SearchItemType.OBSERVATION.value,
title=_strip_nul(f"{obs.category}: {obs.content[:100]}..."),
content_stems=obs_content_stems,
content_snippet=_strip_nul(obs.content),
permalink=obs_permalink,
file_path=entity.file_path,
category=obs.category,
entity_id=entity.id,
metadata={
"tags": obs.tags,
},
created_at=entity.created_at,
updated_at=_mtime_to_datetime(entity),
project_id=entity.project_id,
)
rel_content_stems = _strip_nul(
"\n".join(p for p in self._generate_variants(relation_title) if p and p.strip())
)
rows_to_index.append(
SearchIndexRow(
id=rel.id,
title=relation_title,
permalink=rel.permalink,
content_stems=rel_content_stems,
file_path=entity.file_path,
type=SearchItemType.RELATION.value,
entity_id=entity.id,
from_id=rel.from_id,
to_id=rel.to_id,
relation_type=rel.relation_type,
created_at=entity.created_at,
updated_at=_mtime_to_datetime(entity),
project_id=entity.project_id,
)
)
for rel in entity.outgoing_relations:
relation_title = _strip_nul(
f"{rel.from_entity.title} -> {rel.to_entity.title}"
if rel.to_entity
else f"{rel.from_entity.title}"
)
rel_content_stems = _strip_nul(
"\n".join(
p for p in self._generate_variants(relation_title) if p and p.strip()
)
)
rows_to_index.append(
SearchIndexRow(
id=rel.id,
title=relation_title,
permalink=rel.permalink,
content_stems=rel_content_stems,
file_path=entity.file_path,
type=SearchItemType.RELATION.value,
entity_id=entity.id,
from_id=rel.from_id,
to_id=rel.to_id,
relation_type=rel.relation_type,
created_at=entity.created_at,
updated_at=_mtime_to_datetime(entity),
project_id=entity.project_id,
)
)
with telemetry.scope(
"search.index.bulk_upsert",
phase="bulk_upsert",
result_count=len(rows_to_index),
):
await self.repository.bulk_index_items(rows_to_index)
await self.repository.bulk_index_items(rows_to_index)
async def delete_by_permalink(self, permalink: str):
"""Delete an item from the search index."""
+123 -89
View File
@@ -15,11 +15,17 @@ import aiofiles.os
from loguru import logger
from sqlalchemy.exc import IntegrityError
from basic_memory import telemetry
import logfire
from basic_memory import db
from basic_memory.config import BasicMemoryConfig, ConfigManager
from basic_memory.file_utils import compute_checksum, has_frontmatter
from basic_memory.indexing import BatchIndexer, IndexFileMetadata, IndexInputFile, IndexProgress
from basic_memory.file_utils import ParseError, compute_checksum, remove_frontmatter
from basic_memory.indexing import (
BatchIndexer,
IndexFileMetadata,
IndexInputFile,
IndexProgress,
SyncedMarkdownFile,
)
from basic_memory.indexing.batching import build_index_batches
from basic_memory.indexing.models import (
IndexedEntity,
@@ -308,7 +314,7 @@ class SyncService:
start_time = time.time()
sync_start_timestamp = time.time() # Capture at start for watermark
with telemetry.operation(
with logfire.span(
"sync.project.run",
project_name=project_name,
force_full=force_full,
@@ -319,7 +325,7 @@ class SyncService:
# initial paths from db to sync
# path -> checksum
with telemetry.scope("sync.project.scan", force_full=force_full):
with logfire.span("sync.project.scan", force_full=force_full):
report = await self.scan(directory, force_full=force_full)
# order of sync matters to resolve relations effectively
@@ -328,7 +334,7 @@ class SyncService:
+ f"deleted_files={len(report.deleted)}, moved_files={len(report.moves)}"
)
with telemetry.scope(
with logfire.span(
"sync.project.apply_changes",
new_count=len(report.new),
modified_count=len(report.modified),
@@ -375,9 +381,7 @@ class SyncService:
# not only the lightweight diff summary.
# Outcome: full reindex can heal relation state even when the diff report is empty.
if report.total > 0 or (force_full and indexed_entities):
with telemetry.scope(
"sync.project.resolve_relations", relation_scope="all_pending"
):
with logfire.span("sync.project.resolve_relations", relation_scope="all_pending"):
synced_entity_ids.extend(await self.resolve_relations())
else:
logger.info("Skipping relation resolution - no file changes detected")
@@ -386,7 +390,7 @@ class SyncService:
synced_entity_ids = list(dict.fromkeys(synced_entity_ids))
if synced_entity_ids and sync_embeddings and self.app_config.semantic_search_enabled:
try:
with telemetry.scope(
with logfire.span(
"sync.project.sync_embeddings",
entity_count=len(synced_entity_ids),
):
@@ -410,7 +414,7 @@ class SyncService:
# Update scan watermark after successful sync
# Use the timestamp from sync start (not end) to ensure we catch files
# created during the sync on the next iteration
with telemetry.scope("sync.project.update_watermark"):
with logfire.span("sync.project.update_watermark"):
current_file_count = await self._quick_count_files(directory)
if self.entity_repository.project_id is not None:
project = await self.project_repository.find_by_id(
@@ -757,7 +761,7 @@ class SyncService:
if project is None:
raise ValueError(f"Project not found: {self.entity_repository.project_id}")
with telemetry.scope("sync.project.select_scan_strategy", force_full=force_full):
with logfire.span("sync.project.select_scan_strategy", force_full=force_full):
# Step 1: Quick file count
logger.debug("Counting files in directory")
current_count = await self._quick_count_files(directory)
@@ -801,7 +805,7 @@ class SyncService:
logger.warning("No scan watermark available, falling back to full scan")
scan_coro = self._scan_directory_full(directory)
with telemetry.scope("sync.project.filesystem_scan", scan_type=scan_type):
with logfire.span("sync.project.filesystem_scan", scan_type=scan_type):
file_paths_to_scan = await scan_coro
if scan_type == "incremental":
logger.debug(
@@ -867,7 +871,7 @@ class SyncService:
# Step 4: Detect moves (for both full and incremental scans)
# Check if any "new" files are actually moves by matching checksums
with telemetry.scope("sync.project.detect_moves", new_count=len(report.new)):
with logfire.span("sync.project.detect_moves", new_count=len(report.new)):
for new_path in list(
report.new
): # Use list() to allow modification during iteration
@@ -902,7 +906,7 @@ class SyncService:
# Step 5: Detect deletions (only for full scans)
# Incremental scans can't reliably detect deletions since they only see modified files
if scan_type in ("full_initial", "full_deletions", "full_fallback", "full_forced"):
with telemetry.scope("sync.project.detect_deletions", scan_type=scan_type):
with logfire.span("sync.project.detect_deletions", scan_type=scan_type):
# Use optimized query for just file paths (not full entities)
db_file_paths = await self.entity_repository.get_all_file_paths()
logger.debug(f"Found {len(db_file_paths)} db paths for deletion detection")
@@ -992,7 +996,7 @@ class SyncService:
except FileNotFoundError:
# File exists in database but not on filesystem
# This indicates a database/filesystem inconsistency - treat as deletion
with telemetry.scope(
with logfire.span(
"sync.file.failure",
failure_type="file_not_found",
path=path,
@@ -1014,7 +1018,7 @@ class SyncService:
if isinstance(e, SyncFatalError) or isinstance(
e.__cause__, SyncFatalError
): # pragma: no cover
with telemetry.scope(
with logfire.span(
"sync.file.failure",
failure_type=failure_type,
path=path,
@@ -1027,7 +1031,7 @@ class SyncService:
# Otherwise treat as recoverable file-level error
error_msg = str(e)
with telemetry.scope(
with logfire.span(
"sync.file.failure",
failure_type=failure_type,
path=path,
@@ -1052,91 +1056,121 @@ class SyncService:
Returns:
Tuple of (entity, checksum)
"""
# Parse markdown first to get any existing permalink
synced = await self.sync_one_markdown_file(path, new=new, index_search=False)
return synced.entity, synced.checksum
async def sync_one_markdown_file(
self,
path: str,
*,
new: bool = True,
index_search: bool = True,
resolve_relations: bool = True,
) -> SyncedMarkdownFile:
"""Sync one markdown file and return the final canonical file state.
This method is the fail-fast single-file primitive for callers such as
cloud workers. It does not swallow unexpected exceptions.
"""
logger.debug(f"Parsing markdown file, path: {path}, new: {new}")
file_content = await self.file_service.read_file_content(path)
file_contains_frontmatter = has_frontmatter(file_content)
# Get file timestamps for tracking modification times
try:
initial_markdown_bytes = await self.file_service.read_file_bytes(path)
except FileOperationError as exc:
# Trigger: FileService wraps binary read failures in FileOperationError.
# Why: sync_file() treats bare FileNotFoundError as a deletion race and cleans up the DB row.
# Outcome: preserve that contract while still hashing the exact bytes we loaded.
if isinstance(exc.__cause__, FileNotFoundError):
raise exc.__cause__ from exc
raise
initial_markdown_content = initial_markdown_bytes.decode("utf-8")
file_metadata = await self.file_service.get_file_metadata(path)
created = file_metadata.created_at
modified = file_metadata.modified_at
# Parse markdown content with file metadata (avoids redundant file read/stat)
# This enables cloud implementations (S3FileService) to provide metadata from head_object
abs_path = self.file_service.base_path / path
entity_markdown = await self.entity_parser.parse_markdown_content(
file_path=abs_path,
content=file_content,
mtime=file_metadata.modified_at.timestamp(),
ctime=file_metadata.created_at.timestamp(),
)
# Trigger: markdown file has no frontmatter and frontmatter enforcement is enabled
# Why: watch/sync consumers rely on normalized metadata and stable permalinks
# Outcome: file is updated in-place with derived title/type/permalink metadata
if not file_contains_frontmatter and self.app_config.ensure_frontmatter_on_sync:
permalink = await self.entity_service.resolve_permalink(
path, markdown=entity_markdown, skip_conflict_check=True
initial_checksum = await compute_checksum(initial_markdown_bytes)
existing_entity = await self.entity_repository.get_by_file_path(path)
if existing_entity is not None and existing_entity.checksum == initial_checksum:
logger.debug(
f"Markdown sync skipped unchanged file: path={path}, "
f"entity_id={existing_entity.id}, checksum={initial_checksum[:8]}"
)
frontmatter_updates = {
"title": entity_markdown.frontmatter.title,
"type": entity_markdown.frontmatter.type,
"permalink": permalink,
}
await self.file_service.update_frontmatter(path, frontmatter_updates)
entity_markdown.frontmatter.metadata.update(frontmatter_updates)
# if the file contains frontmatter, resolve a permalink (unless disabled)
if file_contains_frontmatter and not self.app_config.disable_permalinks:
# Resolve permalink - skip conflict checks during bulk sync for performance
permalink = await self.entity_service.resolve_permalink(
path, markdown=entity_markdown, skip_conflict_check=True
return SyncedMarkdownFile(
entity=existing_entity,
checksum=initial_checksum,
markdown_content=initial_markdown_content,
file_path=path,
content_type=self.file_service.content_type(path),
updated_at=file_metadata.modified_at,
size=file_metadata.size,
)
# If permalink changed, update the file
if permalink != entity_markdown.frontmatter.permalink:
logger.debug(
f"Updating permalink for path: {path}, old_permalink: {entity_markdown.frontmatter.permalink}, new_permalink: {permalink}"
)
entity_markdown.frontmatter.metadata["permalink"] = permalink
await self.file_service.update_frontmatter(path, {"permalink": permalink})
# Create/update entity and relations in one path
logger.debug(f"{'Creating' if new else 'Updating'} entity from markdown, path={path}")
entity = await self.entity_service.upsert_entity_from_markdown(
Path(path), entity_markdown, is_new=new
indexed = await self.batch_indexer.index_markdown_file(
IndexInputFile(
path=path,
size=file_metadata.size,
checksum=initial_checksum,
content_type=self.file_service.content_type(path),
last_modified=file_metadata.modified_at,
created_at=file_metadata.created_at,
content=initial_markdown_bytes,
),
new=new,
index_search=False,
resolve_relations=resolve_relations,
)
# After updating relations, we need to compute the checksum again
# This is necessary for files with wikilinks to ensure consistent checksums
# after relation processing is complete
final_checksum = await self.file_service.compute_checksum(path)
# Update checksum, timestamps, and file metadata from file system
# Store mtime/size for efficient change detection in future scans
# This ensures temporal ordering in search and recent activity uses actual file modification times
await self.entity_repository.update(
entity.id,
final_markdown_content = (
indexed.markdown_content
if indexed.markdown_content is not None
else initial_markdown_content
)
file_metadata = await self.file_service.get_file_metadata(path)
refreshed_entities = await self.entity_repository.find_by_ids([indexed.entity_id])
if len(refreshed_entities) != 1: # pragma: no cover
raise ValueError(f"Failed to reload synced markdown entity for {path}")
# Trigger: markdown sync may have rewritten frontmatter after the initial file metadata load.
# Why: the batch indexer persisted checksum/path data from the pre-rewrite IndexInputFile.
# Outcome: refresh size and mtime from the file as it actually exists on disk now.
updated_entity = await self.entity_repository.update(
refreshed_entities[0].id,
{
"checksum": final_checksum,
"created_at": created,
"updated_at": modified,
"checksum": indexed.checksum,
"created_at": file_metadata.created_at,
"updated_at": file_metadata.modified_at,
"mtime": file_metadata.modified_at.timestamp(),
"size": file_metadata.size,
},
)
if updated_entity is None: # pragma: no cover
raise ValueError(f"Failed to update markdown entity metadata for {path}")
if index_search:
# Trigger: markdown may start with '---' as a thematic break or malformed
# frontmatter that the parser already treated as plain content.
# Why: one-file sync should not fail after the entity upsert just because
# strict frontmatter stripping rejects that exact text shape.
# Outcome: fall back to indexing the raw markdown content for these cases.
try:
search_content = remove_frontmatter(final_markdown_content)
except ParseError:
search_content = final_markdown_content
await self.search_service.index_entity_data(
updated_entity,
content=search_content,
)
logger.debug(
f"Markdown sync completed: path={path}, entity_id={entity.id}, "
f"observation_count={len(entity.observations)}, relation_count={len(entity.relations)}, "
f"checksum={final_checksum[:8]}"
f"Markdown sync completed: path={path}, entity_id={updated_entity.id}, "
f"observation_count={len(updated_entity.observations)}, "
f"relation_count={len(updated_entity.relations)}, checksum={indexed.checksum[:8]}"
)
# Return the final checksum to ensure everything is consistent
return entity, final_checksum
return SyncedMarkdownFile(
entity=updated_entity,
checksum=indexed.checksum,
markdown_content=final_markdown_content,
file_path=path,
content_type=self.file_service.content_type(path),
updated_at=file_metadata.modified_at,
size=file_metadata.size,
)
async def sync_regular_file(self, path: str, new: bool = True) -> Tuple[Optional[Entity], str]:
"""Sync a non-markdown file with basic tracking.
@@ -1435,7 +1469,7 @@ class SyncService:
)
affected_entity_ids.add(relation.from_id)
except IntegrityError:
with telemetry.scope(
with logfire.span(
"sync.relation.resolve_conflict",
relation_id=relation.id,
relation_type=relation.relation_type,
@@ -1456,7 +1490,7 @@ class SyncService:
try:
await self.relation_repository.delete(relation.id)
except Exception as e:
with telemetry.scope(
with logfire.span(
"sync.relation.cleanup_failure",
relation_id=relation.id,
relation_type=relation.relation_type,
+60 -26
View File
@@ -85,14 +85,20 @@ class WatchService:
project_repository: ProjectRepository,
quiet: bool = False,
sync_service_factory: Optional[SyncServiceFactory] = None,
constrained_project: Optional[str] = None,
):
self.app_config = app_config
self.project_repository = project_repository
self.state = WatchServiceState()
self.status_path = Path.home() / ".basic-memory" / WATCH_STATUS_JSON
self.status_path = app_config.data_dir_path / WATCH_STATUS_JSON
self.status_path.parent.mkdir(parents=True, exist_ok=True)
self._ignore_patterns_cache: dict[Path, Set[str]] = {}
self._sync_service_factory = sync_service_factory
# When set (typically from BASIC_MEMORY_MCP_PROJECT), the watch cycle
# only observes this project. Without it, each `basic-memory mcp --project X`
# process spawns a watcher over every project and racing writers collide
# on the same files.
self.constrained_project = constrained_project
# quiet mode for mcp so it doesn't mess up stdout
self.console = Console(quiet=quiet)
@@ -149,13 +155,42 @@ class WatchService:
# create coroutines to handle changes
change_handlers = [
self.handle_changes(project, changes) # pyright: ignore
self.handle_changes(project, set(changes))
for project, changes in project_changes.items()
]
# process changes
await asyncio.gather(*change_handlers)
async def _select_projects_to_watch(self) -> list[Project]:
"""Return the set of projects this watch cycle should observe.
Applies two filters in order:
1. ``constrained_project`` if the MCP server was started with
``--project``, only that project is watched. This keeps concurrent
MCP processes from producing duplicate watchers that race on the
same files.
2. Cloud-only projects without a local bisync copy are skipped so we
don't watch a path that does not exist on disk.
"""
projects = await self.project_repository.get_active_projects()
if self.constrained_project:
projects = [p for p in projects if p.name == self.constrained_project]
cloud_skip: list[str] = []
for p in projects:
if self.app_config.get_project_mode(p.name) == ProjectMode.CLOUD:
entry = self.app_config.projects.get(p.name)
if entry and Path(entry.path).is_absolute():
continue # Cloud project with local bisync copy — keep watching
cloud_skip.append(p.name)
if cloud_skip:
projects = [p for p in projects if p.name not in cloud_skip]
logger.debug(f"Skipping cloud-mode projects in watch cycle: {cloud_skip}")
return list(projects)
async def run(self): # pragma: no cover
"""Watch for file changes and sync them"""
@@ -174,23 +209,22 @@ class WatchService:
# Clear ignore patterns cache to pick up any .gitignore changes
self._ignore_patterns_cache.clear()
# Reload projects to catch any new/removed projects
projects = await self.project_repository.get_active_projects()
projects = await self._select_projects_to_watch()
# Trigger: project is configured for cloud routing
# Why: cloud-only projects (no local directory) should not be watched;
# cloud projects with a local bisync copy (absolute path) need watching
# Outcome: watch cycle skips cloud projects without a local directory
cloud_skip = []
for p in projects:
if self.app_config.get_project_mode(p.name) == ProjectMode.CLOUD:
entry = self.app_config.projects.get(p.name)
if entry and Path(entry.path).is_absolute():
continue # Cloud project with local bisync copy — keep watching
cloud_skip.append(p.name)
if cloud_skip:
projects = [p for p in projects if p.name not in cloud_skip]
logger.debug(f"Skipping cloud-mode projects in watch cycle: {cloud_skip}")
# Trigger: no projects selected (e.g. constrained_project names a
# project not in the DB, or every project was filtered out)
# Why: watchfiles.awatch() requires at least one path. Calling it
# with an empty list raises ValueError, which the outer handler
# catches with a 5s sleep — producing a tight error-log loop.
# Outcome: sleep the configured reload interval before retrying, so
# newly added projects get picked up on the next cycle.
if not projects:
logger.warning(
"No projects to watch; sleeping before retry "
f"(constrained_project={self.constrained_project!r})"
)
await asyncio.sleep(self.app_config.watch_project_reload_interval)
continue
project_paths = [project.path for project in projects]
logger.debug(f"Starting watch cycle for directories: {project_paths}")
@@ -502,19 +536,19 @@ class WatchService:
# Add a concise summary instead of a divider
if processed:
changes = [] # pyright: ignore
change_summary: list[str] = []
if add_count > 0:
changes.append(f"[green]{add_count} added[/green]") # pyright: ignore
change_summary.append(f"[green]{add_count} added[/green]")
if modify_count > 0:
changes.append(f"[yellow]{modify_count} modified[/yellow]") # pyright: ignore
change_summary.append(f"[yellow]{modify_count} modified[/yellow]")
if moved_count > 0:
changes.append(f"[blue]{moved_count} moved[/blue]") # pyright: ignore
change_summary.append(f"[blue]{moved_count} moved[/blue]")
if delete_count > 0:
changes.append(f"[red]{delete_count} deleted[/red]") # pyright: ignore
change_summary.append(f"[red]{delete_count} deleted[/red]")
if changes:
self.console.print(f"{', '.join(changes)}", style="dim") # pyright: ignore
logger.info(f"changes: {len(changes)}")
if change_summary:
self.console.print(f"{', '.join(change_summary)}", style="dim")
logger.info(f"changes: {len(change_summary)}")
duration_ms = int((time.time() - start_time) * 1000)
self.state.last_scan = datetime.now()
+15 -196
View File
@@ -1,67 +1,20 @@
"""Optional Logfire telemetry helpers for Basic Memory.
"""Logfire telemetry bootstrap.
Telemetry is disabled by default. When enabled, this module configures Logfire,
exposes a `loguru` handler for trace-aware logging, and provides lightweight
helpers for manual spans and logger context binding.
`configure_telemetry()` wires up the Logfire SDK and returns the loguru
handler. Call sites use `logfire.span(...)` and `logfire.metric_counter(...)`
directly there are no wrappers here.
"""
from __future__ import annotations
from contextlib import contextmanager
from dataclasses import dataclass, field
from typing import Any, Iterator
from typing import Any
from loguru import logger
import logfire
REPOSITORY_URL = "https://github.com/basicmachines-co/basic-memory"
ROOT_PATH = "src/basic_memory"
def _load_logfire() -> Any | None:
"""Load the optional logfire dependency lazily."""
try:
import logfire
except ImportError:
return None
return logfire
@dataclass
class TelemetryState:
"""Process-local Logfire configuration state."""
enabled: bool = False
configured: bool = False
service_name: str | None = None
environment: str | None = None
send_to_logfire: bool = False
warnings: list[str] = field(default_factory=list)
_STATE = TelemetryState()
_LOGFIRE_HANDLER: dict[str, Any] | None = None
_METRICS: dict[tuple[str, str, str, str], Any] = {}
def reset_telemetry_state() -> None:
"""Reset process-local telemetry state.
Primarily used by tests.
"""
global _LOGFIRE_HANDLER
_STATE.enabled = False
_STATE.configured = False
_STATE.service_name = None
_STATE.environment = None
_STATE.send_to_logfire = False
_STATE.warnings.clear()
_LOGFIRE_HANDLER = None
_METRICS.clear()
def _filter_attributes(attrs: dict[str, Any]) -> dict[str, Any]:
"""Drop null attributes so span and log payloads stay compact."""
return {key: value for key, value in attrs.items() if value is not None}
def configure_telemetry(
@@ -73,28 +26,14 @@ def configure_telemetry(
send_to_logfire: bool = False,
log_level: str = "INFO",
) -> bool:
"""Configure optional Logfire instrumentation for the current process."""
"""Configure Logfire for the current process. Returns True when enabled."""
global _LOGFIRE_HANDLER
reset_telemetry_state()
_STATE.service_name = service_name
_STATE.environment = environment
_STATE.send_to_logfire = send_to_logfire
_STATE.enabled = enable_logfire
_LOGFIRE_HANDLER = None
if not enable_logfire:
return False
logfire = _load_logfire()
if logfire is None:
_STATE.enabled = False
_STATE.warnings.append(
"Logfire telemetry was enabled but the 'logfire' package is not installed. "
"Telemetry remains disabled."
)
return False
configure_kwargs = {
kwargs: dict[str, Any] = {
"service_name": service_name,
"environment": environment,
"code_source": logfire.CodeSource(
@@ -107,139 +46,19 @@ def configure_telemetry(
}
try:
logfire.configure(**configure_kwargs)
logfire.configure(**kwargs)
except TypeError:
configure_kwargs.pop("send_to_logfire", None)
logfire.configure(**configure_kwargs)
except Exception as exc: # pragma: no cover
_STATE.enabled = False # pragma: no cover
_STATE.warnings.append(f"Failed to configure Logfire telemetry: {exc}") # pragma: no cover
return False # pragma: no cover
# Older logfire releases don't accept send_to_logfire as a keyword.
kwargs.pop("send_to_logfire", None)
logfire.configure(**kwargs)
_LOGFIRE_HANDLER = logfire.loguru_handler()
_STATE.configured = True
return True
def telemetry_enabled() -> bool:
"""Return True when telemetry is both enabled and configured."""
return _STATE.enabled and _STATE.configured
def get_logfire_handler() -> dict[str, Any] | None:
"""Return the active Logfire `loguru` handler, if any."""
"""Return the active Logfire loguru handler, if any."""
return _LOGFIRE_HANDLER
def pop_telemetry_warnings() -> list[str]:
"""Return and clear pending telemetry warnings."""
warnings = list(_STATE.warnings)
_STATE.warnings.clear()
return warnings
def _get_metric(metric_type: str, name: str, *, unit: str, description: str) -> Any | None:
"""Create or reuse a Logfire metric instrument when telemetry is enabled."""
logfire = _load_logfire()
if logfire is None or not _STATE.configured: # pragma: no cover
return None # pragma: no cover
metric_key = (metric_type, name, unit, description)
cached_metric = _METRICS.get(metric_key)
if cached_metric is not None:
return cached_metric
if metric_type == "counter":
metric = logfire.metric_counter(name, unit=unit, description=description)
elif metric_type == "histogram":
metric = logfire.metric_histogram(name, unit=unit, description=description)
else: # pragma: no cover
raise ValueError(f"Unsupported metric type: {metric_type}") # pragma: no cover
_METRICS[metric_key] = metric
return metric
def add_counter(
name: str,
amount: int | float,
*,
unit: str = "1",
description: str = "",
**attrs: Any,
) -> None:
"""Record a counter increment when telemetry is enabled."""
metric = _get_metric("counter", name, unit=unit, description=description)
if metric is None:
return
metric.add(amount, attributes=_filter_attributes(attrs))
def record_histogram(
name: str,
amount: int | float,
*,
unit: str = "",
description: str = "",
**attrs: Any,
) -> None:
"""Record one histogram sample when telemetry is enabled."""
metric = _get_metric("histogram", name, unit=unit, description=description)
if metric is None:
return
metric.record(amount, attributes=_filter_attributes(attrs))
@contextmanager
def contextualize(**attrs: Any) -> Iterator[None]:
"""Apply filtered telemetry attributes to Loguru calls in this scope."""
with logger.contextualize(**_filter_attributes(attrs)):
yield
@contextmanager
def scope(name: str, **attrs: Any) -> Iterator[None]:
"""Create a span and bind the same stable attributes into Loguru context."""
with contextualize(**attrs):
with span(name, **attrs):
yield
# Alias: `operation` signals a root-level boundary (entrypoint, tool invocation),
# while `scope` signals a nested phase. The distinction is convention only.
operation = scope
@contextmanager
def span(name: str, **attrs: Any) -> Iterator[None]:
"""Create a manual Logfire span when telemetry is enabled."""
with started_span(name, **attrs):
yield
@contextmanager
def started_span(name: str, **attrs: Any) -> Iterator[Any | None]:
"""Create a manual Logfire span and expose the active span handle when available."""
logfire = _load_logfire()
if logfire is None or not _STATE.configured: # pragma: no cover
yield # pragma: no cover
return # pragma: no cover
with logfire.span(name, **_filter_attributes(attrs)) as active_span:
yield active_span
__all__ = [
"add_counter",
"contextualize",
"configure_telemetry",
"get_logfire_handler",
"operation",
"pop_telemetry_warnings",
"record_histogram",
"reset_telemetry_state",
"scope",
"span",
"started_span",
"telemetry_enabled",
]
__all__ = ["configure_telemetry", "get_logfire_handler"]
+37 -8
View File
@@ -224,18 +224,43 @@ def build_canonical_permalink(
project_permalink: Optional[str],
file_path: Union[Path, str, PathLike],
include_project: bool = True,
*,
workspace_permalink: Optional[str] = None,
) -> str:
"""Build a canonical permalink, optionally prefixed with project slug.
"""Build a canonical permalink, optionally prefixed with workspace/project slugs.
Args:
project_permalink: URL-friendly project identifier (slug). If None, no prefix is added.
file_path: Original file path or permalink-like string.
include_project: When True, prefix with project slug.
workspace_permalink: Optional URL-friendly workspace identifier. When provided,
prefix the project-qualified permalink with this workspace slug.
Returns:
Canonical permalink string.
"""
normalized_path = generate_permalink(file_path)
normalized_workspace = generate_permalink(workspace_permalink) if workspace_permalink else None
if normalized_workspace:
if not project_permalink:
raise ValueError("workspace_permalink requires project_permalink")
normalized_project = generate_permalink(project_permalink)
workspace_project_prefix = f"{normalized_workspace}/{normalized_project}"
if normalized_path == workspace_project_prefix or normalized_path.startswith(
f"{workspace_project_prefix}/"
):
return normalized_path
if normalized_path == normalized_project or normalized_path.startswith(
f"{normalized_project}/"
):
project_path = normalized_path
else:
project_path = f"{normalized_project}/{normalized_path}"
return f"{normalized_workspace}/{project_path}"
if not include_project or not project_permalink:
return normalized_path
@@ -244,9 +269,11 @@ def build_canonical_permalink(
if normalized_path == normalized_project or normalized_path.startswith(
f"{normalized_project}/"
):
return normalized_path
project_path = normalized_path
else:
project_path = f"{normalized_project}/{normalized_path}"
return f"{normalized_project}/{normalized_path}"
return project_path
def setup_logging(
@@ -262,7 +289,8 @@ def setup_logging(
Args:
log_level: DEBUG, INFO, WARNING, ERROR
log_to_file: Write to ~/.basic-memory/basic-memory.log with rotation
log_to_file: Write to <basic-memory data dir>/basic-memory.log with rotation
(honors BASIC_MEMORY_CONFIG_DIR)
log_to_stdout: Write to stderr (for Docker/cloud deployments)
structured_context: Bind tenant_id, fly_region, etc. for cloud observability
"""
@@ -281,7 +309,11 @@ def setup_logging(
# Why: multiple basic-memory processes can share the same log directory at once.
# Outcome: use per-process log files on Windows so log rotation stays local.
log_filename = f"basic-memory-{os.getpid()}.log" if os.name == "nt" else "basic-memory.log"
log_path = Path.home() / ".basic-memory" / log_filename
# Deferred import: basic_memory.config imports from this module at load time,
# so resolving the data dir via a top-level import would cycle.
from basic_memory.config import resolve_data_dir
log_path = resolve_data_dir() / log_filename
log_path.parent.mkdir(parents=True, exist_ok=True)
if os.name == "nt":
_cleanup_windows_log_files(log_path.parent, log_path.name)
@@ -322,9 +354,6 @@ def setup_logging(
logging.getLogger("httpx").setLevel(logging.WARNING)
logging.getLogger("watchfiles.main").setLevel(logging.WARNING)
for warning_message in telemetry.pop_telemetry_warnings():
logger.warning(warning_message)
def _cleanup_windows_log_files(log_dir: Path, current_log_name: str) -> None:
"""Trim stale per-process Windows log files so the directory stays bounded."""
+114
View File
@@ -0,0 +1,114 @@
"""Request-local workspace context for canonical permalink generation."""
import re
from contextlib import contextmanager
from contextvars import ContextVar
from dataclasses import dataclass
from typing import Iterator
WORKSPACE_SLUG_HEADER = "X-Basic-Memory-Workspace-Slug"
WORKSPACE_TYPE_HEADER = "X-Basic-Memory-Workspace-Type"
_WORKSPACE_SLUG_PATTERN = re.compile(r"^[a-z0-9_-]+$")
_WORKSPACE_TYPES = {"personal", "organization"}
@dataclass(frozen=True)
class WorkspacePermalinkContext:
"""Workspace metadata needed to build canonical organization permalinks."""
workspace_slug: str
workspace_type: str
@property
def should_prefix_permalinks(self) -> bool:
return self.workspace_type == "organization" and bool(self.workspace_slug)
_workspace_permalink_context: ContextVar[WorkspacePermalinkContext | None] = ContextVar(
"basic_memory_workspace_permalink_context",
default=None,
)
def current_workspace_permalink_context() -> WorkspacePermalinkContext | None:
"""Return the active workspace permalink context, when one is set."""
return _workspace_permalink_context.get()
def validate_workspace_permalink_context_values(
workspace_slug: str | None,
workspace_type: str | None,
) -> None:
"""Validate workspace permalink metadata before it can affect stored permalinks."""
validation_error = workspace_permalink_context_validation_error(workspace_slug, workspace_type)
if validation_error is not None:
raise ValueError(validation_error)
def workspace_permalink_context_validation_error(
workspace_slug: str | None,
workspace_type: str | None,
) -> str | None:
"""Return the validation error for workspace permalink metadata, if any."""
if bool(workspace_slug) != bool(workspace_type):
return "workspace_slug and workspace_type must be provided together"
if not workspace_slug or not workspace_type:
return None
if _WORKSPACE_SLUG_PATTERN.fullmatch(workspace_slug) is None:
return f"{WORKSPACE_SLUG_HEADER} must match [a-z0-9_-]+"
if workspace_type not in _WORKSPACE_TYPES:
allowed = ", ".join(sorted(_WORKSPACE_TYPES))
return f"{WORKSPACE_TYPE_HEADER} must be one of: {allowed}"
return None
@contextmanager
def workspace_permalink_context(
workspace_slug: str | None,
workspace_type: str | None,
) -> Iterator[None]:
"""Set request-local workspace permalink metadata.
Cloud can populate this per request without storing workspace metadata in
local project config. The slug/type pair is all permalink generation needs.
"""
validate_workspace_permalink_context_values(workspace_slug, workspace_type)
if not workspace_slug or not workspace_type:
yield
return
token = _workspace_permalink_context.set(
WorkspacePermalinkContext(
workspace_slug=workspace_slug,
workspace_type=workspace_type,
)
)
try:
yield
finally:
_workspace_permalink_context.reset(token)
def workspace_permalink_headers() -> dict[str, str]:
"""Return HTTP headers for forwarding workspace permalink context."""
context = current_workspace_permalink_context()
if context is None:
return {}
return {
WORKSPACE_SLUG_HEADER: context.workspace_slug,
WORKSPACE_TYPE_HEADER: context.workspace_type,
}
def workspace_slug_for_canonical_permalinks() -> str | None:
"""Return the workspace slug when new permalinks should include it."""
context = current_workspace_permalink_context()
if context and context.should_prefix_permalinks:
return context.workspace_slug
return None
+68 -19
View File
@@ -57,7 +57,12 @@ import pytest
import pytest_asyncio
from pathlib import Path
from sqlalchemy import text
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
from sqlalchemy.ext.asyncio import (
AsyncEngine,
AsyncSession,
async_sessionmaker,
create_async_engine,
)
from sqlalchemy.pool import NullPool
from testcontainers.postgres import PostgresContainer
@@ -118,6 +123,21 @@ def postgres_container(db_backend):
yield postgres
@pytest_asyncio.fixture(autouse=True)
async def cleanup_global_db_after_test() -> AsyncGenerator[None, None]:
"""Close any module-level DB engine created outside fixture ownership."""
yield
# Trigger: integration tests invoke CLI/MCP routes through the production
# client fallback, bypassing this file's engine_factory fixture.
# Why: those fallback engines live in basic_memory.db module state and can
# otherwise leave a non-daemon aiosqlite worker alive after pytest finishes.
# Outcome: every test boundary becomes a cleanup point for fallback engines.
from basic_memory import db
await db.shutdown_db()
POSTGRES_EPHEMERAL_TABLES = [
"search_vector_embeddings",
"search_vector_chunks",
@@ -182,12 +202,36 @@ async def _reset_postgres_integration_schema(engine) -> None:
)
@pytest_asyncio.fixture(scope="session", loop_scope="session")
async def postgres_engine(
db_backend: Literal["sqlite", "postgres"], postgres_container
) -> AsyncGenerator[AsyncEngine | None, None]:
"""Create the shared Postgres engine once per integration test session."""
if db_backend != "postgres":
yield None
return
sync_url = _resolve_postgres_sync_url(postgres_container)
async_url = sync_url.replace("postgresql+psycopg2", "postgresql+asyncpg")
engine = create_async_engine(
async_url,
echo=False,
poolclass=NullPool,
)
try:
yield engine
finally:
await engine.dispose()
@pytest_asyncio.fixture
async def engine_factory(
app_config,
config_manager,
db_backend: Literal["sqlite", "postgres"],
postgres_container,
postgres_engine,
tmp_path,
) -> AsyncGenerator[tuple, None]:
"""Create engine and session factory for the configured database backend."""
@@ -195,18 +239,17 @@ async def engine_factory(
from basic_memory import db
if db_backend == "postgres":
# Postgres mode using testcontainers
sync_url = _resolve_postgres_sync_url(postgres_container)
async_url = sync_url.replace("postgresql+psycopg2", "postgresql+asyncpg")
assert postgres_engine is not None
engine = create_async_engine(
async_url,
echo=False,
poolclass=NullPool,
)
# Trigger: full-stack MCP/CLI tests exercise sync/indexing code that can
# recover from DB errors by rolling back and opening later scoped sessions.
# Why: one savepoint-backed connection is too brittle for that flow.
# Outcome: reuse the engine, but reset rows/schema before each test and
# let app code use normal transaction boundaries.
await _reset_postgres_integration_schema(postgres_engine)
session_maker = async_sessionmaker(
bind=engine,
bind=postgres_engine,
class_=AsyncSession,
expire_on_commit=False,
autoflush=False,
@@ -214,17 +257,17 @@ async def engine_factory(
# Set module-level state to prevent MCP lifespan from re-initializing
# This ensures get_or_create_db() sees an existing engine and skips initialization
db._engine = engine
db._engine = postgres_engine
db._session_maker = session_maker
await _reset_postgres_integration_schema(engine)
yield engine, session_maker
# Clean up module-level state
await engine.dispose()
db._engine = None
db._session_maker = None
try:
yield postgres_engine, session_maker
finally:
# Clean up module-level state
if db._engine is postgres_engine:
db._engine = None
if db._session_maker is session_maker:
db._session_maker = None
else:
# SQLite: Create fresh database (fast with tmp files)
@@ -264,7 +307,13 @@ async def test_project(config_home, engine_factory) -> Project:
@pytest.fixture
def config_home(tmp_path, monkeypatch) -> Path:
# Patch both HOME and USERPROFILE so Path.home() returns the test dir on
# every platform — Path.home() reads HOME on POSIX and USERPROFILE on
# Windows, and ConfigManager.data_dir_path now goes through Path.home()
# via resolve_data_dir(). Must mirror tests/conftest.py:config_home.
monkeypatch.setenv("HOME", str(tmp_path))
if os.name == "nt":
monkeypatch.setenv("USERPROFILE", str(tmp_path))
# Set BASIC_MEMORY_HOME to the test directory
monkeypatch.setenv("BASIC_MEMORY_HOME", str(tmp_path / "basic-memory"))
return tmp_path
@@ -0,0 +1,77 @@
"""
Integration test for long relation_type values (regression guard for issue #721).
When a markdown bullet contains an inline `[[wikilink]]` preceded by long prose,
`parse_relation()` extracts ALL of that prose as the `relation_type`. Previously
the response model `RelationType` had a `MaxLen(200)` constraint that caused
edit_note (which round-trips through the response model when re-indexing) to
fail with:
1 validation error for EntityResponseV2
relations.0.relation_type
String should have at most 200 characters
Commit 01cbad1d removed the cap from `RelationType`. This test locks in that
fix so a future contributor reintroducing `MaxLen` will see the test fail
before shipping it.
Out of scope: improving `parse_relation()` to fall back to a default relation
type when the prose-before-link looks like a sentence rather than a label.
That's a knowledge-graph-quality improvement, not a correctness fix, and is
not required to keep edit_note working.
"""
import pytest
from fastmcp import Client
@pytest.mark.asyncio
async def test_edit_note_handles_long_prose_around_wikilink(mcp_server, app, test_project):
"""edit_note must succeed on notes whose inline wikilinks have >200 chars
of prose preceding them that prose becomes the parsed relation_type."""
long_prose = (
"**Lorem ipsum dolor sit amet** — consectetur adipiscing elit, sed do eiusmod "
"tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, "
"quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo "
"consequat. Trust boundary model documented in"
)
assert len(long_prose) > 200, (
f"setup wrong: prose-before-link must exceed the historical 200-char cap, "
f"got {len(long_prose)} chars"
)
note_body = (
"# Long Relation Type Repro\n\n"
f"- {long_prose} [[Some Note Title]] for additional context.\n"
)
async with Client(mcp_server) as client:
# Create the note (file-write side; would already fail at index time
# if RelationType MaxLen were back).
write_result = await client.call_tool(
"write_note",
{
"project": test_project.name,
"title": "Long Relation Type Repro",
"directory": "issue721",
"content": note_body,
},
)
assert len(write_result.content) == 1
write_text = write_result.content[0].text
assert "Created note" in write_text or "Updated note" in write_text
# Edit the note. This triggers re-index → response model validation.
# With the historical MaxLen(200) cap, this would raise:
# "relations.0.relation_type String should have at most 200 characters"
edit_result = await client.call_tool(
"edit_note",
{
"project": test_project.name,
"identifier": "Long Relation Type Repro",
"operation": "append",
"content": "\n\nappended line\n",
},
)
assert len(edit_result.content) == 1
assert "Edited note (append)" in edit_result.content[0].text
@@ -4,6 +4,7 @@ from __future__ import annotations
import json
from pathlib import Path
from typing import Any
import pytest
from fastmcp import Client
@@ -11,7 +12,7 @@ from fastmcp import Client
from basic_memory.mcp.clients.knowledge import KnowledgeClient
def _json_content(tool_result) -> dict | list:
def _json_content(tool_result) -> Any:
"""Parse a FastMCP tool result content block into JSON."""
assert len(tool_result.content) == 1
assert tool_result.content[0].type == "text"
+2 -1
View File
@@ -7,12 +7,13 @@ results are available.
"""
import json
from typing import Any
import pytest
from fastmcp import Client
def _json_content(tool_result) -> dict | list:
def _json_content(tool_result) -> Any:
"""Parse a FastMCP tool result content block into JSON."""
assert len(tool_result.content) == 1
assert tool_result.content[0].type == "text"
@@ -0,0 +1,540 @@
"""
Integration tests for MCP tool parameter aliases.
Verifies that MCP tools accept training-data-friendly parameter aliases
(via Pydantic AliasChoices) alongside the canonical names, so models
that reach for `offset`/`limit`/`find`/`old_text` etc. don't hit
validation errors on first use.
See: https://github.com/basicmachines-co/basic-memory/issues/690
"""
import pytest
from fastmcp import Client
# --- read_note: pagination params removed in #693 (were no-ops) ---
# The `page` / `page_size` parameters were removed because the API endpoint
# silently dropped them. Search-fallback pagination is unrelated to read_note.
# --- edit_note: find_text / content / section aliases ---
@pytest.mark.asyncio
async def test_edit_note_accepts_find_alias_for_find_text(mcp_server, app, test_project):
"""`find` should map to `find_text` — the highest-frequency miss in the issue."""
async with Client(mcp_server) as client:
await client.call_tool(
"write_note",
{
"project": test_project.name,
"title": "Find Alias Note",
"directory": "test",
"content": "# Find Alias Note\n\nVersion v1.0.0 of the spec.",
},
)
result = await client.call_tool(
"edit_note",
{
"project": test_project.name,
"identifier": "Find Alias Note",
"operation": "find_replace",
"content": "v2.0.0",
"find": "v1.0.0", # alias for find_text
},
)
assert "Edited note (find_replace)" in result.content[0].text
read_result = await client.call_tool(
"read_note",
{"project": test_project.name, "identifier": "Find Alias Note"},
)
assert "v2.0.0" in read_result.content[0].text
assert "v1.0.0" not in read_result.content[0].text
@pytest.mark.asyncio
async def test_edit_note_accepts_old_text_alias(mcp_server, app, test_project):
"""`old_text` (diff/patch convention) should map to `find_text`."""
async with Client(mcp_server) as client:
await client.call_tool(
"write_note",
{
"project": test_project.name,
"title": "Old Text Note",
"directory": "test",
"content": "# Old Text Note\n\nThe quick brown fox.",
},
)
result = await client.call_tool(
"edit_note",
{
"project": test_project.name,
"identifier": "Old Text Note",
"operation": "find_replace",
"content": "lazy",
"old_text": "quick",
},
)
assert "Edited note (find_replace)" in result.content[0].text
@pytest.mark.asyncio
async def test_edit_note_accepts_new_content_alias_for_content(mcp_server, app, test_project):
"""`new_content` should map to `content` — `content` is ambiguous as 'replacement text'."""
async with Client(mcp_server) as client:
await client.call_tool(
"write_note",
{
"project": test_project.name,
"title": "New Content Note",
"directory": "test",
"content": "# New Content Note\n\nplaceholder",
},
)
result = await client.call_tool(
"edit_note",
{
"project": test_project.name,
"identifier": "New Content Note",
"operation": "find_replace",
"new_content": "actual value", # alias for content
"find_text": "placeholder",
},
)
assert "Edited note (find_replace)" in result.content[0].text
read_result = await client.call_tool(
"read_note",
{"project": test_project.name, "identifier": "New Content Note"},
)
assert "actual value" in read_result.content[0].text
@pytest.mark.asyncio
async def test_edit_note_accepts_section_heading_alias(mcp_server, app, test_project):
"""`section_heading` and `heading` should map to `section`."""
async with Client(mcp_server) as client:
await client.call_tool(
"write_note",
{
"project": test_project.name,
"title": "Section Heading Note",
"directory": "test",
"content": "# Section Heading Note\n\n## Notes\n\nold notes\n",
},
)
result = await client.call_tool(
"edit_note",
{
"project": test_project.name,
"identifier": "Section Heading Note",
"operation": "replace_section",
"content": "fresh notes\n",
"section_heading": "## Notes", # alias for section
},
)
assert "Edited note (replace_section)" in result.content[0].text
@pytest.mark.asyncio
async def test_edit_note_canonical_names_still_work(mcp_server, app, test_project):
"""Canonical names must keep working alongside aliases."""
async with Client(mcp_server) as client:
await client.call_tool(
"write_note",
{
"project": test_project.name,
"title": "Edit Canonical Note",
"directory": "test",
"content": "# Edit Canonical Note\n\nold-value here.",
},
)
result = await client.call_tool(
"edit_note",
{
"project": test_project.name,
"identifier": "Edit Canonical Note",
"operation": "find_replace",
"content": "new-value",
"find_text": "old-value",
},
)
assert "Edited note (find_replace)" in result.content[0].text
# --- search_notes aliases ---
@pytest.mark.asyncio
async def test_search_notes_accepts_query_aliases(mcp_server, app, test_project):
"""`q` (HTTP convention), `search`, and `text` should all map to `query`."""
async with Client(mcp_server) as client:
await client.call_tool(
"write_note",
{
"project": test_project.name,
"title": "Searchable Note",
"directory": "test",
"content": "# Searchable Note\n\nUnique-keyword-XYZ here.",
},
)
# Try each alias
for alias_key in ("q", "search", "text"):
result = await client.call_tool(
"search_notes",
{
"project": test_project.name,
alias_key: "Unique-keyword-XYZ",
"limit": 5, # also testing pagination alias
},
)
assert "Searchable Note" in result.content[0].text, f"alias {alias_key} failed"
@pytest.mark.asyncio
async def test_search_notes_accepts_after_date_aliases(mcp_server, app, test_project):
"""`since`/`after`/`from_date` should map to `after_date`."""
async with Client(mcp_server) as client:
await client.call_tool(
"write_note",
{
"project": test_project.name,
"title": "Date Filter Note",
"directory": "test",
"content": "# Date Filter Note\n\nbody",
},
)
# Just verify the alias is accepted at validation time (no error)
result = await client.call_tool(
"search_notes",
{"project": test_project.name, "query": "Date Filter", "since": "1d"},
)
assert result.content # didn't error
# --- recent_activity aliases ---
@pytest.mark.asyncio
async def test_recent_activity_accepts_timeframe_aliases(mcp_server, app, test_project):
"""`since`/`time_range`/`lookback` should map to `timeframe`."""
async with Client(mcp_server) as client:
result = await client.call_tool(
"recent_activity",
{"project": test_project.name, "since": "7d", "limit": 5},
)
assert result.content # accepted, no validation error
# --- list_directory aliases ---
@pytest.mark.asyncio
async def test_list_directory_accepts_directory_alias(mcp_server, app, test_project):
"""`directory`/`folder`/`path`/`dir` should all map to `dir_name`."""
async with Client(mcp_server) as client:
await client.call_tool(
"write_note",
{
"project": test_project.name,
"title": "Dir Test",
"directory": "list-dir-aliases",
"content": "# Dir Test\n\nbody",
},
)
for alias_key in ("directory", "folder", "path", "dir"):
result = await client.call_tool(
"list_directory",
{"project": test_project.name, alias_key: "/list-dir-aliases"},
)
assert "Dir Test" in result.content[0].text, f"alias {alias_key} failed"
@pytest.mark.asyncio
async def test_list_directory_accepts_glob_aliases(mcp_server, app, test_project):
"""`glob`/`pattern`/`filter` should map to `file_name_glob`."""
async with Client(mcp_server) as client:
await client.call_tool(
"write_note",
{
"project": test_project.name,
"title": "Glob Target",
"directory": "glob-test",
"content": "# Glob Target\n\nbody",
},
)
result = await client.call_tool(
"list_directory",
{
"project": test_project.name,
"dir_name": "/glob-test",
"glob": "*.md",
},
)
assert "Glob Target" in result.content[0].text
# --- write_note aliases ---
@pytest.mark.asyncio
async def test_write_note_accepts_directory_aliases(mcp_server, app, test_project):
"""`folder`/`dir`/`path` should map to `directory`."""
async with Client(mcp_server) as client:
result = await client.call_tool(
"write_note",
{
"project": test_project.name,
"title": "Folder Alias Note",
"folder": "folder-alias-test", # alias
"content": "# Folder Alias Note\n\nbody",
},
)
assert "folder-alias-test" in result.content[0].text
@pytest.mark.asyncio
async def test_write_note_accepts_overwrite_aliases(mcp_server, app, test_project):
"""`force`/`replace` should map to `overwrite`."""
async with Client(mcp_server) as client:
# First create
await client.call_tool(
"write_note",
{
"project": test_project.name,
"title": "Overwrite Alias Note",
"directory": "overwrite-test",
"content": "v1",
},
)
# Overwrite using `force` alias
result = await client.call_tool(
"write_note",
{
"project": test_project.name,
"title": "Overwrite Alias Note",
"directory": "overwrite-test",
"content": "v2",
"force": True, # alias for overwrite
},
)
assert "Updated note" in result.content[0].text or "Created note" in result.content[0].text
# --- move_note aliases ---
@pytest.mark.asyncio
async def test_move_note_accepts_destination_aliases(mcp_server, app, test_project):
"""`to`/`dest_path`/`new_path`/`destination` should map to `destination_path`."""
async with Client(mcp_server) as client:
await client.call_tool(
"write_note",
{
"project": test_project.name,
"title": "Move Target",
"directory": "move-src",
"content": "# Move Target\n\nbody",
},
)
result = await client.call_tool(
"move_note",
{
"project": test_project.name,
"identifier": "Move Target",
"to": "move-dest/Move Target.md", # alias for destination_path
},
)
assert "move-dest" in result.content[0].text
# --- read_content aliases ---
@pytest.mark.asyncio
async def test_read_content_accepts_file_path_alias(mcp_server, app, test_project):
"""`file_path`/`filepath`/`file` should map to `path`."""
async with Client(mcp_server) as client:
await client.call_tool(
"write_note",
{
"project": test_project.name,
"title": "Read Content Target",
"directory": "read-content-test",
"content": "# Read Content Target\n\nraw body",
},
)
result = await client.call_tool(
"read_content",
{
"project": test_project.name,
"file_path": "read-content-test/Read Content Target.md",
},
)
# read_content returns a dict; structured content should include the file
text = result.content[0].text if result.content else ""
struct = result.structured_content if hasattr(result, "structured_content") else None
assert "raw body" in text or (struct and "raw body" in str(struct))
# --- build_context aliases ---
@pytest.mark.asyncio
async def test_build_context_accepts_url_aliases(mcp_server, app, test_project):
"""`uri`/`memory_url` should map to `url`."""
async with Client(mcp_server) as client:
await client.call_tool(
"write_note",
{
"project": test_project.name,
"title": "Context Target",
"directory": "build-ctx",
"content": "# Context Target\n\nbody",
},
)
result = await client.call_tool(
"build_context",
{
"project": test_project.name,
"uri": "memory://build-ctx/context-target", # alias for url
},
)
# Just verify validation accepted the alias
assert result.content or result.structured_content
# --- view_note: pagination params removed in #693 (delegates to read_note) ---
# --- delete_note aliases ---
@pytest.mark.asyncio
async def test_delete_note_accepts_is_dir_alias(mcp_server, app, test_project):
"""`is_dir` should map to `is_directory` and route to single-note deletion."""
async with Client(mcp_server) as client:
await client.call_tool(
"write_note",
{
"project": test_project.name,
"title": "Delete Alias Note",
"directory": "delete-alias-test",
"content": "# Delete Alias Note\n\nBody.",
},
)
result = await client.call_tool(
"delete_note",
{
"project": test_project.name,
"identifier": "delete-alias-test/Delete Alias Note",
"is_dir": False, # alias for is_directory
},
)
# delete_note returns a bool/dict on success; just assert no error
assert result.content or result.structured_content
# --- Schema sanity check: aliases must not appear in the advertised schema ---
@pytest.mark.asyncio
async def test_aliases_not_advertised_in_schema(mcp_server, app):
"""The JSON schema sent to models should advertise only canonical names.
Aliases are accepted at validation time but advertising them would defeat
the purpose: we want the model to learn the canonical name, with aliases
as a silent safety net for first-use mistakes.
The `must_not_have` lists below intentionally include both *accepted*
aliases (which must stay hidden from the schema) AND *rejected* aliases
that were considered but deliberately omitted (`offset` for `page`,
`limit_related` for `max_related`). Listing rejected aliases here acts
as a future-contributor guard if anyone re-adds them, this test catches
it before the bad alias ships.
"""
async with Client(mcp_server) as client:
tools = {t.name: t for t in await client.list_tools()}
# tool_name -> (must_have_canonical, must_not_have_aliases)
checks = {
# read_note has no pagination params (#693 — they were no-ops; removed).
# The must_not_have list still includes the rejected aliases so future
# contributors don't reintroduce them.
"read_note": (
[],
["page", "page_size", "offset", "limit", "page_number", "per_page"],
),
"edit_note": (
["find_text", "section", "content"],
["find", "old_text", "old_content", "search", "new_content", "section_heading"],
),
"search_notes": (
["query", "page", "page_size", "note_types", "after_date", "min_similarity"],
[
"q",
"search",
"offset",
"limit",
"note_type",
"types",
"since",
"after",
"threshold",
],
),
"recent_activity": (
["type", "timeframe", "page", "page_size"],
["types", "kind", "since", "time_range", "lookback", "offset", "limit"],
),
"list_directory": (
["dir_name", "file_name_glob"],
["directory", "folder", "path", "dir", "glob", "pattern", "filter"],
),
"write_note": (
["directory", "overwrite"],
["folder", "dir", "path", "force", "replace"],
),
"move_note": (
["destination_path", "destination_folder", "is_directory"],
["dest_path", "new_path", "to", "destination", "is_dir"],
),
"delete_note": (["is_directory"], ["is_dir"]),
"read_content": (["path"], ["file_path", "filepath", "file"]),
# view_note pagination params removed in #693 (delegates to read_note).
"view_note": (
[],
["page", "page_size", "offset", "limit", "page_number", "per_page"],
),
"build_context": (
["url", "timeframe", "page", "page_size", "max_related"],
["uri", "memory_url", "since", "offset", "limit", "max_results", "limit_related"],
),
"canvas": (["directory"], ["folder", "dir", "path"]),
}
for tool_name, (must_have, must_not_have) in checks.items():
assert tool_name in tools, f"tool {tool_name} not registered"
props = tools[tool_name].inputSchema["properties"]
for canonical in must_have:
assert canonical in props, f"{tool_name}: canonical '{canonical}' missing"
for alias in must_not_have:
assert alias not in props, f"{tool_name}: alias '{alias}' leaked into schema"
@@ -588,3 +588,122 @@ async def test_nested_project_paths_rejected(mcp_server, app, test_project, tmp_
# Clean up parent project
await client.call_tool("delete_project", {"project_name": parent_name})
@pytest.mark.asyncio
async def test_create_project_accepts_workspace_in_local_mode(
mcp_server, app, test_project, tmp_path
):
"""Passing workspace via the MCP wire is accepted by the tool schema and
does not break the local create path.
In local mode there is no cloud factory installed, so workspace is a no-op:
the request lands on the ASGI transport which has no workspace concept. This
test guards the schema so a future change can't accidentally drop the parameter.
"""
async with Client(mcp_server) as client:
create_result = await client.call_tool(
"create_memory_project",
{
"project_name": "ws-local-test",
"project_path": str(
tmp_path.parent / (tmp_path.name + "-projects") / "project-ws-local-test"
),
"workspace": "team-paul",
},
)
assert len(create_result.content) == 1
create_text = create_result.content[0].text # pyright: ignore [reportAttributeAccessIssue]
assert "" in create_text
assert "ws-local-test" in create_text
list_result = await client.call_tool("list_memory_projects", {})
assert "ws-local-test" in list_result.content[0].text # pyright: ignore [reportAttributeAccessIssue]
@pytest.mark.asyncio
async def test_create_project_workspace_slug_forwarded_to_factory_as_tenant_id(
mcp_server, app, test_project, tmp_path
):
"""workspace slug resolves before the tenant id flows to the cloud factory.
Simulates the cloud MCP server pattern (set_client_factory) and verifies the
factory receives the workspace argument. This is the chicken-and-egg case:
no project_id exists yet, so workspace is the only way to target a
non-default workspace at create time.
"""
from contextlib import asynccontextmanager
from unittest.mock import AsyncMock, patch
from httpx import ASGITransport, AsyncClient as HttpxAsyncClient
from basic_memory.mcp import async_client
from basic_memory.mcp.tools import project_management
from basic_memory.schemas.cloud import WorkspaceInfo
captured_workspaces: list[str | None] = []
resolved_workspace = WorkspaceInfo(
tenant_id="tenant-cloud-test",
name="Team Paul",
workspace_type="organization",
slug="team-paul",
role="owner",
organization_id="org-team-paul",
is_default=False,
has_active_subscription=True,
)
@asynccontextmanager
async def fake_factory(workspace=None):
captured_workspaces.append(workspace)
# Yield an ASGI-backed httpx client so the create_project HTTP call
# actually reaches the FastAPI app and the project is created in the DB.
async with HttpxAsyncClient(
transport=ASGITransport(app=app), base_url="http://test"
) as inner:
yield inner
original_factory = async_client._client_factory
async_client.set_client_factory(fake_factory)
try:
with patch.object(
project_management,
"resolve_workspace_parameter",
new_callable=AsyncMock,
return_value=resolved_workspace,
) as mock_resolve_workspace:
async with Client(mcp_server) as mcp_client:
create_result = await mcp_client.call_tool(
"create_memory_project",
{
"project_name": "ws-routed-project",
"project_path": str(
tmp_path.parent
/ (tmp_path.name + "-projects")
/ "project-ws-routed-project"
),
"workspace": "team-paul",
},
)
create_text = create_result.content[0].text # pyright: ignore [reportAttributeAccessIssue]
assert "" in create_text
assert "ws-routed-project" in create_text
mock_resolve_workspace.assert_awaited_once()
await_args = mock_resolve_workspace.await_args
assert await_args is not None
assert await_args.kwargs["workspace"] == "team-paul"
# The factory must have been invoked with the tenant id resolved from the slug.
# create_memory_project opens one get_client() context, so the factory is
# called once per tool invocation; both list_projects and create_project
# share that single client.
assert captured_workspaces, "Factory was never invoked"
assert all(ws == "tenant-cloud-test" for ws in captured_workspaces), (
"Expected workspace='tenant-cloud-test' on every factory call, "
f"got {captured_workspaces}"
)
finally:
async_client._client_factory = original_factory
@@ -100,3 +100,71 @@ async def test_read_note_underscored_folder_by_permalink(mcp_server, app, test_p
assert "# Example Note" in result_text
assert "This is a test note in an underscored folder." in result_text
assert f"{test_project.name}/archive/articles/example-note" in result_text # permalink
@pytest.mark.asyncio
async def test_read_note_by_project_id(mcp_server, app, test_project):
"""Read a note by passing project_id (UUID) instead of project name.
Verifies the project_id parameter routes through get_project_client correctly
in pure local mode (no cloud creds), where get_project_mode() would otherwise
default unknown identifiers to CLOUD and break routing.
"""
async with Client(mcp_server) as client:
await client.call_tool(
"write_note",
{
"project": test_project.name,
"title": "By ID Note",
"directory": "test",
"content": "# By ID Note\n\nLooked up by external_id.",
},
)
# Read by external_id (UUID) instead of project name
read_result = await client.call_tool(
"read_note",
{
"project_id": test_project.external_id,
"identifier": "By ID Note",
},
)
assert len(read_result.content) == 1
assert read_result.content[0].type == "text"
result_text = read_result.content[0].text
assert "# By ID Note" in result_text
assert "Looked up by external_id." in result_text
@pytest.mark.asyncio
async def test_read_note_project_id_takes_precedence_over_name(mcp_server, app, test_project):
"""When project_id is passed alongside a wrong project name, project_id wins."""
async with Client(mcp_server) as client:
await client.call_tool(
"write_note",
{
"project": test_project.name,
"title": "Precedence Note",
"directory": "test",
"content": "# Precedence Note\n\nproject_id wins.",
},
)
# Pass an obviously-wrong project name alongside the correct project_id.
# If project_id takes precedence (as documented), the read still succeeds.
read_result = await client.call_tool(
"read_note",
{
"project": "this-project-does-not-exist",
"project_id": test_project.external_id,
"identifier": "Precedence Note",
},
)
assert len(read_result.content) == 1
result_text = read_result.content[0].text
assert "# Precedence Note" in result_text
assert "project_id wins." in result_text
+49 -22
View File
@@ -8,6 +8,7 @@ SearchService for each (backend, provider) combination.
from __future__ import annotations
import os
from collections.abc import AsyncGenerator
from dataclasses import dataclass
from pathlib import Path
@@ -15,7 +16,12 @@ import pytest
import pytest_asyncio
from dotenv import load_dotenv
from sqlalchemy import text
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
from sqlalchemy.ext.asyncio import (
AsyncEngine,
AsyncSession,
async_sessionmaker,
create_async_engine,
)
from sqlalchemy.pool import NullPool
from testcontainers.postgres import PostgresContainer
@@ -152,25 +158,8 @@ async def sqlite_engine_factory(tmp_path):
yield engine, session_maker
@pytest_asyncio.fixture
async def postgres_engine_factory(pgvector_container):
"""Create a Postgres engine + session factory with pgvector extension."""
if pgvector_container is None:
yield None
return
sync_url = pgvector_container.get_connection_url()
async_url = sync_url.replace("postgresql+psycopg2", "postgresql+asyncpg")
engine = create_async_engine(async_url, echo=False, poolclass=NullPool)
session_maker = async_sessionmaker(
bind=engine,
class_=AsyncSession,
expire_on_commit=False,
autoflush=False,
)
# Create schema from scratch for each test
async def _reset_postgres_semantic_schema(engine: AsyncEngine) -> None:
"""Reset the semantic Postgres schema to a clean baseline."""
async with engine.begin() as conn:
await conn.execute(text("DROP TABLE IF EXISTS search_vector_embeddings CASCADE"))
await conn.execute(text("DROP TABLE IF EXISTS search_vector_chunks CASCADE"))
@@ -182,9 +171,47 @@ async def postgres_engine_factory(pgvector_container):
await conn.execute(CREATE_POSTGRES_SEARCH_INDEX_METADATA)
await conn.execute(CREATE_POSTGRES_SEARCH_INDEX_PERMALINK)
yield engine, session_maker
await engine.dispose()
@pytest_asyncio.fixture(scope="session", loop_scope="session")
async def postgres_engine(pgvector_container) -> AsyncGenerator[AsyncEngine | None, None]:
"""Create the shared semantic Postgres engine once per test session."""
if pgvector_container is None:
yield None
return
sync_url = pgvector_container.get_connection_url()
async_url = sync_url.replace("postgresql+psycopg2", "postgresql+asyncpg")
engine = create_async_engine(async_url, echo=False, poolclass=NullPool)
try:
yield engine
finally:
await engine.dispose()
@pytest_asyncio.fixture
async def postgres_engine_factory(postgres_engine):
"""Create a Postgres session factory isolated by schema reset."""
if postgres_engine is None:
yield None
return
# Trigger: semantic provider combos create and rebuild vector tables.
# Why: the main suite showed savepoint-bound shared connections can get
# poisoned by app-level rollback/recovery paths.
# Outcome: keep the pgvector engine warm, but reset schema per test and let
# repository code use normal Postgres transactions.
await _reset_postgres_semantic_schema(postgres_engine)
session_maker = async_sessionmaker(
bind=postgres_engine,
class_=AsyncSession,
expire_on_commit=False,
autoflush=False,
)
yield postgres_engine, session_maker
# --- Embedding provider factories ---
+8 -7
View File
@@ -9,6 +9,8 @@ These tests isolate specific problems with the search pipeline:
from __future__ import annotations
from typing import Any, cast
import pytest
from basic_memory.config import DatabaseBackend
@@ -335,11 +337,10 @@ async def test_similarity_formula_analysis(sqlite_engine_factory, tmp_path):
from basic_memory import db as bm_db
async with bm_db.scoped_session(service.repository.session_maker) as session:
await service.repository._prepare_vector_session(session)
raw_rows = await service.repository._run_vector_query(
session, query_embedding, candidate_limit=20
)
repo = cast(Any, service.repository)
async with bm_db.scoped_session(repo.session_maker) as session:
await repo._prepare_vector_session(session)
raw_rows = await repo._run_vector_query(session, query_embedding, candidate_limit=20)
print(f"\nQuery: '{query_text}'")
print(f" {'chunk_key':<40} {'distance':>10} {'sim_old':>12} {'sim_new':>12}")
@@ -347,7 +348,7 @@ async def test_similarity_formula_analysis(sqlite_engine_factory, tmp_path):
dist = float(row["best_distance"])
sim_old = 1.0 / (1.0 + max(dist, 0.0))
# New formula: L2 distance → cosine similarity for normalized embeddings
sim_new = service.repository._distance_to_similarity(dist)
sim_new = repo._distance_to_similarity(dist)
print(f" {row['chunk_key']:<40} {dist:>10.4f} {sim_old:>12.4f} {sim_new:>12.4f}")
@@ -431,7 +432,7 @@ async def test_chunking_produces_reasonable_chunks(sqlite_engine_factory, tmp_pa
service = await create_search_service(
sqlite_engine_factory, DIAG_COMBO, tmp_path, embedding_provider=provider
)
repo = service.repository
repo = cast(Any, service.repository)
# Simulate a typical entity with observations
text_input = (
+3 -1
View File
@@ -13,6 +13,8 @@ Uses postgres-fastembed combo (no OpenAI dependency) with the pgvector container
from __future__ import annotations
from typing import Any, cast
import pytest
from basic_memory.config import DatabaseBackend
@@ -194,7 +196,7 @@ async def test_postgres_vector_dimension_detection(postgres_engine_factory, tmp_
postgres_engine_factory, PG_FASTEMBED, tmp_path, embedding_provider=provider
)
repo = search_service.repository
repo = cast(Any, search_service.repository)
# First entity triggers _ensure_vector_tables
entity = await search_service.entity_repository.create(
+11 -6
View File
@@ -8,6 +8,11 @@ import pytest
from sqlalchemy import text
def _first_value(row):
assert row is not None
return row[0]
@pytest.mark.asyncio
async def test_wal_mode_enabled(engine_factory, db_backend):
"""Test that WAL mode is enabled on filesystem database connections."""
@@ -19,7 +24,7 @@ async def test_wal_mode_enabled(engine_factory, db_backend):
# Execute a query to verify WAL mode is enabled
async with engine.connect() as conn:
result = await conn.execute(text("PRAGMA journal_mode"))
journal_mode = result.fetchone()[0]
journal_mode = _first_value(result.fetchone())
# WAL mode should be enabled for filesystem databases
assert journal_mode.upper() == "WAL"
@@ -35,7 +40,7 @@ async def test_busy_timeout_configured(engine_factory, db_backend):
async with engine.connect() as conn:
result = await conn.execute(text("PRAGMA busy_timeout"))
busy_timeout = result.fetchone()[0]
busy_timeout = _first_value(result.fetchone())
# Busy timeout should be 10 seconds (10000 milliseconds)
assert busy_timeout == 10000
@@ -51,7 +56,7 @@ async def test_synchronous_mode_configured(engine_factory, db_backend):
async with engine.connect() as conn:
result = await conn.execute(text("PRAGMA synchronous"))
synchronous = result.fetchone()[0]
synchronous = _first_value(result.fetchone())
# Synchronous should be NORMAL (1)
assert synchronous == 1
@@ -67,7 +72,7 @@ async def test_cache_size_configured(engine_factory, db_backend):
async with engine.connect() as conn:
result = await conn.execute(text("PRAGMA cache_size"))
cache_size = result.fetchone()[0]
cache_size = _first_value(result.fetchone())
# Cache size should be -64000 (64MB)
assert cache_size == -64000
@@ -83,7 +88,7 @@ async def test_temp_store_configured(engine_factory, db_backend):
async with engine.connect() as conn:
result = await conn.execute(text("PRAGMA temp_store"))
temp_store = result.fetchone()[0]
temp_store = _first_value(result.fetchone())
# temp_store should be MEMORY (2)
assert temp_store == 2
@@ -114,7 +119,7 @@ async def test_windows_locking_mode_when_on_windows(tmp_path, monkeypatch, confi
):
async with engine.connect() as conn:
result = await conn.execute(text("PRAGMA locking_mode"))
locking_mode = result.fetchone()[0]
locking_mode = _first_value(result.fetchone())
# Locking mode should be NORMAL on Windows
assert locking_mode.upper() == "NORMAL"
+12 -3
View File
@@ -1,5 +1,6 @@
"""Fixtures for V2 API tests."""
from collections.abc import Generator
from typing import Any, AsyncGenerator
import pytest
@@ -13,13 +14,21 @@ from basic_memory.models import Project
@pytest_asyncio.fixture
async def app(test_config, engine_factory, app_config) -> FastAPI:
async def app(test_config, engine_factory, app_config) -> AsyncGenerator[FastAPI, None]:
"""Create FastAPI test application."""
from basic_memory.api.app import app
previous_overrides = dict(app.dependency_overrides)
app.dependency_overrides[get_app_config] = lambda: app_config
app.dependency_overrides[get_engine_factory] = lambda: engine_factory
return app
try:
yield app
finally:
# Trigger: the FastAPI app is a module-level singleton shared across tests.
# Why: dependency overrides that capture a per-test engine can leak into
# later CLI/MCP tests and create connections outside fixture ownership.
# Outcome: each API test leaves the shared app exactly as it found it.
app.dependency_overrides = previous_overrides
@pytest_asyncio.fixture
@@ -30,7 +39,7 @@ async def client(app: FastAPI) -> AsyncGenerator[AsyncClient, None]:
@pytest.fixture(autouse=True)
def task_scheduler_spy(app: FastAPI) -> list[dict[str, Any]]:
def task_scheduler_spy(app: FastAPI) -> Generator[list[dict[str, Any]], None, None]:
"""Capture scheduled task specs without executing them."""
scheduled: list[dict[str, Any]] = []

Some files were not shown because too many files have changed in this diff Show More