mirror of
https://github.com/basicmachines-co/basic-memory
synced 2026-06-21 13:47:35 +00:00
Compare commits
36 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 2d65d1b6bb | |||
| e1adf10f1b | |||
| fa4845119d | |||
| 3589e21278 | |||
| 7f9fc80e27 | |||
| a661e924df | |||
| 05adda1502 | |||
| 0a72d81bb3 | |||
| 5ccf433cad | |||
| b4bf14ebf7 | |||
| 0b335476d6 | |||
| 2fccc74a20 | |||
| c4956cac16 | |||
| 128c2da40c | |||
| 2bfb9c76df | |||
| 3d927b848f | |||
| 953fe20aef | |||
| a4282d9f2f | |||
| 799dd6c629 | |||
| 26e74ea118 | |||
| ee1558ea68 | |||
| 4d62b623db | |||
| 2fe4488eda | |||
| f3e46d7984 | |||
| 56d6f1b4a5 | |||
| 1b39062ecd | |||
| c4cf0aff1e | |||
| 1c343bed66 | |||
| c50d97e548 | |||
| e2e65575d6 | |||
| bf9a6b4a75 | |||
| 474100efef | |||
| b3d5448355 | |||
| 4e53bb83fd | |||
| 8f2b25f0e0 | |||
| 052545b661 |
@@ -3,7 +3,6 @@
|
||||
"env": {
|
||||
"CLAUDE_BASH_MAINTAIN_PROJECT_WORKING_DIR": "1",
|
||||
"CLAUDE_CODE_DISABLE_FEEDBACK_SURVEY": "1",
|
||||
"DISABLE_TELEMETRY": "1",
|
||||
"CLAUDE_CODE_NO_FLICKER": "1",
|
||||
"CLAUDE_CODE_DISABLE_ADAPTIVE_THINKING": "1"
|
||||
},
|
||||
|
||||
@@ -5,9 +5,11 @@ concurrency:
|
||||
cancel-in-progress: true
|
||||
|
||||
on:
|
||||
# Trigger: PR branch pushes already publish commit statuses that show up on the PR.
|
||||
# Why: running the full matrix on both push and pull_request doubles CI time for the
|
||||
# exact same branch head commit.
|
||||
# Outcome: each branch push runs the test suite once, including PR updates.
|
||||
push:
|
||||
pull_request:
|
||||
branches: [ "main" ]
|
||||
|
||||
jobs:
|
||||
static-checks:
|
||||
|
||||
@@ -22,16 +22,16 @@ See the [README.md](README.md) file for a project overview.
|
||||
- Run unit tests (Postgres): `just test-unit-postgres`
|
||||
- Run integration tests (SQLite): `just test-int-sqlite`
|
||||
- Run integration tests (Postgres): `just test-int-postgres`
|
||||
- Run impacted tests: `just testmon` (pytest-testmon)
|
||||
- Run impacted tests: `just testmon` (pytest-testmon; only tests affected by changed code)
|
||||
- Run MCP smoke test: `just test-smoke`
|
||||
- Fast local loop: `just fast-check`
|
||||
- Fast local loop: `just fast-check` (default iteration flow)
|
||||
- Local consistency check: `just doctor`
|
||||
- Generate HTML coverage: `just coverage`
|
||||
- Single test: `pytest tests/path/to/test_file.py::test_function_name`
|
||||
- Run benchmarks: `pytest test-int/test_sync_performance_benchmark.py -v -m "benchmark and not slow"`
|
||||
- Lint: `just lint` or `ruff check . --fix`
|
||||
- Type check: `just typecheck` or `uv run pyright`
|
||||
- Type check (supplemental): `just typecheck-ty` or `uv run ty check src/`
|
||||
- Type check: `just typecheck` or `uv run ty check src tests test-int`
|
||||
- Type check (pyright): `just typecheck-pyright` or `uv run pyright`
|
||||
- Format: `just format` or `uv run ruff format .`
|
||||
- Run all code checks: `just check` (runs lint, format, typecheck, test)
|
||||
- Create db migration: `just migration "Your migration message"`
|
||||
@@ -48,10 +48,12 @@ See the [README.md](README.md) file for a project overview.
|
||||
### Code/Test/Verify Loop (fast path)
|
||||
|
||||
1) **Code:** make changes.
|
||||
2) **Test:** `just fast-check` (lint/format/typecheck + impacted tests + MCP smoke).
|
||||
2) **Test:** `just fast-check` (lint/format/typecheck + pytest-testmon impacted tests for changed code).
|
||||
3) **Verify:** `just doctor` (end-to-end file ↔ DB loop in a temp project).
|
||||
4) **Full gate (when needed):** `just test` or `just check` for SQLite + Postgres.
|
||||
|
||||
Run `just test-smoke` when you specifically need the MCP smoke flow.
|
||||
|
||||
If testmon is “cold,” the first run may be long. Subsequent runs get much faster.
|
||||
|
||||
### Test Structure
|
||||
|
||||
+5
-5
@@ -2125,12 +2125,12 @@ Signed-off-by: phernandez <paul@basicmachines.co>
|
||||
- Update CLAUDE.md ([#33](https://github.com/basicmachines-co/basic-memory/pull/33),
|
||||
[`dfaf0fe`](https://github.com/basicmachines-co/basic-memory/commit/dfaf0fea9cf5b97d169d51a6276ec70162c21a7e))
|
||||
|
||||
fix spelling in CLAUDE.md: enviroment -> environment Signed-off-by: Ikko Eltociear Ashimine
|
||||
fix spelling in CLAUDE.md: environment typo Signed-off-by: Ikko Eltociear Ashimine
|
||||
<eltociear@gmail.com>
|
||||
|
||||
### Refactoring
|
||||
|
||||
- Move project stats into projct subcommand
|
||||
- Move project stats into project subcommand
|
||||
([`2a881b1`](https://github.com/basicmachines-co/basic-memory/commit/2a881b1425c73947f037fbe7ac5539c015b62526))
|
||||
|
||||
Signed-off-by: phernandez <paul@basicmachines.co>
|
||||
@@ -2559,7 +2559,7 @@ Co-authored-by: phernandez <phernandez@basicmachines.co>
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- Refix vitual env in installer build
|
||||
- Refix virtual env in installer build
|
||||
([`052f491`](https://github.com/basicmachines-co/basic-memory/commit/052f491fff629e8ead629c9259f8cb46c608d584))
|
||||
|
||||
|
||||
@@ -2578,7 +2578,7 @@ Co-authored-by: phernandez <phernandez@basicmachines.co>
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- Fix path to intaller app artifact
|
||||
- Fix path to installer app artifact
|
||||
([`53d220d`](https://github.com/basicmachines-co/basic-memory/commit/53d220df585561f9edd0d49a9e88f1d4055059cf))
|
||||
|
||||
|
||||
@@ -2586,7 +2586,7 @@ Co-authored-by: phernandez <phernandez@basicmachines.co>
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- Activate vitualenv in installer build
|
||||
- Activate virtualenv in installer build
|
||||
([`d4c8293`](https://github.com/basicmachines-co/basic-memory/commit/d4c8293687a52eaf3337fe02e2f7b80e4cc9a1bb))
|
||||
|
||||
- Trigger installer build on release
|
||||
|
||||
@@ -516,7 +516,7 @@ canvas(nodes, edges, title, folder) - Generate knowledge visualizations
|
||||
"What have I been working on in the past week?"
|
||||
```
|
||||
|
||||
## Futher info
|
||||
## Further info
|
||||
|
||||
See the [Documentation](https://docs.basicmemory.com?utm_source=github&utm_medium=referral&utm_campaign=readme) for more info, including:
|
||||
|
||||
|
||||
+3
-1
@@ -17,7 +17,9 @@ services:
|
||||
volumes:
|
||||
|
||||
# Persistent storage for configuration and database
|
||||
- basic-memory-config:/root/.basic-memory:rw
|
||||
# Container runs as `appuser` (Dockerfile USER directive), so the CLI
|
||||
# config dir lives under /home/appuser, not /root.
|
||||
- basic-memory-config:/home/appuser/.basic-memory:rw
|
||||
|
||||
# Mount your knowledge directory (required)
|
||||
# Change './knowledge' to your actual Obsidian vault or knowledge directory
|
||||
|
||||
@@ -91,10 +91,11 @@ SQLite Database (Index)
|
||||
# List all projects
|
||||
projects = await list_memory_projects()
|
||||
|
||||
# Response structure:
|
||||
# Response structure (each entry includes external_id you can pass as project_id):
|
||||
# [
|
||||
# {
|
||||
# "name": "main",
|
||||
# "external_id": "550e8400-e29b-41d4-a716-446655440000",
|
||||
# "path": "/Users/name/notes",
|
||||
# "is_default": True,
|
||||
# "note_count": 156,
|
||||
@@ -102,6 +103,7 @@ projects = await list_memory_projects()
|
||||
# },
|
||||
# {
|
||||
# "name": "work",
|
||||
# "external_id": "9f86d081-884c-42a3-b5e3-1c0c5b4c8e52",
|
||||
# "path": "/Users/name/work-notes",
|
||||
# "is_default": False,
|
||||
# "note_count": 89,
|
||||
@@ -164,6 +166,44 @@ active_project = "main"
|
||||
results = await search_notes(query="topic", project=active_project)
|
||||
```
|
||||
|
||||
### `project` vs `project_id`
|
||||
|
||||
Every project has two identifiers:
|
||||
|
||||
- **`project`** — human-readable name (e.g., `"main"`). Easy to use, but can collide across cloud workspaces.
|
||||
- **`project_id`** — stable `external_id` UUID. Always unambiguous; takes precedence over `project` when both are passed.
|
||||
|
||||
**When to prefer `project_id`:**
|
||||
|
||||
1. **Cloud multi-workspace setups.** If the user belongs to more than one workspace (personal + organization, or several organizations) and the same project name might exist in more than one of them, pass `project_id` to route to the exact project. Without it, name resolution falls back to the default workspace, which may not be the one the user means.
|
||||
2. **After `list_memory_projects()`.** Once you have the `external_id`, prefer using it — it's the same number of characters in JSON and saves a name-resolution round-trip.
|
||||
3. **When persisting a project choice across a long session.** UUIDs are stable; names can be renamed.
|
||||
|
||||
**When `project` (name) is fine:**
|
||||
|
||||
- Local single-workspace setups (no collision risk).
|
||||
- One-off operations where the name is clearly visible to the user (e.g., quick `search_notes(project="main", ...)`).
|
||||
- The user explicitly references a project by name in their message.
|
||||
|
||||
**Example — cloud multi-workspace pattern:**
|
||||
|
||||
```python
|
||||
# Discover and pick the right project for this user
|
||||
projects = await list_memory_projects()
|
||||
target = next(p for p in projects if p["name"] == "research" and p["workspace"]["slug"] == "acme")
|
||||
|
||||
# Use the UUID for all subsequent operations — no ambiguity
|
||||
await write_note(
|
||||
title="Meeting Notes",
|
||||
content="...",
|
||||
folder="meetings",
|
||||
project_id=target["external_id"],
|
||||
)
|
||||
results = await search_notes(query="kickoff", project_id=target["external_id"])
|
||||
```
|
||||
|
||||
**Precedence rule:** When both are passed, `project_id` wins. This lets you safely supply `project="main"` for backward compatibility while still routing precisely with `project_id`.
|
||||
|
||||
### Cross-Project Operations
|
||||
|
||||
**Some tools work across all projects when project parameter omitted:**
|
||||
|
||||
+4
-4
@@ -113,7 +113,7 @@ bm project add research --cloud
|
||||
bm project add research --cloud --local-path ~/Documents/research
|
||||
|
||||
# Or configure sync for existing project
|
||||
bm project sync-setup research ~/Documents/research
|
||||
bm cloud sync-setup research ~/Documents/research
|
||||
```
|
||||
|
||||
**What happens under the covers:**
|
||||
@@ -236,7 +236,7 @@ bm project add research --cloud --local-path ~/Documents/research
|
||||
|
||||
```bash
|
||||
# Project already exists on cloud
|
||||
bm project sync-setup research ~/Documents/research
|
||||
bm cloud sync-setup research ~/Documents/research
|
||||
```
|
||||
|
||||
**What this does:**
|
||||
@@ -739,7 +739,7 @@ bm project sync --name research
|
||||
**Solution:**
|
||||
|
||||
```bash
|
||||
bm project sync-setup research ~/Documents/research
|
||||
bm cloud sync-setup research ~/Documents/research
|
||||
bm project bisync --name research --resync
|
||||
```
|
||||
|
||||
@@ -795,7 +795,7 @@ bm project list --local # Local project list
|
||||
bm project list --cloud # Cloud project list
|
||||
bm project add <name> --cloud # Create cloud project (no sync)
|
||||
bm project add <name> --cloud --local-path <path> # Create with local sync
|
||||
bm project sync-setup <name> <path> # Add sync to existing project
|
||||
bm cloud sync-setup <name> <path> # Add sync to existing project
|
||||
bm project rm <name> # Delete project
|
||||
```
|
||||
|
||||
|
||||
@@ -62,20 +62,20 @@ test-int-postgres:
|
||||
fi
|
||||
|
||||
# Run tests impacted by recent changes (requires pytest-testmon)
|
||||
# Pass paths or node ids after `just testmon` to limit the candidate set further.
|
||||
testmon *args:
|
||||
BASIC_MEMORY_ENV=test uv run pytest -p pytest_mock -v --no-cov --testmon --testmon-forceselect {{args}}
|
||||
BASIC_MEMORY_ENV=test uv run pytest -p pytest_mock -v --no-cov --testmon {{args}}
|
||||
|
||||
# Run MCP smoke test (fast end-to-end loop)
|
||||
test-smoke:
|
||||
BASIC_MEMORY_ENV=test uv run pytest -p pytest_mock -v --no-cov -m smoke test-int/mcp/test_smoke_integration.py
|
||||
|
||||
# Fast local loop: lint, format, typecheck, impacted tests
|
||||
# Fast local loop: lint, format, typecheck, impacted tests via pytest-testmon
|
||||
fast-check:
|
||||
just fix
|
||||
just format
|
||||
just typecheck
|
||||
just testmon
|
||||
just test-smoke
|
||||
|
||||
# Reset Postgres test database (drops and recreates schema)
|
||||
# Useful when Alembic migration state gets out of sync during development
|
||||
@@ -170,13 +170,17 @@ lint: fix
|
||||
fix:
|
||||
uv run ruff check --fix --unsafe-fixes src tests test-int
|
||||
|
||||
# Type check code (pyright)
|
||||
# Type check code (ty)
|
||||
typecheck:
|
||||
uv run ty check src tests test-int
|
||||
|
||||
# Type check code (pyright)
|
||||
typecheck-pyright:
|
||||
uv run pyright
|
||||
|
||||
# Type check code (ty)
|
||||
typecheck-ty:
|
||||
uv run ty check src/
|
||||
just typecheck
|
||||
|
||||
# Clean build artifacts and cache files
|
||||
clean:
|
||||
|
||||
+9
-4
@@ -33,7 +33,7 @@ dependencies = [
|
||||
"pyjwt>=2.10.1",
|
||||
"python-dotenv>=1.1.0",
|
||||
"pytest-aio>=1.9.0",
|
||||
"aiofiles>=24.1.0", # Optional observability (disabled by default via config)
|
||||
"aiofiles>=24.1.0",
|
||||
"asyncpg>=0.30.0",
|
||||
"nest-asyncio>=1.6.0", # For Alembic migrations with Postgres
|
||||
"pytest-asyncio>=1.2.0",
|
||||
@@ -47,6 +47,8 @@ dependencies = [
|
||||
"fastembed>=0.7.4",
|
||||
"sqlite-vec>=0.1.6",
|
||||
"openai>=1.100.2",
|
||||
"logfire>=4.19.0",
|
||||
"psutil>=5.9.0",
|
||||
]
|
||||
|
||||
[project.urls]
|
||||
@@ -58,9 +60,6 @@ Documentation = "https://github.com/basicmachines-co/basic-memory#readme"
|
||||
basic-memory = "basic_memory.cli.main:app"
|
||||
bm = "basic_memory.cli.main:app"
|
||||
|
||||
[project.optional-dependencies]
|
||||
telemetry = ["logfire>=4.19.0"]
|
||||
|
||||
[build-system]
|
||||
requires = ["hatchling", "uv-dynamic-versioning>=0.7.0"]
|
||||
build-backend = "hatchling.build"
|
||||
@@ -71,6 +70,12 @@ addopts = "--cov=basic_memory --cov-report term-missing"
|
||||
testpaths = ["tests", "test-int"]
|
||||
asyncio_mode = "strict"
|
||||
asyncio_default_fixture_loop_scope = "function"
|
||||
filterwarnings = [
|
||||
"ignore:The @wait_container_is_ready decorator is deprecated.*:DeprecationWarning:testcontainers\\.core\\.waiting_utils",
|
||||
"ignore:The default datetime adapter is deprecated as of Python 3\\.12.*:DeprecationWarning:aiosqlite\\.core",
|
||||
"ignore:codecs\\.open\\(\\) is deprecated\\. Use open\\(\\) instead\\.:DeprecationWarning:frontmatter",
|
||||
"ignore:Parsing dates involving a day of month without a year specified is ambiguous.*:DeprecationWarning:dateparser\\.utils\\.strptime",
|
||||
]
|
||||
markers = [
|
||||
"benchmark: Performance benchmark tests (deselect with '-m \"not benchmark\"')",
|
||||
"slow: Slow-running tests (deselect with '-m \"not slow\"')",
|
||||
|
||||
@@ -4,6 +4,7 @@ from contextlib import asynccontextmanager
|
||||
|
||||
from fastapi import FastAPI, HTTPException, Request
|
||||
from fastapi.exception_handlers import http_exception_handler
|
||||
from fastapi.responses import JSONResponse
|
||||
from fastapi.routing import APIRouter
|
||||
from loguru import logger
|
||||
|
||||
@@ -25,10 +26,16 @@ from basic_memory.api.v2.routers.project_router import (
|
||||
list_projects,
|
||||
synchronize_projects,
|
||||
)
|
||||
from basic_memory import telemetry
|
||||
import logfire
|
||||
from basic_memory.config import init_api_logging
|
||||
from basic_memory.services.exceptions import EntityAlreadyExistsError
|
||||
from basic_memory.services.initialization import initialize_app
|
||||
from basic_memory.workspace_context import (
|
||||
WORKSPACE_SLUG_HEADER,
|
||||
WORKSPACE_TYPE_HEADER,
|
||||
workspace_permalink_context_validation_error,
|
||||
workspace_permalink_context,
|
||||
)
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
@@ -44,7 +51,7 @@ async def lifespan(app: FastAPI): # pragma: no cover
|
||||
set_container(container)
|
||||
app.state.container = container
|
||||
|
||||
with telemetry.operation(
|
||||
with logfire.span(
|
||||
"api.lifecycle.startup",
|
||||
entrypoint="api",
|
||||
mode=container.mode.name.lower(),
|
||||
@@ -69,7 +76,7 @@ async def lifespan(app: FastAPI): # pragma: no cover
|
||||
yield
|
||||
|
||||
# Shutdown - coordinator handles clean task cancellation
|
||||
with telemetry.operation(
|
||||
with logfire.span(
|
||||
"api.lifecycle.shutdown",
|
||||
entrypoint="api",
|
||||
mode=container.mode.name.lower(),
|
||||
@@ -87,6 +94,32 @@ app = FastAPI(
|
||||
lifespan=lifespan,
|
||||
)
|
||||
|
||||
|
||||
@app.middleware("http")
|
||||
async def workspace_permalink_context_middleware(request: Request, call_next):
|
||||
"""Populate workspace permalink context from request headers."""
|
||||
workspace_slug = request.headers.get(WORKSPACE_SLUG_HEADER)
|
||||
workspace_type = request.headers.get(WORKSPACE_TYPE_HEADER)
|
||||
|
||||
validation_error = workspace_permalink_context_validation_error(workspace_slug, workspace_type)
|
||||
if validation_error is not None:
|
||||
return JSONResponse(
|
||||
status_code=400,
|
||||
content={"detail": validation_error},
|
||||
)
|
||||
|
||||
if not workspace_slug:
|
||||
return await call_next(request)
|
||||
|
||||
# ContextVar state remains active across the awaited downstream handler while
|
||||
# this context manager is open, so entity creation can see request metadata.
|
||||
with workspace_permalink_context(
|
||||
workspace_slug=workspace_slug,
|
||||
workspace_type=workspace_type,
|
||||
):
|
||||
return await call_next(request)
|
||||
|
||||
|
||||
# Include v2 routers FIRST (more specific paths must match before /{project} catch-all)
|
||||
app.include_router(v2_knowledge, prefix="/v2/projects/{project_id}")
|
||||
app.include_router(v2_memory, prefix="/v2/projects/{project_id}")
|
||||
@@ -146,4 +179,7 @@ async def exception_handler(request, exc): # pragma: no cover
|
||||
error_type=type(exc).__name__,
|
||||
error=str(exc),
|
||||
)
|
||||
return await http_exception_handler(request, HTTPException(status_code=500, detail=str(exc)))
|
||||
return await http_exception_handler(
|
||||
request,
|
||||
HTTPException(status_code=500, detail="Internal server error"),
|
||||
)
|
||||
|
||||
@@ -10,10 +10,10 @@ Key improvements:
|
||||
- Simplified caching strategies
|
||||
"""
|
||||
|
||||
from fastapi import APIRouter, HTTPException, BackgroundTasks, Depends, Response, Path, Query
|
||||
from fastapi import APIRouter, HTTPException, Response, Path
|
||||
from loguru import logger
|
||||
|
||||
from basic_memory import telemetry
|
||||
import logfire
|
||||
from basic_memory.deps import (
|
||||
EntityServiceV2ExternalDep,
|
||||
SearchServiceV2ExternalDep,
|
||||
@@ -24,7 +24,6 @@ from basic_memory.deps import (
|
||||
RelationRepositoryV2ExternalDep,
|
||||
ProjectExternalIdPathDep,
|
||||
TaskSchedulerDep,
|
||||
FileServiceV2ExternalDep,
|
||||
)
|
||||
from basic_memory.schemas import DeleteEntitiesResponse
|
||||
from basic_memory.schemas.base import Entity
|
||||
@@ -75,34 +74,40 @@ async def get_graph(
|
||||
Returns a flat node/edge structure optimized for rendering with graph libraries.
|
||||
Only includes resolved relations (where to_id is not null).
|
||||
"""
|
||||
logger.info("API v2 request: get_graph")
|
||||
with logfire.span(
|
||||
"api.request.knowledge.get_graph",
|
||||
entrypoint="api",
|
||||
domain="knowledge",
|
||||
action="get_graph",
|
||||
):
|
||||
logger.info("API v2 request: get_graph")
|
||||
|
||||
# Fetch all entities for this project
|
||||
entities = await entity_repository.find_all(use_load_options=False)
|
||||
nodes = [
|
||||
GraphNode(
|
||||
external_id=entity.external_id,
|
||||
title=entity.title,
|
||||
note_type=entity.note_type,
|
||||
file_path=entity.file_path,
|
||||
)
|
||||
for entity in entities
|
||||
]
|
||||
# Fetch all entities for this project
|
||||
entities = await entity_repository.find_all(use_load_options=False)
|
||||
nodes = [
|
||||
GraphNode(
|
||||
external_id=entity.external_id,
|
||||
title=entity.title,
|
||||
note_type=entity.note_type,
|
||||
file_path=entity.file_path,
|
||||
)
|
||||
for entity in entities
|
||||
]
|
||||
|
||||
# Fetch all resolved relations (to_id is not null) with eager-loaded entities
|
||||
relations = await relation_repository.find_all()
|
||||
edges = [
|
||||
GraphEdge(
|
||||
from_id=relation.from_entity.external_id,
|
||||
to_id=relation.to_entity.external_id,
|
||||
relation_type=relation.relation_type,
|
||||
)
|
||||
for relation in relations
|
||||
if relation.to_entity is not None
|
||||
]
|
||||
# Fetch all resolved relations (to_id is not null) with eager-loaded entities
|
||||
relations = await relation_repository.find_all()
|
||||
edges = [
|
||||
GraphEdge(
|
||||
from_id=relation.from_entity.external_id,
|
||||
to_id=relation.to_entity.external_id,
|
||||
relation_type=relation.relation_type,
|
||||
)
|
||||
for relation in relations
|
||||
if relation.to_entity is not None
|
||||
]
|
||||
|
||||
logger.info(f"API v2 response: graph with {len(nodes)} nodes and {len(edges)} edges")
|
||||
return GraphResponse(nodes=nodes, edges=edges)
|
||||
logger.info(f"API v2 response: graph with {len(nodes)} nodes and {len(edges)} edges")
|
||||
return GraphResponse(nodes=nodes, edges=edges)
|
||||
|
||||
|
||||
## Resolution endpoint
|
||||
@@ -143,7 +148,7 @@ async def resolve_identifier(
|
||||
"resolution_method": "permalink"
|
||||
}
|
||||
"""
|
||||
with telemetry.operation(
|
||||
with logfire.span(
|
||||
"api.request.knowledge.resolve_entity",
|
||||
entrypoint="api",
|
||||
domain="knowledge",
|
||||
@@ -151,25 +156,13 @@ async def resolve_identifier(
|
||||
):
|
||||
logger.info(f"API v2 request: resolve_identifier for '{data.identifier}'")
|
||||
|
||||
with telemetry.scope(
|
||||
"api.knowledge.resolve_entity.lookup_entity",
|
||||
domain="knowledge",
|
||||
action="resolve_entity",
|
||||
phase="lookup_entity",
|
||||
):
|
||||
entity = await entity_repository.get_by_external_id(data.identifier)
|
||||
entity = await entity_repository.get_by_external_id(data.identifier)
|
||||
resolution_method = "external_id" if entity else "search"
|
||||
|
||||
if not entity:
|
||||
with telemetry.scope(
|
||||
"api.knowledge.resolve_entity.resolve_link",
|
||||
domain="knowledge",
|
||||
action="resolve_entity",
|
||||
phase="resolve_link",
|
||||
):
|
||||
entity = await link_resolver.resolve_link(
|
||||
data.identifier, source_path=data.source_path, strict=data.strict
|
||||
)
|
||||
entity = await link_resolver.resolve_link(
|
||||
data.identifier, source_path=data.source_path, strict=data.strict
|
||||
)
|
||||
if entity:
|
||||
if entity.permalink == data.identifier:
|
||||
resolution_method = "permalink"
|
||||
@@ -183,20 +176,14 @@ async def resolve_identifier(
|
||||
if not entity:
|
||||
raise HTTPException(status_code=404, detail=f"Entity not found: '{data.identifier}'")
|
||||
|
||||
with telemetry.scope(
|
||||
"api.knowledge.resolve_entity.shape_response",
|
||||
domain="knowledge",
|
||||
action="resolve_entity",
|
||||
phase="shape_response",
|
||||
):
|
||||
result = EntityResolveResponse(
|
||||
external_id=entity.external_id,
|
||||
entity_id=entity.id,
|
||||
permalink=entity.permalink,
|
||||
file_path=entity.file_path,
|
||||
title=entity.title,
|
||||
resolution_method=resolution_method,
|
||||
)
|
||||
result = EntityResolveResponse(
|
||||
external_id=entity.external_id,
|
||||
entity_id=entity.id,
|
||||
permalink=entity.permalink,
|
||||
file_path=entity.file_path,
|
||||
title=entity.title,
|
||||
resolution_method=resolution_method,
|
||||
)
|
||||
|
||||
logger.debug(
|
||||
f"API v2 response: resolved '{data.identifier}' to external_id={result.external_id} via {resolution_method}"
|
||||
@@ -228,7 +215,7 @@ async def get_entity_by_id(
|
||||
Raises:
|
||||
HTTPException: 404 if entity not found
|
||||
"""
|
||||
with telemetry.operation(
|
||||
with logfire.span(
|
||||
"api.request.knowledge.get_entity",
|
||||
entrypoint="api",
|
||||
domain="knowledge",
|
||||
@@ -236,25 +223,13 @@ async def get_entity_by_id(
|
||||
):
|
||||
logger.info(f"API v2 request: get_entity_by_id entity_id={entity_id}")
|
||||
|
||||
with telemetry.scope(
|
||||
"api.knowledge.get_entity.load_entity",
|
||||
domain="knowledge",
|
||||
action="get_entity",
|
||||
phase="load_entity",
|
||||
):
|
||||
entity = await entity_repository.get_by_external_id(entity_id)
|
||||
entity = await entity_repository.get_by_external_id(entity_id)
|
||||
if not entity:
|
||||
raise HTTPException(
|
||||
status_code=404, detail=f"Entity with external_id '{entity_id}' not found"
|
||||
)
|
||||
|
||||
with telemetry.scope(
|
||||
"api.knowledge.get_entity.shape_response",
|
||||
domain="knowledge",
|
||||
action="get_entity",
|
||||
phase="shape_response",
|
||||
):
|
||||
result = EntityResponseV2.model_validate(entity)
|
||||
result = EntityResponseV2.model_validate(entity)
|
||||
logger.info(f"API v2 response: external_id={entity_id}, title='{result.title}'")
|
||||
|
||||
return result
|
||||
@@ -267,106 +242,44 @@ async def get_entity_by_id(
|
||||
async def create_entity(
|
||||
project_id: ProjectExternalIdPathDep,
|
||||
data: Entity,
|
||||
background_tasks: BackgroundTasks,
|
||||
entity_service: EntityServiceV2ExternalDep,
|
||||
search_service: SearchServiceV2ExternalDep,
|
||||
task_scheduler: TaskSchedulerDep,
|
||||
file_service: FileServiceV2ExternalDep,
|
||||
app_config: AppConfigDep,
|
||||
fast: bool = Query(
|
||||
True, description="If true, write quickly and defer indexing to background tasks."
|
||||
),
|
||||
) -> EntityResponseV2:
|
||||
"""Create a new entity.
|
||||
|
||||
Args:
|
||||
data: Entity data to create
|
||||
fast: If True, defer indexing to background tasks
|
||||
|
||||
Returns:
|
||||
Created entity with generated external_id (UUID) and file content
|
||||
"""
|
||||
with telemetry.operation(
|
||||
with logfire.span(
|
||||
"api.request.knowledge.create_entity",
|
||||
entrypoint="api",
|
||||
domain="knowledge",
|
||||
action="create_entity",
|
||||
fast=fast,
|
||||
):
|
||||
logger.info(
|
||||
"API v2 request", endpoint="create_entity", note_type=data.note_type, title=data.title
|
||||
)
|
||||
|
||||
with telemetry.scope(
|
||||
"api.knowledge.create_entity.write_entity",
|
||||
domain="knowledge",
|
||||
action="create_entity",
|
||||
phase="write_entity",
|
||||
fast=fast,
|
||||
):
|
||||
if fast:
|
||||
entity = await entity_service.fast_write_entity(data)
|
||||
written_content = None
|
||||
search_content = None
|
||||
else:
|
||||
write_result = await entity_service.create_entity_with_content(data)
|
||||
entity = write_result.entity
|
||||
written_content = write_result.content
|
||||
search_content = write_result.search_content
|
||||
|
||||
if fast:
|
||||
with telemetry.scope(
|
||||
"api.knowledge.create_entity.enqueue_reindex",
|
||||
domain="knowledge",
|
||||
action="create_entity",
|
||||
phase="enqueue_reindex",
|
||||
fast=fast,
|
||||
):
|
||||
task_scheduler.schedule(
|
||||
"reindex_entity",
|
||||
entity_id=entity.id,
|
||||
project_id=project_id,
|
||||
)
|
||||
else:
|
||||
with telemetry.scope(
|
||||
"api.knowledge.create_entity.search_index",
|
||||
domain="knowledge",
|
||||
action="create_entity",
|
||||
phase="search_index",
|
||||
):
|
||||
await search_service.index_entity(entity, content=search_content)
|
||||
with telemetry.scope(
|
||||
"api.knowledge.create_entity.vector_sync",
|
||||
domain="knowledge",
|
||||
action="create_entity",
|
||||
phase="vector_sync",
|
||||
):
|
||||
_schedule_vector_sync_if_enabled(
|
||||
task_scheduler=task_scheduler,
|
||||
app_config=app_config,
|
||||
entity_id=entity.id,
|
||||
project_id=project_id,
|
||||
)
|
||||
# Note writes are now internally consistent before the response returns. We only leave
|
||||
# truly derived work, like semantic vectors, on the async scheduler.
|
||||
write_result = await entity_service.create_entity_with_content(data)
|
||||
entity = write_result.entity
|
||||
await search_service.index_entity(entity, content=write_result.search_content)
|
||||
_schedule_vector_sync_if_enabled(
|
||||
task_scheduler=task_scheduler,
|
||||
app_config=app_config,
|
||||
entity_id=entity.id,
|
||||
project_id=project_id,
|
||||
)
|
||||
|
||||
result = EntityResponseV2.model_validate(entity)
|
||||
if fast:
|
||||
result = result.model_copy(update={"observations": [], "relations": []})
|
||||
|
||||
with telemetry.scope(
|
||||
"api.knowledge.create_entity.read_content",
|
||||
domain="knowledge",
|
||||
action="create_entity",
|
||||
phase="read_content",
|
||||
source="file" if fast else "memory",
|
||||
):
|
||||
if fast:
|
||||
content = await file_service.read_file_content(entity.file_path)
|
||||
else:
|
||||
# Non-fast writes already captured the markdown in memory. Reuse it here
|
||||
# instead of re-reading the file; format_on_save is the one config that can
|
||||
# still make the persisted file diverge because write_file only returns a checksum.
|
||||
content = written_content
|
||||
result = result.model_copy(update={"content": content})
|
||||
# The write service already returns the canonical markdown accepted for this request.
|
||||
result = result.model_copy(update={"content": write_result.content})
|
||||
|
||||
logger.info(
|
||||
f"API v2 response: endpoint='create_entity' external_id={entity.external_id}, title={result.title}, permalink={result.permalink}, status_code=201"
|
||||
@@ -381,18 +294,13 @@ async def create_entity(
|
||||
async def update_entity_by_id(
|
||||
data: Entity,
|
||||
response: Response,
|
||||
background_tasks: BackgroundTasks,
|
||||
project_id: ProjectExternalIdPathDep,
|
||||
entity_service: EntityServiceV2ExternalDep,
|
||||
search_service: SearchServiceV2ExternalDep,
|
||||
entity_repository: EntityRepositoryV2ExternalDep,
|
||||
task_scheduler: TaskSchedulerDep,
|
||||
file_service: FileServiceV2ExternalDep,
|
||||
app_config: AppConfigDep,
|
||||
entity_id: str = Path(..., description="Entity external ID (UUID)"),
|
||||
fast: bool = Query(
|
||||
True, description="If true, write quickly and defer indexing to background tasks."
|
||||
),
|
||||
) -> EntityResponseV2:
|
||||
"""Update an entity by external ID.
|
||||
|
||||
@@ -401,121 +309,52 @@ async def update_entity_by_id(
|
||||
Args:
|
||||
entity_id: External ID (UUID string)
|
||||
data: Updated entity data
|
||||
fast: If True, defer indexing to background tasks
|
||||
|
||||
Returns:
|
||||
Updated entity with file content
|
||||
"""
|
||||
with telemetry.operation(
|
||||
with logfire.span(
|
||||
"api.request.knowledge.update_entity",
|
||||
entrypoint="api",
|
||||
domain="knowledge",
|
||||
action="update_entity",
|
||||
fast=fast,
|
||||
):
|
||||
logger.info(f"API v2 request: update_entity_by_id entity_id={entity_id}")
|
||||
|
||||
with telemetry.scope(
|
||||
"api.knowledge.update_entity.load_entity",
|
||||
domain="knowledge",
|
||||
action="update_entity",
|
||||
phase="load_entity",
|
||||
):
|
||||
existing = await entity_repository.get_by_external_id(entity_id)
|
||||
existing = await entity_repository.get_by_external_id(entity_id)
|
||||
created = existing is None
|
||||
|
||||
with telemetry.scope(
|
||||
"api.knowledge.update_entity.write_entity",
|
||||
domain="knowledge",
|
||||
action="update_entity",
|
||||
phase="write_entity",
|
||||
fast=fast,
|
||||
):
|
||||
if fast:
|
||||
entity = await entity_service.fast_write_entity(data, external_id=entity_id)
|
||||
written_content = None
|
||||
search_content = None
|
||||
response.status_code = 200 if existing else 201
|
||||
else:
|
||||
if existing:
|
||||
write_result = await entity_service.update_entity_with_content(existing, data)
|
||||
entity = write_result.entity
|
||||
written_content = write_result.content
|
||||
search_content = write_result.search_content
|
||||
response.status_code = 200
|
||||
else:
|
||||
write_result = await entity_service.create_entity_with_content(data)
|
||||
entity = write_result.entity
|
||||
written_content = write_result.content
|
||||
search_content = write_result.search_content
|
||||
if entity.external_id != entity_id:
|
||||
entity = await entity_repository.update(
|
||||
entity.id,
|
||||
{"external_id": entity_id},
|
||||
)
|
||||
# external_id fixup only changes the DB row. The file content is unchanged,
|
||||
# so the markdown captured during the write remains valid downstream.
|
||||
if not entity:
|
||||
raise HTTPException(
|
||||
status_code=404,
|
||||
detail=f"Entity with external_id '{entity_id}' not found",
|
||||
)
|
||||
response.status_code = 201
|
||||
|
||||
if fast:
|
||||
with telemetry.scope(
|
||||
"api.knowledge.update_entity.enqueue_reindex",
|
||||
domain="knowledge",
|
||||
action="update_entity",
|
||||
phase="enqueue_reindex",
|
||||
fast=fast,
|
||||
):
|
||||
task_scheduler.schedule(
|
||||
"reindex_entity",
|
||||
entity_id=entity.id,
|
||||
project_id=project_id,
|
||||
resolve_relations=created,
|
||||
)
|
||||
if existing:
|
||||
write_result = await entity_service.update_entity_with_content(existing, data)
|
||||
entity = write_result.entity
|
||||
response.status_code = 200
|
||||
else:
|
||||
with telemetry.scope(
|
||||
"api.knowledge.update_entity.search_index",
|
||||
domain="knowledge",
|
||||
action="update_entity",
|
||||
phase="search_index",
|
||||
):
|
||||
await search_service.index_entity(entity, content=search_content)
|
||||
with telemetry.scope(
|
||||
"api.knowledge.update_entity.vector_sync",
|
||||
domain="knowledge",
|
||||
action="update_entity",
|
||||
phase="vector_sync",
|
||||
):
|
||||
_schedule_vector_sync_if_enabled(
|
||||
task_scheduler=task_scheduler,
|
||||
app_config=app_config,
|
||||
entity_id=entity.id,
|
||||
project_id=project_id,
|
||||
write_result = await entity_service.create_entity_with_content(data)
|
||||
entity = write_result.entity
|
||||
if entity.external_id != entity_id:
|
||||
entity = await entity_repository.update(
|
||||
entity.id,
|
||||
{"external_id": entity_id},
|
||||
)
|
||||
# external_id fixup only changes the DB row. The file content is unchanged,
|
||||
# so the markdown captured during the write remains valid downstream.
|
||||
if not entity:
|
||||
raise HTTPException(
|
||||
status_code=404,
|
||||
detail=f"Entity with external_id '{entity_id}' not found",
|
||||
)
|
||||
response.status_code = 201
|
||||
|
||||
await search_service.index_entity(entity, content=write_result.search_content)
|
||||
_schedule_vector_sync_if_enabled(
|
||||
task_scheduler=task_scheduler,
|
||||
app_config=app_config,
|
||||
entity_id=entity.id,
|
||||
project_id=project_id,
|
||||
)
|
||||
|
||||
result = EntityResponseV2.model_validate(entity)
|
||||
if fast:
|
||||
result = result.model_copy(update={"observations": [], "relations": []})
|
||||
|
||||
with telemetry.scope(
|
||||
"api.knowledge.update_entity.read_content",
|
||||
domain="knowledge",
|
||||
action="update_entity",
|
||||
phase="read_content",
|
||||
source="file" if fast else "memory",
|
||||
):
|
||||
if fast:
|
||||
content = await file_service.read_file_content(entity.file_path)
|
||||
else:
|
||||
# Non-fast writes already captured the markdown in memory. Reuse it here
|
||||
# instead of re-reading the file; format_on_save is the one config that can
|
||||
# still make the persisted file diverge because write_file only returns a checksum.
|
||||
content = written_content
|
||||
result = result.model_copy(update={"content": content})
|
||||
result = result.model_copy(update={"content": write_result.content})
|
||||
|
||||
logger.info(
|
||||
f"API v2 response: external_id={entity_id}, created={created}, status_code={response.status_code}"
|
||||
@@ -526,25 +365,19 @@ async def update_entity_by_id(
|
||||
@router.patch("/entities/{entity_id}", response_model=EntityResponseV2)
|
||||
async def edit_entity_by_id(
|
||||
data: EditEntityRequest,
|
||||
background_tasks: BackgroundTasks,
|
||||
project_id: ProjectExternalIdPathDep,
|
||||
entity_service: EntityServiceV2ExternalDep,
|
||||
search_service: SearchServiceV2ExternalDep,
|
||||
entity_repository: EntityRepositoryV2ExternalDep,
|
||||
task_scheduler: TaskSchedulerDep,
|
||||
file_service: FileServiceV2ExternalDep,
|
||||
app_config: AppConfigDep,
|
||||
entity_id: str = Path(..., description="Entity external ID (UUID)"),
|
||||
fast: bool = Query(
|
||||
True, description="If true, write quickly and defer indexing to background tasks."
|
||||
),
|
||||
) -> EntityResponseV2:
|
||||
"""Edit an existing entity by external ID using operations like append, prepend, etc.
|
||||
|
||||
Args:
|
||||
entity_id: External ID (UUID string)
|
||||
data: Edit operation details
|
||||
fast: If True, defer indexing to background tasks
|
||||
|
||||
Returns:
|
||||
Updated entity with file content
|
||||
@@ -552,115 +385,43 @@ async def edit_entity_by_id(
|
||||
Raises:
|
||||
HTTPException: 404 if entity not found, 400 if edit fails
|
||||
"""
|
||||
with telemetry.operation(
|
||||
with logfire.span(
|
||||
"api.request.knowledge.edit_entity",
|
||||
entrypoint="api",
|
||||
domain="knowledge",
|
||||
action="edit_entity",
|
||||
fast=fast,
|
||||
):
|
||||
logger.info(
|
||||
f"API v2 request: edit_entity_by_id entity_id={entity_id}, operation='{data.operation}'"
|
||||
)
|
||||
|
||||
with telemetry.scope(
|
||||
"api.knowledge.edit_entity.load_entity",
|
||||
domain="knowledge",
|
||||
action="edit_entity",
|
||||
phase="load_entity",
|
||||
):
|
||||
entity = await entity_repository.get_by_external_id(entity_id)
|
||||
entity = await entity_repository.get_by_external_id(entity_id)
|
||||
if not entity: # pragma: no cover
|
||||
raise HTTPException(
|
||||
status_code=404, detail=f"Entity with external_id '{entity_id}' not found"
|
||||
)
|
||||
|
||||
try:
|
||||
with telemetry.scope(
|
||||
"api.knowledge.edit_entity.write_entity",
|
||||
domain="knowledge",
|
||||
action="edit_entity",
|
||||
phase="write_entity",
|
||||
fast=fast,
|
||||
):
|
||||
if fast:
|
||||
updated_entity = await entity_service.fast_edit_entity(
|
||||
entity=entity,
|
||||
operation=data.operation,
|
||||
content=data.content,
|
||||
section=data.section,
|
||||
find_text=data.find_text,
|
||||
expected_replacements=data.expected_replacements,
|
||||
)
|
||||
written_content = None
|
||||
search_content = None
|
||||
else:
|
||||
identifier = entity.permalink or entity.file_path
|
||||
write_result = await entity_service.edit_entity_with_content(
|
||||
identifier=identifier,
|
||||
operation=data.operation,
|
||||
content=data.content,
|
||||
section=data.section,
|
||||
find_text=data.find_text,
|
||||
expected_replacements=data.expected_replacements,
|
||||
)
|
||||
updated_entity = write_result.entity
|
||||
written_content = write_result.content
|
||||
search_content = write_result.search_content
|
||||
|
||||
if fast:
|
||||
with telemetry.scope(
|
||||
"api.knowledge.edit_entity.enqueue_reindex",
|
||||
domain="knowledge",
|
||||
action="edit_entity",
|
||||
phase="enqueue_reindex",
|
||||
fast=fast,
|
||||
):
|
||||
task_scheduler.schedule(
|
||||
"reindex_entity",
|
||||
entity_id=updated_entity.id,
|
||||
project_id=project_id,
|
||||
)
|
||||
else:
|
||||
with telemetry.scope(
|
||||
"api.knowledge.edit_entity.search_index",
|
||||
domain="knowledge",
|
||||
action="edit_entity",
|
||||
phase="search_index",
|
||||
):
|
||||
await search_service.index_entity(updated_entity, content=search_content)
|
||||
with telemetry.scope(
|
||||
"api.knowledge.edit_entity.vector_sync",
|
||||
domain="knowledge",
|
||||
action="edit_entity",
|
||||
phase="vector_sync",
|
||||
):
|
||||
_schedule_vector_sync_if_enabled(
|
||||
task_scheduler=task_scheduler,
|
||||
app_config=app_config,
|
||||
entity_id=updated_entity.id,
|
||||
project_id=project_id,
|
||||
)
|
||||
identifier = entity.permalink or entity.file_path
|
||||
write_result = await entity_service.edit_entity_with_content(
|
||||
identifier=identifier,
|
||||
operation=data.operation,
|
||||
content=data.content,
|
||||
section=data.section,
|
||||
find_text=data.find_text,
|
||||
expected_replacements=data.expected_replacements,
|
||||
)
|
||||
updated_entity = write_result.entity
|
||||
await search_service.index_entity(updated_entity, content=write_result.search_content)
|
||||
_schedule_vector_sync_if_enabled(
|
||||
task_scheduler=task_scheduler,
|
||||
app_config=app_config,
|
||||
entity_id=updated_entity.id,
|
||||
project_id=project_id,
|
||||
)
|
||||
|
||||
result = EntityResponseV2.model_validate(updated_entity)
|
||||
if fast:
|
||||
result = result.model_copy(update={"observations": [], "relations": []})
|
||||
|
||||
with telemetry.scope(
|
||||
"api.knowledge.edit_entity.read_content",
|
||||
domain="knowledge",
|
||||
action="edit_entity",
|
||||
phase="read_content",
|
||||
source="file" if fast else "memory",
|
||||
):
|
||||
if fast:
|
||||
content = await file_service.read_file_content(updated_entity.file_path)
|
||||
else:
|
||||
# Non-fast writes already captured the markdown in memory. Reuse it here
|
||||
# instead of re-reading the file; format_on_save is the one config that can
|
||||
# still make the persisted file diverge because write_file only returns a checksum.
|
||||
content = written_content
|
||||
result = result.model_copy(update={"content": content})
|
||||
result = result.model_copy(update={"content": write_result.content})
|
||||
|
||||
logger.info(
|
||||
f"API v2 response: external_id={entity_id}, operation='{data.operation}', status_code=200"
|
||||
@@ -678,12 +439,10 @@ async def edit_entity_by_id(
|
||||
|
||||
@router.delete("/entities/{entity_id}", response_model=DeleteEntitiesResponse)
|
||||
async def delete_entity_by_id(
|
||||
background_tasks: BackgroundTasks,
|
||||
project_id: ProjectExternalIdPathDep,
|
||||
entity_service: EntityServiceV2ExternalDep,
|
||||
entity_repository: EntityRepositoryV2ExternalDep,
|
||||
entity_id: str = Path(..., description="Entity external ID (UUID)"),
|
||||
search_service=Depends(lambda: None), # Optional for now
|
||||
) -> DeleteEntitiesResponse:
|
||||
"""Delete an entity by external ID.
|
||||
|
||||
@@ -695,23 +454,25 @@ async def delete_entity_by_id(
|
||||
|
||||
Note: Returns deleted=False if entity doesn't exist (idempotent)
|
||||
"""
|
||||
logger.info(f"API v2 request: delete_entity_by_id entity_id={entity_id}")
|
||||
with logfire.span(
|
||||
"api.request.knowledge.delete_entity",
|
||||
entrypoint="api",
|
||||
domain="knowledge",
|
||||
action="delete_entity",
|
||||
):
|
||||
logger.info(f"API v2 request: delete_entity_by_id entity_id={entity_id}")
|
||||
|
||||
entity = await entity_repository.get_by_external_id(entity_id)
|
||||
if entity is None:
|
||||
logger.info(f"API v2 response: external_id={entity_id} not found, deleted=False")
|
||||
return DeleteEntitiesResponse(deleted=False)
|
||||
entity = await entity_repository.get_by_external_id(entity_id)
|
||||
if entity is None:
|
||||
logger.info(f"API v2 response: external_id={entity_id} not found, deleted=False")
|
||||
return DeleteEntitiesResponse(deleted=False)
|
||||
|
||||
# Delete the entity using internal ID
|
||||
deleted = await entity_service.delete_entity(entity.id)
|
||||
# Delete the entity using internal ID
|
||||
deleted = await entity_service.delete_entity(entity.id)
|
||||
|
||||
# Remove from search index if search service available
|
||||
if search_service:
|
||||
background_tasks.add_task(search_service.handle_delete, entity) # pragma: no cover
|
||||
logger.info(f"API v2 response: external_id={entity_id}, deleted={deleted}")
|
||||
|
||||
logger.info(f"API v2 response: external_id={entity_id}, deleted={deleted}")
|
||||
|
||||
return DeleteEntitiesResponse(deleted=deleted)
|
||||
return DeleteEntitiesResponse(deleted=deleted)
|
||||
|
||||
|
||||
## Move endpoint
|
||||
@@ -720,7 +481,6 @@ async def delete_entity_by_id(
|
||||
@router.put("/entities/{entity_id}/move", response_model=EntityResponseV2)
|
||||
async def move_entity(
|
||||
data: MoveEntityRequestV2,
|
||||
background_tasks: BackgroundTasks,
|
||||
project_id: ProjectExternalIdPathDep,
|
||||
entity_service: EntityServiceV2ExternalDep,
|
||||
entity_repository: EntityRepositoryV2ExternalDep,
|
||||
@@ -743,48 +503,58 @@ async def move_entity(
|
||||
Returns:
|
||||
Updated entity with new file path
|
||||
"""
|
||||
logger.info(
|
||||
f"API v2 request: move_entity entity_id={entity_id}, destination='{data.destination_path}'"
|
||||
)
|
||||
|
||||
try:
|
||||
# First, get the entity by external_id to verify it exists
|
||||
entity = await entity_repository.get_by_external_id(entity_id)
|
||||
if not entity: # pragma: no cover
|
||||
raise HTTPException(
|
||||
status_code=404, detail=f"Entity with external_id '{entity_id}' not found"
|
||||
)
|
||||
|
||||
# Move the entity using its current file path as identifier
|
||||
moved_entity = await entity_service.move_entity(
|
||||
identifier=entity.file_path, # Use file path for resolution
|
||||
destination_path=data.destination_path,
|
||||
project_config=project_config,
|
||||
app_config=app_config,
|
||||
with logfire.span(
|
||||
"api.request.knowledge.move_entity",
|
||||
entrypoint="api",
|
||||
domain="knowledge",
|
||||
action="move_entity",
|
||||
):
|
||||
logger.info(
|
||||
f"API v2 request: move_entity entity_id={entity_id}, destination='{data.destination_path}'"
|
||||
)
|
||||
|
||||
# Reindex at new location
|
||||
reindexed_entity = await entity_service.link_resolver.resolve_link(data.destination_path)
|
||||
if reindexed_entity:
|
||||
await search_service.index_entity(reindexed_entity)
|
||||
_schedule_vector_sync_if_enabled(
|
||||
task_scheduler=task_scheduler,
|
||||
try:
|
||||
# First, get the entity by external_id to verify it exists
|
||||
entity = await entity_repository.get_by_external_id(entity_id)
|
||||
if not entity: # pragma: no cover
|
||||
raise HTTPException(
|
||||
status_code=404, detail=f"Entity with external_id '{entity_id}' not found"
|
||||
)
|
||||
|
||||
# Move the entity using its current file path as identifier
|
||||
moved_entity = await entity_service.move_entity(
|
||||
identifier=entity.file_path, # Use file path for resolution
|
||||
destination_path=data.destination_path,
|
||||
project_config=project_config,
|
||||
app_config=app_config,
|
||||
entity_id=reindexed_entity.id,
|
||||
project_id=project_id,
|
||||
)
|
||||
|
||||
result = EntityResponseV2.model_validate(moved_entity)
|
||||
# Reindex at new location
|
||||
reindexed_entity = await entity_service.link_resolver.resolve_link(
|
||||
data.destination_path
|
||||
)
|
||||
if reindexed_entity:
|
||||
await search_service.index_entity(reindexed_entity)
|
||||
_schedule_vector_sync_if_enabled(
|
||||
task_scheduler=task_scheduler,
|
||||
app_config=app_config,
|
||||
entity_id=reindexed_entity.id,
|
||||
project_id=project_id,
|
||||
)
|
||||
|
||||
logger.info(f"API v2 response: moved external_id={entity_id} to '{data.destination_path}'")
|
||||
result = EntityResponseV2.model_validate(moved_entity)
|
||||
|
||||
return result
|
||||
logger.info(
|
||||
f"API v2 response: moved external_id={entity_id} to '{data.destination_path}'"
|
||||
)
|
||||
|
||||
except HTTPException: # pragma: no cover
|
||||
raise # pragma: no cover
|
||||
except Exception as e:
|
||||
logger.error(f"Error moving entity: {e}")
|
||||
raise HTTPException(status_code=400, detail=str(e))
|
||||
return result
|
||||
|
||||
except HTTPException: # pragma: no cover
|
||||
raise # pragma: no cover
|
||||
except Exception as e:
|
||||
logger.error(f"Error moving entity: {e}")
|
||||
raise HTTPException(status_code=400, detail=str(e))
|
||||
|
||||
|
||||
## Move directory endpoint
|
||||
@@ -793,7 +563,6 @@ async def move_entity(
|
||||
@router.post("/move-directory", response_model=DirectoryMoveResult)
|
||||
async def move_directory(
|
||||
data: MoveDirectoryRequestV2,
|
||||
background_tasks: BackgroundTasks,
|
||||
project_id: ProjectExternalIdPathDep,
|
||||
entity_service: EntityServiceV2ExternalDep,
|
||||
project_config: ProjectConfigV2ExternalDep,
|
||||
@@ -814,40 +583,46 @@ async def move_directory(
|
||||
Returns:
|
||||
DirectoryMoveResult with counts and details of moved files
|
||||
"""
|
||||
logger.info(
|
||||
f"API v2 request: move_directory source='{data.source_directory}', destination='{data.destination_directory}'"
|
||||
)
|
||||
|
||||
try:
|
||||
# Move the directory using the service
|
||||
result = await entity_service.move_directory(
|
||||
source_directory=data.source_directory,
|
||||
destination_directory=data.destination_directory,
|
||||
project_config=project_config,
|
||||
app_config=app_config,
|
||||
)
|
||||
|
||||
# Reindex moved entities
|
||||
for file_path in result.moved_files:
|
||||
entity = await entity_service.link_resolver.resolve_link(file_path)
|
||||
if entity:
|
||||
await search_service.index_entity(entity)
|
||||
_schedule_vector_sync_if_enabled(
|
||||
task_scheduler=task_scheduler,
|
||||
app_config=app_config,
|
||||
entity_id=entity.id,
|
||||
project_id=project_id,
|
||||
)
|
||||
|
||||
with logfire.span(
|
||||
"api.request.knowledge.move_directory",
|
||||
entrypoint="api",
|
||||
domain="knowledge",
|
||||
action="move_directory",
|
||||
):
|
||||
logger.info(
|
||||
f"API v2 response: move_directory "
|
||||
f"total={result.total_files}, success={result.successful_moves}, failed={result.failed_moves}"
|
||||
f"API v2 request: move_directory source='{data.source_directory}', destination='{data.destination_directory}'"
|
||||
)
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error moving directory: {e}")
|
||||
raise HTTPException(status_code=400, detail=str(e))
|
||||
try:
|
||||
# Move the directory using the service
|
||||
result = await entity_service.move_directory(
|
||||
source_directory=data.source_directory,
|
||||
destination_directory=data.destination_directory,
|
||||
project_config=project_config,
|
||||
app_config=app_config,
|
||||
)
|
||||
|
||||
# Reindex moved entities
|
||||
for file_path in result.moved_files:
|
||||
entity = await entity_service.link_resolver.resolve_link(file_path)
|
||||
if entity:
|
||||
await search_service.index_entity(entity)
|
||||
_schedule_vector_sync_if_enabled(
|
||||
task_scheduler=task_scheduler,
|
||||
app_config=app_config,
|
||||
entity_id=entity.id,
|
||||
project_id=project_id,
|
||||
)
|
||||
|
||||
logger.info(
|
||||
f"API v2 response: move_directory "
|
||||
f"total={result.total_files}, success={result.successful_moves}, failed={result.failed_moves}"
|
||||
)
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error moving directory: {e}")
|
||||
raise HTTPException(status_code=400, detail=str(e))
|
||||
|
||||
|
||||
## Delete directory endpoint
|
||||
@@ -872,20 +647,26 @@ async def delete_directory(
|
||||
Returns:
|
||||
DirectoryDeleteResult with counts and details of deleted files
|
||||
"""
|
||||
logger.info(f"API v2 request: delete_directory directory='{data.directory}'")
|
||||
with logfire.span(
|
||||
"api.request.knowledge.delete_directory",
|
||||
entrypoint="api",
|
||||
domain="knowledge",
|
||||
action="delete_directory",
|
||||
):
|
||||
logger.info(f"API v2 request: delete_directory directory='{data.directory}'")
|
||||
|
||||
try:
|
||||
# Delete the directory using the service
|
||||
result = await entity_service.delete_directory(
|
||||
directory=data.directory,
|
||||
)
|
||||
try:
|
||||
# Delete the directory using the service
|
||||
result = await entity_service.delete_directory(
|
||||
directory=data.directory,
|
||||
)
|
||||
|
||||
logger.info(
|
||||
f"API v2 response: delete_directory "
|
||||
f"total={result.total_files}, success={result.successful_deletes}, failed={result.failed_deletes}"
|
||||
)
|
||||
return result
|
||||
logger.info(
|
||||
f"API v2 response: delete_directory "
|
||||
f"total={result.total_files}, success={result.successful_deletes}, failed={result.failed_deletes}"
|
||||
)
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error deleting directory: {e}")
|
||||
raise HTTPException(status_code=400, detail=str(e))
|
||||
except Exception as e:
|
||||
logger.error(f"Error deleting directory: {e}")
|
||||
raise HTTPException(status_code=400, detail=str(e))
|
||||
|
||||
@@ -9,7 +9,7 @@ from typing import Annotated, Optional
|
||||
from fastapi import APIRouter, Query, Path
|
||||
from loguru import logger
|
||||
|
||||
from basic_memory import telemetry
|
||||
import logfire
|
||||
from basic_memory.deps import ContextServiceV2ExternalDep, EntityRepositoryV2ExternalDep
|
||||
from basic_memory.schemas.base import TimeFrame, parse_timeframe
|
||||
from basic_memory.schemas.memory import (
|
||||
@@ -51,7 +51,7 @@ async def recent(
|
||||
Returns:
|
||||
GraphContext with recent activity and related entities
|
||||
"""
|
||||
with telemetry.operation(
|
||||
with logfire.span(
|
||||
"api.request.memory.recent_activity",
|
||||
entrypoint="api",
|
||||
domain="memory",
|
||||
@@ -72,7 +72,7 @@ async def recent(
|
||||
limit = page_size
|
||||
offset = (page - 1) * page_size
|
||||
|
||||
with telemetry.scope(
|
||||
with logfire.span(
|
||||
"api.memory.recent_activity.build_context",
|
||||
domain="memory",
|
||||
action="recent_activity",
|
||||
@@ -88,7 +88,7 @@ async def recent(
|
||||
offset=offset,
|
||||
max_related=max_related,
|
||||
)
|
||||
with telemetry.scope(
|
||||
with logfire.span(
|
||||
"api.memory.recent_activity.shape_response",
|
||||
domain="memory",
|
||||
action="recent_activity",
|
||||
@@ -137,7 +137,7 @@ async def get_memory_context(
|
||||
Returns:
|
||||
GraphContext with the entity and its related context
|
||||
"""
|
||||
with telemetry.operation(
|
||||
with logfire.span(
|
||||
"api.request.memory.build_context",
|
||||
entrypoint="api",
|
||||
domain="memory",
|
||||
@@ -154,7 +154,7 @@ async def get_memory_context(
|
||||
limit = page_size
|
||||
offset = (page - 1) * page_size
|
||||
|
||||
with telemetry.scope(
|
||||
with logfire.span(
|
||||
"api.memory.build_context.build_context",
|
||||
domain="memory",
|
||||
action="build_context",
|
||||
@@ -170,7 +170,7 @@ async def get_memory_context(
|
||||
offset=offset,
|
||||
max_related=max_related,
|
||||
)
|
||||
with telemetry.scope(
|
||||
with logfire.span(
|
||||
"api.memory.build_context.shape_response",
|
||||
domain="memory",
|
||||
action="build_context",
|
||||
|
||||
@@ -6,6 +6,7 @@ have entity IDs in URLs - they generate formatted prompts from queries.
|
||||
"""
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
from fastapi import APIRouter, HTTPException, status, Path
|
||||
from loguru import logger
|
||||
|
||||
@@ -59,6 +60,7 @@ async def continue_conversation(
|
||||
|
||||
# Initialize search results
|
||||
search_results = []
|
||||
hierarchical_results_for_count = []
|
||||
|
||||
# Get data needed for template
|
||||
if request.topic:
|
||||
@@ -91,7 +93,8 @@ async def continue_conversation(
|
||||
# Limit to a reasonable number of total results
|
||||
all_hierarchical_results = all_hierarchical_results[:10]
|
||||
|
||||
template_context = {
|
||||
hierarchical_results_for_count = all_hierarchical_results
|
||||
template_context: dict[str, Any] = {
|
||||
"topic": request.topic,
|
||||
"timeframe": request.timeframe,
|
||||
"hierarchical_results": all_hierarchical_results,
|
||||
@@ -110,6 +113,7 @@ async def continue_conversation(
|
||||
|
||||
hierarchical_results = recent_context.results[:5] # Limit to top 5 recent items
|
||||
|
||||
hierarchical_results_for_count = hierarchical_results
|
||||
template_context = {
|
||||
"topic": f"Recent Activity from ({request.timeframe})",
|
||||
"timeframe": request.timeframe,
|
||||
@@ -129,9 +133,6 @@ async def continue_conversation(
|
||||
relation_count = 0
|
||||
entity_count = 0
|
||||
|
||||
# Get the hierarchical results from the template context
|
||||
hierarchical_results_for_count = template_context.get("hierarchical_results", [])
|
||||
|
||||
# For topic-based search
|
||||
if request.topic:
|
||||
for item in hierarchical_results_for_count:
|
||||
@@ -159,29 +160,24 @@ async def continue_conversation(
|
||||
elif related.type == "entity": # pragma: no cover
|
||||
entity_count += 1 # pragma: no cover
|
||||
|
||||
# Build metadata
|
||||
metadata = {
|
||||
"query": request.topic,
|
||||
"timeframe": request.timeframe,
|
||||
"search_count": len(search_results)
|
||||
if request.topic
|
||||
else 0, # Original search results count
|
||||
"context_count": len(hierarchical_results_for_count),
|
||||
"observation_count": observation_count,
|
||||
"relation_count": relation_count,
|
||||
"total_items": (
|
||||
prompt_metadata = PromptMetadata(
|
||||
query=request.topic,
|
||||
timeframe=request.timeframe,
|
||||
search_count=len(search_results) if request.topic else 0,
|
||||
context_count=len(hierarchical_results_for_count),
|
||||
observation_count=observation_count,
|
||||
relation_count=relation_count,
|
||||
total_items=(
|
||||
len(hierarchical_results_for_count)
|
||||
+ observation_count
|
||||
+ relation_count
|
||||
+ entity_count
|
||||
),
|
||||
"search_limit": request.search_items_limit,
|
||||
"context_depth": request.depth,
|
||||
"related_limit": request.related_items_limit,
|
||||
"generated_at": datetime.now(timezone.utc).isoformat(),
|
||||
}
|
||||
|
||||
prompt_metadata = PromptMetadata(**metadata)
|
||||
search_limit=request.search_items_limit,
|
||||
context_depth=request.depth,
|
||||
related_limit=request.related_items_limit,
|
||||
generated_at=datetime.now(timezone.utc).isoformat(),
|
||||
)
|
||||
|
||||
return PromptResponse(
|
||||
prompt=rendered_prompt, context=template_context, metadata=prompt_metadata
|
||||
@@ -229,7 +225,7 @@ async def search_prompt(
|
||||
results = await search_service.search(query, limit=limit, offset=offset)
|
||||
search_results = await to_search_results(entity_service, results)
|
||||
|
||||
template_context = {
|
||||
template_context: dict[str, Any] = {
|
||||
"query": request.query,
|
||||
"timeframe": request.timeframe,
|
||||
"results": search_results,
|
||||
@@ -241,22 +237,19 @@ async def search_prompt(
|
||||
# Render template
|
||||
rendered_prompt = await template_loader.render("prompts/search.hbs", template_context)
|
||||
|
||||
# Build metadata
|
||||
metadata = {
|
||||
"query": request.query,
|
||||
"timeframe": request.timeframe,
|
||||
"search_count": len(search_results),
|
||||
"context_count": len(search_results),
|
||||
"observation_count": 0, # Search results don't include observations
|
||||
"relation_count": 0, # Search results don't include relations
|
||||
"total_items": len(search_results),
|
||||
"search_limit": limit,
|
||||
"context_depth": 0, # No context depth for basic search
|
||||
"related_limit": 0, # No related items for basic search
|
||||
"generated_at": datetime.now(timezone.utc).isoformat(),
|
||||
}
|
||||
|
||||
prompt_metadata = PromptMetadata(**metadata)
|
||||
prompt_metadata = PromptMetadata(
|
||||
query=request.query,
|
||||
timeframe=request.timeframe,
|
||||
search_count=len(search_results),
|
||||
context_count=len(search_results),
|
||||
observation_count=0,
|
||||
relation_count=0,
|
||||
total_items=len(search_results),
|
||||
search_limit=limit,
|
||||
context_depth=0,
|
||||
related_limit=0,
|
||||
generated_at=datetime.now(timezone.utc).isoformat(),
|
||||
)
|
||||
|
||||
return PromptResponse(
|
||||
prompt=rendered_prompt, context=template_context, metadata=prompt_metadata
|
||||
|
||||
@@ -15,7 +15,7 @@ from pathlib import Path as PathLib
|
||||
from fastapi import APIRouter, HTTPException, Response, Path
|
||||
from loguru import logger
|
||||
|
||||
from basic_memory import telemetry
|
||||
import logfire
|
||||
from basic_memory.deps import (
|
||||
ProjectConfigV2ExternalDep,
|
||||
FileServiceV2ExternalDep,
|
||||
@@ -56,7 +56,7 @@ async def get_resource_content(
|
||||
Raises:
|
||||
HTTPException: 404 if entity or file not found
|
||||
"""
|
||||
with telemetry.operation(
|
||||
with logfire.span(
|
||||
"api.request.resource.get_content",
|
||||
entrypoint="api",
|
||||
domain="resource",
|
||||
@@ -64,7 +64,7 @@ async def get_resource_content(
|
||||
):
|
||||
logger.debug(f"V2 Getting content for project {project_id}, entity_id: {entity_id}")
|
||||
|
||||
with telemetry.scope(
|
||||
with logfire.span(
|
||||
"api.resource.get_content.load_entity",
|
||||
domain="resource",
|
||||
action="get_content",
|
||||
@@ -74,7 +74,7 @@ async def get_resource_content(
|
||||
if not entity:
|
||||
raise HTTPException(status_code=404, detail=f"Entity {entity_id} not found")
|
||||
|
||||
with telemetry.scope(
|
||||
with logfire.span(
|
||||
"api.resource.get_content.validate_path",
|
||||
domain="resource",
|
||||
action="get_content",
|
||||
@@ -90,7 +90,7 @@ async def get_resource_content(
|
||||
detail="Entity contains invalid file path",
|
||||
)
|
||||
|
||||
with telemetry.scope(
|
||||
with logfire.span(
|
||||
"api.resource.get_content.ensure_exists",
|
||||
domain="resource",
|
||||
action="get_content",
|
||||
@@ -102,7 +102,7 @@ async def get_resource_content(
|
||||
detail=f"File not found: {entity.file_path}",
|
||||
)
|
||||
|
||||
with telemetry.scope(
|
||||
with logfire.span(
|
||||
"api.resource.get_content.read_content",
|
||||
domain="resource",
|
||||
action="get_content",
|
||||
@@ -139,7 +139,7 @@ async def create_resource(
|
||||
Raises:
|
||||
HTTPException: 400 for invalid file paths, 409 if file already exists
|
||||
"""
|
||||
with telemetry.operation(
|
||||
with logfire.span(
|
||||
"api.request.resource.create",
|
||||
entrypoint="api",
|
||||
domain="resource",
|
||||
@@ -166,7 +166,7 @@ async def create_resource(
|
||||
f"Use PUT /resource/{existing_entity.external_id} to update it.",
|
||||
)
|
||||
|
||||
with telemetry.scope(
|
||||
with logfire.span(
|
||||
"api.resource.create.write_file",
|
||||
domain="resource",
|
||||
action="create",
|
||||
@@ -175,7 +175,7 @@ async def create_resource(
|
||||
await file_service.ensure_directory(PathLib(data.file_path).parent)
|
||||
checksum = await file_service.write_file(data.file_path, data.content)
|
||||
|
||||
with telemetry.scope(
|
||||
with logfire.span(
|
||||
"api.resource.create.read_metadata",
|
||||
domain="resource",
|
||||
action="create",
|
||||
@@ -197,7 +197,7 @@ async def create_resource(
|
||||
created_at=file_metadata.created_at,
|
||||
updated_at=file_metadata.modified_at,
|
||||
)
|
||||
with telemetry.scope(
|
||||
with logfire.span(
|
||||
"api.resource.create.upsert_entity",
|
||||
domain="resource",
|
||||
action="create",
|
||||
@@ -205,13 +205,13 @@ async def create_resource(
|
||||
):
|
||||
entity = await entity_repository.add(entity)
|
||||
|
||||
with telemetry.scope(
|
||||
with logfire.span(
|
||||
"api.resource.create.search_index",
|
||||
domain="resource",
|
||||
action="create",
|
||||
phase="search_index",
|
||||
):
|
||||
await search_service.index_entity(entity) # pyright: ignore
|
||||
await search_service.index_entity(entity)
|
||||
|
||||
return ResourceResponse(
|
||||
entity_id=entity.id,
|
||||
@@ -258,7 +258,7 @@ async def update_resource(
|
||||
Raises:
|
||||
HTTPException: 404 if entity not found, 400 for invalid paths
|
||||
"""
|
||||
with telemetry.operation(
|
||||
with logfire.span(
|
||||
"api.request.resource.update",
|
||||
entrypoint="api",
|
||||
domain="resource",
|
||||
@@ -282,7 +282,7 @@ async def update_resource(
|
||||
"Path must be relative and stay within project boundaries.",
|
||||
)
|
||||
|
||||
with telemetry.scope(
|
||||
with logfire.span(
|
||||
"api.resource.update.write_file",
|
||||
domain="resource",
|
||||
action="update",
|
||||
@@ -297,7 +297,7 @@ async def update_resource(
|
||||
|
||||
checksum = await file_service.write_file(target_file_path, data.content)
|
||||
|
||||
with telemetry.scope(
|
||||
with logfire.span(
|
||||
"api.resource.update.read_metadata",
|
||||
domain="resource",
|
||||
action="update",
|
||||
@@ -309,7 +309,7 @@ async def update_resource(
|
||||
content_type = file_service.content_type(target_file_path)
|
||||
note_type = "canvas" if target_file_path.endswith(".canvas") else "file"
|
||||
|
||||
with telemetry.scope(
|
||||
with logfire.span(
|
||||
"api.resource.update.update_entity",
|
||||
domain="resource",
|
||||
action="update",
|
||||
@@ -326,14 +326,16 @@ async def update_resource(
|
||||
"updated_at": file_metadata.modified_at,
|
||||
},
|
||||
)
|
||||
if updated_entity is None:
|
||||
raise HTTPException(status_code=404, detail=f"Entity {entity_id} not found")
|
||||
|
||||
with telemetry.scope(
|
||||
with logfire.span(
|
||||
"api.resource.update.search_index",
|
||||
domain="resource",
|
||||
action="update",
|
||||
phase="search_index",
|
||||
):
|
||||
await search_service.index_entity(updated_entity) # pyright: ignore
|
||||
await search_service.index_entity(updated_entity)
|
||||
|
||||
return ResourceResponse(
|
||||
entity_id=entity.id,
|
||||
|
||||
@@ -4,15 +4,17 @@ This router uses external_id UUIDs for stable, API-friendly routing.
|
||||
V1 uses string-based project names which are less efficient and less stable.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Path
|
||||
|
||||
from basic_memory import telemetry
|
||||
import logfire
|
||||
from basic_memory.api.v2.utils import to_search_results
|
||||
from basic_memory.repository.semantic_errors import (
|
||||
SemanticDependenciesMissingError,
|
||||
SemanticSearchDisabledError,
|
||||
)
|
||||
from basic_memory.schemas.search import SearchQuery, SearchResponse
|
||||
from basic_memory.schemas.search import SearchQuery, SearchResponse, SearchRetrievalMode
|
||||
from basic_memory.deps import (
|
||||
SearchServiceV2ExternalDep,
|
||||
EntityServiceV2ExternalDep,
|
||||
@@ -48,7 +50,7 @@ async def search(
|
||||
Returns:
|
||||
SearchResponse with paginated search results
|
||||
"""
|
||||
with telemetry.operation(
|
||||
with logfire.span(
|
||||
"api.request.search",
|
||||
entrypoint="api",
|
||||
domain="search",
|
||||
@@ -65,9 +67,9 @@ async def search(
|
||||
has_filters=bool(query.note_types or query.entity_types or query.metadata_filters),
|
||||
):
|
||||
offset = (page - 1) * page_size
|
||||
fetch_limit = page_size + 1
|
||||
exact_count_available = query.retrieval_mode == SearchRetrievalMode.FTS
|
||||
try:
|
||||
with telemetry.scope(
|
||||
with logfire.span(
|
||||
"api.search.search.execute_query",
|
||||
domain="search",
|
||||
action="search",
|
||||
@@ -75,7 +77,14 @@ async def search(
|
||||
page=page,
|
||||
page_size=page_size,
|
||||
):
|
||||
results = await search_service.search(query, limit=fetch_limit, offset=offset)
|
||||
if exact_count_available:
|
||||
results, total = await asyncio.gather(
|
||||
search_service.search(query, limit=page_size, offset=offset),
|
||||
search_service.count(query),
|
||||
)
|
||||
else:
|
||||
results = await search_service.search(query, limit=page_size + 1, offset=offset)
|
||||
total = 0
|
||||
except SemanticSearchDisabledError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
except SemanticDependenciesMissingError as exc:
|
||||
@@ -83,18 +92,24 @@ async def search(
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
|
||||
with telemetry.scope(
|
||||
with logfire.span(
|
||||
"api.search.search.paginate_results",
|
||||
domain="search",
|
||||
action="search",
|
||||
phase="paginate_results",
|
||||
result_count=len(results),
|
||||
):
|
||||
has_more = len(results) > page_size
|
||||
if has_more:
|
||||
results = results[:page_size]
|
||||
if exact_count_available:
|
||||
has_more = offset + len(results) < total
|
||||
else:
|
||||
# Trigger: semantic modes would need another vector/hybrid retrieval to count.
|
||||
# Why: search requests should not pay for a second semantic pass.
|
||||
# Outcome: preserve probe pagination for semantic search and leave total at 0.
|
||||
has_more = len(results) > page_size
|
||||
if has_more:
|
||||
results = results[:page_size]
|
||||
|
||||
with telemetry.scope(
|
||||
with logfire.span(
|
||||
"api.search.search.hydrate_results",
|
||||
domain="search",
|
||||
action="search",
|
||||
@@ -102,7 +117,7 @@ async def search(
|
||||
result_count=len(results),
|
||||
):
|
||||
search_results = await to_search_results(entity_service, results)
|
||||
with telemetry.scope(
|
||||
with logfire.span(
|
||||
"api.search.search.build_response",
|
||||
domain="search",
|
||||
action="search",
|
||||
@@ -113,6 +128,7 @@ async def search(
|
||||
results=search_results,
|
||||
current_page=page,
|
||||
page_size=page_size,
|
||||
total=total,
|
||||
has_more=has_more,
|
||||
)
|
||||
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
from typing import Optional, List
|
||||
from typing import Any, Protocol, Optional, List, Sequence
|
||||
|
||||
from basic_memory import telemetry
|
||||
from basic_memory.models import Entity as EntityModel
|
||||
from basic_memory.repository import EntityRepository
|
||||
import logfire
|
||||
from basic_memory.repository.search_repository import SearchIndexRow
|
||||
from basic_memory.schemas.memory import (
|
||||
EntitySummary,
|
||||
@@ -13,20 +11,39 @@ from basic_memory.schemas.memory import (
|
||||
ContextResult,
|
||||
)
|
||||
from basic_memory.schemas.search import SearchItemType, SearchResult
|
||||
from basic_memory.services import EntityService
|
||||
from basic_memory.services.context_service import (
|
||||
ContextResultRow,
|
||||
ContextResult as ServiceContextResult,
|
||||
)
|
||||
|
||||
|
||||
class EntityBatchLookup(Protocol):
|
||||
async def find_by_ids(self, ids: List[int]) -> Sequence[Any]: ...
|
||||
|
||||
|
||||
class EntityServiceBatchLookup(Protocol):
|
||||
async def get_entities_by_id(self, ids: List[int]) -> Sequence[Any]: ...
|
||||
|
||||
|
||||
def _required_str(value: str | None, field_name: str) -> str:
|
||||
"""Return a required search field or fail before producing invalid response data."""
|
||||
if value is None:
|
||||
raise ValueError(f"Search result is missing required field: {field_name}")
|
||||
return value
|
||||
|
||||
|
||||
def _search_item_type(value: str | SearchItemType) -> SearchItemType:
|
||||
"""Normalize repository row type strings into the public search enum."""
|
||||
return value if isinstance(value, SearchItemType) else SearchItemType(value)
|
||||
|
||||
|
||||
async def to_graph_context(
|
||||
context_result: ServiceContextResult,
|
||||
entity_repository: EntityRepository,
|
||||
entity_repository: EntityBatchLookup,
|
||||
page: Optional[int] = None,
|
||||
page_size: Optional[int] = None,
|
||||
):
|
||||
with telemetry.scope(
|
||||
) -> GraphContext:
|
||||
with logfire.span(
|
||||
"memory.hydrate_context",
|
||||
domain="memory",
|
||||
action="build_context",
|
||||
@@ -44,17 +61,18 @@ async def to_graph_context(
|
||||
+ context_item.observations
|
||||
+ context_item.related_results
|
||||
):
|
||||
if item.type == SearchItemType.ENTITY:
|
||||
item_type = _search_item_type(item.type)
|
||||
if item_type == SearchItemType.ENTITY:
|
||||
# Entity's own ID for its external_id
|
||||
entity_ids_needed.add(item.id)
|
||||
elif item.type == SearchItemType.OBSERVATION:
|
||||
elif item_type == SearchItemType.OBSERVATION:
|
||||
# Parent entity ID for entity_external_id
|
||||
if item.entity_id: # pyright: ignore
|
||||
entity_ids_needed.add(item.entity_id) # pyright: ignore
|
||||
elif item.type == SearchItemType.RELATION:
|
||||
if item.entity_id:
|
||||
entity_ids_needed.add(item.entity_id)
|
||||
elif item_type == SearchItemType.RELATION:
|
||||
# Source and target entity IDs for external_ids
|
||||
if item.from_id: # pyright: ignore
|
||||
entity_ids_needed.add(item.from_id) # pyright: ignore
|
||||
if item.from_id:
|
||||
entity_ids_needed.add(item.from_id)
|
||||
if item.to_id:
|
||||
entity_ids_needed.add(item.to_id)
|
||||
|
||||
@@ -62,7 +80,7 @@ async def to_graph_context(
|
||||
entity_title_lookup: dict[int, str] = {}
|
||||
entity_external_id_lookup: dict[int, str] = {}
|
||||
if entity_ids_needed:
|
||||
with telemetry.scope(
|
||||
with logfire.span(
|
||||
"memory.hydrate_context.lookup_entities",
|
||||
domain="memory",
|
||||
action="build_context",
|
||||
@@ -75,59 +93,62 @@ async def to_graph_context(
|
||||
entity_external_id_lookup[e.id] = e.external_id
|
||||
|
||||
# Helper function to convert items to summaries
|
||||
def to_summary(item: SearchIndexRow | ContextResultRow):
|
||||
match item.type:
|
||||
def to_summary(
|
||||
item: SearchIndexRow | ContextResultRow,
|
||||
) -> EntitySummary | ObservationSummary | RelationSummary:
|
||||
item_type = _search_item_type(item.type)
|
||||
match item_type:
|
||||
case SearchItemType.ENTITY:
|
||||
return EntitySummary(
|
||||
external_id=entity_external_id_lookup.get(item.id, ""),
|
||||
entity_id=item.id,
|
||||
title=item.title, # pyright: ignore
|
||||
title=_required_str(item.title, "title"),
|
||||
permalink=item.permalink,
|
||||
content=item.content,
|
||||
file_path=item.file_path,
|
||||
file_path=_required_str(item.file_path, "file_path"),
|
||||
created_at=item.created_at,
|
||||
)
|
||||
case SearchItemType.OBSERVATION:
|
||||
entity_ext_id = None
|
||||
if item.entity_id: # pyright: ignore
|
||||
entity_ext_id = entity_external_id_lookup.get(item.entity_id) # pyright: ignore
|
||||
entity_title = None
|
||||
if item.entity_id:
|
||||
entity_ext_id = entity_external_id_lookup.get(item.entity_id)
|
||||
entity_title = entity_title_lookup.get(item.entity_id)
|
||||
return ObservationSummary(
|
||||
observation_id=item.id,
|
||||
entity_id=item.entity_id, # pyright: ignore
|
||||
entity_id=item.entity_id,
|
||||
entity_external_id=entity_ext_id,
|
||||
title=entity_title_lookup.get(item.entity_id), # pyright: ignore
|
||||
file_path=item.file_path,
|
||||
category=item.category, # pyright: ignore
|
||||
content=item.content, # pyright: ignore
|
||||
permalink=item.permalink, # pyright: ignore
|
||||
title=entity_title,
|
||||
file_path=_required_str(item.file_path, "file_path"),
|
||||
category=_required_str(item.category, "category"),
|
||||
content=_required_str(item.content, "content"),
|
||||
permalink=_required_str(item.permalink, "permalink"),
|
||||
created_at=item.created_at,
|
||||
)
|
||||
case SearchItemType.RELATION:
|
||||
from_title = entity_title_lookup.get(item.from_id) if item.from_id else None # pyright: ignore
|
||||
from_title = entity_title_lookup.get(item.from_id) if item.from_id else None
|
||||
to_title = entity_title_lookup.get(item.to_id) if item.to_id else None
|
||||
from_ext_id = (
|
||||
entity_external_id_lookup.get(item.from_id) if item.from_id else None
|
||||
) # pyright: ignore
|
||||
)
|
||||
to_ext_id = entity_external_id_lookup.get(item.to_id) if item.to_id else None
|
||||
return RelationSummary(
|
||||
relation_id=item.id,
|
||||
entity_id=item.entity_id, # pyright: ignore
|
||||
title=item.title, # pyright: ignore
|
||||
file_path=item.file_path,
|
||||
permalink=item.permalink, # pyright: ignore
|
||||
relation_type=item.relation_type, # pyright: ignore
|
||||
entity_id=item.entity_id,
|
||||
title=_required_str(item.title, "title"),
|
||||
file_path=_required_str(item.file_path, "file_path"),
|
||||
permalink=_required_str(item.permalink, "permalink"),
|
||||
relation_type=_required_str(item.relation_type, "relation_type"),
|
||||
from_entity=from_title,
|
||||
from_entity_id=item.from_id, # pyright: ignore
|
||||
from_entity_id=item.from_id,
|
||||
from_entity_external_id=from_ext_id,
|
||||
to_entity=to_title,
|
||||
to_entity_id=item.to_id,
|
||||
to_entity_external_id=to_ext_id,
|
||||
created_at=item.created_at,
|
||||
)
|
||||
case _: # pragma: no cover
|
||||
raise ValueError(f"Unexpected type: {item.type}")
|
||||
|
||||
with telemetry.scope(
|
||||
with logfire.span(
|
||||
"memory.hydrate_context.shape_results",
|
||||
domain="memory",
|
||||
action="build_context",
|
||||
@@ -137,12 +158,16 @@ async def to_graph_context(
|
||||
hierarchical_results = []
|
||||
for context_item in context_result.results:
|
||||
primary_result = to_summary(context_item.primary_result)
|
||||
observations = [to_summary(obs) for obs in context_item.observations]
|
||||
observations = [
|
||||
summary
|
||||
for summary in (to_summary(obs) for obs in context_item.observations)
|
||||
if isinstance(summary, ObservationSummary)
|
||||
]
|
||||
related = [to_summary(rel) for rel in context_item.related_results]
|
||||
hierarchical_results.append(
|
||||
ContextResult(
|
||||
primary_result=primary_result,
|
||||
observations=observations, # pyright: ignore[reportArgumentType]
|
||||
observations=observations,
|
||||
related_results=related,
|
||||
)
|
||||
)
|
||||
@@ -170,8 +195,10 @@ async def to_graph_context(
|
||||
)
|
||||
|
||||
|
||||
async def to_search_results(entity_service: EntityService, results: List[SearchIndexRow]):
|
||||
with telemetry.scope(
|
||||
async def to_search_results(
|
||||
entity_service: EntityServiceBatchLookup, results: List[SearchIndexRow]
|
||||
) -> list[SearchResult]:
|
||||
with logfire.span(
|
||||
"search.hydrate_results",
|
||||
domain="search",
|
||||
action="search",
|
||||
@@ -187,8 +214,8 @@ async def to_search_results(entity_service: EntityService, results: List[SearchI
|
||||
all_entity_ids.add(eid)
|
||||
|
||||
# Single batch fetch for all entities
|
||||
entities_by_id: dict[int, EntityModel] = {}
|
||||
with telemetry.scope(
|
||||
entities_by_id: dict[int, Any] = {}
|
||||
with logfire.span(
|
||||
"search.hydrate_results.fetch_entities",
|
||||
domain="search",
|
||||
action="search",
|
||||
@@ -200,7 +227,7 @@ async def to_search_results(entity_service: EntityService, results: List[SearchI
|
||||
entities_by_id = {e.id: e for e in entities}
|
||||
|
||||
search_results = []
|
||||
with telemetry.scope(
|
||||
with logfire.span(
|
||||
"search.hydrate_results.shape_results",
|
||||
domain="search",
|
||||
action="search",
|
||||
@@ -222,20 +249,20 @@ async def to_search_results(entity_service: EntityService, results: List[SearchI
|
||||
entity_id = result.entity_id
|
||||
|
||||
# Look up entities by their specific IDs
|
||||
parent_entity = entities_by_id.get(result.entity_id) if result.entity_id else None # pyright: ignore
|
||||
from_entity = entities_by_id.get(result.from_id) if result.from_id else None # pyright: ignore
|
||||
parent_entity = entities_by_id.get(result.entity_id) if result.entity_id else None
|
||||
from_entity = entities_by_id.get(result.from_id) if result.from_id else None
|
||||
to_entity = entities_by_id.get(result.to_id) if result.to_id else None
|
||||
|
||||
search_results.append(
|
||||
SearchResult(
|
||||
title=result.title, # pyright: ignore
|
||||
type=result.type, # pyright: ignore
|
||||
title=_required_str(result.title, "title"),
|
||||
type=_search_item_type(result.type),
|
||||
permalink=result.permalink,
|
||||
score=result.score, # pyright: ignore
|
||||
score=result.score if result.score is not None else 0.0,
|
||||
entity=parent_entity.permalink if parent_entity else None,
|
||||
content=result.content,
|
||||
matched_chunk=result.matched_chunk_text,
|
||||
file_path=result.file_path,
|
||||
file_path=_required_str(result.file_path, "file_path"),
|
||||
metadata=result.metadata,
|
||||
entity_id=entity_id,
|
||||
observation_id=observation_id,
|
||||
|
||||
@@ -12,7 +12,7 @@ from basic_memory.cli.auto_update import maybe_run_periodic_auto_update # noqa:
|
||||
from basic_memory.cli.container import CliContainer, set_container # noqa: E402
|
||||
from basic_memory.cli.promo import maybe_show_cloud_promo, maybe_show_init_line # noqa: E402
|
||||
from basic_memory.config import init_cli_logging # noqa: E402
|
||||
from basic_memory import telemetry # noqa: E402
|
||||
import logfire # noqa: E402
|
||||
|
||||
|
||||
def version_callback(value: bool) -> None:
|
||||
@@ -45,7 +45,7 @@ def app_callback(
|
||||
init_cli_logging()
|
||||
command_name = ctx.invoked_subcommand or "root"
|
||||
ctx.with_resource(
|
||||
telemetry.operation(
|
||||
logfire.span(
|
||||
f"cli.command.{command_name}",
|
||||
entrypoint="cli",
|
||||
command_name=command_name,
|
||||
|
||||
@@ -99,41 +99,39 @@ async def make_api_request(
|
||||
response = await client.request(method=method, url=url, headers=headers, json=json_data)
|
||||
response.raise_for_status()
|
||||
return response
|
||||
except httpx.HTTPStatusError as e:
|
||||
response = e.response
|
||||
|
||||
# Try to parse error detail from response
|
||||
error_detail = None
|
||||
try:
|
||||
error_detail = response.json()
|
||||
except Exception:
|
||||
# If JSON parsing fails, we'll handle it as a generic error
|
||||
pass
|
||||
|
||||
# Check for subscription_required error (403)
|
||||
if response.status_code == 403 and isinstance(error_detail, dict):
|
||||
# Handle both FastAPI HTTPException format (nested under "detail")
|
||||
# and direct format
|
||||
detail_obj = error_detail.get("detail", error_detail)
|
||||
if (
|
||||
isinstance(detail_obj, dict)
|
||||
and detail_obj.get("error") == "subscription_required"
|
||||
):
|
||||
message = detail_obj.get("message", "Active subscription required")
|
||||
subscribe_url = detail_obj.get(
|
||||
"subscribe_url", "https://basicmemory.com/subscribe"
|
||||
)
|
||||
raise SubscriptionRequiredError(
|
||||
message=message, subscribe_url=subscribe_url
|
||||
) from e
|
||||
|
||||
# Raise generic CloudAPIError with status code and detail
|
||||
raise CloudAPIError(
|
||||
f"API request failed: {e}",
|
||||
status_code=response.status_code,
|
||||
detail=error_detail if isinstance(error_detail, dict) else {},
|
||||
) from e
|
||||
except httpx.HTTPError as e:
|
||||
# Check if this is a response error with response details
|
||||
if hasattr(e, "response") and e.response is not None: # pyright: ignore [reportAttributeAccessIssue]
|
||||
response = e.response # type: ignore
|
||||
|
||||
# Try to parse error detail from response
|
||||
error_detail = None
|
||||
try:
|
||||
error_detail = response.json()
|
||||
except Exception:
|
||||
# If JSON parsing fails, we'll handle it as a generic error
|
||||
pass
|
||||
|
||||
# Check for subscription_required error (403)
|
||||
if response.status_code == 403 and isinstance(error_detail, dict):
|
||||
# Handle both FastAPI HTTPException format (nested under "detail")
|
||||
# and direct format
|
||||
detail_obj = error_detail.get("detail", error_detail)
|
||||
if (
|
||||
isinstance(detail_obj, dict)
|
||||
and detail_obj.get("error") == "subscription_required"
|
||||
):
|
||||
message = detail_obj.get("message", "Active subscription required")
|
||||
subscribe_url = detail_obj.get(
|
||||
"subscribe_url", "https://basicmemory.com/subscribe"
|
||||
)
|
||||
raise SubscriptionRequiredError(
|
||||
message=message, subscribe_url=subscribe_url
|
||||
) from e
|
||||
|
||||
# Raise generic CloudAPIError with status code and detail
|
||||
raise CloudAPIError(
|
||||
f"API request failed: {e}",
|
||||
status_code=response.status_code,
|
||||
detail=error_detail if isinstance(error_detail, dict) else {},
|
||||
) from e
|
||||
|
||||
raise CloudAPIError(f"API request failed: {e}") from e
|
||||
|
||||
@@ -76,10 +76,22 @@ def login():
|
||||
|
||||
@cloud_app.command()
|
||||
def logout():
|
||||
"""Remove stored OAuth tokens."""
|
||||
config = ConfigManager().config
|
||||
"""Remove stored OAuth tokens and clear cached workspace selection."""
|
||||
config_manager = ConfigManager()
|
||||
config = config_manager.config
|
||||
auth = CLIAuth(client_id=config.cloud_client_id, authkit_domain=config.cloud_domain)
|
||||
auth.logout()
|
||||
|
||||
# Trigger: ending a session must invalidate the cached workspace.
|
||||
# Why: a follow-up `bm cloud login` (often as a different user, or returning
|
||||
# from an org workspace to personal) inherits the previous selection
|
||||
# and silently routes everything through the wrong tenant. See #755.
|
||||
# Outcome: re-login starts from a clean slate; the user picks again via
|
||||
# `bm cloud workspace set-default` or per-project --workspace.
|
||||
if config.default_workspace is not None:
|
||||
config.default_workspace = None
|
||||
config_manager.save_config(config)
|
||||
|
||||
console.print("[dim]API key (if configured) remains available for cloud project routing.[/dim]")
|
||||
|
||||
|
||||
@@ -167,7 +179,7 @@ def setup() -> None:
|
||||
console.print("1. Add a project with local sync path:")
|
||||
console.print(" bm project add research --cloud --local-path ~/Documents/research")
|
||||
console.print("\n Or configure sync for an existing project:")
|
||||
console.print(" bm project sync-setup research ~/Documents/research")
|
||||
console.print(" bm cloud sync-setup research ~/Documents/research")
|
||||
console.print("\n2. Preview the initial sync (recommended):")
|
||||
console.print(" bm project bisync --name research --resync --dry-run")
|
||||
console.print("\n3. If all looks good, run the actual sync:")
|
||||
|
||||
@@ -20,6 +20,7 @@ from loguru import logger
|
||||
from rich.console import Console
|
||||
|
||||
from basic_memory.cli.commands.cloud.rclone_installer import is_rclone_installed
|
||||
from basic_memory.config import resolve_data_dir
|
||||
from basic_memory.utils import normalize_project_path
|
||||
|
||||
console = Console()
|
||||
@@ -138,13 +139,16 @@ def get_bmignore_filter_path() -> Path:
|
||||
def get_project_bisync_state(project_name: str) -> Path:
|
||||
"""Get path to project's bisync state directory.
|
||||
|
||||
Honors ``BASIC_MEMORY_CONFIG_DIR`` so isolated instances each keep their
|
||||
own bisync state alongside their config.
|
||||
|
||||
Args:
|
||||
project_name: Name of the project
|
||||
|
||||
Returns:
|
||||
Path to bisync state directory for this project
|
||||
"""
|
||||
return Path.home() / ".basic-memory" / "bisync-state" / project_name
|
||||
return resolve_data_dir() / "bisync-state" / project_name
|
||||
|
||||
|
||||
def bisync_initialized(project_name: str) -> bool:
|
||||
|
||||
@@ -1,8 +1,10 @@
|
||||
"""Database management commands."""
|
||||
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from pathlib import Path, PurePosixPath, PureWindowsPath
|
||||
|
||||
import psutil
|
||||
import typer
|
||||
from loguru import logger
|
||||
from rich.console import Console
|
||||
@@ -21,6 +23,103 @@ from basic_memory.sync.sync_service import get_sync_service
|
||||
console = Console()
|
||||
|
||||
|
||||
def _is_basic_memory_mcp(cmdline: list[str]) -> bool:
|
||||
"""Heuristic: does this argv represent a `basic-memory mcp` server?
|
||||
|
||||
The MCP server can be launched any of:
|
||||
basic-memory mcp
|
||||
bm mcp # entrypoint alias from pyproject.toml
|
||||
python -m basic_memory.cli.main mcp # module form
|
||||
uv run basic-memory mcp / uv run bm mcp # uv wrappers
|
||||
/abs/path/to/{bm,basic-memory}[.exe] mcp
|
||||
|
||||
A reliable match needs both signals:
|
||||
1. "mcp" appears as an exact argv token (not "mcp-foo").
|
||||
2. Some argv token names the basic-memory entrypoint — either by
|
||||
hyphen/underscore form, or as a `bm` script (covers `/usr/local/bin/bm`,
|
||||
`bm.exe`, etc. via Path.stem).
|
||||
"""
|
||||
if "mcp" not in cmdline:
|
||||
return False
|
||||
for arg in cmdline:
|
||||
if "basic-memory" in arg or "basic_memory" in arg:
|
||||
return True
|
||||
# Try both POSIX and Windows path interpretations so a test on
|
||||
# macOS still recognizes `C:\\...\\bm.exe`, and a real Windows
|
||||
# run still recognizes `/usr/local/bin/bm`. Path() alone uses
|
||||
# the host OS, which gives wrong stems for foreign separators.
|
||||
if PurePosixPath(arg).stem == "bm" or PureWindowsPath(arg).stem == "bm":
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _find_live_mcp_processes() -> list[tuple[int, str]]:
|
||||
"""Return (pid, joined_cmdline) for live `basic-memory mcp` processes.
|
||||
|
||||
Why this exists (issue #765):
|
||||
On POSIX, `Path.unlink()` removes the directory entry but the inode
|
||||
survives as long as any process holds the file open. A `bm reset`
|
||||
run while Claude Desktop (or another MCP client) is alive will
|
||||
therefore "succeed" — but the still-running MCP keeps reading the
|
||||
old, now-invisible memory.db inode and returns phantom rows. On
|
||||
Windows the OS naturally raises PermissionError on `unlink()`, so
|
||||
the bug is POSIX-specific. We detect proactively to give the same
|
||||
error experience on every platform before doing damage.
|
||||
|
||||
The current process is excluded so this can be called from inside a
|
||||
`bm reset` invocation. NoSuchProcess / AccessDenied are swallowed
|
||||
because process tables race with the scan and we don't want a
|
||||
transient permission error to mask a real zombie.
|
||||
"""
|
||||
me = os.getpid()
|
||||
matches: list[tuple[int, str]] = []
|
||||
for proc in psutil.process_iter(["pid", "cmdline"]):
|
||||
try:
|
||||
pid = proc.info.get("pid")
|
||||
if pid is None or pid == me:
|
||||
continue
|
||||
cmdline = proc.info.get("cmdline") or []
|
||||
if not cmdline:
|
||||
continue
|
||||
if _is_basic_memory_mcp(cmdline):
|
||||
matches.append((pid, " ".join(cmdline)))
|
||||
except (psutil.NoSuchProcess, psutil.AccessDenied):
|
||||
continue
|
||||
return matches
|
||||
|
||||
|
||||
def _abort_if_mcp_processes_alive() -> None:
|
||||
"""Refuse `bm reset` while basic-memory MCP processes are still running.
|
||||
|
||||
See _find_live_mcp_processes for the underlying POSIX-vs-Windows
|
||||
rationale. Prints a per-PID list and platform-appropriate cleanup
|
||||
instructions, then exits non-zero so destructive work never starts.
|
||||
"""
|
||||
zombies = _find_live_mcp_processes()
|
||||
if not zombies:
|
||||
return
|
||||
|
||||
console.print("[red]Refusing to reset:[/red] basic-memory MCP processes are still running.")
|
||||
console.print(
|
||||
"[yellow]On macOS/Linux these would keep reading the deleted memory.db inode "
|
||||
"and return phantom search results (see #765).[/yellow]"
|
||||
)
|
||||
for pid, cmd in zombies:
|
||||
console.print(f" PID {pid}: {cmd}")
|
||||
console.print("\n[bold]How to clean up:[/bold]")
|
||||
console.print(" 1. Quit Claude Desktop and any other MCP clients.")
|
||||
if os.name == "nt":
|
||||
console.print(
|
||||
" 2. Verify nothing remains: "
|
||||
"[green]Get-CimInstance Win32_Process | "
|
||||
"Where-Object {$_.CommandLine -like '*basic-memory*mcp*'}[/green]"
|
||||
)
|
||||
else:
|
||||
console.print(" 2. Verify nothing remains: [green]pgrep -fa 'basic-memory mcp'[/green]")
|
||||
console.print(" 3. Re-run [green]bm reset[/green].")
|
||||
raise typer.Exit(1)
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class EmbeddingProgress:
|
||||
"""Typed CLI progress payload for embedding backfills."""
|
||||
@@ -86,6 +185,16 @@ async def _reindex_projects(app_config):
|
||||
@app.command()
|
||||
def reset(
|
||||
reindex: bool = typer.Option(False, "--reindex", help="Rebuild db index from filesystem"),
|
||||
force: bool = typer.Option(
|
||||
False,
|
||||
"--force",
|
||||
help=(
|
||||
"Skip the pre-flight check that refuses to reset while "
|
||||
"basic-memory MCP processes are running. Use only in "
|
||||
"automated workflows where you've already ensured no MCP "
|
||||
"clients are attached to the database."
|
||||
),
|
||||
),
|
||||
): # pragma: no cover
|
||||
"""Reset database (drop all tables and recreate)."""
|
||||
console.print(
|
||||
@@ -94,6 +203,14 @@ def reset(
|
||||
"Use [green]bm reset --reindex[/green] to automatically rebuild the index afterward."
|
||||
)
|
||||
if typer.confirm("Reset the database index?"):
|
||||
# Pre-flight: refuse to proceed if MCP processes still hold the DB
|
||||
# file open. POSIX would silently let us unlink the inode while
|
||||
# they keep reading it; Windows would error here anyway. See
|
||||
# _find_live_mcp_processes for the full story. --force is the
|
||||
# documented escape hatch for scripted/CI runs.
|
||||
if not force:
|
||||
_abort_if_mcp_processes_alive()
|
||||
|
||||
logger.info("Resetting database...")
|
||||
config_manager = ConfigManager()
|
||||
app_config = config_manager.config
|
||||
|
||||
@@ -69,7 +69,7 @@ async def run_doctor() -> None:
|
||||
content=f"# {api_note_title}\n\n- [note] API to file check",
|
||||
entity_metadata={"tags": ["doctor"]},
|
||||
)
|
||||
api_result = await knowledge_client.create_entity(api_note.model_dump(), fast=False)
|
||||
api_result = await knowledge_client.create_entity(api_note.model_dump())
|
||||
|
||||
api_file = project_path / api_result.file_path
|
||||
if not api_file.exists():
|
||||
|
||||
@@ -26,7 +26,7 @@ from basic_memory.cli.commands.cloud.rclone_commands import (
|
||||
)
|
||||
from basic_memory.cli.commands.command_utils import get_project_info, run_with_cleanup
|
||||
from basic_memory.cli.commands.routing import force_routing, validate_routing_flags
|
||||
from basic_memory.config import ConfigManager, ProjectEntry, ProjectMode
|
||||
from basic_memory.config import BasicMemoryConfig, ConfigManager, ProjectEntry, ProjectMode
|
||||
from basic_memory.mcp.async_client import get_client, resolve_configured_workspace
|
||||
from basic_memory.mcp.clients import ProjectClient
|
||||
from basic_memory.schemas.cloud import (
|
||||
@@ -838,6 +838,76 @@ def move_project(
|
||||
raise typer.Exit(1)
|
||||
|
||||
|
||||
async def _detach_local_project_row(app_config: BasicMemoryConfig, name: str) -> bool:
|
||||
"""Drop the project's row from the local index DB.
|
||||
|
||||
Trigger: `bm project set-cloud` is making a project cloud-only.
|
||||
Why: the local row is what causes `_merge_projects` to report
|
||||
`source: "local+cloud"` after the toggle (#680). Removing it
|
||||
forces the merged listing to honor the user's chosen mode.
|
||||
Outcome: returns True if a row was deleted, False if there was
|
||||
nothing to clean up. On-disk note files are not touched.
|
||||
"""
|
||||
from basic_memory import db
|
||||
from basic_memory.repository import ProjectRepository
|
||||
|
||||
_, session_maker = await db.get_or_create_db(
|
||||
db_path=app_config.database_path,
|
||||
db_type=db.DatabaseType.FILESYSTEM,
|
||||
)
|
||||
try:
|
||||
repo = ProjectRepository(session_maker)
|
||||
existing = await repo.get_by_name(name)
|
||||
if existing is None:
|
||||
return False
|
||||
await repo.delete(existing.id)
|
||||
return True
|
||||
finally:
|
||||
# CLI-only: safe to tear down the global DB singleton here since
|
||||
# set-cloud/set-local never run inside a long-lived MCP/API server.
|
||||
await db.shutdown_db()
|
||||
|
||||
|
||||
async def _attach_local_project_row(app_config: BasicMemoryConfig, name: str, path: str) -> None:
|
||||
"""Ensure the project has a row in the local index DB at the given path.
|
||||
|
||||
Trigger: `bm project set-local` is making a previously cloud-only
|
||||
project local again.
|
||||
Why: without a row in the local DB, every local-side tool (`list`,
|
||||
`info`, sync, indexing) would skip this project.
|
||||
Outcome: a row is created if missing, or its path is updated to match
|
||||
the new local home if it already exists. On-disk files are not
|
||||
touched — the caller is responsible for ensuring the directory
|
||||
exists.
|
||||
"""
|
||||
from basic_memory import db
|
||||
from basic_memory.repository import ProjectRepository
|
||||
|
||||
_, session_maker = await db.get_or_create_db(
|
||||
db_path=app_config.database_path,
|
||||
db_type=db.DatabaseType.FILESYSTEM,
|
||||
)
|
||||
try:
|
||||
repo = ProjectRepository(session_maker)
|
||||
existing = await repo.get_by_name(name)
|
||||
if existing is None:
|
||||
await repo.create(
|
||||
{
|
||||
"name": name,
|
||||
"path": path,
|
||||
"permalink": generate_permalink(name),
|
||||
"is_active": True,
|
||||
}
|
||||
)
|
||||
return
|
||||
if existing.path != path:
|
||||
await repo.update_path(existing.id, path)
|
||||
finally:
|
||||
# CLI-only: safe to tear down the global DB singleton here since
|
||||
# set-cloud/set-local never run inside a long-lived MCP/API server.
|
||||
await db.shutdown_db()
|
||||
|
||||
|
||||
@project_app.command("set-cloud")
|
||||
def set_cloud(
|
||||
name: str = typer.Argument(..., help="Name of the project to route through cloud"),
|
||||
@@ -855,6 +925,13 @@ def set_cloud(
|
||||
If omitted, uses the default workspace (if set) or auto-selects when
|
||||
only one workspace is available.
|
||||
|
||||
This is a one-way cutover: the project's row in the local index DB is
|
||||
removed and the local path in config is cleared so the project's
|
||||
configured state is purely cloud. On-disk note files are preserved —
|
||||
the caller can keep, archive, or delete them as they see fit. To
|
||||
return to local mode use `bm project set-local <name> --local-path
|
||||
<path>`.
|
||||
|
||||
Examples:
|
||||
bm project set-cloud research --workspace Personal
|
||||
bm project set-cloud research --workspace 11111111-...
|
||||
@@ -883,27 +960,52 @@ def set_cloud(
|
||||
|
||||
resolved_workspace_id = _resolve_workspace_id(config, workspace)
|
||||
|
||||
# Drop the local DB row first so the user-visible state stays consistent
|
||||
# even if the config save below raises for some reason. Idempotent: a
|
||||
# second `set-cloud` simply finds no row and returns False.
|
||||
previous_path = config.projects[name].path
|
||||
detached = run_with_cleanup(_detach_local_project_row(config, name))
|
||||
|
||||
config.set_project_mode(name, ProjectMode.CLOUD)
|
||||
if resolved_workspace_id:
|
||||
config.projects[name].workspace_id = resolved_workspace_id
|
||||
# Clear local path: source-of-truth for this project is now the cloud
|
||||
config.projects[name].path = ""
|
||||
config_manager.save_config(config)
|
||||
|
||||
console.print(f"[green]Project '{name}' set to cloud mode[/green]")
|
||||
if resolved_workspace_id:
|
||||
console.print(f"[dim]Workspace: {resolved_workspace_id}[/dim]")
|
||||
if detached and previous_path:
|
||||
console.print(
|
||||
f"[dim]Local index entry removed. Files at {previous_path} are preserved on disk.[/dim]"
|
||||
)
|
||||
console.print("[dim]MCP tools and CLI commands for this project will route through cloud[/dim]")
|
||||
|
||||
|
||||
@project_app.command("set-local")
|
||||
def set_local(
|
||||
name: str = typer.Argument(..., help="Name of the project to revert to local mode"),
|
||||
local_path: str = typer.Option(
|
||||
None,
|
||||
"--local-path",
|
||||
help=(
|
||||
"Local filesystem path for this project. Required unless the project "
|
||||
"was previously local and its prior path is still in config."
|
||||
),
|
||||
),
|
||||
) -> None:
|
||||
"""Revert a project to local mode (use in-process ASGI transport).
|
||||
|
||||
Clears any associated cloud workspace.
|
||||
Recreates the project's row in the local index DB and clears any
|
||||
associated cloud workspace. If the project was previously local and
|
||||
its prior path is still in config (e.g. an older version that didn't
|
||||
blank `path` on `set-cloud`), `--local-path` may be omitted and that
|
||||
path will be reused.
|
||||
|
||||
Example:
|
||||
bm project set-local research
|
||||
Examples:
|
||||
bm project set-local research --local-path ~/Documents/research
|
||||
bm project set-local research # reuse prior path
|
||||
"""
|
||||
config_manager = ConfigManager()
|
||||
config = config_manager.config
|
||||
@@ -913,11 +1015,31 @@ def set_local(
|
||||
console.print(f"[red]Error: Project '{name}' not found in config[/red]")
|
||||
raise typer.Exit(1)
|
||||
|
||||
entry = config.projects[name]
|
||||
candidate = local_path or entry.path
|
||||
if not candidate:
|
||||
console.print(
|
||||
f"[red]Error: --local-path is required for '{name}' "
|
||||
"(no previous local path is recorded)[/red]"
|
||||
)
|
||||
raise typer.Exit(1)
|
||||
|
||||
resolved_path = Path(os.path.abspath(os.path.expanduser(candidate))).as_posix()
|
||||
|
||||
# Recreate the local DB row. Idempotent: if the row exists with the
|
||||
# same path it's a no-op; if it exists at a stale path the path is
|
||||
# updated. The directory itself is not auto-created — the user is
|
||||
# expected to know whether they want to start a fresh project tree
|
||||
# or point at an existing one.
|
||||
run_with_cleanup(_attach_local_project_row(config, name, resolved_path))
|
||||
|
||||
config.set_project_mode(name, ProjectMode.LOCAL)
|
||||
config.projects[name].workspace_id = None
|
||||
config.projects[name].path = resolved_path
|
||||
config_manager.save_config(config)
|
||||
|
||||
console.print(f"[green]Project '{name}' set to local mode[/green]")
|
||||
console.print(f"[dim]Path: {resolved_path}[/dim]")
|
||||
console.print("[dim]MCP tools and CLI commands for this project will use local transport[/dim]")
|
||||
|
||||
|
||||
|
||||
@@ -62,9 +62,12 @@ def write_note(
|
||||
help="The project to write to. If not provided, the default project will be used."
|
||||
),
|
||||
] = None,
|
||||
workspace: Annotated[
|
||||
project_id: Annotated[
|
||||
Optional[str],
|
||||
typer.Option(help="Cloud workspace tenant ID or unique name to route this request."),
|
||||
typer.Option(
|
||||
"--project-id",
|
||||
help="Project external_id (UUID). Takes precedence over --project; use to disambiguate same-named projects across cloud workspaces.",
|
||||
),
|
||||
] = None,
|
||||
local: bool = typer.Option(
|
||||
False, "--local", help="Force local API routing (ignore cloud mode)"
|
||||
@@ -106,7 +109,7 @@ def write_note(
|
||||
content=content,
|
||||
directory=folder,
|
||||
project=project,
|
||||
workspace=workspace,
|
||||
project_id=project_id,
|
||||
tags=tags,
|
||||
output_format="json",
|
||||
)
|
||||
@@ -128,15 +131,16 @@ def read_note(
|
||||
include_frontmatter: bool = typer.Option(
|
||||
False, "--include-frontmatter", help="Include YAML frontmatter in output"
|
||||
),
|
||||
page: int = typer.Option(1, "--page", help="Page number for pagination"),
|
||||
page_size: int = typer.Option(10, "--page-size", help="Number of results per page"),
|
||||
project: Annotated[
|
||||
Optional[str],
|
||||
typer.Option(help="The project to use. If not provided, the default project will be used."),
|
||||
] = None,
|
||||
workspace: Annotated[
|
||||
project_id: Annotated[
|
||||
Optional[str],
|
||||
typer.Option(help="Cloud workspace tenant ID or unique name to route this request."),
|
||||
typer.Option(
|
||||
"--project-id",
|
||||
help="Project external_id (UUID). Takes precedence over --project; use to disambiguate same-named projects across cloud workspaces.",
|
||||
),
|
||||
] = None,
|
||||
local: bool = typer.Option(
|
||||
False, "--local", help="Force local API routing (ignore cloud mode)"
|
||||
@@ -149,7 +153,6 @@ def read_note(
|
||||
|
||||
bm tool read-note my-note
|
||||
bm tool read-note my-note --include-frontmatter
|
||||
bm tool read-note my-note --page 2 --page-size 5
|
||||
"""
|
||||
try:
|
||||
validate_routing_flags(local, cloud)
|
||||
@@ -159,9 +162,7 @@ def read_note(
|
||||
mcp_read_note(
|
||||
identifier=identifier,
|
||||
project=project,
|
||||
workspace=workspace,
|
||||
page=page,
|
||||
page_size=page_size,
|
||||
project_id=project_id,
|
||||
include_frontmatter=include_frontmatter,
|
||||
output_format="json",
|
||||
)
|
||||
@@ -200,9 +201,12 @@ def edit_note(
|
||||
help="The project to edit. If not provided, the default project will be used."
|
||||
),
|
||||
] = None,
|
||||
workspace: Annotated[
|
||||
project_id: Annotated[
|
||||
Optional[str],
|
||||
typer.Option(help="Cloud workspace tenant ID or unique name to route this request."),
|
||||
typer.Option(
|
||||
"--project-id",
|
||||
help="Project external_id (UUID). Takes precedence over --project; use to disambiguate same-named projects across cloud workspaces.",
|
||||
),
|
||||
] = None,
|
||||
local: bool = typer.Option(
|
||||
False, "--local", help="Force local API routing (ignore cloud mode)"
|
||||
@@ -227,7 +231,7 @@ def edit_note(
|
||||
operation=operation,
|
||||
content=content,
|
||||
project=project,
|
||||
workspace=workspace,
|
||||
project_id=project_id,
|
||||
section=section,
|
||||
find_text=find_text,
|
||||
expected_replacements=expected_replacements,
|
||||
@@ -265,9 +269,12 @@ def build_context(
|
||||
Optional[str],
|
||||
typer.Option(help="The project to use. If not provided, the default project will be used."),
|
||||
] = None,
|
||||
workspace: Annotated[
|
||||
project_id: Annotated[
|
||||
Optional[str],
|
||||
typer.Option(help="Cloud workspace tenant ID or unique name to route this request."),
|
||||
typer.Option(
|
||||
"--project-id",
|
||||
help="Project external_id (UUID). Takes precedence over --project; use to disambiguate same-named projects across cloud workspaces.",
|
||||
),
|
||||
] = None,
|
||||
local: bool = typer.Option(
|
||||
False, "--local", help="Force local API routing (ignore cloud mode)"
|
||||
@@ -289,7 +296,7 @@ def build_context(
|
||||
mcp_build_context(
|
||||
url=url,
|
||||
project=project,
|
||||
workspace=workspace,
|
||||
project_id=project_id,
|
||||
depth=depth,
|
||||
timeframe=timeframe,
|
||||
page=page,
|
||||
@@ -322,9 +329,12 @@ def recent_activity(
|
||||
Optional[str],
|
||||
typer.Option(help="The project to use. If not provided, the default project will be used."),
|
||||
] = None,
|
||||
workspace: Annotated[
|
||||
project_id: Annotated[
|
||||
Optional[str],
|
||||
typer.Option(help="Cloud workspace tenant ID or unique name to route this request."),
|
||||
typer.Option(
|
||||
"--project-id",
|
||||
help="Project external_id (UUID). Takes precedence over --project; use to disambiguate same-named projects across cloud workspaces.",
|
||||
),
|
||||
] = None,
|
||||
local: bool = typer.Option(
|
||||
False, "--local", help="Force local API routing (ignore cloud mode)"
|
||||
@@ -345,13 +355,13 @@ def recent_activity(
|
||||
with force_routing(local=local, cloud=cloud):
|
||||
result = run_with_cleanup(
|
||||
mcp_recent_activity(
|
||||
type=type, # pyright: ignore[reportArgumentType]
|
||||
type=type or "",
|
||||
depth=depth if depth is not None else 1,
|
||||
timeframe=timeframe if timeframe is not None else "7d",
|
||||
page=page,
|
||||
page_size=page_size,
|
||||
project=project,
|
||||
workspace=workspace,
|
||||
project_id=project_id,
|
||||
output_format="json",
|
||||
)
|
||||
)
|
||||
@@ -413,9 +423,12 @@ def search_notes(
|
||||
Optional[str],
|
||||
typer.Option(help="The project to use. If not provided, the default project will be used."),
|
||||
] = None,
|
||||
workspace: Annotated[
|
||||
project_id: Annotated[
|
||||
Optional[str],
|
||||
typer.Option(help="Cloud workspace tenant ID or unique name to route this request."),
|
||||
typer.Option(
|
||||
"--project-id",
|
||||
help="Project external_id (UUID). Takes precedence over --project; use to disambiguate same-named projects across cloud workspaces.",
|
||||
),
|
||||
] = None,
|
||||
local: bool = typer.Option(
|
||||
False, "--local", help="Force local API routing (ignore cloud mode)"
|
||||
@@ -487,7 +500,7 @@ def search_notes(
|
||||
mcp_search(
|
||||
query=query or None,
|
||||
project=project,
|
||||
workspace=workspace,
|
||||
project_id=project_id,
|
||||
search_type=search_type,
|
||||
output_format="json",
|
||||
page=page,
|
||||
@@ -597,9 +610,12 @@ def schema_validate(
|
||||
Optional[str],
|
||||
typer.Option(help="The project to use. If not provided, the default project will be used."),
|
||||
] = None,
|
||||
workspace: Annotated[
|
||||
project_id: Annotated[
|
||||
Optional[str],
|
||||
typer.Option(help="Cloud workspace tenant ID or unique name to route this request."),
|
||||
typer.Option(
|
||||
"--project-id",
|
||||
help="Project external_id (UUID). Takes precedence over --project; use to disambiguate same-named projects across cloud workspaces.",
|
||||
),
|
||||
] = None,
|
||||
local: bool = typer.Option(
|
||||
False, "--local", help="Force local API routing (ignore cloud mode)"
|
||||
@@ -634,7 +650,7 @@ def schema_validate(
|
||||
note_type=note_type,
|
||||
identifier=identifier,
|
||||
project=project,
|
||||
workspace=workspace,
|
||||
project_id=project_id,
|
||||
output_format="json",
|
||||
)
|
||||
)
|
||||
@@ -665,9 +681,12 @@ def schema_infer(
|
||||
Optional[str],
|
||||
typer.Option(help="The project to use. If not provided, the default project will be used."),
|
||||
] = None,
|
||||
workspace: Annotated[
|
||||
project_id: Annotated[
|
||||
Optional[str],
|
||||
typer.Option(help="Cloud workspace tenant ID or unique name to route this request."),
|
||||
typer.Option(
|
||||
"--project-id",
|
||||
help="Project external_id (UUID). Takes precedence over --project; use to disambiguate same-named projects across cloud workspaces.",
|
||||
),
|
||||
] = None,
|
||||
local: bool = typer.Option(
|
||||
False, "--local", help="Force local API routing (ignore cloud mode)"
|
||||
@@ -691,7 +710,7 @@ def schema_infer(
|
||||
note_type=note_type,
|
||||
threshold=threshold,
|
||||
project=project,
|
||||
workspace=workspace,
|
||||
project_id=project_id,
|
||||
output_format="json",
|
||||
)
|
||||
)
|
||||
@@ -719,9 +738,12 @@ def schema_diff(
|
||||
Optional[str],
|
||||
typer.Option(help="The project to use. If not provided, the default project will be used."),
|
||||
] = None,
|
||||
workspace: Annotated[
|
||||
project_id: Annotated[
|
||||
Optional[str],
|
||||
typer.Option(help="Cloud workspace tenant ID or unique name to route this request."),
|
||||
typer.Option(
|
||||
"--project-id",
|
||||
help="Project external_id (UUID). Takes precedence over --project; use to disambiguate same-named projects across cloud workspaces.",
|
||||
),
|
||||
] = None,
|
||||
local: bool = typer.Option(
|
||||
False, "--local", help="Force local API routing (ignore cloud mode)"
|
||||
@@ -743,7 +765,7 @@ def schema_diff(
|
||||
mcp_schema_diff(
|
||||
note_type=note_type,
|
||||
project=project,
|
||||
workspace=workspace,
|
||||
project_id=project_id,
|
||||
output_format="json",
|
||||
)
|
||||
)
|
||||
|
||||
+53
-17
@@ -8,7 +8,7 @@ from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from enum import Enum
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, Literal, Optional, List, Tuple
|
||||
from typing import TYPE_CHECKING, Any, Dict, Literal, Optional, List, Tuple
|
||||
|
||||
from loguru import logger
|
||||
from pydantic import AliasChoices, BaseModel, Field, model_validator
|
||||
@@ -50,6 +50,44 @@ def _default_semantic_search_enabled() -> bool:
|
||||
)
|
||||
|
||||
|
||||
def resolve_data_dir() -> Path:
|
||||
"""Resolve the Basic Memory data directory.
|
||||
|
||||
Single source of truth for the per-user state directory. Honors
|
||||
``BASIC_MEMORY_CONFIG_DIR`` so each process/worktree can isolate config
|
||||
and database state; otherwise falls back to ``<user home>/.basic-memory``.
|
||||
|
||||
Cross-platform: ``Path.home()`` reads ``$HOME`` on POSIX and
|
||||
``%USERPROFILE%`` on Windows, so there's no need to check ``$HOME``
|
||||
explicitly here.
|
||||
"""
|
||||
if config_dir := os.getenv("BASIC_MEMORY_CONFIG_DIR"):
|
||||
return Path(config_dir)
|
||||
return Path.home() / DATA_DIR_NAME
|
||||
|
||||
|
||||
def default_fastembed_cache_dir() -> str:
|
||||
"""Return the default cache directory used for FastEmbed model artifacts.
|
||||
|
||||
Resolution order:
|
||||
1. ``FASTEMBED_CACHE_PATH`` env var — honors FastEmbed's own convention
|
||||
so users who already configure it through the environment keep working.
|
||||
2. ``<basic-memory data dir>/fastembed_cache`` — the same stable,
|
||||
user-writable directory Basic Memory already uses for config and
|
||||
the default SQLite database. Honors ``BASIC_MEMORY_CONFIG_DIR``.
|
||||
|
||||
Why not ``tempfile.gettempdir()``?
|
||||
FastEmbed's own default is ``<system tmp>/fastembed_cache``, which is
|
||||
ephemeral in many sandboxed MCP runtimes (e.g. Codex CLI wipes /tmp
|
||||
between invocations). The model then disappears and every subsequent
|
||||
ONNX load raises ``NO_SUCHFILE``. Persisting the cache under the
|
||||
per-user data directory works identically on macOS, Linux, and Windows.
|
||||
"""
|
||||
if env_override := os.getenv("FASTEMBED_CACHE_PATH"):
|
||||
return env_override
|
||||
return str(resolve_data_dir() / "fastembed_cache")
|
||||
|
||||
|
||||
@dataclass
|
||||
class ProjectConfig:
|
||||
"""Configuration for a specific basic-memory project."""
|
||||
@@ -122,6 +160,11 @@ class ProjectEntry(BaseModel):
|
||||
class BasicMemoryConfig(BaseSettings):
|
||||
"""Pydantic model for Basic Memory global configuration."""
|
||||
|
||||
if TYPE_CHECKING:
|
||||
# Pydantic accepts raw constructor data and validates/coerces it at runtime.
|
||||
# Model attributes remain strongly typed after initialization.
|
||||
def __init__(self, **data: Any) -> None: ...
|
||||
|
||||
env: Environment = Field(default="dev", description="Environment name")
|
||||
|
||||
projects: Dict[str, ProjectEntry] = Field(
|
||||
@@ -217,7 +260,13 @@ class BasicMemoryConfig(BaseSettings):
|
||||
)
|
||||
semantic_embedding_cache_dir: str | None = Field(
|
||||
default=None,
|
||||
description="Optional cache directory for FastEmbed model artifacts.",
|
||||
description=(
|
||||
"Optional override for the FastEmbed model cache directory. "
|
||||
"When unset, Basic Memory resolves this at runtime to "
|
||||
"<basic-memory data dir>/fastembed_cache (or FASTEMBED_CACHE_PATH "
|
||||
"when that env var is set) so the model persists across runs "
|
||||
"without hardcoding a path into config.json."
|
||||
),
|
||||
)
|
||||
semantic_embedding_threads: int | None = Field(
|
||||
default=None,
|
||||
@@ -704,11 +753,7 @@ class BasicMemoryConfig(BaseSettings):
|
||||
@property
|
||||
def data_dir_path(self) -> Path:
|
||||
"""Get app state directory for config and default SQLite database."""
|
||||
if config_dir := os.getenv("BASIC_MEMORY_CONFIG_DIR"):
|
||||
return Path(config_dir)
|
||||
|
||||
home = os.getenv("HOME", Path.home())
|
||||
return Path(home) / DATA_DIR_NAME
|
||||
return resolve_data_dir()
|
||||
|
||||
|
||||
# Module-level cache for configuration
|
||||
@@ -726,16 +771,7 @@ class ConfigManager:
|
||||
|
||||
def __init__(self) -> None:
|
||||
"""Initialize the configuration manager."""
|
||||
home = os.getenv("HOME", Path.home())
|
||||
if isinstance(home, str):
|
||||
home = Path(home)
|
||||
|
||||
# Allow override via environment variable
|
||||
if config_dir := os.getenv("BASIC_MEMORY_CONFIG_DIR"):
|
||||
self.config_dir = Path(config_dir)
|
||||
else:
|
||||
self.config_dir = home / DATA_DIR_NAME
|
||||
|
||||
self.config_dir = resolve_data_dir()
|
||||
self.config_file = self.config_dir / CONFIG_FILE_NAME
|
||||
|
||||
# Ensure config directory exists
|
||||
|
||||
@@ -492,7 +492,6 @@ class LocalTaskScheduler:
|
||||
|
||||
|
||||
async def get_task_scheduler(
|
||||
entity_service: EntityServiceV2ExternalDep,
|
||||
sync_service: SyncServiceV2ExternalDep,
|
||||
search_service: SearchServiceV2ExternalDep,
|
||||
project_config: ProjectConfigV2ExternalDep,
|
||||
@@ -500,28 +499,6 @@ async def get_task_scheduler(
|
||||
) -> TaskScheduler:
|
||||
"""Create a scheduler that maps task specs to coroutines."""
|
||||
|
||||
scheduler: LocalTaskScheduler | None = None
|
||||
|
||||
async def _reindex_entity(
|
||||
entity_id: int,
|
||||
resolve_relations: bool = False,
|
||||
**_: Any,
|
||||
) -> None:
|
||||
await entity_service.reindex_entity(entity_id)
|
||||
# Trigger: caller requests relation resolution
|
||||
# Why: resolve forward references created before the entity existed
|
||||
# Outcome: updates unresolved relations pointing to this entity
|
||||
if resolve_relations:
|
||||
await sync_service.resolve_relations(entity_id=entity_id)
|
||||
# Trigger: semantic search enabled in local config.
|
||||
# Why: vector chunks are derived and should refresh after canonical reindex completes.
|
||||
# Outcome: schedules out-of-band vector sync without extending write latency.
|
||||
if app_config.semantic_search_enabled and scheduler is not None:
|
||||
scheduler.schedule("sync_entity_vectors", entity_id=entity_id)
|
||||
|
||||
async def _resolve_relations(entity_id: int, **_: Any) -> None:
|
||||
await sync_service.resolve_relations(entity_id=entity_id)
|
||||
|
||||
async def _sync_entity_vectors(entity_id: int, **_: Any) -> None:
|
||||
await search_service.sync_entity_vectors(entity_id)
|
||||
|
||||
@@ -537,8 +514,6 @@ async def get_task_scheduler(
|
||||
|
||||
scheduler = LocalTaskScheduler(
|
||||
{
|
||||
"reindex_entity": _reindex_entity,
|
||||
"resolve_relations": _resolve_relations,
|
||||
"sync_entity_vectors": _sync_entity_vectors,
|
||||
"sync_project": _sync_project,
|
||||
"reindex_project": _reindex_project,
|
||||
|
||||
@@ -4,6 +4,8 @@ import fnmatch
|
||||
from pathlib import Path
|
||||
from typing import Set
|
||||
|
||||
from basic_memory.config import resolve_data_dir
|
||||
|
||||
|
||||
# Common directories and patterns to ignore by default
|
||||
# These are used as fallback if .bmignore doesn't exist
|
||||
@@ -61,9 +63,11 @@ def get_bmignore_path() -> Path:
|
||||
"""Get path to .bmignore file.
|
||||
|
||||
Returns:
|
||||
Path to ~/.basic-memory/.bmignore
|
||||
Path to <basic-memory data dir>/.bmignore, honoring
|
||||
``BASIC_MEMORY_CONFIG_DIR`` so isolated instances each keep their
|
||||
own ignore file.
|
||||
"""
|
||||
return Path.home() / ".basic-memory" / ".bmignore"
|
||||
return resolve_data_dir() / ".bmignore"
|
||||
|
||||
|
||||
def create_default_bmignore() -> None:
|
||||
@@ -176,7 +180,8 @@ def load_gitignore_patterns(base_path: Path, use_gitignore: bool = True) -> Set[
|
||||
"""Load gitignore patterns from .gitignore file and .bmignore.
|
||||
|
||||
Combines patterns from:
|
||||
1. ~/.basic-memory/.bmignore (user's global ignore patterns)
|
||||
1. <basic-memory data dir>/.bmignore (user's global ignore patterns, honors
|
||||
BASIC_MEMORY_CONFIG_DIR)
|
||||
2. {base_path}/.gitignore (project-specific patterns, if use_gitignore=True)
|
||||
|
||||
Args:
|
||||
|
||||
@@ -39,23 +39,24 @@ def format_timestamp(timestamp: Any) -> str: # pragma: no cover
|
||||
Returns:
|
||||
A formatted string representation of the timestamp.
|
||||
"""
|
||||
parsed_timestamp = timestamp
|
||||
if isinstance(timestamp, str):
|
||||
try:
|
||||
# Try ISO format
|
||||
timestamp = datetime.fromisoformat(timestamp.replace("Z", "+00:00"))
|
||||
parsed_timestamp = datetime.fromisoformat(timestamp.replace("Z", "+00:00"))
|
||||
except ValueError:
|
||||
try:
|
||||
# Try unix timestamp as string
|
||||
timestamp = datetime.fromtimestamp(float(timestamp)).astimezone()
|
||||
parsed_timestamp = datetime.fromtimestamp(float(timestamp)).astimezone()
|
||||
except ValueError:
|
||||
# Return as is if we can't parse it
|
||||
return timestamp
|
||||
elif isinstance(timestamp, (int, float)):
|
||||
# Unix timestamp
|
||||
timestamp = datetime.fromtimestamp(timestamp).astimezone()
|
||||
parsed_timestamp = datetime.fromtimestamp(timestamp).astimezone()
|
||||
|
||||
if isinstance(timestamp, datetime):
|
||||
return timestamp.strftime("%Y-%m-%d %H:%M:%S")
|
||||
if isinstance(parsed_timestamp, datetime):
|
||||
return parsed_timestamp.strftime("%Y-%m-%d %H:%M:%S")
|
||||
|
||||
# Return as is if we can't format it
|
||||
return str(timestamp) # pragma: no cover
|
||||
return str(parsed_timestamp) # pragma: no cover
|
||||
|
||||
@@ -12,6 +12,7 @@ from basic_memory.indexing.models import (
|
||||
IndexingBatchResult,
|
||||
IndexInputFile,
|
||||
IndexProgress,
|
||||
SyncedMarkdownFile,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
@@ -25,5 +26,6 @@ __all__ = [
|
||||
"IndexingBatchResult",
|
||||
"IndexInputFile",
|
||||
"IndexProgress",
|
||||
"SyncedMarkdownFile",
|
||||
"build_index_batches",
|
||||
]
|
||||
|
||||
@@ -11,8 +11,9 @@ from typing import Awaitable, Callable, Mapping, TypeVar
|
||||
from loguru import logger
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
|
||||
import logfire
|
||||
from basic_memory.config import BasicMemoryConfig
|
||||
from basic_memory.file_utils import compute_checksum, has_frontmatter
|
||||
from basic_memory.file_utils import compute_checksum, has_frontmatter, remove_frontmatter
|
||||
from basic_memory.markdown.schemas import EntityMarkdown
|
||||
from basic_memory.indexing.models import (
|
||||
IndexedEntity,
|
||||
@@ -43,12 +44,19 @@ class _PreparedMarkdownFile:
|
||||
class _PreparedEntity:
|
||||
path: str
|
||||
entity_id: int
|
||||
permalink: str | None
|
||||
checksum: str
|
||||
content_type: str | None
|
||||
search_content: str | None
|
||||
markdown_content: str | None = None
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class _PersistedMarkdownFile:
|
||||
prepared: _PreparedMarkdownFile
|
||||
entity: Entity
|
||||
|
||||
|
||||
class BatchIndexer:
|
||||
"""Index already-loaded files without assuming where they came from."""
|
||||
|
||||
@@ -118,6 +126,9 @@ class BatchIndexer:
|
||||
)
|
||||
error_by_path.update(markdown_errors)
|
||||
prepared_entities.update(markdown_upserts)
|
||||
if existing_permalink_by_path is not None:
|
||||
for path, prepared_entity in markdown_upserts.items():
|
||||
existing_permalink_by_path[path] = prepared_entity.permalink
|
||||
|
||||
regular_upserts, regular_errors = await self._run_bounded(
|
||||
regular_paths,
|
||||
@@ -168,6 +179,76 @@ class BatchIndexer:
|
||||
search_indexed=search_indexed,
|
||||
)
|
||||
|
||||
async def index_markdown_file(
|
||||
self,
|
||||
file: IndexInputFile,
|
||||
*,
|
||||
new: bool | None = None,
|
||||
existing_permalink_by_path: dict[str, str | None] | None = None,
|
||||
index_search: bool = True,
|
||||
resolve_relations: bool = True,
|
||||
) -> IndexedEntity:
|
||||
"""Index one markdown file using the same normalization and upsert path as batches."""
|
||||
if not self._is_markdown(file):
|
||||
raise ValueError(f"index_markdown_file requires markdown input: {file.path}")
|
||||
|
||||
with logfire.span("index.markdown_file.prepare", path=file.path):
|
||||
prepared = await self._prepare_markdown_file(file)
|
||||
if existing_permalink_by_path is None:
|
||||
with logfire.span("index.markdown_file.load_permalink_map", path=file.path):
|
||||
existing_permalink_by_path = {
|
||||
path: permalink
|
||||
for path, permalink in (
|
||||
await self.entity_repository.get_file_path_to_permalink_map()
|
||||
).items()
|
||||
}
|
||||
|
||||
reserved_permalinks = {
|
||||
permalink
|
||||
for path, permalink in existing_permalink_by_path.items()
|
||||
if path != file.path and permalink
|
||||
}
|
||||
with logfire.span("index.markdown_file.normalize", path=file.path):
|
||||
prepared = await self._normalize_markdown_file(prepared, reserved_permalinks)
|
||||
existing_permalink_by_path[file.path] = prepared.markdown.frontmatter.permalink
|
||||
|
||||
with logfire.span("index.markdown_file.persist", path=file.path, is_new=new):
|
||||
persisted = await self._persist_markdown_file(
|
||||
prepared,
|
||||
is_new=new,
|
||||
resolve_relations=resolve_relations,
|
||||
reload_entity=False,
|
||||
)
|
||||
existing_permalink_by_path[file.path] = persisted.entity.permalink
|
||||
|
||||
with logfire.span(
|
||||
"index.markdown_file.reload_entity",
|
||||
path=file.path,
|
||||
entity_id=persisted.entity.id,
|
||||
):
|
||||
refreshed = await self.entity_repository.find_by_ids([persisted.entity.id])
|
||||
if len(refreshed) != 1: # pragma: no cover
|
||||
raise ValueError(f"Failed to reload indexed entity for {file.path}")
|
||||
entity = refreshed[0]
|
||||
prepared_entity = self._build_prepared_entity(persisted.prepared, entity)
|
||||
|
||||
if index_search:
|
||||
with logfire.span(
|
||||
"index.markdown_file.refresh_search_index",
|
||||
path=file.path,
|
||||
entity_id=entity.id,
|
||||
):
|
||||
return await self._refresh_search_index(prepared_entity, entity)
|
||||
|
||||
return IndexedEntity(
|
||||
path=prepared_entity.path,
|
||||
entity_id=entity.id,
|
||||
permalink=entity.permalink,
|
||||
checksum=prepared_entity.checksum,
|
||||
content_type=prepared_entity.content_type,
|
||||
markdown_content=prepared_entity.markdown_content,
|
||||
)
|
||||
|
||||
# --- Preparation ---
|
||||
|
||||
async def _prepare_markdown_file(self, file: IndexInputFile) -> _PreparedMarkdownFile:
|
||||
@@ -320,34 +401,8 @@ class BatchIndexer:
|
||||
# --- Persistence ---
|
||||
|
||||
async def _upsert_markdown_file(self, prepared: _PreparedMarkdownFile) -> _PreparedEntity:
|
||||
existing = await self.entity_repository.get_by_file_path(
|
||||
prepared.file.path,
|
||||
load_relations=False,
|
||||
)
|
||||
entity = await self.entity_service.upsert_entity_from_markdown(
|
||||
Path(prepared.file.path),
|
||||
prepared.markdown,
|
||||
is_new=existing is None,
|
||||
)
|
||||
updated = await self.entity_repository.update(
|
||||
entity.id,
|
||||
self._entity_metadata_updates(prepared.file, prepared.final_checksum),
|
||||
)
|
||||
if updated is None:
|
||||
raise ValueError(f"Failed to update markdown entity metadata for {prepared.file.path}")
|
||||
|
||||
return _PreparedEntity(
|
||||
path=prepared.file.path,
|
||||
entity_id=updated.id,
|
||||
checksum=prepared.final_checksum,
|
||||
content_type=prepared.file.content_type,
|
||||
search_content=(
|
||||
prepared.markdown.content
|
||||
if prepared.markdown.content is not None
|
||||
else prepared.content
|
||||
),
|
||||
markdown_content=prepared.content,
|
||||
)
|
||||
persisted = await self._persist_markdown_file(prepared)
|
||||
return self._build_prepared_entity(persisted.prepared, persisted.entity)
|
||||
|
||||
async def _upsert_regular_file(self, file: IndexInputFile) -> _PreparedEntity:
|
||||
checksum = await self._resolve_checksum(file)
|
||||
@@ -405,6 +460,7 @@ class BatchIndexer:
|
||||
return _PreparedEntity(
|
||||
path=file.path,
|
||||
entity_id=updated.id,
|
||||
permalink=updated.permalink,
|
||||
checksum=checksum,
|
||||
content_type=file.content_type,
|
||||
search_content=None,
|
||||
@@ -495,6 +551,99 @@ class BatchIndexer:
|
||||
|
||||
# --- Helpers ---
|
||||
|
||||
async def _persist_markdown_file(
|
||||
self,
|
||||
prepared: _PreparedMarkdownFile,
|
||||
*,
|
||||
is_new: bool | None = None,
|
||||
resolve_relations: bool = True,
|
||||
reload_entity: bool = True,
|
||||
) -> _PersistedMarkdownFile:
|
||||
existing = await self.entity_repository.get_by_file_path(
|
||||
prepared.file.path,
|
||||
load_relations=False,
|
||||
)
|
||||
if is_new is None:
|
||||
is_new = existing is None
|
||||
entity = await self.entity_service.upsert_entity_from_markdown(
|
||||
Path(prepared.file.path),
|
||||
prepared.markdown,
|
||||
is_new=is_new,
|
||||
existing_entity=existing,
|
||||
resolve_relations=resolve_relations,
|
||||
reload_entity=reload_entity,
|
||||
)
|
||||
prepared = await self._reconcile_persisted_permalink(prepared, entity)
|
||||
metadata_updates = self._entity_metadata_updates(prepared.file, prepared.final_checksum)
|
||||
updated = await self.entity_repository.update_fields(
|
||||
entity.id,
|
||||
metadata_updates,
|
||||
)
|
||||
if not updated:
|
||||
raise ValueError(f"Failed to update markdown entity metadata for {prepared.file.path}")
|
||||
self._apply_entity_metadata_updates(entity, metadata_updates)
|
||||
return _PersistedMarkdownFile(prepared=prepared, entity=entity)
|
||||
|
||||
async def _reconcile_persisted_permalink(
|
||||
self,
|
||||
prepared: _PreparedMarkdownFile,
|
||||
entity: Entity,
|
||||
) -> _PreparedMarkdownFile:
|
||||
# Trigger: the source file started without frontmatter and sync is configured
|
||||
# to leave frontmatterless files alone.
|
||||
# Why: upsert may still assign a DB permalink even when disk content should stay untouched.
|
||||
# Outcome: skip reconciliation writes that would silently inject frontmatter.
|
||||
if (
|
||||
self.app_config.disable_permalinks
|
||||
or (
|
||||
not prepared.file_contains_frontmatter
|
||||
and not self.app_config.ensure_frontmatter_on_sync
|
||||
)
|
||||
or entity.permalink is None
|
||||
or entity.permalink == prepared.markdown.frontmatter.permalink
|
||||
):
|
||||
return prepared
|
||||
|
||||
logger.debug(
|
||||
"Updating permalink after upsert conflict resolution",
|
||||
path=prepared.file.path,
|
||||
old_permalink=prepared.markdown.frontmatter.permalink,
|
||||
new_permalink=entity.permalink,
|
||||
)
|
||||
prepared.markdown.frontmatter.metadata["permalink"] = entity.permalink
|
||||
write_result = await self.file_writer.write_frontmatter(
|
||||
IndexFrontmatterUpdate(
|
||||
path=prepared.file.path,
|
||||
metadata={"permalink": entity.permalink},
|
||||
)
|
||||
)
|
||||
return _PreparedMarkdownFile(
|
||||
file=prepared.file,
|
||||
content=write_result.content,
|
||||
final_checksum=write_result.checksum,
|
||||
markdown=prepared.markdown,
|
||||
file_contains_frontmatter=prepared.file_contains_frontmatter,
|
||||
)
|
||||
|
||||
def _build_prepared_entity(
|
||||
self,
|
||||
prepared: _PreparedMarkdownFile,
|
||||
entity: Entity,
|
||||
) -> _PreparedEntity:
|
||||
return _PreparedEntity(
|
||||
path=prepared.file.path,
|
||||
entity_id=entity.id,
|
||||
permalink=entity.permalink,
|
||||
checksum=prepared.final_checksum,
|
||||
content_type=prepared.file.content_type,
|
||||
search_content=(
|
||||
prepared.markdown.content
|
||||
if prepared.markdown.content is not None
|
||||
else remove_frontmatter(prepared.content)
|
||||
),
|
||||
markdown_content=prepared.content,
|
||||
)
|
||||
|
||||
async def _resolve_checksum(self, file: IndexInputFile) -> str:
|
||||
if file.checksum is not None:
|
||||
return file.checksum
|
||||
@@ -523,6 +672,11 @@ class BatchIndexer:
|
||||
updates["content_type"] = file.content_type
|
||||
return updates
|
||||
|
||||
def _apply_entity_metadata_updates(self, entity: Entity, updates: dict[str, object]) -> None:
|
||||
"""Keep the returned entity aligned with metadata written without reload."""
|
||||
for key, value in updates.items():
|
||||
setattr(entity, key, value)
|
||||
|
||||
def _is_markdown(self, file: IndexInputFile) -> bool:
|
||||
if file.content_type is not None:
|
||||
return file.content_type == "text/markdown"
|
||||
|
||||
@@ -4,7 +4,10 @@ from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime
|
||||
from typing import Any, Protocol
|
||||
from typing import Any, Protocol, TYPE_CHECKING
|
||||
|
||||
if TYPE_CHECKING: # pragma: no cover
|
||||
from basic_memory.models import Entity
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
@@ -75,6 +78,19 @@ class IndexedEntity:
|
||||
markdown_content: str | None = None
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class SyncedMarkdownFile:
|
||||
"""Canonical result for syncing one markdown file end-to-end."""
|
||||
|
||||
entity: Entity
|
||||
checksum: str
|
||||
markdown_content: str
|
||||
file_path: str
|
||||
content_type: str
|
||||
updated_at: datetime
|
||||
size: int
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class IndexingBatchResult:
|
||||
"""Outcome for one batch execution."""
|
||||
|
||||
@@ -180,6 +180,9 @@ def observation_plugin(md: MarkdownIt) -> None:
|
||||
def observation_rule(state: Any) -> None:
|
||||
"""Process observations in token stream."""
|
||||
tokens = state.tokens
|
||||
# Track blockquote nesting so Obsidian callouts (`> [!info] Title`)
|
||||
# don't get parsed as observations with category `!info`.
|
||||
blockquote_depth = 0
|
||||
|
||||
for idx in range(len(tokens)):
|
||||
token = tokens[idx]
|
||||
@@ -187,6 +190,18 @@ def observation_plugin(md: MarkdownIt) -> None:
|
||||
# Initialize meta for all tokens
|
||||
token.meta = token.meta or {}
|
||||
|
||||
if token.type == "blockquote_open":
|
||||
blockquote_depth += 1
|
||||
continue
|
||||
if token.type == "blockquote_close":
|
||||
blockquote_depth -= 1
|
||||
continue
|
||||
|
||||
# Skip parsing inside blockquotes — that's Obsidian callout
|
||||
# territory, not Basic Memory observation syntax.
|
||||
if blockquote_depth > 0:
|
||||
continue
|
||||
|
||||
# Parse observations in list items
|
||||
if token.type == "inline" and is_observation(token):
|
||||
obs = parse_observation(token)
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
"""Schema models for entity markdown files."""
|
||||
|
||||
from datetime import datetime
|
||||
from typing import List, Optional
|
||||
from typing import TYPE_CHECKING, Any, List, Optional
|
||||
|
||||
from pydantic import BaseModel
|
||||
from pydantic import BaseModel, Field, model_validator
|
||||
|
||||
|
||||
class Observation(BaseModel):
|
||||
@@ -38,23 +38,47 @@ class Relation(BaseModel):
|
||||
class EntityFrontmatter(BaseModel):
|
||||
"""Required frontmatter fields for an entity."""
|
||||
|
||||
metadata: dict = {}
|
||||
if TYPE_CHECKING:
|
||||
# Frontmatter may be built from raw YAML keys. The validator below
|
||||
# gathers those keys into the metadata mapping used at runtime.
|
||||
def __init__(self, **data: Any) -> None: ...
|
||||
|
||||
metadata: dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
@model_validator(mode="before")
|
||||
@classmethod
|
||||
def collect_metadata(cls, data: Any) -> Any:
|
||||
if not isinstance(data, dict):
|
||||
return data
|
||||
|
||||
if "metadata" not in data:
|
||||
return {"metadata": data}
|
||||
|
||||
metadata = data.get("metadata") or {}
|
||||
extras = {key: value for key, value in data.items() if key != "metadata"}
|
||||
if extras:
|
||||
return {"metadata": {**extras, **metadata}}
|
||||
return data
|
||||
|
||||
@property
|
||||
def tags(self) -> List[str]:
|
||||
return self.metadata.get("tags") if self.metadata else None # pyright: ignore
|
||||
tags = self.metadata.get("tags")
|
||||
return [str(tag) for tag in tags] if isinstance(tags, list) else []
|
||||
|
||||
@property
|
||||
def title(self) -> str:
|
||||
return self.metadata.get("title") if self.metadata else None # pyright: ignore
|
||||
title = self.metadata.get("title")
|
||||
return title if isinstance(title, str) else ""
|
||||
|
||||
@property
|
||||
def type(self) -> str:
|
||||
return self.metadata.get("type", "note") if self.metadata else "note" # pyright: ignore
|
||||
note_type = self.metadata.get("type", "note")
|
||||
return note_type if isinstance(note_type, str) else "note"
|
||||
|
||||
@property
|
||||
def permalink(self) -> str:
|
||||
return self.metadata.get("permalink") if self.metadata else None # pyright: ignore
|
||||
def permalink(self) -> Optional[str]:
|
||||
permalink = self.metadata.get("permalink")
|
||||
return permalink if isinstance(permalink, str) else None
|
||||
|
||||
|
||||
class EntityMarkdown(BaseModel):
|
||||
|
||||
@@ -5,7 +5,7 @@ from typing import AsyncIterator, Callable, Optional
|
||||
from httpx import ASGITransport, AsyncClient, Timeout
|
||||
from loguru import logger
|
||||
|
||||
from basic_memory import telemetry
|
||||
import logfire
|
||||
from basic_memory.api.app import app as fastapi_app
|
||||
from basic_memory.config import ConfigManager, ProjectMode
|
||||
|
||||
@@ -44,7 +44,7 @@ def _asgi_client(timeout: Timeout) -> AsyncClient:
|
||||
|
||||
async def _resolve_cloud_token(config) -> str:
|
||||
"""Resolve cloud token with API key preferred, OAuth fallback."""
|
||||
with telemetry.span(
|
||||
with logfire.span(
|
||||
"routing.resolve_cloud_credentials",
|
||||
has_api_key=bool(config.cloud_api_key),
|
||||
):
|
||||
@@ -94,9 +94,12 @@ async def _cloud_client(
|
||||
workspace: Optional[str] = None,
|
||||
) -> AsyncIterator[AsyncClient]:
|
||||
"""Create a cloud proxy client with resolved credentials."""
|
||||
from basic_memory.workspace_context import workspace_permalink_headers
|
||||
|
||||
token = await _resolve_cloud_token(config)
|
||||
proxy_base_url = f"{config.cloud_host}/proxy"
|
||||
headers = {"Authorization": f"Bearer {token}"}
|
||||
headers.update(workspace_permalink_headers())
|
||||
if workspace:
|
||||
headers["X-Workspace-ID"] = workspace
|
||||
logger.info(f"Creating HTTP client for cloud proxy at: {proxy_base_url}")
|
||||
|
||||
@@ -7,7 +7,7 @@ from typing import Any
|
||||
|
||||
from httpx import AsyncClient
|
||||
|
||||
from basic_memory import telemetry
|
||||
import logfire
|
||||
from basic_memory.mcp.tools.utils import call_get, call_post, call_put, call_patch, call_delete
|
||||
from basic_memory.schemas.response import (
|
||||
EntityResponse,
|
||||
@@ -44,9 +44,7 @@ class KnowledgeClient:
|
||||
|
||||
# --- Entity CRUD Operations ---
|
||||
|
||||
async def create_entity(
|
||||
self, entity_data: dict[str, Any], *, fast: bool | None = None
|
||||
) -> EntityResponse:
|
||||
async def create_entity(self, entity_data: dict[str, Any]) -> EntityResponse:
|
||||
"""Create a new entity.
|
||||
|
||||
Args:
|
||||
@@ -58,18 +56,15 @@ class KnowledgeClient:
|
||||
Raises:
|
||||
ToolError: If the request fails
|
||||
"""
|
||||
params = {"fast": fast} if fast is not None else None
|
||||
with telemetry.scope(
|
||||
with logfire.span(
|
||||
"mcp.client.knowledge.create_entity",
|
||||
client_name="knowledge",
|
||||
operation="create_entity",
|
||||
fast=fast,
|
||||
):
|
||||
response = await call_post(
|
||||
self.http_client,
|
||||
f"{self._base_path}/entities",
|
||||
json=entity_data,
|
||||
params=params,
|
||||
client_name="knowledge",
|
||||
operation="create_entity",
|
||||
path_template="/v2/projects/{project_id}/knowledge/entities",
|
||||
@@ -80,8 +75,6 @@ class KnowledgeClient:
|
||||
self,
|
||||
entity_id: str,
|
||||
entity_data: dict[str, Any],
|
||||
*,
|
||||
fast: bool | None = None,
|
||||
) -> EntityResponse:
|
||||
"""Update an existing entity (full replacement).
|
||||
|
||||
@@ -95,18 +88,15 @@ class KnowledgeClient:
|
||||
Raises:
|
||||
ToolError: If the request fails
|
||||
"""
|
||||
params = {"fast": fast} if fast is not None else None
|
||||
with telemetry.scope(
|
||||
with logfire.span(
|
||||
"mcp.client.knowledge.update_entity",
|
||||
client_name="knowledge",
|
||||
operation="update_entity",
|
||||
fast=fast,
|
||||
):
|
||||
response = await call_put(
|
||||
self.http_client,
|
||||
f"{self._base_path}/entities/{entity_id}",
|
||||
json=entity_data,
|
||||
params=params,
|
||||
client_name="knowledge",
|
||||
operation="update_entity",
|
||||
path_template="/v2/projects/{project_id}/knowledge/entities/{entity_id}",
|
||||
@@ -125,7 +115,7 @@ class KnowledgeClient:
|
||||
Raises:
|
||||
ToolError: If the entity is not found or request fails
|
||||
"""
|
||||
with telemetry.scope(
|
||||
with logfire.span(
|
||||
"mcp.client.knowledge.get_entity",
|
||||
client_name="knowledge",
|
||||
operation="get_entity",
|
||||
@@ -143,8 +133,6 @@ class KnowledgeClient:
|
||||
self,
|
||||
entity_id: str,
|
||||
patch_data: dict[str, Any],
|
||||
*,
|
||||
fast: bool | None = None,
|
||||
) -> EntityResponse:
|
||||
"""Partially update an entity.
|
||||
|
||||
@@ -158,18 +146,15 @@ class KnowledgeClient:
|
||||
Raises:
|
||||
ToolError: If the request fails
|
||||
"""
|
||||
params = {"fast": fast} if fast is not None else None
|
||||
with telemetry.scope(
|
||||
with logfire.span(
|
||||
"mcp.client.knowledge.patch_entity",
|
||||
client_name="knowledge",
|
||||
operation="patch_entity",
|
||||
fast=fast,
|
||||
):
|
||||
response = await call_patch(
|
||||
self.http_client,
|
||||
f"{self._base_path}/entities/{entity_id}",
|
||||
json=patch_data,
|
||||
params=params,
|
||||
client_name="knowledge",
|
||||
operation="patch_entity",
|
||||
path_template="/v2/projects/{project_id}/knowledge/entities/{entity_id}",
|
||||
@@ -188,7 +173,7 @@ class KnowledgeClient:
|
||||
Raises:
|
||||
ToolError: If the entity is not found or request fails
|
||||
"""
|
||||
with telemetry.scope(
|
||||
with logfire.span(
|
||||
"mcp.client.knowledge.delete_entity",
|
||||
client_name="knowledge",
|
||||
operation="delete_entity",
|
||||
@@ -215,7 +200,7 @@ class KnowledgeClient:
|
||||
Raises:
|
||||
ToolError: If the request fails
|
||||
"""
|
||||
with telemetry.scope(
|
||||
with logfire.span(
|
||||
"mcp.client.knowledge.move_entity",
|
||||
client_name="knowledge",
|
||||
operation="move_entity",
|
||||
@@ -245,7 +230,7 @@ class KnowledgeClient:
|
||||
Raises:
|
||||
ToolError: If the request fails
|
||||
"""
|
||||
with telemetry.scope(
|
||||
with logfire.span(
|
||||
"mcp.client.knowledge.move_directory",
|
||||
client_name="knowledge",
|
||||
operation="move_directory",
|
||||
@@ -275,7 +260,7 @@ class KnowledgeClient:
|
||||
Raises:
|
||||
ToolError: If the request fails
|
||||
"""
|
||||
with telemetry.scope(
|
||||
with logfire.span(
|
||||
"mcp.client.knowledge.delete_directory",
|
||||
client_name="knowledge",
|
||||
operation="delete_directory",
|
||||
@@ -305,7 +290,7 @@ class KnowledgeClient:
|
||||
Raises:
|
||||
ToolError: If the identifier cannot be resolved
|
||||
"""
|
||||
with telemetry.scope(
|
||||
with logfire.span(
|
||||
"mcp.client.knowledge.resolve_entity",
|
||||
client_name="knowledge",
|
||||
operation="resolve_entity",
|
||||
|
||||
@@ -7,7 +7,7 @@ from typing import Optional
|
||||
|
||||
from httpx import AsyncClient
|
||||
|
||||
from basic_memory import telemetry
|
||||
import logfire
|
||||
from basic_memory.mcp.tools.utils import call_get
|
||||
from basic_memory.schemas.memory import GraphContext
|
||||
|
||||
@@ -72,7 +72,7 @@ class MemoryClient:
|
||||
if timeframe:
|
||||
params["timeframe"] = timeframe
|
||||
|
||||
with telemetry.scope(
|
||||
with logfire.span(
|
||||
"mcp.client.memory.build_context",
|
||||
client_name="memory",
|
||||
operation="build_context",
|
||||
@@ -123,7 +123,7 @@ class MemoryClient:
|
||||
# Join types as comma-separated string if provided
|
||||
params["type"] = ",".join(types) if isinstance(types, list) else types
|
||||
|
||||
with telemetry.scope(
|
||||
with logfire.span(
|
||||
"mcp.client.memory.recent_activity",
|
||||
client_name="memory",
|
||||
operation="recent_activity",
|
||||
|
||||
@@ -3,11 +3,9 @@
|
||||
Encapsulates all /v2/projects/{project_id}/resource/* endpoints.
|
||||
"""
|
||||
|
||||
from typing import Optional
|
||||
|
||||
from httpx import AsyncClient, Response
|
||||
|
||||
from basic_memory import telemetry
|
||||
import logfire
|
||||
from basic_memory.mcp.tools.utils import call_get
|
||||
|
||||
|
||||
@@ -39,19 +37,11 @@ class ResourceClient:
|
||||
self.project_id = project_id
|
||||
self._base_path = f"/v2/projects/{project_id}/resource"
|
||||
|
||||
async def read(
|
||||
self,
|
||||
entity_id: str,
|
||||
*,
|
||||
page: Optional[int] = None,
|
||||
page_size: Optional[int] = None,
|
||||
) -> Response:
|
||||
async def read(self, entity_id: str) -> Response:
|
||||
"""Read a resource by entity ID.
|
||||
|
||||
Args:
|
||||
entity_id: Entity external_id (UUID)
|
||||
page: Optional page number for paginated content
|
||||
page_size: Optional page size for paginated content
|
||||
|
||||
Returns:
|
||||
Raw HTTP Response (caller handles text/binary content)
|
||||
@@ -59,23 +49,14 @@ class ResourceClient:
|
||||
Raises:
|
||||
ToolError: If the resource is not found or request fails
|
||||
"""
|
||||
params: dict = {}
|
||||
if page is not None:
|
||||
params["page"] = page
|
||||
if page_size is not None:
|
||||
params["page_size"] = page_size
|
||||
|
||||
with telemetry.scope(
|
||||
with logfire.span(
|
||||
"mcp.client.resource.read",
|
||||
client_name="resource",
|
||||
operation="read",
|
||||
page=page,
|
||||
page_size=page_size,
|
||||
):
|
||||
return await call_get(
|
||||
self.http_client,
|
||||
f"{self._base_path}/{entity_id}",
|
||||
params=params if params else None,
|
||||
client_name="resource",
|
||||
operation="read",
|
||||
path_template="/v2/projects/{project_id}/resource/{entity_id}",
|
||||
|
||||
@@ -7,7 +7,7 @@ from typing import Any
|
||||
|
||||
from httpx import AsyncClient
|
||||
|
||||
from basic_memory import telemetry
|
||||
import logfire
|
||||
from basic_memory.mcp.tools.utils import call_post
|
||||
from basic_memory.schemas.search import SearchResponse
|
||||
|
||||
@@ -57,7 +57,7 @@ class SearchClient:
|
||||
Raises:
|
||||
ToolError: If the request fails
|
||||
"""
|
||||
with telemetry.scope(
|
||||
with logfire.span(
|
||||
"mcp.client.search.search",
|
||||
client_name="search",
|
||||
operation="search",
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -95,8 +95,8 @@ def format_prompt_context(context: PromptContext) -> str:
|
||||
sections = []
|
||||
|
||||
# Process each context
|
||||
for context in context.results: # pyright: ignore
|
||||
for primary in context.primary_results: # pyright: ignore
|
||||
for context_item in context.results:
|
||||
for primary in context_item.primary_results:
|
||||
if primary.permalink not in added_permalinks:
|
||||
primary_permalink = primary.permalink
|
||||
|
||||
@@ -121,8 +121,8 @@ def format_prompt_context(context: PromptContext) -> str:
|
||||
section += f"- **Created**: {primary.created_at.strftime('%Y-%m-%d %H:%M')}\n"
|
||||
|
||||
# Add content snippet
|
||||
if hasattr(primary, "content") and primary.content: # pyright: ignore
|
||||
content = primary.content or "" # pyright: ignore # pragma: no cover
|
||||
if hasattr(primary, "content") and primary.content:
|
||||
content = primary.content or "" # pragma: no cover
|
||||
if content: # pragma: no cover
|
||||
section += f"\n**Excerpt**:\n{content}\n" # pragma: no cover
|
||||
|
||||
@@ -132,14 +132,14 @@ def format_prompt_context(context: PromptContext) -> str:
|
||||
""")
|
||||
sections.append(section)
|
||||
|
||||
if context.related_results: # pyright: ignore
|
||||
section += dedent( # pyright: ignore
|
||||
if context_item.related_results:
|
||||
section += dedent(
|
||||
"""
|
||||
## Related Context
|
||||
"""
|
||||
)
|
||||
|
||||
for related in context.related_results: # pyright: ignore
|
||||
for related in context_item.related_results:
|
||||
section_content = dedent(f"""
|
||||
- type: **{related.type}**
|
||||
- title: {related.title}
|
||||
|
||||
@@ -18,13 +18,15 @@ Basic Memory creates a semantic knowledge graph from markdown files. Focus on bu
|
||||
|
||||
**Resolution priority:**
|
||||
1. CLI constraint: `BASIC_MEMORY_MCP_PROJECT` env var (highest priority)
|
||||
2. Explicit parameter: `project="name"` in tool calls
|
||||
2. Explicit parameter: `project_id="<uuid>"` (preferred when known) or `project="name"` in tool calls
|
||||
3. Default project: `default_project` in config (fallback)
|
||||
|
||||
**`project` vs `project_id`:** Every project has a stable `external_id` (UUID) returned by `list_memory_projects()`. Pass it as `project_id=...` to address a project unambiguously — required when the same project name exists in multiple cloud workspaces. For local single-project setups, the `project` name is fine.
|
||||
|
||||
### Quick Setup Check
|
||||
|
||||
```python
|
||||
# Discover projects
|
||||
# Discover projects (each entry includes external_id you can pass as project_id)
|
||||
projects = await list_memory_projects()
|
||||
```
|
||||
|
||||
@@ -169,6 +171,14 @@ await write_note(
|
||||
**Multi-project users:**
|
||||
- Always specify project explicitly in tool calls
|
||||
|
||||
**Cloud multi-workspace users:** project names can collide across workspaces. After calling `list_memory_projects()`, prefer the project's `external_id` via `project_id=...` for any subsequent tool calls — it routes to the exact project regardless of name collisions. The `project` name parameter falls back to the default workspace on ambiguity, which may not be what you want.
|
||||
|
||||
```python
|
||||
# Cloud / multi-workspace: prefer project_id (UUID) once you've discovered it
|
||||
projects = await list_memory_projects()
|
||||
results = await search_notes(query="auth", project_id=projects[0]["external_id"])
|
||||
```
|
||||
|
||||
**Discovery:**
|
||||
```python
|
||||
# Start with discovery
|
||||
|
||||
@@ -15,7 +15,7 @@ from basic_memory.cli.auth import CLIAuth
|
||||
from basic_memory.db import scoped_session
|
||||
from basic_memory.mcp.container import McpContainer, set_container
|
||||
from basic_memory.services.initialization import initialize_app
|
||||
from basic_memory import telemetry
|
||||
import logfire
|
||||
|
||||
|
||||
async def _log_embedding_status(session_maker: async_sessionmaker[AsyncSession]) -> None:
|
||||
@@ -63,7 +63,7 @@ async def lifespan(app: FastMCP):
|
||||
set_container(container)
|
||||
|
||||
config = container.config
|
||||
with telemetry.operation(
|
||||
with logfire.span(
|
||||
"mcp.lifecycle.startup",
|
||||
entrypoint="mcp",
|
||||
mode=container.mode.name.lower(),
|
||||
@@ -124,7 +124,7 @@ async def lifespan(app: FastMCP):
|
||||
yield
|
||||
finally:
|
||||
# Shutdown - coordinator handles clean task cancellation
|
||||
with telemetry.operation(
|
||||
with logfire.span(
|
||||
"mcp.lifecycle.shutdown",
|
||||
entrypoint="mcp",
|
||||
mode=container.mode.name.lower(),
|
||||
|
||||
@@ -1,14 +1,15 @@
|
||||
"""Build context tool for Basic Memory MCP server."""
|
||||
|
||||
from typing import Optional, Literal
|
||||
from typing import Annotated, Optional, Literal
|
||||
|
||||
import logfire
|
||||
from loguru import logger
|
||||
from fastmcp import Context
|
||||
from pydantic import AliasChoices, Field
|
||||
|
||||
from basic_memory.config import ConfigManager
|
||||
from basic_memory import telemetry
|
||||
from basic_memory.mcp.project_context import (
|
||||
detect_project_from_url_prefix,
|
||||
detect_project_from_memory_url_prefix,
|
||||
get_project_client,
|
||||
resolve_project_and_path,
|
||||
)
|
||||
@@ -133,14 +134,34 @@ def _format_context_markdown(graph: GraphContext, project: str) -> str:
|
||||
annotations={"readOnlyHint": True, "openWorldHint": False},
|
||||
)
|
||||
async def build_context(
|
||||
url: MemoryUrl,
|
||||
url: Annotated[
|
||||
MemoryUrl,
|
||||
Field(validation_alias=AliasChoices("url", "uri", "memory_url")),
|
||||
],
|
||||
project: Optional[str] = None,
|
||||
workspace: Optional[str] = None,
|
||||
project_id: Optional[str] = None,
|
||||
depth: str | int | None = 1,
|
||||
timeframe: Optional[TimeFrame] = "7d",
|
||||
page: int = 1,
|
||||
page_size: int = 10,
|
||||
max_related: int = 10,
|
||||
timeframe: Annotated[
|
||||
Optional[TimeFrame],
|
||||
Field(
|
||||
default="7d",
|
||||
validation_alias=AliasChoices("timeframe", "since", "time_range", "lookback"),
|
||||
),
|
||||
] = "7d",
|
||||
# `offset` is intentionally NOT aliased: it has different semantics
|
||||
# (item-indexed vs. 1-indexed page-number).
|
||||
page: Annotated[
|
||||
int,
|
||||
Field(default=1, validation_alias=AliasChoices("page", "page_number")),
|
||||
] = 1,
|
||||
page_size: Annotated[
|
||||
int,
|
||||
Field(default=10, validation_alias=AliasChoices("page_size", "limit", "per_page")),
|
||||
] = 10,
|
||||
max_related: Annotated[
|
||||
int,
|
||||
Field(default=10, validation_alias=AliasChoices("max_related", "max_results")),
|
||||
] = 10,
|
||||
output_format: Literal["json", "text"] = "json",
|
||||
context: Context | None = None,
|
||||
) -> dict | str:
|
||||
@@ -158,6 +179,9 @@ async def build_context(
|
||||
Args:
|
||||
project: Project name to build context from. Optional - server will resolve using hierarchy.
|
||||
If unknown, use list_memory_projects() to discover available projects.
|
||||
project_id: Project external_id (UUID). Prefer this over `project` when known —
|
||||
it routes to the exact project regardless of name collisions across cloud
|
||||
workspaces. Takes precedence over `project`. Get from list_memory_projects().
|
||||
url: memory:// URI pointing to discussion content (e.g. memory://specs/search)
|
||||
depth: How many relation hops to traverse (1-3 recommended for performance)
|
||||
timeframe: How far back to look. Supports natural language like "2 days ago", "last week"
|
||||
@@ -185,9 +209,14 @@ async def build_context(
|
||||
Raises:
|
||||
ToolError: If project doesn't exist or depth parameter is invalid
|
||||
"""
|
||||
# Detect project from memory URL prefix before routing
|
||||
if project is None:
|
||||
detected = detect_project_from_url_prefix(url, ConfigManager().config)
|
||||
# Detect project from memory URL prefix before routing.
|
||||
# project_id routes by external UUID, so it bypasses URL discovery entirely.
|
||||
if project is None and project_id is None:
|
||||
detected = await detect_project_from_memory_url_prefix(
|
||||
url,
|
||||
ConfigManager().config,
|
||||
context=context,
|
||||
)
|
||||
if detected:
|
||||
project = detected
|
||||
|
||||
@@ -202,12 +231,12 @@ async def build_context(
|
||||
|
||||
# URL is already validated and normalized by MemoryUrl type annotation
|
||||
|
||||
with telemetry.operation(
|
||||
with logfire.span(
|
||||
"mcp.tool.build_context",
|
||||
entrypoint="mcp",
|
||||
tool_name="build_context",
|
||||
requested_project=project,
|
||||
workspace_id=workspace,
|
||||
requested_project_id=project_id,
|
||||
depth=depth or 1,
|
||||
timeframe=timeframe,
|
||||
page=page,
|
||||
@@ -216,48 +245,46 @@ async def build_context(
|
||||
output_format=output_format,
|
||||
is_memory_url=str(url).startswith("memory://"),
|
||||
):
|
||||
async with get_project_client(project, workspace, context) as (client, active_project):
|
||||
with telemetry.contextualize(
|
||||
project_name=active_project.name,
|
||||
workspace_id=workspace,
|
||||
tool_name="build_context",
|
||||
):
|
||||
logger.info(
|
||||
f"MCP tool call tool=build_context project={active_project.name} "
|
||||
f"url={url} depth={depth} timeframe={timeframe} output_format={output_format}"
|
||||
)
|
||||
async with get_project_client(project, context=context, project_id=project_id) as (
|
||||
client,
|
||||
active_project,
|
||||
):
|
||||
logger.info(
|
||||
f"MCP tool call tool=build_context project={active_project.name} "
|
||||
f"url={url} depth={depth} timeframe={timeframe} output_format={output_format}"
|
||||
)
|
||||
|
||||
# Resolve memory:// identifier with project-prefix awareness
|
||||
_, resolved_path, _ = await resolve_project_and_path(
|
||||
client,
|
||||
url,
|
||||
active_project.name,
|
||||
context,
|
||||
)
|
||||
# Resolve memory:// identifier with project-prefix awareness
|
||||
_, resolved_path, _ = await resolve_project_and_path(
|
||||
client,
|
||||
url,
|
||||
active_project.name,
|
||||
context,
|
||||
)
|
||||
|
||||
# Import here to avoid circular import
|
||||
from basic_memory.mcp.clients import MemoryClient
|
||||
# Import here to avoid circular import
|
||||
from basic_memory.mcp.clients import MemoryClient
|
||||
|
||||
# Use typed MemoryClient for API calls
|
||||
memory_client = MemoryClient(client, active_project.external_id)
|
||||
graph = await memory_client.build_context(
|
||||
resolved_path,
|
||||
depth=depth or 1,
|
||||
timeframe=timeframe,
|
||||
page=page,
|
||||
page_size=page_size,
|
||||
max_related=max_related,
|
||||
)
|
||||
# Use typed MemoryClient for API calls
|
||||
memory_client = MemoryClient(client, active_project.external_id)
|
||||
graph = await memory_client.build_context(
|
||||
resolved_path,
|
||||
depth=depth or 1,
|
||||
timeframe=timeframe,
|
||||
page=page,
|
||||
page_size=page_size,
|
||||
max_related=max_related,
|
||||
)
|
||||
|
||||
logger.info(
|
||||
f"MCP tool response: tool=build_context project={active_project.name} "
|
||||
f"uri={graph.metadata.uri or resolved_path} "
|
||||
f"primary_count={graph.metadata.primary_count or 0} "
|
||||
f"related_count={graph.metadata.related_count or 0} "
|
||||
f"output_format={output_format}"
|
||||
)
|
||||
logger.info(
|
||||
f"MCP tool response: tool=build_context project={active_project.name} "
|
||||
f"uri={graph.metadata.uri or resolved_path} "
|
||||
f"primary_count={graph.metadata.primary_count or 0} "
|
||||
f"related_count={graph.metadata.related_count or 0} "
|
||||
f"output_format={output_format}"
|
||||
)
|
||||
|
||||
if output_format == "text":
|
||||
return _format_context_markdown(graph, active_project.name)
|
||||
if output_format == "text":
|
||||
return _format_context_markdown(graph, active_project.name)
|
||||
|
||||
return graph.model_dump()
|
||||
return graph.model_dump()
|
||||
|
||||
@@ -8,7 +8,7 @@ from typing import Annotated, Dict, List, Any, Optional
|
||||
|
||||
from loguru import logger
|
||||
from fastmcp import Context
|
||||
from pydantic import BeforeValidator
|
||||
from pydantic import AliasChoices, BeforeValidator, Field
|
||||
|
||||
from basic_memory.mcp.project_context import get_project_client
|
||||
from basic_memory.utils import coerce_list
|
||||
@@ -24,9 +24,12 @@ async def canvas(
|
||||
nodes: Annotated[List[Dict[str, Any]], BeforeValidator(coerce_list)],
|
||||
edges: Annotated[List[Dict[str, Any]], BeforeValidator(coerce_list)],
|
||||
title: str,
|
||||
directory: str,
|
||||
directory: Annotated[
|
||||
str,
|
||||
Field(validation_alias=AliasChoices("directory", "folder", "dir", "path")),
|
||||
],
|
||||
project: Optional[str] = None,
|
||||
workspace: Optional[str] = None,
|
||||
project_id: Optional[str] = None,
|
||||
context: Context | None = None,
|
||||
) -> str:
|
||||
"""Create an Obsidian canvas file with the provided nodes and edges.
|
||||
@@ -43,6 +46,9 @@ async def canvas(
|
||||
Args:
|
||||
project: Project name to create canvas in. Optional - server will resolve using hierarchy.
|
||||
If unknown, use list_memory_projects() to discover available projects.
|
||||
project_id: Project external_id (UUID). Prefer this over `project` when known —
|
||||
it routes to the exact project regardless of name collisions across cloud
|
||||
workspaces. Takes precedence over `project`. Get from list_memory_projects().
|
||||
nodes: List of node objects following JSON Canvas 1.0 spec
|
||||
edges: List of edge objects following JSON Canvas 1.0 spec
|
||||
title: The title of the canvas (will be saved as title.canvas)
|
||||
@@ -97,7 +103,10 @@ async def canvas(
|
||||
Raises:
|
||||
ToolError: If project doesn't exist or directory path is invalid
|
||||
"""
|
||||
async with get_project_client(project, workspace, context) as (client, active_project):
|
||||
async with get_project_client(project, context=context, project_id=project_id) as (
|
||||
client,
|
||||
active_project,
|
||||
):
|
||||
# Ensure path has .canvas extension
|
||||
file_title = title if title.endswith(".canvas") else f"{title}.canvas"
|
||||
file_path = f"{directory}/{file_title}"
|
||||
|
||||
@@ -181,8 +181,6 @@ async def fetch(
|
||||
content = str(
|
||||
await read_note(
|
||||
identifier=id,
|
||||
page=1,
|
||||
page_size=10,
|
||||
context=context,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -1,13 +1,21 @@
|
||||
from textwrap import dedent
|
||||
from typing import Optional, Literal
|
||||
from typing import Annotated, Optional, Literal
|
||||
|
||||
from loguru import logger
|
||||
from fastmcp import Context
|
||||
from mcp.server.fastmcp.exceptions import ToolError
|
||||
from pydantic import AliasChoices, Field
|
||||
|
||||
from basic_memory.config import ConfigManager
|
||||
from basic_memory.mcp.project_context import detect_project_from_url_prefix, get_project_client
|
||||
from basic_memory.mcp.project_context import (
|
||||
detect_project_from_memory_url_prefix,
|
||||
get_project_client,
|
||||
resolve_project_and_path,
|
||||
)
|
||||
from basic_memory.mcp.server import mcp
|
||||
from basic_memory.schemas.project_info import ProjectItem
|
||||
from basic_memory.utils import generate_permalink, normalize_project_reference
|
||||
from basic_memory.workspace_context import current_workspace_permalink_context
|
||||
|
||||
|
||||
def _format_delete_error_response(project: str, error_message: str, identifier: str) -> str:
|
||||
@@ -147,15 +155,44 @@ delete_note("{project}", "correct-identifier-from-search")
|
||||
If the note should be deleted but the operation keeps failing, send a message to support@basicmemory.com."""
|
||||
|
||||
|
||||
def _directory_path_for_delete(
|
||||
target_identifier: str,
|
||||
active_project: ProjectItem,
|
||||
*,
|
||||
include_project_prefix: bool,
|
||||
) -> str:
|
||||
"""Return the project-relative directory path expected by the delete API."""
|
||||
directory = normalize_project_reference(target_identifier).strip("/")
|
||||
project_permalink = active_project.permalink
|
||||
|
||||
route_prefixes: list[str] = []
|
||||
workspace_context = current_workspace_permalink_context()
|
||||
if workspace_context and workspace_context.should_prefix_permalinks:
|
||||
route_prefixes.append(
|
||||
f"{generate_permalink(workspace_context.workspace_slug)}/{project_permalink}"
|
||||
)
|
||||
if include_project_prefix:
|
||||
route_prefixes.append(project_permalink)
|
||||
|
||||
for route_prefix in route_prefixes:
|
||||
if directory.startswith(f"{route_prefix}/"):
|
||||
return directory.removeprefix(f"{route_prefix}/")
|
||||
|
||||
return directory
|
||||
|
||||
|
||||
@mcp.tool(
|
||||
description="Delete a note or directory by title, permalink, or path",
|
||||
annotations={"destructiveHint": True, "openWorldHint": False},
|
||||
)
|
||||
async def delete_note(
|
||||
identifier: str,
|
||||
is_directory: bool = False,
|
||||
is_directory: Annotated[
|
||||
bool,
|
||||
Field(default=False, validation_alias=AliasChoices("is_directory", "is_dir")),
|
||||
] = False,
|
||||
project: Optional[str] = None,
|
||||
workspace: Optional[str] = None,
|
||||
project_id: Optional[str] = None,
|
||||
output_format: Literal["text", "json"] = "text",
|
||||
context: Context | None = None,
|
||||
) -> bool | str | dict:
|
||||
@@ -179,6 +216,9 @@ async def delete_note(
|
||||
(without file extensions). Defaults to False.
|
||||
project: Project name to delete from. Optional - server will resolve using hierarchy.
|
||||
If unknown, use list_memory_projects() to discover available projects.
|
||||
project_id: Project external_id (UUID). Prefer this over `project` when known —
|
||||
it routes to the exact project regardless of name collisions across cloud
|
||||
workspaces. Takes precedence over `project`. Get from list_memory_projects().
|
||||
output_format: "text" preserves existing behavior (bool/string). "json"
|
||||
returns machine-readable deletion metadata.
|
||||
context: Optional FastMCP context for performance caching.
|
||||
@@ -224,16 +264,23 @@ async def delete_note(
|
||||
commands and alternative formats to try.
|
||||
"""
|
||||
# Detect project from memory URL prefix before routing
|
||||
# Trigger: identifier starts with memory:// and no explicit project was provided
|
||||
# Trigger: identifier starts with memory:// and no explicit project/project_id was provided
|
||||
# Why: only gate on memory:// to avoid misrouting plain paths like "research/note"
|
||||
# where "research" is a directory, not a project name
|
||||
# Outcome: project is set from the URL prefix, routing goes to the correct project
|
||||
if project is None and identifier.strip().startswith("memory://"):
|
||||
detected = detect_project_from_url_prefix(identifier, ConfigManager().config)
|
||||
if project is None and project_id is None and identifier.strip().startswith("memory://"):
|
||||
detected = await detect_project_from_memory_url_prefix(
|
||||
identifier,
|
||||
ConfigManager().config,
|
||||
context=context,
|
||||
)
|
||||
if detected:
|
||||
project = detected
|
||||
|
||||
async with get_project_client(project, workspace, context) as (client, active_project):
|
||||
async with get_project_client(project, context=context, project_id=project_id) as (
|
||||
client,
|
||||
active_project,
|
||||
):
|
||||
logger.debug(
|
||||
f"Deleting {'directory' if is_directory else 'note'}: {identifier} in project: {active_project.name}"
|
||||
)
|
||||
@@ -243,11 +290,30 @@ async def delete_note(
|
||||
|
||||
# Use typed KnowledgeClient for API calls
|
||||
knowledge_client = KnowledgeClient(client, active_project.external_id)
|
||||
_, target_identifier, is_memory_url = await resolve_project_and_path(
|
||||
client,
|
||||
identifier,
|
||||
active_project.name,
|
||||
context,
|
||||
)
|
||||
|
||||
# Handle directory deletes
|
||||
if is_directory:
|
||||
try:
|
||||
result = await knowledge_client.delete_directory(identifier)
|
||||
# Trigger: directory input was routed from a memory:// URL.
|
||||
# Why: resolve_project_and_path returns canonical permalinks, while
|
||||
# delete_directory filters by project-relative file_path prefixes.
|
||||
# Outcome: strip only the route prefix before calling the delete API.
|
||||
directory_identifier = (
|
||||
_directory_path_for_delete(
|
||||
target_identifier,
|
||||
active_project,
|
||||
include_project_prefix=ConfigManager().config.permalinks_include_project,
|
||||
)
|
||||
if is_memory_url
|
||||
else target_identifier
|
||||
)
|
||||
result = await knowledge_client.delete_directory(directory_identifier)
|
||||
if output_format == "json":
|
||||
return {
|
||||
"deleted": result.failed_deletes == 0,
|
||||
@@ -329,7 +395,7 @@ delete_note("path/to/file.md")
|
||||
note_file_path = None
|
||||
try:
|
||||
# Resolve identifier to entity ID
|
||||
entity_id = await knowledge_client.resolve_entity(identifier, strict=True)
|
||||
entity_id = await knowledge_client.resolve_entity(target_identifier, strict=True)
|
||||
if output_format == "json":
|
||||
entity = await knowledge_client.get_entity(entity_id)
|
||||
note_title = entity.title
|
||||
|
||||
@@ -1,16 +1,18 @@
|
||||
"""Edit note tool for Basic Memory MCP server."""
|
||||
|
||||
from typing import Optional, Literal
|
||||
from typing import Annotated, Optional, Literal
|
||||
|
||||
import logfire
|
||||
from loguru import logger
|
||||
from fastmcp import Context
|
||||
from pydantic import AliasChoices, Field
|
||||
|
||||
from basic_memory.config import ConfigManager
|
||||
from basic_memory import telemetry
|
||||
from basic_memory.mcp.project_context import (
|
||||
detect_project_from_url_prefix,
|
||||
detect_project_from_memory_url_prefix,
|
||||
get_project_client,
|
||||
add_project_metadata,
|
||||
resolve_project_and_path,
|
||||
)
|
||||
from basic_memory.mcp.server import mcp
|
||||
from basic_memory.schemas.base import Entity
|
||||
@@ -170,11 +172,33 @@ Error editing note '{identifier}': {error_message}
|
||||
async def edit_note(
|
||||
identifier: str,
|
||||
operation: str,
|
||||
content: str,
|
||||
# Accept common replacement-content aliases. Models trained on diff/patch
|
||||
# APIs reach for new_content/replacement/replace_with on first try.
|
||||
content: Annotated[
|
||||
str,
|
||||
Field(
|
||||
validation_alias=AliasChoices("content", "new_content", "replacement", "replace_with")
|
||||
),
|
||||
],
|
||||
project: Optional[str] = None,
|
||||
workspace: Optional[str] = None,
|
||||
section: Optional[str] = None,
|
||||
find_text: Optional[str] = None,
|
||||
project_id: Optional[str] = None,
|
||||
# Section/heading naming varies across tools; accept the descriptive forms.
|
||||
section: Annotated[
|
||||
Optional[str],
|
||||
Field(
|
||||
default=None,
|
||||
validation_alias=AliasChoices("section", "section_heading", "heading"),
|
||||
),
|
||||
] = None,
|
||||
# find_text is the highest-frequency miss per the issue: models reach for
|
||||
# find/old_text/old_content/search before find_text every time.
|
||||
find_text: Annotated[
|
||||
Optional[str],
|
||||
Field(
|
||||
default=None,
|
||||
validation_alias=AliasChoices("find_text", "find", "old_text", "old_content", "search"),
|
||||
),
|
||||
] = None,
|
||||
expected_replacements: Optional[int] = None,
|
||||
output_format: Literal["text", "json"] = "text",
|
||||
context: Context | None = None,
|
||||
@@ -201,6 +225,9 @@ async def edit_note(
|
||||
content: The content to add or use for replacement
|
||||
project: Project name to edit in. Optional - server will resolve using hierarchy.
|
||||
If unknown, use list_memory_projects() to discover available projects.
|
||||
project_id: Project external_id (UUID). Prefer this over `project` when known —
|
||||
it routes to the exact project regardless of name collisions across cloud
|
||||
workspaces. Takes precedence over `project`. Get from list_memory_projects().
|
||||
section: For replace_section operation - the markdown header to replace content under (e.g., "## Notes", "### Implementation")
|
||||
find_text: For find_replace operation - the text to find and replace
|
||||
expected_replacements: For find_replace operation - the expected number of replacements (validation will fail if actual doesn't match)
|
||||
@@ -262,252 +289,255 @@ async def edit_note(
|
||||
effective_replacements = expected_replacements if expected_replacements is not None else 1
|
||||
|
||||
# Detect project from memory URL prefix before routing
|
||||
# Trigger: identifier starts with memory:// and no explicit project was provided
|
||||
# Trigger: identifier starts with memory:// and no explicit project/project_id was provided
|
||||
# Why: only gate on memory:// to avoid misrouting plain paths like "research/note"
|
||||
# where "research" is a directory, not a project name
|
||||
# Outcome: project is set from the URL prefix, routing goes to the correct project
|
||||
if project is None and identifier.strip().startswith("memory://"):
|
||||
detected = detect_project_from_url_prefix(identifier, ConfigManager().config)
|
||||
if project is None and project_id is None and identifier.strip().startswith("memory://"):
|
||||
detected = await detect_project_from_memory_url_prefix(
|
||||
identifier,
|
||||
ConfigManager().config,
|
||||
context=context,
|
||||
)
|
||||
if detected:
|
||||
project = detected
|
||||
|
||||
with telemetry.operation(
|
||||
with logfire.span(
|
||||
"mcp.tool.edit_note",
|
||||
entrypoint="mcp",
|
||||
tool_name="edit_note",
|
||||
requested_project=project,
|
||||
workspace_id=workspace,
|
||||
requested_project_id=project_id,
|
||||
edit_operation=operation,
|
||||
output_format=output_format,
|
||||
has_section=bool(section),
|
||||
has_find_text=bool(find_text),
|
||||
expected_replacements=effective_replacements,
|
||||
):
|
||||
async with get_project_client(project, workspace, context) as (client, active_project):
|
||||
with telemetry.contextualize(
|
||||
project_name=active_project.name,
|
||||
workspace_id=workspace,
|
||||
tool_name="edit_note",
|
||||
):
|
||||
logger.info(
|
||||
f"MCP tool call tool=edit_note project={active_project.name} "
|
||||
f"identifier={identifier} operation={operation} output_format={output_format}"
|
||||
async with get_project_client(project, context=context, project_id=project_id) as (
|
||||
client,
|
||||
active_project,
|
||||
):
|
||||
logger.info(
|
||||
f"MCP tool call tool=edit_note project={active_project.name} "
|
||||
f"identifier={identifier} operation={operation} output_format={output_format}"
|
||||
)
|
||||
|
||||
# Validate operation
|
||||
valid_operations = [
|
||||
"append",
|
||||
"prepend",
|
||||
"find_replace",
|
||||
"replace_section",
|
||||
"insert_before_section",
|
||||
"insert_after_section",
|
||||
]
|
||||
if operation not in valid_operations:
|
||||
raise ValueError(
|
||||
f"Invalid operation '{operation}'. Must be one of: {', '.join(valid_operations)}"
|
||||
)
|
||||
|
||||
# Validate operation
|
||||
valid_operations = [
|
||||
"append",
|
||||
"prepend",
|
||||
"find_replace",
|
||||
"replace_section",
|
||||
"insert_before_section",
|
||||
"insert_after_section",
|
||||
]
|
||||
if operation not in valid_operations:
|
||||
raise ValueError(
|
||||
f"Invalid operation '{operation}'. Must be one of: {', '.join(valid_operations)}"
|
||||
)
|
||||
# Validate required parameters for specific operations
|
||||
if operation == "find_replace" and not find_text:
|
||||
raise ValueError("find_text parameter is required for find_replace operation")
|
||||
section_ops = ("replace_section", "insert_before_section", "insert_after_section")
|
||||
if operation in section_ops and not section:
|
||||
raise ValueError("section parameter is required for section-based operations")
|
||||
|
||||
# Validate required parameters for specific operations
|
||||
if operation == "find_replace" and not find_text:
|
||||
raise ValueError("find_text parameter is required for find_replace operation")
|
||||
section_ops = ("replace_section", "insert_before_section", "insert_after_section")
|
||||
if operation in section_ops and not section:
|
||||
raise ValueError("section parameter is required for section-based operations")
|
||||
# Use the PATCH endpoint to edit the entity
|
||||
try:
|
||||
# Import here to avoid circular import
|
||||
from basic_memory.mcp.clients import KnowledgeClient
|
||||
|
||||
# Use the PATCH endpoint to edit the entity
|
||||
# Use typed KnowledgeClient for API calls
|
||||
knowledge_client = KnowledgeClient(client, active_project.external_id)
|
||||
_, entity_identifier, _ = await resolve_project_and_path(
|
||||
client,
|
||||
identifier,
|
||||
active_project.name,
|
||||
context,
|
||||
)
|
||||
|
||||
file_created = False
|
||||
entity_id = ""
|
||||
result: EntityResponse | None = None
|
||||
|
||||
# Try to resolve the entity; for append/prepend, create it if not found
|
||||
try:
|
||||
# Import here to avoid circular import
|
||||
from basic_memory.mcp.clients import KnowledgeClient
|
||||
entity_id = await knowledge_client.resolve_entity(
|
||||
entity_identifier,
|
||||
strict=True,
|
||||
)
|
||||
except Exception as resolve_error:
|
||||
# Trigger: entity does not exist yet
|
||||
# Why: append/prepend can meaningfully create a new note from the content,
|
||||
# while find_replace/replace_section require existing content to modify
|
||||
# Outcome: note is created via the same path as write_note
|
||||
error_msg = str(resolve_error).lower()
|
||||
is_not_found = "entity not found" in error_msg or "not found" in error_msg
|
||||
|
||||
# Use typed KnowledgeClient for API calls
|
||||
knowledge_client = KnowledgeClient(client, active_project.external_id)
|
||||
if is_not_found and operation in ("append", "prepend"):
|
||||
title, directory = _parse_identifier_to_title_and_directory(identifier)
|
||||
|
||||
file_created = False
|
||||
entity_id = ""
|
||||
result: EntityResponse | None = None
|
||||
|
||||
# Try to resolve the entity; for append/prepend, create it if not found
|
||||
try:
|
||||
entity_id = await knowledge_client.resolve_entity(identifier, strict=True)
|
||||
except Exception as resolve_error:
|
||||
# Trigger: entity does not exist yet
|
||||
# Why: append/prepend can meaningfully create a new note from the content,
|
||||
# while find_replace/replace_section require existing content to modify
|
||||
# Outcome: note is created via the same path as write_note
|
||||
error_msg = str(resolve_error).lower()
|
||||
is_not_found = "entity not found" in error_msg or "not found" in error_msg
|
||||
|
||||
if is_not_found and operation in ("append", "prepend"):
|
||||
title, directory = _parse_identifier_to_title_and_directory(identifier)
|
||||
|
||||
# Validate directory path (same security check as write_note)
|
||||
project_path = active_project.home
|
||||
if directory and not validate_project_path(directory, project_path):
|
||||
logger.warning(
|
||||
"Attempted path traversal attack blocked",
|
||||
directory=directory,
|
||||
project=active_project.name,
|
||||
)
|
||||
if output_format == "json":
|
||||
return {
|
||||
"title": title,
|
||||
"permalink": None,
|
||||
"file_path": None,
|
||||
"checksum": None,
|
||||
"operation": operation,
|
||||
"fileCreated": False,
|
||||
"error": "SECURITY_VALIDATION_ERROR",
|
||||
}
|
||||
return f"# Error\n\nDirectory path '{directory}' is not allowed - paths must stay within project boundaries"
|
||||
|
||||
entity = Entity(
|
||||
title=title,
|
||||
# Validate directory path (same security check as write_note)
|
||||
project_path = active_project.home
|
||||
if directory and not validate_project_path(directory, project_path):
|
||||
logger.warning(
|
||||
"Attempted path traversal attack blocked",
|
||||
directory=directory,
|
||||
content_type="text/markdown",
|
||||
content=content,
|
||||
project=active_project.name,
|
||||
)
|
||||
if output_format == "json":
|
||||
return {
|
||||
"title": title,
|
||||
"permalink": None,
|
||||
"file_path": None,
|
||||
"checksum": None,
|
||||
"operation": operation,
|
||||
"fileCreated": False,
|
||||
"error": "SECURITY_VALIDATION_ERROR",
|
||||
}
|
||||
return f"# Error\n\nDirectory path '{directory}' is not allowed - paths must stay within project boundaries"
|
||||
|
||||
logger.info(
|
||||
"Creating note via edit_note auto-create",
|
||||
title=title,
|
||||
directory=directory,
|
||||
operation=operation,
|
||||
)
|
||||
result = await knowledge_client.create_entity(
|
||||
entity.model_dump(), fast=False
|
||||
)
|
||||
file_created = True
|
||||
else:
|
||||
# find_replace/replace_section require existing content — re-raise
|
||||
raise resolve_error
|
||||
|
||||
# --- Standard edit path (entity already existed) ---
|
||||
if not file_created:
|
||||
# Prepare the edit request data
|
||||
edit_data = {
|
||||
"operation": operation,
|
||||
"content": content,
|
||||
}
|
||||
|
||||
# Add optional parameters
|
||||
if section:
|
||||
edit_data["section"] = section
|
||||
if find_text:
|
||||
edit_data["find_text"] = find_text
|
||||
if effective_replacements != 1: # Only send if different from default
|
||||
edit_data["expected_replacements"] = str(effective_replacements)
|
||||
|
||||
# Call the PATCH endpoint
|
||||
result = await knowledge_client.patch_entity(
|
||||
entity_id, edit_data, fast=False
|
||||
entity = Entity(
|
||||
title=title,
|
||||
directory=directory,
|
||||
content_type="text/markdown",
|
||||
content=content,
|
||||
)
|
||||
|
||||
# --- Format response ---
|
||||
# result is always set: either by create_entity (auto-create) or patch_entity (edit)
|
||||
assert result is not None
|
||||
if file_created:
|
||||
summary = [
|
||||
f"# Created note ({operation})",
|
||||
f"project: {active_project.name}",
|
||||
f"file_path: {result.file_path}",
|
||||
f"permalink: {result.permalink}",
|
||||
f"checksum: {result.checksum[:8] if result.checksum else 'unknown'}",
|
||||
"fileCreated: true",
|
||||
]
|
||||
lines_added = len(content.split("\n"))
|
||||
summary.append(f"operation: Created note with {lines_added} lines")
|
||||
logger.info(
|
||||
"Creating note via edit_note auto-create",
|
||||
title=title,
|
||||
directory=directory,
|
||||
operation=operation,
|
||||
)
|
||||
result = await knowledge_client.create_entity(entity.model_dump())
|
||||
file_created = True
|
||||
else:
|
||||
summary = [
|
||||
f"# Edited note ({operation})",
|
||||
f"project: {active_project.name}",
|
||||
f"file_path: {result.file_path}",
|
||||
f"permalink: {result.permalink}",
|
||||
f"checksum: {result.checksum[:8] if result.checksum else 'unknown'}",
|
||||
]
|
||||
# find_replace/replace_section require existing content — re-raise
|
||||
raise resolve_error
|
||||
|
||||
# Add operation-specific details
|
||||
if operation == "append":
|
||||
lines_added = len(content.split("\n"))
|
||||
summary.append(f"operation: Added {lines_added} lines to end of note")
|
||||
elif operation == "prepend":
|
||||
lines_added = len(content.split("\n"))
|
||||
summary.append(
|
||||
f"operation: Added {lines_added} lines to beginning of note"
|
||||
)
|
||||
elif operation == "find_replace":
|
||||
# For find_replace, we can't easily count replacements from here
|
||||
# since we don't have the original content, but the server handled it
|
||||
summary.append("operation: Find and replace operation completed")
|
||||
elif operation == "replace_section":
|
||||
summary.append(f"operation: Replaced content under section '{section}'")
|
||||
elif operation == "insert_before_section":
|
||||
summary.append(
|
||||
f"operation: Inserted content before section '{section}'"
|
||||
)
|
||||
elif operation == "insert_after_section":
|
||||
summary.append(f"operation: Inserted content after section '{section}'")
|
||||
# --- Standard edit path (entity already existed) ---
|
||||
if not file_created:
|
||||
# Prepare the edit request data
|
||||
edit_data = {
|
||||
"operation": operation,
|
||||
"content": content,
|
||||
}
|
||||
|
||||
# Count observations by category (reuse logic from write_note)
|
||||
categories = {}
|
||||
if result.observations:
|
||||
for obs in result.observations:
|
||||
categories[obs.category] = categories.get(obs.category, 0) + 1
|
||||
# Add optional parameters
|
||||
if section:
|
||||
edit_data["section"] = section
|
||||
if find_text:
|
||||
edit_data["find_text"] = find_text
|
||||
if effective_replacements != 1: # Only send if different from default
|
||||
edit_data["expected_replacements"] = str(effective_replacements)
|
||||
|
||||
summary.append("\n## Observations")
|
||||
for category, count in sorted(categories.items()):
|
||||
summary.append(f"- {category}: {count}")
|
||||
# Call the PATCH endpoint
|
||||
result = await knowledge_client.patch_entity(entity_id, edit_data)
|
||||
|
||||
# Count resolved/unresolved relations
|
||||
unresolved = 0
|
||||
resolved = 0
|
||||
if result.relations:
|
||||
unresolved = sum(1 for r in result.relations if not r.to_id)
|
||||
resolved = len(result.relations) - unresolved
|
||||
# --- Format response ---
|
||||
# result is always set: either by create_entity (auto-create) or patch_entity (edit)
|
||||
assert result is not None
|
||||
if file_created:
|
||||
summary = [
|
||||
f"# Created note ({operation})",
|
||||
f"project: {active_project.name}",
|
||||
f"file_path: {result.file_path}",
|
||||
f"permalink: {result.permalink}",
|
||||
f"checksum: {result.checksum[:8] if result.checksum else 'unknown'}",
|
||||
"fileCreated: true",
|
||||
]
|
||||
lines_added = len(content.split("\n"))
|
||||
summary.append(f"operation: Created note with {lines_added} lines")
|
||||
else:
|
||||
summary = [
|
||||
f"# Edited note ({operation})",
|
||||
f"project: {active_project.name}",
|
||||
f"file_path: {result.file_path}",
|
||||
f"permalink: {result.permalink}",
|
||||
f"checksum: {result.checksum[:8] if result.checksum else 'unknown'}",
|
||||
]
|
||||
|
||||
summary.append("\n## Relations")
|
||||
summary.append(f"- Resolved: {resolved}")
|
||||
if unresolved:
|
||||
summary.append(f"- Unresolved: {unresolved}")
|
||||
# Add operation-specific details
|
||||
if operation == "append":
|
||||
lines_added = len(content.split("\n"))
|
||||
summary.append(f"operation: Added {lines_added} lines to end of note")
|
||||
elif operation == "prepend":
|
||||
lines_added = len(content.split("\n"))
|
||||
summary.append(f"operation: Added {lines_added} lines to beginning of note")
|
||||
elif operation == "find_replace":
|
||||
# For find_replace, we can't easily count replacements from here
|
||||
# since we don't have the original content, but the server handled it
|
||||
summary.append("operation: Find and replace operation completed")
|
||||
elif operation == "replace_section":
|
||||
summary.append(f"operation: Replaced content under section '{section}'")
|
||||
elif operation == "insert_before_section":
|
||||
summary.append(f"operation: Inserted content before section '{section}'")
|
||||
elif operation == "insert_after_section":
|
||||
summary.append(f"operation: Inserted content after section '{section}'")
|
||||
|
||||
logger.info(
|
||||
f"MCP tool response: tool=edit_note project={active_project.name} "
|
||||
f"operation={operation} permalink={result.permalink} "
|
||||
f"observations_count={len(result.observations)} "
|
||||
f"relations_count={len(result.relations)} "
|
||||
f"file_created={str(file_created).lower()}"
|
||||
)
|
||||
# Count observations by category (reuse logic from write_note)
|
||||
categories = {}
|
||||
if result.observations:
|
||||
for obs in result.observations:
|
||||
categories[obs.category] = categories.get(obs.category, 0) + 1
|
||||
|
||||
if output_format == "json":
|
||||
return {
|
||||
"title": result.title,
|
||||
"permalink": result.permalink,
|
||||
"file_path": result.file_path,
|
||||
"checksum": result.checksum,
|
||||
"operation": operation,
|
||||
"fileCreated": file_created,
|
||||
}
|
||||
summary.append("\n## Observations")
|
||||
for category, count in sorted(categories.items()):
|
||||
summary.append(f"- {category}: {count}")
|
||||
|
||||
summary_result = "\n".join(summary)
|
||||
return add_project_metadata(summary_result, active_project.name)
|
||||
# Count resolved/unresolved relations
|
||||
unresolved = 0
|
||||
resolved = 0
|
||||
if result.relations:
|
||||
unresolved = sum(1 for r in result.relations if not r.to_id)
|
||||
resolved = len(result.relations) - unresolved
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error editing note: {e}")
|
||||
if output_format == "json":
|
||||
return {
|
||||
"title": None,
|
||||
"permalink": None,
|
||||
"file_path": None,
|
||||
"checksum": None,
|
||||
"operation": operation,
|
||||
"fileCreated": False,
|
||||
"error": str(e),
|
||||
}
|
||||
return _format_error_response(
|
||||
str(e),
|
||||
operation,
|
||||
identifier,
|
||||
find_text,
|
||||
effective_replacements,
|
||||
active_project.name,
|
||||
)
|
||||
summary.append("\n## Relations")
|
||||
summary.append(f"- Resolved: {resolved}")
|
||||
if unresolved:
|
||||
summary.append(f"- Unresolved: {unresolved}")
|
||||
|
||||
logger.info(
|
||||
f"MCP tool response: tool=edit_note project={active_project.name} "
|
||||
f"operation={operation} permalink={result.permalink} "
|
||||
f"observations_count={len(result.observations)} "
|
||||
f"relations_count={len(result.relations)} "
|
||||
f"file_created={str(file_created).lower()}"
|
||||
)
|
||||
|
||||
if output_format == "json":
|
||||
return {
|
||||
"title": result.title,
|
||||
"permalink": result.permalink,
|
||||
"file_path": result.file_path,
|
||||
"checksum": result.checksum,
|
||||
"operation": operation,
|
||||
"fileCreated": file_created,
|
||||
}
|
||||
|
||||
summary_result = "\n".join(summary)
|
||||
return add_project_metadata(summary_result, active_project.name)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error editing note: {e}")
|
||||
if output_format == "json":
|
||||
return {
|
||||
"title": None,
|
||||
"permalink": None,
|
||||
"file_path": None,
|
||||
"checksum": None,
|
||||
"operation": operation,
|
||||
"fileCreated": False,
|
||||
"error": str(e),
|
||||
}
|
||||
return _format_error_response(
|
||||
str(e),
|
||||
operation,
|
||||
identifier,
|
||||
find_text,
|
||||
effective_replacements,
|
||||
active_project.name,
|
||||
)
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
"""List directory tool for Basic Memory MCP server."""
|
||||
|
||||
from typing import Optional
|
||||
from typing import Annotated, Optional
|
||||
|
||||
from loguru import logger
|
||||
from fastmcp import Context
|
||||
from pydantic import AliasChoices, Field
|
||||
|
||||
from basic_memory.mcp.project_context import get_project_client
|
||||
from basic_memory.mcp.server import mcp
|
||||
@@ -14,11 +15,24 @@ from basic_memory.mcp.server import mcp
|
||||
annotations={"readOnlyHint": True, "openWorldHint": False},
|
||||
)
|
||||
async def list_directory(
|
||||
dir_name: str = "/",
|
||||
# `dir_name` is unusual; models reach for directory/folder/path/dir.
|
||||
dir_name: Annotated[
|
||||
str,
|
||||
Field(
|
||||
default="/",
|
||||
validation_alias=AliasChoices("dir_name", "directory", "folder", "path", "dir"),
|
||||
),
|
||||
] = "/",
|
||||
depth: int = 1,
|
||||
file_name_glob: Optional[str] = None,
|
||||
file_name_glob: Annotated[
|
||||
Optional[str],
|
||||
Field(
|
||||
default=None,
|
||||
validation_alias=AliasChoices("file_name_glob", "glob", "pattern", "filter"),
|
||||
),
|
||||
] = None,
|
||||
project: Optional[str] = None,
|
||||
workspace: Optional[str] = None,
|
||||
project_id: Optional[str] = None,
|
||||
context: Context | None = None,
|
||||
) -> str:
|
||||
"""List directory contents from the knowledge base with optional filtering.
|
||||
@@ -36,6 +50,9 @@ async def list_directory(
|
||||
Examples: "*.md", "*meeting*", "project_*"
|
||||
project: Project name to list directory from. Optional - server will resolve using hierarchy.
|
||||
If unknown, use list_memory_projects() to discover available projects.
|
||||
project_id: Project external_id (UUID). Prefer this over `project` when known —
|
||||
it routes to the exact project regardless of name collisions across cloud
|
||||
workspaces. Takes precedence over `project`. Get from list_memory_projects().
|
||||
context: Optional FastMCP context for performance caching.
|
||||
|
||||
Returns:
|
||||
@@ -63,7 +80,10 @@ async def list_directory(
|
||||
Raises:
|
||||
ToolError: If project doesn't exist or directory path is invalid
|
||||
"""
|
||||
async with get_project_client(project, workspace, context) as (client, active_project):
|
||||
async with get_project_client(project, context=context, project_id=project_id) as (
|
||||
client,
|
||||
active_project,
|
||||
):
|
||||
logger.debug(
|
||||
f"Listing directory '{dir_name}' in project {project} with depth={depth}, glob='{file_name_glob}'"
|
||||
)
|
||||
|
||||
@@ -2,11 +2,12 @@
|
||||
|
||||
from pathlib import Path, PureWindowsPath
|
||||
from textwrap import dedent
|
||||
from typing import Optional, Literal
|
||||
from typing import Annotated, Optional, Literal
|
||||
|
||||
from loguru import logger
|
||||
from fastmcp import Context
|
||||
from mcp.server.fastmcp.exceptions import ToolError
|
||||
from pydantic import AliasChoices, Field
|
||||
|
||||
from basic_memory.mcp.server import mcp
|
||||
from basic_memory.mcp.project_context import get_project_client
|
||||
@@ -348,11 +349,29 @@ delete_note("{identifier}")
|
||||
)
|
||||
async def move_note(
|
||||
identifier: str,
|
||||
destination_path: str = "",
|
||||
destination_folder: Optional[str] = None,
|
||||
is_directory: bool = False,
|
||||
# Move/rename APIs across the ecosystem use `to`/`destination`/`new_path`.
|
||||
destination_path: Annotated[
|
||||
str,
|
||||
Field(
|
||||
default="",
|
||||
validation_alias=AliasChoices(
|
||||
"destination_path", "dest_path", "new_path", "to", "destination"
|
||||
),
|
||||
),
|
||||
] = "",
|
||||
destination_folder: Annotated[
|
||||
Optional[str],
|
||||
Field(
|
||||
default=None,
|
||||
validation_alias=AliasChoices("destination_folder", "dest_folder", "to_folder"),
|
||||
),
|
||||
] = None,
|
||||
is_directory: Annotated[
|
||||
bool,
|
||||
Field(default=False, validation_alias=AliasChoices("is_directory", "is_dir")),
|
||||
] = False,
|
||||
project: Optional[str] = None,
|
||||
workspace: Optional[str] = None,
|
||||
project_id: Optional[str] = None,
|
||||
output_format: Literal["text", "json"] = "text",
|
||||
context: Context | None = None,
|
||||
) -> str | dict:
|
||||
@@ -377,6 +396,9 @@ async def move_note(
|
||||
(without file extensions). Defaults to False.
|
||||
project: Project name to move within. Optional - server will resolve using hierarchy.
|
||||
If unknown, use list_memory_projects() to discover available projects.
|
||||
project_id: Project external_id (UUID). Prefer this over `project` when known —
|
||||
it routes to the exact project regardless of name collisions across cloud
|
||||
workspaces. Takes precedence over `project`. Get from list_memory_projects().
|
||||
output_format: "text" returns existing markdown guidance/success text. "json"
|
||||
returns machine-readable move metadata.
|
||||
context: Optional FastMCP context for performance caching.
|
||||
@@ -476,7 +498,10 @@ async def move_note(
|
||||
"error": "DESTINATION_FOLDER_NOT_FOR_DIRECTORIES",
|
||||
}
|
||||
return f"# Move Failed - Invalid Parameters\n\n{error_msg}"
|
||||
async with get_project_client(project, workspace, context) as (client, active_project):
|
||||
async with get_project_client(project, context=context, project_id=project_id) as (
|
||||
client,
|
||||
active_project,
|
||||
):
|
||||
destination_target = destination_folder or destination_path
|
||||
logger.info(
|
||||
f"MCP tool call tool=move_note project={active_project.name} "
|
||||
|
||||
@@ -11,7 +11,17 @@ from fastmcp import Context
|
||||
from loguru import logger
|
||||
|
||||
from basic_memory.config import ConfigManager, has_cloud_credentials
|
||||
from basic_memory.mcp.async_client import get_client, get_cloud_proxy_client, is_factory_mode
|
||||
from basic_memory.mcp.async_client import (
|
||||
_explicit_routing,
|
||||
_force_local_mode,
|
||||
get_client,
|
||||
is_factory_mode,
|
||||
)
|
||||
from basic_memory.mcp.project_context import (
|
||||
WorkspaceProjectEntry,
|
||||
ensure_workspace_project_index,
|
||||
resolve_workspace_parameter,
|
||||
)
|
||||
from basic_memory.mcp.server import mcp
|
||||
from basic_memory.schemas.project_info import ProjectInfoRequest, ProjectItem, ProjectList
|
||||
from basic_memory.utils import generate_permalink
|
||||
@@ -20,30 +30,6 @@ from basic_memory.utils import generate_permalink
|
||||
# --- Helpers for dual-fetch + merge ---
|
||||
|
||||
|
||||
async def _fetch_cloud_projects(
|
||||
workspace: str | None = None,
|
||||
context: Context | None = None,
|
||||
) -> ProjectList | None:
|
||||
"""Fetch projects from the cloud API, returning None on failure.
|
||||
|
||||
Logs warnings on failure so the caller can fall back to local-only results.
|
||||
"""
|
||||
try:
|
||||
from basic_memory.mcp.clients import ProjectClient
|
||||
|
||||
async with get_cloud_proxy_client(workspace=workspace) as cloud_client:
|
||||
cloud_project_client = ProjectClient(cloud_client)
|
||||
cloud_list = await cloud_project_client.list_projects()
|
||||
if context: # pragma: no cover
|
||||
await context.info(f"Discovered {len(cloud_list.projects)} cloud projects")
|
||||
return cloud_list
|
||||
except Exception as exc:
|
||||
logger.warning(f"Cloud project discovery failed: {exc}")
|
||||
if context: # pragma: no cover
|
||||
await context.info("Cloud project discovery failed, showing local projects only")
|
||||
return None
|
||||
|
||||
|
||||
def _merge_projects(
|
||||
local_list: ProjectList | None,
|
||||
cloud_list: ProjectList | None,
|
||||
@@ -51,6 +37,8 @@ def _merge_projects(
|
||||
cloud_workspace_name: str | None = None,
|
||||
cloud_workspace_type: str | None = None,
|
||||
cloud_workspace_tenant_id: str | None = None,
|
||||
cloud_workspace_slug: str | None = None,
|
||||
cloud_workspace_is_default: bool = False,
|
||||
) -> list[dict]:
|
||||
"""Merge local and cloud project lists by permalink.
|
||||
|
||||
@@ -113,9 +101,13 @@ def _merge_projects(
|
||||
ws_type = cloud_workspace_type if cloud_proj else None
|
||||
ws_tenant_id = cloud_workspace_tenant_id if cloud_proj else None
|
||||
|
||||
proj = cloud_proj or local_proj
|
||||
external_id = proj.external_id if proj else None
|
||||
|
||||
merged.append(
|
||||
{
|
||||
"name": name,
|
||||
"external_id": external_id,
|
||||
"path": path,
|
||||
"local_path": local_path,
|
||||
"cloud_path": cloud_path,
|
||||
@@ -126,21 +118,120 @@ def _merge_projects(
|
||||
"workspace_name": ws_name,
|
||||
"workspace_type": ws_type,
|
||||
"workspace_tenant_id": ws_tenant_id,
|
||||
"workspace_slug": cloud_workspace_slug if cloud_proj else None,
|
||||
"workspace_is_default": cloud_workspace_is_default if cloud_proj else False,
|
||||
"qualified_name": (
|
||||
f"{cloud_workspace_slug}/{permalink}"
|
||||
if cloud_proj and cloud_workspace_slug
|
||||
else None
|
||||
),
|
||||
}
|
||||
)
|
||||
|
||||
return merged
|
||||
|
||||
|
||||
def _merge_workspace_projects(
|
||||
local_list: ProjectList | None,
|
||||
cloud_entries: tuple[WorkspaceProjectEntry, ...],
|
||||
) -> list[dict]:
|
||||
"""Merge local projects with cloud projects from every accessible workspace."""
|
||||
local_by_permalink: dict[str, ProjectItem] = {}
|
||||
if local_list:
|
||||
for project in local_list.projects:
|
||||
local_by_permalink[project.permalink] = project
|
||||
|
||||
cloud_permalinks = {entry.project.permalink for entry in cloud_entries}
|
||||
merged: list[dict] = []
|
||||
|
||||
for entry in sorted(
|
||||
cloud_entries,
|
||||
key=lambda item: (
|
||||
not item.workspace.is_default,
|
||||
item.workspace.workspace_type != "personal",
|
||||
item.workspace.name.casefold(),
|
||||
item.project.permalink,
|
||||
),
|
||||
):
|
||||
permalink = entry.project.permalink
|
||||
local_proj = local_by_permalink.get(permalink)
|
||||
cloud_proj = entry.project
|
||||
source = "local+cloud" if local_proj else "cloud"
|
||||
local_path = local_proj.path if local_proj else None
|
||||
cloud_path = cloud_proj.path
|
||||
|
||||
merged.append(
|
||||
{
|
||||
"name": cloud_proj.name,
|
||||
"external_id": cloud_proj.external_id,
|
||||
"path": local_path or cloud_path,
|
||||
"local_path": local_path,
|
||||
"cloud_path": cloud_path,
|
||||
"source": source,
|
||||
"is_default": bool((local_proj and local_proj.is_default) or cloud_proj.is_default),
|
||||
"is_private": cloud_proj.is_private,
|
||||
"display_name": cloud_proj.display_name,
|
||||
"workspace_name": entry.workspace.name,
|
||||
"workspace_type": entry.workspace.workspace_type,
|
||||
"workspace_tenant_id": entry.workspace.tenant_id,
|
||||
"workspace_slug": entry.workspace.slug,
|
||||
"workspace_is_default": entry.workspace.is_default,
|
||||
"qualified_name": entry.qualified_name,
|
||||
}
|
||||
)
|
||||
|
||||
if local_list:
|
||||
for project in sorted(local_list.projects, key=lambda item: item.permalink):
|
||||
if project.permalink in cloud_permalinks:
|
||||
continue
|
||||
merged.append(
|
||||
{
|
||||
"name": project.name,
|
||||
"external_id": project.external_id,
|
||||
"path": project.path,
|
||||
"local_path": project.path,
|
||||
"cloud_path": None,
|
||||
"source": "local",
|
||||
"is_default": project.is_default,
|
||||
"is_private": project.is_private,
|
||||
"display_name": project.display_name,
|
||||
"workspace_name": None,
|
||||
"workspace_type": None,
|
||||
"workspace_tenant_id": None,
|
||||
"workspace_slug": None,
|
||||
"workspace_is_default": False,
|
||||
"qualified_name": None,
|
||||
}
|
||||
)
|
||||
|
||||
return merged
|
||||
|
||||
|
||||
def _format_project_list_text(merged: list[dict]) -> str:
|
||||
"""Format merged project list as human-readable text."""
|
||||
result = "Available projects:\n"
|
||||
|
||||
current_workspace: tuple[str | None, str | None] | None = None
|
||||
for project in merged:
|
||||
workspace_slug = project.get("workspace_slug")
|
||||
workspace_name = project.get("workspace_name")
|
||||
if workspace_slug:
|
||||
workspace_key = (workspace_slug, workspace_name)
|
||||
if workspace_key != current_workspace:
|
||||
default_label = " default" if project.get("workspace_is_default") else ""
|
||||
result += f"\nWorkspace: {workspace_name} ({workspace_slug}{default_label})\n"
|
||||
current_workspace = workspace_key
|
||||
elif current_workspace is not None:
|
||||
result += "\nLocal projects:\n"
|
||||
current_workspace = None
|
||||
|
||||
display_name = project["display_name"]
|
||||
name = project["name"]
|
||||
label = f"{display_name} ({name})" if display_name else name
|
||||
source = project["source"]
|
||||
result += f"• {label} ({source})\n"
|
||||
external_id = project.get("external_id", "")
|
||||
id_suffix = f" [{external_id}]" if external_id else ""
|
||||
result += f"- {label} ({source}){id_suffix}\n"
|
||||
|
||||
result += "\n" + "─" * 40 + "\n"
|
||||
result += "Next: Ask which project to use for this session.\n"
|
||||
@@ -172,7 +263,6 @@ def _format_project_list_json(
|
||||
)
|
||||
async def list_memory_projects(
|
||||
output_format: Literal["text", "json"] = "text",
|
||||
workspace: str | None = None,
|
||||
context: Context | None = None,
|
||||
) -> str | dict:
|
||||
"""List all available projects with their status.
|
||||
@@ -180,11 +270,14 @@ async def list_memory_projects(
|
||||
Shows projects from both local and cloud sources when cloud credentials
|
||||
are available, merging by permalink to give a unified view.
|
||||
|
||||
Each project entry includes an `external_id` (UUID). Pass that value as the
|
||||
`project_id` parameter on other tools to address a specific project
|
||||
unambiguously across cloud workspaces — useful when the same project name
|
||||
exists in more than one workspace.
|
||||
|
||||
Args:
|
||||
output_format: "text" returns the existing human-readable project list.
|
||||
"json" returns structured project metadata.
|
||||
workspace: Cloud workspace name or tenant_id. Falls back to
|
||||
config.default_workspace when not specified.
|
||||
context: Optional FastMCP context for progress/status logging.
|
||||
"""
|
||||
if context: # pragma: no cover
|
||||
@@ -196,18 +289,21 @@ async def list_memory_projects(
|
||||
|
||||
# --- Factory mode (cloud app) ---
|
||||
# Trigger: set_client_factory() was called (e.g., basic-memory-cloud)
|
||||
# Why: there is no local ASGI server; the factory IS the only source
|
||||
# Outcome: single fetch, no merge needed
|
||||
# Why: there is no local ASGI server; the factory IS the cloud source
|
||||
# Outcome: fetch every accessible workspace so callers can discover cross-workspace IDs
|
||||
if is_factory_mode():
|
||||
async with get_client() as client:
|
||||
project_client = ProjectClient(client)
|
||||
project_list = await project_client.list_projects()
|
||||
|
||||
merged = _merge_projects(project_list, None)
|
||||
workspace_index = await ensure_workspace_project_index(context=context)
|
||||
merged = _merge_workspace_projects(None, workspace_index.entries)
|
||||
default_project = next(
|
||||
(
|
||||
entry.project.name
|
||||
for entry in workspace_index.entries
|
||||
if entry.workspace.is_default and entry.project.is_default
|
||||
),
|
||||
None,
|
||||
)
|
||||
if output_format == "json":
|
||||
return _format_project_list_json(
|
||||
merged, project_list.default_project, constrained_project
|
||||
)
|
||||
return _format_project_list_json(merged, default_project, constrained_project)
|
||||
if constrained_project:
|
||||
return _format_constrained_text(constrained_project)
|
||||
return _format_project_list_text(merged)
|
||||
@@ -220,39 +316,40 @@ async def list_memory_projects(
|
||||
|
||||
# Fetch cloud projects when credentials are available
|
||||
cloud_list: ProjectList | None = None
|
||||
cloud_entries: tuple[WorkspaceProjectEntry, ...] = ()
|
||||
cloud_ws_name: str | None = None
|
||||
cloud_ws_type: str | None = None
|
||||
cloud_ws_tenant_id: str | None = None
|
||||
cloud_ws_slug: str | None = None
|
||||
cloud_ws_is_default = False
|
||||
config = ConfigManager().config
|
||||
if has_cloud_credentials(config):
|
||||
# Use explicit workspace, fall back to config default
|
||||
effective_workspace = workspace or config.default_workspace
|
||||
cloud_list = await _fetch_cloud_projects(effective_workspace, context)
|
||||
|
||||
# Resolve workspace metadata so each cloud project carries its workspace info
|
||||
if cloud_list:
|
||||
cloud_ws_tenant_id = effective_workspace
|
||||
try:
|
||||
from basic_memory.mcp.project_context import get_available_workspaces
|
||||
|
||||
workspaces = await get_available_workspaces(context)
|
||||
matched = next(
|
||||
(ws for ws in workspaces if ws.tenant_id == effective_workspace),
|
||||
None,
|
||||
try:
|
||||
workspace_index = await ensure_workspace_project_index(context=context)
|
||||
cloud_entries = workspace_index.entries
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
f"Cloud workspace project index discovery failed while listing projects; "
|
||||
f"showing local-only project list: {exc}"
|
||||
)
|
||||
if context: # pragma: no cover
|
||||
await context.info(
|
||||
"Cloud workspace project discovery failed while listing projects; "
|
||||
"showing local projects only"
|
||||
)
|
||||
if matched:
|
||||
cloud_ws_name = matched.name
|
||||
cloud_ws_type = matched.workspace_type
|
||||
except Exception:
|
||||
pass # workspace lookup is best-effort
|
||||
|
||||
merged = _merge_projects(
|
||||
local_list,
|
||||
cloud_list,
|
||||
cloud_workspace_name=cloud_ws_name,
|
||||
cloud_workspace_type=cloud_ws_type,
|
||||
cloud_workspace_tenant_id=cloud_ws_tenant_id,
|
||||
)
|
||||
if cloud_entries:
|
||||
merged = _merge_workspace_projects(local_list, cloud_entries)
|
||||
else:
|
||||
merged = _merge_projects(
|
||||
local_list,
|
||||
cloud_list,
|
||||
cloud_workspace_name=cloud_ws_name,
|
||||
cloud_workspace_type=cloud_ws_type,
|
||||
cloud_workspace_tenant_id=cloud_ws_tenant_id,
|
||||
cloud_workspace_slug=cloud_ws_slug,
|
||||
cloud_workspace_is_default=cloud_ws_is_default,
|
||||
)
|
||||
default_project = local_list.default_project
|
||||
|
||||
if output_format == "json":
|
||||
@@ -272,6 +369,31 @@ def _format_constrained_text(constrained_project: str) -> str:
|
||||
return result
|
||||
|
||||
|
||||
async def _resolve_create_project_workspace(
|
||||
workspace: str | None,
|
||||
context: Context | None,
|
||||
) -> str | None:
|
||||
"""Resolve the create-project workspace selector to the routing tenant id."""
|
||||
if workspace is None:
|
||||
return None
|
||||
|
||||
explicit_cloud_routing = _explicit_routing() and not _force_local_mode()
|
||||
config = ConfigManager().config
|
||||
should_resolve_workspace = is_factory_mode() or (
|
||||
explicit_cloud_routing and has_cloud_credentials(config)
|
||||
)
|
||||
if not should_resolve_workspace:
|
||||
return workspace
|
||||
|
||||
# Trigger: cloud routing can use workspace discovery and the caller supplied
|
||||
# a friendly selector such as a slug, name, or tenant id.
|
||||
# Why: MCP callers should not need to paste UUIDs, but the transport still
|
||||
# uses X-Workspace-ID with the tenant id as its routing authority.
|
||||
# Outcome: resolve once at create time and pass only the tenant id downstream.
|
||||
resolved_workspace = await resolve_workspace_parameter(workspace=workspace, context=context)
|
||||
return resolved_workspace.tenant_id
|
||||
|
||||
|
||||
@mcp.tool(
|
||||
"create_memory_project",
|
||||
annotations={"destructiveHint": False, "openWorldHint": False},
|
||||
@@ -280,6 +402,7 @@ async def create_memory_project(
|
||||
project_name: str,
|
||||
project_path: str,
|
||||
set_default: bool = False,
|
||||
workspace: str | None = None,
|
||||
output_format: Literal["text", "json"] = "text",
|
||||
context: Context | None = None,
|
||||
) -> str | dict:
|
||||
@@ -292,6 +415,11 @@ async def create_memory_project(
|
||||
project_name: Name for the new project (must be unique)
|
||||
project_path: File system path where the project will be stored
|
||||
set_default: Whether to set this project as the default (optional, defaults to False)
|
||||
workspace: Optional cloud workspace selector to create the project in. Slug is
|
||||
preferred for AI callers, but tenant_id and unique name are also accepted.
|
||||
When omitted, the connection's default workspace is used. Discover values
|
||||
via `list_workspaces`. Only meaningful in cloud mode; ignored for local
|
||||
projects.
|
||||
output_format: "text" returns the existing human-readable result text.
|
||||
"json" returns structured project creation metadata.
|
||||
context: Optional FastMCP context for progress/status logging.
|
||||
@@ -302,8 +430,15 @@ async def create_memory_project(
|
||||
Example:
|
||||
create_memory_project("my-research", "~/Documents/research")
|
||||
create_memory_project("work-notes", "/home/user/work", set_default=True)
|
||||
create_memory_project("team-notes", "/team/notes", workspace="team-paul")
|
||||
"""
|
||||
async with get_client() as client:
|
||||
workspace_id = await _resolve_create_project_workspace(workspace, context)
|
||||
|
||||
# workspace targets a non-default cloud workspace at create time.
|
||||
# Trigger: caller passed workspace (e.g. a slug discovered via list_workspaces).
|
||||
# Why: there is no project_id yet for per-project routing — the project doesn't exist.
|
||||
# Outcome: cloud factory routes the create request to the resolved workspace tenant id.
|
||||
async with get_client(workspace=workspace_id) as client:
|
||||
# Check if server is constrained to a specific project
|
||||
constrained_project = os.environ.get("BASIC_MEMORY_MCP_PROJECT")
|
||||
if constrained_project:
|
||||
@@ -347,6 +482,7 @@ async def create_memory_project(
|
||||
if output_format == "json":
|
||||
return {
|
||||
"name": existing_match.name,
|
||||
"external_id": existing_match.external_id,
|
||||
"path": existing_match.path,
|
||||
"is_default": is_default,
|
||||
"created": False,
|
||||
@@ -356,17 +492,22 @@ async def create_memory_project(
|
||||
f"✓ Project already exists: {existing_match.name}\n\n"
|
||||
f"Project Details:\n"
|
||||
f"• Name: {existing_match.name}\n"
|
||||
f"• External ID: {existing_match.external_id}\n"
|
||||
f"• Path: {existing_match.path}\n"
|
||||
f"{'• Set as default project\n' if is_default else ''}"
|
||||
"\nProject is already available for use in tool calls.\n"
|
||||
)
|
||||
|
||||
status_response = await project_client.create_project(project_request.model_dump())
|
||||
from basic_memory.mcp.project_context import invalidate_workspace_project_index
|
||||
|
||||
await invalidate_workspace_project_index(context)
|
||||
|
||||
if output_format == "json":
|
||||
new_project = status_response.new_project
|
||||
return {
|
||||
"name": new_project.name if new_project else project_name,
|
||||
"external_id": new_project.external_id if new_project else None,
|
||||
"path": new_project.path if new_project else project_path,
|
||||
"is_default": bool(
|
||||
(new_project.is_default if new_project else False) or set_default
|
||||
@@ -380,6 +521,7 @@ async def create_memory_project(
|
||||
if status_response.new_project:
|
||||
result += "Project Details:\n"
|
||||
result += f"• Name: {status_response.new_project.name}\n"
|
||||
result += f"• External ID: {status_response.new_project.external_id}\n"
|
||||
result += f"• Path: {status_response.new_project.path}\n"
|
||||
|
||||
if set_default:
|
||||
@@ -451,6 +593,9 @@ async def delete_project(project_name: str, context: Context | None = None) -> s
|
||||
|
||||
# Delete project using project external_id
|
||||
status_response = await project_client.delete_project(target_project.external_id)
|
||||
from basic_memory.mcp.project_context import invalidate_workspace_project_index
|
||||
|
||||
await invalidate_workspace_project_index(context)
|
||||
|
||||
result = f"✓ {status_response.message}\n\n"
|
||||
|
||||
|
||||
@@ -8,16 +8,17 @@ Files are read directly without any knowledge graph processing.
|
||||
import base64
|
||||
import io
|
||||
|
||||
from typing import Optional
|
||||
from typing import Annotated, Optional
|
||||
|
||||
from loguru import logger
|
||||
from PIL import Image as PILImage
|
||||
from fastmcp import Context
|
||||
from pydantic import AliasChoices, Field
|
||||
from mcp.server.fastmcp.exceptions import ToolError
|
||||
|
||||
from basic_memory.config import ConfigManager
|
||||
from basic_memory.mcp.project_context import (
|
||||
detect_project_from_url_prefix,
|
||||
detect_project_from_memory_url_prefix,
|
||||
get_project_client,
|
||||
resolve_project_and_path,
|
||||
)
|
||||
@@ -158,9 +159,12 @@ def optimize_image(img, content_length, max_output_bytes=350000):
|
||||
annotations={"readOnlyHint": True, "openWorldHint": False},
|
||||
)
|
||||
async def read_content(
|
||||
path: str,
|
||||
path: Annotated[
|
||||
str,
|
||||
Field(validation_alias=AliasChoices("path", "file_path", "filepath", "file")),
|
||||
],
|
||||
project: Optional[str] = None,
|
||||
workspace: Optional[str] = None,
|
||||
project_id: Optional[str] = None,
|
||||
context: Context | None = None,
|
||||
) -> dict:
|
||||
"""Read a file's raw content by path or permalink.
|
||||
@@ -181,6 +185,9 @@ async def read_content(
|
||||
- A permalink (docs/example)
|
||||
project: Project name to read from. Optional - server will resolve using hierarchy.
|
||||
If unknown, use list_memory_projects() to discover available projects.
|
||||
project_id: Project external_id (UUID). Prefer this over `project` when known —
|
||||
it routes to the exact project regardless of name collisions across cloud
|
||||
workspaces. Takes precedence over `project`. Get from list_memory_projects().
|
||||
context: Optional FastMCP context for performance caching.
|
||||
|
||||
Returns:
|
||||
@@ -210,17 +217,27 @@ async def read_content(
|
||||
HTTPError: If project doesn't exist or is inaccessible
|
||||
SecurityError: If path attempts path traversal
|
||||
"""
|
||||
# Detect project from memory URL prefix before routing
|
||||
if project is None:
|
||||
detected = detect_project_from_url_prefix(path, ConfigManager().config)
|
||||
# Detect project from memory URL prefix before routing.
|
||||
# project_id routes by external UUID, so it bypasses URL discovery entirely.
|
||||
if project is None and project_id is None:
|
||||
detected = await detect_project_from_memory_url_prefix(
|
||||
path,
|
||||
ConfigManager().config,
|
||||
context=context,
|
||||
)
|
||||
if detected:
|
||||
project = detected
|
||||
|
||||
logger.info(f"MCP tool call tool=read_content project={project} path={path}")
|
||||
|
||||
async with get_project_client(project, workspace, context) as (client, active_project):
|
||||
# Resolve path with project-prefix awareness for memory:// URLs
|
||||
_, url, _ = await resolve_project_and_path(client, path, project, context)
|
||||
async with get_project_client(project, context=context, project_id=project_id) as (
|
||||
client,
|
||||
active_project,
|
||||
):
|
||||
# Resolve path with project-prefix awareness for memory:// URLs.
|
||||
# Use active_project.name so resolution stays consistent when project_id
|
||||
# was used or `project` was wrong/ambiguous (matches the cached resolution).
|
||||
_, url, _ = await resolve_project_and_path(client, path, active_project.name, context)
|
||||
|
||||
# Validate path to prevent path traversal attacks
|
||||
# For memory:// URLs, validate the extracted path (not the raw URL which
|
||||
|
||||
@@ -1,17 +1,17 @@
|
||||
"""Read note tool for Basic Memory MCP server."""
|
||||
|
||||
from textwrap import dedent
|
||||
from typing import Optional, Literal
|
||||
from typing import Optional, Literal, cast
|
||||
|
||||
import logfire
|
||||
import yaml
|
||||
|
||||
from loguru import logger
|
||||
from fastmcp import Context
|
||||
|
||||
from basic_memory import telemetry
|
||||
from basic_memory.config import ConfigManager
|
||||
from basic_memory.mcp.project_context import (
|
||||
detect_project_from_url_prefix,
|
||||
detect_project_from_memory_url_prefix,
|
||||
get_project_client,
|
||||
resolve_project_and_path,
|
||||
)
|
||||
@@ -33,10 +33,10 @@ def _parse_opening_frontmatter(content: str) -> tuple[str, dict | None]:
|
||||
If parsing fails or frontmatter is not a mapping, returns body unchanged and None.
|
||||
"""
|
||||
original_content = content
|
||||
if not content.startswith("---\n"):
|
||||
lines = content.splitlines(keepends=True)
|
||||
if not lines or lines[0].strip() != "---":
|
||||
return original_content, None
|
||||
|
||||
lines = content.splitlines(keepends=True)
|
||||
closing_index = None
|
||||
for i in range(1, len(lines)):
|
||||
if lines[i].strip() == "---":
|
||||
@@ -70,9 +70,7 @@ def _parse_opening_frontmatter(content: str) -> tuple[str, dict | None]:
|
||||
async def read_note(
|
||||
identifier: str,
|
||||
project: Optional[str] = None,
|
||||
workspace: Optional[str] = None,
|
||||
page: int = 1,
|
||||
page_size: int = 10,
|
||||
project_id: Optional[str] = None,
|
||||
output_format: Literal["text", "json"] = "text",
|
||||
include_frontmatter: bool = False,
|
||||
context: Context | None = None,
|
||||
@@ -96,10 +94,11 @@ async def read_note(
|
||||
project: Project name to read from. Optional - server will resolve using the
|
||||
hierarchy above. If unknown, use list_memory_projects() to discover
|
||||
available projects.
|
||||
project_id: Project external_id (UUID). Prefer this over `project` when known —
|
||||
it routes to the exact project regardless of name collisions across cloud
|
||||
workspaces. Takes precedence over `project`. Get from list_memory_projects().
|
||||
identifier: The title or permalink of the note to read
|
||||
Can be a full memory:// URL, a permalink, a title, or search text
|
||||
page: Page number for paginated results (default: 1)
|
||||
page_size: Number of items per page (default: 10)
|
||||
output_format: "text" returns markdown content or guidance text.
|
||||
"json" returns a structured object with title/permalink/file_path/content/frontmatter.
|
||||
include_frontmatter: When output_format="json", whether content should include the
|
||||
@@ -120,9 +119,6 @@ async def read_note(
|
||||
# Read with memory URL
|
||||
read_note("my-research", "memory://specs/search-spec")
|
||||
|
||||
# Read with pagination
|
||||
read_note("work-project", "Project Updates", page=2, page_size=5)
|
||||
|
||||
# Read recent meeting notes
|
||||
read_note("team-docs", "Weekly Standup")
|
||||
|
||||
@@ -134,230 +130,239 @@ async def read_note(
|
||||
If the exact note isn't found, this tool provides helpful suggestions
|
||||
including related notes, search commands, and note creation templates.
|
||||
"""
|
||||
# Detect project from memory URL prefix before routing
|
||||
if project is None:
|
||||
detected = detect_project_from_url_prefix(identifier, ConfigManager().config)
|
||||
# Detect project from memory URL prefix before routing.
|
||||
# project_id routes by external UUID, so it bypasses URL discovery entirely.
|
||||
if project is None and project_id is None:
|
||||
detected = await detect_project_from_memory_url_prefix(
|
||||
identifier,
|
||||
ConfigManager().config,
|
||||
context=context,
|
||||
)
|
||||
if detected:
|
||||
project = detected
|
||||
|
||||
with telemetry.operation(
|
||||
with logfire.span(
|
||||
"mcp.tool.read_note",
|
||||
entrypoint="mcp",
|
||||
tool_name="read_note",
|
||||
requested_project=project,
|
||||
workspace_id=workspace,
|
||||
requested_project_id=project_id,
|
||||
output_format=output_format,
|
||||
page=page,
|
||||
page_size=page_size,
|
||||
include_frontmatter=include_frontmatter,
|
||||
):
|
||||
async with get_project_client(project, workspace, context) as (client, active_project):
|
||||
with telemetry.contextualize(
|
||||
project_name=active_project.name,
|
||||
workspace_id=workspace,
|
||||
tool_name="read_note",
|
||||
async with get_project_client(project, context=context, project_id=project_id) as (
|
||||
client,
|
||||
active_project,
|
||||
):
|
||||
# Resolve identifier with project-prefix awareness for memory:// URLs.
|
||||
# Pass active_project.name (the canonical resolved name) rather than the
|
||||
# original `project` arg so the inner get_active_project cache hits even
|
||||
# when project_id was used or `project` was wrong/ambiguous.
|
||||
_, entity_path, _ = await resolve_project_and_path(
|
||||
client, identifier, active_project.name, context
|
||||
)
|
||||
|
||||
# Validate identifier to prevent path traversal attacks
|
||||
# For memory:// URLs, validate the extracted path (not the raw URL which
|
||||
# has a scheme prefix that confuses path validation)
|
||||
raw_path = (
|
||||
memory_url_path(identifier) if identifier.startswith("memory://") else identifier
|
||||
)
|
||||
processed_path = entity_path
|
||||
project_path = active_project.home
|
||||
|
||||
if not validate_project_path(raw_path, project_path) or not validate_project_path(
|
||||
processed_path, project_path
|
||||
):
|
||||
# Resolve identifier with project-prefix awareness for memory:// URLs
|
||||
_, entity_path, _ = await resolve_project_and_path(
|
||||
client, identifier, project, context
|
||||
logger.warning(
|
||||
"Attempted path traversal attack blocked",
|
||||
identifier=identifier,
|
||||
processed_path=processed_path,
|
||||
project=active_project.name,
|
||||
)
|
||||
|
||||
# Validate identifier to prevent path traversal attacks
|
||||
# For memory:// URLs, validate the extracted path (not the raw URL which
|
||||
# has a scheme prefix that confuses path validation)
|
||||
raw_path = (
|
||||
memory_url_path(identifier)
|
||||
if identifier.startswith("memory://")
|
||||
else identifier
|
||||
)
|
||||
processed_path = entity_path
|
||||
project_path = active_project.home
|
||||
|
||||
if not validate_project_path(raw_path, project_path) or not validate_project_path(
|
||||
processed_path, project_path
|
||||
):
|
||||
logger.warning(
|
||||
"Attempted path traversal attack blocked",
|
||||
identifier=identifier,
|
||||
processed_path=processed_path,
|
||||
project=active_project.name,
|
||||
)
|
||||
if output_format == "json":
|
||||
return {
|
||||
"title": None,
|
||||
"permalink": None,
|
||||
"file_path": None,
|
||||
"content": None,
|
||||
"frontmatter": None,
|
||||
"error": "SECURITY_VALIDATION_ERROR",
|
||||
}
|
||||
return f"# Error\n\nIdentifier '{identifier}' is not allowed - paths must stay within project boundaries"
|
||||
|
||||
# Get the file via REST API - first try direct identifier resolution
|
||||
logger.info(
|
||||
f"Attempting to read note from Project: {active_project.name} identifier: {entity_path}"
|
||||
)
|
||||
|
||||
# Import here to avoid circular import
|
||||
from basic_memory.mcp.clients import KnowledgeClient, ResourceClient
|
||||
|
||||
# Use typed clients for API calls
|
||||
knowledge_client = KnowledgeClient(client, active_project.external_id)
|
||||
resource_client = ResourceClient(client, active_project.external_id)
|
||||
|
||||
async def _read_json_payload(entity_id: str) -> dict:
|
||||
with telemetry.scope(
|
||||
"mcp.read_note.shape_response",
|
||||
domain="mcp",
|
||||
action="read_note",
|
||||
phase="shape_response",
|
||||
):
|
||||
entity = await knowledge_client.get_entity(entity_id)
|
||||
response = await resource_client.read(
|
||||
entity_id, page=page, page_size=page_size
|
||||
)
|
||||
content_text = response.text
|
||||
body_content, parsed_frontmatter = _parse_opening_frontmatter(content_text)
|
||||
return {
|
||||
"title": entity.title,
|
||||
"permalink": entity.permalink,
|
||||
"file_path": entity.file_path,
|
||||
"content": content_text if include_frontmatter else body_content,
|
||||
"frontmatter": parsed_frontmatter,
|
||||
}
|
||||
|
||||
def _empty_json_payload() -> dict:
|
||||
if output_format == "json":
|
||||
return {
|
||||
"title": None,
|
||||
"permalink": None,
|
||||
"file_path": None,
|
||||
"content": None,
|
||||
"frontmatter": None,
|
||||
"error": "SECURITY_VALIDATION_ERROR",
|
||||
}
|
||||
return f"# Error\n\nIdentifier '{identifier}' is not allowed - paths must stay within project boundaries"
|
||||
|
||||
# Get the file via REST API - first try direct identifier resolution
|
||||
logger.info(
|
||||
f"Attempting to read note from Project: {active_project.name} identifier: {entity_path}"
|
||||
)
|
||||
|
||||
# Import here to avoid circular import
|
||||
from basic_memory.mcp.clients import KnowledgeClient, ResourceClient
|
||||
|
||||
# Use typed clients for API calls
|
||||
knowledge_client = KnowledgeClient(client, active_project.external_id)
|
||||
resource_client = ResourceClient(client, active_project.external_id)
|
||||
|
||||
async def _read_json_payload(entity_id: str) -> dict:
|
||||
with logfire.span(
|
||||
"mcp.read_note.shape_response",
|
||||
domain="mcp",
|
||||
action="read_note",
|
||||
phase="shape_response",
|
||||
):
|
||||
entity = await knowledge_client.get_entity(entity_id)
|
||||
response = await resource_client.read(entity_id)
|
||||
content_text = response.text
|
||||
body_content, parsed_frontmatter = _parse_opening_frontmatter(content_text)
|
||||
return {
|
||||
"title": entity.title,
|
||||
"permalink": entity.permalink,
|
||||
"file_path": entity.file_path,
|
||||
"content": content_text if include_frontmatter else body_content,
|
||||
"frontmatter": parsed_frontmatter,
|
||||
}
|
||||
|
||||
def _search_results(payload: object) -> list[dict]:
|
||||
if not isinstance(payload, dict):
|
||||
return []
|
||||
results = payload.get("results")
|
||||
return results if isinstance(results, list) else []
|
||||
def _empty_json_payload() -> dict:
|
||||
return {
|
||||
"title": None,
|
||||
"permalink": None,
|
||||
"file_path": None,
|
||||
"content": None,
|
||||
"frontmatter": None,
|
||||
}
|
||||
|
||||
async def _search_candidates(identifier_text: str, *, title_only: bool) -> dict:
|
||||
# Trigger: direct entity resolution failed for the caller's identifier.
|
||||
# Why: search_notes applies the same memory:// normalization and tool-level
|
||||
# query handling as the rest of MCP routing, which raw client calls skip.
|
||||
# Outcome: unresolved memory URLs still fall back through normalized search.
|
||||
search_type = "title" if title_only else "text"
|
||||
response = await search_notes(
|
||||
project=active_project.name,
|
||||
workspace=workspace,
|
||||
query=identifier_text,
|
||||
search_type=search_type,
|
||||
page=page,
|
||||
page_size=page_size,
|
||||
output_format="json",
|
||||
context=context,
|
||||
)
|
||||
return response if isinstance(response, dict) else {}
|
||||
def _search_results(payload: object) -> list[dict[str, object]]:
|
||||
if not isinstance(payload, dict):
|
||||
return []
|
||||
payload_dict = cast(dict[str, object], payload)
|
||||
results = payload_dict.get("results")
|
||||
if not isinstance(results, list):
|
||||
return []
|
||||
return [
|
||||
cast(dict[str, object], result)
|
||||
for result in results
|
||||
if isinstance(result, dict)
|
||||
]
|
||||
|
||||
def _result_title(item: dict) -> str:
|
||||
return str(item.get("title") or "")
|
||||
async def _search_candidates(
|
||||
identifier_text: str, *, title_only: bool
|
||||
) -> dict[str, object]:
|
||||
# Trigger: direct entity resolution failed for the caller's identifier.
|
||||
# Why: search_notes applies the same memory:// normalization and tool-level
|
||||
# query handling as the rest of MCP routing, which raw client calls skip.
|
||||
# Outcome: unresolved memory URLs still fall back through normalized search.
|
||||
# Pass project_id (external_id UUID) so the workspace selection from the
|
||||
# outer get_project_client() is preserved across the inner re-resolution.
|
||||
# Without this, project names that collide across workspaces could re-resolve
|
||||
# to a different tenant via the default-workspace fallback (CLI/context=None).
|
||||
search_type = "title" if title_only else "text"
|
||||
response = await search_notes(
|
||||
project=active_project.name,
|
||||
project_id=active_project.external_id,
|
||||
query=identifier_text,
|
||||
search_type=search_type,
|
||||
output_format="json",
|
||||
context=context,
|
||||
)
|
||||
return cast(dict[str, object], response) if isinstance(response, dict) else {}
|
||||
|
||||
def _result_permalink(item: dict) -> Optional[str]:
|
||||
value = item.get("permalink")
|
||||
return str(value) if value else None
|
||||
def _result_title(item: dict[str, object]) -> str:
|
||||
return str(item.get("title") or "")
|
||||
|
||||
def _result_file_path(item: dict) -> Optional[str]:
|
||||
value = item.get("file_path")
|
||||
return str(value) if value else None
|
||||
def _result_permalink(item: dict[str, object]) -> Optional[str]:
|
||||
value = item.get("permalink")
|
||||
return str(value) if value else None
|
||||
|
||||
try:
|
||||
# Try to resolve identifier to entity ID
|
||||
entity_id = await knowledge_client.resolve_entity(entity_path, strict=True)
|
||||
def _result_file_path(item: dict[str, object]) -> Optional[str]:
|
||||
value = item.get("file_path")
|
||||
return str(value) if value else None
|
||||
|
||||
# Fetch content using entity ID
|
||||
response = await resource_client.read(entity_id, page=page, page_size=page_size)
|
||||
try:
|
||||
# Try to resolve identifier to entity ID
|
||||
entity_id = await knowledge_client.resolve_entity(entity_path, strict=True)
|
||||
|
||||
# If successful, return the content
|
||||
if response.status_code == 200:
|
||||
logger.info(
|
||||
"Returning read_note result from resource: {path}", path=entity_path
|
||||
)
|
||||
if output_format == "json":
|
||||
return await _read_json_payload(entity_id)
|
||||
return response.text
|
||||
except Exception as e: # pragma: no cover
|
||||
logger.info(f"Direct lookup failed for '{entity_path}': {e}")
|
||||
# Continue to fallback methods
|
||||
# Fetch content using entity ID
|
||||
response = await resource_client.read(entity_id)
|
||||
|
||||
# Fallback 1: Try title search via API
|
||||
logger.info(f"Search title for: {identifier}")
|
||||
title_results = await _search_candidates(identifier, title_only=True)
|
||||
|
||||
title_candidates = _search_results(title_results)
|
||||
if title_candidates:
|
||||
# Trigger: direct resolution failed and title search returned candidates.
|
||||
# Why: avoid returning unrelated notes when search yields only fuzzy matches.
|
||||
# Outcome: fetch content only when a true exact title match exists.
|
||||
result = next(
|
||||
(
|
||||
candidate
|
||||
for candidate in title_candidates
|
||||
if _is_exact_title_match(identifier, _result_title(candidate))
|
||||
),
|
||||
None,
|
||||
)
|
||||
if not result:
|
||||
logger.info(f"No exact title match found for: {identifier}")
|
||||
elif _result_permalink(result):
|
||||
try:
|
||||
# Resolve the permalink to entity ID
|
||||
entity_id = await knowledge_client.resolve_entity(
|
||||
_result_permalink(result) or "", strict=True
|
||||
)
|
||||
|
||||
# Fetch content using the entity ID
|
||||
response = await resource_client.read(
|
||||
entity_id, page=page, page_size=page_size
|
||||
)
|
||||
|
||||
if response.status_code == 200:
|
||||
logger.info(
|
||||
f"Found note by exact title search: {_result_permalink(result)}"
|
||||
)
|
||||
if output_format == "json":
|
||||
return await _read_json_payload(entity_id)
|
||||
return response.text
|
||||
except Exception as e: # pragma: no cover
|
||||
logger.info(
|
||||
f"Failed to fetch content for found title match {_result_permalink(result)}: {e}"
|
||||
)
|
||||
else:
|
||||
# If successful, return the content
|
||||
if response.status_code == 200:
|
||||
logger.info(
|
||||
f"No results in title search for: {identifier} in project {active_project.name}"
|
||||
"Returning read_note result from resource: {path}", path=entity_path
|
||||
)
|
||||
|
||||
# Fallback 2: Text search as a last resort
|
||||
logger.info(f"Title search failed, trying text search for: {identifier}")
|
||||
text_results = await _search_candidates(identifier, title_only=False)
|
||||
|
||||
# We didn't find a direct match, construct a helpful error message
|
||||
text_candidates = _search_results(text_results)
|
||||
if not text_candidates:
|
||||
if output_format == "json":
|
||||
return _empty_json_payload()
|
||||
return format_not_found_message(active_project.name, identifier)
|
||||
return await _read_json_payload(entity_id)
|
||||
return response.text
|
||||
except Exception as e: # pragma: no cover
|
||||
logger.info(f"Direct lookup failed for '{entity_path}': {e}")
|
||||
# Continue to fallback methods
|
||||
|
||||
# Fallback 1: Try title search via API
|
||||
logger.info(f"Search title for: {identifier}")
|
||||
title_results = await _search_candidates(identifier, title_only=True)
|
||||
|
||||
title_candidates = _search_results(title_results)
|
||||
if title_candidates:
|
||||
# Trigger: direct resolution failed and title search returned candidates.
|
||||
# Why: avoid returning unrelated notes when search yields only fuzzy matches.
|
||||
# Outcome: fetch content only when a true exact title match exists.
|
||||
result = next(
|
||||
(
|
||||
candidate
|
||||
for candidate in title_candidates
|
||||
if _is_exact_title_match(identifier, _result_title(candidate))
|
||||
),
|
||||
None,
|
||||
)
|
||||
if not result:
|
||||
logger.info(f"No exact title match found for: {identifier}")
|
||||
elif _result_permalink(result):
|
||||
try:
|
||||
# Resolve the permalink to entity ID
|
||||
entity_id = await knowledge_client.resolve_entity(
|
||||
_result_permalink(result) or "", strict=True
|
||||
)
|
||||
|
||||
# Fetch content using the entity ID
|
||||
response = await resource_client.read(entity_id)
|
||||
|
||||
if response.status_code == 200:
|
||||
logger.info(
|
||||
f"Found note by exact title search: {_result_permalink(result)}"
|
||||
)
|
||||
if output_format == "json":
|
||||
return await _read_json_payload(entity_id)
|
||||
return response.text
|
||||
except Exception as e: # pragma: no cover
|
||||
logger.info(
|
||||
f"Failed to fetch content for found title match {_result_permalink(result)}: {e}"
|
||||
)
|
||||
else:
|
||||
logger.info(
|
||||
f"No results in title search for: {identifier} in project {active_project.name}"
|
||||
)
|
||||
|
||||
# Fallback 2: Text search as a last resort
|
||||
logger.info(f"Title search failed, trying text search for: {identifier}")
|
||||
text_results = await _search_candidates(identifier, title_only=False)
|
||||
|
||||
# We didn't find a direct match, construct a helpful error message
|
||||
text_candidates = _search_results(text_results)
|
||||
if not text_candidates:
|
||||
if output_format == "json":
|
||||
payload = _empty_json_payload()
|
||||
payload["related_results"] = [
|
||||
{
|
||||
"title": _result_title(result),
|
||||
"permalink": _result_permalink(result),
|
||||
"file_path": _result_file_path(result),
|
||||
}
|
||||
for result in text_candidates[:5]
|
||||
]
|
||||
return payload
|
||||
return format_related_results(active_project.name, identifier, text_candidates[:5])
|
||||
return _empty_json_payload()
|
||||
return format_not_found_message(active_project.name, identifier)
|
||||
if output_format == "json":
|
||||
payload = _empty_json_payload()
|
||||
payload["related_results"] = [
|
||||
{
|
||||
"title": _result_title(result),
|
||||
"permalink": _result_permalink(result),
|
||||
"file_path": _result_file_path(result),
|
||||
}
|
||||
for result in text_candidates[:5]
|
||||
]
|
||||
return payload
|
||||
return format_related_results(active_project.name, identifier, text_candidates[:5])
|
||||
|
||||
|
||||
def format_not_found_message(project: str | None, identifier: str) -> str:
|
||||
|
||||
@@ -2,10 +2,11 @@
|
||||
|
||||
from datetime import timezone
|
||||
from pathlib import PurePosixPath
|
||||
from typing import List, Union, Optional, Literal
|
||||
from typing import Annotated, List, Union, Optional, Literal
|
||||
|
||||
from loguru import logger
|
||||
from fastmcp import Context
|
||||
from pydantic import AliasChoices, Field
|
||||
|
||||
from basic_memory.mcp.async_client import get_client
|
||||
from basic_memory.mcp.project_context import (
|
||||
@@ -38,13 +39,30 @@ from basic_memory.schemas.search import SearchItemType
|
||||
annotations={"readOnlyHint": True, "openWorldHint": False},
|
||||
)
|
||||
async def recent_activity(
|
||||
type: Union[str, List[str]] = "",
|
||||
type: Annotated[
|
||||
Union[str, List[str]],
|
||||
Field(default="", validation_alias=AliasChoices("type", "types", "kind")),
|
||||
] = "",
|
||||
depth: int = 1,
|
||||
timeframe: TimeFrame = "7d",
|
||||
page: int = 1,
|
||||
page_size: int = 10,
|
||||
timeframe: Annotated[
|
||||
TimeFrame,
|
||||
Field(
|
||||
default="7d",
|
||||
validation_alias=AliasChoices("timeframe", "since", "time_range", "lookback"),
|
||||
),
|
||||
] = "7d",
|
||||
# `offset` is intentionally NOT aliased: it has different semantics
|
||||
# (item-indexed vs. 1-indexed page-number).
|
||||
page: Annotated[
|
||||
int,
|
||||
Field(default=1, validation_alias=AliasChoices("page", "page_number")),
|
||||
] = 1,
|
||||
page_size: Annotated[
|
||||
int,
|
||||
Field(default=10, validation_alias=AliasChoices("page_size", "limit", "per_page")),
|
||||
] = 10,
|
||||
project: Optional[str] = None,
|
||||
workspace: Optional[str] = None,
|
||||
project_id: Optional[str] = None,
|
||||
output_format: Literal["text", "json"] = "text",
|
||||
context: Context | None = None,
|
||||
) -> str | list[dict]:
|
||||
@@ -86,6 +104,9 @@ async def recent_activity(
|
||||
project: Project name to query. Optional - server will resolve using the
|
||||
hierarchy above. If unknown, use list_memory_projects() to discover
|
||||
available projects.
|
||||
project_id: Project external_id (UUID). Prefer this over `project` when known —
|
||||
it routes to the exact project regardless of name collisions across cloud
|
||||
workspaces. Takes precedence over `project`. Get from list_memory_projects().
|
||||
output_format: "text" returns human-readable summary text. "json" returns
|
||||
a flat list of recent items.
|
||||
context: Optional FastMCP context for performance caching.
|
||||
@@ -161,9 +182,12 @@ async def recent_activity(
|
||||
if "type" not in params:
|
||||
params["type"] = [SearchItemType.ENTITY.value]
|
||||
|
||||
# Resolve project parameter using the three-tier hierarchy
|
||||
# allow_discovery=True enables Discovery Mode, so a project is not required
|
||||
resolved_project = await resolve_project_parameter(project, allow_discovery=True)
|
||||
# Resolve project parameter using the three-tier hierarchy.
|
||||
# allow_discovery=True enables Discovery Mode, so a project is not required.
|
||||
# project_id (UUID) takes precedence over project name — without this fallback,
|
||||
# callers passing only project_id would fall into Discovery Mode.
|
||||
effective_identifier = project_id if project_id else project
|
||||
resolved_project = await resolve_project_parameter(effective_identifier, allow_discovery=True)
|
||||
|
||||
if resolved_project is None:
|
||||
# Discovery Mode: Get activity across all projects
|
||||
@@ -278,7 +302,7 @@ async def recent_activity(
|
||||
f"Getting recent activity from project {resolved_project}: type={type}, depth={depth}, timeframe={timeframe}"
|
||||
)
|
||||
|
||||
async with get_project_client(resolved_project, workspace, context) as (
|
||||
async with get_project_client(resolved_project, context=context, project_id=project_id) as (
|
||||
client,
|
||||
active_project,
|
||||
):
|
||||
@@ -464,10 +488,12 @@ def _format_project_output(
|
||||
elif result.primary_result.type == "observation":
|
||||
observations.append(result.primary_result)
|
||||
|
||||
# Show entities (notes/documents)
|
||||
# Show entities (notes/documents). Render every row the API returned —
|
||||
# `page_size` is the single knob for how much comes back, so heading count
|
||||
# and body row count always agree (regression: #784 silent truncation).
|
||||
if entities:
|
||||
lines.append(f"\n**📄 Recent Notes & Documents ({len(entities)}):**")
|
||||
for entity in entities[:5]: # Show top 5
|
||||
for entity in entities:
|
||||
title = entity.title or "Untitled"
|
||||
# Get folder from file_path
|
||||
folder = ""
|
||||
@@ -500,7 +526,7 @@ def _format_project_output(
|
||||
# Show relations (connections)
|
||||
if relations:
|
||||
lines.append(f"\n**🔗 Recent Connections ({len(relations)}):**")
|
||||
for rel in relations[:5]: # Show top 5
|
||||
for rel in relations:
|
||||
rel_type = rel.relation_type
|
||||
from_entity = rel.from_entity or "Unknown"
|
||||
to_entity = rel.to_entity
|
||||
|
||||
@@ -211,7 +211,7 @@ async def schema_validate(
|
||||
note_type: Optional[str] = None,
|
||||
identifier: Optional[str] = None,
|
||||
project: Optional[str] = None,
|
||||
workspace: Optional[str] = None,
|
||||
project_id: Optional[str] = None,
|
||||
output_format: Literal["text", "json"] = "text",
|
||||
context: Context | None = None,
|
||||
) -> ValidationReport | str | dict:
|
||||
@@ -236,6 +236,9 @@ async def schema_validate(
|
||||
identifier: Specific note to validate (permalink, title, or path).
|
||||
If provided, validates only this note.
|
||||
project: Project name. Optional -- server will resolve.
|
||||
project_id: Project external_id (UUID). Prefer this over `project` when known —
|
||||
it routes to the exact project regardless of name collisions across cloud
|
||||
workspaces. Takes precedence over `project`. Get from list_memory_projects().
|
||||
context: Optional FastMCP context for performance caching.
|
||||
|
||||
Returns:
|
||||
@@ -251,7 +254,10 @@ async def schema_validate(
|
||||
# Validate in a specific project
|
||||
schema_validate(note_type="person", project="my-research")
|
||||
"""
|
||||
async with get_project_client(project, workspace, context) as (client, active_project):
|
||||
async with get_project_client(project, context=context, project_id=project_id) as (
|
||||
client,
|
||||
active_project,
|
||||
):
|
||||
logger.info(
|
||||
f"MCP tool call tool=schema_validate project={active_project.name} "
|
||||
f"note_type={note_type} identifier={identifier}"
|
||||
@@ -318,7 +324,7 @@ async def schema_infer(
|
||||
note_type: str,
|
||||
threshold: float = 0.25,
|
||||
project: Optional[str] = None,
|
||||
workspace: Optional[str] = None,
|
||||
project_id: Optional[str] = None,
|
||||
output_format: Literal["text", "json"] = "text",
|
||||
context: Context | None = None,
|
||||
) -> str | dict:
|
||||
@@ -342,6 +348,9 @@ async def schema_infer(
|
||||
threshold: Minimum frequency (0-1) for a field to be suggested as optional.
|
||||
Default 0.25 (25%). Fields above 95% become required.
|
||||
project: Project name. Optional -- server will resolve.
|
||||
project_id: Project external_id (UUID). Prefer this over `project` when known —
|
||||
it routes to the exact project regardless of name collisions across cloud
|
||||
workspaces. Takes precedence over `project`. Get from list_memory_projects().
|
||||
context: Optional FastMCP context for performance caching.
|
||||
|
||||
Returns:
|
||||
@@ -357,7 +366,10 @@ async def schema_infer(
|
||||
# Infer in a specific project
|
||||
schema_infer("person", project="my-research")
|
||||
"""
|
||||
async with get_project_client(project, workspace, context) as (client, active_project):
|
||||
async with get_project_client(project, context=context, project_id=project_id) as (
|
||||
client,
|
||||
active_project,
|
||||
):
|
||||
logger.info(
|
||||
f"MCP tool call tool=schema_infer project={active_project.name} "
|
||||
f"note_type={note_type} threshold={threshold}"
|
||||
@@ -432,7 +444,7 @@ async def schema_infer(
|
||||
async def schema_diff(
|
||||
note_type: str,
|
||||
project: Optional[str] = None,
|
||||
workspace: Optional[str] = None,
|
||||
project_id: Optional[str] = None,
|
||||
output_format: Literal["text", "json"] = "text",
|
||||
context: Context | None = None,
|
||||
) -> str | dict:
|
||||
@@ -453,6 +465,9 @@ async def schema_diff(
|
||||
Args:
|
||||
note_type: The note type to check for drift (e.g., "person").
|
||||
project: Project name. Optional -- server will resolve.
|
||||
project_id: Project external_id (UUID). Prefer this over `project` when known —
|
||||
it routes to the exact project regardless of name collisions across cloud
|
||||
workspaces. Takes precedence over `project`. Get from list_memory_projects().
|
||||
context: Optional FastMCP context for performance caching.
|
||||
|
||||
Returns:
|
||||
@@ -466,7 +481,10 @@ async def schema_diff(
|
||||
# Check drift in a specific project
|
||||
schema_diff("person", project="my-research")
|
||||
"""
|
||||
async with get_project_client(project, workspace, context) as (client, active_project):
|
||||
async with get_project_client(project, context=context, project_id=project_id) as (
|
||||
client,
|
||||
active_project,
|
||||
):
|
||||
logger.info(
|
||||
f"MCP tool call tool=schema_diff project={active_project.name} note_type={note_type}"
|
||||
)
|
||||
|
||||
@@ -4,16 +4,16 @@ import re
|
||||
from textwrap import dedent
|
||||
from typing import Annotated, List, Optional, Dict, Any, Literal
|
||||
|
||||
import logfire
|
||||
from loguru import logger
|
||||
from fastmcp import Context
|
||||
from pydantic import BeforeValidator
|
||||
from pydantic import AliasChoices, BeforeValidator, Field
|
||||
|
||||
from basic_memory import telemetry
|
||||
from basic_memory.config import ConfigManager
|
||||
from basic_memory.utils import coerce_dict, coerce_list
|
||||
from basic_memory.mcp.container import get_container
|
||||
from basic_memory.mcp.project_context import (
|
||||
detect_project_from_url_prefix,
|
||||
detect_project_from_memory_url_prefix,
|
||||
get_project_client,
|
||||
resolve_project_and_path,
|
||||
)
|
||||
@@ -301,27 +301,51 @@ def _format_search_markdown(result: SearchResponse, project: str, query: str | N
|
||||
annotations={"readOnlyHint": True, "openWorldHint": False},
|
||||
)
|
||||
async def search_notes(
|
||||
query: Optional[str] = None,
|
||||
# Accept common search-query aliases models reach for from training data.
|
||||
# `q` is the universal HTTP convention; `search`/`text` are common in NL APIs.
|
||||
query: Annotated[
|
||||
Optional[str],
|
||||
Field(default=None, validation_alias=AliasChoices("query", "q", "search", "text")),
|
||||
] = None,
|
||||
project: Optional[str] = None,
|
||||
workspace: Optional[str] = None,
|
||||
page: int = 1,
|
||||
page_size: int = 10,
|
||||
project_id: Optional[str] = None,
|
||||
# `offset` is intentionally NOT aliased to `page`: offset is item-indexed
|
||||
# (skip N items) while page is 1-indexed page-number. Direct aliasing would
|
||||
# silently return the wrong slice.
|
||||
page: Annotated[
|
||||
int,
|
||||
Field(default=1, validation_alias=AliasChoices("page", "page_number")),
|
||||
] = 1,
|
||||
page_size: Annotated[
|
||||
int,
|
||||
Field(default=10, validation_alias=AliasChoices("page_size", "limit", "per_page")),
|
||||
] = 10,
|
||||
search_type: str | None = None,
|
||||
output_format: Literal["text", "json"] = "text",
|
||||
# Plural-vs-singular trips models constantly. Accept the singular too.
|
||||
note_types: Annotated[
|
||||
List[str] | None,
|
||||
BeforeValidator(coerce_list),
|
||||
Field(default=None, validation_alias=AliasChoices("note_types", "note_type", "types")),
|
||||
"Filter by the 'type' field in note frontmatter (e.g. 'note', 'chapter', 'person'). "
|
||||
"Case-insensitive.",
|
||||
] = None,
|
||||
entity_types: Annotated[
|
||||
List[str] | None,
|
||||
BeforeValidator(coerce_list),
|
||||
Field(default=None, validation_alias=AliasChoices("entity_types", "entity_type")),
|
||||
"Filter by knowledge graph item type: 'entity' (whole notes), 'observation', or "
|
||||
"'relation'. Defaults to 'entity'. Do NOT pass schema/frontmatter types like "
|
||||
"'Chapter' here — use note_types instead.",
|
||||
] = None,
|
||||
after_date: Optional[str] = None,
|
||||
# Time-filter naming varies wildly across APIs.
|
||||
after_date: Annotated[
|
||||
Optional[str],
|
||||
Field(
|
||||
default=None,
|
||||
validation_alias=AliasChoices("after_date", "since", "after", "from_date"),
|
||||
),
|
||||
] = None,
|
||||
metadata_filters: Annotated[
|
||||
Dict[str, Any] | None,
|
||||
BeforeValidator(coerce_dict),
|
||||
@@ -331,7 +355,13 @@ async def search_notes(
|
||||
BeforeValidator(coerce_list),
|
||||
] = None,
|
||||
status: Optional[str] = None,
|
||||
min_similarity: Optional[float] = None,
|
||||
min_similarity: Annotated[
|
||||
Optional[float],
|
||||
Field(
|
||||
default=None,
|
||||
validation_alias=AliasChoices("min_similarity", "threshold", "similarity_threshold"),
|
||||
),
|
||||
] = None,
|
||||
context: Context | None = None,
|
||||
) -> dict | str:
|
||||
"""Search across all content in the knowledge base with comprehensive syntax support.
|
||||
@@ -415,6 +445,9 @@ async def search_notes(
|
||||
Omit or pass None for filter-only searches using metadata_filters, tags, or status.
|
||||
project: Project name to search in. Optional - server will resolve using hierarchy.
|
||||
If unknown, use list_memory_projects() to discover available projects.
|
||||
project_id: Project external_id (UUID). Prefer this over `project` when known —
|
||||
it routes to the exact project regardless of name collisions across cloud
|
||||
workspaces. Takes precedence over `project`. Get from list_memory_projects().
|
||||
page: The page number of results to return (default 1)
|
||||
page_size: The number of results to return per page (default 10)
|
||||
search_type: Type of search to perform, one of:
|
||||
@@ -518,18 +551,23 @@ async def search_notes(
|
||||
remainder = re.sub(r"\b(AND|OR|NOT)\b", "", remainder).strip()
|
||||
query = remainder or None
|
||||
|
||||
# Detect project from memory URL prefix before routing
|
||||
if project is None and query is not None:
|
||||
detected = detect_project_from_url_prefix(query, ConfigManager().config)
|
||||
# Detect project from memory URL prefix before routing.
|
||||
# project_id routes by external UUID, so it bypasses URL discovery entirely.
|
||||
if project is None and project_id is None and query is not None:
|
||||
detected = await detect_project_from_memory_url_prefix(
|
||||
query,
|
||||
ConfigManager().config,
|
||||
context=context,
|
||||
)
|
||||
if detected:
|
||||
project = detected
|
||||
|
||||
with telemetry.operation(
|
||||
with logfire.span(
|
||||
"mcp.tool.search_notes",
|
||||
entrypoint="mcp",
|
||||
tool_name="search_notes",
|
||||
requested_project=project,
|
||||
workspace_id=workspace,
|
||||
requested_project_id=project_id,
|
||||
search_type=search_type or "default",
|
||||
output_format=output_format,
|
||||
page=page,
|
||||
@@ -543,140 +581,140 @@ async def search_notes(
|
||||
has_tags_filter=bool(tags),
|
||||
has_status_filter=bool(status),
|
||||
):
|
||||
async with get_project_client(project, workspace, context) as (client, active_project):
|
||||
with telemetry.contextualize(
|
||||
project_name=active_project.name,
|
||||
workspace_id=workspace,
|
||||
tool_name="search_notes",
|
||||
):
|
||||
# Handle memory:// URLs by resolving to permalink search
|
||||
is_memory_url = False
|
||||
if query is not None:
|
||||
_, resolved_query, is_memory_url = await resolve_project_and_path(
|
||||
client, query, project, context
|
||||
)
|
||||
if is_memory_url:
|
||||
query = resolved_query
|
||||
effective_search_type = search_type or _default_search_type()
|
||||
async with get_project_client(project, context=context, project_id=project_id) as (
|
||||
client,
|
||||
active_project,
|
||||
):
|
||||
# Handle memory:// URLs by resolving to permalink search.
|
||||
# Use active_project.name so resolution hits the cached active project
|
||||
# when project_id was used or `project` was wrong/ambiguous.
|
||||
is_memory_url = False
|
||||
if query is not None:
|
||||
_, resolved_query, is_memory_url = await resolve_project_and_path(
|
||||
client, query, active_project.name, context
|
||||
)
|
||||
if is_memory_url:
|
||||
effective_search_type = "permalink"
|
||||
query = resolved_query
|
||||
effective_search_type = search_type or _default_search_type()
|
||||
if is_memory_url:
|
||||
effective_search_type = "permalink"
|
||||
|
||||
try:
|
||||
# Create a SearchQuery object based on the parameters
|
||||
search_query = SearchQuery()
|
||||
try:
|
||||
# Create a SearchQuery object based on the parameters
|
||||
search_query = SearchQuery()
|
||||
|
||||
# Only map search_type to query fields when there is an actual query string.
|
||||
# When query is None/empty, skip the search mode block — filters-only path.
|
||||
effective_query = (query or "").strip()
|
||||
if effective_query:
|
||||
valid_search_types = {
|
||||
"text",
|
||||
"title",
|
||||
"permalink",
|
||||
"vector",
|
||||
"semantic",
|
||||
"hybrid",
|
||||
}
|
||||
if effective_search_type == "text":
|
||||
search_query.text = effective_query
|
||||
search_query.retrieval_mode = SearchRetrievalMode.FTS
|
||||
elif effective_search_type in ("vector", "semantic"):
|
||||
search_query.text = effective_query
|
||||
search_query.retrieval_mode = SearchRetrievalMode.VECTOR
|
||||
elif effective_search_type == "hybrid":
|
||||
search_query.text = effective_query
|
||||
search_query.retrieval_mode = SearchRetrievalMode.HYBRID
|
||||
elif effective_search_type == "title":
|
||||
search_query.title = effective_query
|
||||
elif effective_search_type == "permalink" and "*" in effective_query:
|
||||
search_query.permalink_match = effective_query
|
||||
elif effective_search_type == "permalink":
|
||||
search_query.permalink = effective_query
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Invalid search_type '{effective_search_type}'. "
|
||||
f"Valid options: {', '.join(sorted(valid_search_types))}"
|
||||
)
|
||||
|
||||
# Add optional filters if provided (empty lists are treated as no filter)
|
||||
if entity_types:
|
||||
search_query.entity_types = [SearchItemType(t) for t in entity_types]
|
||||
if note_types:
|
||||
search_query.note_types = note_types
|
||||
if after_date:
|
||||
search_query.after_date = after_date
|
||||
if metadata_filters:
|
||||
# Alias common column/model names to their frontmatter key equivalents.
|
||||
# Users often pass "note_type" (the entity model column) when the
|
||||
# frontmatter field is actually "type".
|
||||
_METADATA_KEY_ALIASES = {"note_type": "type"}
|
||||
metadata_filters = {
|
||||
_METADATA_KEY_ALIASES.get(k, k): v for k, v in metadata_filters.items()
|
||||
}
|
||||
search_query.metadata_filters = metadata_filters
|
||||
if tags:
|
||||
search_query.tags = tags
|
||||
if status:
|
||||
search_query.status = status
|
||||
if min_similarity is not None:
|
||||
search_query.min_similarity = min_similarity
|
||||
|
||||
# Reject searches with no criteria at all
|
||||
if search_query.no_criteria():
|
||||
return (
|
||||
"# No Search Criteria\n\n"
|
||||
"Please provide at least one of: `query`, `metadata_filters`, "
|
||||
"`tags`, `status`, `note_types`, `entity_types`, or `after_date`."
|
||||
# Only map search_type to query fields when there is an actual query string.
|
||||
# When query is None/empty, skip the search mode block — filters-only path.
|
||||
effective_query = (query or "").strip()
|
||||
if effective_query:
|
||||
valid_search_types = {
|
||||
"text",
|
||||
"title",
|
||||
"permalink",
|
||||
"vector",
|
||||
"semantic",
|
||||
"hybrid",
|
||||
}
|
||||
if effective_search_type == "text":
|
||||
search_query.text = effective_query
|
||||
search_query.retrieval_mode = SearchRetrievalMode.FTS
|
||||
elif effective_search_type in ("vector", "semantic"):
|
||||
search_query.text = effective_query
|
||||
search_query.retrieval_mode = SearchRetrievalMode.VECTOR
|
||||
elif effective_search_type == "hybrid":
|
||||
search_query.text = effective_query
|
||||
search_query.retrieval_mode = SearchRetrievalMode.HYBRID
|
||||
elif effective_search_type == "title":
|
||||
search_query.title = effective_query
|
||||
elif effective_search_type == "permalink" and "*" in effective_query:
|
||||
search_query.permalink_match = effective_query
|
||||
elif effective_search_type == "permalink":
|
||||
search_query.permalink = effective_query
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Invalid search_type '{effective_search_type}'. "
|
||||
f"Valid options: {', '.join(sorted(valid_search_types))}"
|
||||
)
|
||||
|
||||
# Default to entity-level results to avoid returning individual
|
||||
# observations/relations as separate search results (see issue #31).
|
||||
# Applied after no_criteria() so that the implicit default doesn't
|
||||
# mask a truly empty search request.
|
||||
if not search_query.entity_types:
|
||||
search_query.entity_types = [SearchItemType("entity")]
|
||||
# Add optional filters if provided (empty lists are treated as no filter)
|
||||
if entity_types:
|
||||
search_query.entity_types = [SearchItemType(t) for t in entity_types]
|
||||
if note_types:
|
||||
search_query.note_types = note_types
|
||||
if after_date:
|
||||
search_query.after_date = after_date
|
||||
if metadata_filters:
|
||||
# Alias common column/model names to their frontmatter key equivalents.
|
||||
# Users often pass "note_type" (the entity model column) when the
|
||||
# frontmatter field is actually "type".
|
||||
_METADATA_KEY_ALIASES = {"note_type": "type"}
|
||||
metadata_filters = {
|
||||
_METADATA_KEY_ALIASES.get(k, k): v for k, v in metadata_filters.items()
|
||||
}
|
||||
search_query.metadata_filters = metadata_filters
|
||||
if tags:
|
||||
search_query.tags = tags
|
||||
if status:
|
||||
search_query.status = status
|
||||
if min_similarity is not None:
|
||||
search_query.min_similarity = min_similarity
|
||||
|
||||
# Reject searches with no criteria at all
|
||||
if search_query.no_criteria():
|
||||
return (
|
||||
"# No Search Criteria\n\n"
|
||||
"Please provide at least one of: `query`, `metadata_filters`, "
|
||||
"`tags`, `status`, `note_types`, `entity_types`, or `after_date`."
|
||||
)
|
||||
|
||||
# Default to entity-level results to avoid returning individual
|
||||
# observations/relations as separate search results (see issue #31).
|
||||
# Applied after no_criteria() so that the implicit default doesn't
|
||||
# mask a truly empty search request.
|
||||
if not search_query.entity_types:
|
||||
search_query.entity_types = [SearchItemType("entity")]
|
||||
|
||||
logger.debug(
|
||||
f"Search request: project={active_project.name} "
|
||||
f"search_type={effective_search_type} "
|
||||
f"query={effective_query or '<filters-only>'} "
|
||||
f"note_types={len(note_types)} entity_types={len(search_query.entity_types or [])} "
|
||||
f"page={page} page_size={page_size}"
|
||||
)
|
||||
# Import here to avoid circular import (tools → clients → utils → tools)
|
||||
from basic_memory.mcp.clients import SearchClient
|
||||
|
||||
# Use typed SearchClient for API calls
|
||||
search_client = SearchClient(client, active_project.external_id)
|
||||
result = await search_client.search(
|
||||
search_query.model_dump(),
|
||||
page=page,
|
||||
page_size=page_size,
|
||||
)
|
||||
logger.debug(
|
||||
f"Search response: project={active_project.name} "
|
||||
f"results={len(result.results)} has_more={str(result.has_more).lower()} "
|
||||
f"page={result.current_page} page_size={result.page_size}"
|
||||
)
|
||||
|
||||
# Check if we got no results and provide helpful guidance
|
||||
if not result.results:
|
||||
logger.debug(
|
||||
f"Search request: project={active_project.name} "
|
||||
f"search_type={effective_search_type} "
|
||||
f"query={effective_query or '<filters-only>'} "
|
||||
f"note_types={len(note_types)} entity_types={len(search_query.entity_types or [])} "
|
||||
f"page={page} page_size={page_size}"
|
||||
f"Search returned no results for query: {query} in project {active_project.name}"
|
||||
)
|
||||
# Import here to avoid circular import (tools → clients → utils → tools)
|
||||
from basic_memory.mcp.clients import SearchClient
|
||||
# Don't treat this as an error, but the user might want guidance
|
||||
# We return the empty result as normal - the user can decide if they need help
|
||||
|
||||
# Use typed SearchClient for API calls
|
||||
search_client = SearchClient(client, active_project.external_id)
|
||||
result = await search_client.search(
|
||||
search_query.model_dump(),
|
||||
page=page,
|
||||
page_size=page_size,
|
||||
)
|
||||
logger.debug(
|
||||
f"Search response: project={active_project.name} "
|
||||
f"results={len(result.results)} has_more={str(result.has_more).lower()} "
|
||||
f"page={result.current_page} page_size={result.page_size}"
|
||||
)
|
||||
if output_format == "json":
|
||||
return result.model_dump(mode="json", exclude_none=True)
|
||||
|
||||
# Check if we got no results and provide helpful guidance
|
||||
if not result.results:
|
||||
logger.debug(
|
||||
f"Search returned no results for query: {query} in project {active_project.name}"
|
||||
)
|
||||
# Don't treat this as an error, but the user might want guidance
|
||||
# We return the empty result as normal - the user can decide if they need help
|
||||
return _format_search_markdown(result, active_project.name, query)
|
||||
|
||||
if output_format == "json":
|
||||
return result.model_dump(mode="json", exclude_none=True)
|
||||
|
||||
return _format_search_markdown(result, active_project.name, query)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
f"Search failed for query '{query or ''}': {e}, project: {active_project.name}"
|
||||
)
|
||||
# Return formatted error message as string for better user experience
|
||||
return _format_search_error_response(
|
||||
active_project.name, str(e), query or "", effective_search_type
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
f"Search failed for query '{query or ''}': {e}, project: {active_project.name}"
|
||||
)
|
||||
# Return formatted error message as string for better user experience
|
||||
return _format_search_error_response(
|
||||
active_project.name, str(e), query or "", effective_search_type
|
||||
)
|
||||
|
||||
@@ -25,6 +25,7 @@ def _text_block(message: str) -> List[ContentBlock]:
|
||||
async def search_notes_ui(
|
||||
query: str,
|
||||
project: Optional[str] = None,
|
||||
project_id: Optional[str] = None,
|
||||
page: int = 1,
|
||||
page_size: int = 10,
|
||||
search_type: Optional[str] = None,
|
||||
@@ -49,6 +50,7 @@ async def search_notes_ui(
|
||||
result = await search_notes(
|
||||
query=query,
|
||||
project=project,
|
||||
project_id=project_id,
|
||||
page=page,
|
||||
page_size=page_size,
|
||||
search_type=search_type,
|
||||
@@ -97,16 +99,14 @@ async def search_notes_ui(
|
||||
async def read_note_ui(
|
||||
identifier: str,
|
||||
project: Optional[str] = None,
|
||||
page: int = 1,
|
||||
page_size: int = 10,
|
||||
project_id: Optional[str] = None,
|
||||
context: Context | None = None,
|
||||
) -> List[ContentBlock]:
|
||||
"""Return a note preview UI as an embedded MCP-UI resource."""
|
||||
content = await read_note(
|
||||
identifier=identifier,
|
||||
project=project,
|
||||
page=page,
|
||||
page_size=page_size,
|
||||
project_id=project_id,
|
||||
output_format="text",
|
||||
context=context,
|
||||
)
|
||||
@@ -114,8 +114,6 @@ async def read_note_ui(
|
||||
render_data = {
|
||||
"toolInput": {
|
||||
"identifier": identifier,
|
||||
"page": page,
|
||||
"page_size": page_size,
|
||||
},
|
||||
"toolOutput": content,
|
||||
}
|
||||
|
||||
@@ -5,9 +5,9 @@ to the Basic Memory API, with improved error handling and logging.
|
||||
"""
|
||||
|
||||
import typing
|
||||
from contextlib import contextmanager
|
||||
from typing import Optional
|
||||
from typing import Any, Optional
|
||||
|
||||
import logfire
|
||||
from httpx import Response, URL, AsyncClient, HTTPStatusError
|
||||
from httpx._client import UseClientDefault, USE_CLIENT_DEFAULT
|
||||
from httpx._types import (
|
||||
@@ -24,7 +24,6 @@ from httpx._types import (
|
||||
from loguru import logger
|
||||
from mcp.server.fastmcp.exceptions import ToolError
|
||||
|
||||
from basic_memory import telemetry
|
||||
from basic_memory.config import ConfigManager
|
||||
|
||||
|
||||
@@ -41,43 +40,22 @@ def _classify_http_outcome(status_code: int) -> str:
|
||||
return "unknown" # pragma: no cover
|
||||
|
||||
|
||||
class _RequestSpan:
|
||||
"""Small adapter for attaching outcome metadata to a live request span."""
|
||||
def _response_span_attrs(response: Response) -> dict[str, Any]:
|
||||
"""Attributes to attach to a request span after a response lands."""
|
||||
return {
|
||||
"status_code": response.status_code,
|
||||
"is_success": response.is_success,
|
||||
"outcome": _classify_http_outcome(response.status_code),
|
||||
}
|
||||
|
||||
def __init__(self, active_span: typing.Any | None):
|
||||
self._active_span = active_span
|
||||
|
||||
def record_response(self, response: Response) -> None:
|
||||
self._set_attributes(
|
||||
{
|
||||
"status_code": response.status_code,
|
||||
"is_success": response.is_success,
|
||||
"outcome": _classify_http_outcome(response.status_code),
|
||||
}
|
||||
)
|
||||
|
||||
def record_transport_error(self, exc: Exception) -> None:
|
||||
self._set_attributes(
|
||||
{
|
||||
"is_success": False,
|
||||
"outcome": "transport_error",
|
||||
"error_type": type(exc).__name__,
|
||||
}
|
||||
)
|
||||
|
||||
def _set_attributes(self, attrs: dict[str, typing.Any]) -> None:
|
||||
if self._active_span is None:
|
||||
return
|
||||
|
||||
set_attributes = getattr(self._active_span, "set_attributes", None)
|
||||
if callable(set_attributes):
|
||||
set_attributes(attrs)
|
||||
return
|
||||
|
||||
set_attribute = getattr(self._active_span, "set_attribute", None)
|
||||
if callable(set_attribute):
|
||||
for key, value in attrs.items():
|
||||
set_attribute(key, value)
|
||||
def _transport_error_span_attrs(exc: Exception) -> dict[str, Any]:
|
||||
"""Attributes to attach when the transport layer fails before any response."""
|
||||
return {
|
||||
"is_success": False,
|
||||
"outcome": "transport_error",
|
||||
"error_type": type(exc).__name__,
|
||||
}
|
||||
|
||||
|
||||
def get_error_message(
|
||||
@@ -130,15 +108,20 @@ def get_error_message(
|
||||
return f"HTTP error {status_code}: {method} request to '{path}' failed"
|
||||
|
||||
|
||||
def _extract_response_data(response: Response) -> typing.Any:
|
||||
"""Safely decode response payload for error reporting."""
|
||||
try:
|
||||
return response.json()
|
||||
except Exception:
|
||||
def _extract_response_data(response: Response) -> Any:
|
||||
"""Decode the JSON payload of an API response for error reporting.
|
||||
|
||||
Upstream gateways (Fly, Cloudflare, load balancers) can return HTML
|
||||
error pages before the request reaches our FastAPI app; those have no
|
||||
structured `detail` to surface, so we skip them. A malformed body with
|
||||
a JSON content-type is a server bug and we let it raise.
|
||||
"""
|
||||
if "application/json" not in response.headers.get("content-type", ""):
|
||||
return None
|
||||
return response.json()
|
||||
|
||||
|
||||
def _response_detail_text(response_data: typing.Any) -> str | None:
|
||||
def _response_detail_text(response_data: Any) -> str | None:
|
||||
"""Extract textual error detail from API payloads."""
|
||||
if isinstance(response_data, dict):
|
||||
detail = response_data.get("detail")
|
||||
@@ -189,31 +172,6 @@ def _resolve_error_message(
|
||||
return get_error_message(status_code, url, method)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _request_scope(
|
||||
method: str,
|
||||
*,
|
||||
client_name: str | None,
|
||||
operation: str | None,
|
||||
path_template: str | None,
|
||||
params: QueryParamTypes | None = None,
|
||||
has_body: bool = False,
|
||||
):
|
||||
"""Create the shared MCP transport span used by all HTTP helpers."""
|
||||
attrs = {
|
||||
"method": method,
|
||||
"client_name": client_name,
|
||||
"operation": operation,
|
||||
"path_template": path_template,
|
||||
"phase": "request",
|
||||
"has_query": bool(params),
|
||||
"has_body": has_body,
|
||||
}
|
||||
with telemetry.contextualize(**attrs):
|
||||
with telemetry.started_span("mcp.http.request", **attrs) as active_span:
|
||||
yield _RequestSpan(active_span)
|
||||
|
||||
|
||||
async def call_get(
|
||||
client: AsyncClient,
|
||||
url: URL | str,
|
||||
@@ -250,15 +208,18 @@ async def call_get(
|
||||
"""
|
||||
logger.debug(f"Calling GET '{url}' params: '{params}'")
|
||||
error_message = None
|
||||
request_span: _RequestSpan | None = None
|
||||
request_span: logfire.LogfireSpan | None = None
|
||||
|
||||
try:
|
||||
with _request_scope(
|
||||
"GET",
|
||||
with logfire.span(
|
||||
"mcp.http.request",
|
||||
method="GET",
|
||||
client_name=client_name,
|
||||
operation=operation,
|
||||
path_template=path_template,
|
||||
params=params,
|
||||
phase="request",
|
||||
has_query=bool(params),
|
||||
has_body=False,
|
||||
) as request_span:
|
||||
response = await client.get(
|
||||
url,
|
||||
@@ -270,7 +231,7 @@ async def call_get(
|
||||
timeout=timeout,
|
||||
extensions=extensions,
|
||||
)
|
||||
request_span.record_response(response)
|
||||
request_span.set_attributes(_response_span_attrs(response))
|
||||
|
||||
if response.is_success:
|
||||
return response
|
||||
@@ -299,7 +260,7 @@ async def call_get(
|
||||
raise ToolError(error_message) from e
|
||||
except Exception as e:
|
||||
if request_span is not None:
|
||||
request_span.record_transport_error(e)
|
||||
request_span.set_attributes(_transport_error_span_attrs(e))
|
||||
raise
|
||||
|
||||
|
||||
@@ -347,15 +308,17 @@ async def call_put(
|
||||
"""
|
||||
logger.debug(f"Calling PUT '{url}'")
|
||||
error_message = None
|
||||
request_span: _RequestSpan | None = None
|
||||
request_span: logfire.LogfireSpan | None = None
|
||||
|
||||
try:
|
||||
with _request_scope(
|
||||
"PUT",
|
||||
with logfire.span(
|
||||
"mcp.http.request",
|
||||
method="PUT",
|
||||
client_name=client_name,
|
||||
operation=operation,
|
||||
path_template=path_template,
|
||||
params=params,
|
||||
phase="request",
|
||||
has_query=bool(params),
|
||||
has_body=any(value is not None for value in (content, data, files, json)),
|
||||
) as request_span:
|
||||
response = await client.put(
|
||||
@@ -372,7 +335,7 @@ async def call_put(
|
||||
timeout=timeout,
|
||||
extensions=extensions,
|
||||
)
|
||||
request_span.record_response(response)
|
||||
request_span.set_attributes(_response_span_attrs(response))
|
||||
|
||||
if response.is_success:
|
||||
return response
|
||||
@@ -402,7 +365,7 @@ async def call_put(
|
||||
raise ToolError(error_message) from e
|
||||
except Exception as e:
|
||||
if request_span is not None:
|
||||
request_span.record_transport_error(e)
|
||||
request_span.set_attributes(_transport_error_span_attrs(e))
|
||||
raise
|
||||
|
||||
|
||||
@@ -449,15 +412,17 @@ async def call_patch(
|
||||
ToolError: If the request fails with an appropriate error message
|
||||
"""
|
||||
logger.debug(f"Calling PATCH '{url}'")
|
||||
request_span: _RequestSpan | None = None
|
||||
request_span: logfire.LogfireSpan | None = None
|
||||
|
||||
try:
|
||||
with _request_scope(
|
||||
"PATCH",
|
||||
with logfire.span(
|
||||
"mcp.http.request",
|
||||
method="PATCH",
|
||||
client_name=client_name,
|
||||
operation=operation,
|
||||
path_template=path_template,
|
||||
params=params,
|
||||
phase="request",
|
||||
has_query=bool(params),
|
||||
has_body=any(value is not None for value in (content, data, files, json)),
|
||||
) as request_span:
|
||||
response = await client.patch(
|
||||
@@ -474,7 +439,7 @@ async def call_patch(
|
||||
timeout=timeout,
|
||||
extensions=extensions,
|
||||
)
|
||||
request_span.record_response(response)
|
||||
request_span.set_attributes(_response_span_attrs(response))
|
||||
|
||||
if response.is_success:
|
||||
return response
|
||||
@@ -509,7 +474,7 @@ async def call_patch(
|
||||
raise ToolError(error_message) from e
|
||||
except Exception as e:
|
||||
if request_span is not None:
|
||||
request_span.record_transport_error(e)
|
||||
request_span.set_attributes(_transport_error_span_attrs(e))
|
||||
raise
|
||||
|
||||
|
||||
@@ -557,15 +522,17 @@ async def call_post(
|
||||
"""
|
||||
logger.debug(f"Calling POST '{url}'")
|
||||
error_message = None
|
||||
request_span: _RequestSpan | None = None
|
||||
request_span: logfire.LogfireSpan | None = None
|
||||
|
||||
try:
|
||||
with _request_scope(
|
||||
"POST",
|
||||
with logfire.span(
|
||||
"mcp.http.request",
|
||||
method="POST",
|
||||
client_name=client_name,
|
||||
operation=operation,
|
||||
path_template=path_template,
|
||||
params=params,
|
||||
phase="request",
|
||||
has_query=bool(params),
|
||||
has_body=any(value is not None for value in (content, data, files, json)),
|
||||
) as request_span:
|
||||
response = await client.post(
|
||||
@@ -582,7 +549,7 @@ async def call_post(
|
||||
timeout=timeout,
|
||||
extensions=extensions,
|
||||
)
|
||||
request_span.record_response(response)
|
||||
request_span.set_attributes(_response_span_attrs(response))
|
||||
logger.debug(f"response: {_extract_response_data(response)}")
|
||||
|
||||
if response.is_success:
|
||||
@@ -612,7 +579,7 @@ async def call_post(
|
||||
raise ToolError(error_message) from e
|
||||
except Exception as e:
|
||||
if request_span is not None:
|
||||
request_span.record_transport_error(e)
|
||||
request_span.set_attributes(_transport_error_span_attrs(e))
|
||||
raise
|
||||
|
||||
|
||||
@@ -684,15 +651,18 @@ async def call_delete(
|
||||
"""
|
||||
logger.debug(f"Calling DELETE '{url}'")
|
||||
error_message = None
|
||||
request_span: _RequestSpan | None = None
|
||||
request_span: logfire.LogfireSpan | None = None
|
||||
|
||||
try:
|
||||
with _request_scope(
|
||||
"DELETE",
|
||||
with logfire.span(
|
||||
"mcp.http.request",
|
||||
method="DELETE",
|
||||
client_name=client_name,
|
||||
operation=operation,
|
||||
path_template=path_template,
|
||||
params=params,
|
||||
phase="request",
|
||||
has_query=bool(params),
|
||||
has_body=False,
|
||||
) as request_span:
|
||||
response = await client.delete(
|
||||
url=url,
|
||||
@@ -704,7 +674,7 @@ async def call_delete(
|
||||
timeout=timeout,
|
||||
extensions=extensions,
|
||||
)
|
||||
request_span.record_response(response)
|
||||
request_span.set_attributes(_response_span_attrs(response))
|
||||
|
||||
if response.is_success:
|
||||
return response
|
||||
@@ -733,5 +703,5 @@ async def call_delete(
|
||||
raise ToolError(error_message) from e
|
||||
except Exception as e:
|
||||
if request_span is not None:
|
||||
request_span.record_transport_error(e)
|
||||
request_span.set_attributes(_transport_error_span_attrs(e))
|
||||
raise
|
||||
|
||||
@@ -17,9 +17,7 @@ from basic_memory.mcp.tools.read_note import read_note
|
||||
async def view_note(
|
||||
identifier: str,
|
||||
project: Optional[str] = None,
|
||||
workspace: Optional[str] = None,
|
||||
page: int = 1,
|
||||
page_size: int = 10,
|
||||
project_id: Optional[str] = None,
|
||||
context: Context | None = None,
|
||||
) -> str:
|
||||
"""View a markdown note as a formatted artifact.
|
||||
@@ -32,8 +30,9 @@ async def view_note(
|
||||
identifier: The title or permalink of the note to view
|
||||
project: Project name to read from. Optional - server will resolve using hierarchy.
|
||||
If unknown, use list_memory_projects() to discover available projects.
|
||||
page: Page number for paginated results (default: 1)
|
||||
page_size: Number of items per page (default: 10)
|
||||
project_id: Project external_id (UUID). Prefer this over `project` when known —
|
||||
it routes to the exact project regardless of name collisions across cloud
|
||||
workspaces. Takes precedence over `project`. Get from list_memory_projects().
|
||||
context: Optional FastMCP context for performance caching.
|
||||
|
||||
Returns:
|
||||
@@ -46,9 +45,6 @@ async def view_note(
|
||||
# View a note by permalink
|
||||
view_note("meetings/weekly-standup")
|
||||
|
||||
# View with pagination
|
||||
view_note("large-document", page=2, page_size=5)
|
||||
|
||||
# Explicit project specification
|
||||
view_note("Meeting Notes", project="my-project")
|
||||
|
||||
@@ -63,9 +59,7 @@ async def view_note(
|
||||
await read_note(
|
||||
identifier=identifier,
|
||||
project=project,
|
||||
workspace=workspace,
|
||||
page=page,
|
||||
page_size=page_size,
|
||||
project_id=project_id,
|
||||
context=context,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -6,6 +6,42 @@ from fastmcp import Context
|
||||
|
||||
from basic_memory.mcp.project_context import get_available_workspaces
|
||||
from basic_memory.mcp.server import mcp
|
||||
from basic_memory.schemas.cloud import WorkspaceInfo, WorkspaceListResponse
|
||||
|
||||
|
||||
def _personal_workspace() -> WorkspaceInfo:
|
||||
"""Return a display-only personal workspace when discovery has no rows.
|
||||
|
||||
This keeps list_workspaces friendly for non-teams or local-only users. It is not
|
||||
the cloud routing source of truth; project-scoped routing still depends on real
|
||||
workspace discovery and the workspace project index in project_context.
|
||||
"""
|
||||
return WorkspaceInfo(
|
||||
tenant_id="personal",
|
||||
workspace_type="personal",
|
||||
slug="personal",
|
||||
name="Personal",
|
||||
role="owner",
|
||||
is_default=True,
|
||||
has_active_subscription=True,
|
||||
)
|
||||
|
||||
|
||||
def _workspace_list_response(workspaces: list[WorkspaceInfo]) -> WorkspaceListResponse:
|
||||
"""Build the structured MCP response from the shared cloud workspace schema."""
|
||||
if not workspaces:
|
||||
workspaces = [_personal_workspace()]
|
||||
|
||||
default_workspace_id = next(
|
||||
(workspace.tenant_id for workspace in workspaces if workspace.is_default),
|
||||
None,
|
||||
)
|
||||
return WorkspaceListResponse(
|
||||
workspaces=workspaces,
|
||||
count=len(workspaces),
|
||||
default_workspace_id=default_workspace_id,
|
||||
current_workspace_id=None,
|
||||
)
|
||||
|
||||
|
||||
@mcp.tool(
|
||||
@@ -24,40 +60,18 @@ async def list_workspaces(
|
||||
context: Optional FastMCP context for progress/status logging.
|
||||
"""
|
||||
workspaces = await get_available_workspaces(context=context)
|
||||
response = _workspace_list_response(workspaces)
|
||||
|
||||
if output_format == "json":
|
||||
return {
|
||||
"workspaces": [
|
||||
{
|
||||
"tenant_id": ws.tenant_id,
|
||||
"name": ws.name,
|
||||
"workspace_type": ws.workspace_type,
|
||||
"role": ws.role,
|
||||
"organization_id": ws.organization_id,
|
||||
"has_active_subscription": ws.has_active_subscription,
|
||||
}
|
||||
for ws in workspaces
|
||||
],
|
||||
"count": len(workspaces),
|
||||
}
|
||||
return response.model_dump(mode="json")
|
||||
|
||||
if not workspaces:
|
||||
return (
|
||||
"# No Workspaces Available\n\n"
|
||||
"No accessible workspaces were found for this account. "
|
||||
"Ensure the account has an active subscription and tenant access."
|
||||
)
|
||||
|
||||
lines = [
|
||||
f"# Available Workspaces ({len(workspaces)})",
|
||||
"",
|
||||
"Use `workspace` as either the `tenant_id` or unique `name` in project-scoped tool calls.",
|
||||
"",
|
||||
]
|
||||
for workspace in workspaces:
|
||||
lines = [f"# Available Workspaces ({response.count})", ""]
|
||||
for workspace in response.workspaces:
|
||||
default_label = ", default" if workspace.is_default else ""
|
||||
lines.append(
|
||||
f"- {workspace.name} "
|
||||
f"(type={workspace.workspace_type}, role={workspace.role}, tenant_id={workspace.tenant_id})"
|
||||
f"(slug={workspace.slug}, type={workspace.workspace_type}, "
|
||||
f"role={workspace.role}{default_label}, tenant_id={workspace.tenant_id})"
|
||||
)
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
@@ -3,10 +3,10 @@
|
||||
import textwrap
|
||||
from typing import Annotated, List, Union, Optional, Literal
|
||||
|
||||
import logfire
|
||||
from loguru import logger
|
||||
from pydantic import BeforeValidator
|
||||
from pydantic import AliasChoices, BeforeValidator, Field
|
||||
|
||||
from basic_memory import telemetry
|
||||
from basic_memory.config import ConfigManager
|
||||
from basic_memory.mcp.project_context import get_project_client, add_project_metadata
|
||||
from basic_memory.mcp.server import mcp
|
||||
@@ -25,13 +25,21 @@ TagType = Union[List[str], str, None]
|
||||
async def write_note(
|
||||
title: str,
|
||||
content: str,
|
||||
directory: str,
|
||||
# Folder/dir/path are interchangeable in models' training data.
|
||||
directory: Annotated[
|
||||
str,
|
||||
Field(validation_alias=AliasChoices("directory", "folder", "dir", "path")),
|
||||
],
|
||||
project: Optional[str] = None,
|
||||
workspace: Optional[str] = None,
|
||||
project_id: Optional[str] = None,
|
||||
tags: list[str] | str | None = None,
|
||||
note_type: str = "note",
|
||||
metadata: Annotated[dict | None, BeforeValidator(coerce_dict)] = None,
|
||||
overwrite: bool | None = None,
|
||||
# Force/replace are the file-write idioms models default to.
|
||||
overwrite: Annotated[
|
||||
bool | None,
|
||||
Field(default=None, validation_alias=AliasChoices("overwrite", "force", "replace")),
|
||||
] = None,
|
||||
output_format: Literal["text", "json"] = "text",
|
||||
context: Context | None = None,
|
||||
) -> str | dict:
|
||||
@@ -74,6 +82,9 @@ async def write_note(
|
||||
project: Project name to write to. Optional - server will resolve using the
|
||||
hierarchy above. If unknown, use list_memory_projects() to discover
|
||||
available projects.
|
||||
project_id: Project external_id (UUID). Prefer this over `project` when known —
|
||||
it routes to the exact project regardless of name collisions across cloud
|
||||
workspaces. Takes precedence over `project`. Get from list_memory_projects().
|
||||
tags: Tags to categorize the note. Can be a list of strings, a comma-separated string, or None.
|
||||
Note: If passing from external MCP clients, use a string format (e.g. "tag1,tag2,tag3")
|
||||
note_type: Type of note to create (stored in frontmatter). Defaults to "note".
|
||||
@@ -149,180 +160,174 @@ async def write_note(
|
||||
overwrite if overwrite is not None else ConfigManager().config.write_note_overwrite_default
|
||||
)
|
||||
|
||||
with telemetry.operation(
|
||||
with logfire.span(
|
||||
"mcp.tool.write_note",
|
||||
entrypoint="mcp",
|
||||
tool_name="write_note",
|
||||
requested_project=project,
|
||||
workspace_id=workspace,
|
||||
requested_project_id=project_id,
|
||||
note_type=note_type,
|
||||
overwrite=effective_overwrite,
|
||||
output_format=output_format,
|
||||
):
|
||||
async with get_project_client(project, workspace, context) as (client, active_project):
|
||||
with telemetry.contextualize(
|
||||
project_name=active_project.name,
|
||||
workspace_id=workspace,
|
||||
tool_name="write_note",
|
||||
):
|
||||
logger.info(
|
||||
f"MCP tool call tool=write_note project={active_project.name} directory={directory}, title={title}, tags={tags}"
|
||||
)
|
||||
async with get_project_client(project, context=context, project_id=project_id) as (
|
||||
client,
|
||||
active_project,
|
||||
):
|
||||
logger.info(
|
||||
f"MCP tool call tool=write_note project={active_project.name} directory={directory}, title={title}, tags={tags}"
|
||||
)
|
||||
|
||||
# Normalize "/" to empty string for root directory (must happen before validation)
|
||||
if directory == "/":
|
||||
directory = ""
|
||||
# Normalize "/" to empty string for root directory (must happen before validation)
|
||||
if directory == "/":
|
||||
directory = ""
|
||||
|
||||
# Validate directory path to prevent path traversal attacks
|
||||
project_path = active_project.home
|
||||
if directory and not validate_project_path(directory, project_path):
|
||||
logger.warning(
|
||||
"Attempted path traversal attack blocked",
|
||||
directory=directory,
|
||||
project=active_project.name,
|
||||
)
|
||||
if output_format == "json":
|
||||
return {
|
||||
"title": title,
|
||||
"permalink": None,
|
||||
"file_path": None,
|
||||
"checksum": None,
|
||||
"action": "created",
|
||||
"error": "SECURITY_VALIDATION_ERROR",
|
||||
}
|
||||
return f"# Error\n\nDirectory path '{directory}' is not allowed - paths must stay within project boundaries"
|
||||
|
||||
# Process tags using the helper function
|
||||
tag_list = parse_tags(tags)
|
||||
|
||||
# Build entity_metadata from optional metadata, then explicit tags on top
|
||||
# Order matters: explicit tags parameter takes precedence over metadata["tags"]
|
||||
entity_metadata = {}
|
||||
if metadata:
|
||||
entity_metadata.update(metadata)
|
||||
if tag_list:
|
||||
entity_metadata["tags"] = tag_list
|
||||
|
||||
entity = Entity(
|
||||
title=title,
|
||||
# Validate directory path to prevent path traversal attacks
|
||||
project_path = active_project.home
|
||||
if directory and not validate_project_path(directory, project_path):
|
||||
logger.warning(
|
||||
"Attempted path traversal attack blocked",
|
||||
directory=directory,
|
||||
note_type=note_type,
|
||||
content_type="text/markdown",
|
||||
content=content,
|
||||
entity_metadata=entity_metadata or None,
|
||||
)
|
||||
|
||||
# Import here to avoid circular import
|
||||
from basic_memory.mcp.clients import KnowledgeClient
|
||||
|
||||
# Use typed KnowledgeClient for API calls
|
||||
knowledge_client = KnowledgeClient(client, active_project.external_id)
|
||||
|
||||
# Try to create the entity first (optimistic create)
|
||||
logger.debug(f"Attempting to create entity permalink={entity.permalink}")
|
||||
action = "Created" # Default to created
|
||||
try:
|
||||
result = await knowledge_client.create_entity(entity.model_dump(), fast=False)
|
||||
action = "Created"
|
||||
except Exception as e:
|
||||
# If creation failed due to conflict (already exists), try to update
|
||||
if (
|
||||
"409" in str(e)
|
||||
or "conflict" in str(e).lower()
|
||||
or "already exists" in str(e).lower()
|
||||
):
|
||||
# Guard: block overwrite unless explicitly enabled
|
||||
if not effective_overwrite:
|
||||
logger.warning(
|
||||
f"write_note blocked: note already exists (overwrite not enabled) "
|
||||
f"permalink={entity.permalink}"
|
||||
)
|
||||
if output_format == "json":
|
||||
return {
|
||||
"title": title,
|
||||
"permalink": entity.permalink,
|
||||
"file_path": None,
|
||||
"checksum": None,
|
||||
"action": "conflict",
|
||||
"error": "NOTE_ALREADY_EXISTS",
|
||||
}
|
||||
return _format_overwrite_error(
|
||||
title, entity.permalink, active_project.name
|
||||
)
|
||||
|
||||
logger.debug(
|
||||
f"Entity exists, updating instead permalink={entity.permalink}"
|
||||
)
|
||||
try:
|
||||
if not entity.permalink:
|
||||
raise ValueError(
|
||||
"Entity permalink is required for updates"
|
||||
) # pragma: no cover
|
||||
entity_id = await knowledge_client.resolve_entity(entity.permalink)
|
||||
result = await knowledge_client.update_entity(
|
||||
entity_id, entity.model_dump(), fast=False
|
||||
)
|
||||
action = "Updated"
|
||||
except Exception as update_error: # pragma: no cover
|
||||
# Re-raise the original error if update also fails
|
||||
raise e from update_error # pragma: no cover
|
||||
else:
|
||||
# Re-raise if it's not a conflict error
|
||||
raise # pragma: no cover
|
||||
summary = [
|
||||
f"# {action} note",
|
||||
f"project: {active_project.name}",
|
||||
f"file_path: {result.file_path}",
|
||||
f"permalink: {result.permalink}",
|
||||
f"checksum: {result.checksum[:8] if result.checksum else 'unknown'}",
|
||||
]
|
||||
|
||||
# Count observations by category
|
||||
categories = {}
|
||||
if result.observations:
|
||||
for obs in result.observations:
|
||||
categories[obs.category] = categories.get(obs.category, 0) + 1
|
||||
|
||||
summary.append("\n## Observations")
|
||||
for category, count in sorted(categories.items()):
|
||||
summary.append(f"- {category}: {count}")
|
||||
|
||||
# Count resolved/unresolved relations
|
||||
unresolved = 0
|
||||
resolved = 0
|
||||
if result.relations:
|
||||
unresolved = sum(1 for r in result.relations if not r.to_id)
|
||||
resolved = len(result.relations) - unresolved
|
||||
|
||||
summary.append("\n## Relations")
|
||||
summary.append(f"- Resolved: {resolved}")
|
||||
if unresolved:
|
||||
summary.append(f"- Unresolved: {unresolved}")
|
||||
summary.append(
|
||||
"\nNote: Unresolved relations point to entities that don't exist yet."
|
||||
)
|
||||
summary.append(
|
||||
"They will be automatically resolved when target entities are created or during sync operations."
|
||||
)
|
||||
|
||||
if tag_list:
|
||||
summary.append(f"\n## Tags\n- {', '.join(tag_list)}")
|
||||
|
||||
# Log the response with structured data
|
||||
logger.info(
|
||||
f"MCP tool response: tool=write_note project={active_project.name} action={action} permalink={result.permalink} observations_count={len(result.observations)} relations_count={len(result.relations)} resolved_relations={resolved} unresolved_relations={unresolved}"
|
||||
project=active_project.name,
|
||||
)
|
||||
if output_format == "json":
|
||||
return {
|
||||
"title": result.title,
|
||||
"permalink": result.permalink,
|
||||
"file_path": result.file_path,
|
||||
"checksum": result.checksum,
|
||||
"action": action.lower(),
|
||||
"title": title,
|
||||
"permalink": None,
|
||||
"file_path": None,
|
||||
"checksum": None,
|
||||
"action": "created",
|
||||
"error": "SECURITY_VALIDATION_ERROR",
|
||||
}
|
||||
return f"# Error\n\nDirectory path '{directory}' is not allowed - paths must stay within project boundaries"
|
||||
|
||||
summary_result = "\n".join(summary)
|
||||
return add_project_metadata(summary_result, active_project.name)
|
||||
# Process tags using the helper function
|
||||
tag_list = parse_tags(tags)
|
||||
|
||||
# Build entity_metadata from optional metadata, then explicit tags on top
|
||||
# Order matters: explicit tags parameter takes precedence over metadata["tags"]
|
||||
entity_metadata = {}
|
||||
if metadata:
|
||||
entity_metadata.update(metadata)
|
||||
if tag_list:
|
||||
entity_metadata["tags"] = tag_list
|
||||
|
||||
entity = Entity(
|
||||
title=title,
|
||||
directory=directory,
|
||||
note_type=note_type,
|
||||
content_type="text/markdown",
|
||||
content=content,
|
||||
entity_metadata=entity_metadata or None,
|
||||
)
|
||||
|
||||
# Import here to avoid circular import
|
||||
from basic_memory.mcp.clients import KnowledgeClient
|
||||
|
||||
# Use typed KnowledgeClient for API calls
|
||||
knowledge_client = KnowledgeClient(client, active_project.external_id)
|
||||
|
||||
# Try to create the entity first (optimistic create)
|
||||
logger.debug(f"Attempting to create entity permalink={entity.permalink}")
|
||||
action = "Created" # Default to created
|
||||
try:
|
||||
result = await knowledge_client.create_entity(entity.model_dump())
|
||||
action = "Created"
|
||||
except Exception as e:
|
||||
# If creation failed due to conflict (already exists), try to update
|
||||
if (
|
||||
"409" in str(e)
|
||||
or "conflict" in str(e).lower()
|
||||
or "already exists" in str(e).lower()
|
||||
):
|
||||
# Guard: block overwrite unless explicitly enabled
|
||||
if not effective_overwrite:
|
||||
logger.warning(
|
||||
f"write_note blocked: note already exists (overwrite not enabled) "
|
||||
f"permalink={entity.permalink}"
|
||||
)
|
||||
if output_format == "json":
|
||||
return {
|
||||
"title": title,
|
||||
"permalink": entity.permalink,
|
||||
"file_path": None,
|
||||
"checksum": None,
|
||||
"action": "conflict",
|
||||
"error": "NOTE_ALREADY_EXISTS",
|
||||
}
|
||||
return _format_overwrite_error(title, entity.permalink, active_project.name)
|
||||
|
||||
logger.debug(f"Entity exists, updating instead permalink={entity.permalink}")
|
||||
try:
|
||||
if not entity.permalink:
|
||||
raise ValueError(
|
||||
"Entity permalink is required for updates"
|
||||
) # pragma: no cover
|
||||
entity_id = await knowledge_client.resolve_entity(entity.permalink)
|
||||
result = await knowledge_client.update_entity(
|
||||
entity_id, entity.model_dump()
|
||||
)
|
||||
action = "Updated"
|
||||
except Exception as update_error: # pragma: no cover
|
||||
# Re-raise the original error if update also fails
|
||||
raise e from update_error # pragma: no cover
|
||||
else:
|
||||
# Re-raise if it's not a conflict error
|
||||
raise # pragma: no cover
|
||||
summary = [
|
||||
f"# {action} note",
|
||||
f"project: {active_project.name}",
|
||||
f"file_path: {result.file_path}",
|
||||
f"permalink: {result.permalink}",
|
||||
f"checksum: {result.checksum[:8] if result.checksum else 'unknown'}",
|
||||
]
|
||||
|
||||
# Count observations by category
|
||||
categories = {}
|
||||
if result.observations:
|
||||
for obs in result.observations:
|
||||
categories[obs.category] = categories.get(obs.category, 0) + 1
|
||||
|
||||
summary.append("\n## Observations")
|
||||
for category, count in sorted(categories.items()):
|
||||
summary.append(f"- {category}: {count}")
|
||||
|
||||
# Count resolved/unresolved relations
|
||||
unresolved = 0
|
||||
resolved = 0
|
||||
if result.relations:
|
||||
unresolved = sum(1 for r in result.relations if not r.to_id)
|
||||
resolved = len(result.relations) - unresolved
|
||||
|
||||
summary.append("\n## Relations")
|
||||
summary.append(f"- Resolved: {resolved}")
|
||||
if unresolved:
|
||||
summary.append(f"- Unresolved: {unresolved}")
|
||||
summary.append(
|
||||
"\nNote: Unresolved relations point to entities that don't exist yet."
|
||||
)
|
||||
summary.append(
|
||||
"They will be automatically resolved when target entities are created or during sync operations."
|
||||
)
|
||||
|
||||
if tag_list:
|
||||
summary.append(f"\n## Tags\n- {', '.join(tag_list)}")
|
||||
|
||||
# Log the response with structured data
|
||||
logger.info(
|
||||
f"MCP tool response: tool=write_note project={active_project.name} action={action} permalink={result.permalink} observations_count={len(result.observations)} relations_count={len(result.relations)} resolved_relations={resolved} unresolved_relations={unresolved}"
|
||||
)
|
||||
if output_format == "json":
|
||||
return {
|
||||
"title": result.title,
|
||||
"permalink": result.permalink,
|
||||
"file_path": result.file_path,
|
||||
"checksum": result.checksum,
|
||||
"action": action.lower(),
|
||||
}
|
||||
|
||||
summary_result = "\n".join(summary)
|
||||
return add_project_metadata(summary_result, active_project.name)
|
||||
|
||||
|
||||
def _format_overwrite_error(title: str, permalink: str | None, project_name: str) -> str:
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
"""Base model class for SQLAlchemy models."""
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from sqlalchemy.ext.asyncio import AsyncAttrs
|
||||
from sqlalchemy.orm import DeclarativeBase
|
||||
|
||||
@@ -7,4 +9,5 @@ from sqlalchemy.orm import DeclarativeBase
|
||||
class Base(AsyncAttrs, DeclarativeBase):
|
||||
"""Base class for all models"""
|
||||
|
||||
pass
|
||||
if TYPE_CHECKING:
|
||||
id: int
|
||||
|
||||
@@ -62,7 +62,7 @@ class Entity(Base):
|
||||
)
|
||||
|
||||
# Core identity
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True) # pyright: ignore [reportIncompatibleVariableOverride]
|
||||
# External UUID for API references - stable identifier that won't change
|
||||
external_id: Mapped[str] = mapped_column(String, unique=True, default=lambda: str(uuid.uuid4()))
|
||||
title: Mapped[str] = mapped_column(String)
|
||||
@@ -229,7 +229,7 @@ class Observation(Base):
|
||||
Index("ix_observation_category", "category"), # Add category index
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True) # pyright: ignore [reportIncompatibleVariableOverride]
|
||||
project_id: Mapped[int] = mapped_column(Integer, ForeignKey("project.id"), index=True)
|
||||
entity_id: Mapped[int] = mapped_column(Integer, ForeignKey("entity.id", ondelete="CASCADE"))
|
||||
content: Mapped[str] = mapped_column(Text)
|
||||
@@ -276,7 +276,7 @@ class Relation(Base):
|
||||
Index("ix_relation_to_id", "to_id"),
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True) # pyright: ignore [reportIncompatibleVariableOverride]
|
||||
project_id: Mapped[int] = mapped_column(Integer, ForeignKey("project.id"), index=True)
|
||||
from_id: Mapped[int] = mapped_column(Integer, ForeignKey("entity.id", ondelete="CASCADE"))
|
||||
to_id: Mapped[Optional[int]] = mapped_column(
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
import os
|
||||
from threading import Lock
|
||||
|
||||
from basic_memory.config import BasicMemoryConfig
|
||||
from basic_memory.config import BasicMemoryConfig, default_fastembed_cache_dir
|
||||
from basic_memory.repository.embedding_provider import EmbeddingProvider
|
||||
|
||||
type ProviderCacheKey = tuple[
|
||||
@@ -12,7 +12,7 @@ type ProviderCacheKey = tuple[
|
||||
int | None,
|
||||
int,
|
||||
int,
|
||||
str | None,
|
||||
str,
|
||||
int | None,
|
||||
int | None,
|
||||
]
|
||||
@@ -22,6 +22,20 @@ _EMBEDDING_PROVIDER_CACHE_LOCK = Lock()
|
||||
_FASTEMBED_MAX_THREADS = 8
|
||||
|
||||
|
||||
def _resolve_cache_dir(app_config: BasicMemoryConfig) -> str:
|
||||
"""Resolve the effective FastEmbed cache dir for this config.
|
||||
|
||||
Uses an explicit ``is not None`` check — an empty string override from
|
||||
config or ``BASIC_MEMORY_SEMANTIC_EMBEDDING_CACHE_DIR`` is an invalid
|
||||
path, not a request to fall back to the default, and FastEmbed's error
|
||||
message is clearer than silently swapping in a different directory.
|
||||
"""
|
||||
configured = app_config.semantic_embedding_cache_dir
|
||||
if configured is not None:
|
||||
return configured
|
||||
return default_fastembed_cache_dir()
|
||||
|
||||
|
||||
def _available_cpu_count() -> int | None:
|
||||
"""Return the CPU budget available to this process when the runtime exposes it."""
|
||||
process_cpu_count = getattr(os, "process_cpu_count", None)
|
||||
@@ -61,7 +75,12 @@ def _resolve_fastembed_runtime_knobs(
|
||||
|
||||
|
||||
def _provider_cache_key(app_config: BasicMemoryConfig) -> ProviderCacheKey:
|
||||
"""Build a stable cache key from provider-relevant semantic embedding config."""
|
||||
"""Build a stable cache key from provider-relevant semantic embedding config.
|
||||
|
||||
Uses the *resolved* cache dir — not the raw config field — so different
|
||||
FASTEMBED_CACHE_PATH values produce distinct cache keys even when the
|
||||
config field itself is unset.
|
||||
"""
|
||||
resolved_threads, resolved_parallel = _resolve_fastembed_runtime_knobs(app_config)
|
||||
return (
|
||||
app_config.semantic_embedding_provider.strip().lower(),
|
||||
@@ -69,7 +88,7 @@ def _provider_cache_key(app_config: BasicMemoryConfig) -> ProviderCacheKey:
|
||||
app_config.semantic_embedding_dimensions,
|
||||
app_config.semantic_embedding_batch_size,
|
||||
app_config.semantic_embedding_request_concurrency,
|
||||
app_config.semantic_embedding_cache_dir,
|
||||
_resolve_cache_dir(app_config),
|
||||
resolved_threads,
|
||||
resolved_parallel,
|
||||
)
|
||||
@@ -103,8 +122,12 @@ def create_embedding_provider(app_config: BasicMemoryConfig) -> EmbeddingProvide
|
||||
from basic_memory.repository.fastembed_provider import FastEmbedEmbeddingProvider
|
||||
|
||||
resolved_threads, resolved_parallel = _resolve_fastembed_runtime_knobs(app_config)
|
||||
if app_config.semantic_embedding_cache_dir is not None:
|
||||
extra_kwargs["cache_dir"] = app_config.semantic_embedding_cache_dir
|
||||
# Trigger: cache_dir is resolved rather than passed through directly.
|
||||
# Why: FastEmbed's own default caches to <system tmp>/fastembed_cache,
|
||||
# which disappears in sandboxed MCP runtimes (e.g. Codex CLI). See #741.
|
||||
# Outcome: always pass an explicit, user-writable cache dir so the ONNX
|
||||
# model persists across runs.
|
||||
extra_kwargs["cache_dir"] = _resolve_cache_dir(app_config)
|
||||
if resolved_threads is not None:
|
||||
extra_kwargs["threads"] = resolved_threads
|
||||
if resolved_parallel is not None:
|
||||
|
||||
@@ -33,7 +33,7 @@ class EntityRepository(Repository[Entity]):
|
||||
"""
|
||||
super().__init__(session_maker, Entity, project_id=project_id)
|
||||
|
||||
async def get_by_id(self, entity_id: int) -> Optional[Entity]: # pragma: no cover
|
||||
async def get_by_id(self, entity_id: int, *, load_relations: bool = True) -> Optional[Entity]:
|
||||
"""Get entity by numeric ID.
|
||||
|
||||
Args:
|
||||
@@ -43,6 +43,10 @@ class EntityRepository(Repository[Entity]):
|
||||
Entity if found, None otherwise
|
||||
"""
|
||||
async with db.scoped_session(self.session_maker) as session:
|
||||
if not load_relations:
|
||||
result = await session.execute(self.select().where(Entity.id == entity_id))
|
||||
return result.scalars().one_or_none()
|
||||
|
||||
return await self.select_by_id(session, entity_id)
|
||||
|
||||
async def _find_one_by_query(self, query, *, load_relations: bool) -> Optional[Entity]:
|
||||
|
||||
@@ -11,7 +11,7 @@ from basic_memory.repository.embedding_provider import EmbeddingProvider
|
||||
from basic_memory.repository.semantic_errors import SemanticDependenciesMissingError
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from fastembed import TextEmbedding # type: ignore[import-not-found] # pragma: no cover
|
||||
from fastembed import TextEmbedding # pragma: no cover
|
||||
|
||||
|
||||
class FastEmbedEmbeddingProvider(EmbeddingProvider):
|
||||
@@ -62,7 +62,7 @@ class FastEmbedEmbeddingProvider(EmbeddingProvider):
|
||||
|
||||
def _create_model() -> "TextEmbedding":
|
||||
try:
|
||||
from fastembed import TextEmbedding # type: ignore[import-not-found]
|
||||
from fastembed import TextEmbedding
|
||||
except (
|
||||
ImportError
|
||||
) as exc: # pragma: no cover - exercised via tests with monkeypatch
|
||||
|
||||
@@ -50,7 +50,7 @@ class OpenAIEmbeddingProvider(EmbeddingProvider):
|
||||
return self._client
|
||||
|
||||
try:
|
||||
from openai import AsyncOpenAI # type: ignore[import-not-found]
|
||||
from openai import AsyncOpenAI
|
||||
except ImportError as exc: # pragma: no cover - covered via monkeypatch tests
|
||||
raise SemanticDependenciesMissingError(
|
||||
"OpenAI dependency is missing. "
|
||||
|
||||
@@ -686,7 +686,17 @@ class PostgresSearchRepository(SearchRepositoryBase):
|
||||
# FTS search (Postgres-specific)
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
async def search(
|
||||
@staticmethod
|
||||
def _is_tsquery_syntax_error(exc: Exception) -> bool:
|
||||
msg = str(exc).lower()
|
||||
return (
|
||||
"syntax error in tsquery" in msg
|
||||
or "invalid input syntax for type tsquery" in msg
|
||||
or "no operand in tsquery" in msg
|
||||
or "no operator in tsquery" in msg
|
||||
)
|
||||
|
||||
async def _build_fts_query_parts(
|
||||
self,
|
||||
search_text: Optional[str] = None,
|
||||
permalink: Optional[str] = None,
|
||||
@@ -695,32 +705,11 @@ class PostgresSearchRepository(SearchRepositoryBase):
|
||||
note_types: Optional[List[str]] = None,
|
||||
after_date: Optional[datetime] = None,
|
||||
search_item_types: Optional[List[SearchItemType]] = None,
|
||||
observation_categories: Optional[List[str]] = None,
|
||||
relation_types: Optional[List[str]] = None,
|
||||
metadata_filters: Optional[dict] = None,
|
||||
retrieval_mode: SearchRetrievalMode = SearchRetrievalMode.FTS,
|
||||
min_similarity: Optional[float] = None,
|
||||
limit: int = 10,
|
||||
offset: int = 0,
|
||||
) -> List[SearchIndexRow]:
|
||||
"""Search across all indexed content using PostgreSQL tsvector."""
|
||||
# --- Dispatch vector / hybrid modes (shared logic) ---
|
||||
dispatched = await self._dispatch_retrieval_mode(
|
||||
search_text=search_text,
|
||||
permalink=permalink,
|
||||
permalink_match=permalink_match,
|
||||
title=title,
|
||||
note_types=note_types,
|
||||
after_date=after_date,
|
||||
search_item_types=search_item_types,
|
||||
metadata_filters=metadata_filters,
|
||||
retrieval_mode=retrieval_mode,
|
||||
min_similarity=min_similarity,
|
||||
limit=limit,
|
||||
offset=offset,
|
||||
)
|
||||
if dispatched is not None:
|
||||
return dispatched
|
||||
|
||||
# --- FTS mode (Postgres-specific) ---
|
||||
) -> tuple[str, str, dict, str, str]:
|
||||
"""Build Postgres FTS FROM/WHERE params shared by search and count."""
|
||||
conditions = []
|
||||
params = {}
|
||||
order_by_clause = ""
|
||||
@@ -766,14 +755,28 @@ class PostgresSearchRepository(SearchRepositoryBase):
|
||||
else:
|
||||
conditions.append("search_index.permalink = :permalink")
|
||||
|
||||
# Handle search item type filter (parameterized for defense-in-depth)
|
||||
if search_item_types:
|
||||
type_placeholders = []
|
||||
for idx, t in enumerate(search_item_types):
|
||||
param_name = f"search_type_{idx}"
|
||||
params[param_name] = t.value
|
||||
type_placeholders.append(f":{param_name}")
|
||||
conditions.append(f"search_index.type IN ({', '.join(type_placeholders)})")
|
||||
# Handle typed search row filters (parameterized for defense-in-depth)
|
||||
self._append_in_filter(
|
||||
conditions,
|
||||
params,
|
||||
column="search_index.type",
|
||||
values=search_item_types,
|
||||
param_prefix="search_type",
|
||||
)
|
||||
self._append_in_filter(
|
||||
conditions,
|
||||
params,
|
||||
column="search_index.category",
|
||||
values=observation_categories,
|
||||
param_prefix="observation_category",
|
||||
)
|
||||
self._append_in_filter(
|
||||
conditions,
|
||||
params,
|
||||
column="search_index.relation_type",
|
||||
values=relation_types,
|
||||
param_prefix="relation_type",
|
||||
)
|
||||
|
||||
# Handle note type filter using JSONB containment (parameterized)
|
||||
if note_types:
|
||||
@@ -868,10 +871,6 @@ class PostgresSearchRepository(SearchRepositoryBase):
|
||||
params["project_id"] = self.project_id
|
||||
conditions.append("search_index.project_id = :project_id")
|
||||
|
||||
# set limit and offset
|
||||
params["limit"] = limit
|
||||
params["offset"] = offset
|
||||
|
||||
# Build WHERE clause
|
||||
where_clause = " AND ".join(conditions) if conditions else "1=1"
|
||||
|
||||
@@ -884,6 +883,70 @@ class PostgresSearchRepository(SearchRepositoryBase):
|
||||
else:
|
||||
score_expr = "0"
|
||||
|
||||
return from_clause, where_clause, params, order_by_clause, score_expr
|
||||
|
||||
async def search(
|
||||
self,
|
||||
search_text: Optional[str] = None,
|
||||
permalink: Optional[str] = None,
|
||||
permalink_match: Optional[str] = None,
|
||||
title: Optional[str] = None,
|
||||
note_types: Optional[List[str]] = None,
|
||||
after_date: Optional[datetime] = None,
|
||||
search_item_types: Optional[List[SearchItemType]] = None,
|
||||
observation_categories: Optional[List[str]] = None,
|
||||
relation_types: Optional[List[str]] = None,
|
||||
metadata_filters: Optional[dict] = None,
|
||||
retrieval_mode: SearchRetrievalMode = SearchRetrievalMode.FTS,
|
||||
min_similarity: Optional[float] = None,
|
||||
limit: int = 10,
|
||||
offset: int = 0,
|
||||
) -> List[SearchIndexRow]:
|
||||
"""Search across all indexed content using PostgreSQL tsvector."""
|
||||
# --- Dispatch vector / hybrid modes (shared logic) ---
|
||||
dispatched = await self._dispatch_retrieval_mode(
|
||||
search_text=search_text,
|
||||
permalink=permalink,
|
||||
permalink_match=permalink_match,
|
||||
title=title,
|
||||
note_types=note_types,
|
||||
after_date=after_date,
|
||||
search_item_types=search_item_types,
|
||||
observation_categories=observation_categories,
|
||||
relation_types=relation_types,
|
||||
metadata_filters=metadata_filters,
|
||||
retrieval_mode=retrieval_mode,
|
||||
min_similarity=min_similarity,
|
||||
limit=limit,
|
||||
offset=offset,
|
||||
)
|
||||
if dispatched is not None:
|
||||
return dispatched
|
||||
|
||||
# --- FTS mode (Postgres-specific) ---
|
||||
(
|
||||
from_clause,
|
||||
where_clause,
|
||||
params,
|
||||
order_by_clause,
|
||||
score_expr,
|
||||
) = await self._build_fts_query_parts(
|
||||
search_text=search_text,
|
||||
permalink=permalink,
|
||||
permalink_match=permalink_match,
|
||||
title=title,
|
||||
note_types=note_types,
|
||||
after_date=after_date,
|
||||
search_item_types=search_item_types,
|
||||
observation_categories=observation_categories,
|
||||
relation_types=relation_types,
|
||||
metadata_filters=metadata_filters,
|
||||
)
|
||||
|
||||
# set limit and offset
|
||||
params["limit"] = limit
|
||||
params["offset"] = offset
|
||||
|
||||
sql = f"""
|
||||
SELECT
|
||||
search_index.project_id,
|
||||
@@ -915,17 +978,7 @@ class PostgresSearchRepository(SearchRepositoryBase):
|
||||
result = await session.execute(text(sql), params)
|
||||
rows = result.fetchall()
|
||||
except Exception as e:
|
||||
# Handle tsquery syntax errors (and only those).
|
||||
#
|
||||
# Important: Postgres errors for other failures (e.g. missing table) will still mention
|
||||
# `to_tsquery(...)` in the SQL text, so checking for the substring "tsquery" is too broad.
|
||||
msg = str(e).lower()
|
||||
if (
|
||||
"syntax error in tsquery" in msg
|
||||
or "invalid input syntax for type tsquery" in msg
|
||||
or "no operand in tsquery" in msg
|
||||
or "no operator in tsquery" in msg
|
||||
):
|
||||
if self._is_tsquery_syntax_error(e):
|
||||
logger.warning(f"tsquery syntax error for search term: {search_text}, error: {e}")
|
||||
return []
|
||||
|
||||
@@ -966,3 +1019,66 @@ class PostgresSearchRepository(SearchRepositoryBase):
|
||||
)
|
||||
|
||||
return results
|
||||
|
||||
async def count(
|
||||
self,
|
||||
search_text: Optional[str] = None,
|
||||
permalink: Optional[str] = None,
|
||||
permalink_match: Optional[str] = None,
|
||||
title: Optional[str] = None,
|
||||
note_types: Optional[List[str]] = None,
|
||||
after_date: Optional[datetime] = None,
|
||||
search_item_types: Optional[List[SearchItemType]] = None,
|
||||
observation_categories: Optional[List[str]] = None,
|
||||
relation_types: Optional[List[str]] = None,
|
||||
metadata_filters: Optional[dict] = None,
|
||||
retrieval_mode: SearchRetrievalMode = SearchRetrievalMode.FTS,
|
||||
min_similarity: Optional[float] = None,
|
||||
) -> int:
|
||||
"""Count indexed content matching the Postgres FTS query."""
|
||||
if retrieval_mode != SearchRetrievalMode.FTS:
|
||||
return await super().count(
|
||||
search_text=search_text,
|
||||
permalink=permalink,
|
||||
permalink_match=permalink_match,
|
||||
title=title,
|
||||
note_types=note_types,
|
||||
after_date=after_date,
|
||||
search_item_types=search_item_types,
|
||||
observation_categories=observation_categories,
|
||||
relation_types=relation_types,
|
||||
metadata_filters=metadata_filters,
|
||||
retrieval_mode=retrieval_mode,
|
||||
min_similarity=min_similarity,
|
||||
)
|
||||
|
||||
(
|
||||
from_clause,
|
||||
where_clause,
|
||||
params,
|
||||
_order_by_clause,
|
||||
_score_expr,
|
||||
) = await self._build_fts_query_parts(
|
||||
search_text=search_text,
|
||||
permalink=permalink,
|
||||
permalink_match=permalink_match,
|
||||
title=title,
|
||||
note_types=note_types,
|
||||
after_date=after_date,
|
||||
search_item_types=search_item_types,
|
||||
observation_categories=observation_categories,
|
||||
relation_types=relation_types,
|
||||
metadata_filters=metadata_filters,
|
||||
)
|
||||
sql = f"SELECT COUNT(*) FROM {from_clause} WHERE {where_clause}"
|
||||
logger.trace(f"Count {sql} params: {params}")
|
||||
try:
|
||||
async with db.scoped_session(self.session_maker) as session:
|
||||
result = await session.execute(text(sql), params)
|
||||
return int(result.scalar_one())
|
||||
except Exception as e:
|
||||
if self._is_tsquery_syntax_error(e):
|
||||
logger.warning(f"tsquery syntax error for search term: {search_text}, error: {e}")
|
||||
return 0
|
||||
logger.error(f"Database error during search count: {e}")
|
||||
raise
|
||||
|
||||
@@ -13,6 +13,7 @@ from sqlalchemy import (
|
||||
Result,
|
||||
and_,
|
||||
delete,
|
||||
update as sqlalchemy_update,
|
||||
)
|
||||
from sqlalchemy.engine import CursorResult
|
||||
from sqlalchemy.exc import NoResultFound
|
||||
@@ -140,6 +141,20 @@ class Repository[T: Base]:
|
||||
# Query within same session
|
||||
return await self.select_by_ids(session, [m.id for m in models]) # pyright: ignore [reportAttributeAccessIssue]
|
||||
|
||||
async def add_all_no_return(self, models: List[T]) -> int:
|
||||
"""Insert models without reloading them afterward."""
|
||||
if not models:
|
||||
return 0
|
||||
|
||||
async with db.scoped_session(self.session_maker) as session:
|
||||
for model in models:
|
||||
self._set_project_id_if_needed(model)
|
||||
|
||||
session.add_all(models)
|
||||
await session.flush()
|
||||
logger.debug(f"Added {len(models)} {self.Model.__name__} records")
|
||||
return len(models)
|
||||
|
||||
def select(self, *entities: Any) -> Select:
|
||||
"""Create a new SELECT statement.
|
||||
|
||||
@@ -268,7 +283,7 @@ class Repository[T: Base]:
|
||||
|
||||
return await self.select_by_ids(session, [model.id for model in model_list]) # pyright: ignore [reportAttributeAccessIssue]
|
||||
|
||||
async def update(self, entity_id: int, entity_data: dict | T) -> Optional[T]:
|
||||
async def update(self, entity_id: int, entity_data: dict[str, Any] | T) -> Optional[T]:
|
||||
"""Update an entity with the given data."""
|
||||
logger.debug(f"Updating {self.Model.__name__} {entity_id} with data: {entity_data}")
|
||||
async with db.scoped_session(self.session_maker) as session:
|
||||
@@ -279,12 +294,13 @@ class Repository[T: Base]:
|
||||
entity = result.scalars().one()
|
||||
|
||||
if isinstance(entity_data, dict):
|
||||
for key, value in entity_data.items():
|
||||
if key in self.valid_columns:
|
||||
setattr(entity, key, value)
|
||||
update_data = cast(dict[str, Any], entity_data)
|
||||
for key in self.valid_columns:
|
||||
if key in update_data:
|
||||
setattr(entity, key, update_data[key])
|
||||
|
||||
elif isinstance(entity_data, self.Model):
|
||||
for column in self.Model.__table__.columns.keys():
|
||||
for column in self.valid_columns:
|
||||
setattr(entity, column, getattr(entity_data, column))
|
||||
|
||||
await session.flush() # Make sure changes are flushed
|
||||
@@ -297,6 +313,25 @@ class Repository[T: Base]:
|
||||
logger.debug(f"No {self.Model.__name__} found to update: {entity_id}")
|
||||
return None
|
||||
|
||||
async def update_fields(self, entity_id: Any, entity_data: dict[str, Any]) -> bool:
|
||||
"""Update columns without reloading the model graph afterward."""
|
||||
update_data = {k: v for k, v in entity_data.items() if k in self.valid_columns}
|
||||
if not update_data:
|
||||
return True
|
||||
|
||||
async with db.scoped_session(self.session_maker) as session:
|
||||
conditions = [self.primary_key == entity_id]
|
||||
if self.has_project_id and self.project_id is not None:
|
||||
conditions.append(getattr(self.Model, "project_id") == self.project_id)
|
||||
|
||||
result = cast(
|
||||
CursorResult[Any],
|
||||
await session.execute(
|
||||
sqlalchemy_update(self.Model).where(and_(*conditions)).values(**update_data)
|
||||
),
|
||||
)
|
||||
return result.rowcount > 0
|
||||
|
||||
async def delete(self, entity_id: int) -> bool:
|
||||
"""Delete an entity from the database."""
|
||||
logger.debug(f"Deleting {self.Model.__name__}: {entity_id}")
|
||||
|
||||
@@ -41,6 +41,8 @@ class SearchRepository(Protocol):
|
||||
note_types: Optional[List[str]] = None,
|
||||
after_date: Optional[datetime] = None,
|
||||
search_item_types: Optional[List[SearchItemType]] = None,
|
||||
observation_categories: Optional[List[str]] = None,
|
||||
relation_types: Optional[List[str]] = None,
|
||||
metadata_filters: Optional[dict] = None,
|
||||
retrieval_mode: SearchRetrievalMode = SearchRetrievalMode.FTS,
|
||||
min_similarity: Optional[float] = None,
|
||||
@@ -50,6 +52,24 @@ class SearchRepository(Protocol):
|
||||
"""Search across indexed content."""
|
||||
...
|
||||
|
||||
async def count(
|
||||
self,
|
||||
search_text: Optional[str] = None,
|
||||
permalink: Optional[str] = None,
|
||||
permalink_match: Optional[str] = None,
|
||||
title: Optional[str] = None,
|
||||
note_types: Optional[List[str]] = None,
|
||||
after_date: Optional[datetime] = None,
|
||||
search_item_types: Optional[List[SearchItemType]] = None,
|
||||
observation_categories: Optional[List[str]] = None,
|
||||
relation_types: Optional[List[str]] = None,
|
||||
metadata_filters: Optional[dict] = None,
|
||||
retrieval_mode: SearchRetrievalMode = SearchRetrievalMode.FTS,
|
||||
min_similarity: Optional[float] = None,
|
||||
) -> int:
|
||||
"""Count indexed content matching the same filters as search."""
|
||||
...
|
||||
|
||||
async def index_item(self, search_index_row: SearchIndexRow) -> None:
|
||||
"""Index a single item."""
|
||||
...
|
||||
|
||||
@@ -12,11 +12,12 @@ from dataclasses import dataclass, field, replace
|
||||
from datetime import datetime
|
||||
from typing import Any, Callable, Dict, List, Optional
|
||||
|
||||
import logfire
|
||||
from loguru import logger
|
||||
from sqlalchemy import Executable, Result, text
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from basic_memory import db, telemetry
|
||||
from basic_memory import db
|
||||
from basic_memory.repository.embedding_provider import EmbeddingProvider
|
||||
from basic_memory.repository.search_index_row import SearchIndexRow
|
||||
from basic_memory.repository.semantic_errors import (
|
||||
@@ -217,6 +218,8 @@ class SearchRepositoryBase(ABC):
|
||||
note_types: Optional[List[str]] = None,
|
||||
after_date: Optional[datetime] = None,
|
||||
search_item_types: Optional[List[SearchItemType]] = None,
|
||||
observation_categories: Optional[List[str]] = None,
|
||||
relation_types: Optional[List[str]] = None,
|
||||
metadata_filters: Optional[Dict[str, Any]] = None,
|
||||
retrieval_mode: SearchRetrievalMode = SearchRetrievalMode.FTS,
|
||||
min_similarity: Optional[float] = None,
|
||||
@@ -233,6 +236,8 @@ class SearchRepositoryBase(ABC):
|
||||
note_types: Filter by note types (from metadata.note_type)
|
||||
after_date: Filter by created_at > after_date
|
||||
search_item_types: Filter by SearchItemType (ENTITY, OBSERVATION, RELATION)
|
||||
observation_categories: Filter observation rows by category
|
||||
relation_types: Filter relation rows by relation_type
|
||||
metadata_filters: Structured frontmatter metadata filters
|
||||
limit: Maximum results to return
|
||||
offset: Number of results to skip
|
||||
@@ -246,6 +251,26 @@ class SearchRepositoryBase(ABC):
|
||||
"""
|
||||
pass
|
||||
|
||||
async def count(
|
||||
self,
|
||||
search_text: Optional[str] = None,
|
||||
permalink: Optional[str] = None,
|
||||
permalink_match: Optional[str] = None,
|
||||
title: Optional[str] = None,
|
||||
note_types: Optional[List[str]] = None,
|
||||
after_date: Optional[datetime] = None,
|
||||
search_item_types: Optional[List[SearchItemType]] = None,
|
||||
observation_categories: Optional[List[str]] = None,
|
||||
relation_types: Optional[List[str]] = None,
|
||||
metadata_filters: Optional[Dict[str, Any]] = None,
|
||||
retrieval_mode: SearchRetrievalMode = SearchRetrievalMode.FTS,
|
||||
min_similarity: Optional[float] = None,
|
||||
) -> int:
|
||||
"""Count results when a backend-specific COUNT query is available."""
|
||||
if retrieval_mode != SearchRetrievalMode.FTS:
|
||||
raise ValueError("Exact counts are only supported for full-text search retrieval.")
|
||||
raise NotImplementedError("Backend search repositories must implement full-text counts.")
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Abstract methods — semantic search (backend-specific DB operations)
|
||||
# ------------------------------------------------------------------
|
||||
@@ -454,6 +479,26 @@ class SearchRepositoryBase(ABC):
|
||||
logger.debug(f"Query executed successfully in {elapsed_time:.2f}s.")
|
||||
return result
|
||||
|
||||
@staticmethod
|
||||
def _append_in_filter(
|
||||
conditions: list[str],
|
||||
params: dict[str, Any],
|
||||
*,
|
||||
column: str,
|
||||
values: list[Any] | None,
|
||||
param_prefix: str,
|
||||
) -> None:
|
||||
"""Append a parameterized SQL IN clause for controlled column names."""
|
||||
if not values:
|
||||
return
|
||||
|
||||
placeholders: list[str] = []
|
||||
for idx, value in enumerate(values):
|
||||
param_name = f"{param_prefix}_{idx}"
|
||||
params[param_name] = value.value if isinstance(value, SearchItemType) else value
|
||||
placeholders.append(f":{param_name}")
|
||||
conditions.append(f"{column} IN ({', '.join(placeholders)})")
|
||||
|
||||
async def delete_entity_vector_rows(self, entity_id: int) -> None:
|
||||
"""Delete one entity's derived vector rows using the backend's cleanup path."""
|
||||
await self._ensure_vector_tables()
|
||||
@@ -845,7 +890,7 @@ class SearchRepositoryBase(ABC):
|
||||
progress_callback(entity_id, completed_entities, total_entities)
|
||||
|
||||
prepare_window_size = self._vector_prepare_window_size()
|
||||
with telemetry.started_span(
|
||||
with logfire.span(
|
||||
"basic_memory.vector_sync.batch",
|
||||
project_id=self.project_id,
|
||||
backend=backend_name,
|
||||
@@ -1068,39 +1113,42 @@ class SearchRepositoryBase(ABC):
|
||||
write_seconds_total=result.write_seconds_total,
|
||||
)
|
||||
batch_total_seconds = time.perf_counter() - batch_start
|
||||
metric_attrs = {
|
||||
batch_attrs = {
|
||||
"backend": backend_name,
|
||||
"skip_only_batch": result.embedding_jobs_total == 0,
|
||||
}
|
||||
telemetry.record_histogram(
|
||||
"vector_sync_batch_total_seconds",
|
||||
batch_total_seconds,
|
||||
unit="s",
|
||||
**metric_attrs,
|
||||
logfire.metric_histogram("vector_sync_batch_total_seconds", unit="s").record(
|
||||
batch_total_seconds, attributes=batch_attrs
|
||||
)
|
||||
telemetry.add_counter(
|
||||
"vector_sync_entities_total", result.entities_total, **metric_attrs
|
||||
logfire.metric_histogram("vector_sync_prepare_seconds", unit="s").record(
|
||||
result.prepare_seconds_total, attributes=batch_attrs
|
||||
)
|
||||
telemetry.add_counter(
|
||||
"vector_sync_entities_skipped",
|
||||
result.entities_skipped,
|
||||
**metric_attrs,
|
||||
logfire.metric_histogram("vector_sync_queue_wait_seconds", unit="s").record(
|
||||
result.queue_wait_seconds_total, attributes=batch_attrs
|
||||
)
|
||||
telemetry.add_counter(
|
||||
"vector_sync_entities_deferred",
|
||||
result.entities_deferred,
|
||||
**metric_attrs,
|
||||
logfire.metric_histogram("vector_sync_embed_seconds", unit="s").record(
|
||||
result.embed_seconds_total, attributes=batch_attrs
|
||||
)
|
||||
telemetry.add_counter(
|
||||
"vector_sync_embedding_jobs_total",
|
||||
result.embedding_jobs_total,
|
||||
**metric_attrs,
|
||||
logfire.metric_histogram("vector_sync_write_seconds", unit="s").record(
|
||||
result.write_seconds_total, attributes=batch_attrs
|
||||
)
|
||||
telemetry.add_counter("vector_sync_chunks_total", result.chunks_total, **metric_attrs)
|
||||
telemetry.add_counter(
|
||||
"vector_sync_chunks_skipped",
|
||||
result.chunks_skipped,
|
||||
**metric_attrs,
|
||||
logfire.metric_counter("vector_sync_entities_total").add(
|
||||
result.entities_total, attributes=batch_attrs
|
||||
)
|
||||
logfire.metric_counter("vector_sync_entities_skipped").add(
|
||||
result.entities_skipped, attributes=batch_attrs
|
||||
)
|
||||
logfire.metric_counter("vector_sync_entities_deferred").add(
|
||||
result.entities_deferred, attributes=batch_attrs
|
||||
)
|
||||
logfire.metric_counter("vector_sync_embedding_jobs_total").add(
|
||||
result.embedding_jobs_total, attributes=batch_attrs
|
||||
)
|
||||
logfire.metric_counter("vector_sync_chunks_total").add(
|
||||
result.chunks_total, attributes=batch_attrs
|
||||
)
|
||||
logfire.metric_counter("vector_sync_chunks_skipped").add(
|
||||
result.chunks_skipped, attributes=batch_attrs
|
||||
)
|
||||
if batch_span is not None:
|
||||
batch_span.set_attributes(
|
||||
@@ -1673,36 +1721,12 @@ class SearchRepositoryBase(ABC):
|
||||
shard_count: int,
|
||||
remaining_jobs_after_shard: int,
|
||||
) -> None:
|
||||
"""Log completion and slow-entity warnings with a consistent format."""
|
||||
backend_name = type(self).__name__.removesuffix("SearchRepository").lower()
|
||||
metric_attrs = {
|
||||
"backend": backend_name,
|
||||
"skip_only_entity": entity_skipped and embedding_jobs_count == 0,
|
||||
}
|
||||
telemetry.record_histogram(
|
||||
"vector_sync_prepare_seconds",
|
||||
prepare_seconds,
|
||||
unit="s",
|
||||
**metric_attrs,
|
||||
)
|
||||
telemetry.record_histogram(
|
||||
"vector_sync_queue_wait_seconds",
|
||||
queue_wait_seconds,
|
||||
unit="s",
|
||||
**metric_attrs,
|
||||
)
|
||||
telemetry.record_histogram(
|
||||
"vector_sync_embed_seconds",
|
||||
embed_seconds,
|
||||
unit="s",
|
||||
**metric_attrs,
|
||||
)
|
||||
telemetry.record_histogram(
|
||||
"vector_sync_write_seconds",
|
||||
write_seconds,
|
||||
unit="s",
|
||||
**metric_attrs,
|
||||
)
|
||||
"""Log completion and slow-entity warnings with a consistent format.
|
||||
|
||||
Per-entity timings are aggregated into `VectorSyncBatchResult` and
|
||||
recorded as batch-level histograms once the batch completes — this
|
||||
function stays on the per-entity hot path so it only emits logs.
|
||||
"""
|
||||
if total_seconds > 10:
|
||||
logger.warning(
|
||||
"Vector sync slow entity: project_id={project_id} entity_id={entity_id} "
|
||||
@@ -1779,6 +1803,8 @@ class SearchRepositoryBase(ABC):
|
||||
note_types: Optional[List[str]],
|
||||
after_date: Optional[datetime],
|
||||
search_item_types: Optional[List[SearchItemType]],
|
||||
observation_categories: Optional[List[str]],
|
||||
relation_types: Optional[List[str]],
|
||||
metadata_filters: Optional[dict],
|
||||
retrieval_mode: SearchRetrievalMode,
|
||||
min_similarity: Optional[float] = None,
|
||||
@@ -1812,6 +1838,8 @@ class SearchRepositoryBase(ABC):
|
||||
note_types=note_types,
|
||||
after_date=after_date,
|
||||
search_item_types=search_item_types,
|
||||
observation_categories=observation_categories,
|
||||
relation_types=relation_types,
|
||||
metadata_filters=metadata_filters,
|
||||
min_similarity=min_similarity,
|
||||
limit=limit,
|
||||
@@ -1831,6 +1859,8 @@ class SearchRepositoryBase(ABC):
|
||||
note_types=note_types,
|
||||
after_date=after_date,
|
||||
search_item_types=search_item_types,
|
||||
observation_categories=observation_categories,
|
||||
relation_types=relation_types,
|
||||
metadata_filters=metadata_filters,
|
||||
min_similarity=min_similarity,
|
||||
limit=limit,
|
||||
@@ -1860,6 +1890,8 @@ class SearchRepositoryBase(ABC):
|
||||
note_types: Optional[List[str]],
|
||||
after_date: Optional[datetime],
|
||||
search_item_types: Optional[List[SearchItemType]],
|
||||
observation_categories: Optional[List[str]],
|
||||
relation_types: Optional[List[str]],
|
||||
metadata_filters: Optional[dict],
|
||||
min_similarity: Optional[float] = None,
|
||||
limit: int,
|
||||
@@ -1970,6 +2002,8 @@ class SearchRepositoryBase(ABC):
|
||||
note_types,
|
||||
after_date,
|
||||
search_item_types,
|
||||
observation_categories,
|
||||
relation_types,
|
||||
metadata_filters,
|
||||
]
|
||||
)
|
||||
@@ -1983,6 +2017,8 @@ class SearchRepositoryBase(ABC):
|
||||
note_types=note_types,
|
||||
after_date=after_date,
|
||||
search_item_types=search_item_types,
|
||||
observation_categories=observation_categories,
|
||||
relation_types=relation_types,
|
||||
metadata_filters=metadata_filters,
|
||||
retrieval_mode=SearchRetrievalMode.FTS,
|
||||
limit=VECTOR_FILTER_SCAN_LIMIT,
|
||||
@@ -2133,6 +2169,8 @@ class SearchRepositoryBase(ABC):
|
||||
note_types: Optional[List[str]],
|
||||
after_date: Optional[datetime],
|
||||
search_item_types: Optional[List[SearchItemType]],
|
||||
observation_categories: Optional[List[str]],
|
||||
relation_types: Optional[List[str]],
|
||||
metadata_filters: Optional[dict],
|
||||
min_similarity: Optional[float] = None,
|
||||
limit: int,
|
||||
@@ -2157,6 +2195,8 @@ class SearchRepositoryBase(ABC):
|
||||
note_types=note_types,
|
||||
after_date=after_date,
|
||||
search_item_types=search_item_types,
|
||||
observation_categories=observation_categories,
|
||||
relation_types=relation_types,
|
||||
metadata_filters=metadata_filters,
|
||||
retrieval_mode=SearchRetrievalMode.FTS,
|
||||
limit=candidate_limit,
|
||||
@@ -2172,6 +2212,8 @@ class SearchRepositoryBase(ABC):
|
||||
note_types=note_types,
|
||||
after_date=after_date,
|
||||
search_item_types=search_item_types,
|
||||
observation_categories=observation_categories,
|
||||
relation_types=relation_types,
|
||||
metadata_filters=metadata_filters,
|
||||
min_similarity=min_similarity,
|
||||
limit=candidate_limit,
|
||||
|
||||
@@ -94,9 +94,22 @@ class SQLiteSearchRepository(SearchRepositoryBase):
|
||||
raise e
|
||||
|
||||
# Fail fast: create vector tables at startup so missing sqlite-vec
|
||||
# or embedding provider errors surface immediately
|
||||
# or embedding provider errors surface immediately.
|
||||
# Trigger: the runtime semantic stack (sqlite-vec extension or embedding
|
||||
# provider) is unavailable at startup.
|
||||
# Why: failing the whole MCP boot for a search-only feature blocks
|
||||
# Claude Desktop's handshake (#711). Keyword-only search is a
|
||||
# reasonable fallback while the user resolves the dependency.
|
||||
# Outcome: log the cause, mark this repository as semantic-disabled so
|
||||
# downstream calls short-circuit cleanly, and let init complete.
|
||||
if self._semantic_enabled:
|
||||
await self._ensure_vector_tables()
|
||||
try:
|
||||
await self._ensure_vector_tables()
|
||||
except SemanticDependenciesMissingError as exc:
|
||||
logger.warning(
|
||||
f"Semantic search disabled: {exc}. Falling back to keyword-only search."
|
||||
)
|
||||
self._semantic_enabled = False
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# FTS5 query preparation (backend-specific)
|
||||
@@ -350,7 +363,7 @@ class SQLiteSearchRepository(SearchRepositoryBase):
|
||||
pass
|
||||
|
||||
try:
|
||||
import sqlite_vec # type: ignore[import-not-found]
|
||||
import sqlite_vec
|
||||
except ImportError as exc:
|
||||
raise SemanticDependenciesMissingError(
|
||||
"sqlite-vec package is missing. "
|
||||
@@ -374,6 +387,25 @@ class SQLiteSearchRepository(SearchRepositoryBase):
|
||||
async_connection = await session.connection()
|
||||
raw_connection = await async_connection.get_raw_connection()
|
||||
driver_connection = raw_connection.driver_connection
|
||||
|
||||
# Trigger: the underlying CPython was built without sqlite extension support.
|
||||
# Why: python.org's macOS installer ships a stripped sqlite3 module with no
|
||||
# enable_load_extension; when uvx happens to pick that interpreter (#711),
|
||||
# the AttributeError surfaces here and previously crashed startup before
|
||||
# Claude Desktop could complete its MCP handshake.
|
||||
# Outcome: convert to SemanticDependenciesMissingError so the init-time
|
||||
# handler can degrade gracefully to keyword search instead of dying.
|
||||
if not hasattr(driver_connection, "enable_load_extension"):
|
||||
raise SemanticDependenciesMissingError(
|
||||
"This Python build does not support SQLite extension loading "
|
||||
"(no enable_load_extension on sqlite3.Connection). "
|
||||
"Common cause: python.org Python on macOS. "
|
||||
"Reinstall basic-memory under a Python that ships extension "
|
||||
"support (uv-managed CPython, Homebrew Python, or the official "
|
||||
"Docker image), or set semantic_search_enabled=false in config "
|
||||
"to silence this and use keyword-only search."
|
||||
)
|
||||
|
||||
await driver_connection.enable_load_extension(True)
|
||||
await driver_connection.load_extension(sqlite_vec.loadable_path())
|
||||
await driver_connection.enable_load_extension(False)
|
||||
@@ -669,7 +701,11 @@ class SQLiteSearchRepository(SearchRepositoryBase):
|
||||
# FTS search (backend-specific)
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
async def search(
|
||||
@staticmethod
|
||||
def _is_fts5_syntax_error(exc: Exception) -> bool:
|
||||
return "fts5: syntax error" in str(exc).lower()
|
||||
|
||||
async def _build_fts_query_parts(
|
||||
self,
|
||||
search_text: Optional[str] = None,
|
||||
permalink: Optional[str] = None,
|
||||
@@ -678,32 +714,11 @@ class SQLiteSearchRepository(SearchRepositoryBase):
|
||||
note_types: Optional[List[str]] = None,
|
||||
after_date: Optional[datetime] = None,
|
||||
search_item_types: Optional[List[SearchItemType]] = None,
|
||||
observation_categories: Optional[List[str]] = None,
|
||||
relation_types: Optional[List[str]] = None,
|
||||
metadata_filters: Optional[dict] = None,
|
||||
retrieval_mode: SearchRetrievalMode = SearchRetrievalMode.FTS,
|
||||
min_similarity: Optional[float] = None,
|
||||
limit: int = 10,
|
||||
offset: int = 0,
|
||||
) -> List[SearchIndexRow]:
|
||||
"""Search across all indexed content using SQLite FTS5."""
|
||||
# --- Dispatch vector / hybrid modes (shared logic) ---
|
||||
dispatched = await self._dispatch_retrieval_mode(
|
||||
search_text=search_text,
|
||||
permalink=permalink,
|
||||
permalink_match=permalink_match,
|
||||
title=title,
|
||||
note_types=note_types,
|
||||
after_date=after_date,
|
||||
search_item_types=search_item_types,
|
||||
metadata_filters=metadata_filters,
|
||||
retrieval_mode=retrieval_mode,
|
||||
min_similarity=min_similarity,
|
||||
limit=limit,
|
||||
offset=offset,
|
||||
)
|
||||
if dispatched is not None:
|
||||
return dispatched
|
||||
|
||||
# --- FTS mode (SQLite-specific) ---
|
||||
) -> tuple[str, str, dict, str]:
|
||||
"""Build SQLite FTS FROM/WHERE params shared by search and count."""
|
||||
conditions = []
|
||||
match_conditions = []
|
||||
params = {}
|
||||
@@ -753,14 +768,28 @@ class SQLiteSearchRepository(SearchRepositoryBase):
|
||||
params["permalink"] = permalink_text
|
||||
match_conditions.append("search_index.permalink MATCH :permalink")
|
||||
|
||||
# Handle entity type filter (parameterized for defense-in-depth)
|
||||
if search_item_types:
|
||||
type_placeholders = []
|
||||
for idx, t in enumerate(search_item_types):
|
||||
param_name = f"search_type_{idx}"
|
||||
params[param_name] = t.value
|
||||
type_placeholders.append(f":{param_name}")
|
||||
conditions.append(f"search_index.type IN ({', '.join(type_placeholders)})")
|
||||
# Handle typed search row filters (parameterized for defense-in-depth)
|
||||
self._append_in_filter(
|
||||
conditions,
|
||||
params,
|
||||
column="search_index.type",
|
||||
values=search_item_types,
|
||||
param_prefix="search_type",
|
||||
)
|
||||
self._append_in_filter(
|
||||
conditions,
|
||||
params,
|
||||
column="search_index.category",
|
||||
values=observation_categories,
|
||||
param_prefix="observation_category",
|
||||
)
|
||||
self._append_in_filter(
|
||||
conditions,
|
||||
params,
|
||||
column="search_index.relation_type",
|
||||
values=relation_types,
|
||||
param_prefix="relation_type",
|
||||
)
|
||||
|
||||
# Handle note type filter (frontmatter type field, parameterized)
|
||||
if note_types:
|
||||
@@ -879,13 +908,66 @@ class SQLiteSearchRepository(SearchRepositoryBase):
|
||||
params["project_id"] = self.project_id
|
||||
conditions.append("search_index.project_id = :project_id")
|
||||
|
||||
# Build WHERE clause
|
||||
where_clause = " AND ".join(conditions) if conditions else "1=1"
|
||||
return from_clause, where_clause, params, order_by_clause
|
||||
|
||||
async def search(
|
||||
self,
|
||||
search_text: Optional[str] = None,
|
||||
permalink: Optional[str] = None,
|
||||
permalink_match: Optional[str] = None,
|
||||
title: Optional[str] = None,
|
||||
note_types: Optional[List[str]] = None,
|
||||
after_date: Optional[datetime] = None,
|
||||
search_item_types: Optional[List[SearchItemType]] = None,
|
||||
observation_categories: Optional[List[str]] = None,
|
||||
relation_types: Optional[List[str]] = None,
|
||||
metadata_filters: Optional[dict] = None,
|
||||
retrieval_mode: SearchRetrievalMode = SearchRetrievalMode.FTS,
|
||||
min_similarity: Optional[float] = None,
|
||||
limit: int = 10,
|
||||
offset: int = 0,
|
||||
) -> List[SearchIndexRow]:
|
||||
"""Search across all indexed content using SQLite FTS5."""
|
||||
# --- Dispatch vector / hybrid modes (shared logic) ---
|
||||
dispatched = await self._dispatch_retrieval_mode(
|
||||
search_text=search_text,
|
||||
permalink=permalink,
|
||||
permalink_match=permalink_match,
|
||||
title=title,
|
||||
note_types=note_types,
|
||||
after_date=after_date,
|
||||
search_item_types=search_item_types,
|
||||
observation_categories=observation_categories,
|
||||
relation_types=relation_types,
|
||||
metadata_filters=metadata_filters,
|
||||
retrieval_mode=retrieval_mode,
|
||||
min_similarity=min_similarity,
|
||||
limit=limit,
|
||||
offset=offset,
|
||||
)
|
||||
if dispatched is not None:
|
||||
return dispatched
|
||||
|
||||
# --- FTS mode (SQLite-specific) ---
|
||||
from_clause, where_clause, params, order_by_clause = await self._build_fts_query_parts(
|
||||
search_text=search_text,
|
||||
permalink=permalink,
|
||||
permalink_match=permalink_match,
|
||||
title=title,
|
||||
note_types=note_types,
|
||||
after_date=after_date,
|
||||
search_item_types=search_item_types,
|
||||
observation_categories=observation_categories,
|
||||
relation_types=relation_types,
|
||||
metadata_filters=metadata_filters,
|
||||
)
|
||||
|
||||
# set limit on search query
|
||||
params["limit"] = limit
|
||||
params["offset"] = offset
|
||||
|
||||
# Build WHERE clause
|
||||
where_clause = " AND ".join(conditions) if conditions else "1=1"
|
||||
|
||||
sql = f"""
|
||||
SELECT
|
||||
search_index.project_id,
|
||||
@@ -918,7 +1000,7 @@ class SQLiteSearchRepository(SearchRepositoryBase):
|
||||
rows = result.fetchall()
|
||||
except Exception as e:
|
||||
# Handle FTS5 syntax errors and provide user-friendly feedback
|
||||
if "fts5: syntax error" in str(e).lower(): # pragma: no cover
|
||||
if self._is_fts5_syntax_error(e): # pragma: no cover
|
||||
logger.warning(f"FTS5 syntax error for search term: {search_text}, error: {e}")
|
||||
# Return empty results rather than crashing
|
||||
return []
|
||||
@@ -956,3 +1038,60 @@ class SQLiteSearchRepository(SearchRepositoryBase):
|
||||
)
|
||||
|
||||
return results
|
||||
|
||||
async def count(
|
||||
self,
|
||||
search_text: Optional[str] = None,
|
||||
permalink: Optional[str] = None,
|
||||
permalink_match: Optional[str] = None,
|
||||
title: Optional[str] = None,
|
||||
note_types: Optional[List[str]] = None,
|
||||
after_date: Optional[datetime] = None,
|
||||
search_item_types: Optional[List[SearchItemType]] = None,
|
||||
observation_categories: Optional[List[str]] = None,
|
||||
relation_types: Optional[List[str]] = None,
|
||||
metadata_filters: Optional[dict] = None,
|
||||
retrieval_mode: SearchRetrievalMode = SearchRetrievalMode.FTS,
|
||||
min_similarity: Optional[float] = None,
|
||||
) -> int:
|
||||
"""Count indexed content matching the SQLite FTS query."""
|
||||
if retrieval_mode != SearchRetrievalMode.FTS:
|
||||
return await super().count(
|
||||
search_text=search_text,
|
||||
permalink=permalink,
|
||||
permalink_match=permalink_match,
|
||||
title=title,
|
||||
note_types=note_types,
|
||||
after_date=after_date,
|
||||
search_item_types=search_item_types,
|
||||
observation_categories=observation_categories,
|
||||
relation_types=relation_types,
|
||||
metadata_filters=metadata_filters,
|
||||
retrieval_mode=retrieval_mode,
|
||||
min_similarity=min_similarity,
|
||||
)
|
||||
|
||||
from_clause, where_clause, params, _order_by_clause = await self._build_fts_query_parts(
|
||||
search_text=search_text,
|
||||
permalink=permalink,
|
||||
permalink_match=permalink_match,
|
||||
title=title,
|
||||
note_types=note_types,
|
||||
after_date=after_date,
|
||||
search_item_types=search_item_types,
|
||||
observation_categories=observation_categories,
|
||||
relation_types=relation_types,
|
||||
metadata_filters=metadata_filters,
|
||||
)
|
||||
sql = f"SELECT COUNT(*) FROM {from_clause} WHERE {where_clause}"
|
||||
logger.trace(f"Count {sql} params: {params}")
|
||||
try:
|
||||
async with db.scoped_session(self.session_maker) as session:
|
||||
result = await session.execute(text(sql), params)
|
||||
return int(result.scalar_one())
|
||||
except Exception as e:
|
||||
if self._is_fts5_syntax_error(e): # pragma: no cover
|
||||
logger.warning(f"FTS5 syntax error for search term: {search_text}, error: {e}")
|
||||
return 0
|
||||
logger.error(f"Database error during search count: {e}")
|
||||
raise
|
||||
|
||||
@@ -63,8 +63,10 @@ class WorkspaceInfo(BaseModel):
|
||||
|
||||
tenant_id: str = Field(..., description="Workspace tenant identifier")
|
||||
workspace_type: str = Field(..., description="Workspace type (personal or organization)")
|
||||
slug: str = Field(..., description="Stable workspace slug for qualified project routing")
|
||||
name: str = Field(..., description="Workspace display name")
|
||||
role: str = Field(..., description="Current user's role in the workspace")
|
||||
is_default: bool = Field(..., description="Whether this is the default cloud workspace")
|
||||
organization_id: str | None = Field(None, description="Organization ID for org workspaces")
|
||||
has_active_subscription: bool = Field(
|
||||
default=False, description="Whether the workspace has an active subscription"
|
||||
@@ -78,6 +80,9 @@ class WorkspaceListResponse(BaseModel):
|
||||
default_factory=list, description="Available workspaces"
|
||||
)
|
||||
count: int = Field(default=0, description="Number of available workspaces")
|
||||
default_workspace_id: str | None = Field(
|
||||
default=None, description="Default workspace tenant ID when available"
|
||||
)
|
||||
current_workspace_id: str | None = Field(
|
||||
default=None, description="Current workspace tenant ID when available"
|
||||
)
|
||||
|
||||
@@ -103,7 +103,7 @@ MemoryUrl = Annotated[
|
||||
memory_url = TypeAdapter(MemoryUrl)
|
||||
|
||||
|
||||
def memory_url_path(url: memory_url) -> str: # pyright: ignore
|
||||
def memory_url_path(url: str) -> str:
|
||||
"""
|
||||
Returns the uri for a url value by removing the prefix "memory://" from a given MemoryUrl.
|
||||
|
||||
|
||||
@@ -194,7 +194,7 @@ class EntityResponse(SQLAlchemyModel):
|
||||
note_type: NoteType
|
||||
|
||||
# COMPAT(v0.18): old clients expect entity_type; remove when no longer needed
|
||||
@computed_field # type: ignore[prop-decorator]
|
||||
@computed_field
|
||||
@property
|
||||
def entity_type(self) -> str:
|
||||
return self.note_type
|
||||
|
||||
@@ -46,6 +46,8 @@ class SearchQuery(BaseModel):
|
||||
- metadata_filters: Structured frontmatter filters (field -> value)
|
||||
- tags: Convenience frontmatter tag filter
|
||||
- status: Convenience frontmatter status filter
|
||||
- observation_categories: Limit observation results to categories
|
||||
- relation_types: Limit relation results to relationship types
|
||||
|
||||
Boolean search examples:
|
||||
- "python AND flask" - Find items with both terms
|
||||
@@ -67,6 +69,8 @@ class SearchQuery(BaseModel):
|
||||
metadata_filters: Optional[dict[str, Any]] = None # Structured frontmatter filters
|
||||
tags: Optional[List[str]] = None # Convenience tag filter
|
||||
status: Optional[str] = None # Convenience status filter
|
||||
observation_categories: Optional[List[str]] = None # Filter observations by category
|
||||
relation_types: Optional[List[str]] = None # Filter relations by relation_type
|
||||
retrieval_mode: SearchRetrievalMode = SearchRetrievalMode.FTS
|
||||
min_similarity: Optional[float] = None # Per-query override for semantic_min_similarity
|
||||
|
||||
@@ -85,6 +89,8 @@ class SearchQuery(BaseModel):
|
||||
status_is_empty = self.status is None or (isinstance(self.status, str) and not self.status)
|
||||
note_types_is_empty = not self.note_types
|
||||
entity_types_is_empty = not self.entity_types
|
||||
observation_categories_is_empty = not self.observation_categories
|
||||
relation_types_is_empty = not self.relation_types
|
||||
return (
|
||||
self.permalink is None
|
||||
and self.permalink_match is None
|
||||
@@ -96,6 +102,8 @@ class SearchQuery(BaseModel):
|
||||
and metadata_is_empty
|
||||
and tags_is_empty
|
||||
and status_is_empty
|
||||
and observation_categories_is_empty
|
||||
and relation_types_is_empty
|
||||
)
|
||||
|
||||
def has_boolean_operators(self) -> bool:
|
||||
@@ -142,4 +150,5 @@ class SearchResponse(BaseModel):
|
||||
results: List[SearchResult]
|
||||
current_page: int
|
||||
page_size: int
|
||||
total: int = 0
|
||||
has_more: bool = False
|
||||
|
||||
@@ -5,7 +5,7 @@ from typing import TYPE_CHECKING, Dict, List, Set
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
# avoid cirular imports
|
||||
# avoid circular imports
|
||||
if TYPE_CHECKING: # pragma: no cover
|
||||
from basic_memory.sync.sync_service import SyncReport
|
||||
|
||||
|
||||
@@ -10,7 +10,7 @@ from typing import List, Optional, Tuple, TYPE_CHECKING
|
||||
from loguru import logger
|
||||
from sqlalchemy import text
|
||||
|
||||
from basic_memory import telemetry
|
||||
import logfire
|
||||
from basic_memory.repository.entity_repository import EntityRepository
|
||||
from basic_memory.repository.observation_repository import ObservationRepository
|
||||
from basic_memory.repository.postgres_search_repository import PostgresSearchRepository
|
||||
@@ -111,7 +111,7 @@ class ContextService:
|
||||
f"Building context for URI: '{memory_url}' depth: '{depth}' since: '{since}' limit: '{limit}' offset: '{offset}' max_related: '{max_related}'"
|
||||
)
|
||||
|
||||
with telemetry.scope(
|
||||
with logfire.span(
|
||||
"memory.build_context",
|
||||
domain="memory",
|
||||
action="build_context",
|
||||
@@ -122,7 +122,7 @@ class ContextService:
|
||||
fetch_limit = limit + 1
|
||||
|
||||
normalized_path: Optional[str] = None
|
||||
with telemetry.scope(
|
||||
with logfire.span(
|
||||
"memory.build_context.resolve_primary",
|
||||
domain="memory",
|
||||
action="build_context",
|
||||
@@ -180,7 +180,7 @@ class ContextService:
|
||||
type_id_pairs = [(r.type, r.id) for r in primary] if primary else []
|
||||
logger.debug(f"found primary type_id_pairs: {len(type_id_pairs)}")
|
||||
|
||||
with telemetry.scope(
|
||||
with logfire.span(
|
||||
"memory.build_context.find_related",
|
||||
domain="memory",
|
||||
action="build_context",
|
||||
@@ -202,7 +202,7 @@ class ContextService:
|
||||
|
||||
observations_by_entity = {}
|
||||
if include_observations and entity_ids:
|
||||
with telemetry.scope(
|
||||
with logfire.span(
|
||||
"memory.build_context.load_observations",
|
||||
domain="memory",
|
||||
action="build_context",
|
||||
@@ -226,7 +226,7 @@ class ContextService:
|
||||
has_more=has_more,
|
||||
)
|
||||
|
||||
with telemetry.scope(
|
||||
with logfire.span(
|
||||
"memory.build_context.shape_results",
|
||||
domain="memory",
|
||||
action="build_context",
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -12,7 +12,7 @@ import aiofiles
|
||||
|
||||
import yaml
|
||||
|
||||
from basic_memory import telemetry
|
||||
import logfire
|
||||
from basic_memory import file_utils
|
||||
|
||||
if TYPE_CHECKING: # pragma: no cover
|
||||
@@ -89,7 +89,7 @@ class FileService:
|
||||
"""
|
||||
logger.debug(f"Reading entity content, entity_id={entity.id}, permalink={entity.permalink}")
|
||||
|
||||
with telemetry.scope(
|
||||
with logfire.span(
|
||||
"file_service.read_content",
|
||||
domain="file_service",
|
||||
action="read_content",
|
||||
@@ -191,7 +191,7 @@ class FileService:
|
||||
full_path = path_obj if path_obj.is_absolute() else self.base_path / path_obj
|
||||
|
||||
try:
|
||||
with telemetry.scope(
|
||||
with logfire.span(
|
||||
"file_service.write",
|
||||
domain="file_service",
|
||||
action="write",
|
||||
@@ -249,7 +249,7 @@ class FileService:
|
||||
full_path = path_obj if path_obj.is_absolute() else self.base_path / path_obj
|
||||
|
||||
try:
|
||||
with telemetry.scope(
|
||||
with logfire.span(
|
||||
"file_service.read_content",
|
||||
domain="file_service",
|
||||
action="read_content",
|
||||
@@ -273,6 +273,9 @@ class FileService:
|
||||
logger.warning("File not found", operation="read_file_content", path=str(full_path))
|
||||
raise
|
||||
except Exception as e:
|
||||
if isinstance(e, FileNotFoundError):
|
||||
logger.warning("File not found", operation="read_file", path=str(full_path))
|
||||
raise
|
||||
logger.exception("File read error", path=str(full_path), error=str(e))
|
||||
raise FileOperationError(f"Failed to read file: {e}")
|
||||
|
||||
@@ -296,7 +299,7 @@ class FileService:
|
||||
full_path = path_obj if path_obj.is_absolute() else self.base_path / path_obj
|
||||
|
||||
try:
|
||||
with telemetry.scope(
|
||||
with logfire.span(
|
||||
"file_service.read_content",
|
||||
domain="file_service",
|
||||
action="read_content",
|
||||
@@ -339,7 +342,7 @@ class FileService:
|
||||
full_path = path_obj if path_obj.is_absolute() else self.base_path / path_obj
|
||||
|
||||
try:
|
||||
with telemetry.scope(
|
||||
with logfire.span(
|
||||
"file_service.read",
|
||||
domain="file_service",
|
||||
action="read",
|
||||
@@ -366,6 +369,9 @@ class FileService:
|
||||
)
|
||||
return content, checksum
|
||||
|
||||
except FileNotFoundError as e:
|
||||
logger.warning("File not found", operation="read_file", path=str(full_path))
|
||||
raise FileOperationError(f"Failed to read file: {e}") from e
|
||||
except Exception as e:
|
||||
logger.exception("File read error", path=str(full_path), error=str(e))
|
||||
raise FileOperationError(f"Failed to read file: {e}")
|
||||
|
||||
@@ -99,18 +99,23 @@ async def initialize_file_sync(
|
||||
)
|
||||
project_repository = ProjectRepository(session_maker)
|
||||
|
||||
# Filter to constrained project if MCP server was started with --project.
|
||||
# Applied to both the initial background sync and the watch service so that
|
||||
# running multiple `basic-memory mcp --project X` processes does not produce
|
||||
# duplicate watchers fighting over the same files.
|
||||
constrained_project = os.environ.get("BASIC_MEMORY_MCP_PROJECT")
|
||||
|
||||
# Initialize watch service
|
||||
watch_service = WatchService(
|
||||
app_config=app_config,
|
||||
project_repository=project_repository,
|
||||
quiet=quiet,
|
||||
constrained_project=constrained_project,
|
||||
)
|
||||
|
||||
# Get active projects
|
||||
active_projects = await project_repository.get_active_projects()
|
||||
|
||||
# Filter to constrained project if MCP server was started with --project
|
||||
constrained_project = os.environ.get("BASIC_MEMORY_MCP_PROJECT")
|
||||
if constrained_project:
|
||||
active_projects = [p for p in active_projects if p.name == constrained_project]
|
||||
logger.info(f"Background sync constrained to project: {constrained_project}")
|
||||
@@ -154,7 +159,10 @@ async def initialize_file_sync(
|
||||
# Don't await the tasks - let them run in background while we continue
|
||||
|
||||
# Then start the watch service in the background
|
||||
logger.info("Starting watch service for all projects")
|
||||
if constrained_project:
|
||||
logger.info(f"Starting watch service constrained to project: {constrained_project}")
|
||||
else:
|
||||
logger.info("Starting watch service for all projects")
|
||||
|
||||
# run the watch service
|
||||
await watch_service.run()
|
||||
|
||||
@@ -1137,12 +1137,10 @@ class ProjectService:
|
||||
|
||||
# Get watch service status if available
|
||||
watch_status = None
|
||||
watch_status_path = Path.home() / ".basic-memory" / WATCH_STATUS_JSON
|
||||
watch_status_path = self.config_manager.config.data_dir_path / WATCH_STATUS_JSON
|
||||
if watch_status_path.exists():
|
||||
try: # pragma: no cover
|
||||
watch_status = json.loads( # pragma: no cover
|
||||
watch_status_path.read_text(encoding="utf-8")
|
||||
)
|
||||
try:
|
||||
watch_status = json.loads(watch_status_path.read_text(encoding="utf-8"))
|
||||
except Exception: # pragma: no cover
|
||||
pass
|
||||
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
import asyncio
|
||||
import ast
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from typing import List, Optional, Set, Dict, Any
|
||||
|
||||
@@ -11,7 +12,8 @@ from fastapi import BackgroundTasks
|
||||
from loguru import logger
|
||||
from sqlalchemy import text
|
||||
|
||||
from basic_memory import telemetry
|
||||
import logfire
|
||||
|
||||
from basic_memory.models import Entity
|
||||
from basic_memory.repository import EntityRepository
|
||||
from basic_memory.repository.search_repository import (
|
||||
@@ -63,6 +65,24 @@ FTS_RELAXED_STOPWORDS = {
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _PreparedSearchQuery:
|
||||
"""Normalized query inputs shared by search and count."""
|
||||
|
||||
search_text: str | None
|
||||
permalink: str | None
|
||||
permalink_match: str | None
|
||||
title: str | None
|
||||
note_types: list[str] | None
|
||||
search_item_types: list[SearchItemType] | None
|
||||
observation_categories: list[str] | None
|
||||
relation_types: list[str] | None
|
||||
after_date: datetime | None
|
||||
metadata_filters: dict[str, Any] | None
|
||||
retrieval_mode: SearchRetrievalMode
|
||||
min_similarity: float | None
|
||||
|
||||
|
||||
def _strip_nul(value: str) -> str:
|
||||
"""Strip NUL bytes that PostgreSQL text columns cannot store.
|
||||
|
||||
@@ -131,27 +151,20 @@ class SearchService:
|
||||
|
||||
logger.info("Reindex complete")
|
||||
|
||||
async def search(self, query: SearchQuery, limit=10, offset=0) -> List[SearchIndexRow]:
|
||||
"""Search across all indexed content.
|
||||
def _prepare_query(self, query: SearchQuery) -> _PreparedSearchQuery | None:
|
||||
"""Normalize a SearchQuery into repository arguments."""
|
||||
search_text = query.text
|
||||
tags = query.tags
|
||||
|
||||
Supports three modes:
|
||||
1. Exact permalink: finds direct matches for a specific path
|
||||
2. Pattern match: handles * wildcards in paths
|
||||
3. Text search: full-text search across title/content
|
||||
"""
|
||||
# Support tag:<tag> shorthand by mapping to tags filter
|
||||
if query.text:
|
||||
text = query.text.strip()
|
||||
if text.lower().startswith("tag:"):
|
||||
tag_values = re.split(r"[,\s]+", text[4:].strip())
|
||||
tags = [t for t in tag_values if t]
|
||||
if tags:
|
||||
query.tags = tags
|
||||
query.text = None
|
||||
|
||||
if query.no_criteria():
|
||||
logger.debug("no criteria passed to query")
|
||||
return []
|
||||
# Support tag:<tag> shorthand by mapping to tags filter.
|
||||
if search_text is not None:
|
||||
search_text = search_text.strip() or None
|
||||
if search_text and search_text.lower().startswith("tag:"):
|
||||
tag_values = re.split(r"[,\s]+", search_text[4:].strip())
|
||||
parsed_tags = [t for t in tag_values if t]
|
||||
if parsed_tags:
|
||||
tags = parsed_tags
|
||||
search_text = None
|
||||
|
||||
after_date = (
|
||||
(
|
||||
@@ -163,67 +176,146 @@ class SearchService:
|
||||
else None
|
||||
)
|
||||
|
||||
# Merge structured metadata filters (explicit + convenience fields)
|
||||
# Merge structured metadata filters (explicit + convenience fields).
|
||||
metadata_filters: Optional[Dict[str, Any]] = None
|
||||
if query.metadata_filters or query.tags or query.status:
|
||||
if query.metadata_filters or tags or query.status:
|
||||
metadata_filters = dict(query.metadata_filters or {})
|
||||
if query.tags:
|
||||
metadata_filters.setdefault("tags", query.tags)
|
||||
if tags:
|
||||
metadata_filters.setdefault("tags", tags)
|
||||
if query.status:
|
||||
metadata_filters.setdefault("status", query.status)
|
||||
|
||||
retrieval_mode = query.retrieval_mode or SearchRetrievalMode.FTS
|
||||
strict_search_text = query.text
|
||||
has_query = bool(
|
||||
strict_search_text or query.title or query.permalink or query.permalink_match
|
||||
)
|
||||
has_filters = bool(
|
||||
metadata_filters
|
||||
or query.note_types
|
||||
or query.entity_types
|
||||
or after_date
|
||||
or query.tags
|
||||
or query.status
|
||||
prepared = _PreparedSearchQuery(
|
||||
search_text=search_text,
|
||||
permalink=query.permalink,
|
||||
permalink_match=query.permalink_match,
|
||||
title=query.title,
|
||||
note_types=query.note_types,
|
||||
search_item_types=query.entity_types,
|
||||
observation_categories=query.observation_categories,
|
||||
relation_types=query.relation_types,
|
||||
after_date=after_date,
|
||||
metadata_filters=metadata_filters,
|
||||
retrieval_mode=query.retrieval_mode or SearchRetrievalMode.FTS,
|
||||
min_similarity=query.min_similarity,
|
||||
)
|
||||
|
||||
with telemetry.scope(
|
||||
has_criteria = bool(
|
||||
prepared.search_text
|
||||
or prepared.permalink
|
||||
or prepared.permalink_match
|
||||
or prepared.title
|
||||
or prepared.note_types
|
||||
or prepared.search_item_types
|
||||
or prepared.observation_categories
|
||||
or prepared.relation_types
|
||||
or prepared.after_date
|
||||
or prepared.metadata_filters
|
||||
)
|
||||
if not has_criteria:
|
||||
logger.debug("no criteria passed to query")
|
||||
return None
|
||||
return prepared
|
||||
|
||||
@staticmethod
|
||||
def _prepared_has_filters(prepared: _PreparedSearchQuery) -> bool:
|
||||
return bool(
|
||||
prepared.metadata_filters
|
||||
or prepared.note_types
|
||||
or prepared.search_item_types
|
||||
or prepared.observation_categories
|
||||
or prepared.relation_types
|
||||
or prepared.after_date
|
||||
)
|
||||
|
||||
async def _search_repository(
|
||||
self,
|
||||
prepared: _PreparedSearchQuery,
|
||||
*,
|
||||
search_text: str | None,
|
||||
limit: int,
|
||||
offset: int,
|
||||
) -> List[SearchIndexRow]:
|
||||
return await self.repository.search(
|
||||
search_text=search_text,
|
||||
permalink=prepared.permalink,
|
||||
permalink_match=prepared.permalink_match,
|
||||
title=prepared.title,
|
||||
note_types=prepared.note_types,
|
||||
search_item_types=prepared.search_item_types,
|
||||
observation_categories=prepared.observation_categories,
|
||||
relation_types=prepared.relation_types,
|
||||
after_date=prepared.after_date,
|
||||
metadata_filters=prepared.metadata_filters,
|
||||
retrieval_mode=prepared.retrieval_mode,
|
||||
min_similarity=prepared.min_similarity,
|
||||
limit=limit,
|
||||
offset=offset,
|
||||
)
|
||||
|
||||
async def _count_repository(
|
||||
self,
|
||||
prepared: _PreparedSearchQuery,
|
||||
*,
|
||||
search_text: str | None,
|
||||
) -> int:
|
||||
return await self.repository.count(
|
||||
search_text=search_text,
|
||||
permalink=prepared.permalink,
|
||||
permalink_match=prepared.permalink_match,
|
||||
title=prepared.title,
|
||||
note_types=prepared.note_types,
|
||||
search_item_types=prepared.search_item_types,
|
||||
observation_categories=prepared.observation_categories,
|
||||
relation_types=prepared.relation_types,
|
||||
after_date=prepared.after_date,
|
||||
metadata_filters=prepared.metadata_filters,
|
||||
retrieval_mode=prepared.retrieval_mode,
|
||||
min_similarity=prepared.min_similarity,
|
||||
)
|
||||
|
||||
async def search(self, query: SearchQuery, limit=10, offset=0) -> List[SearchIndexRow]:
|
||||
"""Search across all indexed content.
|
||||
|
||||
Supports three modes:
|
||||
1. Exact permalink: finds direct matches for a specific path
|
||||
2. Pattern match: handles * wildcards in paths
|
||||
3. Text search: full-text search across title/content
|
||||
"""
|
||||
prepared = self._prepare_query(query)
|
||||
if prepared is None:
|
||||
return []
|
||||
|
||||
strict_search_text = prepared.search_text
|
||||
has_query = bool(
|
||||
strict_search_text or prepared.title or prepared.permalink or prepared.permalink_match
|
||||
)
|
||||
has_filters = self._prepared_has_filters(prepared)
|
||||
|
||||
with logfire.span(
|
||||
"search.execute",
|
||||
retrieval_mode=retrieval_mode.value,
|
||||
retrieval_mode=prepared.retrieval_mode.value,
|
||||
has_query=has_query,
|
||||
has_filters=has_filters,
|
||||
limit=limit,
|
||||
offset=offset,
|
||||
):
|
||||
logger.trace(f"Searching with query: {query}")
|
||||
with telemetry.scope(
|
||||
"search.repository_query",
|
||||
retrieval_mode=retrieval_mode.value,
|
||||
phase="repository_query",
|
||||
has_query=has_query,
|
||||
has_filters=has_filters,
|
||||
):
|
||||
# First pass: preserve existing strict search behavior.
|
||||
results = await self.repository.search(
|
||||
search_text=strict_search_text,
|
||||
permalink=query.permalink,
|
||||
permalink_match=query.permalink_match,
|
||||
title=query.title,
|
||||
note_types=query.note_types,
|
||||
search_item_types=query.entity_types,
|
||||
after_date=after_date,
|
||||
metadata_filters=metadata_filters,
|
||||
retrieval_mode=retrieval_mode,
|
||||
min_similarity=query.min_similarity,
|
||||
limit=limit,
|
||||
offset=offset,
|
||||
)
|
||||
results = await self._search_repository(
|
||||
prepared,
|
||||
search_text=strict_search_text,
|
||||
limit=limit,
|
||||
offset=offset,
|
||||
)
|
||||
|
||||
# Trigger: strict FTS with plain multi-term text returned no results.
|
||||
# Why: natural-language queries often include stopwords that over-constrain implicit AND.
|
||||
# Outcome: retry once with relaxed OR terms while preserving explicit boolean intent.
|
||||
if results:
|
||||
return results
|
||||
if not self._is_relaxed_fts_fallback_eligible(query, strict_search_text, retrieval_mode):
|
||||
if not self._is_relaxed_fts_fallback_eligible(
|
||||
query, strict_search_text, prepared.retrieval_mode
|
||||
):
|
||||
return results
|
||||
|
||||
assert strict_search_text is not None
|
||||
@@ -235,34 +327,58 @@ class SearchService:
|
||||
"Strict FTS returned 0 results; retrying relaxed FTS query "
|
||||
f"strict='{strict_search_text}' relaxed='{relaxed_search_text}'"
|
||||
)
|
||||
with telemetry.scope(
|
||||
with logfire.span(
|
||||
"search.relaxed_fts_retry",
|
||||
retrieval_mode=retrieval_mode.value,
|
||||
retrieval_mode=prepared.retrieval_mode.value,
|
||||
token_count=len(self._tokenize_fts_text(strict_search_text)),
|
||||
limit=limit,
|
||||
offset=offset,
|
||||
):
|
||||
with telemetry.scope(
|
||||
"search.repository_query",
|
||||
retrieval_mode=retrieval_mode.value,
|
||||
phase="repository_query",
|
||||
has_query=has_query,
|
||||
has_filters=has_filters,
|
||||
):
|
||||
return await self.repository.search(
|
||||
search_text=relaxed_search_text,
|
||||
permalink=query.permalink,
|
||||
permalink_match=query.permalink_match,
|
||||
title=query.title,
|
||||
note_types=query.note_types,
|
||||
search_item_types=query.entity_types,
|
||||
after_date=after_date,
|
||||
metadata_filters=metadata_filters,
|
||||
retrieval_mode=retrieval_mode,
|
||||
min_similarity=query.min_similarity,
|
||||
limit=limit,
|
||||
offset=offset,
|
||||
)
|
||||
return await self._search_repository(
|
||||
prepared,
|
||||
search_text=relaxed_search_text,
|
||||
limit=limit,
|
||||
offset=offset,
|
||||
)
|
||||
|
||||
async def count(self, query: SearchQuery) -> int:
|
||||
"""Count all indexed rows matching a query."""
|
||||
prepared = self._prepare_query(query)
|
||||
if prepared is None:
|
||||
return 0
|
||||
|
||||
strict_search_text = prepared.search_text
|
||||
has_query = bool(
|
||||
strict_search_text or prepared.title or prepared.permalink or prepared.permalink_match
|
||||
)
|
||||
has_filters = self._prepared_has_filters(prepared)
|
||||
|
||||
with logfire.span(
|
||||
"search.count",
|
||||
retrieval_mode=prepared.retrieval_mode.value,
|
||||
has_query=has_query,
|
||||
has_filters=has_filters,
|
||||
):
|
||||
total = await self._count_repository(prepared, search_text=strict_search_text)
|
||||
|
||||
if total > 0:
|
||||
return total
|
||||
if not self._is_relaxed_fts_fallback_eligible(
|
||||
query, strict_search_text, prepared.retrieval_mode
|
||||
):
|
||||
return total
|
||||
|
||||
assert strict_search_text is not None
|
||||
relaxed_search_text = self._build_relaxed_fts_query(strict_search_text)
|
||||
if relaxed_search_text == strict_search_text:
|
||||
return total
|
||||
|
||||
with logfire.span(
|
||||
"search.count.relaxed_fts_retry",
|
||||
retrieval_mode=prepared.retrieval_mode.value,
|
||||
token_count=len(self._tokenize_fts_text(strict_search_text)),
|
||||
):
|
||||
return await self._count_repository(prepared, search_text=relaxed_search_text)
|
||||
|
||||
@staticmethod
|
||||
def _tokenize_fts_text(search_text: str) -> list[str]:
|
||||
@@ -396,17 +512,8 @@ class SearchService:
|
||||
f"permalink={entity.permalink} project_id={entity.project_id}"
|
||||
)
|
||||
try:
|
||||
with telemetry.scope(
|
||||
"search.index_entity_data",
|
||||
phase="index_entity_data",
|
||||
result_count=1,
|
||||
):
|
||||
with telemetry.scope(
|
||||
"search.index.delete_existing",
|
||||
phase="delete_existing",
|
||||
result_count=1,
|
||||
):
|
||||
await self.repository.delete_by_entity_id(entity_id=entity.id)
|
||||
with logfire.span("search.index_entity_data", entity_id=entity.id):
|
||||
await self.repository.delete_by_entity_id(entity_id=entity.id)
|
||||
|
||||
if entity.is_markdown:
|
||||
await self.index_entity_markdown(entity, content)
|
||||
@@ -676,28 +783,23 @@ class SearchService:
|
||||
self,
|
||||
entity: Entity,
|
||||
) -> None:
|
||||
with telemetry.scope(
|
||||
"search.index_file",
|
||||
phase="index_file",
|
||||
result_count=1,
|
||||
):
|
||||
# Index entity file with no content
|
||||
await self.repository.index_item(
|
||||
SearchIndexRow(
|
||||
id=entity.id,
|
||||
entity_id=entity.id,
|
||||
type=SearchItemType.ENTITY.value,
|
||||
title=_strip_nul(entity.title),
|
||||
permalink=entity.permalink, # Required for Postgres NOT NULL constraint
|
||||
file_path=entity.file_path,
|
||||
metadata={
|
||||
"note_type": entity.note_type,
|
||||
},
|
||||
created_at=entity.created_at,
|
||||
updated_at=_mtime_to_datetime(entity),
|
||||
project_id=entity.project_id,
|
||||
)
|
||||
# Index entity file with no content
|
||||
await self.repository.index_item(
|
||||
SearchIndexRow(
|
||||
id=entity.id,
|
||||
entity_id=entity.id,
|
||||
type=SearchItemType.ENTITY.value,
|
||||
title=_strip_nul(entity.title),
|
||||
permalink=entity.permalink, # Required for Postgres NOT NULL constraint
|
||||
file_path=entity.file_path,
|
||||
metadata={
|
||||
"note_type": entity.note_type,
|
||||
},
|
||||
created_at=entity.created_at,
|
||||
updated_at=_mtime_to_datetime(entity),
|
||||
project_id=entity.project_id,
|
||||
)
|
||||
)
|
||||
|
||||
async def index_entity_markdown(
|
||||
self,
|
||||
@@ -730,11 +832,7 @@ class SearchService:
|
||||
The project_id is automatically added by the repository when indexing.
|
||||
"""
|
||||
|
||||
with telemetry.scope(
|
||||
"search.index_markdown",
|
||||
phase="index_markdown",
|
||||
result_count=1,
|
||||
):
|
||||
with logfire.span("search.index_markdown", entity_id=entity.id):
|
||||
rows_to_index = []
|
||||
|
||||
content_stems = []
|
||||
@@ -743,51 +841,76 @@ class SearchService:
|
||||
content_stems.extend(title_variants)
|
||||
|
||||
if content is None:
|
||||
with telemetry.scope(
|
||||
"search.index.read_content",
|
||||
phase="read_content",
|
||||
result_count=1,
|
||||
):
|
||||
content = await self.file_service.read_entity_content(entity)
|
||||
content = await self.file_service.read_entity_content(entity)
|
||||
if content:
|
||||
content_stems.append(content)
|
||||
content_snippet = _strip_nul(content)
|
||||
|
||||
with telemetry.scope(
|
||||
"search.index.build_rows",
|
||||
phase="build_rows",
|
||||
result_count=1,
|
||||
):
|
||||
if entity.permalink:
|
||||
content_stems.extend(self._generate_variants(entity.permalink))
|
||||
if entity.permalink:
|
||||
content_stems.extend(self._generate_variants(entity.permalink))
|
||||
|
||||
content_stems.extend(self._generate_variants(entity.file_path))
|
||||
content_stems.extend(self._generate_variants(entity.file_path))
|
||||
|
||||
entity_tags = self._extract_entity_tags(entity)
|
||||
if entity_tags:
|
||||
content_stems.extend(entity_tags)
|
||||
entity_tags = self._extract_entity_tags(entity)
|
||||
if entity_tags:
|
||||
content_stems.extend(entity_tags)
|
||||
|
||||
entity_content_stems = _strip_nul(
|
||||
"\n".join(p for p in content_stems if p and p.strip())
|
||||
entity_content_stems = _strip_nul(
|
||||
"\n".join(p for p in content_stems if p and p.strip())
|
||||
)
|
||||
|
||||
if len(entity_content_stems) > MAX_CONTENT_STEMS_SIZE: # pragma: no cover
|
||||
entity_content_stems = entity_content_stems[
|
||||
:MAX_CONTENT_STEMS_SIZE
|
||||
] # pragma: no cover
|
||||
|
||||
rows_to_index.append(
|
||||
SearchIndexRow(
|
||||
id=entity.id,
|
||||
type=SearchItemType.ENTITY.value,
|
||||
title=_strip_nul(entity.title),
|
||||
content_stems=entity_content_stems,
|
||||
content_snippet=content_snippet,
|
||||
permalink=entity.permalink,
|
||||
file_path=entity.file_path,
|
||||
entity_id=entity.id,
|
||||
metadata={
|
||||
"note_type": entity.note_type,
|
||||
},
|
||||
created_at=entity.created_at,
|
||||
updated_at=_mtime_to_datetime(entity),
|
||||
project_id=entity.project_id,
|
||||
)
|
||||
)
|
||||
|
||||
if len(entity_content_stems) > MAX_CONTENT_STEMS_SIZE: # pragma: no cover
|
||||
entity_content_stems = entity_content_stems[
|
||||
seen_permalinks: set[str] = {entity.permalink} if entity.permalink else set()
|
||||
for obs in entity.observations:
|
||||
obs_permalink = obs.permalink
|
||||
if obs_permalink in seen_permalinks:
|
||||
logger.debug(f"Skipping duplicate observation permalink: {obs_permalink}")
|
||||
continue
|
||||
seen_permalinks.add(obs_permalink)
|
||||
|
||||
obs_content_stems = _strip_nul(
|
||||
"\n".join(p for p in self._generate_variants(obs.content) if p and p.strip())
|
||||
)
|
||||
if len(obs_content_stems) > MAX_CONTENT_STEMS_SIZE: # pragma: no cover
|
||||
obs_content_stems = obs_content_stems[
|
||||
:MAX_CONTENT_STEMS_SIZE
|
||||
] # pragma: no cover
|
||||
|
||||
rows_to_index.append(
|
||||
SearchIndexRow(
|
||||
id=entity.id,
|
||||
type=SearchItemType.ENTITY.value,
|
||||
title=_strip_nul(entity.title),
|
||||
content_stems=entity_content_stems,
|
||||
content_snippet=content_snippet,
|
||||
permalink=entity.permalink,
|
||||
id=obs.id,
|
||||
type=SearchItemType.OBSERVATION.value,
|
||||
title=_strip_nul(f"{obs.category}: {obs.content[:100]}..."),
|
||||
content_stems=obs_content_stems,
|
||||
content_snippet=_strip_nul(obs.content),
|
||||
permalink=obs_permalink,
|
||||
file_path=entity.file_path,
|
||||
category=obs.category,
|
||||
entity_id=entity.id,
|
||||
metadata={
|
||||
"note_type": entity.note_type,
|
||||
"tags": obs.tags,
|
||||
},
|
||||
created_at=entity.created_at,
|
||||
updated_at=_mtime_to_datetime(entity),
|
||||
@@ -795,79 +918,35 @@ class SearchService:
|
||||
)
|
||||
)
|
||||
|
||||
seen_permalinks: set[str] = {entity.permalink} if entity.permalink else set()
|
||||
for obs in entity.observations:
|
||||
obs_permalink = obs.permalink
|
||||
if obs_permalink in seen_permalinks:
|
||||
logger.debug(f"Skipping duplicate observation permalink: {obs_permalink}")
|
||||
continue
|
||||
seen_permalinks.add(obs_permalink)
|
||||
for rel in entity.outgoing_relations:
|
||||
relation_title = _strip_nul(
|
||||
f"{rel.from_entity.title} -> {rel.to_entity.title}"
|
||||
if rel.to_entity
|
||||
else f"{rel.from_entity.title}"
|
||||
)
|
||||
|
||||
obs_content_stems = _strip_nul(
|
||||
"\n".join(
|
||||
p for p in self._generate_variants(obs.content) if p and p.strip()
|
||||
)
|
||||
)
|
||||
if len(obs_content_stems) > MAX_CONTENT_STEMS_SIZE: # pragma: no cover
|
||||
obs_content_stems = obs_content_stems[
|
||||
:MAX_CONTENT_STEMS_SIZE
|
||||
] # pragma: no cover
|
||||
rows_to_index.append(
|
||||
SearchIndexRow(
|
||||
id=obs.id,
|
||||
type=SearchItemType.OBSERVATION.value,
|
||||
title=_strip_nul(f"{obs.category}: {obs.content[:100]}..."),
|
||||
content_stems=obs_content_stems,
|
||||
content_snippet=_strip_nul(obs.content),
|
||||
permalink=obs_permalink,
|
||||
file_path=entity.file_path,
|
||||
category=obs.category,
|
||||
entity_id=entity.id,
|
||||
metadata={
|
||||
"tags": obs.tags,
|
||||
},
|
||||
created_at=entity.created_at,
|
||||
updated_at=_mtime_to_datetime(entity),
|
||||
project_id=entity.project_id,
|
||||
)
|
||||
rel_content_stems = _strip_nul(
|
||||
"\n".join(p for p in self._generate_variants(relation_title) if p and p.strip())
|
||||
)
|
||||
rows_to_index.append(
|
||||
SearchIndexRow(
|
||||
id=rel.id,
|
||||
title=relation_title,
|
||||
permalink=rel.permalink,
|
||||
content_stems=rel_content_stems,
|
||||
file_path=entity.file_path,
|
||||
type=SearchItemType.RELATION.value,
|
||||
entity_id=entity.id,
|
||||
from_id=rel.from_id,
|
||||
to_id=rel.to_id,
|
||||
relation_type=rel.relation_type,
|
||||
created_at=entity.created_at,
|
||||
updated_at=_mtime_to_datetime(entity),
|
||||
project_id=entity.project_id,
|
||||
)
|
||||
)
|
||||
|
||||
for rel in entity.outgoing_relations:
|
||||
relation_title = _strip_nul(
|
||||
f"{rel.from_entity.title} -> {rel.to_entity.title}"
|
||||
if rel.to_entity
|
||||
else f"{rel.from_entity.title}"
|
||||
)
|
||||
|
||||
rel_content_stems = _strip_nul(
|
||||
"\n".join(
|
||||
p for p in self._generate_variants(relation_title) if p and p.strip()
|
||||
)
|
||||
)
|
||||
rows_to_index.append(
|
||||
SearchIndexRow(
|
||||
id=rel.id,
|
||||
title=relation_title,
|
||||
permalink=rel.permalink,
|
||||
content_stems=rel_content_stems,
|
||||
file_path=entity.file_path,
|
||||
type=SearchItemType.RELATION.value,
|
||||
entity_id=entity.id,
|
||||
from_id=rel.from_id,
|
||||
to_id=rel.to_id,
|
||||
relation_type=rel.relation_type,
|
||||
created_at=entity.created_at,
|
||||
updated_at=_mtime_to_datetime(entity),
|
||||
project_id=entity.project_id,
|
||||
)
|
||||
)
|
||||
|
||||
with telemetry.scope(
|
||||
"search.index.bulk_upsert",
|
||||
phase="bulk_upsert",
|
||||
result_count=len(rows_to_index),
|
||||
):
|
||||
await self.repository.bulk_index_items(rows_to_index)
|
||||
await self.repository.bulk_index_items(rows_to_index)
|
||||
|
||||
async def delete_by_permalink(self, permalink: str):
|
||||
"""Delete an item from the search index."""
|
||||
|
||||
@@ -15,11 +15,17 @@ import aiofiles.os
|
||||
from loguru import logger
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
|
||||
from basic_memory import telemetry
|
||||
import logfire
|
||||
from basic_memory import db
|
||||
from basic_memory.config import BasicMemoryConfig, ConfigManager
|
||||
from basic_memory.file_utils import compute_checksum, has_frontmatter
|
||||
from basic_memory.indexing import BatchIndexer, IndexFileMetadata, IndexInputFile, IndexProgress
|
||||
from basic_memory.file_utils import ParseError, compute_checksum, remove_frontmatter
|
||||
from basic_memory.indexing import (
|
||||
BatchIndexer,
|
||||
IndexFileMetadata,
|
||||
IndexInputFile,
|
||||
IndexProgress,
|
||||
SyncedMarkdownFile,
|
||||
)
|
||||
from basic_memory.indexing.batching import build_index_batches
|
||||
from basic_memory.indexing.models import (
|
||||
IndexedEntity,
|
||||
@@ -308,7 +314,7 @@ class SyncService:
|
||||
|
||||
start_time = time.time()
|
||||
sync_start_timestamp = time.time() # Capture at start for watermark
|
||||
with telemetry.operation(
|
||||
with logfire.span(
|
||||
"sync.project.run",
|
||||
project_name=project_name,
|
||||
force_full=force_full,
|
||||
@@ -319,7 +325,7 @@ class SyncService:
|
||||
|
||||
# initial paths from db to sync
|
||||
# path -> checksum
|
||||
with telemetry.scope("sync.project.scan", force_full=force_full):
|
||||
with logfire.span("sync.project.scan", force_full=force_full):
|
||||
report = await self.scan(directory, force_full=force_full)
|
||||
|
||||
# order of sync matters to resolve relations effectively
|
||||
@@ -328,7 +334,7 @@ class SyncService:
|
||||
+ f"deleted_files={len(report.deleted)}, moved_files={len(report.moves)}"
|
||||
)
|
||||
|
||||
with telemetry.scope(
|
||||
with logfire.span(
|
||||
"sync.project.apply_changes",
|
||||
new_count=len(report.new),
|
||||
modified_count=len(report.modified),
|
||||
@@ -375,9 +381,7 @@ class SyncService:
|
||||
# not only the lightweight diff summary.
|
||||
# Outcome: full reindex can heal relation state even when the diff report is empty.
|
||||
if report.total > 0 or (force_full and indexed_entities):
|
||||
with telemetry.scope(
|
||||
"sync.project.resolve_relations", relation_scope="all_pending"
|
||||
):
|
||||
with logfire.span("sync.project.resolve_relations", relation_scope="all_pending"):
|
||||
synced_entity_ids.extend(await self.resolve_relations())
|
||||
else:
|
||||
logger.info("Skipping relation resolution - no file changes detected")
|
||||
@@ -386,7 +390,7 @@ class SyncService:
|
||||
synced_entity_ids = list(dict.fromkeys(synced_entity_ids))
|
||||
if synced_entity_ids and sync_embeddings and self.app_config.semantic_search_enabled:
|
||||
try:
|
||||
with telemetry.scope(
|
||||
with logfire.span(
|
||||
"sync.project.sync_embeddings",
|
||||
entity_count=len(synced_entity_ids),
|
||||
):
|
||||
@@ -410,7 +414,7 @@ class SyncService:
|
||||
# Update scan watermark after successful sync
|
||||
# Use the timestamp from sync start (not end) to ensure we catch files
|
||||
# created during the sync on the next iteration
|
||||
with telemetry.scope("sync.project.update_watermark"):
|
||||
with logfire.span("sync.project.update_watermark"):
|
||||
current_file_count = await self._quick_count_files(directory)
|
||||
if self.entity_repository.project_id is not None:
|
||||
project = await self.project_repository.find_by_id(
|
||||
@@ -757,7 +761,7 @@ class SyncService:
|
||||
if project is None:
|
||||
raise ValueError(f"Project not found: {self.entity_repository.project_id}")
|
||||
|
||||
with telemetry.scope("sync.project.select_scan_strategy", force_full=force_full):
|
||||
with logfire.span("sync.project.select_scan_strategy", force_full=force_full):
|
||||
# Step 1: Quick file count
|
||||
logger.debug("Counting files in directory")
|
||||
current_count = await self._quick_count_files(directory)
|
||||
@@ -801,7 +805,7 @@ class SyncService:
|
||||
logger.warning("No scan watermark available, falling back to full scan")
|
||||
scan_coro = self._scan_directory_full(directory)
|
||||
|
||||
with telemetry.scope("sync.project.filesystem_scan", scan_type=scan_type):
|
||||
with logfire.span("sync.project.filesystem_scan", scan_type=scan_type):
|
||||
file_paths_to_scan = await scan_coro
|
||||
if scan_type == "incremental":
|
||||
logger.debug(
|
||||
@@ -867,7 +871,7 @@ class SyncService:
|
||||
|
||||
# Step 4: Detect moves (for both full and incremental scans)
|
||||
# Check if any "new" files are actually moves by matching checksums
|
||||
with telemetry.scope("sync.project.detect_moves", new_count=len(report.new)):
|
||||
with logfire.span("sync.project.detect_moves", new_count=len(report.new)):
|
||||
for new_path in list(
|
||||
report.new
|
||||
): # Use list() to allow modification during iteration
|
||||
@@ -902,7 +906,7 @@ class SyncService:
|
||||
# Step 5: Detect deletions (only for full scans)
|
||||
# Incremental scans can't reliably detect deletions since they only see modified files
|
||||
if scan_type in ("full_initial", "full_deletions", "full_fallback", "full_forced"):
|
||||
with telemetry.scope("sync.project.detect_deletions", scan_type=scan_type):
|
||||
with logfire.span("sync.project.detect_deletions", scan_type=scan_type):
|
||||
# Use optimized query for just file paths (not full entities)
|
||||
db_file_paths = await self.entity_repository.get_all_file_paths()
|
||||
logger.debug(f"Found {len(db_file_paths)} db paths for deletion detection")
|
||||
@@ -992,7 +996,7 @@ class SyncService:
|
||||
except FileNotFoundError:
|
||||
# File exists in database but not on filesystem
|
||||
# This indicates a database/filesystem inconsistency - treat as deletion
|
||||
with telemetry.scope(
|
||||
with logfire.span(
|
||||
"sync.file.failure",
|
||||
failure_type="file_not_found",
|
||||
path=path,
|
||||
@@ -1014,7 +1018,7 @@ class SyncService:
|
||||
if isinstance(e, SyncFatalError) or isinstance(
|
||||
e.__cause__, SyncFatalError
|
||||
): # pragma: no cover
|
||||
with telemetry.scope(
|
||||
with logfire.span(
|
||||
"sync.file.failure",
|
||||
failure_type=failure_type,
|
||||
path=path,
|
||||
@@ -1027,7 +1031,7 @@ class SyncService:
|
||||
|
||||
# Otherwise treat as recoverable file-level error
|
||||
error_msg = str(e)
|
||||
with telemetry.scope(
|
||||
with logfire.span(
|
||||
"sync.file.failure",
|
||||
failure_type=failure_type,
|
||||
path=path,
|
||||
@@ -1052,91 +1056,121 @@ class SyncService:
|
||||
Returns:
|
||||
Tuple of (entity, checksum)
|
||||
"""
|
||||
# Parse markdown first to get any existing permalink
|
||||
synced = await self.sync_one_markdown_file(path, new=new, index_search=False)
|
||||
return synced.entity, synced.checksum
|
||||
|
||||
async def sync_one_markdown_file(
|
||||
self,
|
||||
path: str,
|
||||
*,
|
||||
new: bool = True,
|
||||
index_search: bool = True,
|
||||
resolve_relations: bool = True,
|
||||
) -> SyncedMarkdownFile:
|
||||
"""Sync one markdown file and return the final canonical file state.
|
||||
|
||||
This method is the fail-fast single-file primitive for callers such as
|
||||
cloud workers. It does not swallow unexpected exceptions.
|
||||
"""
|
||||
logger.debug(f"Parsing markdown file, path: {path}, new: {new}")
|
||||
|
||||
file_content = await self.file_service.read_file_content(path)
|
||||
file_contains_frontmatter = has_frontmatter(file_content)
|
||||
|
||||
# Get file timestamps for tracking modification times
|
||||
try:
|
||||
initial_markdown_bytes = await self.file_service.read_file_bytes(path)
|
||||
except FileOperationError as exc:
|
||||
# Trigger: FileService wraps binary read failures in FileOperationError.
|
||||
# Why: sync_file() treats bare FileNotFoundError as a deletion race and cleans up the DB row.
|
||||
# Outcome: preserve that contract while still hashing the exact bytes we loaded.
|
||||
if isinstance(exc.__cause__, FileNotFoundError):
|
||||
raise exc.__cause__ from exc
|
||||
raise
|
||||
initial_markdown_content = initial_markdown_bytes.decode("utf-8")
|
||||
file_metadata = await self.file_service.get_file_metadata(path)
|
||||
created = file_metadata.created_at
|
||||
modified = file_metadata.modified_at
|
||||
|
||||
# Parse markdown content with file metadata (avoids redundant file read/stat)
|
||||
# This enables cloud implementations (S3FileService) to provide metadata from head_object
|
||||
abs_path = self.file_service.base_path / path
|
||||
entity_markdown = await self.entity_parser.parse_markdown_content(
|
||||
file_path=abs_path,
|
||||
content=file_content,
|
||||
mtime=file_metadata.modified_at.timestamp(),
|
||||
ctime=file_metadata.created_at.timestamp(),
|
||||
)
|
||||
|
||||
# Trigger: markdown file has no frontmatter and frontmatter enforcement is enabled
|
||||
# Why: watch/sync consumers rely on normalized metadata and stable permalinks
|
||||
# Outcome: file is updated in-place with derived title/type/permalink metadata
|
||||
if not file_contains_frontmatter and self.app_config.ensure_frontmatter_on_sync:
|
||||
permalink = await self.entity_service.resolve_permalink(
|
||||
path, markdown=entity_markdown, skip_conflict_check=True
|
||||
initial_checksum = await compute_checksum(initial_markdown_bytes)
|
||||
existing_entity = await self.entity_repository.get_by_file_path(path)
|
||||
if existing_entity is not None and existing_entity.checksum == initial_checksum:
|
||||
logger.debug(
|
||||
f"Markdown sync skipped unchanged file: path={path}, "
|
||||
f"entity_id={existing_entity.id}, checksum={initial_checksum[:8]}"
|
||||
)
|
||||
frontmatter_updates = {
|
||||
"title": entity_markdown.frontmatter.title,
|
||||
"type": entity_markdown.frontmatter.type,
|
||||
"permalink": permalink,
|
||||
}
|
||||
await self.file_service.update_frontmatter(path, frontmatter_updates)
|
||||
entity_markdown.frontmatter.metadata.update(frontmatter_updates)
|
||||
|
||||
# if the file contains frontmatter, resolve a permalink (unless disabled)
|
||||
if file_contains_frontmatter and not self.app_config.disable_permalinks:
|
||||
# Resolve permalink - skip conflict checks during bulk sync for performance
|
||||
permalink = await self.entity_service.resolve_permalink(
|
||||
path, markdown=entity_markdown, skip_conflict_check=True
|
||||
return SyncedMarkdownFile(
|
||||
entity=existing_entity,
|
||||
checksum=initial_checksum,
|
||||
markdown_content=initial_markdown_content,
|
||||
file_path=path,
|
||||
content_type=self.file_service.content_type(path),
|
||||
updated_at=file_metadata.modified_at,
|
||||
size=file_metadata.size,
|
||||
)
|
||||
|
||||
# If permalink changed, update the file
|
||||
if permalink != entity_markdown.frontmatter.permalink:
|
||||
logger.debug(
|
||||
f"Updating permalink for path: {path}, old_permalink: {entity_markdown.frontmatter.permalink}, new_permalink: {permalink}"
|
||||
)
|
||||
|
||||
entity_markdown.frontmatter.metadata["permalink"] = permalink
|
||||
await self.file_service.update_frontmatter(path, {"permalink": permalink})
|
||||
|
||||
# Create/update entity and relations in one path
|
||||
logger.debug(f"{'Creating' if new else 'Updating'} entity from markdown, path={path}")
|
||||
entity = await self.entity_service.upsert_entity_from_markdown(
|
||||
Path(path), entity_markdown, is_new=new
|
||||
indexed = await self.batch_indexer.index_markdown_file(
|
||||
IndexInputFile(
|
||||
path=path,
|
||||
size=file_metadata.size,
|
||||
checksum=initial_checksum,
|
||||
content_type=self.file_service.content_type(path),
|
||||
last_modified=file_metadata.modified_at,
|
||||
created_at=file_metadata.created_at,
|
||||
content=initial_markdown_bytes,
|
||||
),
|
||||
new=new,
|
||||
index_search=False,
|
||||
resolve_relations=resolve_relations,
|
||||
)
|
||||
|
||||
# After updating relations, we need to compute the checksum again
|
||||
# This is necessary for files with wikilinks to ensure consistent checksums
|
||||
# after relation processing is complete
|
||||
final_checksum = await self.file_service.compute_checksum(path)
|
||||
|
||||
# Update checksum, timestamps, and file metadata from file system
|
||||
# Store mtime/size for efficient change detection in future scans
|
||||
# This ensures temporal ordering in search and recent activity uses actual file modification times
|
||||
await self.entity_repository.update(
|
||||
entity.id,
|
||||
final_markdown_content = (
|
||||
indexed.markdown_content
|
||||
if indexed.markdown_content is not None
|
||||
else initial_markdown_content
|
||||
)
|
||||
file_metadata = await self.file_service.get_file_metadata(path)
|
||||
refreshed_entities = await self.entity_repository.find_by_ids([indexed.entity_id])
|
||||
if len(refreshed_entities) != 1: # pragma: no cover
|
||||
raise ValueError(f"Failed to reload synced markdown entity for {path}")
|
||||
# Trigger: markdown sync may have rewritten frontmatter after the initial file metadata load.
|
||||
# Why: the batch indexer persisted checksum/path data from the pre-rewrite IndexInputFile.
|
||||
# Outcome: refresh size and mtime from the file as it actually exists on disk now.
|
||||
updated_entity = await self.entity_repository.update(
|
||||
refreshed_entities[0].id,
|
||||
{
|
||||
"checksum": final_checksum,
|
||||
"created_at": created,
|
||||
"updated_at": modified,
|
||||
"checksum": indexed.checksum,
|
||||
"created_at": file_metadata.created_at,
|
||||
"updated_at": file_metadata.modified_at,
|
||||
"mtime": file_metadata.modified_at.timestamp(),
|
||||
"size": file_metadata.size,
|
||||
},
|
||||
)
|
||||
if updated_entity is None: # pragma: no cover
|
||||
raise ValueError(f"Failed to update markdown entity metadata for {path}")
|
||||
|
||||
if index_search:
|
||||
# Trigger: markdown may start with '---' as a thematic break or malformed
|
||||
# frontmatter that the parser already treated as plain content.
|
||||
# Why: one-file sync should not fail after the entity upsert just because
|
||||
# strict frontmatter stripping rejects that exact text shape.
|
||||
# Outcome: fall back to indexing the raw markdown content for these cases.
|
||||
try:
|
||||
search_content = remove_frontmatter(final_markdown_content)
|
||||
except ParseError:
|
||||
search_content = final_markdown_content
|
||||
await self.search_service.index_entity_data(
|
||||
updated_entity,
|
||||
content=search_content,
|
||||
)
|
||||
|
||||
logger.debug(
|
||||
f"Markdown sync completed: path={path}, entity_id={entity.id}, "
|
||||
f"observation_count={len(entity.observations)}, relation_count={len(entity.relations)}, "
|
||||
f"checksum={final_checksum[:8]}"
|
||||
f"Markdown sync completed: path={path}, entity_id={updated_entity.id}, "
|
||||
f"observation_count={len(updated_entity.observations)}, "
|
||||
f"relation_count={len(updated_entity.relations)}, checksum={indexed.checksum[:8]}"
|
||||
)
|
||||
|
||||
# Return the final checksum to ensure everything is consistent
|
||||
return entity, final_checksum
|
||||
return SyncedMarkdownFile(
|
||||
entity=updated_entity,
|
||||
checksum=indexed.checksum,
|
||||
markdown_content=final_markdown_content,
|
||||
file_path=path,
|
||||
content_type=self.file_service.content_type(path),
|
||||
updated_at=file_metadata.modified_at,
|
||||
size=file_metadata.size,
|
||||
)
|
||||
|
||||
async def sync_regular_file(self, path: str, new: bool = True) -> Tuple[Optional[Entity], str]:
|
||||
"""Sync a non-markdown file with basic tracking.
|
||||
@@ -1435,7 +1469,7 @@ class SyncService:
|
||||
)
|
||||
affected_entity_ids.add(relation.from_id)
|
||||
except IntegrityError:
|
||||
with telemetry.scope(
|
||||
with logfire.span(
|
||||
"sync.relation.resolve_conflict",
|
||||
relation_id=relation.id,
|
||||
relation_type=relation.relation_type,
|
||||
@@ -1456,7 +1490,7 @@ class SyncService:
|
||||
try:
|
||||
await self.relation_repository.delete(relation.id)
|
||||
except Exception as e:
|
||||
with telemetry.scope(
|
||||
with logfire.span(
|
||||
"sync.relation.cleanup_failure",
|
||||
relation_id=relation.id,
|
||||
relation_type=relation.relation_type,
|
||||
|
||||
@@ -85,14 +85,20 @@ class WatchService:
|
||||
project_repository: ProjectRepository,
|
||||
quiet: bool = False,
|
||||
sync_service_factory: Optional[SyncServiceFactory] = None,
|
||||
constrained_project: Optional[str] = None,
|
||||
):
|
||||
self.app_config = app_config
|
||||
self.project_repository = project_repository
|
||||
self.state = WatchServiceState()
|
||||
self.status_path = Path.home() / ".basic-memory" / WATCH_STATUS_JSON
|
||||
self.status_path = app_config.data_dir_path / WATCH_STATUS_JSON
|
||||
self.status_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
self._ignore_patterns_cache: dict[Path, Set[str]] = {}
|
||||
self._sync_service_factory = sync_service_factory
|
||||
# When set (typically from BASIC_MEMORY_MCP_PROJECT), the watch cycle
|
||||
# only observes this project. Without it, each `basic-memory mcp --project X`
|
||||
# process spawns a watcher over every project and racing writers collide
|
||||
# on the same files.
|
||||
self.constrained_project = constrained_project
|
||||
|
||||
# quiet mode for mcp so it doesn't mess up stdout
|
||||
self.console = Console(quiet=quiet)
|
||||
@@ -149,13 +155,42 @@ class WatchService:
|
||||
|
||||
# create coroutines to handle changes
|
||||
change_handlers = [
|
||||
self.handle_changes(project, changes) # pyright: ignore
|
||||
self.handle_changes(project, set(changes))
|
||||
for project, changes in project_changes.items()
|
||||
]
|
||||
|
||||
# process changes
|
||||
await asyncio.gather(*change_handlers)
|
||||
|
||||
async def _select_projects_to_watch(self) -> list[Project]:
|
||||
"""Return the set of projects this watch cycle should observe.
|
||||
|
||||
Applies two filters in order:
|
||||
1. ``constrained_project`` — if the MCP server was started with
|
||||
``--project``, only that project is watched. This keeps concurrent
|
||||
MCP processes from producing duplicate watchers that race on the
|
||||
same files.
|
||||
2. Cloud-only projects without a local bisync copy are skipped so we
|
||||
don't watch a path that does not exist on disk.
|
||||
"""
|
||||
projects = await self.project_repository.get_active_projects()
|
||||
|
||||
if self.constrained_project:
|
||||
projects = [p for p in projects if p.name == self.constrained_project]
|
||||
|
||||
cloud_skip: list[str] = []
|
||||
for p in projects:
|
||||
if self.app_config.get_project_mode(p.name) == ProjectMode.CLOUD:
|
||||
entry = self.app_config.projects.get(p.name)
|
||||
if entry and Path(entry.path).is_absolute():
|
||||
continue # Cloud project with local bisync copy — keep watching
|
||||
cloud_skip.append(p.name)
|
||||
if cloud_skip:
|
||||
projects = [p for p in projects if p.name not in cloud_skip]
|
||||
logger.debug(f"Skipping cloud-mode projects in watch cycle: {cloud_skip}")
|
||||
|
||||
return list(projects)
|
||||
|
||||
async def run(self): # pragma: no cover
|
||||
"""Watch for file changes and sync them"""
|
||||
|
||||
@@ -174,23 +209,22 @@ class WatchService:
|
||||
# Clear ignore patterns cache to pick up any .gitignore changes
|
||||
self._ignore_patterns_cache.clear()
|
||||
|
||||
# Reload projects to catch any new/removed projects
|
||||
projects = await self.project_repository.get_active_projects()
|
||||
projects = await self._select_projects_to_watch()
|
||||
|
||||
# Trigger: project is configured for cloud routing
|
||||
# Why: cloud-only projects (no local directory) should not be watched;
|
||||
# cloud projects with a local bisync copy (absolute path) need watching
|
||||
# Outcome: watch cycle skips cloud projects without a local directory
|
||||
cloud_skip = []
|
||||
for p in projects:
|
||||
if self.app_config.get_project_mode(p.name) == ProjectMode.CLOUD:
|
||||
entry = self.app_config.projects.get(p.name)
|
||||
if entry and Path(entry.path).is_absolute():
|
||||
continue # Cloud project with local bisync copy — keep watching
|
||||
cloud_skip.append(p.name)
|
||||
if cloud_skip:
|
||||
projects = [p for p in projects if p.name not in cloud_skip]
|
||||
logger.debug(f"Skipping cloud-mode projects in watch cycle: {cloud_skip}")
|
||||
# Trigger: no projects selected (e.g. constrained_project names a
|
||||
# project not in the DB, or every project was filtered out)
|
||||
# Why: watchfiles.awatch() requires at least one path. Calling it
|
||||
# with an empty list raises ValueError, which the outer handler
|
||||
# catches with a 5s sleep — producing a tight error-log loop.
|
||||
# Outcome: sleep the configured reload interval before retrying, so
|
||||
# newly added projects get picked up on the next cycle.
|
||||
if not projects:
|
||||
logger.warning(
|
||||
"No projects to watch; sleeping before retry "
|
||||
f"(constrained_project={self.constrained_project!r})"
|
||||
)
|
||||
await asyncio.sleep(self.app_config.watch_project_reload_interval)
|
||||
continue
|
||||
|
||||
project_paths = [project.path for project in projects]
|
||||
logger.debug(f"Starting watch cycle for directories: {project_paths}")
|
||||
@@ -502,19 +536,19 @@ class WatchService:
|
||||
|
||||
# Add a concise summary instead of a divider
|
||||
if processed:
|
||||
changes = [] # pyright: ignore
|
||||
change_summary: list[str] = []
|
||||
if add_count > 0:
|
||||
changes.append(f"[green]{add_count} added[/green]") # pyright: ignore
|
||||
change_summary.append(f"[green]{add_count} added[/green]")
|
||||
if modify_count > 0:
|
||||
changes.append(f"[yellow]{modify_count} modified[/yellow]") # pyright: ignore
|
||||
change_summary.append(f"[yellow]{modify_count} modified[/yellow]")
|
||||
if moved_count > 0:
|
||||
changes.append(f"[blue]{moved_count} moved[/blue]") # pyright: ignore
|
||||
change_summary.append(f"[blue]{moved_count} moved[/blue]")
|
||||
if delete_count > 0:
|
||||
changes.append(f"[red]{delete_count} deleted[/red]") # pyright: ignore
|
||||
change_summary.append(f"[red]{delete_count} deleted[/red]")
|
||||
|
||||
if changes:
|
||||
self.console.print(f"{', '.join(changes)}", style="dim") # pyright: ignore
|
||||
logger.info(f"changes: {len(changes)}")
|
||||
if change_summary:
|
||||
self.console.print(f"{', '.join(change_summary)}", style="dim")
|
||||
logger.info(f"changes: {len(change_summary)}")
|
||||
|
||||
duration_ms = int((time.time() - start_time) * 1000)
|
||||
self.state.last_scan = datetime.now()
|
||||
|
||||
+15
-196
@@ -1,67 +1,20 @@
|
||||
"""Optional Logfire telemetry helpers for Basic Memory.
|
||||
"""Logfire telemetry bootstrap.
|
||||
|
||||
Telemetry is disabled by default. When enabled, this module configures Logfire,
|
||||
exposes a `loguru` handler for trace-aware logging, and provides lightweight
|
||||
helpers for manual spans and logger context binding.
|
||||
`configure_telemetry()` wires up the Logfire SDK and returns the loguru
|
||||
handler. Call sites use `logfire.span(...)` and `logfire.metric_counter(...)`
|
||||
directly — there are no wrappers here.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Iterator
|
||||
from typing import Any
|
||||
|
||||
from loguru import logger
|
||||
import logfire
|
||||
|
||||
REPOSITORY_URL = "https://github.com/basicmachines-co/basic-memory"
|
||||
ROOT_PATH = "src/basic_memory"
|
||||
|
||||
|
||||
def _load_logfire() -> Any | None:
|
||||
"""Load the optional logfire dependency lazily."""
|
||||
try:
|
||||
import logfire
|
||||
except ImportError:
|
||||
return None
|
||||
return logfire
|
||||
|
||||
|
||||
@dataclass
|
||||
class TelemetryState:
|
||||
"""Process-local Logfire configuration state."""
|
||||
|
||||
enabled: bool = False
|
||||
configured: bool = False
|
||||
service_name: str | None = None
|
||||
environment: str | None = None
|
||||
send_to_logfire: bool = False
|
||||
warnings: list[str] = field(default_factory=list)
|
||||
|
||||
|
||||
_STATE = TelemetryState()
|
||||
_LOGFIRE_HANDLER: dict[str, Any] | None = None
|
||||
_METRICS: dict[tuple[str, str, str, str], Any] = {}
|
||||
|
||||
|
||||
def reset_telemetry_state() -> None:
|
||||
"""Reset process-local telemetry state.
|
||||
|
||||
Primarily used by tests.
|
||||
"""
|
||||
global _LOGFIRE_HANDLER
|
||||
_STATE.enabled = False
|
||||
_STATE.configured = False
|
||||
_STATE.service_name = None
|
||||
_STATE.environment = None
|
||||
_STATE.send_to_logfire = False
|
||||
_STATE.warnings.clear()
|
||||
_LOGFIRE_HANDLER = None
|
||||
_METRICS.clear()
|
||||
|
||||
|
||||
def _filter_attributes(attrs: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Drop null attributes so span and log payloads stay compact."""
|
||||
return {key: value for key, value in attrs.items() if value is not None}
|
||||
|
||||
|
||||
def configure_telemetry(
|
||||
@@ -73,28 +26,14 @@ def configure_telemetry(
|
||||
send_to_logfire: bool = False,
|
||||
log_level: str = "INFO",
|
||||
) -> bool:
|
||||
"""Configure optional Logfire instrumentation for the current process."""
|
||||
"""Configure Logfire for the current process. Returns True when enabled."""
|
||||
global _LOGFIRE_HANDLER
|
||||
|
||||
reset_telemetry_state()
|
||||
_STATE.service_name = service_name
|
||||
_STATE.environment = environment
|
||||
_STATE.send_to_logfire = send_to_logfire
|
||||
_STATE.enabled = enable_logfire
|
||||
_LOGFIRE_HANDLER = None
|
||||
|
||||
if not enable_logfire:
|
||||
return False
|
||||
|
||||
logfire = _load_logfire()
|
||||
if logfire is None:
|
||||
_STATE.enabled = False
|
||||
_STATE.warnings.append(
|
||||
"Logfire telemetry was enabled but the 'logfire' package is not installed. "
|
||||
"Telemetry remains disabled."
|
||||
)
|
||||
return False
|
||||
|
||||
configure_kwargs = {
|
||||
kwargs: dict[str, Any] = {
|
||||
"service_name": service_name,
|
||||
"environment": environment,
|
||||
"code_source": logfire.CodeSource(
|
||||
@@ -107,139 +46,19 @@ def configure_telemetry(
|
||||
}
|
||||
|
||||
try:
|
||||
logfire.configure(**configure_kwargs)
|
||||
logfire.configure(**kwargs)
|
||||
except TypeError:
|
||||
configure_kwargs.pop("send_to_logfire", None)
|
||||
logfire.configure(**configure_kwargs)
|
||||
except Exception as exc: # pragma: no cover
|
||||
_STATE.enabled = False # pragma: no cover
|
||||
_STATE.warnings.append(f"Failed to configure Logfire telemetry: {exc}") # pragma: no cover
|
||||
return False # pragma: no cover
|
||||
# Older logfire releases don't accept send_to_logfire as a keyword.
|
||||
kwargs.pop("send_to_logfire", None)
|
||||
logfire.configure(**kwargs)
|
||||
|
||||
_LOGFIRE_HANDLER = logfire.loguru_handler()
|
||||
_STATE.configured = True
|
||||
return True
|
||||
|
||||
|
||||
def telemetry_enabled() -> bool:
|
||||
"""Return True when telemetry is both enabled and configured."""
|
||||
return _STATE.enabled and _STATE.configured
|
||||
|
||||
|
||||
def get_logfire_handler() -> dict[str, Any] | None:
|
||||
"""Return the active Logfire `loguru` handler, if any."""
|
||||
"""Return the active Logfire loguru handler, if any."""
|
||||
return _LOGFIRE_HANDLER
|
||||
|
||||
|
||||
def pop_telemetry_warnings() -> list[str]:
|
||||
"""Return and clear pending telemetry warnings."""
|
||||
warnings = list(_STATE.warnings)
|
||||
_STATE.warnings.clear()
|
||||
return warnings
|
||||
|
||||
|
||||
def _get_metric(metric_type: str, name: str, *, unit: str, description: str) -> Any | None:
|
||||
"""Create or reuse a Logfire metric instrument when telemetry is enabled."""
|
||||
logfire = _load_logfire()
|
||||
if logfire is None or not _STATE.configured: # pragma: no cover
|
||||
return None # pragma: no cover
|
||||
|
||||
metric_key = (metric_type, name, unit, description)
|
||||
cached_metric = _METRICS.get(metric_key)
|
||||
if cached_metric is not None:
|
||||
return cached_metric
|
||||
|
||||
if metric_type == "counter":
|
||||
metric = logfire.metric_counter(name, unit=unit, description=description)
|
||||
elif metric_type == "histogram":
|
||||
metric = logfire.metric_histogram(name, unit=unit, description=description)
|
||||
else: # pragma: no cover
|
||||
raise ValueError(f"Unsupported metric type: {metric_type}") # pragma: no cover
|
||||
|
||||
_METRICS[metric_key] = metric
|
||||
return metric
|
||||
|
||||
|
||||
def add_counter(
|
||||
name: str,
|
||||
amount: int | float,
|
||||
*,
|
||||
unit: str = "1",
|
||||
description: str = "",
|
||||
**attrs: Any,
|
||||
) -> None:
|
||||
"""Record a counter increment when telemetry is enabled."""
|
||||
metric = _get_metric("counter", name, unit=unit, description=description)
|
||||
if metric is None:
|
||||
return
|
||||
metric.add(amount, attributes=_filter_attributes(attrs))
|
||||
|
||||
|
||||
def record_histogram(
|
||||
name: str,
|
||||
amount: int | float,
|
||||
*,
|
||||
unit: str = "",
|
||||
description: str = "",
|
||||
**attrs: Any,
|
||||
) -> None:
|
||||
"""Record one histogram sample when telemetry is enabled."""
|
||||
metric = _get_metric("histogram", name, unit=unit, description=description)
|
||||
if metric is None:
|
||||
return
|
||||
metric.record(amount, attributes=_filter_attributes(attrs))
|
||||
|
||||
|
||||
@contextmanager
|
||||
def contextualize(**attrs: Any) -> Iterator[None]:
|
||||
"""Apply filtered telemetry attributes to Loguru calls in this scope."""
|
||||
with logger.contextualize(**_filter_attributes(attrs)):
|
||||
yield
|
||||
|
||||
|
||||
@contextmanager
|
||||
def scope(name: str, **attrs: Any) -> Iterator[None]:
|
||||
"""Create a span and bind the same stable attributes into Loguru context."""
|
||||
with contextualize(**attrs):
|
||||
with span(name, **attrs):
|
||||
yield
|
||||
|
||||
|
||||
# Alias: `operation` signals a root-level boundary (entrypoint, tool invocation),
|
||||
# while `scope` signals a nested phase. The distinction is convention only.
|
||||
operation = scope
|
||||
|
||||
|
||||
@contextmanager
|
||||
def span(name: str, **attrs: Any) -> Iterator[None]:
|
||||
"""Create a manual Logfire span when telemetry is enabled."""
|
||||
with started_span(name, **attrs):
|
||||
yield
|
||||
|
||||
|
||||
@contextmanager
|
||||
def started_span(name: str, **attrs: Any) -> Iterator[Any | None]:
|
||||
"""Create a manual Logfire span and expose the active span handle when available."""
|
||||
logfire = _load_logfire()
|
||||
if logfire is None or not _STATE.configured: # pragma: no cover
|
||||
yield # pragma: no cover
|
||||
return # pragma: no cover
|
||||
|
||||
with logfire.span(name, **_filter_attributes(attrs)) as active_span:
|
||||
yield active_span
|
||||
|
||||
|
||||
__all__ = [
|
||||
"add_counter",
|
||||
"contextualize",
|
||||
"configure_telemetry",
|
||||
"get_logfire_handler",
|
||||
"operation",
|
||||
"pop_telemetry_warnings",
|
||||
"record_histogram",
|
||||
"reset_telemetry_state",
|
||||
"scope",
|
||||
"span",
|
||||
"started_span",
|
||||
"telemetry_enabled",
|
||||
]
|
||||
__all__ = ["configure_telemetry", "get_logfire_handler"]
|
||||
|
||||
@@ -224,18 +224,43 @@ def build_canonical_permalink(
|
||||
project_permalink: Optional[str],
|
||||
file_path: Union[Path, str, PathLike],
|
||||
include_project: bool = True,
|
||||
*,
|
||||
workspace_permalink: Optional[str] = None,
|
||||
) -> str:
|
||||
"""Build a canonical permalink, optionally prefixed with project slug.
|
||||
"""Build a canonical permalink, optionally prefixed with workspace/project slugs.
|
||||
|
||||
Args:
|
||||
project_permalink: URL-friendly project identifier (slug). If None, no prefix is added.
|
||||
file_path: Original file path or permalink-like string.
|
||||
include_project: When True, prefix with project slug.
|
||||
workspace_permalink: Optional URL-friendly workspace identifier. When provided,
|
||||
prefix the project-qualified permalink with this workspace slug.
|
||||
|
||||
Returns:
|
||||
Canonical permalink string.
|
||||
"""
|
||||
normalized_path = generate_permalink(file_path)
|
||||
normalized_workspace = generate_permalink(workspace_permalink) if workspace_permalink else None
|
||||
|
||||
if normalized_workspace:
|
||||
if not project_permalink:
|
||||
raise ValueError("workspace_permalink requires project_permalink")
|
||||
|
||||
normalized_project = generate_permalink(project_permalink)
|
||||
workspace_project_prefix = f"{normalized_workspace}/{normalized_project}"
|
||||
if normalized_path == workspace_project_prefix or normalized_path.startswith(
|
||||
f"{workspace_project_prefix}/"
|
||||
):
|
||||
return normalized_path
|
||||
|
||||
if normalized_path == normalized_project or normalized_path.startswith(
|
||||
f"{normalized_project}/"
|
||||
):
|
||||
project_path = normalized_path
|
||||
else:
|
||||
project_path = f"{normalized_project}/{normalized_path}"
|
||||
|
||||
return f"{normalized_workspace}/{project_path}"
|
||||
|
||||
if not include_project or not project_permalink:
|
||||
return normalized_path
|
||||
@@ -244,9 +269,11 @@ def build_canonical_permalink(
|
||||
if normalized_path == normalized_project or normalized_path.startswith(
|
||||
f"{normalized_project}/"
|
||||
):
|
||||
return normalized_path
|
||||
project_path = normalized_path
|
||||
else:
|
||||
project_path = f"{normalized_project}/{normalized_path}"
|
||||
|
||||
return f"{normalized_project}/{normalized_path}"
|
||||
return project_path
|
||||
|
||||
|
||||
def setup_logging(
|
||||
@@ -262,7 +289,8 @@ def setup_logging(
|
||||
|
||||
Args:
|
||||
log_level: DEBUG, INFO, WARNING, ERROR
|
||||
log_to_file: Write to ~/.basic-memory/basic-memory.log with rotation
|
||||
log_to_file: Write to <basic-memory data dir>/basic-memory.log with rotation
|
||||
(honors BASIC_MEMORY_CONFIG_DIR)
|
||||
log_to_stdout: Write to stderr (for Docker/cloud deployments)
|
||||
structured_context: Bind tenant_id, fly_region, etc. for cloud observability
|
||||
"""
|
||||
@@ -281,7 +309,11 @@ def setup_logging(
|
||||
# Why: multiple basic-memory processes can share the same log directory at once.
|
||||
# Outcome: use per-process log files on Windows so log rotation stays local.
|
||||
log_filename = f"basic-memory-{os.getpid()}.log" if os.name == "nt" else "basic-memory.log"
|
||||
log_path = Path.home() / ".basic-memory" / log_filename
|
||||
# Deferred import: basic_memory.config imports from this module at load time,
|
||||
# so resolving the data dir via a top-level import would cycle.
|
||||
from basic_memory.config import resolve_data_dir
|
||||
|
||||
log_path = resolve_data_dir() / log_filename
|
||||
log_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
if os.name == "nt":
|
||||
_cleanup_windows_log_files(log_path.parent, log_path.name)
|
||||
@@ -322,9 +354,6 @@ def setup_logging(
|
||||
logging.getLogger("httpx").setLevel(logging.WARNING)
|
||||
logging.getLogger("watchfiles.main").setLevel(logging.WARNING)
|
||||
|
||||
for warning_message in telemetry.pop_telemetry_warnings():
|
||||
logger.warning(warning_message)
|
||||
|
||||
|
||||
def _cleanup_windows_log_files(log_dir: Path, current_log_name: str) -> None:
|
||||
"""Trim stale per-process Windows log files so the directory stays bounded."""
|
||||
|
||||
@@ -0,0 +1,114 @@
|
||||
"""Request-local workspace context for canonical permalink generation."""
|
||||
|
||||
import re
|
||||
from contextlib import contextmanager
|
||||
from contextvars import ContextVar
|
||||
from dataclasses import dataclass
|
||||
from typing import Iterator
|
||||
|
||||
WORKSPACE_SLUG_HEADER = "X-Basic-Memory-Workspace-Slug"
|
||||
WORKSPACE_TYPE_HEADER = "X-Basic-Memory-Workspace-Type"
|
||||
_WORKSPACE_SLUG_PATTERN = re.compile(r"^[a-z0-9_-]+$")
|
||||
_WORKSPACE_TYPES = {"personal", "organization"}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class WorkspacePermalinkContext:
|
||||
"""Workspace metadata needed to build canonical organization permalinks."""
|
||||
|
||||
workspace_slug: str
|
||||
workspace_type: str
|
||||
|
||||
@property
|
||||
def should_prefix_permalinks(self) -> bool:
|
||||
return self.workspace_type == "organization" and bool(self.workspace_slug)
|
||||
|
||||
|
||||
_workspace_permalink_context: ContextVar[WorkspacePermalinkContext | None] = ContextVar(
|
||||
"basic_memory_workspace_permalink_context",
|
||||
default=None,
|
||||
)
|
||||
|
||||
|
||||
def current_workspace_permalink_context() -> WorkspacePermalinkContext | None:
|
||||
"""Return the active workspace permalink context, when one is set."""
|
||||
return _workspace_permalink_context.get()
|
||||
|
||||
|
||||
def validate_workspace_permalink_context_values(
|
||||
workspace_slug: str | None,
|
||||
workspace_type: str | None,
|
||||
) -> None:
|
||||
"""Validate workspace permalink metadata before it can affect stored permalinks."""
|
||||
validation_error = workspace_permalink_context_validation_error(workspace_slug, workspace_type)
|
||||
if validation_error is not None:
|
||||
raise ValueError(validation_error)
|
||||
|
||||
|
||||
def workspace_permalink_context_validation_error(
|
||||
workspace_slug: str | None,
|
||||
workspace_type: str | None,
|
||||
) -> str | None:
|
||||
"""Return the validation error for workspace permalink metadata, if any."""
|
||||
if bool(workspace_slug) != bool(workspace_type):
|
||||
return "workspace_slug and workspace_type must be provided together"
|
||||
|
||||
if not workspace_slug or not workspace_type:
|
||||
return None
|
||||
|
||||
if _WORKSPACE_SLUG_PATTERN.fullmatch(workspace_slug) is None:
|
||||
return f"{WORKSPACE_SLUG_HEADER} must match [a-z0-9_-]+"
|
||||
|
||||
if workspace_type not in _WORKSPACE_TYPES:
|
||||
allowed = ", ".join(sorted(_WORKSPACE_TYPES))
|
||||
return f"{WORKSPACE_TYPE_HEADER} must be one of: {allowed}"
|
||||
|
||||
return None
|
||||
|
||||
|
||||
@contextmanager
|
||||
def workspace_permalink_context(
|
||||
workspace_slug: str | None,
|
||||
workspace_type: str | None,
|
||||
) -> Iterator[None]:
|
||||
"""Set request-local workspace permalink metadata.
|
||||
|
||||
Cloud can populate this per request without storing workspace metadata in
|
||||
local project config. The slug/type pair is all permalink generation needs.
|
||||
"""
|
||||
validate_workspace_permalink_context_values(workspace_slug, workspace_type)
|
||||
|
||||
if not workspace_slug or not workspace_type:
|
||||
yield
|
||||
return
|
||||
|
||||
token = _workspace_permalink_context.set(
|
||||
WorkspacePermalinkContext(
|
||||
workspace_slug=workspace_slug,
|
||||
workspace_type=workspace_type,
|
||||
)
|
||||
)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
_workspace_permalink_context.reset(token)
|
||||
|
||||
|
||||
def workspace_permalink_headers() -> dict[str, str]:
|
||||
"""Return HTTP headers for forwarding workspace permalink context."""
|
||||
context = current_workspace_permalink_context()
|
||||
if context is None:
|
||||
return {}
|
||||
|
||||
return {
|
||||
WORKSPACE_SLUG_HEADER: context.workspace_slug,
|
||||
WORKSPACE_TYPE_HEADER: context.workspace_type,
|
||||
}
|
||||
|
||||
|
||||
def workspace_slug_for_canonical_permalinks() -> str | None:
|
||||
"""Return the workspace slug when new permalinks should include it."""
|
||||
context = current_workspace_permalink_context()
|
||||
if context and context.should_prefix_permalinks:
|
||||
return context.workspace_slug
|
||||
return None
|
||||
+68
-19
@@ -57,7 +57,12 @@ import pytest
|
||||
import pytest_asyncio
|
||||
from pathlib import Path
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
|
||||
from sqlalchemy.ext.asyncio import (
|
||||
AsyncEngine,
|
||||
AsyncSession,
|
||||
async_sessionmaker,
|
||||
create_async_engine,
|
||||
)
|
||||
from sqlalchemy.pool import NullPool
|
||||
from testcontainers.postgres import PostgresContainer
|
||||
|
||||
@@ -118,6 +123,21 @@ def postgres_container(db_backend):
|
||||
yield postgres
|
||||
|
||||
|
||||
@pytest_asyncio.fixture(autouse=True)
|
||||
async def cleanup_global_db_after_test() -> AsyncGenerator[None, None]:
|
||||
"""Close any module-level DB engine created outside fixture ownership."""
|
||||
yield
|
||||
|
||||
# Trigger: integration tests invoke CLI/MCP routes through the production
|
||||
# client fallback, bypassing this file's engine_factory fixture.
|
||||
# Why: those fallback engines live in basic_memory.db module state and can
|
||||
# otherwise leave a non-daemon aiosqlite worker alive after pytest finishes.
|
||||
# Outcome: every test boundary becomes a cleanup point for fallback engines.
|
||||
from basic_memory import db
|
||||
|
||||
await db.shutdown_db()
|
||||
|
||||
|
||||
POSTGRES_EPHEMERAL_TABLES = [
|
||||
"search_vector_embeddings",
|
||||
"search_vector_chunks",
|
||||
@@ -182,12 +202,36 @@ async def _reset_postgres_integration_schema(engine) -> None:
|
||||
)
|
||||
|
||||
|
||||
@pytest_asyncio.fixture(scope="session", loop_scope="session")
|
||||
async def postgres_engine(
|
||||
db_backend: Literal["sqlite", "postgres"], postgres_container
|
||||
) -> AsyncGenerator[AsyncEngine | None, None]:
|
||||
"""Create the shared Postgres engine once per integration test session."""
|
||||
if db_backend != "postgres":
|
||||
yield None
|
||||
return
|
||||
|
||||
sync_url = _resolve_postgres_sync_url(postgres_container)
|
||||
async_url = sync_url.replace("postgresql+psycopg2", "postgresql+asyncpg")
|
||||
engine = create_async_engine(
|
||||
async_url,
|
||||
echo=False,
|
||||
poolclass=NullPool,
|
||||
)
|
||||
|
||||
try:
|
||||
yield engine
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def engine_factory(
|
||||
app_config,
|
||||
config_manager,
|
||||
db_backend: Literal["sqlite", "postgres"],
|
||||
postgres_container,
|
||||
postgres_engine,
|
||||
tmp_path,
|
||||
) -> AsyncGenerator[tuple, None]:
|
||||
"""Create engine and session factory for the configured database backend."""
|
||||
@@ -195,18 +239,17 @@ async def engine_factory(
|
||||
from basic_memory import db
|
||||
|
||||
if db_backend == "postgres":
|
||||
# Postgres mode using testcontainers
|
||||
sync_url = _resolve_postgres_sync_url(postgres_container)
|
||||
async_url = sync_url.replace("postgresql+psycopg2", "postgresql+asyncpg")
|
||||
assert postgres_engine is not None
|
||||
|
||||
engine = create_async_engine(
|
||||
async_url,
|
||||
echo=False,
|
||||
poolclass=NullPool,
|
||||
)
|
||||
# Trigger: full-stack MCP/CLI tests exercise sync/indexing code that can
|
||||
# recover from DB errors by rolling back and opening later scoped sessions.
|
||||
# Why: one savepoint-backed connection is too brittle for that flow.
|
||||
# Outcome: reuse the engine, but reset rows/schema before each test and
|
||||
# let app code use normal transaction boundaries.
|
||||
await _reset_postgres_integration_schema(postgres_engine)
|
||||
|
||||
session_maker = async_sessionmaker(
|
||||
bind=engine,
|
||||
bind=postgres_engine,
|
||||
class_=AsyncSession,
|
||||
expire_on_commit=False,
|
||||
autoflush=False,
|
||||
@@ -214,17 +257,17 @@ async def engine_factory(
|
||||
|
||||
# Set module-level state to prevent MCP lifespan from re-initializing
|
||||
# This ensures get_or_create_db() sees an existing engine and skips initialization
|
||||
db._engine = engine
|
||||
db._engine = postgres_engine
|
||||
db._session_maker = session_maker
|
||||
|
||||
await _reset_postgres_integration_schema(engine)
|
||||
|
||||
yield engine, session_maker
|
||||
|
||||
# Clean up module-level state
|
||||
await engine.dispose()
|
||||
db._engine = None
|
||||
db._session_maker = None
|
||||
try:
|
||||
yield postgres_engine, session_maker
|
||||
finally:
|
||||
# Clean up module-level state
|
||||
if db._engine is postgres_engine:
|
||||
db._engine = None
|
||||
if db._session_maker is session_maker:
|
||||
db._session_maker = None
|
||||
|
||||
else:
|
||||
# SQLite: Create fresh database (fast with tmp files)
|
||||
@@ -264,7 +307,13 @@ async def test_project(config_home, engine_factory) -> Project:
|
||||
|
||||
@pytest.fixture
|
||||
def config_home(tmp_path, monkeypatch) -> Path:
|
||||
# Patch both HOME and USERPROFILE so Path.home() returns the test dir on
|
||||
# every platform — Path.home() reads HOME on POSIX and USERPROFILE on
|
||||
# Windows, and ConfigManager.data_dir_path now goes through Path.home()
|
||||
# via resolve_data_dir(). Must mirror tests/conftest.py:config_home.
|
||||
monkeypatch.setenv("HOME", str(tmp_path))
|
||||
if os.name == "nt":
|
||||
monkeypatch.setenv("USERPROFILE", str(tmp_path))
|
||||
# Set BASIC_MEMORY_HOME to the test directory
|
||||
monkeypatch.setenv("BASIC_MEMORY_HOME", str(tmp_path / "basic-memory"))
|
||||
return tmp_path
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
"""
|
||||
Integration test for long relation_type values (regression guard for issue #721).
|
||||
|
||||
When a markdown bullet contains an inline `[[wikilink]]` preceded by long prose,
|
||||
`parse_relation()` extracts ALL of that prose as the `relation_type`. Previously
|
||||
the response model `RelationType` had a `MaxLen(200)` constraint that caused
|
||||
edit_note (which round-trips through the response model when re-indexing) to
|
||||
fail with:
|
||||
|
||||
1 validation error for EntityResponseV2
|
||||
relations.0.relation_type
|
||||
String should have at most 200 characters
|
||||
|
||||
Commit 01cbad1d removed the cap from `RelationType`. This test locks in that
|
||||
fix so a future contributor reintroducing `MaxLen` will see the test fail
|
||||
before shipping it.
|
||||
|
||||
Out of scope: improving `parse_relation()` to fall back to a default relation
|
||||
type when the prose-before-link looks like a sentence rather than a label.
|
||||
That's a knowledge-graph-quality improvement, not a correctness fix, and is
|
||||
not required to keep edit_note working.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
from fastmcp import Client
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_edit_note_handles_long_prose_around_wikilink(mcp_server, app, test_project):
|
||||
"""edit_note must succeed on notes whose inline wikilinks have >200 chars
|
||||
of prose preceding them — that prose becomes the parsed relation_type."""
|
||||
long_prose = (
|
||||
"**Lorem ipsum dolor sit amet** — consectetur adipiscing elit, sed do eiusmod "
|
||||
"tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, "
|
||||
"quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo "
|
||||
"consequat. Trust boundary model documented in"
|
||||
)
|
||||
assert len(long_prose) > 200, (
|
||||
f"setup wrong: prose-before-link must exceed the historical 200-char cap, "
|
||||
f"got {len(long_prose)} chars"
|
||||
)
|
||||
|
||||
note_body = (
|
||||
"# Long Relation Type Repro\n\n"
|
||||
f"- {long_prose} [[Some Note Title]] for additional context.\n"
|
||||
)
|
||||
|
||||
async with Client(mcp_server) as client:
|
||||
# Create the note (file-write side; would already fail at index time
|
||||
# if RelationType MaxLen were back).
|
||||
write_result = await client.call_tool(
|
||||
"write_note",
|
||||
{
|
||||
"project": test_project.name,
|
||||
"title": "Long Relation Type Repro",
|
||||
"directory": "issue721",
|
||||
"content": note_body,
|
||||
},
|
||||
)
|
||||
assert len(write_result.content) == 1
|
||||
write_text = write_result.content[0].text
|
||||
assert "Created note" in write_text or "Updated note" in write_text
|
||||
|
||||
# Edit the note. This triggers re-index → response model validation.
|
||||
# With the historical MaxLen(200) cap, this would raise:
|
||||
# "relations.0.relation_type String should have at most 200 characters"
|
||||
edit_result = await client.call_tool(
|
||||
"edit_note",
|
||||
{
|
||||
"project": test_project.name,
|
||||
"identifier": "Long Relation Type Repro",
|
||||
"operation": "append",
|
||||
"content": "\n\nappended line\n",
|
||||
},
|
||||
)
|
||||
assert len(edit_result.content) == 1
|
||||
assert "Edited note (append)" in edit_result.content[0].text
|
||||
@@ -4,6 +4,7 @@ from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from fastmcp import Client
|
||||
@@ -11,7 +12,7 @@ from fastmcp import Client
|
||||
from basic_memory.mcp.clients.knowledge import KnowledgeClient
|
||||
|
||||
|
||||
def _json_content(tool_result) -> dict | list:
|
||||
def _json_content(tool_result) -> Any:
|
||||
"""Parse a FastMCP tool result content block into JSON."""
|
||||
assert len(tool_result.content) == 1
|
||||
assert tool_result.content[0].type == "text"
|
||||
|
||||
@@ -7,12 +7,13 @@ results are available.
|
||||
"""
|
||||
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from fastmcp import Client
|
||||
|
||||
|
||||
def _json_content(tool_result) -> dict | list:
|
||||
def _json_content(tool_result) -> Any:
|
||||
"""Parse a FastMCP tool result content block into JSON."""
|
||||
assert len(tool_result.content) == 1
|
||||
assert tool_result.content[0].type == "text"
|
||||
|
||||
@@ -0,0 +1,540 @@
|
||||
"""
|
||||
Integration tests for MCP tool parameter aliases.
|
||||
|
||||
Verifies that MCP tools accept training-data-friendly parameter aliases
|
||||
(via Pydantic AliasChoices) alongside the canonical names, so models
|
||||
that reach for `offset`/`limit`/`find`/`old_text` etc. don't hit
|
||||
validation errors on first use.
|
||||
|
||||
See: https://github.com/basicmachines-co/basic-memory/issues/690
|
||||
"""
|
||||
|
||||
import pytest
|
||||
from fastmcp import Client
|
||||
|
||||
|
||||
# --- read_note: pagination params removed in #693 (were no-ops) ---
|
||||
# The `page` / `page_size` parameters were removed because the API endpoint
|
||||
# silently dropped them. Search-fallback pagination is unrelated to read_note.
|
||||
|
||||
|
||||
# --- edit_note: find_text / content / section aliases ---
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_edit_note_accepts_find_alias_for_find_text(mcp_server, app, test_project):
|
||||
"""`find` should map to `find_text` — the highest-frequency miss in the issue."""
|
||||
async with Client(mcp_server) as client:
|
||||
await client.call_tool(
|
||||
"write_note",
|
||||
{
|
||||
"project": test_project.name,
|
||||
"title": "Find Alias Note",
|
||||
"directory": "test",
|
||||
"content": "# Find Alias Note\n\nVersion v1.0.0 of the spec.",
|
||||
},
|
||||
)
|
||||
|
||||
result = await client.call_tool(
|
||||
"edit_note",
|
||||
{
|
||||
"project": test_project.name,
|
||||
"identifier": "Find Alias Note",
|
||||
"operation": "find_replace",
|
||||
"content": "v2.0.0",
|
||||
"find": "v1.0.0", # alias for find_text
|
||||
},
|
||||
)
|
||||
|
||||
assert "Edited note (find_replace)" in result.content[0].text
|
||||
|
||||
read_result = await client.call_tool(
|
||||
"read_note",
|
||||
{"project": test_project.name, "identifier": "Find Alias Note"},
|
||||
)
|
||||
assert "v2.0.0" in read_result.content[0].text
|
||||
assert "v1.0.0" not in read_result.content[0].text
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_edit_note_accepts_old_text_alias(mcp_server, app, test_project):
|
||||
"""`old_text` (diff/patch convention) should map to `find_text`."""
|
||||
async with Client(mcp_server) as client:
|
||||
await client.call_tool(
|
||||
"write_note",
|
||||
{
|
||||
"project": test_project.name,
|
||||
"title": "Old Text Note",
|
||||
"directory": "test",
|
||||
"content": "# Old Text Note\n\nThe quick brown fox.",
|
||||
},
|
||||
)
|
||||
|
||||
result = await client.call_tool(
|
||||
"edit_note",
|
||||
{
|
||||
"project": test_project.name,
|
||||
"identifier": "Old Text Note",
|
||||
"operation": "find_replace",
|
||||
"content": "lazy",
|
||||
"old_text": "quick",
|
||||
},
|
||||
)
|
||||
|
||||
assert "Edited note (find_replace)" in result.content[0].text
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_edit_note_accepts_new_content_alias_for_content(mcp_server, app, test_project):
|
||||
"""`new_content` should map to `content` — `content` is ambiguous as 'replacement text'."""
|
||||
async with Client(mcp_server) as client:
|
||||
await client.call_tool(
|
||||
"write_note",
|
||||
{
|
||||
"project": test_project.name,
|
||||
"title": "New Content Note",
|
||||
"directory": "test",
|
||||
"content": "# New Content Note\n\nplaceholder",
|
||||
},
|
||||
)
|
||||
|
||||
result = await client.call_tool(
|
||||
"edit_note",
|
||||
{
|
||||
"project": test_project.name,
|
||||
"identifier": "New Content Note",
|
||||
"operation": "find_replace",
|
||||
"new_content": "actual value", # alias for content
|
||||
"find_text": "placeholder",
|
||||
},
|
||||
)
|
||||
|
||||
assert "Edited note (find_replace)" in result.content[0].text
|
||||
|
||||
read_result = await client.call_tool(
|
||||
"read_note",
|
||||
{"project": test_project.name, "identifier": "New Content Note"},
|
||||
)
|
||||
assert "actual value" in read_result.content[0].text
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_edit_note_accepts_section_heading_alias(mcp_server, app, test_project):
|
||||
"""`section_heading` and `heading` should map to `section`."""
|
||||
async with Client(mcp_server) as client:
|
||||
await client.call_tool(
|
||||
"write_note",
|
||||
{
|
||||
"project": test_project.name,
|
||||
"title": "Section Heading Note",
|
||||
"directory": "test",
|
||||
"content": "# Section Heading Note\n\n## Notes\n\nold notes\n",
|
||||
},
|
||||
)
|
||||
|
||||
result = await client.call_tool(
|
||||
"edit_note",
|
||||
{
|
||||
"project": test_project.name,
|
||||
"identifier": "Section Heading Note",
|
||||
"operation": "replace_section",
|
||||
"content": "fresh notes\n",
|
||||
"section_heading": "## Notes", # alias for section
|
||||
},
|
||||
)
|
||||
|
||||
assert "Edited note (replace_section)" in result.content[0].text
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_edit_note_canonical_names_still_work(mcp_server, app, test_project):
|
||||
"""Canonical names must keep working alongside aliases."""
|
||||
async with Client(mcp_server) as client:
|
||||
await client.call_tool(
|
||||
"write_note",
|
||||
{
|
||||
"project": test_project.name,
|
||||
"title": "Edit Canonical Note",
|
||||
"directory": "test",
|
||||
"content": "# Edit Canonical Note\n\nold-value here.",
|
||||
},
|
||||
)
|
||||
|
||||
result = await client.call_tool(
|
||||
"edit_note",
|
||||
{
|
||||
"project": test_project.name,
|
||||
"identifier": "Edit Canonical Note",
|
||||
"operation": "find_replace",
|
||||
"content": "new-value",
|
||||
"find_text": "old-value",
|
||||
},
|
||||
)
|
||||
|
||||
assert "Edited note (find_replace)" in result.content[0].text
|
||||
|
||||
|
||||
# --- search_notes aliases ---
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_search_notes_accepts_query_aliases(mcp_server, app, test_project):
|
||||
"""`q` (HTTP convention), `search`, and `text` should all map to `query`."""
|
||||
async with Client(mcp_server) as client:
|
||||
await client.call_tool(
|
||||
"write_note",
|
||||
{
|
||||
"project": test_project.name,
|
||||
"title": "Searchable Note",
|
||||
"directory": "test",
|
||||
"content": "# Searchable Note\n\nUnique-keyword-XYZ here.",
|
||||
},
|
||||
)
|
||||
|
||||
# Try each alias
|
||||
for alias_key in ("q", "search", "text"):
|
||||
result = await client.call_tool(
|
||||
"search_notes",
|
||||
{
|
||||
"project": test_project.name,
|
||||
alias_key: "Unique-keyword-XYZ",
|
||||
"limit": 5, # also testing pagination alias
|
||||
},
|
||||
)
|
||||
assert "Searchable Note" in result.content[0].text, f"alias {alias_key} failed"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_search_notes_accepts_after_date_aliases(mcp_server, app, test_project):
|
||||
"""`since`/`after`/`from_date` should map to `after_date`."""
|
||||
async with Client(mcp_server) as client:
|
||||
await client.call_tool(
|
||||
"write_note",
|
||||
{
|
||||
"project": test_project.name,
|
||||
"title": "Date Filter Note",
|
||||
"directory": "test",
|
||||
"content": "# Date Filter Note\n\nbody",
|
||||
},
|
||||
)
|
||||
|
||||
# Just verify the alias is accepted at validation time (no error)
|
||||
result = await client.call_tool(
|
||||
"search_notes",
|
||||
{"project": test_project.name, "query": "Date Filter", "since": "1d"},
|
||||
)
|
||||
assert result.content # didn't error
|
||||
|
||||
|
||||
# --- recent_activity aliases ---
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_recent_activity_accepts_timeframe_aliases(mcp_server, app, test_project):
|
||||
"""`since`/`time_range`/`lookback` should map to `timeframe`."""
|
||||
async with Client(mcp_server) as client:
|
||||
result = await client.call_tool(
|
||||
"recent_activity",
|
||||
{"project": test_project.name, "since": "7d", "limit": 5},
|
||||
)
|
||||
assert result.content # accepted, no validation error
|
||||
|
||||
|
||||
# --- list_directory aliases ---
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_list_directory_accepts_directory_alias(mcp_server, app, test_project):
|
||||
"""`directory`/`folder`/`path`/`dir` should all map to `dir_name`."""
|
||||
async with Client(mcp_server) as client:
|
||||
await client.call_tool(
|
||||
"write_note",
|
||||
{
|
||||
"project": test_project.name,
|
||||
"title": "Dir Test",
|
||||
"directory": "list-dir-aliases",
|
||||
"content": "# Dir Test\n\nbody",
|
||||
},
|
||||
)
|
||||
|
||||
for alias_key in ("directory", "folder", "path", "dir"):
|
||||
result = await client.call_tool(
|
||||
"list_directory",
|
||||
{"project": test_project.name, alias_key: "/list-dir-aliases"},
|
||||
)
|
||||
assert "Dir Test" in result.content[0].text, f"alias {alias_key} failed"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_list_directory_accepts_glob_aliases(mcp_server, app, test_project):
|
||||
"""`glob`/`pattern`/`filter` should map to `file_name_glob`."""
|
||||
async with Client(mcp_server) as client:
|
||||
await client.call_tool(
|
||||
"write_note",
|
||||
{
|
||||
"project": test_project.name,
|
||||
"title": "Glob Target",
|
||||
"directory": "glob-test",
|
||||
"content": "# Glob Target\n\nbody",
|
||||
},
|
||||
)
|
||||
|
||||
result = await client.call_tool(
|
||||
"list_directory",
|
||||
{
|
||||
"project": test_project.name,
|
||||
"dir_name": "/glob-test",
|
||||
"glob": "*.md",
|
||||
},
|
||||
)
|
||||
assert "Glob Target" in result.content[0].text
|
||||
|
||||
|
||||
# --- write_note aliases ---
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_write_note_accepts_directory_aliases(mcp_server, app, test_project):
|
||||
"""`folder`/`dir`/`path` should map to `directory`."""
|
||||
async with Client(mcp_server) as client:
|
||||
result = await client.call_tool(
|
||||
"write_note",
|
||||
{
|
||||
"project": test_project.name,
|
||||
"title": "Folder Alias Note",
|
||||
"folder": "folder-alias-test", # alias
|
||||
"content": "# Folder Alias Note\n\nbody",
|
||||
},
|
||||
)
|
||||
assert "folder-alias-test" in result.content[0].text
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_write_note_accepts_overwrite_aliases(mcp_server, app, test_project):
|
||||
"""`force`/`replace` should map to `overwrite`."""
|
||||
async with Client(mcp_server) as client:
|
||||
# First create
|
||||
await client.call_tool(
|
||||
"write_note",
|
||||
{
|
||||
"project": test_project.name,
|
||||
"title": "Overwrite Alias Note",
|
||||
"directory": "overwrite-test",
|
||||
"content": "v1",
|
||||
},
|
||||
)
|
||||
# Overwrite using `force` alias
|
||||
result = await client.call_tool(
|
||||
"write_note",
|
||||
{
|
||||
"project": test_project.name,
|
||||
"title": "Overwrite Alias Note",
|
||||
"directory": "overwrite-test",
|
||||
"content": "v2",
|
||||
"force": True, # alias for overwrite
|
||||
},
|
||||
)
|
||||
assert "Updated note" in result.content[0].text or "Created note" in result.content[0].text
|
||||
|
||||
|
||||
# --- move_note aliases ---
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_move_note_accepts_destination_aliases(mcp_server, app, test_project):
|
||||
"""`to`/`dest_path`/`new_path`/`destination` should map to `destination_path`."""
|
||||
async with Client(mcp_server) as client:
|
||||
await client.call_tool(
|
||||
"write_note",
|
||||
{
|
||||
"project": test_project.name,
|
||||
"title": "Move Target",
|
||||
"directory": "move-src",
|
||||
"content": "# Move Target\n\nbody",
|
||||
},
|
||||
)
|
||||
result = await client.call_tool(
|
||||
"move_note",
|
||||
{
|
||||
"project": test_project.name,
|
||||
"identifier": "Move Target",
|
||||
"to": "move-dest/Move Target.md", # alias for destination_path
|
||||
},
|
||||
)
|
||||
assert "move-dest" in result.content[0].text
|
||||
|
||||
|
||||
# --- read_content aliases ---
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_read_content_accepts_file_path_alias(mcp_server, app, test_project):
|
||||
"""`file_path`/`filepath`/`file` should map to `path`."""
|
||||
async with Client(mcp_server) as client:
|
||||
await client.call_tool(
|
||||
"write_note",
|
||||
{
|
||||
"project": test_project.name,
|
||||
"title": "Read Content Target",
|
||||
"directory": "read-content-test",
|
||||
"content": "# Read Content Target\n\nraw body",
|
||||
},
|
||||
)
|
||||
result = await client.call_tool(
|
||||
"read_content",
|
||||
{
|
||||
"project": test_project.name,
|
||||
"file_path": "read-content-test/Read Content Target.md",
|
||||
},
|
||||
)
|
||||
# read_content returns a dict; structured content should include the file
|
||||
text = result.content[0].text if result.content else ""
|
||||
struct = result.structured_content if hasattr(result, "structured_content") else None
|
||||
assert "raw body" in text or (struct and "raw body" in str(struct))
|
||||
|
||||
|
||||
# --- build_context aliases ---
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_build_context_accepts_url_aliases(mcp_server, app, test_project):
|
||||
"""`uri`/`memory_url` should map to `url`."""
|
||||
async with Client(mcp_server) as client:
|
||||
await client.call_tool(
|
||||
"write_note",
|
||||
{
|
||||
"project": test_project.name,
|
||||
"title": "Context Target",
|
||||
"directory": "build-ctx",
|
||||
"content": "# Context Target\n\nbody",
|
||||
},
|
||||
)
|
||||
result = await client.call_tool(
|
||||
"build_context",
|
||||
{
|
||||
"project": test_project.name,
|
||||
"uri": "memory://build-ctx/context-target", # alias for url
|
||||
},
|
||||
)
|
||||
# Just verify validation accepted the alias
|
||||
assert result.content or result.structured_content
|
||||
|
||||
|
||||
# --- view_note: pagination params removed in #693 (delegates to read_note) ---
|
||||
|
||||
|
||||
# --- delete_note aliases ---
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_delete_note_accepts_is_dir_alias(mcp_server, app, test_project):
|
||||
"""`is_dir` should map to `is_directory` and route to single-note deletion."""
|
||||
async with Client(mcp_server) as client:
|
||||
await client.call_tool(
|
||||
"write_note",
|
||||
{
|
||||
"project": test_project.name,
|
||||
"title": "Delete Alias Note",
|
||||
"directory": "delete-alias-test",
|
||||
"content": "# Delete Alias Note\n\nBody.",
|
||||
},
|
||||
)
|
||||
|
||||
result = await client.call_tool(
|
||||
"delete_note",
|
||||
{
|
||||
"project": test_project.name,
|
||||
"identifier": "delete-alias-test/Delete Alias Note",
|
||||
"is_dir": False, # alias for is_directory
|
||||
},
|
||||
)
|
||||
|
||||
# delete_note returns a bool/dict on success; just assert no error
|
||||
assert result.content or result.structured_content
|
||||
|
||||
|
||||
# --- Schema sanity check: aliases must not appear in the advertised schema ---
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_aliases_not_advertised_in_schema(mcp_server, app):
|
||||
"""The JSON schema sent to models should advertise only canonical names.
|
||||
|
||||
Aliases are accepted at validation time but advertising them would defeat
|
||||
the purpose: we want the model to learn the canonical name, with aliases
|
||||
as a silent safety net for first-use mistakes.
|
||||
|
||||
The `must_not_have` lists below intentionally include both *accepted*
|
||||
aliases (which must stay hidden from the schema) AND *rejected* aliases
|
||||
that were considered but deliberately omitted (`offset` for `page`,
|
||||
`limit_related` for `max_related`). Listing rejected aliases here acts
|
||||
as a future-contributor guard — if anyone re-adds them, this test catches
|
||||
it before the bad alias ships.
|
||||
"""
|
||||
async with Client(mcp_server) as client:
|
||||
tools = {t.name: t for t in await client.list_tools()}
|
||||
|
||||
# tool_name -> (must_have_canonical, must_not_have_aliases)
|
||||
checks = {
|
||||
# read_note has no pagination params (#693 — they were no-ops; removed).
|
||||
# The must_not_have list still includes the rejected aliases so future
|
||||
# contributors don't reintroduce them.
|
||||
"read_note": (
|
||||
[],
|
||||
["page", "page_size", "offset", "limit", "page_number", "per_page"],
|
||||
),
|
||||
"edit_note": (
|
||||
["find_text", "section", "content"],
|
||||
["find", "old_text", "old_content", "search", "new_content", "section_heading"],
|
||||
),
|
||||
"search_notes": (
|
||||
["query", "page", "page_size", "note_types", "after_date", "min_similarity"],
|
||||
[
|
||||
"q",
|
||||
"search",
|
||||
"offset",
|
||||
"limit",
|
||||
"note_type",
|
||||
"types",
|
||||
"since",
|
||||
"after",
|
||||
"threshold",
|
||||
],
|
||||
),
|
||||
"recent_activity": (
|
||||
["type", "timeframe", "page", "page_size"],
|
||||
["types", "kind", "since", "time_range", "lookback", "offset", "limit"],
|
||||
),
|
||||
"list_directory": (
|
||||
["dir_name", "file_name_glob"],
|
||||
["directory", "folder", "path", "dir", "glob", "pattern", "filter"],
|
||||
),
|
||||
"write_note": (
|
||||
["directory", "overwrite"],
|
||||
["folder", "dir", "path", "force", "replace"],
|
||||
),
|
||||
"move_note": (
|
||||
["destination_path", "destination_folder", "is_directory"],
|
||||
["dest_path", "new_path", "to", "destination", "is_dir"],
|
||||
),
|
||||
"delete_note": (["is_directory"], ["is_dir"]),
|
||||
"read_content": (["path"], ["file_path", "filepath", "file"]),
|
||||
# view_note pagination params removed in #693 (delegates to read_note).
|
||||
"view_note": (
|
||||
[],
|
||||
["page", "page_size", "offset", "limit", "page_number", "per_page"],
|
||||
),
|
||||
"build_context": (
|
||||
["url", "timeframe", "page", "page_size", "max_related"],
|
||||
["uri", "memory_url", "since", "offset", "limit", "max_results", "limit_related"],
|
||||
),
|
||||
"canvas": (["directory"], ["folder", "dir", "path"]),
|
||||
}
|
||||
|
||||
for tool_name, (must_have, must_not_have) in checks.items():
|
||||
assert tool_name in tools, f"tool {tool_name} not registered"
|
||||
props = tools[tool_name].inputSchema["properties"]
|
||||
for canonical in must_have:
|
||||
assert canonical in props, f"{tool_name}: canonical '{canonical}' missing"
|
||||
for alias in must_not_have:
|
||||
assert alias not in props, f"{tool_name}: alias '{alias}' leaked into schema"
|
||||
@@ -588,3 +588,122 @@ async def test_nested_project_paths_rejected(mcp_server, app, test_project, tmp_
|
||||
|
||||
# Clean up parent project
|
||||
await client.call_tool("delete_project", {"project_name": parent_name})
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_create_project_accepts_workspace_in_local_mode(
|
||||
mcp_server, app, test_project, tmp_path
|
||||
):
|
||||
"""Passing workspace via the MCP wire is accepted by the tool schema and
|
||||
does not break the local create path.
|
||||
|
||||
In local mode there is no cloud factory installed, so workspace is a no-op:
|
||||
the request lands on the ASGI transport which has no workspace concept. This
|
||||
test guards the schema so a future change can't accidentally drop the parameter.
|
||||
"""
|
||||
|
||||
async with Client(mcp_server) as client:
|
||||
create_result = await client.call_tool(
|
||||
"create_memory_project",
|
||||
{
|
||||
"project_name": "ws-local-test",
|
||||
"project_path": str(
|
||||
tmp_path.parent / (tmp_path.name + "-projects") / "project-ws-local-test"
|
||||
),
|
||||
"workspace": "team-paul",
|
||||
},
|
||||
)
|
||||
|
||||
assert len(create_result.content) == 1
|
||||
create_text = create_result.content[0].text # pyright: ignore [reportAttributeAccessIssue]
|
||||
assert "✓" in create_text
|
||||
assert "ws-local-test" in create_text
|
||||
|
||||
list_result = await client.call_tool("list_memory_projects", {})
|
||||
assert "ws-local-test" in list_result.content[0].text # pyright: ignore [reportAttributeAccessIssue]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_create_project_workspace_slug_forwarded_to_factory_as_tenant_id(
|
||||
mcp_server, app, test_project, tmp_path
|
||||
):
|
||||
"""workspace slug resolves before the tenant id flows to the cloud factory.
|
||||
|
||||
Simulates the cloud MCP server pattern (set_client_factory) and verifies the
|
||||
factory receives the workspace argument. This is the chicken-and-egg case:
|
||||
no project_id exists yet, so workspace is the only way to target a
|
||||
non-default workspace at create time.
|
||||
"""
|
||||
from contextlib import asynccontextmanager
|
||||
from unittest.mock import AsyncMock, patch
|
||||
|
||||
from httpx import ASGITransport, AsyncClient as HttpxAsyncClient
|
||||
|
||||
from basic_memory.mcp import async_client
|
||||
from basic_memory.mcp.tools import project_management
|
||||
from basic_memory.schemas.cloud import WorkspaceInfo
|
||||
|
||||
captured_workspaces: list[str | None] = []
|
||||
resolved_workspace = WorkspaceInfo(
|
||||
tenant_id="tenant-cloud-test",
|
||||
name="Team Paul",
|
||||
workspace_type="organization",
|
||||
slug="team-paul",
|
||||
role="owner",
|
||||
organization_id="org-team-paul",
|
||||
is_default=False,
|
||||
has_active_subscription=True,
|
||||
)
|
||||
|
||||
@asynccontextmanager
|
||||
async def fake_factory(workspace=None):
|
||||
captured_workspaces.append(workspace)
|
||||
# Yield an ASGI-backed httpx client so the create_project HTTP call
|
||||
# actually reaches the FastAPI app and the project is created in the DB.
|
||||
async with HttpxAsyncClient(
|
||||
transport=ASGITransport(app=app), base_url="http://test"
|
||||
) as inner:
|
||||
yield inner
|
||||
|
||||
original_factory = async_client._client_factory
|
||||
async_client.set_client_factory(fake_factory)
|
||||
try:
|
||||
with patch.object(
|
||||
project_management,
|
||||
"resolve_workspace_parameter",
|
||||
new_callable=AsyncMock,
|
||||
return_value=resolved_workspace,
|
||||
) as mock_resolve_workspace:
|
||||
async with Client(mcp_server) as mcp_client:
|
||||
create_result = await mcp_client.call_tool(
|
||||
"create_memory_project",
|
||||
{
|
||||
"project_name": "ws-routed-project",
|
||||
"project_path": str(
|
||||
tmp_path.parent
|
||||
/ (tmp_path.name + "-projects")
|
||||
/ "project-ws-routed-project"
|
||||
),
|
||||
"workspace": "team-paul",
|
||||
},
|
||||
)
|
||||
|
||||
create_text = create_result.content[0].text # pyright: ignore [reportAttributeAccessIssue]
|
||||
assert "✓" in create_text
|
||||
assert "ws-routed-project" in create_text
|
||||
|
||||
mock_resolve_workspace.assert_awaited_once()
|
||||
await_args = mock_resolve_workspace.await_args
|
||||
assert await_args is not None
|
||||
assert await_args.kwargs["workspace"] == "team-paul"
|
||||
# The factory must have been invoked with the tenant id resolved from the slug.
|
||||
# create_memory_project opens one get_client() context, so the factory is
|
||||
# called once per tool invocation; both list_projects and create_project
|
||||
# share that single client.
|
||||
assert captured_workspaces, "Factory was never invoked"
|
||||
assert all(ws == "tenant-cloud-test" for ws in captured_workspaces), (
|
||||
"Expected workspace='tenant-cloud-test' on every factory call, "
|
||||
f"got {captured_workspaces}"
|
||||
)
|
||||
finally:
|
||||
async_client._client_factory = original_factory
|
||||
|
||||
@@ -100,3 +100,71 @@ async def test_read_note_underscored_folder_by_permalink(mcp_server, app, test_p
|
||||
assert "# Example Note" in result_text
|
||||
assert "This is a test note in an underscored folder." in result_text
|
||||
assert f"{test_project.name}/archive/articles/example-note" in result_text # permalink
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_read_note_by_project_id(mcp_server, app, test_project):
|
||||
"""Read a note by passing project_id (UUID) instead of project name.
|
||||
|
||||
Verifies the project_id parameter routes through get_project_client correctly
|
||||
in pure local mode (no cloud creds), where get_project_mode() would otherwise
|
||||
default unknown identifiers to CLOUD and break routing.
|
||||
"""
|
||||
|
||||
async with Client(mcp_server) as client:
|
||||
await client.call_tool(
|
||||
"write_note",
|
||||
{
|
||||
"project": test_project.name,
|
||||
"title": "By ID Note",
|
||||
"directory": "test",
|
||||
"content": "# By ID Note\n\nLooked up by external_id.",
|
||||
},
|
||||
)
|
||||
|
||||
# Read by external_id (UUID) instead of project name
|
||||
read_result = await client.call_tool(
|
||||
"read_note",
|
||||
{
|
||||
"project_id": test_project.external_id,
|
||||
"identifier": "By ID Note",
|
||||
},
|
||||
)
|
||||
|
||||
assert len(read_result.content) == 1
|
||||
assert read_result.content[0].type == "text"
|
||||
result_text = read_result.content[0].text
|
||||
assert "# By ID Note" in result_text
|
||||
assert "Looked up by external_id." in result_text
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_read_note_project_id_takes_precedence_over_name(mcp_server, app, test_project):
|
||||
"""When project_id is passed alongside a wrong project name, project_id wins."""
|
||||
|
||||
async with Client(mcp_server) as client:
|
||||
await client.call_tool(
|
||||
"write_note",
|
||||
{
|
||||
"project": test_project.name,
|
||||
"title": "Precedence Note",
|
||||
"directory": "test",
|
||||
"content": "# Precedence Note\n\nproject_id wins.",
|
||||
},
|
||||
)
|
||||
|
||||
# Pass an obviously-wrong project name alongside the correct project_id.
|
||||
# If project_id takes precedence (as documented), the read still succeeds.
|
||||
read_result = await client.call_tool(
|
||||
"read_note",
|
||||
{
|
||||
"project": "this-project-does-not-exist",
|
||||
"project_id": test_project.external_id,
|
||||
"identifier": "Precedence Note",
|
||||
},
|
||||
)
|
||||
|
||||
assert len(read_result.content) == 1
|
||||
result_text = read_result.content[0].text
|
||||
assert "# Precedence Note" in result_text
|
||||
assert "project_id wins." in result_text
|
||||
|
||||
@@ -8,6 +8,7 @@ SearchService for each (backend, provider) combination.
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from collections.abc import AsyncGenerator
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
@@ -15,7 +16,12 @@ import pytest
|
||||
import pytest_asyncio
|
||||
from dotenv import load_dotenv
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
|
||||
from sqlalchemy.ext.asyncio import (
|
||||
AsyncEngine,
|
||||
AsyncSession,
|
||||
async_sessionmaker,
|
||||
create_async_engine,
|
||||
)
|
||||
from sqlalchemy.pool import NullPool
|
||||
from testcontainers.postgres import PostgresContainer
|
||||
|
||||
@@ -152,25 +158,8 @@ async def sqlite_engine_factory(tmp_path):
|
||||
yield engine, session_maker
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def postgres_engine_factory(pgvector_container):
|
||||
"""Create a Postgres engine + session factory with pgvector extension."""
|
||||
if pgvector_container is None:
|
||||
yield None
|
||||
return
|
||||
|
||||
sync_url = pgvector_container.get_connection_url()
|
||||
async_url = sync_url.replace("postgresql+psycopg2", "postgresql+asyncpg")
|
||||
|
||||
engine = create_async_engine(async_url, echo=False, poolclass=NullPool)
|
||||
session_maker = async_sessionmaker(
|
||||
bind=engine,
|
||||
class_=AsyncSession,
|
||||
expire_on_commit=False,
|
||||
autoflush=False,
|
||||
)
|
||||
|
||||
# Create schema from scratch for each test
|
||||
async def _reset_postgres_semantic_schema(engine: AsyncEngine) -> None:
|
||||
"""Reset the semantic Postgres schema to a clean baseline."""
|
||||
async with engine.begin() as conn:
|
||||
await conn.execute(text("DROP TABLE IF EXISTS search_vector_embeddings CASCADE"))
|
||||
await conn.execute(text("DROP TABLE IF EXISTS search_vector_chunks CASCADE"))
|
||||
@@ -182,9 +171,47 @@ async def postgres_engine_factory(pgvector_container):
|
||||
await conn.execute(CREATE_POSTGRES_SEARCH_INDEX_METADATA)
|
||||
await conn.execute(CREATE_POSTGRES_SEARCH_INDEX_PERMALINK)
|
||||
|
||||
yield engine, session_maker
|
||||
|
||||
await engine.dispose()
|
||||
@pytest_asyncio.fixture(scope="session", loop_scope="session")
|
||||
async def postgres_engine(pgvector_container) -> AsyncGenerator[AsyncEngine | None, None]:
|
||||
"""Create the shared semantic Postgres engine once per test session."""
|
||||
if pgvector_container is None:
|
||||
yield None
|
||||
return
|
||||
|
||||
sync_url = pgvector_container.get_connection_url()
|
||||
async_url = sync_url.replace("postgresql+psycopg2", "postgresql+asyncpg")
|
||||
|
||||
engine = create_async_engine(async_url, echo=False, poolclass=NullPool)
|
||||
|
||||
try:
|
||||
yield engine
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def postgres_engine_factory(postgres_engine):
|
||||
"""Create a Postgres session factory isolated by schema reset."""
|
||||
if postgres_engine is None:
|
||||
yield None
|
||||
return
|
||||
|
||||
# Trigger: semantic provider combos create and rebuild vector tables.
|
||||
# Why: the main suite showed savepoint-bound shared connections can get
|
||||
# poisoned by app-level rollback/recovery paths.
|
||||
# Outcome: keep the pgvector engine warm, but reset schema per test and let
|
||||
# repository code use normal Postgres transactions.
|
||||
await _reset_postgres_semantic_schema(postgres_engine)
|
||||
|
||||
session_maker = async_sessionmaker(
|
||||
bind=postgres_engine,
|
||||
class_=AsyncSession,
|
||||
expire_on_commit=False,
|
||||
autoflush=False,
|
||||
)
|
||||
|
||||
yield postgres_engine, session_maker
|
||||
|
||||
|
||||
# --- Embedding provider factories ---
|
||||
|
||||
@@ -9,6 +9,8 @@ These tests isolate specific problems with the search pipeline:
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, cast
|
||||
|
||||
import pytest
|
||||
|
||||
from basic_memory.config import DatabaseBackend
|
||||
@@ -335,11 +337,10 @@ async def test_similarity_formula_analysis(sqlite_engine_factory, tmp_path):
|
||||
|
||||
from basic_memory import db as bm_db
|
||||
|
||||
async with bm_db.scoped_session(service.repository.session_maker) as session:
|
||||
await service.repository._prepare_vector_session(session)
|
||||
raw_rows = await service.repository._run_vector_query(
|
||||
session, query_embedding, candidate_limit=20
|
||||
)
|
||||
repo = cast(Any, service.repository)
|
||||
async with bm_db.scoped_session(repo.session_maker) as session:
|
||||
await repo._prepare_vector_session(session)
|
||||
raw_rows = await repo._run_vector_query(session, query_embedding, candidate_limit=20)
|
||||
|
||||
print(f"\nQuery: '{query_text}'")
|
||||
print(f" {'chunk_key':<40} {'distance':>10} {'sim_old':>12} {'sim_new':>12}")
|
||||
@@ -347,7 +348,7 @@ async def test_similarity_formula_analysis(sqlite_engine_factory, tmp_path):
|
||||
dist = float(row["best_distance"])
|
||||
sim_old = 1.0 / (1.0 + max(dist, 0.0))
|
||||
# New formula: L2 distance → cosine similarity for normalized embeddings
|
||||
sim_new = service.repository._distance_to_similarity(dist)
|
||||
sim_new = repo._distance_to_similarity(dist)
|
||||
print(f" {row['chunk_key']:<40} {dist:>10.4f} {sim_old:>12.4f} {sim_new:>12.4f}")
|
||||
|
||||
|
||||
@@ -431,7 +432,7 @@ async def test_chunking_produces_reasonable_chunks(sqlite_engine_factory, tmp_pa
|
||||
service = await create_search_service(
|
||||
sqlite_engine_factory, DIAG_COMBO, tmp_path, embedding_provider=provider
|
||||
)
|
||||
repo = service.repository
|
||||
repo = cast(Any, service.repository)
|
||||
|
||||
# Simulate a typical entity with observations
|
||||
text_input = (
|
||||
|
||||
@@ -13,6 +13,8 @@ Uses postgres-fastembed combo (no OpenAI dependency) with the pgvector container
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, cast
|
||||
|
||||
import pytest
|
||||
|
||||
from basic_memory.config import DatabaseBackend
|
||||
@@ -194,7 +196,7 @@ async def test_postgres_vector_dimension_detection(postgres_engine_factory, tmp_
|
||||
postgres_engine_factory, PG_FASTEMBED, tmp_path, embedding_provider=provider
|
||||
)
|
||||
|
||||
repo = search_service.repository
|
||||
repo = cast(Any, search_service.repository)
|
||||
|
||||
# First entity triggers _ensure_vector_tables
|
||||
entity = await search_service.entity_repository.create(
|
||||
|
||||
@@ -8,6 +8,11 @@ import pytest
|
||||
from sqlalchemy import text
|
||||
|
||||
|
||||
def _first_value(row):
|
||||
assert row is not None
|
||||
return row[0]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_wal_mode_enabled(engine_factory, db_backend):
|
||||
"""Test that WAL mode is enabled on filesystem database connections."""
|
||||
@@ -19,7 +24,7 @@ async def test_wal_mode_enabled(engine_factory, db_backend):
|
||||
# Execute a query to verify WAL mode is enabled
|
||||
async with engine.connect() as conn:
|
||||
result = await conn.execute(text("PRAGMA journal_mode"))
|
||||
journal_mode = result.fetchone()[0]
|
||||
journal_mode = _first_value(result.fetchone())
|
||||
|
||||
# WAL mode should be enabled for filesystem databases
|
||||
assert journal_mode.upper() == "WAL"
|
||||
@@ -35,7 +40,7 @@ async def test_busy_timeout_configured(engine_factory, db_backend):
|
||||
|
||||
async with engine.connect() as conn:
|
||||
result = await conn.execute(text("PRAGMA busy_timeout"))
|
||||
busy_timeout = result.fetchone()[0]
|
||||
busy_timeout = _first_value(result.fetchone())
|
||||
|
||||
# Busy timeout should be 10 seconds (10000 milliseconds)
|
||||
assert busy_timeout == 10000
|
||||
@@ -51,7 +56,7 @@ async def test_synchronous_mode_configured(engine_factory, db_backend):
|
||||
|
||||
async with engine.connect() as conn:
|
||||
result = await conn.execute(text("PRAGMA synchronous"))
|
||||
synchronous = result.fetchone()[0]
|
||||
synchronous = _first_value(result.fetchone())
|
||||
|
||||
# Synchronous should be NORMAL (1)
|
||||
assert synchronous == 1
|
||||
@@ -67,7 +72,7 @@ async def test_cache_size_configured(engine_factory, db_backend):
|
||||
|
||||
async with engine.connect() as conn:
|
||||
result = await conn.execute(text("PRAGMA cache_size"))
|
||||
cache_size = result.fetchone()[0]
|
||||
cache_size = _first_value(result.fetchone())
|
||||
|
||||
# Cache size should be -64000 (64MB)
|
||||
assert cache_size == -64000
|
||||
@@ -83,7 +88,7 @@ async def test_temp_store_configured(engine_factory, db_backend):
|
||||
|
||||
async with engine.connect() as conn:
|
||||
result = await conn.execute(text("PRAGMA temp_store"))
|
||||
temp_store = result.fetchone()[0]
|
||||
temp_store = _first_value(result.fetchone())
|
||||
|
||||
# temp_store should be MEMORY (2)
|
||||
assert temp_store == 2
|
||||
@@ -114,7 +119,7 @@ async def test_windows_locking_mode_when_on_windows(tmp_path, monkeypatch, confi
|
||||
):
|
||||
async with engine.connect() as conn:
|
||||
result = await conn.execute(text("PRAGMA locking_mode"))
|
||||
locking_mode = result.fetchone()[0]
|
||||
locking_mode = _first_value(result.fetchone())
|
||||
|
||||
# Locking mode should be NORMAL on Windows
|
||||
assert locking_mode.upper() == "NORMAL"
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
"""Fixtures for V2 API tests."""
|
||||
|
||||
from collections.abc import Generator
|
||||
from typing import Any, AsyncGenerator
|
||||
|
||||
import pytest
|
||||
@@ -13,13 +14,21 @@ from basic_memory.models import Project
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def app(test_config, engine_factory, app_config) -> FastAPI:
|
||||
async def app(test_config, engine_factory, app_config) -> AsyncGenerator[FastAPI, None]:
|
||||
"""Create FastAPI test application."""
|
||||
from basic_memory.api.app import app
|
||||
|
||||
previous_overrides = dict(app.dependency_overrides)
|
||||
app.dependency_overrides[get_app_config] = lambda: app_config
|
||||
app.dependency_overrides[get_engine_factory] = lambda: engine_factory
|
||||
return app
|
||||
try:
|
||||
yield app
|
||||
finally:
|
||||
# Trigger: the FastAPI app is a module-level singleton shared across tests.
|
||||
# Why: dependency overrides that capture a per-test engine can leak into
|
||||
# later CLI/MCP tests and create connections outside fixture ownership.
|
||||
# Outcome: each API test leaves the shared app exactly as it found it.
|
||||
app.dependency_overrides = previous_overrides
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
@@ -30,7 +39,7 @@ async def client(app: FastAPI) -> AsyncGenerator[AsyncClient, None]:
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def task_scheduler_spy(app: FastAPI) -> list[dict[str, Any]]:
|
||||
def task_scheduler_spy(app: FastAPI) -> Generator[list[dict[str, Any]], None, None]:
|
||||
"""Capture scheduled task specs without executing them."""
|
||||
scheduled: list[dict[str, Any]] = []
|
||||
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user