mirror of
https://github.com/basicmachines-co/basic-memory
synced 2026-06-21 13:47:35 +00:00
Compare commits
265 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 5ae8a733ea | |||
| b94ef01f82 | |||
| 12af7930de | |||
| 60ec6728de | |||
| a84e77f4af | |||
| 6910620968 | |||
| 214a54d740 | |||
| 5a90c7cedf | |||
| 9e3fe26a83 | |||
| 47ee982041 | |||
| 4d22c398c6 | |||
| 8ac2d975f9 | |||
| 34830bfad7 | |||
| f5e0c42047 | |||
| 3f98da8c67 | |||
| 14ff77d1c2 | |||
| 4e4da6128d | |||
| 3bed6d8890 | |||
| 4cba7ba01c | |||
| 8eeec64e28 | |||
| c6fa185bf3 | |||
| 415c2b3d6e | |||
| 4aa0cbdd62 | |||
| 55f314237d | |||
| 9862ef5411 | |||
| df5e8d805f | |||
| 831dc1ecdc | |||
| 26381aeed1 | |||
| 7918e5c6bf | |||
| f312341020 | |||
| e871298e95 | |||
| 177ae21ba7 | |||
| 3415fd1014 | |||
| b08659e228 | |||
| 09a4b09436 | |||
| a661e924df | |||
| 05adda1502 | |||
| 0a72d81bb3 | |||
| 5ccf433cad | |||
| b4bf14ebf7 | |||
| 0b335476d6 | |||
| 2fccc74a20 | |||
| c4956cac16 | |||
| 128c2da40c | |||
| 2bfb9c76df | |||
| 3d927b848f | |||
| 953fe20aef | |||
| a4282d9f2f | |||
| 799dd6c629 | |||
| 26e74ea118 | |||
| ee1558ea68 | |||
| 4d62b623db | |||
| 2fe4488eda | |||
| f3e46d7984 | |||
| 56d6f1b4a5 | |||
| 1b39062ecd | |||
| c4cf0aff1e | |||
| 1c343bed66 | |||
| c50d97e548 | |||
| e2e65575d6 | |||
| bf9a6b4a75 | |||
| 474100efef | |||
| b3d5448355 | |||
| 4e53bb83fd | |||
| 8f2b25f0e0 | |||
| 052545b661 | |||
| abd4a5a6da | |||
| a872947e03 | |||
| 093c94fea5 | |||
| cc104f761f | |||
| 7945c1e2f7 | |||
| d7f3f6a96f | |||
| 540da418b3 | |||
| 3e40cb9657 | |||
| 8c81d3ce17 | |||
| b35d594ef0 | |||
| e982900084 | |||
| b3403e96b3 | |||
| fe04a0b2a2 | |||
| 86ad639890 | |||
| 88c8f18200 | |||
| 41a16b93cb | |||
| 367fcaac50 | |||
| 69808b23ca | |||
| 6f207c20c0 | |||
| cff31c5797 | |||
| 2d1ccfa36c | |||
| a2e0f935d6 | |||
| e6b98a15c7 | |||
| 733c4f7514 | |||
| cfa70004be | |||
| 7696fca826 | |||
| 98a2a3cbaf | |||
| 01cbad1dbe | |||
| a4e0422926 | |||
| 552a835669 | |||
| 888e3c2909 | |||
| d1320f671e | |||
| 94bdfe77e4 | |||
| 4791e19685 | |||
| 36848410a1 | |||
| a77b51a28e | |||
| 1a6a65571e | |||
| c8b00449d2 | |||
| 013864ebf0 | |||
| dd91b49054 | |||
| 7c96a0777d | |||
| 148e07c580 | |||
| 21334cc29b | |||
| db60942267 | |||
| 7bfac158df | |||
| 87616924ff | |||
| 5cb0502ed2 | |||
| a94a717b1b | |||
| 6e4bb72f10 | |||
| 11b0e31e24 | |||
| a5c9e77f16 | |||
| 30a89357cb | |||
| 222ec5d3b6 | |||
| d42aec7ea9 | |||
| 9809b469c6 | |||
| 76ac880f2d | |||
| ad3f2650d9 | |||
| d6508d985c | |||
| 7b95b9f37b | |||
| 0bce4be1a6 | |||
| a316424edf | |||
| af71cf4896 | |||
| e846ae85d8 | |||
| 63e4bcdf1d | |||
| f23dd0474b | |||
| fced804438 | |||
| 1fdc9fdc69 | |||
| 2feecdfaf7 | |||
| 195229f78e | |||
| d15f6a8427 | |||
| b8a3a14ad2 | |||
| 9b199c6dcb | |||
| fe4a7b1622 | |||
| 7d2012a82c | |||
| 7f2d4d2a6f | |||
| 8b7f39ee9b | |||
| a368d06fd2 | |||
| 3a2b80b7e9 | |||
| bd5099120f | |||
| 8e64caf784 | |||
| ccb5740920 | |||
| 373893a7ee | |||
| 1987581ee3 | |||
| 59134affa4 | |||
| c7d97decd6 | |||
| 5d5efa02a5 | |||
| 4f12182e28 | |||
| 2c5b63c4a2 | |||
| c4c9f842ea | |||
| dc0ca71c18 | |||
| 236ae268aa | |||
| bd5923a370 | |||
| 4ea5396ddd | |||
| e97eafa55a | |||
| 254e30423d | |||
| 496af07ced | |||
| b5667f9b55 | |||
| 54b968b93c | |||
| e59b5cb6d9 | |||
| f0335b998e | |||
| 66effb04a7 | |||
| 9331126ba1 | |||
| 3bbb44af0b | |||
| f9b2a075a9 | |||
| 0a3f3f07f8 | |||
| 184ea6d9fa | |||
| f5a0e942b0 | |||
| e46555bf2c | |||
| 74e6afdc0f | |||
| aa635b8a8b | |||
| 1856d4b462 | |||
| 73413486bc | |||
| b09eca1698 | |||
| 3004d0d1fe | |||
| b6369d3d14 | |||
| db6d0dcd9e | |||
| 5a5eb443ea | |||
| e1df23d793 | |||
| 0462a7d4ba | |||
| 79db876bd6 | |||
| 0130573342 | |||
| c372dfb09f | |||
| d763d86798 | |||
| 538af97cba | |||
| 054178d155 | |||
| 18f00f861d | |||
| da4d369c32 | |||
| 0f3889fdd0 | |||
| c44291830c | |||
| e1cccba72d | |||
| 6ff39076a0 | |||
| 56cefbaafd | |||
| 3337c7d1ff | |||
| e248763a73 | |||
| 2e5813d31e | |||
| 2cde8d2659 | |||
| 9515130b2a | |||
| b86dd6fb53 | |||
| 8451f2b1d7 | |||
| ee0397513d | |||
| 9c9ff2931d | |||
| bbe6c1e8f3 | |||
| edb7991ccf | |||
| d7faeb754e | |||
| deef89a724 | |||
| 30499a9f61 | |||
| 1ac65b944c | |||
| 306e562281 | |||
| ed82b0c417 | |||
| 0cb3f95d67 | |||
| 0a36256f8a | |||
| 36e67e6eec | |||
| f2683291e4 | |||
| 8c05a9ec80 | |||
| a6d8d4c0f6 | |||
| 0239f4abb4 | |||
| 9259a7eb59 | |||
| 55d675e278 | |||
| 6afe4fd0cc | |||
| 113d1b6f1b | |||
| 545804f194 | |||
| 8bc03d1357 | |||
| f6e0a5b5bb | |||
| 7624a20d8d | |||
| d84708ca7f | |||
| 1428d18de1 | |||
| 312662f382 | |||
| ed9487708e | |||
| 8df88e4d02 | |||
| 07778790d3 | |||
| b609c4e531 | |||
| f1a065bce3 | |||
| 2b94d9a278 | |||
| 344e651693 | |||
| c97733d785 | |||
| 00537272c6 | |||
| b057912452 | |||
| 8489a3d37e | |||
| a47c9c021f | |||
| c46d7a6833 | |||
| 343a6e118b | |||
| a0e754b7ae | |||
| 24ca5f6804 | |||
| f1d50c2ba7 | |||
| 8072449a78 | |||
| 45d3f58e4d | |||
| d9c8923148 | |||
| 15bd6b95ef | |||
| 0715dcff3d | |||
| 009e84926d | |||
| 8838571509 | |||
| 530cbac73f | |||
| e3ced49d9d | |||
| 8f962fdd87 | |||
| fbb497f6dc | |||
| 0023e736ab | |||
| 0b2080114b | |||
| 8730067f3a | |||
| e14ba92631 |
@@ -0,0 +1,244 @@
|
||||
---
|
||||
name: instrumentation
|
||||
description: Add Pydantic Logfire observability to applications. Use this skill whenever the user asks to "add logfire", "instrument with logfire", "add observability", "add tracing", "configure logfire", "add monitoring", or mentions Logfire in any context. Supports Python, JavaScript/TypeScript, and Rust. Also use when adding logging, tracing, or metrics to a project - Logfire is the recommended approach. Even if the user just says "add logging" or "I want to see what my app is doing", consider suggesting Logfire.
|
||||
---
|
||||
|
||||
# Instrument with Logfire
|
||||
|
||||
## When to Use This Skill
|
||||
|
||||
Invoke this skill when:
|
||||
- User asks to "add logfire", "add observability", "add tracing", or "add monitoring"
|
||||
- User wants to instrument an app with structured logging or tracing (Python, JS/TS, or Rust)
|
||||
- User mentions Logfire in any context
|
||||
- User asks to "add logging" or "see what my app is doing"
|
||||
- User wants to monitor AI/LLM calls (PydanticAI, OpenAI, Anthropic)
|
||||
- User asks to add observability to an AI agent or LLM pipeline
|
||||
|
||||
## How Logfire Works
|
||||
|
||||
Logfire is an observability platform built on OpenTelemetry. It captures traces, logs, and metrics from applications. Logfire has native SDKs for Python, JavaScript/TypeScript, and Rust, plus support for any language via OpenTelemetry.
|
||||
|
||||
The reason this skill exists is that Claude tends to get a few things subtly wrong with Logfire - especially the ordering of `configure()` vs `instrument_*()` calls, the structured logging syntax, and which extras to install. These matter because a misconfigured setup silently drops traces.
|
||||
|
||||
## Step 1: Detect Language and Frameworks
|
||||
|
||||
Identify the project language and instrumentable libraries:
|
||||
|
||||
- **Python**: Read `pyproject.toml` or `requirements.txt`. Common instrumentable libraries: FastAPI, httpx, asyncpg, SQLAlchemy, psycopg, Redis, Celery, Django, Flask, requests, PydanticAI.
|
||||
- **JavaScript/TypeScript**: Read `package.json`. Common frameworks: Express, Next.js, Fastify. Also check for Cloudflare Workers or Deno.
|
||||
- **Rust**: Read `Cargo.toml`.
|
||||
|
||||
Then follow the language-specific steps below.
|
||||
|
||||
---
|
||||
|
||||
## Python
|
||||
|
||||
### Install with Extras
|
||||
|
||||
Install `logfire` with extras matching the detected frameworks. Each instrumented library needs its corresponding extra - without it, the `instrument_*()` call will fail at runtime with a missing dependency error.
|
||||
|
||||
```bash
|
||||
uv add 'logfire[fastapi,httpx,asyncpg]'
|
||||
```
|
||||
|
||||
The full list of available extras: `fastapi`, `starlette`, `django`, `flask`, `httpx`, `requests`, `asyncpg`, `psycopg`, `psycopg2`, `sqlalchemy`, `redis`, `pymongo`, `mysql`, `sqlite3`, `celery`, `aiohttp`, `aws-lambda`, `system-metrics`, `litellm`, `dspy`, `google-genai`.
|
||||
|
||||
### Configure and Instrument
|
||||
|
||||
This is where ordering matters. `logfire.configure()` initializes the SDK and must come before everything else. The `instrument_*()` calls register hooks into each library. If you call `instrument_*()` before `configure()`, the hooks register but traces go nowhere.
|
||||
|
||||
```python
|
||||
import logfire
|
||||
|
||||
# 1. Configure first - always
|
||||
logfire.configure()
|
||||
|
||||
# 2. Instrument libraries - after configure, before app starts
|
||||
logfire.instrument_fastapi(app)
|
||||
logfire.instrument_httpx()
|
||||
logfire.instrument_asyncpg()
|
||||
```
|
||||
|
||||
Placement rules:
|
||||
- `logfire.configure()` goes in the application entry point (`main.py`, or the module that creates the app)
|
||||
- Call it **once per process** - not inside request handlers, not in library code
|
||||
- `instrument_*()` calls go right after `configure()`
|
||||
- Web framework instrumentors (`instrument_fastapi`, `instrument_flask`, `instrument_django`) need the app instance as an argument. HTTP client and database instrumentors (`instrument_httpx`, `instrument_asyncpg`) are global and take no arguments.
|
||||
- In **Gunicorn** deployments, call `logfire.configure()` inside the `post_fork` hook, not at module level - each worker is a separate process
|
||||
|
||||
### Structured Logging
|
||||
|
||||
Replace `print()` and `logging.*()` calls with Logfire's structured logging. The key pattern: use `{key}` placeholders with keyword arguments, never f-strings.
|
||||
|
||||
```python
|
||||
# Correct - each {key} becomes a searchable attribute in the Logfire UI
|
||||
logfire.info("Created user {user_id}", user_id=uid)
|
||||
logfire.error("Payment failed {amount} {currency}", amount=100, currency="USD")
|
||||
|
||||
# Wrong - creates a flat string, nothing is searchable
|
||||
logfire.info(f"Created user {uid}")
|
||||
```
|
||||
|
||||
For grouping related operations and measuring duration, use spans:
|
||||
|
||||
```python
|
||||
with logfire.span("Processing order {order_id}", order_id=order_id):
|
||||
items = await fetch_items(order_id)
|
||||
total = calculate_total(items)
|
||||
logfire.info("Calculated total {total}", total=total)
|
||||
```
|
||||
|
||||
For exceptions, use `logfire.exception()` which automatically captures the traceback:
|
||||
|
||||
```python
|
||||
try:
|
||||
await process_order(order_id)
|
||||
except Exception:
|
||||
logfire.exception("Failed to process order {order_id}", order_id=order_id)
|
||||
raise
|
||||
```
|
||||
|
||||
### AI/LLM Instrumentation (Python)
|
||||
|
||||
Logfire auto-instruments AI libraries to capture LLM calls, token usage, tool invocations, and agent runs.
|
||||
|
||||
```bash
|
||||
uv add 'logfire[pydantic-ai]'
|
||||
# or: uv add 'logfire[openai]' / uv add 'logfire[anthropic]'
|
||||
```
|
||||
|
||||
Available AI extras: `pydantic-ai`, `openai`, `anthropic`, `litellm`, `dspy`, `google-genai`.
|
||||
|
||||
```python
|
||||
logfire.configure()
|
||||
logfire.instrument_pydantic_ai() # captures agent runs, tool calls, LLM request/response
|
||||
# or:
|
||||
logfire.instrument_openai() # captures chat completions, embeddings, token counts
|
||||
logfire.instrument_anthropic() # captures messages, token usage
|
||||
```
|
||||
|
||||
For PydanticAI, each agent run becomes a parent span containing child spans for every tool call and LLM request.
|
||||
|
||||
---
|
||||
|
||||
## JavaScript / TypeScript
|
||||
|
||||
### Install
|
||||
|
||||
```bash
|
||||
# Node.js
|
||||
npm install @pydantic/logfire-node
|
||||
|
||||
# Cloudflare Workers
|
||||
npm install @pydantic/logfire-cf-workers logfire
|
||||
|
||||
# Next.js / generic
|
||||
npm install logfire
|
||||
```
|
||||
|
||||
### Configure
|
||||
|
||||
**Node.js (Express, Fastify, etc.)** - create an `instrumentation.ts` loaded before your app:
|
||||
|
||||
```typescript
|
||||
import * as logfire from '@pydantic/logfire-node'
|
||||
logfire.configure()
|
||||
```
|
||||
|
||||
Launch with: `node --require ./instrumentation.js app.js`
|
||||
|
||||
The SDK auto-instruments common libraries when loaded before the app. Set `LOGFIRE_TOKEN` in your environment or pass `token` to `configure()`.
|
||||
|
||||
**Cloudflare Workers** - wrap your handler with `instrument()`:
|
||||
|
||||
```typescript
|
||||
import { instrument } from '@pydantic/logfire-cf-workers'
|
||||
|
||||
export default instrument(handler, {
|
||||
service: { name: 'my-worker', version: '1.0.0' }
|
||||
})
|
||||
```
|
||||
|
||||
**Next.js** - set environment variables for OpenTelemetry export:
|
||||
|
||||
```
|
||||
OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=https://logfire-api.pydantic.dev/v1/traces
|
||||
OTEL_EXPORTER_OTLP_HEADERS=Authorization=<your-write-token>
|
||||
```
|
||||
|
||||
### Structured Logging (JS/TS)
|
||||
|
||||
```typescript
|
||||
// Structured attributes as second argument
|
||||
logfire.info('Created user', { user_id: uid })
|
||||
logfire.error('Payment failed', { amount: 100, currency: 'USD' })
|
||||
|
||||
// Spans
|
||||
logfire.span('Processing order', { order_id }, {}, async () => {
|
||||
logfire.info('Processing step completed')
|
||||
})
|
||||
|
||||
// Error reporting
|
||||
logfire.reportError('order processing', error)
|
||||
```
|
||||
|
||||
Log levels: `trace`, `debug`, `info`, `notice`, `warn`, `error`, `fatal`.
|
||||
|
||||
---
|
||||
|
||||
## Rust
|
||||
|
||||
### Install
|
||||
|
||||
```toml
|
||||
[dependencies]
|
||||
logfire = "0.6"
|
||||
```
|
||||
|
||||
### Configure
|
||||
|
||||
```rust
|
||||
let shutdown_handler = logfire::configure()
|
||||
.install_panic_handler()
|
||||
.finish()?;
|
||||
```
|
||||
|
||||
Set `LOGFIRE_TOKEN` in your environment or use the Logfire CLI to select a project.
|
||||
|
||||
### Structured Logging (Rust)
|
||||
|
||||
The Rust SDK is built on `tracing` and `opentelemetry` - existing `tracing` macros work automatically.
|
||||
|
||||
```rust
|
||||
// Spans
|
||||
logfire::span!("processing order", order_id = order_id).in_scope(|| {
|
||||
// traced code
|
||||
});
|
||||
|
||||
// Events
|
||||
logfire::info!("Created user {user_id}", user_id = uid);
|
||||
```
|
||||
|
||||
Always call `shutdown_handler.shutdown()` before program exit to flush data.
|
||||
|
||||
---
|
||||
|
||||
## Verify
|
||||
|
||||
After instrumentation, verify the setup works:
|
||||
|
||||
1. Run `logfire auth` to check authentication (or set `LOGFIRE_TOKEN`)
|
||||
2. Start the app and trigger a request
|
||||
3. Check https://logfire.pydantic.dev/ for traces
|
||||
|
||||
If traces aren't appearing: check that `configure()` is called before `instrument_*()` (Python), check that `LOGFIRE_TOKEN` is set, and check that the correct packages/extras are installed.
|
||||
|
||||
## References
|
||||
|
||||
Detailed patterns and integration tables, organized by language:
|
||||
|
||||
- **Python**: `${CLAUDE_PLUGIN_ROOT}/skills/instrumentation/references/python/logging-patterns.md` (log levels, spans, stdlib integration, metrics, capfire testing) and `${CLAUDE_PLUGIN_ROOT}/skills/instrumentation/references/python/integrations.md` (full instrumentor table with extras)
|
||||
- **JavaScript/TypeScript**: `${CLAUDE_PLUGIN_ROOT}/skills/instrumentation/references/javascript/patterns.md` (log levels, spans, error handling, config) and `${CLAUDE_PLUGIN_ROOT}/skills/instrumentation/references/javascript/frameworks.md` (Node.js, Cloudflare Workers, Next.js, Deno setup)
|
||||
- **Rust**: `${CLAUDE_PLUGIN_ROOT}/skills/instrumentation/references/rust/patterns.md` (macros, spans, tracing/log crate integration, async, shutdown)
|
||||
@@ -0,0 +1,78 @@
|
||||
# JavaScript Framework Setup
|
||||
|
||||
## Node.js (Express, Fastify, etc.)
|
||||
|
||||
Create `instrumentation.ts` and load it before your app:
|
||||
|
||||
```typescript
|
||||
// instrumentation.ts
|
||||
import * as logfire from '@pydantic/logfire-node'
|
||||
import 'dotenv/config'
|
||||
|
||||
logfire.configure()
|
||||
```
|
||||
|
||||
Launch:
|
||||
|
||||
```bash
|
||||
node --require ./instrumentation.js app.js
|
||||
# or with ts-node:
|
||||
npx ts-node --require ./instrumentation.ts app.ts
|
||||
```
|
||||
|
||||
The SDK auto-instruments common libraries (http, fetch, express, etc.) when loaded before the app via `--require`.
|
||||
|
||||
## Cloudflare Workers
|
||||
|
||||
```typescript
|
||||
import { instrument } from '@pydantic/logfire-cf-workers'
|
||||
|
||||
const handler = {
|
||||
async fetch(request: Request, env: Env, ctx: ExecutionContext) {
|
||||
return new Response('Hello')
|
||||
},
|
||||
}
|
||||
|
||||
export default instrument(handler, {
|
||||
service: { name: 'my-worker', version: '1.0.0' },
|
||||
})
|
||||
```
|
||||
|
||||
Add `LOGFIRE_TOKEN` to `.dev.vars` and enable `nodejs_compat` in `wrangler.toml`:
|
||||
|
||||
```toml
|
||||
compatibility_flags = ["nodejs_compat"]
|
||||
```
|
||||
|
||||
## Next.js / Vercel
|
||||
|
||||
Set environment variables in `.env.local` or Vercel dashboard:
|
||||
|
||||
```bash
|
||||
OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=https://logfire-api.pydantic.dev/v1/traces
|
||||
OTEL_EXPORTER_OTLP_METRICS_ENDPOINT=https://logfire-api.pydantic.dev/v1/metrics
|
||||
OTEL_EXPORTER_OTLP_HEADERS=Authorization=<your-write-token>
|
||||
```
|
||||
|
||||
Optionally use the `logfire` package for manual spans in server components and API routes:
|
||||
|
||||
```typescript
|
||||
import * as logfire from 'logfire'
|
||||
|
||||
logfire.info('Server action executed', { action: 'createUser' })
|
||||
```
|
||||
|
||||
## Deno
|
||||
|
||||
Deno has built-in OpenTelemetry support. Set environment variables:
|
||||
|
||||
```bash
|
||||
OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=https://logfire-api.pydantic.dev/v1/traces
|
||||
OTEL_EXPORTER_OTLP_HEADERS=Authorization=<your-write-token>
|
||||
```
|
||||
|
||||
Run with telemetry enabled:
|
||||
|
||||
```bash
|
||||
deno run --allow-env --unstable-otel app.ts
|
||||
```
|
||||
@@ -0,0 +1,75 @@
|
||||
# JavaScript / TypeScript Patterns
|
||||
|
||||
## Log Levels
|
||||
|
||||
From lowest to highest severity:
|
||||
|
||||
```typescript
|
||||
logfire.trace('Detailed trace', { detail: x })
|
||||
logfire.debug('Debug info', { state: s })
|
||||
logfire.info('Normal operation', { event: e })
|
||||
logfire.notice('Notable event', { event: e })
|
||||
logfire.warn('Warning', { issue: i })
|
||||
logfire.error('Error occurred', { error: err })
|
||||
logfire.fatal('Fatal error', { error: err })
|
||||
```
|
||||
|
||||
All methods accept `(message, attributes?, options?)`. Options can include `{ tags: ['tag1'] }`.
|
||||
|
||||
## Spans
|
||||
|
||||
### Callback-based (auto-closes)
|
||||
|
||||
```typescript
|
||||
await logfire.span('Processing order', { order_id }, {}, async () => {
|
||||
const items = await fetchItems(order_id)
|
||||
logfire.info('Fetched items', { count: items.length })
|
||||
return processItems(items)
|
||||
})
|
||||
```
|
||||
|
||||
### Manual control
|
||||
|
||||
```typescript
|
||||
const span = logfire.startSpan('Long operation', { job_id })
|
||||
try {
|
||||
await doWork()
|
||||
} finally {
|
||||
span.end()
|
||||
}
|
||||
```
|
||||
|
||||
Child spans reference their parent via the `parentSpan` option.
|
||||
|
||||
## Error Handling
|
||||
|
||||
```typescript
|
||||
try {
|
||||
await processOrder(orderId)
|
||||
} catch (error) {
|
||||
logfire.reportError('order processing', error)
|
||||
throw error
|
||||
}
|
||||
```
|
||||
|
||||
`reportError` automatically extracts stack traces and error details into structured span attributes.
|
||||
|
||||
## Configuration
|
||||
|
||||
### Environment variables
|
||||
|
||||
```bash
|
||||
LOGFIRE_TOKEN=your-write-token
|
||||
LOGFIRE_SERVICE_NAME=my-service
|
||||
LOGFIRE_SERVICE_VERSION=1.0.0
|
||||
```
|
||||
|
||||
### Programmatic
|
||||
|
||||
```typescript
|
||||
logfire.configure({
|
||||
token: process.env.LOGFIRE_TOKEN,
|
||||
serviceName: 'my-service',
|
||||
serviceVersion: '1.0.0',
|
||||
})
|
||||
```
|
||||
@@ -0,0 +1,67 @@
|
||||
# Python Integration Reference
|
||||
|
||||
## Web Frameworks
|
||||
|
||||
| Framework | Instrumentor | Needs app instance | Extra |
|
||||
|-----------|-------------|-------------------|-------|
|
||||
| FastAPI | `logfire.instrument_fastapi(app)` | Yes | `fastapi` |
|
||||
| Django | `logfire.instrument_django(app)` | Yes | `django` |
|
||||
| Flask | `logfire.instrument_flask(app)` | Yes | `flask` |
|
||||
| Starlette | `logfire.instrument_starlette(app)` | Yes | `starlette` |
|
||||
| AIOHTTP | `logfire.instrument_aiohttp_client()` | No | `aiohttp` |
|
||||
|
||||
## HTTP Clients
|
||||
|
||||
| Library | Instrumentor | Extra |
|
||||
|---------|-------------|-------|
|
||||
| httpx | `logfire.instrument_httpx()` | `httpx` |
|
||||
| requests | `logfire.instrument_requests()` | `requests` |
|
||||
|
||||
## Databases
|
||||
|
||||
| Library | Instrumentor | Extra |
|
||||
|---------|-------------|-------|
|
||||
| asyncpg | `logfire.instrument_asyncpg()` | `asyncpg` |
|
||||
| psycopg | `logfire.instrument_psycopg()` | `psycopg` |
|
||||
| psycopg2 | `logfire.instrument_psycopg2()` | `psycopg2` |
|
||||
| SQLAlchemy | `logfire.instrument_sqlalchemy()` | `sqlalchemy` |
|
||||
| PyMongo | `logfire.instrument_pymongo()` | `pymongo` |
|
||||
| MySQL | `logfire.instrument_mysql()` | `mysql` |
|
||||
| SQLite3 | `logfire.instrument_sqlite3()` | `sqlite3` |
|
||||
| Redis | `logfire.instrument_redis()` | `redis` |
|
||||
|
||||
## AI/LLM Frameworks
|
||||
|
||||
| Framework | Instrumentor | Extra |
|
||||
|-----------|-------------|-------|
|
||||
| PydanticAI | `logfire.instrument_pydantic_ai()` | `pydantic-ai` |
|
||||
| OpenAI | `logfire.instrument_openai()` | `openai` |
|
||||
| Anthropic | `logfire.instrument_anthropic()` | `anthropic` |
|
||||
| LiteLLM | `logfire.instrument_litellm()` | `litellm` |
|
||||
| DSPy | `logfire.instrument_dspy()` | `dspy` |
|
||||
| Google GenAI | `logfire.instrument_google_genai()` | `google-genai` |
|
||||
|
||||
## Task Queues
|
||||
|
||||
| Framework | Instrumentor | Extra |
|
||||
|-----------|-------------|-------|
|
||||
| Celery | `logfire.instrument_celery()` | `celery` |
|
||||
|
||||
## Other
|
||||
|
||||
| Feature | Instrumentor | Extra |
|
||||
|---------|-------------|-------|
|
||||
| System Metrics | `logfire.instrument_system_metrics()` | `system-metrics` |
|
||||
| Pydantic Models | `logfire.instrument_pydantic()` | - (built-in) |
|
||||
| AWS Lambda | handler wrapper | `aws-lambda` |
|
||||
|
||||
## Gunicorn Configuration
|
||||
|
||||
```python
|
||||
# gunicorn.conf.py
|
||||
import logfire
|
||||
|
||||
def post_fork(server, worker):
|
||||
logfire.configure()
|
||||
logfire.instrument_fastapi(app)
|
||||
```
|
||||
@@ -0,0 +1,101 @@
|
||||
# Python Logging Patterns
|
||||
|
||||
## Log Levels
|
||||
|
||||
From lowest to highest severity:
|
||||
|
||||
```python
|
||||
logfire.trace("Detailed trace {detail}", detail=x)
|
||||
logfire.debug("Debug info {state}", state=s)
|
||||
logfire.info("Normal operation {event}", event=e)
|
||||
logfire.notice("Notable event {event}", event=e)
|
||||
logfire.warn("Warning {issue}", issue=i)
|
||||
logfire.error("Error occurred {error}", error=err)
|
||||
logfire.fatal("Fatal error {error}", error=err)
|
||||
```
|
||||
|
||||
## Nested Spans
|
||||
|
||||
Spans nest to create a tree visible in the Logfire UI. Use them to show the structure of an operation, not just that it happened:
|
||||
|
||||
```python
|
||||
with logfire.span("HTTP request {method} {url}", method="POST", url=url):
|
||||
with logfire.span("Serialize payload"):
|
||||
payload = model.model_dump_json()
|
||||
with logfire.span("Send request"):
|
||||
response = await client.post(url, content=payload)
|
||||
logfire.info("Response {status}", status=response.status_code)
|
||||
```
|
||||
|
||||
## Standard Library Logging Integration
|
||||
|
||||
For projects that already use Python's `logging` module, route existing log calls through Logfire rather than rewriting them all:
|
||||
|
||||
```python
|
||||
from logging import basicConfig
|
||||
import logfire
|
||||
|
||||
logfire.configure()
|
||||
basicConfig(handlers=[logfire.LogfireLoggingHandler()])
|
||||
```
|
||||
|
||||
Or with `dictConfig`:
|
||||
|
||||
```python
|
||||
from logging.config import dictConfig
|
||||
import logfire
|
||||
|
||||
logfire.configure()
|
||||
dictConfig({
|
||||
'version': 1,
|
||||
'handlers': {
|
||||
'logfire': {'class': 'logfire.LogfireLoggingHandler'},
|
||||
},
|
||||
'root': {'handlers': ['logfire']},
|
||||
})
|
||||
```
|
||||
|
||||
## Suppressing Noisy Libraries
|
||||
|
||||
Some libraries emit excessive debug logs. Silence them at the `logging` level:
|
||||
|
||||
```python
|
||||
import logging
|
||||
|
||||
logging.getLogger('httpcore').setLevel(logging.WARNING)
|
||||
logging.getLogger('httpx').setLevel(logging.WARNING)
|
||||
```
|
||||
|
||||
## Custom Metrics
|
||||
|
||||
For dashboards and alerting, create metrics:
|
||||
|
||||
```python
|
||||
counter = logfire.metric_counter("orders_processed", unit="1")
|
||||
counter.add(1, {"status": "success"})
|
||||
|
||||
histogram = logfire.metric_histogram("request_duration", unit="s")
|
||||
histogram.record(0.123, {"endpoint": "/api/users"})
|
||||
|
||||
gauge = logfire.metric_gauge("active_connections")
|
||||
gauge.set(42)
|
||||
```
|
||||
|
||||
## Testing with capfire
|
||||
|
||||
Use the `capfire` pytest fixture to assert on emitted spans without sending data to production:
|
||||
|
||||
```python
|
||||
from logfire.testing import CaptureLogfire
|
||||
|
||||
def test_order_processing(capfire: CaptureLogfire) -> None:
|
||||
process_order(order_id=123)
|
||||
|
||||
spans = capfire.exporter.exported_spans_as_dict()
|
||||
assert any(
|
||||
span['attributes'].get('order_id') == 123
|
||||
for span in spans
|
||||
)
|
||||
```
|
||||
|
||||
Configure logfire with `send_to_logfire=False` in test fixtures to prevent production data leakage.
|
||||
@@ -0,0 +1,106 @@
|
||||
# Rust Patterns
|
||||
|
||||
## Core Macros
|
||||
|
||||
The Rust SDK is built on `tracing` and `opentelemetry`. All `tracing` macros work automatically with Logfire.
|
||||
|
||||
### Events (log points)
|
||||
|
||||
```rust
|
||||
logfire::trace!("Detailed trace {detail}", detail = x);
|
||||
logfire::debug!("Debug info {state}", state = s);
|
||||
logfire::info!("Normal operation {event}", event = e);
|
||||
logfire::warn!("Warning {issue}", issue = i);
|
||||
logfire::error!("Error occurred {err}", err = e);
|
||||
```
|
||||
|
||||
### Spans
|
||||
|
||||
```rust
|
||||
// Scoped - span closes when closure completes
|
||||
logfire::span!("Processing order {order_id}", order_id = id).in_scope(|| {
|
||||
let items = fetch_items(id);
|
||||
logfire::info!("Fetched {count} items", count = items.len());
|
||||
process_items(items)
|
||||
});
|
||||
|
||||
// Guard-based - span closes when guard is dropped
|
||||
let _guard = logfire::span!("Long operation {job_id}", job_id = id).entered();
|
||||
do_work();
|
||||
// span ends when _guard goes out of scope
|
||||
```
|
||||
|
||||
## Configuration
|
||||
|
||||
```rust
|
||||
use logfire;
|
||||
|
||||
fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
let shutdown_handler = logfire::configure()
|
||||
.install_panic_handler() // captures panics as error spans
|
||||
.finish()?;
|
||||
|
||||
// application code...
|
||||
|
||||
shutdown_handler.shutdown()?; // flush all pending spans
|
||||
Ok(())
|
||||
}
|
||||
```
|
||||
|
||||
Set `LOGFIRE_TOKEN` in your environment or use the Logfire CLI (`logfire auth`).
|
||||
|
||||
## Tracing Crate Compatibility
|
||||
|
||||
Any library using `tracing` macros automatically sends data through Logfire:
|
||||
|
||||
```rust
|
||||
use tracing;
|
||||
|
||||
tracing::info!("This also appears in Logfire");
|
||||
|
||||
#[tracing::instrument]
|
||||
fn my_function(param: &str) {
|
||||
// automatically creates a span with param as an attribute
|
||||
}
|
||||
```
|
||||
|
||||
## Log Crate Integration
|
||||
|
||||
The `log` crate is automatically captured and forwarded to Logfire. Libraries using `log::info!()`, `log::error!()`, etc. will appear in your Logfire dashboard without any additional configuration.
|
||||
|
||||
## Async Spans
|
||||
|
||||
```rust
|
||||
use tracing::Instrument;
|
||||
|
||||
async fn process_order(order_id: u64) {
|
||||
let span = logfire::span!("process order {order_id}", order_id = order_id);
|
||||
async {
|
||||
fetch_items(order_id).await;
|
||||
logfire::info!("Order processed");
|
||||
}
|
||||
.instrument(span)
|
||||
.await;
|
||||
}
|
||||
```
|
||||
|
||||
## Shutdown
|
||||
|
||||
Always call `shutdown()` before program exit to flush pending data:
|
||||
|
||||
```rust
|
||||
// In main()
|
||||
let shutdown_handler = logfire::configure().finish()?;
|
||||
|
||||
// ... app runs ...
|
||||
|
||||
// Before exit
|
||||
shutdown_handler.shutdown()?;
|
||||
```
|
||||
|
||||
For web servers using `tokio`, handle shutdown via signal:
|
||||
|
||||
```rust
|
||||
tokio::signal::ctrl_c().await?;
|
||||
shutdown_handler.shutdown()?;
|
||||
```
|
||||
+19
-3
@@ -1,5 +1,21 @@
|
||||
{
|
||||
"enabledPlugins": {
|
||||
"basic-memory@basicmachines": true
|
||||
}
|
||||
"$schema": "https://json.schemastore.org/claude-code-settings.json",
|
||||
"env": {
|
||||
"CLAUDE_BASH_MAINTAIN_PROJECT_WORKING_DIR": "1",
|
||||
"CLAUDE_CODE_DISABLE_FEEDBACK_SURVEY": "1",
|
||||
"CLAUDE_CODE_NO_FLICKER": "1",
|
||||
"CLAUDE_CODE_DISABLE_ADAPTIVE_THINKING": "1"
|
||||
},
|
||||
"permissions": {
|
||||
"allow": [
|
||||
"Bash(just fast-check)",
|
||||
"Bash(just check)",
|
||||
"Bash(just fix)",
|
||||
"Bash(just typecheck)",
|
||||
"Bash(just lint)",
|
||||
"Bash(just test)"
|
||||
],
|
||||
"deny": []
|
||||
},
|
||||
"enableAllProjectMcpServers": true
|
||||
}
|
||||
|
||||
Symlink
+1
@@ -0,0 +1 @@
|
||||
../../.agents/skills/instrumentation
|
||||
@@ -10,6 +10,9 @@ on:
|
||||
# - "src/**/*.js"
|
||||
# - "src/**/*.jsx"
|
||||
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
|
||||
jobs:
|
||||
claude-review:
|
||||
# Only run for organization members and collaborators
|
||||
@@ -27,7 +30,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
fetch-depth: 1
|
||||
|
||||
|
||||
@@ -4,6 +4,9 @@ on:
|
||||
issues:
|
||||
types: [opened]
|
||||
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
|
||||
jobs:
|
||||
triage:
|
||||
runs-on: ubuntu-latest
|
||||
@@ -12,7 +15,7 @@ jobs:
|
||||
id-token: write
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
fetch-depth: 1
|
||||
|
||||
@@ -68,4 +71,4 @@ jobs:
|
||||
|
||||
Read the issue carefully and provide helpful triage with appropriate labels.
|
||||
|
||||
claude_args: '--allowed-tools "Bash(gh issue:*),Bash(gh search:*),Read"'
|
||||
claude_args: '--allowed-tools "Bash(gh issue:*),Bash(gh search:*),Read"'
|
||||
|
||||
@@ -12,6 +12,9 @@ on:
|
||||
pull_request_target:
|
||||
types: [opened, synchronize]
|
||||
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
|
||||
jobs:
|
||||
claude:
|
||||
if: |
|
||||
@@ -41,7 +44,7 @@ jobs:
|
||||
actions: read # Required for Claude to read CI results on PRs
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
# For pull_request_target, checkout the PR head to review the actual changes
|
||||
ref: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.head.sha || github.sha }}
|
||||
@@ -65,4 +68,3 @@ jobs:
|
||||
# See https://github.com/anthropics/claude-code-action/blob/main/docs/usage.md
|
||||
# or https://docs.claude.com/en/docs/claude-code/sdk#command-line for available options
|
||||
# claude_args: '--model claude-opus-4-1-20250805 --allowed-tools Bash(gh pr:*)'
|
||||
|
||||
|
||||
@@ -5,6 +5,9 @@ on:
|
||||
branches: [main]
|
||||
workflow_dispatch: # Allow manual triggering
|
||||
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
|
||||
jobs:
|
||||
dev-release:
|
||||
runs-on: ubuntu-latest
|
||||
@@ -13,12 +16,12 @@ jobs:
|
||||
contents: write
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/checkout@v6
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: "3.12"
|
||||
|
||||
@@ -50,4 +53,4 @@ jobs:
|
||||
uses: pypa/gh-action-pypi-publish@release/v1
|
||||
with:
|
||||
password: ${{ secrets.PYPI_TOKEN }}
|
||||
skip-existing: true # Don't fail if version already exists
|
||||
skip-existing: true # Don't fail if version already exists
|
||||
|
||||
@@ -9,6 +9,7 @@ on:
|
||||
env:
|
||||
REGISTRY: ghcr.io
|
||||
IMAGE_NAME: basicmachines-co/basic-memory
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
|
||||
jobs:
|
||||
docker:
|
||||
@@ -19,17 +20,17 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
uses: docker/setup-buildx-action@v4
|
||||
with:
|
||||
platforms: linux/amd64,linux/arm64
|
||||
|
||||
- name: Log in to GitHub Container Registry
|
||||
uses: docker/login-action@v3
|
||||
uses: docker/login-action@v4
|
||||
with:
|
||||
registry: ${{ env.REGISTRY }}
|
||||
username: ${{ github.actor }}
|
||||
@@ -37,7 +38,7 @@ jobs:
|
||||
|
||||
- name: Extract metadata
|
||||
id: meta
|
||||
uses: docker/metadata-action@v5
|
||||
uses: docker/metadata-action@v6
|
||||
with:
|
||||
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
|
||||
tags: |
|
||||
@@ -48,7 +49,7 @@ jobs:
|
||||
type=raw,value=latest,enable={{is_default_branch}}
|
||||
|
||||
- name: Build and push Docker image
|
||||
uses: docker/build-push-action@v5
|
||||
uses: docker/build-push-action@v7
|
||||
with:
|
||||
context: .
|
||||
file: ./Dockerfile
|
||||
@@ -58,4 +59,3 @@ jobs:
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
|
||||
|
||||
@@ -7,11 +7,14 @@ on:
|
||||
- edited
|
||||
- synchronize
|
||||
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
|
||||
jobs:
|
||||
main:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: amannn/action-semantic-pull-request@v5
|
||||
- uses: amannn/action-semantic-pull-request@v6
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
with:
|
||||
@@ -38,4 +41,4 @@ jobs:
|
||||
deps
|
||||
installer
|
||||
# Allow breaking changes (needs "!" after type/scope)
|
||||
requireScopeForBreakingChange: true
|
||||
requireScopeForBreakingChange: true
|
||||
|
||||
@@ -5,6 +5,9 @@ on:
|
||||
tags:
|
||||
- 'v*' # Trigger on version tags like v1.0.0, v0.13.0, etc.
|
||||
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
|
||||
jobs:
|
||||
release:
|
||||
runs-on: ubuntu-latest
|
||||
@@ -13,12 +16,12 @@ jobs:
|
||||
contents: write
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/checkout@v6
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: "3.12"
|
||||
|
||||
@@ -39,7 +42,7 @@ jobs:
|
||||
echo "Build completed successfully"
|
||||
|
||||
- name: Create GitHub Release
|
||||
uses: softprops/action-gh-release@v2
|
||||
uses: softprops/action-gh-release@v3
|
||||
with:
|
||||
files: |
|
||||
dist/*.whl
|
||||
@@ -60,26 +63,63 @@ jobs:
|
||||
# Only run for stable releases (not dev, beta, or rc versions)
|
||||
if: ${{ !contains(github.ref_name, 'dev') && !contains(github.ref_name, 'b') && !contains(github.ref_name, 'rc') }}
|
||||
permissions:
|
||||
contents: write
|
||||
actions: read
|
||||
contents: read
|
||||
steps:
|
||||
# Inline bump replaces mislav/bump-homebrew-formula-action@v4.x.
|
||||
# The action does a HEAD request to api.github.com /repos/.../tarball/<ref>
|
||||
# with the bearer token and expects a 302 redirect. GitHub now returns
|
||||
# 303 on that endpoint when authenticated, which the action treats as a
|
||||
# fatal error. Re-implementing the bump as plain git+sed keeps the same
|
||||
# contract (update url + sha256, commit, push) with no third-party action.
|
||||
- name: Update Homebrew formula
|
||||
uses: mislav/bump-homebrew-formula-action@v3
|
||||
with:
|
||||
# Formula name in homebrew-basic-memory repo
|
||||
formula-name: basic-memory
|
||||
# The tap repository
|
||||
homebrew-tap: basicmachines-co/homebrew-basic-memory
|
||||
# Base branch of the tap repository
|
||||
base-branch: main
|
||||
# Download URL will be automatically constructed from the tag
|
||||
download-url: https://github.com/basicmachines-co/basic-memory/archive/refs/tags/${{ github.ref_name }}.tar.gz
|
||||
# Commit message for the formula update
|
||||
commit-message: |
|
||||
{{formulaName}} {{version}}
|
||||
|
||||
Created by https://github.com/basicmachines-co/basic-memory/actions/runs/${{ github.run_id }}
|
||||
env:
|
||||
# Personal Access Token with repo scope for homebrew-basic-memory repo
|
||||
COMMITTER_TOKEN: ${{ secrets.HOMEBREW_TOKEN }}
|
||||
HOMEBREW_TOKEN: ${{ secrets.HOMEBREW_TOKEN }}
|
||||
REF: ${{ github.ref_name }}
|
||||
REPO: ${{ github.repository }}
|
||||
RUN_URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
VERSION="${REF#v}"
|
||||
ARCHIVE_URL="https://github.com/${REPO}/archive/refs/tags/${REF}.tar.gz"
|
||||
|
||||
echo "::group::Compute tarball sha256"
|
||||
SHA256="$(curl --fail --silent --location "$ARCHIVE_URL" | sha256sum | awk '{print $1}')"
|
||||
test -n "$SHA256"
|
||||
echo "sha256: $SHA256"
|
||||
echo "::endgroup::"
|
||||
|
||||
echo "::group::Clone tap"
|
||||
git clone \
|
||||
--depth 1 \
|
||||
"https://x-access-token:${HOMEBREW_TOKEN}@github.com/basicmachines-co/homebrew-basic-memory.git" \
|
||||
tap
|
||||
cd tap
|
||||
git config user.name "github-actions[bot]"
|
||||
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
|
||||
echo "::endgroup::"
|
||||
|
||||
echo "::group::Patch Formula/basic-memory.rb"
|
||||
# Pipe-delimited sed because the URL contains slashes. The Formula
|
||||
# only has one `url` and one `sha256` directive, so a first-match
|
||||
# replacement is unambiguous. POSIX character classes ([[:space:]])
|
||||
# keep this portable across BSD and GNU sed.
|
||||
sed -i -E \
|
||||
-e "s|^([[:space:]]*url[[:space:]]+)\"[^\"]+\"|\1\"${ARCHIVE_URL}\"|" \
|
||||
-e "s|^([[:space:]]*sha256[[:space:]]+)\"[^\"]+\"|\1\"${SHA256}\"|" \
|
||||
Formula/basic-memory.rb
|
||||
git --no-pager diff Formula/basic-memory.rb
|
||||
echo "::endgroup::"
|
||||
|
||||
if git diff --quiet Formula/basic-memory.rb; then
|
||||
echo "Formula already at ${REF}; nothing to do."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "::group::Commit & push"
|
||||
git add Formula/basic-memory.rb
|
||||
git commit -m "basic-memory ${VERSION}
|
||||
|
||||
Created by ${RUN_URL}"
|
||||
git push origin HEAD:main
|
||||
echo "::endgroup::"
|
||||
|
||||
+210
-83
@@ -1,53 +1,41 @@
|
||||
name: Tests
|
||||
|
||||
concurrency:
|
||||
group: bm-ci-${{ github.workflow }}-${{ github.repository }}-${{ github.head_ref || github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
on:
|
||||
# Trigger: PR branch pushes already publish commit statuses that show up on the PR.
|
||||
# Why: running the full matrix on both push and pull_request doubles CI time for the
|
||||
# exact same branch head commit.
|
||||
# Outcome: each branch push runs the test suite once, including PR updates.
|
||||
push:
|
||||
branches: [ "main" ]
|
||||
pull_request:
|
||||
branches: [ "main" ]
|
||||
# pull_request_target runs on the BASE of the PR, not the merge result.
|
||||
# It has write permissions and access to secrets.
|
||||
# It's useful for PRs from forks or automated PRs but requires careful use for security reasons.
|
||||
# See: https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request_target
|
||||
pull_request_target:
|
||||
branches: [ "main" ]
|
||||
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
|
||||
jobs:
|
||||
test-sqlite:
|
||||
name: Test SQLite (${{ matrix.os }}, Python ${{ matrix.python-version }})
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
os: [ubuntu-latest, windows-latest]
|
||||
python-version: [ "3.12", "3.13", "3.14" ]
|
||||
runs-on: ${{ matrix.os }}
|
||||
static-checks:
|
||||
name: Static Checks (Python 3.12)
|
||||
timeout-minutes: 20
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/checkout@v6
|
||||
with:
|
||||
submodules: true
|
||||
|
||||
- name: Set up Python ${{ matrix.python-version }}
|
||||
uses: actions/setup-python@v4
|
||||
- name: Set up Python 3.12
|
||||
uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
cache: 'pip'
|
||||
python-version: "3.12"
|
||||
cache: "pip"
|
||||
|
||||
- name: Install uv
|
||||
run: |
|
||||
pip install uv
|
||||
|
||||
- name: Install just (Linux/macOS)
|
||||
if: runner.os != 'Windows'
|
||||
run: |
|
||||
curl --proto '=https' --tlsv1.2 -sSf https://just.systems/install.sh | bash -s -- --to /usr/local/bin
|
||||
|
||||
- name: Install just (Windows)
|
||||
if: runner.os == 'Windows'
|
||||
run: |
|
||||
# Install just using Chocolatey (pre-installed on GitHub Actions Windows runners)
|
||||
choco install just --yes
|
||||
shell: pwsh
|
||||
- uses: extractions/setup-just@v4
|
||||
|
||||
- name: Create virtual env
|
||||
run: |
|
||||
@@ -55,7 +43,7 @@ jobs:
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
uv pip install -e .[dev]
|
||||
uv pip install -e ".[dev]"
|
||||
|
||||
- name: Run type checks
|
||||
run: |
|
||||
@@ -65,28 +53,30 @@ jobs:
|
||||
run: |
|
||||
just lint
|
||||
|
||||
- name: Run tests (SQLite)
|
||||
run: |
|
||||
uv pip install pytest pytest-cov
|
||||
just test-sqlite
|
||||
|
||||
test-postgres:
|
||||
name: Test Postgres (Python ${{ matrix.python-version }})
|
||||
test-sqlite-unit:
|
||||
name: Test SQLite Unit (${{ matrix.os }}, Python ${{ matrix.python-version }})
|
||||
timeout-minutes: 45
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
python-version: [ "3.12", "3.13", "3.14" ]
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
# Note: No services section needed - testcontainers handles Postgres in Docker
|
||||
include:
|
||||
- os: ubuntu-latest
|
||||
python-version: "3.12"
|
||||
- os: ubuntu-latest
|
||||
python-version: "3.13"
|
||||
- os: ubuntu-latest
|
||||
python-version: "3.14"
|
||||
- os: windows-latest
|
||||
python-version: "3.12"
|
||||
runs-on: ${{ matrix.os }}
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/checkout@v6
|
||||
with:
|
||||
submodules: true
|
||||
|
||||
- name: Set up Python ${{ matrix.python-version }}
|
||||
uses: actions/setup-python@v4
|
||||
uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
cache: 'pip'
|
||||
@@ -95,9 +85,7 @@ jobs:
|
||||
run: |
|
||||
pip install uv
|
||||
|
||||
- name: Install just
|
||||
run: |
|
||||
curl --proto '=https' --tlsv1.2 -sSf https://just.systems/install.sh | bash -s -- --to /usr/local/bin
|
||||
- uses: extractions/setup-just@v4
|
||||
|
||||
- name: Create virtual env
|
||||
run: |
|
||||
@@ -105,24 +93,184 @@ jobs:
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
uv pip install -e .[dev]
|
||||
uv pip install -e ".[dev]"
|
||||
|
||||
- name: Run tests (Postgres via testcontainers)
|
||||
- name: Run tests
|
||||
run: |
|
||||
uv pip install pytest pytest-cov
|
||||
just test-postgres
|
||||
just test-unit-sqlite
|
||||
|
||||
coverage:
|
||||
name: Coverage Summary (combined, Python 3.12)
|
||||
test-sqlite-integration:
|
||||
name: Test SQLite Integration (${{ matrix.os }}, Python ${{ matrix.python-version }})
|
||||
timeout-minutes: 45
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- os: ubuntu-latest
|
||||
python-version: "3.12"
|
||||
- os: ubuntu-latest
|
||||
python-version: "3.13"
|
||||
- os: ubuntu-latest
|
||||
python-version: "3.14"
|
||||
- os: windows-latest
|
||||
python-version: "3.12"
|
||||
runs-on: ${{ matrix.os }}
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
with:
|
||||
submodules: true
|
||||
|
||||
- name: Set up Python ${{ matrix.python-version }}
|
||||
uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
cache: 'pip'
|
||||
|
||||
- name: Install uv
|
||||
run: |
|
||||
pip install uv
|
||||
|
||||
- uses: extractions/setup-just@v4
|
||||
|
||||
- name: Create virtual env
|
||||
run: |
|
||||
uv venv
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
uv pip install -e ".[dev]"
|
||||
|
||||
- name: Run tests
|
||||
run: |
|
||||
just test-int-sqlite
|
||||
|
||||
test-postgres-unit:
|
||||
name: Test Postgres Unit (Python ${{ matrix.python-version }})
|
||||
timeout-minutes: 60
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- python-version: "3.12"
|
||||
- python-version: "3.13"
|
||||
- python-version: "3.14"
|
||||
runs-on: ubuntu-latest
|
||||
services:
|
||||
postgres:
|
||||
image: pgvector/pgvector:pg16
|
||||
env:
|
||||
POSTGRES_USER: basic_memory_user
|
||||
POSTGRES_PASSWORD: dev_password
|
||||
POSTGRES_DB: basic_memory_test
|
||||
ports:
|
||||
- 5432:5432
|
||||
options: >-
|
||||
--health-cmd "pg_isready -U basic_memory_user -d basic_memory_test"
|
||||
--health-interval 10s
|
||||
--health-timeout 5s
|
||||
--health-retries 5
|
||||
env:
|
||||
BASIC_MEMORY_TEST_POSTGRES_URL: postgresql://basic_memory_user:dev_password@127.0.0.1:5432/basic_memory_test
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
with:
|
||||
submodules: true
|
||||
|
||||
- name: Set up Python ${{ matrix.python-version }}
|
||||
uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
cache: 'pip'
|
||||
|
||||
- name: Install uv
|
||||
run: |
|
||||
pip install uv
|
||||
|
||||
- uses: extractions/setup-just@v4
|
||||
|
||||
- name: Create virtual env
|
||||
run: |
|
||||
uv venv
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
uv pip install -e ".[dev]"
|
||||
|
||||
- name: Run tests
|
||||
run: |
|
||||
just test-unit-postgres
|
||||
|
||||
test-postgres-integration:
|
||||
name: Test Postgres Integration (Python ${{ matrix.python-version }})
|
||||
timeout-minutes: 45
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- python-version: "3.12"
|
||||
- python-version: "3.13"
|
||||
- python-version: "3.14"
|
||||
runs-on: ubuntu-latest
|
||||
services:
|
||||
postgres:
|
||||
image: pgvector/pgvector:pg16
|
||||
env:
|
||||
POSTGRES_USER: basic_memory_user
|
||||
POSTGRES_PASSWORD: dev_password
|
||||
POSTGRES_DB: basic_memory_test
|
||||
ports:
|
||||
- 5432:5432
|
||||
options: >-
|
||||
--health-cmd "pg_isready -U basic_memory_user -d basic_memory_test"
|
||||
--health-interval 10s
|
||||
--health-timeout 5s
|
||||
--health-retries 5
|
||||
env:
|
||||
BASIC_MEMORY_TEST_POSTGRES_URL: postgresql://basic_memory_user:dev_password@127.0.0.1:5432/basic_memory_test
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
with:
|
||||
submodules: true
|
||||
|
||||
- name: Set up Python ${{ matrix.python-version }}
|
||||
uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
cache: 'pip'
|
||||
|
||||
- name: Install uv
|
||||
run: |
|
||||
pip install uv
|
||||
|
||||
- uses: extractions/setup-just@v4
|
||||
|
||||
- name: Create virtual env
|
||||
run: |
|
||||
uv venv
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
uv pip install -e ".[dev]"
|
||||
|
||||
- name: Run tests
|
||||
run: |
|
||||
just test-int-postgres
|
||||
|
||||
test-semantic:
|
||||
name: Test Semantic (Python 3.12)
|
||||
timeout-minutes: 45
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/checkout@v6
|
||||
with:
|
||||
submodules: true
|
||||
|
||||
- name: Set up Python 3.12
|
||||
uses: actions/setup-python@v4
|
||||
uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: "3.12"
|
||||
cache: "pip"
|
||||
@@ -131,9 +279,7 @@ jobs:
|
||||
run: |
|
||||
pip install uv
|
||||
|
||||
- name: Install just
|
||||
run: |
|
||||
curl --proto '=https' --tlsv1.2 -sSf https://just.systems/install.sh | bash -s -- --to /usr/local/bin
|
||||
- uses: extractions/setup-just@v4
|
||||
|
||||
- name: Create virtual env
|
||||
run: |
|
||||
@@ -141,27 +287,8 @@ jobs:
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
uv pip install -e .[dev]
|
||||
uv pip install -e ".[dev]"
|
||||
|
||||
- name: Run combined coverage (SQLite + Postgres)
|
||||
- name: Run tests
|
||||
run: |
|
||||
uv pip install pytest pytest-cov
|
||||
just coverage
|
||||
|
||||
- name: Add coverage report to job summary
|
||||
if: always()
|
||||
run: |
|
||||
{
|
||||
echo "## Coverage"
|
||||
echo ""
|
||||
echo '```'
|
||||
uv run coverage report -m
|
||||
echo '```'
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
- name: Upload HTML coverage report
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: htmlcov
|
||||
path: htmlcov/
|
||||
just test-semantic
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
*.py[cod]
|
||||
__pycache__/
|
||||
.pytest_cache/
|
||||
.testmondata*
|
||||
.coverage
|
||||
htmlcov/
|
||||
|
||||
@@ -55,3 +56,5 @@ claude-output
|
||||
**/.claude/settings.local.json
|
||||
.mcp.json
|
||||
.mcpregistry_*
|
||||
/.testmondata
|
||||
.benchmarks/
|
||||
|
||||
@@ -0,0 +1,461 @@
|
||||
# AGENTS.md - Basic Memory Project Guide
|
||||
|
||||
## Project Overview
|
||||
|
||||
Basic Memory is a local-first knowledge management system built on the Model Context Protocol (MCP). It enables
|
||||
bidirectional communication between LLMs (like Claude) and markdown files, creating a personal knowledge graph that can
|
||||
be traversed using links between documents.
|
||||
|
||||
## CODEBASE DEVELOPMENT
|
||||
|
||||
### Project information
|
||||
|
||||
See the [README.md](README.md) file for a project overview.
|
||||
|
||||
### Build and Test Commands
|
||||
|
||||
- Install: `just install` or `pip install -e ".[dev]"`
|
||||
- Run all tests (SQLite + Postgres): `just test`
|
||||
- Run all tests against SQLite: `just test-sqlite`
|
||||
- Run all tests against Postgres: `just test-postgres` (uses testcontainers)
|
||||
- Run unit tests (SQLite): `just test-unit-sqlite`
|
||||
- Run unit tests (Postgres): `just test-unit-postgres`
|
||||
- Run integration tests (SQLite): `just test-int-sqlite`
|
||||
- Run integration tests (Postgres): `just test-int-postgres`
|
||||
- Run impacted tests: `just testmon` (pytest-testmon; only tests affected by changed code)
|
||||
- Run MCP smoke test: `just test-smoke`
|
||||
- Fast local loop: `just fast-check` (default iteration flow)
|
||||
- Local consistency check: `just doctor`
|
||||
- Generate HTML coverage: `just coverage`
|
||||
- Single test: `pytest tests/path/to/test_file.py::test_function_name`
|
||||
- Run benchmarks: `pytest test-int/test_sync_performance_benchmark.py -v -m "benchmark and not slow"`
|
||||
- Lint: `just lint` or `ruff check . --fix`
|
||||
- Type check: `just typecheck` or `uv run ty check src tests test-int`
|
||||
- Type check (pyright): `just typecheck-pyright` or `uv run pyright`
|
||||
- Format: `just format` or `uv run ruff format .`
|
||||
- Run all code checks: `just check` (runs lint, format, typecheck, test)
|
||||
- Create db migration: `just migration "Your migration message"`
|
||||
- Run development MCP Inspector: `just run-inspector`
|
||||
|
||||
**Note:** Project requires Python 3.12+ (uses type parameter syntax and `type` aliases introduced in 3.12)
|
||||
|
||||
**Postgres Testing:** Uses [testcontainers](https://testcontainers-python.readthedocs.io/) which automatically spins up a Postgres instance in Docker. No manual database setup required - just have Docker running.
|
||||
|
||||
**Doctor Note:** `just doctor` runs with a temporary HOME/config so it won't touch your local Basic Memory settings. It leaves temp dirs in `/tmp` (safe to ignore or remove).
|
||||
|
||||
**Testmon Note:** When no files have changed, `just testmon` may collect 0 tests. That's expected and means no impacted tests were detected.
|
||||
|
||||
### Code/Test/Verify Loop (fast path)
|
||||
|
||||
1) **Code:** make changes.
|
||||
2) **Test:** `just fast-check` (lint/format/typecheck + pytest-testmon impacted tests for changed code).
|
||||
3) **Verify:** `just doctor` (end-to-end file ↔ DB loop in a temp project).
|
||||
4) **Full gate (when needed):** `just test` or `just check` for SQLite + Postgres.
|
||||
|
||||
Run `just test-smoke` when you specifically need the MCP smoke flow.
|
||||
|
||||
If testmon is “cold,” the first run may be long. Subsequent runs get much faster.
|
||||
|
||||
### PR CI Gate
|
||||
|
||||
Before opening or updating a PR, run the checks that mirror the common required CI failures:
|
||||
|
||||
- Run `just typecheck` in addition to targeted `ruff` and `pytest` commands when tests were added or changed.
|
||||
- Sign commits with `git commit -s` so DCO passes. If a PR branch already has unsigned commits, rewrite the branch with signed-off commits before asking for review.
|
||||
- Use a semantic PR title accepted by `.github/workflows/pr-title.yml`: `type(scope): summary`.
|
||||
- Use one of the allowed scopes: `core`, `cli`, `api`, `mcp`, `sync`, `ui`, `deps`, `installer`.
|
||||
|
||||
### Test Structure
|
||||
|
||||
- `tests/` - Unit tests for individual components (mocked, fast)
|
||||
- `test-int/` - Integration tests for real-world scenarios (no mocks, realistic)
|
||||
- Both directories are covered by unified coverage reporting
|
||||
- Benchmark tests in `test-int/` are marked with `@pytest.mark.benchmark`
|
||||
- Slow tests are marked with `@pytest.mark.slow`
|
||||
- Smoke tests are marked with `@pytest.mark.smoke`
|
||||
|
||||
### Code Style Guidelines
|
||||
|
||||
- Line length: 100 characters max
|
||||
- Python 3.12+ with full type annotations (uses type parameters and type aliases)
|
||||
- Format with ruff (consistent styling)
|
||||
- Import order: standard lib, third-party, local imports
|
||||
- Naming: snake_case for functions/variables, PascalCase for classes
|
||||
- Prefer async patterns with SQLAlchemy 2.0
|
||||
- Use Pydantic v2 for data validation and schemas
|
||||
- CLI uses Typer for command structure
|
||||
- API uses FastAPI for endpoints
|
||||
- Follow the repository pattern for data access
|
||||
- Tools communicate to api routers via the httpx ASGI client (in process)
|
||||
|
||||
### Code Change Guidelines
|
||||
|
||||
- **Full file read before edits**: Before editing any file, read it in full first to ensure complete context; partial reads lead to corrupted edits
|
||||
- **Minimize diffs**: Prefer the smallest change that satisfies the request. Avoid unrelated refactors or style rewrites unless necessary for correctness
|
||||
- **No speculative getattr**: Never use `getattr(obj, "attr", default)` when unsure about attribute names. Check the class definition or source code first
|
||||
- **Fail fast**: Write code with fail-fast logic by default. Do not swallow exceptions with errors or warnings
|
||||
- **No fallback logic**: Do not add fallback logic unless explicitly told to and agreed with the user
|
||||
- **No guessing**: Do not say "The issue is..." before you actually know what the issue is. Investigate first.
|
||||
|
||||
### Literate Programming Style
|
||||
|
||||
Code should tell a story. Comments must explain the "why" and narrative flow, not just the "what".
|
||||
|
||||
**Section Headers:**
|
||||
For files with multiple phases of logic, add section headers so the control flow reads like chapters:
|
||||
```python
|
||||
# --- Authentication ---
|
||||
# ... auth logic ...
|
||||
|
||||
# --- Data Validation ---
|
||||
# ... validation logic ...
|
||||
|
||||
# --- Business Logic ---
|
||||
# ... core logic ...
|
||||
```
|
||||
|
||||
**Decision Point Comments:**
|
||||
For conditionals that materially change behavior (gates, fallbacks, retries, feature flags), add comments with:
|
||||
- **Trigger**: what condition causes this branch
|
||||
- **Why**: the rationale (cost, correctness, UX, determinism)
|
||||
- **Outcome**: what changes downstream
|
||||
|
||||
```python
|
||||
# Trigger: project has no active sync watcher
|
||||
# Why: avoid duplicate file system watchers consuming resources
|
||||
# Outcome: starts new watcher, registers in active_watchers dict
|
||||
if project_id not in active_watchers:
|
||||
start_watcher(project_id)
|
||||
```
|
||||
|
||||
**Constraint Comments:**
|
||||
If code exists because of a constraint (async requirements, rate limits, schema compatibility), explain the constraint near the code:
|
||||
```python
|
||||
# SQLite requires WAL mode for concurrent read/write access
|
||||
connection.execute("PRAGMA journal_mode=WAL")
|
||||
```
|
||||
|
||||
**What NOT to Comment:**
|
||||
Avoid comments that restate obvious code:
|
||||
```python
|
||||
# Bad - restates code
|
||||
counter += 1 # increment counter
|
||||
|
||||
# Good - explains why
|
||||
counter += 1 # track retries for backoff calculation
|
||||
```
|
||||
|
||||
### Codebase Architecture
|
||||
|
||||
See [docs/ARCHITECTURE.md](docs/ARCHITECTURE.md) for detailed architecture documentation.
|
||||
|
||||
**Directory Structure:**
|
||||
- `/alembic` - Alembic db migrations
|
||||
- `/api` - FastAPI REST endpoints + `container.py` composition root
|
||||
- `/cli` - Typer CLI + `container.py` composition root
|
||||
- `/deps` - Feature-scoped FastAPI dependencies (config, db, projects, repositories, services, importers)
|
||||
- `/importers` - Import functionality for Claude, ChatGPT, and other sources
|
||||
- `/markdown` - Markdown parsing and processing
|
||||
- `/mcp` - MCP server + `container.py` composition root + `clients/` typed API clients
|
||||
- `/models` - SQLAlchemy ORM models
|
||||
- `/repository` - Data access layer
|
||||
- `/schemas` - Pydantic models for validation
|
||||
- `/services` - Business logic layer
|
||||
- `/sync` - File synchronization services + `coordinator.py` for lifecycle management
|
||||
|
||||
**Composition Roots:**
|
||||
Each entrypoint (API, MCP, CLI) has a composition root that:
|
||||
- Reads `ConfigManager` (the only place that reads global config)
|
||||
- Resolves runtime mode via `RuntimeMode` enum (TEST > CLOUD > LOCAL)
|
||||
- Provides dependencies to downstream code explicitly
|
||||
|
||||
**Typed API Clients (MCP):**
|
||||
MCP tools use typed clients in `mcp/clients/` to communicate with the API:
|
||||
- `KnowledgeClient` - Entity CRUD operations
|
||||
- `SearchClient` - Search operations
|
||||
- `MemoryClient` - Context building
|
||||
- `DirectoryClient` - Directory listing
|
||||
- `ResourceClient` - Resource reading
|
||||
- `ProjectClient` - Project management
|
||||
|
||||
Flow: MCP Tool → Typed Client → HTTP API → Router → Service → Repository
|
||||
|
||||
### Development Notes
|
||||
|
||||
- MCP tools are defined in src/basic_memory/mcp/tools/
|
||||
- MCP prompts are defined in src/basic_memory/mcp/prompts/
|
||||
- MCP tools should be atomic, composable operations
|
||||
- Use `textwrap.dedent()` for multi-line string formatting in prompts and tools
|
||||
- MCP Prompts are used to invoke tools and format content with instructions for an LLM
|
||||
- Schema changes require Alembic migrations
|
||||
- SQLite is used for indexing and full text search, files are source of truth
|
||||
- Testing uses pytest with asyncio support (strict mode)
|
||||
- Unit tests (`tests/`) use mocks when necessary; integration tests (`test-int/`) use real implementations
|
||||
- By default, tests run against SQLite (fast, no Docker needed)
|
||||
- Set `BASIC_MEMORY_TEST_POSTGRES=1` to run against Postgres (uses testcontainers - Docker required)
|
||||
- Each test runs in a standalone environment with isolated database and tmp_path directory
|
||||
- CI runs SQLite and Postgres tests in parallel for faster feedback
|
||||
- Performance benchmarks are in `test-int/test_sync_performance_benchmark.py`
|
||||
- Use pytest markers: `@pytest.mark.benchmark` for benchmarks, `@pytest.mark.slow` for slow tests
|
||||
- **Coverage must stay at 100%**: Write tests for new code. Only use `# pragma: no cover` when tests would require excessive mocking (e.g., TYPE_CHECKING blocks, error handlers that need failure injection, runtime-mode-dependent code paths)
|
||||
|
||||
### Async Client Pattern (Important!)
|
||||
|
||||
**MCP tools use `get_project_client()` for per-project routing:**
|
||||
|
||||
```python
|
||||
from basic_memory.mcp.project_context import get_project_client
|
||||
|
||||
@mcp.tool()
|
||||
async def my_tool(project: str | None = None, context: Context | None = None):
|
||||
async with get_project_client(project, context) as (client, active_project):
|
||||
# client is routed based on project's mode (local ASGI or cloud HTTP)
|
||||
response = await call_get(client, "/path")
|
||||
return response
|
||||
```
|
||||
|
||||
**CLI commands and non-project-scoped code use `get_client()` directly:**
|
||||
|
||||
```python
|
||||
from basic_memory.mcp.async_client import get_client
|
||||
|
||||
async def my_cli_command():
|
||||
async with get_client() as client:
|
||||
response = await call_get(client, "/path")
|
||||
return response
|
||||
|
||||
# Per-project routing (when project name is known):
|
||||
async with get_client(project_name="research") as client:
|
||||
...
|
||||
```
|
||||
|
||||
**Do NOT use:**
|
||||
- ❌ `from basic_memory.mcp.async_client import client` (deprecated module-level client)
|
||||
- ❌ Manual auth header management
|
||||
- ❌ `inject_auth_header()` (deleted)
|
||||
- ❌ Separate `get_client()` + `get_active_project()` in MCP tools (use `get_project_client()` instead)
|
||||
|
||||
**Key principles:**
|
||||
- Auth happens at client creation, not per-request
|
||||
- Proper resource management via context managers
|
||||
- Per-project routing: each project can be LOCAL or CLOUD independently
|
||||
- Cloud projects use API key (`cloud_api_key` in config) as Bearer token
|
||||
- Routing priority: factory injection > force-local > per-project cloud > global cloud > local ASGI
|
||||
- Factory pattern enables dependency injection for cloud consolidation
|
||||
|
||||
**For cloud app integration:**
|
||||
```python
|
||||
from basic_memory.mcp import async_client
|
||||
|
||||
# Set custom factory before importing tools
|
||||
async_client.set_client_factory(your_custom_factory)
|
||||
```
|
||||
|
||||
See SPEC-16 for full context manager refactor details.
|
||||
|
||||
## BASIC MEMORY PRODUCT USAGE
|
||||
|
||||
### Knowledge Structure
|
||||
|
||||
- Entity: Any concept, document, or idea represented as a markdown file
|
||||
- Observation: A categorized fact about an entity (`- [category] content`)
|
||||
- Relation: A directional link between entities (`- relation_type [[Target]]`)
|
||||
- Frontmatter: YAML metadata at the top of markdown files
|
||||
- Knowledge representation follows precise markdown format:
|
||||
- Observations with [category] prefixes
|
||||
- Relations with WikiLinks [[Entity]]
|
||||
- Frontmatter with metadata
|
||||
|
||||
### Basic Memory Commands
|
||||
|
||||
**Local Commands:**
|
||||
- Check sync status: `basic-memory status`
|
||||
- Doctor check (file <-> DB loop): `basic-memory doctor`
|
||||
- Import from Claude: `basic-memory import claude conversations`
|
||||
- Import from ChatGPT: `basic-memory import chatgpt`
|
||||
- Import from Memory JSON: `basic-memory import memory-json`
|
||||
- Tool access: `basic-memory tool` (provides CLI access to MCP tools)
|
||||
- Continue: `basic-memory tool continue-conversation --topic="search"`
|
||||
|
||||
**Project Management:**
|
||||
- List projects: `basic-memory project list`
|
||||
- Add project: `basic-memory project add "name" ~/path`
|
||||
- Project info: `basic-memory project info`
|
||||
- Set cloud mode: `basic-memory project set-cloud "name"`
|
||||
- Set local mode: `basic-memory project set-local "name"`
|
||||
- One-way sync (local -> cloud): `basic-memory project sync`
|
||||
- Bidirectional sync: `basic-memory project bisync`
|
||||
- Integrity check: `basic-memory project check`
|
||||
|
||||
**Cloud Commands (requires subscription):**
|
||||
- Authenticate (global): `basic-memory cloud login`
|
||||
- Logout (global): `basic-memory cloud logout`
|
||||
- Check cloud status: `basic-memory cloud status`
|
||||
- Setup cloud sync: `basic-memory cloud setup`
|
||||
- Save API key: `basic-memory cloud set-key bmc_...`
|
||||
- Create API key: `basic-memory cloud create-key "name"`
|
||||
- Manage snapshots: `basic-memory cloud snapshot [create|list|delete|show|browse]`
|
||||
- Restore from snapshot: `basic-memory cloud restore <path> --snapshot <id>`
|
||||
|
||||
### MCP Capabilities
|
||||
|
||||
- Basic Memory exposes these MCP tools to LLMs:
|
||||
|
||||
**Content Management:**
|
||||
- `write_note(title, content, directory, tags)` - Create/update markdown notes with semantic observations and relations
|
||||
- `read_note(identifier, page, page_size)` - Read notes by title, permalink, or memory:// URL with knowledge graph awareness
|
||||
- `read_content(path)` - Read raw file content (text, images, binaries) without knowledge graph processing
|
||||
- `view_note(identifier, page, page_size)` - View notes as formatted artifacts for better readability
|
||||
- `edit_note(identifier, operation, content)` - Edit notes incrementally (append, prepend, find/replace, replace_section)
|
||||
- `move_note(identifier, destination_path, is_directory)` - Move notes or directories to new locations, updating database and maintaining links
|
||||
- `delete_note(identifier, is_directory)` - Delete notes or directories from the knowledge base
|
||||
|
||||
**Knowledge Graph Navigation:**
|
||||
- `build_context(url, depth, timeframe)` - Navigate the knowledge graph via memory:// URLs for conversation continuity
|
||||
- `recent_activity(type, depth, timeframe)` - Get recently updated information with specified timeframe (e.g., "1d", "1 week")
|
||||
- `list_directory(dir_name, depth, file_name_glob)` - Browse directory contents with filtering and depth control
|
||||
|
||||
**Search & Discovery:**
|
||||
- `search_notes(query, page, page_size, search_type, types, entity_types, after_date)` - Full-text search across all content with advanced filtering options
|
||||
|
||||
**Project Management:**
|
||||
- `list_memory_projects()` - List all available projects with their status
|
||||
- `create_memory_project(project_name, project_path, set_default)` - Create new Basic Memory projects
|
||||
- `delete_project(project_name)` - Delete a project from configuration
|
||||
|
||||
**Visualization:**
|
||||
- `canvas(nodes, edges, title, directory)` - Generate Obsidian canvas files for knowledge graph visualization
|
||||
|
||||
**ChatGPT-Compatible Tools:**
|
||||
- `search(query)` - Search across knowledge base (OpenAI actions compatible)
|
||||
- `fetch(id)` - Fetch full content of a search result document
|
||||
|
||||
- MCP Prompts for better AI interaction:
|
||||
- `ai_assistant_guide()` - Guidance on effectively using Basic Memory tools for AI assistants
|
||||
- `continue_conversation(topic, timeframe)` - Continue previous conversations with relevant historical context
|
||||
- `search(query, after_date)` - Search with detailed, formatted results for better context understanding
|
||||
- `recent_activity(timeframe)` - View recently changed items with formatted output
|
||||
|
||||
### Cloud Features (v0.15.0+)
|
||||
|
||||
Basic Memory now supports cloud synchronization and storage (requires active subscription):
|
||||
|
||||
**Authentication:**
|
||||
- JWT-based authentication with subscription validation
|
||||
- Secure session management with token refresh
|
||||
- Support for multiple cloud projects
|
||||
|
||||
**Bidirectional Sync:**
|
||||
- rclone bisync integration for two-way synchronization
|
||||
- Conflict resolution and integrity verification
|
||||
- Real-time sync with change detection
|
||||
- Mount/unmount cloud storage for direct file access
|
||||
|
||||
**Cloud Project Management:**
|
||||
- Create and manage projects in the cloud
|
||||
- Toggle between local and cloud modes
|
||||
- Per-project sync configuration
|
||||
- Subscription-based access control
|
||||
|
||||
**Security & Performance:**
|
||||
- Removed .env file loading for improved security
|
||||
- .gitignore integration (respects gitignored files)
|
||||
- WAL mode for SQLite performance
|
||||
- Background relation resolution (non-blocking startup)
|
||||
- API performance optimizations (SPEC-11)
|
||||
|
||||
**Per-Project Cloud Routing:**
|
||||
|
||||
Individual projects can be routed through the cloud while others stay local, using an API key:
|
||||
|
||||
```bash
|
||||
# Save API key and set project to cloud mode
|
||||
basic-memory cloud set-key bmc_abc123...
|
||||
basic-memory project set-cloud research # route through cloud
|
||||
basic-memory project set-local research # revert to local
|
||||
```
|
||||
|
||||
MCP tools use `get_project_client()` which automatically routes based on the project's mode. Cloud projects use the `cloud_api_key` from config as Bearer token.
|
||||
|
||||
**CLI Routing Flags (Global Cloud Mode):**
|
||||
|
||||
When global cloud mode is enabled, CLI commands route to the cloud API by default. Use `--local` and `--cloud` flags to override:
|
||||
|
||||
```bash
|
||||
# Force local routing (ignore cloud mode)
|
||||
basic-memory status --local
|
||||
basic-memory project list --local
|
||||
|
||||
# Force cloud routing (when cloud mode is disabled)
|
||||
basic-memory status --cloud
|
||||
basic-memory project info my-project --cloud
|
||||
```
|
||||
|
||||
Key behaviors:
|
||||
- The local MCP server (`basic-memory mcp`) automatically uses local routing
|
||||
- This allows simultaneous use of local Claude Desktop and cloud-based clients
|
||||
- Some commands (like `project default`, `project sync-config`, `project move`) require `--local` in cloud mode since they modify local configuration
|
||||
- Environment variable `BASIC_MEMORY_FORCE_LOCAL=true` forces local routing globally
|
||||
- Per-project cloud routing via API key works independently of global cloud mode
|
||||
|
||||
## AI-Human Collaborative Development
|
||||
|
||||
Basic Memory emerged from and enables a new kind of development process that combines human and AI capabilities. Instead
|
||||
of using AI just for code generation, we've developed a true collaborative workflow:
|
||||
|
||||
1. AI (LLM) writes initial implementation based on specifications and context
|
||||
2. Human reviews, runs tests, and commits code with any necessary adjustments
|
||||
3. Knowledge persists across conversations using Basic Memory's knowledge graph
|
||||
4. Development continues seamlessly across different AI sessions with consistent context
|
||||
5. Results improve through iterative collaboration and shared understanding
|
||||
|
||||
This approach has allowed us to tackle more complex challenges and build a more robust system than either humans or AI
|
||||
could achieve independently.
|
||||
|
||||
**Problem-Solving Guidance:**
|
||||
- If a solution isn't working after reasonable effort, suggest alternative approaches
|
||||
- Don't persist with a problematic library or pattern when better alternatives exist
|
||||
- Example: When py-pglite caused cascading test failures, switching to testcontainers-postgres was the right call
|
||||
|
||||
## GitHub Integration
|
||||
|
||||
Basic Memory has taken AI-Human collaboration to the next level by integrating Claude directly into the development workflow through GitHub:
|
||||
|
||||
### GitHub MCP Tools
|
||||
|
||||
Using the GitHub Model Context Protocol server, Claude can now:
|
||||
|
||||
- **Repository Management**:
|
||||
- View repository files and structure
|
||||
- Read file contents
|
||||
- Create new branches
|
||||
- Create and update files
|
||||
|
||||
- **Issue Management**:
|
||||
- Create new issues
|
||||
- Comment on existing issues
|
||||
- Close and update issues
|
||||
- Search across issues
|
||||
|
||||
- **Pull Request Workflow**:
|
||||
- Create pull requests
|
||||
- Review code changes
|
||||
- Add comments to PRs
|
||||
|
||||
This integration enables Claude to participate as a full team member in the development process, not just as a code generation tool. Claude's GitHub account ([bm-claudeai](https://github.com/bm-claudeai)) is a member of the Basic Machines organization with direct contributor access to the codebase.
|
||||
|
||||
### Collaborative Development Process
|
||||
|
||||
With GitHub integration, the development workflow includes:
|
||||
|
||||
1. **Direct code review** - Claude can analyze PRs and provide detailed feedback
|
||||
2. **Contribution tracking** - All of Claude's contributions are properly attributed in the Git history
|
||||
3. **Branch management** - Claude can create feature branches for implementations
|
||||
4. **Documentation maintenance** - Claude can keep documentation updated as the code evolves
|
||||
5. **Code Commits**: ALWAYS sign off commits with `git commit -s`
|
||||
6. **Pull Request Titles**: PR titles must follow the semantic format enforced by `.github/workflows/pr-title.yml`: `type(scope): summary`
|
||||
- Allowed types: `feat`, `fix`, `chore`, `docs`, `style`, `refactor`, `perf`, `test`, `build`, `ci`
|
||||
- Allowed scopes: `core`, `cli`, `api`, `mcp`, `sync`, `ui`, `deps`, `installer`
|
||||
- Example: `fix(cli): propagate cloud workspace routing`
|
||||
|
||||
This level of integration represents a new paradigm in AI-human collaboration, where the AI assistant becomes a full-fledged team member rather than just a tool for generating code snippets.
|
||||
+446
-5
@@ -1,5 +1,446 @@
|
||||
# CHANGELOG
|
||||
|
||||
## v0.21.1 (2026-05-16)
|
||||
|
||||
CI-only release. No user-facing changes.
|
||||
|
||||
### Maintenance
|
||||
|
||||
- **#833**: Replace `mislav/bump-homebrew-formula-action` with an inline
|
||||
bash bump step. The action's `resolveRedirect` HEAD-requests
|
||||
`api.github.com /repos/.../tarball/<ref>` expecting HTTP 302; GitHub now
|
||||
returns 303 for authenticated requests on that endpoint, which broke
|
||||
the v0.21.0 Homebrew job and required a manual tap bump. The inline
|
||||
step does the same work (curl + sha256sum, clone tap, sed-bump
|
||||
`url`/`sha256`, commit, push) without any third-party dependency.
|
||||
|
||||
## v0.21.0 (2026-05-16)
|
||||
|
||||
Workspace-aware everywhere: every MCP tool and CLI command now routes through the
|
||||
same workspace/project model, the search and sync paths are noticeably faster,
|
||||
and a handful of long-standing parsing and routing footguns are gone.
|
||||
|
||||
### Breaking Changes
|
||||
|
||||
- Relation parsing no longer treats unquoted multi-word text before a wikilink as a
|
||||
custom relation type. Use single-token relation types like `relates_to [[Target]]`,
|
||||
or quote multi-word relation types like `"relates to" [[Target]]` or
|
||||
`'relates to' [[Target]]`.
|
||||
- Bare list wikilinks like `- [[Target]]` now index as `links_to`.
|
||||
- Prose list items like `- some other thing [[Target]]` now index as `links_to`.
|
||||
- To preserve existing multi-word relation types on re-sync, quote them before upgrading.
|
||||
- See **#824**.
|
||||
|
||||
### Features
|
||||
|
||||
- **#816**: `bm orphan` CLI command surfaces entities whose underlying markdown
|
||||
files are gone, with a flag to clean them out.
|
||||
- **#789**: Create projects directly by cloud workspace slug from MCP
|
||||
(`create_memory_project(workspace=...)`).
|
||||
- **#757**: Discover projects across every accessible cloud workspace in MCP's
|
||||
project list — no more per-workspace blind spots.
|
||||
- **#766**: MCP tools accept training-data-friendly parameter aliases
|
||||
(`q`/`search`/`text` for `query`, etc.) so models reach for them naturally.
|
||||
- **#776**: `bm db reset` refuses to run while a `basic-memory mcp` process is
|
||||
alive, so resets can no longer corrupt an open session.
|
||||
- **#791**: Search responses include result totals so pagers can stop guessing.
|
||||
- **#719**: Cloud `note_content` tenant schema primitive lands on the backend.
|
||||
- **#715**: `bm project add` accepts a `--visibility` flag for cloud projects.
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
#### Search and recent activity
|
||||
- **#832**: SQLite project deletion now sweeps `search_index`,
|
||||
`search_vector_chunks`, and `search_vector_embeddings`, so a project that
|
||||
reuses a recycled auto-increment id can't inherit the previous tenant's content.
|
||||
- **#812**: `recent_activity` orders and filters by `updated_at`, so edits bubble
|
||||
to the top instead of staying pinned to creation time.
|
||||
- **#807**: Multi-project `search_notes` is opt-in (`search_all_projects=True`);
|
||||
default search stays scoped to the resolved project.
|
||||
- **#785**: `recent_activity` caps responses and emits an explicit truncation
|
||||
footer instead of silently dropping rows.
|
||||
- **#713**: Eliminated an N+1 query in search result hydration.
|
||||
|
||||
#### Workspace / project routing
|
||||
- **#822**: `bm project list` now includes projects from every workspace, not
|
||||
just the current one.
|
||||
- **#813**, **#808**, **#806**, **#803**, **#801**, **#795**, **#790**, **#778**,
|
||||
**#777**, **#722**, **#712**, **#704**: Workspace-qualified permalink routing
|
||||
is centralized and applied consistently across `edit_note`, `delete_project`,
|
||||
`build_context`, `memory://` URLs, factory-mode project listing, cloud uploads,
|
||||
and the API client.
|
||||
|
||||
#### Sync
|
||||
- **#827**: `rclone bisync` filters from `.bmignore` are preserved across syncs.
|
||||
- **#815**: Watch service ignores hidden paths relative to the watched project,
|
||||
not just relative to `$HOME`.
|
||||
- **#814**: `scan` subprocesses no longer go through the shell, avoiding quoting
|
||||
issues with paths that contain special characters.
|
||||
- **#759**: Watch service stays inside `--project` scope.
|
||||
- **#746**: Canonical markdown is preserved during single-file sync.
|
||||
|
||||
#### Parsing
|
||||
- **#796**: Picoschema modifier descriptions (`field?(modifier, description)`)
|
||||
parse correctly.
|
||||
- **#769**: Obsidian callout blocks are skipped by the observation parser
|
||||
instead of being mis-extracted as observations.
|
||||
|
||||
#### CLI
|
||||
- **#775**: `bm project set-cloud` / `set-local` cleans up local DB state for
|
||||
the affected project.
|
||||
- **#773**: `bm cloud logout` clears `default_workspace`.
|
||||
- **#780**: `bm cloud setup` hint points at `bm cloud sync-setup`.
|
||||
- **#734**: `bm project info` shows cloud index freshness.
|
||||
- **#718**: Private cloud projects display under their `display_name` instead of
|
||||
raw UUID.
|
||||
- **#768**: `read_note` / `view_note` drop no-op pagination params from their
|
||||
signatures.
|
||||
|
||||
#### Stability
|
||||
- **#774**: `sqlite-vec` failures during init degrade gracefully to keyword-only
|
||||
search instead of crashing startup.
|
||||
- **#733**: Delete-vector and cloud-sync cleanup is now consistent.
|
||||
- **#702**: Race conditions in concurrent `delete_entity` are resolved.
|
||||
- **#724**: `external_id` is preserved when entities are re-upserted during a
|
||||
re-index.
|
||||
- **#728**: Vector init no longer issues runtime `ALTER TABLE`.
|
||||
- **#744**: `BASIC_MEMORY_CONFIG_DIR` is honored across remaining call sites.
|
||||
- **#743**: FastEmbed cache lives under the data dir instead of `/tmp`.
|
||||
- **#752**: Cloud projects report `source=cloud` in factory mode.
|
||||
- Stripped null bytes from markdown content before DB insert.
|
||||
- Allowed long `relation_type` values in API responses.
|
||||
|
||||
#### Installer
|
||||
- **#772**: Docker-compose config volume mounts under the `appuser` home.
|
||||
- **#695**: Bumped `brew outdated` timeout from 15s to 60s.
|
||||
|
||||
### Performance
|
||||
|
||||
- **#828**: CLI startup no longer pulls in the local ASGI FastAPI app when it
|
||||
isn't needed.
|
||||
- **#751**, **#726**, **#717**, **#714**, single-file/batch indexing: marked
|
||||
speedups on the sync hot path; unchanged markdown is skipped entirely.
|
||||
- **#731**, **#723**: Vector sync is faster on both backends, with tuned
|
||||
fastembed defaults.
|
||||
|
||||
### Maintenance
|
||||
|
||||
- **#825**: Dependency refresh + security hardening.
|
||||
- Updated to `fastmcp` 3.3.1.
|
||||
- **#754**: Removed in-house telemetry wrappers in favor of direct `logfire`
|
||||
usage.
|
||||
- **#736**: `ty` is now the default typechecker.
|
||||
- **#771**, **#770**, **#716**: New regression guards for vector-row cleanup,
|
||||
long relation types, and recent activity hydration.
|
||||
|
||||
## v0.20.3 (2026-03-26)
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- **#698**: CLI cloud commands now use API key when configured
|
||||
- `get_authenticated_headers()` only checked OAuth tokens, ignoring `config.cloud_api_key`
|
||||
- All CLI cloud commands (`upload`, `status`, `snapshot`, `restore`, etc.) failed for API-key-only users while MCP tools worked fine
|
||||
- Now mirrors the same credential priority as MCP: API key first, OAuth fallback
|
||||
- Fixes `bm cloud upload --project` returning "project does not exist" when authenticated with `bmc_*` API key
|
||||
|
||||
## v0.20.2 (2026-03-10)
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- Fix auto-update Homebrew detection: `brew outdated` exits 1 when a formula is outdated, not on error
|
||||
- Previously treated exit code 1 as a failure, causing "Automatic update check failed" instead of detecting the available update
|
||||
|
||||
## v0.20.1 (2026-03-10)
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- **#661**: Fix `bm project list` MCP column to show transport type (stdio/https) instead of DB presence
|
||||
- Renamed "MCP (stdio)" column to "MCP"
|
||||
- Shows actual routing mode: `stdio` for local, `https` for cloud projects
|
||||
- Clears local path display for cloud-mode projects
|
||||
- **#662**: Invalidate config cache when file is modified by another process
|
||||
- Adds mtime-based cache validation to `ConfigManager.load_config()`
|
||||
- Long-lived processes (MCP stdio server) now detect external config changes
|
||||
- Fixes `bm project set-cloud` having no effect on running MCP server
|
||||
|
||||
## v0.20.0 (2026-03-10)
|
||||
|
||||
### Features
|
||||
|
||||
- **#643**: Default-on auto-update system and `bm update` command
|
||||
- Automatic background update checks for CLI installs (uv tool, Homebrew)
|
||||
- Install-source detection (homebrew, uv_tool, uvx, unknown) with uvx skip behavior
|
||||
- Periodic check gating via `auto_update_last_checked_at` + `update_check_interval` config
|
||||
- Manager-specific update flows: Homebrew (`brew upgrade`) and uv tool (`uv tool upgrade`)
|
||||
- Silent, non-blocking MCP behavior via daemon thread before server run
|
||||
- Manual commands: `bm update` (force check + apply) and `bm update --check` (check only)
|
||||
- New config fields: `auto_update`, `update_check_interval`, `auto_update_last_checked_at`
|
||||
|
||||
## v0.19.2 (2026-03-09)
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- **#657**: Coerce string params to list/dict in MCP tools
|
||||
- MCP clients that serialize `list`/`dict` arguments as JSON strings no longer fail Pydantic validation
|
||||
- Adds `BeforeValidator` coercion to `search_notes` (`entity_types`, `note_types`, `tags`, `metadata_filters`), `write_note` (`metadata`), and `canvas` (`nodes`, `edges`)
|
||||
- **#655**: Handle SQLite and Windows semantic search regressions
|
||||
- Fix embedding status query for non-semantic SQLite databases
|
||||
- Windows-safe log file rotation with per-process log filenames
|
||||
- Robust `setup_logging` that handles all environments cleanly
|
||||
|
||||
## v0.19.1 (2026-03-08)
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- **#649**: Enforce strict entity resolution in destructive MCP tools (`edit_note`, `move_note`, `delete_note`)
|
||||
- Prevents fuzzy-match fallback from silently editing/moving/deleting the wrong note
|
||||
- DST-related timeframe validation fix (round instead of truncate days)
|
||||
|
||||
### Features
|
||||
|
||||
- **#648**: Add `insert_before_section` and `insert_after_section` edit operations
|
||||
- Add `GET /knowledge/graph` endpoint for full graph visualization
|
||||
|
||||
### Dependencies
|
||||
|
||||
- Bump authlib from 1.6.6 to 1.6.7
|
||||
|
||||
## v0.19.0 (2026-03-07)
|
||||
|
||||
### Highlights
|
||||
|
||||
- **Semantic vector search** for SQLite and Postgres with FastEmbed embeddings
|
||||
- **Schema system** for validating and inferring knowledge base structure
|
||||
- **Per-project cloud routing** with API key authentication
|
||||
- **Upgraded to FastMCP 3.0** with tool annotations
|
||||
- **CLI overhaul** with JSON output, workspace awareness, and project dashboard
|
||||
|
||||
### Features
|
||||
|
||||
- **#550**: Add semantic vector search for SQLite and Postgres
|
||||
- FastEmbed-based embeddings with automatic backfill
|
||||
- Hybrid search combining full-text and vector similarity
|
||||
- Score-based fusion replacing RRF for better ranking
|
||||
- `min_similarity` override for tuning search precision
|
||||
- Semantic dependencies are now default, with optional extras fallback
|
||||
|
||||
- **#549**: Schema system for Basic Memory
|
||||
- `schema_infer` — infer schema from existing notes
|
||||
- `schema_validate` — validate notes against a schema definition
|
||||
- `schema_diff` — compare schemas across projects
|
||||
- Frontmatter validation support (#597)
|
||||
- Read schema definitions from file instead of stale DB metadata (#635)
|
||||
|
||||
- **#555**: Per-project local/cloud routing with API key auth
|
||||
- Individual projects route through cloud while others stay local
|
||||
- `basic-memory cloud set-key` and `basic-memory project set-cloud/set-local`
|
||||
- Stdio MCP honors per-project cloud routing (#590)
|
||||
|
||||
- **#598**: Upgrade FastMCP 2.12.3 to 3.0.1 with tool annotations
|
||||
|
||||
- **#585**: Add JSON output mode for MCP tools (default text)
|
||||
- `--json` output for CLI commands for scripting and CI
|
||||
|
||||
- **#576**: Add workspace selection flow for MCP and CLI
|
||||
- Workspace-aware cloud project listing
|
||||
- CLI refactoring for workspace support
|
||||
|
||||
- **#544**: Project-prefixed permalinks and memory URL routing
|
||||
|
||||
- **#632**: Add overwrite guard to `write_note` tool
|
||||
|
||||
- **#614**: `edit_note` append/prepend auto-creates note if not found
|
||||
|
||||
- **#609**: Richer content context in search results
|
||||
- Return matched chunk text in search results (#601)
|
||||
- Improved content hit rate
|
||||
|
||||
- **#602**: Add `created_by` and `last_updated_by` user tracking to Entity
|
||||
|
||||
- **#600**: Rename `entity_type` to `note_type` across codebase
|
||||
|
||||
- **#574**: Add `display_name` and `is_private` to ProjectItem
|
||||
|
||||
- **#569**: Expose `external_id` in EntityResponse and link resolver
|
||||
|
||||
- **#567**: Isolate default SQLite DB by config dir
|
||||
|
||||
- **#560**: Enable `default_project_mode` by default
|
||||
|
||||
- **#559**: Add `basic-memory watch` CLI command
|
||||
|
||||
- **#546**: Add cloud discovery touchpoints to CLI and MCP
|
||||
|
||||
- **#572**: CLI analytics via Umami event collector
|
||||
|
||||
- Replace project info with htop-inspired dashboard
|
||||
|
||||
- Merge `search_by_metadata` into `search_notes` with optional query
|
||||
|
||||
- Add `--strip-frontmatter` to `basic-memory tool read-note`
|
||||
|
||||
- Add `destination_folder` parameter to `move_note` tool
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- **#644**: Fix default project resolution in cloud mode
|
||||
- ChatGPT search/fetch tools broken in cloud mode
|
||||
- `resolve_project_parameter` falls back to projects API
|
||||
|
||||
- **#638**: Restore API backward compatibility for v0.18.x clients
|
||||
|
||||
- **#637**: Create backup before config migration overwrites old format
|
||||
|
||||
- **#636**: `list_workspaces` bypasses factory pattern on cloud MCP server
|
||||
|
||||
- **#631**: `build_context` related_results schema validation failure
|
||||
|
||||
- **#613**: Reduce excessive log volume by demoting per-request noise to DEBUG
|
||||
|
||||
- **#612**: Handle quoted picoschema enum strings in YAML frontmatter
|
||||
|
||||
- **#607**: Guard against closed streams in promo and missing vector tables
|
||||
|
||||
- **#606**: Accept null for `expected_replacements` in `edit_note`
|
||||
|
||||
- **#595**: `recent_activity` dedup and pagination across MCP tools
|
||||
|
||||
- **#593**: Backend-specific distance-to-similarity conversion
|
||||
|
||||
- **#582**: Use LinkResolver fallback in `build_context` for flexible identifier matching
|
||||
|
||||
- **#577**: Replace RRF with score-based fusion in hybrid search
|
||||
|
||||
- **#575**: Remove hardcoded "main" default from `default_project`
|
||||
|
||||
- **#534**: Speed up `bm --version` startup
|
||||
|
||||
- Fix semantic embeddings not generated on fresh DB or upgrade
|
||||
|
||||
- Clarify `search_notes` parameter naming and fix `note_types` case sensitivity
|
||||
|
||||
- Parse `tag:` prefix at MCP tool level to avoid hybrid search failure
|
||||
|
||||
- Cap sqlite-vec knn k parameter at 4096 limit
|
||||
|
||||
- Parameterize SQL queries in search repository type filters
|
||||
|
||||
- Coerce list frontmatter values to strings for title and type fields
|
||||
|
||||
- Avoid `Post(**metadata)` crash when frontmatter contains 'content' or 'handler' keys
|
||||
|
||||
- Upgrade cryptography and python-multipart for security advisories
|
||||
|
||||
### Internal
|
||||
|
||||
- **#594**: Add `ty` as supplemental type checker
|
||||
- Batched vector sync orchestration across repositories
|
||||
- FastEmbed parallel guardrails and provider caching
|
||||
- Improved cloud CLI status and error messages
|
||||
- CI coverage and Postgres test fixes
|
||||
|
||||
## v0.18.5 (2026-02-13)
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- Strip NUL bytes from content before PostgreSQL search indexing
|
||||
([`ec9b2c4`](https://github.com/basicmachines-co/basic-memory/commit/ec9b2c4))
|
||||
|
||||
## v0.18.4 (2026-02-12)
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- Use global `--header` flag for Tigris consistency on all rclone transactions
|
||||
([`0eae0e1`](https://github.com/basicmachines-co/basic-memory/commit/0eae0e1))
|
||||
- `--header-download` / `--header-upload` only apply to GET/PUT requests, missing S3
|
||||
ListObjectsV2 calls that bisync issues first. Non-US users saw stale edge-cached metadata.
|
||||
- `--header` applies to ALL HTTP transactions (list, download, upload), fixing bisync for
|
||||
users outside the Tigris origin region.
|
||||
|
||||
## v0.18.2 (2026-02-11)
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- **#562**: Use VIRTUAL instead of STORED columns in SQLite migration
|
||||
([`344e651`](https://github.com/basicmachines-co/basic-memory/commit/344e651))
|
||||
- Fixes compatibility issue with SQLite STORED generated columns
|
||||
|
||||
## v0.18.1 (2026-02-11)
|
||||
|
||||
### Features
|
||||
|
||||
- **#552**: Add `--format json` to CLI tool commands
|
||||
([`a47c9c0`](https://github.com/basicmachines-co/basic-memory/commit/a47c9c0))
|
||||
- CLI tool commands now support `--format json` for machine-readable output
|
||||
|
||||
- **#535**: Support `tag:` query shorthand in search
|
||||
([`f1d50c2`](https://github.com/basicmachines-co/basic-memory/commit/f1d50c2))
|
||||
- Use `tag:mytag` as a convenient shorthand in search queries
|
||||
|
||||
- **#532**: Fast edit entities, refactors for webui, enhanced search
|
||||
([`530cbac`](https://github.com/basicmachines-co/basic-memory/commit/530cbac))
|
||||
- Performance improvements for entity editing and search operations
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- **#558**: Add X-Tigris-Consistent headers to all rclone commands
|
||||
([`8489a3d`](https://github.com/basicmachines-co/basic-memory/commit/8489a3d))
|
||||
- Ensures consistent reads from Tigris object storage during sync
|
||||
|
||||
- **#541**: Handle EntityCreationError as conflict
|
||||
([`343a6e1`](https://github.com/basicmachines-co/basic-memory/commit/343a6e1))
|
||||
|
||||
- **#536**: Stabilize metadata filters on Postgres
|
||||
([`009e849`](https://github.com/basicmachines-co/basic-memory/commit/009e849))
|
||||
|
||||
- **#533**: Fix recent_activity prompt defaults
|
||||
([`24ca5f6`](https://github.com/basicmachines-co/basic-memory/commit/24ca5f6))
|
||||
|
||||
- **#530**: Prevent spurious `metadata: {}` in frontmatter output
|
||||
([`e3ced49`](https://github.com/basicmachines-co/basic-memory/commit/e3ced49))
|
||||
|
||||
- Add POST legacy compat routes for v0.18.0 CLI
|
||||
([`c46d7a6`](https://github.com/basicmachines-co/basic-memory/commit/c46d7a6))
|
||||
|
||||
- Restore legacy `/projects/projects` endpoint for older CLI versions
|
||||
([`a0e754b`](https://github.com/basicmachines-co/basic-memory/commit/a0e754b))
|
||||
|
||||
### Internal
|
||||
|
||||
- **#538**: Add fast feedback loop tooling (`just fast-check`, `just doctor`, `just testmon`)
|
||||
([`8072449`](https://github.com/basicmachines-co/basic-memory/commit/8072449))
|
||||
|
||||
## v0.18.0 (2026-01-28)
|
||||
|
||||
### Features
|
||||
|
||||
- **#527**: Add context-aware wiki link resolution with source_path support
|
||||
([`0023e73`](https://github.com/basicmachines-co/basic-memory/commit/0023e73))
|
||||
- Add `source_path` parameter to `resolve_link()` for context-aware resolution
|
||||
- Relative path resolution: `[[nested/note]]` from `folder/file.md` → `folder/nested/note.md`
|
||||
- Proximity-based resolution for duplicate titles (prefers notes in same folder)
|
||||
- Strict mode to disable fuzzy search fallback for wiki links
|
||||
|
||||
- **#518**: Add directory support to move_note and delete_note tools
|
||||
([`0b20801`](https://github.com/basicmachines-co/basic-memory/commit/0b20801))
|
||||
- Add `is_directory` parameter to `move_note` and `delete_note` MCP tools
|
||||
- New `POST /move-directory` and delete directory API endpoints
|
||||
- Rename `folder` → `directory` parameter across codebase for consistency
|
||||
|
||||
- **#522**: Local MCP cloud mode routing
|
||||
([`8730067`](https://github.com/basicmachines-co/basic-memory/commit/8730067))
|
||||
- Add `--local` and `--cloud` CLI routing flags
|
||||
- Local MCP server (`basic-memory mcp`) automatically uses local routing
|
||||
- Enables simultaneous use of local Claude Desktop and cloud-based clients
|
||||
- Environment variable `BASIC_MEMORY_FORCE_LOCAL=true` for global override
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- **#524**: Fix MCP prompt rendering errors
|
||||
([`e14ba92`](https://github.com/basicmachines-co/basic-memory/commit/e14ba92))
|
||||
- Fix "Error rendering prompt recent_activity" error
|
||||
- Change `TimeFrame` to `str` in prompt type annotations for FastMCP compatibility
|
||||
|
||||
## v0.17.9 (2026-01-24)
|
||||
|
||||
### Bug Fixes
|
||||
@@ -1815,12 +2256,12 @@ Signed-off-by: phernandez <paul@basicmachines.co>
|
||||
- Update CLAUDE.md ([#33](https://github.com/basicmachines-co/basic-memory/pull/33),
|
||||
[`dfaf0fe`](https://github.com/basicmachines-co/basic-memory/commit/dfaf0fea9cf5b97d169d51a6276ec70162c21a7e))
|
||||
|
||||
fix spelling in CLAUDE.md: enviroment -> environment Signed-off-by: Ikko Eltociear Ashimine
|
||||
fix spelling in CLAUDE.md: environment typo Signed-off-by: Ikko Eltociear Ashimine
|
||||
<eltociear@gmail.com>
|
||||
|
||||
### Refactoring
|
||||
|
||||
- Move project stats into projct subcommand
|
||||
- Move project stats into project subcommand
|
||||
([`2a881b1`](https://github.com/basicmachines-co/basic-memory/commit/2a881b1425c73947f037fbe7ac5539c015b62526))
|
||||
|
||||
Signed-off-by: phernandez <paul@basicmachines.co>
|
||||
@@ -2249,7 +2690,7 @@ Co-authored-by: phernandez <phernandez@basicmachines.co>
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- Refix vitual env in installer build
|
||||
- Refix virtual env in installer build
|
||||
([`052f491`](https://github.com/basicmachines-co/basic-memory/commit/052f491fff629e8ead629c9259f8cb46c608d584))
|
||||
|
||||
|
||||
@@ -2268,7 +2709,7 @@ Co-authored-by: phernandez <phernandez@basicmachines.co>
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- Fix path to intaller app artifact
|
||||
- Fix path to installer app artifact
|
||||
([`53d220d`](https://github.com/basicmachines-co/basic-memory/commit/53d220df585561f9edd0d49a9e88f1d4055059cf))
|
||||
|
||||
|
||||
@@ -2276,7 +2717,7 @@ Co-authored-by: phernandez <phernandez@basicmachines.co>
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- Activate vitualenv in installer build
|
||||
- Activate virtualenv in installer build
|
||||
([`d4c8293`](https://github.com/basicmachines-co/basic-memory/commit/d4c8293687a52eaf3337fe02e2f7b80e4cc9a1bb))
|
||||
|
||||
- Trigger installer build on release
|
||||
|
||||
@@ -1,369 +0,0 @@
|
||||
# CLAUDE.md - Basic Memory Project Guide
|
||||
|
||||
## Project Overview
|
||||
|
||||
Basic Memory is a local-first knowledge management system built on the Model Context Protocol (MCP). It enables
|
||||
bidirectional communication between LLMs (like Claude) and markdown files, creating a personal knowledge graph that can
|
||||
be traversed using links between documents.
|
||||
|
||||
## CODEBASE DEVELOPMENT
|
||||
|
||||
### Project information
|
||||
|
||||
See the [README.md](README.md) file for a project overview.
|
||||
|
||||
### Build and Test Commands
|
||||
|
||||
- Install: `just install` or `pip install -e ".[dev]"`
|
||||
- Run all tests (SQLite + Postgres): `just test`
|
||||
- Run all tests against SQLite: `just test-sqlite`
|
||||
- Run all tests against Postgres: `just test-postgres` (uses testcontainers)
|
||||
- Run unit tests (SQLite): `just test-unit-sqlite`
|
||||
- Run unit tests (Postgres): `just test-unit-postgres`
|
||||
- Run integration tests (SQLite): `just test-int-sqlite`
|
||||
- Run integration tests (Postgres): `just test-int-postgres`
|
||||
- Generate HTML coverage: `just coverage`
|
||||
- Single test: `pytest tests/path/to/test_file.py::test_function_name`
|
||||
- Run benchmarks: `pytest test-int/test_sync_performance_benchmark.py -v -m "benchmark and not slow"`
|
||||
- Lint: `just lint` or `ruff check . --fix`
|
||||
- Type check: `just typecheck` or `uv run pyright`
|
||||
- Format: `just format` or `uv run ruff format .`
|
||||
- Run all code checks: `just check` (runs lint, format, typecheck, test)
|
||||
- Create db migration: `just migration "Your migration message"`
|
||||
- Run development MCP Inspector: `just run-inspector`
|
||||
|
||||
**Note:** Project requires Python 3.12+ (uses type parameter syntax and `type` aliases introduced in 3.12)
|
||||
|
||||
**Postgres Testing:** Uses [testcontainers](https://testcontainers-python.readthedocs.io/) which automatically spins up a Postgres instance in Docker. No manual database setup required - just have Docker running.
|
||||
|
||||
### Test Structure
|
||||
|
||||
- `tests/` - Unit tests for individual components (mocked, fast)
|
||||
- `test-int/` - Integration tests for real-world scenarios (no mocks, realistic)
|
||||
- Both directories are covered by unified coverage reporting
|
||||
- Benchmark tests in `test-int/` are marked with `@pytest.mark.benchmark`
|
||||
- Slow tests are marked with `@pytest.mark.slow`
|
||||
|
||||
### Code Style Guidelines
|
||||
|
||||
- Line length: 100 characters max
|
||||
- Python 3.12+ with full type annotations (uses type parameters and type aliases)
|
||||
- Format with ruff (consistent styling)
|
||||
- Import order: standard lib, third-party, local imports
|
||||
- Naming: snake_case for functions/variables, PascalCase for classes
|
||||
- Prefer async patterns with SQLAlchemy 2.0
|
||||
- Use Pydantic v2 for data validation and schemas
|
||||
- CLI uses Typer for command structure
|
||||
- API uses FastAPI for endpoints
|
||||
- Follow the repository pattern for data access
|
||||
- Tools communicate to api routers via the httpx ASGI client (in process)
|
||||
|
||||
### Code Change Guidelines
|
||||
|
||||
- **Full file read before edits**: Before editing any file, read it in full first to ensure complete context; partial reads lead to corrupted edits
|
||||
- **Minimize diffs**: Prefer the smallest change that satisfies the request. Avoid unrelated refactors or style rewrites unless necessary for correctness
|
||||
- **No speculative getattr**: Never use `getattr(obj, "attr", default)` when unsure about attribute names. Check the class definition or source code first
|
||||
- **Fail fast**: Write code with fail-fast logic by default. Do not swallow exceptions with errors or warnings
|
||||
- **No fallback logic**: Do not add fallback logic unless explicitly told to and agreed with the user
|
||||
- **No guessing**: Do not say "The issue is..." before you actually know what the issue is. Investigate first.
|
||||
|
||||
### Literate Programming Style
|
||||
|
||||
Code should tell a story. Comments must explain the "why" and narrative flow, not just the "what".
|
||||
|
||||
**Section Headers:**
|
||||
For files with multiple phases of logic, add section headers so the control flow reads like chapters:
|
||||
```python
|
||||
# --- Authentication ---
|
||||
# ... auth logic ...
|
||||
|
||||
# --- Data Validation ---
|
||||
# ... validation logic ...
|
||||
|
||||
# --- Business Logic ---
|
||||
# ... core logic ...
|
||||
```
|
||||
|
||||
**Decision Point Comments:**
|
||||
For conditionals that materially change behavior (gates, fallbacks, retries, feature flags), add comments with:
|
||||
- **Trigger**: what condition causes this branch
|
||||
- **Why**: the rationale (cost, correctness, UX, determinism)
|
||||
- **Outcome**: what changes downstream
|
||||
|
||||
```python
|
||||
# Trigger: project has no active sync watcher
|
||||
# Why: avoid duplicate file system watchers consuming resources
|
||||
# Outcome: starts new watcher, registers in active_watchers dict
|
||||
if project_id not in active_watchers:
|
||||
start_watcher(project_id)
|
||||
```
|
||||
|
||||
**Constraint Comments:**
|
||||
If code exists because of a constraint (async requirements, rate limits, schema compatibility), explain the constraint near the code:
|
||||
```python
|
||||
# SQLite requires WAL mode for concurrent read/write access
|
||||
connection.execute("PRAGMA journal_mode=WAL")
|
||||
```
|
||||
|
||||
**What NOT to Comment:**
|
||||
Avoid comments that restate obvious code:
|
||||
```python
|
||||
# Bad - restates code
|
||||
counter += 1 # increment counter
|
||||
|
||||
# Good - explains why
|
||||
counter += 1 # track retries for backoff calculation
|
||||
```
|
||||
|
||||
### Codebase Architecture
|
||||
|
||||
See [docs/ARCHITECTURE.md](docs/ARCHITECTURE.md) for detailed architecture documentation.
|
||||
|
||||
**Directory Structure:**
|
||||
- `/alembic` - Alembic db migrations
|
||||
- `/api` - FastAPI REST endpoints + `container.py` composition root
|
||||
- `/cli` - Typer CLI + `container.py` composition root
|
||||
- `/deps` - Feature-scoped FastAPI dependencies (config, db, projects, repositories, services, importers)
|
||||
- `/importers` - Import functionality for Claude, ChatGPT, and other sources
|
||||
- `/markdown` - Markdown parsing and processing
|
||||
- `/mcp` - MCP server + `container.py` composition root + `clients/` typed API clients
|
||||
- `/models` - SQLAlchemy ORM models
|
||||
- `/repository` - Data access layer
|
||||
- `/schemas` - Pydantic models for validation
|
||||
- `/services` - Business logic layer
|
||||
- `/sync` - File synchronization services + `coordinator.py` for lifecycle management
|
||||
|
||||
**Composition Roots:**
|
||||
Each entrypoint (API, MCP, CLI) has a composition root that:
|
||||
- Reads `ConfigManager` (the only place that reads global config)
|
||||
- Resolves runtime mode via `RuntimeMode` enum (TEST > CLOUD > LOCAL)
|
||||
- Provides dependencies to downstream code explicitly
|
||||
|
||||
**Typed API Clients (MCP):**
|
||||
MCP tools use typed clients in `mcp/clients/` to communicate with the API:
|
||||
- `KnowledgeClient` - Entity CRUD operations
|
||||
- `SearchClient` - Search operations
|
||||
- `MemoryClient` - Context building
|
||||
- `DirectoryClient` - Directory listing
|
||||
- `ResourceClient` - Resource reading
|
||||
- `ProjectClient` - Project management
|
||||
|
||||
Flow: MCP Tool → Typed Client → HTTP API → Router → Service → Repository
|
||||
|
||||
### Development Notes
|
||||
|
||||
- MCP tools are defined in src/basic_memory/mcp/tools/
|
||||
- MCP prompts are defined in src/basic_memory/mcp/prompts/
|
||||
- MCP tools should be atomic, composable operations
|
||||
- Use `textwrap.dedent()` for multi-line string formatting in prompts and tools
|
||||
- MCP Prompts are used to invoke tools and format content with instructions for an LLM
|
||||
- Schema changes require Alembic migrations
|
||||
- SQLite is used for indexing and full text search, files are source of truth
|
||||
- Testing uses pytest with asyncio support (strict mode)
|
||||
- Unit tests (`tests/`) use mocks when necessary; integration tests (`test-int/`) use real implementations
|
||||
- By default, tests run against SQLite (fast, no Docker needed)
|
||||
- Set `BASIC_MEMORY_TEST_POSTGRES=1` to run against Postgres (uses testcontainers - Docker required)
|
||||
- Each test runs in a standalone environment with isolated database and tmp_path directory
|
||||
- CI runs SQLite and Postgres tests in parallel for faster feedback
|
||||
- Performance benchmarks are in `test-int/test_sync_performance_benchmark.py`
|
||||
- Use pytest markers: `@pytest.mark.benchmark` for benchmarks, `@pytest.mark.slow` for slow tests
|
||||
- **Coverage must stay at 100%**: Write tests for new code. Only use `# pragma: no cover` when tests would require excessive mocking (e.g., TYPE_CHECKING blocks, error handlers that need failure injection, runtime-mode-dependent code paths)
|
||||
|
||||
### Async Client Pattern (Important!)
|
||||
|
||||
**All MCP tools and CLI commands use the context manager pattern for HTTP clients:**
|
||||
|
||||
```python
|
||||
from basic_memory.mcp.async_client import get_client
|
||||
|
||||
async def my_mcp_tool():
|
||||
async with get_client() as client:
|
||||
# Use client for API calls
|
||||
response = await call_get(client, "/path")
|
||||
return response
|
||||
```
|
||||
|
||||
**Do NOT use:**
|
||||
- ❌ `from basic_memory.mcp.async_client import client` (deprecated module-level client)
|
||||
- ❌ Manual auth header management
|
||||
- ❌ `inject_auth_header()` (deleted)
|
||||
|
||||
**Key principles:**
|
||||
- Auth happens at client creation, not per-request
|
||||
- Proper resource management via context managers
|
||||
- Supports three modes: Local (ASGI), CLI cloud (HTTP + auth), Cloud app (factory injection)
|
||||
- Factory pattern enables dependency injection for cloud consolidation
|
||||
|
||||
**For cloud app integration:**
|
||||
```python
|
||||
from basic_memory.mcp import async_client
|
||||
|
||||
# Set custom factory before importing tools
|
||||
async_client.set_client_factory(your_custom_factory)
|
||||
```
|
||||
|
||||
See SPEC-16 for full context manager refactor details.
|
||||
|
||||
## BASIC MEMORY PRODUCT USAGE
|
||||
|
||||
### Knowledge Structure
|
||||
|
||||
- Entity: Any concept, document, or idea represented as a markdown file
|
||||
- Observation: A categorized fact about an entity (`- [category] content`)
|
||||
- Relation: A directional link between entities (`- relation_type [[Target]]`)
|
||||
- Frontmatter: YAML metadata at the top of markdown files
|
||||
- Knowledge representation follows precise markdown format:
|
||||
- Observations with [category] prefixes
|
||||
- Relations with WikiLinks [[Entity]]
|
||||
- Frontmatter with metadata
|
||||
|
||||
### Basic Memory Commands
|
||||
|
||||
**Local Commands:**
|
||||
- Check sync status: `basic-memory status`
|
||||
- Import from Claude: `basic-memory import claude conversations`
|
||||
- Import from ChatGPT: `basic-memory import chatgpt`
|
||||
- Import from Memory JSON: `basic-memory import memory-json`
|
||||
- Tool access: `basic-memory tool` (provides CLI access to MCP tools)
|
||||
- Continue: `basic-memory tool continue-conversation --topic="search"`
|
||||
|
||||
**Project Management:**
|
||||
- List projects: `basic-memory project list`
|
||||
- Add project: `basic-memory project add "name" ~/path`
|
||||
- Project info: `basic-memory project info`
|
||||
- One-way sync (local -> cloud): `basic-memory project sync`
|
||||
- Bidirectional sync: `basic-memory project bisync`
|
||||
- Integrity check: `basic-memory project check`
|
||||
|
||||
**Cloud Commands (requires subscription):**
|
||||
- Authenticate: `basic-memory cloud login`
|
||||
- Logout: `basic-memory cloud logout`
|
||||
- Check cloud status: `basic-memory cloud status`
|
||||
- Setup cloud sync: `basic-memory cloud setup`
|
||||
- Manage snapshots: `basic-memory cloud snapshot [create|list|delete|show|browse]`
|
||||
- Restore from snapshot: `basic-memory cloud restore <path> --snapshot <id>`
|
||||
|
||||
### MCP Capabilities
|
||||
|
||||
- Basic Memory exposes these MCP tools to LLMs:
|
||||
|
||||
**Content Management:**
|
||||
- `write_note(title, content, folder, tags)` - Create/update markdown notes with semantic observations and relations
|
||||
- `read_note(identifier, page, page_size)` - Read notes by title, permalink, or memory:// URL with knowledge graph awareness
|
||||
- `read_content(path)` - Read raw file content (text, images, binaries) without knowledge graph processing
|
||||
- `view_note(identifier, page, page_size)` - View notes as formatted artifacts for better readability
|
||||
- `edit_note(identifier, operation, content)` - Edit notes incrementally (append, prepend, find/replace, replace_section)
|
||||
- `move_note(identifier, destination_path)` - Move notes to new locations, updating database and maintaining links
|
||||
- `delete_note(identifier)` - Delete notes from the knowledge base
|
||||
|
||||
**Knowledge Graph Navigation:**
|
||||
- `build_context(url, depth, timeframe)` - Navigate the knowledge graph via memory:// URLs for conversation continuity
|
||||
- `recent_activity(type, depth, timeframe)` - Get recently updated information with specified timeframe (e.g., "1d", "1 week")
|
||||
- `list_directory(dir_name, depth, file_name_glob)` - Browse directory contents with filtering and depth control
|
||||
|
||||
**Search & Discovery:**
|
||||
- `search_notes(query, page, page_size, search_type, types, entity_types, after_date)` - Full-text search across all content with advanced filtering options
|
||||
|
||||
**Project Management:**
|
||||
- `list_memory_projects()` - List all available projects with their status
|
||||
- `create_memory_project(project_name, project_path, set_default)` - Create new Basic Memory projects
|
||||
- `delete_project(project_name)` - Delete a project from configuration
|
||||
|
||||
**Visualization:**
|
||||
- `canvas(nodes, edges, title, folder)` - Generate Obsidian canvas files for knowledge graph visualization
|
||||
|
||||
**ChatGPT-Compatible Tools:**
|
||||
- `search(query)` - Search across knowledge base (OpenAI actions compatible)
|
||||
- `fetch(id)` - Fetch full content of a search result document
|
||||
|
||||
- MCP Prompts for better AI interaction:
|
||||
- `ai_assistant_guide()` - Guidance on effectively using Basic Memory tools for AI assistants
|
||||
- `continue_conversation(topic, timeframe)` - Continue previous conversations with relevant historical context
|
||||
- `search(query, after_date)` - Search with detailed, formatted results for better context understanding
|
||||
- `recent_activity(timeframe)` - View recently changed items with formatted output
|
||||
|
||||
### Cloud Features (v0.15.0+)
|
||||
|
||||
Basic Memory now supports cloud synchronization and storage (requires active subscription):
|
||||
|
||||
**Authentication:**
|
||||
- JWT-based authentication with subscription validation
|
||||
- Secure session management with token refresh
|
||||
- Support for multiple cloud projects
|
||||
|
||||
**Bidirectional Sync:**
|
||||
- rclone bisync integration for two-way synchronization
|
||||
- Conflict resolution and integrity verification
|
||||
- Real-time sync with change detection
|
||||
- Mount/unmount cloud storage for direct file access
|
||||
|
||||
**Cloud Project Management:**
|
||||
- Create and manage projects in the cloud
|
||||
- Toggle between local and cloud modes
|
||||
- Per-project sync configuration
|
||||
- Subscription-based access control
|
||||
|
||||
**Security & Performance:**
|
||||
- Removed .env file loading for improved security
|
||||
- .gitignore integration (respects gitignored files)
|
||||
- WAL mode for SQLite performance
|
||||
- Background relation resolution (non-blocking startup)
|
||||
- API performance optimizations (SPEC-11)
|
||||
|
||||
## AI-Human Collaborative Development
|
||||
|
||||
Basic Memory emerged from and enables a new kind of development process that combines human and AI capabilities. Instead
|
||||
of using AI just for code generation, we've developed a true collaborative workflow:
|
||||
|
||||
1. AI (LLM) writes initial implementation based on specifications and context
|
||||
2. Human reviews, runs tests, and commits code with any necessary adjustments
|
||||
3. Knowledge persists across conversations using Basic Memory's knowledge graph
|
||||
4. Development continues seamlessly across different AI sessions with consistent context
|
||||
5. Results improve through iterative collaboration and shared understanding
|
||||
|
||||
This approach has allowed us to tackle more complex challenges and build a more robust system than either humans or AI
|
||||
could achieve independently.
|
||||
|
||||
**Problem-Solving Guidance:**
|
||||
- If a solution isn't working after reasonable effort, suggest alternative approaches
|
||||
- Don't persist with a problematic library or pattern when better alternatives exist
|
||||
- Example: When py-pglite caused cascading test failures, switching to testcontainers-postgres was the right call
|
||||
|
||||
## GitHub Integration
|
||||
|
||||
Basic Memory has taken AI-Human collaboration to the next level by integrating Claude directly into the development workflow through GitHub:
|
||||
|
||||
### GitHub MCP Tools
|
||||
|
||||
Using the GitHub Model Context Protocol server, Claude can now:
|
||||
|
||||
- **Repository Management**:
|
||||
- View repository files and structure
|
||||
- Read file contents
|
||||
- Create new branches
|
||||
- Create and update files
|
||||
|
||||
- **Issue Management**:
|
||||
- Create new issues
|
||||
- Comment on existing issues
|
||||
- Close and update issues
|
||||
- Search across issues
|
||||
|
||||
- **Pull Request Workflow**:
|
||||
- Create pull requests
|
||||
- Review code changes
|
||||
- Add comments to PRs
|
||||
|
||||
This integration enables Claude to participate as a full team member in the development process, not just as a code generation tool. Claude's GitHub account ([bm-claudeai](https://github.com/bm-claudeai)) is a member of the Basic Machines organization with direct contributor access to the codebase.
|
||||
|
||||
### Collaborative Development Process
|
||||
|
||||
With GitHub integration, the development workflow includes:
|
||||
|
||||
1. **Direct code review** - Claude can analyze PRs and provide detailed feedback
|
||||
2. **Contribution tracking** - All of Claude's contributions are properly attributed in the Git history
|
||||
3. **Branch management** - Claude can create feature branches for implementations
|
||||
4. **Documentation maintenance** - Claude can keep documentation updated as the code evolves
|
||||
5. **Code Commits**: ALWAYS sign off commits with `git commit -s`
|
||||
|
||||
This level of integration represents a new paradigm in AI-human collaboration, where the AI assistant becomes a full-fledged team member rather than just a tool for generating code snippets.
|
||||
+494
@@ -0,0 +1,494 @@
|
||||
# Note Format Reference
|
||||
|
||||
Every document in Basic Memory is a plain Markdown file. Files are the source of truth — changes to files automatically update the knowledge graph in the database. You maintain complete ownership, files work with git, and knowledge persists independently of any AI conversation.
|
||||
|
||||
## Document Structure
|
||||
|
||||
A note has three parts: YAML frontmatter, content (observations), and relations.
|
||||
|
||||
```markdown
|
||||
---
|
||||
title: Coffee Brewing Methods
|
||||
type: note
|
||||
tags: [coffee, brewing]
|
||||
permalink: coffee-brewing-methods
|
||||
---
|
||||
|
||||
# Coffee Brewing Methods
|
||||
|
||||
## Observations
|
||||
- [method] Pour over provides more flavor clarity than French press
|
||||
- [technique] Water temperature at 205°F extracts optimal compounds #brewing
|
||||
- [preference] Ethiopian beans work well with lighter roasts (personal experience)
|
||||
|
||||
## Relations
|
||||
- relates_to [[Coffee Bean Origins]]
|
||||
- requires [[Proper Grinding Technique]]
|
||||
- contrasts_with [[Tea Brewing Methods]]
|
||||
```
|
||||
|
||||
The `## Observations` and `## Relations` headings are conventional but not required — the parser detects observations and relations by their syntax patterns anywhere in the document.
|
||||
|
||||
## Frontmatter
|
||||
|
||||
YAML metadata between `---` fences at the top of the file.
|
||||
|
||||
| Field | Required | Default | Description |
|
||||
|-------|----------|---------|-------------|
|
||||
| `title` | No | filename stem | Used for linking and references. Auto-set from filename if missing. |
|
||||
| `type` | No | `note` | Entity type. Used for schema resolution and filtering. |
|
||||
| `tags` | No | `[]` | List or comma-separated string. Used for organization and search. |
|
||||
| `permalink` | No | generated from title | Stable identifier. Persists even if the file moves. |
|
||||
| `schema` | No | none | Schema attachment — dict (inline), string (reference), or omitted (implicit). |
|
||||
|
||||
Custom fields are allowed. Any key not in the standard set is stored as `entity_metadata` and indexed for search and filtering.
|
||||
|
||||
```yaml
|
||||
---
|
||||
title: Paul Graham
|
||||
type: Person
|
||||
tags: [startups, essays, lisp]
|
||||
permalink: paul-graham
|
||||
status: active
|
||||
source: wikipedia
|
||||
---
|
||||
```
|
||||
|
||||
Here `status` and `source` are custom fields stored in `entity_metadata`.
|
||||
|
||||
### Frontmatter Value Handling
|
||||
|
||||
YAML automatically converts some values to native types. Basic Memory normalizes them:
|
||||
|
||||
- Date strings (`2025-10-24`) → kept as ISO format strings
|
||||
- Numbers (`1.0`) → converted to strings
|
||||
- Booleans (`true`) → converted to strings (`"True"`)
|
||||
- Lists and dicts → preserved, items normalized recursively
|
||||
|
||||
This prevents errors when downstream code expects string values.
|
||||
|
||||
## Observations
|
||||
|
||||
An observation is a categorized fact about the entity. Written as a Markdown list item.
|
||||
|
||||
**Syntax:**
|
||||
|
||||
```
|
||||
- [category] content text #tag1 #tag2 (context)
|
||||
```
|
||||
|
||||
| Part | Required | Description |
|
||||
|------|----------|-------------|
|
||||
| `[category]` | Yes | Classification in square brackets. Any text except `[]()` chars. |
|
||||
| content | Yes | The fact or statement. |
|
||||
| `#tags` | No | Inline tags. Space-separated, each starting with `#`. |
|
||||
| `(context)` | No | Parenthesized text at end of line. Supporting details or source. |
|
||||
|
||||
### Examples
|
||||
|
||||
```markdown
|
||||
- [tech] Uses SQLite for storage #database
|
||||
- [design] Follows local-first architecture #architecture
|
||||
- [decision] Selected bcrypt for passwords #security (based on OWASP audit)
|
||||
- [name] Paul Graham
|
||||
- [expertise] Startups
|
||||
- [expertise] Lisp
|
||||
- [expertise] Essay writing
|
||||
```
|
||||
|
||||
Array-like fields use repeated categories — multiple `[expertise]` observations above.
|
||||
|
||||
### What Is Not an Observation
|
||||
|
||||
The parser excludes these list item patterns:
|
||||
|
||||
| Pattern | Example | Reason |
|
||||
|---------|---------|--------|
|
||||
| Checkboxes | `- [ ] Todo item`, `- [x] Done`, `- [-] Cancelled` | Task list syntax |
|
||||
| Markdown links | `- [text](url)` | URL link syntax |
|
||||
| Bare wiki links | `- [[Target]]` | Treated as a relation instead |
|
||||
|
||||
A list item with `#tags` but no `[category]` is still parsed — the tags are extracted and the category defaults to `Note`.
|
||||
|
||||
## Relations
|
||||
|
||||
Relations connect documents to form the knowledge graph. There are two kinds.
|
||||
|
||||
### Explicit Relations
|
||||
|
||||
Written as list items with a relation type and a `[[wiki link]]` target.
|
||||
|
||||
**Syntax:**
|
||||
|
||||
```
|
||||
- relation_type [[Target Entity]] (context)
|
||||
```
|
||||
|
||||
| Part | Required | Description |
|
||||
|------|----------|-------------|
|
||||
| `relation_type` | No | Text before `[[`. Defaults to `relates_to` if omitted. |
|
||||
| `[[Target]]` | Yes | Wiki link to the target entity. Matched by title or permalink. |
|
||||
| `(context)` | No | Parenthesized text after `]]`. Supporting details. |
|
||||
|
||||
### Examples
|
||||
|
||||
```markdown
|
||||
- implements [[Search Design]]
|
||||
- depends_on [[Database Schema]]
|
||||
- works_at [[Y Combinator]] (co-founder)
|
||||
- [[Some Entity]]
|
||||
```
|
||||
|
||||
The last example — a bare `[[wiki link]]` in a list item — gets relation type `relates_to`.
|
||||
|
||||
Common relation types:
|
||||
- `implements`, `depends_on`, `relates_to`, `inspired_by`
|
||||
- `extends`, `part_of`, `contains`, `pairs_with`
|
||||
- `works_at`, `authored`, `collaborated_with`
|
||||
|
||||
Any text works as a relation type. These are conventions, not a fixed set.
|
||||
|
||||
### Inline References
|
||||
|
||||
Wiki links appearing in regular prose (not as list items) create implicit `links_to` relations.
|
||||
|
||||
```markdown
|
||||
This builds on [[Core Design]] and uses [[Utility Functions]].
|
||||
```
|
||||
|
||||
This creates two relations: `links_to [[Core Design]]` and `links_to [[Utility Functions]]`.
|
||||
|
||||
### Forward References
|
||||
|
||||
Relations can link to entities that don't exist yet. Basic Memory resolves them when the target is created.
|
||||
|
||||
## Permalinks and memory:// URLs
|
||||
|
||||
Every document has a unique **permalink** — a stable identifier derived from its title. You can set one explicitly in frontmatter, or let the system generate it.
|
||||
|
||||
```yaml
|
||||
permalink: auth-approaches-2024
|
||||
```
|
||||
|
||||
Permalinks form the basis of `memory://` URLs:
|
||||
|
||||
```
|
||||
memory://auth-approaches-2024 # By permalink
|
||||
memory://Authentication Approaches # By title (auto-resolves)
|
||||
memory://project/auth-approaches # By path
|
||||
```
|
||||
|
||||
Pattern matching is supported:
|
||||
|
||||
```
|
||||
memory://auth* # Starts with "auth"
|
||||
memory://*/approaches # Ends with "approaches"
|
||||
memory://project/*/requirements # Nested wildcard
|
||||
```
|
||||
|
||||
## Schemas
|
||||
|
||||
Schemas declare the expected structure of a note — which observation categories and relation types a well-formed note should have. They use Picoschema, a compact notation from Google's Dotprompt that fits naturally in YAML frontmatter.
|
||||
|
||||
### Picoschema Syntax
|
||||
|
||||
```yaml
|
||||
schema:
|
||||
name: string, full name # required field with description
|
||||
email?: string, contact email # ? = optional
|
||||
role?: string, job title
|
||||
works_at?: Organization, employer # capitalized type = entity reference
|
||||
tags?(array): string, categories # array of type
|
||||
status?(enum): [active, inactive] # enum with allowed values
|
||||
metadata?(object): # nested object
|
||||
updated_at?: string
|
||||
source?: string
|
||||
```
|
||||
|
||||
| Notation | Meaning | Example |
|
||||
|----------|---------|---------|
|
||||
| `field: type` | Required field | `name: string` |
|
||||
| `field?: type` | Optional field | `role?: string` |
|
||||
| `field(array): type` | Array of values | `expertise(array): string` |
|
||||
| `field?(enum): [vals]` | Enum with allowed values | `status?(enum): [active, inactive]` |
|
||||
| `field?(object):` | Nested object with sub-fields | `metadata?(object):` |
|
||||
| `, description` | Description after comma | `name: string, full name` |
|
||||
| `EntityName` | Capitalized type = entity reference | `works_at?: Organization` |
|
||||
|
||||
**Scalar types:** `string`, `integer`, `number`, `boolean`, `any`
|
||||
|
||||
Any type not in that set whose first letter is uppercase is treated as an entity reference (a relation target).
|
||||
|
||||
### Schema-to-Note Mapping
|
||||
|
||||
Schemas validate against existing observation/relation syntax. Note authors don't learn new syntax.
|
||||
|
||||
| Schema Declaration | Maps To | Example in Note |
|
||||
|--------------------|---------|-----------------|
|
||||
| `field: string` | Observation `[field] value` | `- [name] Paul Graham` |
|
||||
| `field?(array): string` | Multiple `[field]` observations | `- [expertise] Lisp` (repeated) |
|
||||
| `field?: EntityType` | Relation `field [[Target]]` | `- works_at [[Y Combinator]]` |
|
||||
| `field?(array): EntityType` | Multiple `field` relations | `- authored [[Book]]` (repeated) |
|
||||
| `tags` | Frontmatter `tags` array | `tags: [startups, essays]` |
|
||||
| `field?(enum): [vals]` | Observation `[field] value` where value is in the set | `- [status] active` |
|
||||
|
||||
Observations and relations not covered by the schema are valid — schemas describe a subset, not a straitjacket.
|
||||
|
||||
### Schema Attachment
|
||||
|
||||
Three ways to attach a schema to a note, resolved in priority order:
|
||||
|
||||
**1. Inline schema** — `schema` is a dict in frontmatter:
|
||||
|
||||
```yaml
|
||||
---
|
||||
title: Team Standup 2024-01-15
|
||||
type: meeting
|
||||
schema:
|
||||
attendees(array): string, who was there
|
||||
decisions(array): string, what was decided
|
||||
action_items(array): string, follow-ups
|
||||
blockers?(array): string, anything stuck
|
||||
---
|
||||
```
|
||||
|
||||
Good for one-off structured notes or prototyping a schema before extracting it.
|
||||
|
||||
**2. Explicit reference** — `schema` is a string naming a schema note:
|
||||
|
||||
```yaml
|
||||
---
|
||||
title: Basic Memory
|
||||
schema: SoftwareProject
|
||||
---
|
||||
```
|
||||
|
||||
or by permalink:
|
||||
|
||||
```yaml
|
||||
---
|
||||
title: LLM Memory Patterns
|
||||
schema: schema/research-project
|
||||
---
|
||||
```
|
||||
|
||||
Use when the note's `type` differs from the schema it should validate against, or when multiple schema variants exist.
|
||||
|
||||
**3. Implicit by type** — no `schema` field, resolved by matching `type`:
|
||||
|
||||
```yaml
|
||||
---
|
||||
title: Paul Graham
|
||||
type: Person
|
||||
---
|
||||
```
|
||||
|
||||
The system looks up a schema note where `entity: Person`. If found, it applies. If not, no validation occurs.
|
||||
|
||||
**4. No schema** — perfectly fine. Most notes don't need one.
|
||||
|
||||
### Schema Notes
|
||||
|
||||
A schema is itself a Basic Memory note with `type: schema`. It lives anywhere (though `schema/` is the conventional directory).
|
||||
|
||||
```yaml
|
||||
# schema/Person.md
|
||||
---
|
||||
title: Person
|
||||
type: schema
|
||||
entity: Person
|
||||
version: 1
|
||||
schema:
|
||||
name: string, full name
|
||||
role?: string, job title or position
|
||||
works_at?: Organization, employer
|
||||
expertise?(array): string, areas of knowledge
|
||||
email?: string, contact email
|
||||
settings:
|
||||
validation: warn
|
||||
---
|
||||
|
||||
# Person
|
||||
|
||||
A human individual in the knowledge graph.
|
||||
```
|
||||
|
||||
| Field | Required | Description |
|
||||
|-------|----------|-------------|
|
||||
| `type` | Yes | Must be `schema` |
|
||||
| `entity` | Yes | The entity type this schema describes (e.g., `Person`) |
|
||||
| `version` | No | Schema version number (default: `1`) |
|
||||
| `schema` | Yes | Picoschema dict defining the fields |
|
||||
| `settings.validation` | No | Validation mode (default: `warn`) |
|
||||
|
||||
Schema notes are regular notes — they show up in search, can have observations and relations, and participate in the knowledge graph.
|
||||
|
||||
### Validation Modes
|
||||
|
||||
| Mode | Behavior |
|
||||
|------|----------|
|
||||
| `warn` | Warnings in output, doesn't block (default) |
|
||||
| `strict` | Errors that block sync, for CI/CD enforcement |
|
||||
| `off` | No validation |
|
||||
|
||||
### Validation Output
|
||||
|
||||
```
|
||||
$ bm schema validate people/ada-lovelace.md
|
||||
|
||||
⚠ Person schema validation:
|
||||
- Missing required field: name (expected [name] observation)
|
||||
- Missing optional field: role
|
||||
- Missing optional field: works_at (no relation found)
|
||||
|
||||
ℹ Unmatched observations: [fact] ×2, [born] ×1
|
||||
ℹ Unmatched relations: collaborated_with
|
||||
```
|
||||
|
||||
"Unmatched" items are informational — observations and relations the schema doesn't cover.
|
||||
|
||||
### Schema Inference
|
||||
|
||||
Generate schemas from existing notes by analyzing observation and relation frequency:
|
||||
|
||||
```
|
||||
$ bm schema infer Person
|
||||
|
||||
Analyzing 30 notes with type: Person...
|
||||
|
||||
Observations found:
|
||||
[name] 30/30 100% → name: string
|
||||
[role] 27/30 90% → role?: string
|
||||
[expertise] 18/30 60% → expertise?(array): string
|
||||
[email] 8/30 27% → email?: string
|
||||
|
||||
Relations found:
|
||||
works_at 22/30 73% → works_at?: Organization
|
||||
|
||||
Suggested schema:
|
||||
name: string, full name
|
||||
role?: string, job title
|
||||
expertise?(array): string, areas of knowledge
|
||||
email?: string, contact email
|
||||
works_at?: Organization, employer
|
||||
|
||||
Save to schema/Person.md? [y/n]
|
||||
```
|
||||
|
||||
Frequency thresholds:
|
||||
- **100% present** → required field
|
||||
- **25%+ present** → optional field
|
||||
- **Below 25%** → excluded from suggestion
|
||||
|
||||
### Schema Drift Detection
|
||||
|
||||
Track how usage patterns shift over time:
|
||||
|
||||
```
|
||||
$ bm schema diff Person
|
||||
|
||||
Schema drift detected:
|
||||
|
||||
+ expertise: now in 81% of notes (was 12%)
|
||||
- department: dropped to 3% of notes
|
||||
~ works_at: cardinality changed (one → many)
|
||||
|
||||
Update schema? [y/n/review]
|
||||
```
|
||||
|
||||
## Complete Examples
|
||||
|
||||
### Simple Note (No Schema)
|
||||
|
||||
```markdown
|
||||
---
|
||||
title: Project Ideas
|
||||
type: note
|
||||
tags: [ideas, brainstorm]
|
||||
---
|
||||
|
||||
# Project Ideas
|
||||
|
||||
## Observations
|
||||
- [idea] Build a CLI tool for markdown linting #tooling
|
||||
- [idea] Create a recipe knowledge base #cooking
|
||||
- [priority] Focus on developer tools first (Q1 goal)
|
||||
|
||||
## Relations
|
||||
- inspired_by [[Developer Workflow Research]]
|
||||
- part_of [[Q1 Planning]]
|
||||
```
|
||||
|
||||
### Schema-Validated Note
|
||||
|
||||
Schema at `schema/Person.md`:
|
||||
|
||||
```yaml
|
||||
---
|
||||
title: Person
|
||||
type: schema
|
||||
entity: Person
|
||||
version: 1
|
||||
schema:
|
||||
name: string, full name
|
||||
role?: string, job title or position
|
||||
works_at?: Organization, employer
|
||||
expertise?(array): string, areas of knowledge
|
||||
email?: string, contact email
|
||||
settings:
|
||||
validation: warn
|
||||
---
|
||||
|
||||
# Person
|
||||
|
||||
A human individual in the knowledge graph.
|
||||
```
|
||||
|
||||
Note at `people/paul-graham.md`:
|
||||
|
||||
```markdown
|
||||
---
|
||||
title: Paul Graham
|
||||
type: Person
|
||||
tags: [startups, essays, lisp]
|
||||
---
|
||||
|
||||
# Paul Graham
|
||||
|
||||
## Observations
|
||||
- [name] Paul Graham
|
||||
- [role] Essayist and investor
|
||||
- [expertise] Startups
|
||||
- [expertise] Lisp
|
||||
- [expertise] Essay writing
|
||||
- [fact] Created Viaweb, the first web app
|
||||
|
||||
## Relations
|
||||
- works_at [[Y Combinator]]
|
||||
- authored [[Hackers and Painters]]
|
||||
```
|
||||
|
||||
The `[fact]` observation and `authored` relation are not in the schema — they're valid, just unmatched. The schema only checks that `[name]` exists (required) and looks for optional fields like `[role]`, `[expertise]`, and `works_at`.
|
||||
|
||||
### Inline Schema Note
|
||||
|
||||
```markdown
|
||||
---
|
||||
title: Team Standup 2024-01-15
|
||||
type: meeting
|
||||
schema:
|
||||
attendees(array): string, who was there
|
||||
decisions(array): string, what was decided
|
||||
action_items(array): string, follow-ups
|
||||
blockers?(array): string, anything stuck
|
||||
---
|
||||
|
||||
# Team Standup 2024-01-15
|
||||
|
||||
## Observations
|
||||
- [attendees] Paul
|
||||
- [attendees] Sarah
|
||||
- [decisions] Ship v2 by Friday
|
||||
- [action_items] Paul to review PR #42
|
||||
- [blockers] Waiting on API credentials
|
||||
```
|
||||
@@ -7,259 +7,232 @@
|
||||

|
||||

|
||||
|
||||
## 🚀 Basic Memory Cloud is Live!
|
||||
## Skip the install — try Basic Memory in the cloud
|
||||
|
||||
- **Cross-device and multi-platform support is here.** Your knowledge graph now works on desktop, web, and mobile - seamlessly synced across all your AI tools (Claude, ChatGPT, Gemini, Claude Code, and Codex)
|
||||
- **Early Supporter Pricing:** Early users get 25% off forever.
|
||||
The open source project continues as always. Cloud just makes it work everywhere.
|
||||
Claude, Codex, or Cursor connected in 30 seconds. No Python, no JSON, no
|
||||
terminal. **$14.25/mo locked in for life** (regular price $19). 7-day free
|
||||
trial — cancel any time before day 7 if it's not for you. Beta pricing —
|
||||
sign up now and your rate never goes up. OSS users: code `BMFOSS` takes
|
||||
another 20% off for 3 months.
|
||||
|
||||
[Sign up now →](https://basicmemory.com)
|
||||
[Start free trial →](https://basicmemory.com?utm_source=github&utm_medium=referral&utm_campaign=readme&utm_content=banner)
|
||||
|
||||
with a 7 day free trial
|
||||
---
|
||||
|
||||
# Basic Memory
|
||||
|
||||
Basic Memory lets you build persistent knowledge through natural conversations with Large Language Models (LLMs) like
|
||||
Claude, while keeping everything in simple Markdown files on your computer. It uses the Model Context Protocol (MCP) to
|
||||
enable any compatible LLM to read and write to your local knowledge base.
|
||||
### Your AI never forgets again.
|
||||
|
||||
- Website: https://basicmemory.com
|
||||
- Documentation: https://docs.basicmemory.com
|
||||
Pick up right where you left off — in Claude, Codex, Cursor, ChatGPT, or
|
||||
anything that speaks [MCP](https://modelcontextprotocol.io). Your knowledge
|
||||
lives as Markdown files that both you and your AI can read, write, and
|
||||
search.
|
||||
|
||||
## Pick up your conversation right where you left off
|
||||
- **Local-first.** Plain text on your disk. Forever.
|
||||
- **Two-way.** AI and humans write to the same files; sync keeps them in step.
|
||||
- **A real knowledge graph.** Observations and wikilinks compound into context.
|
||||
- **Semantic search.** Find notes by meaning, not just keywords.
|
||||
- **MCP-native.** Works with every major AI client and IDE.
|
||||
- **Progressive tool discovery.** Every tool is tagged with behavior hints
|
||||
(read-only, destructive, idempotent) so agents pick the right tool on
|
||||
demand — no wasted context trying things to see what they do.
|
||||
- **Cloud, optional.** Sync across devices when you want — never required.
|
||||
|
||||
- AI assistants can load context from local files in a new conversation
|
||||
- Notes are saved locally as Markdown files in real time
|
||||
- No project knowledge or special prompting required
|
||||
## Get started
|
||||
|
||||
Pick the path that fits you. Both run the same product on the same Markdown.
|
||||
|
||||
<table>
|
||||
<tr>
|
||||
<th width="50%">☁️ Cloud</th>
|
||||
<th width="50%">💻 Local install</th>
|
||||
</tr>
|
||||
<tr>
|
||||
<td valign="top">
|
||||
|
||||
**30 seconds.** Sign up, connect your AI client, done.
|
||||
|
||||
- Works in any browser
|
||||
- Mobile, web, desktop
|
||||
- Cross-device sync built in
|
||||
- We handle hosting, backups, snapshots
|
||||
|
||||
**$14.25/mo locked for life** · 7-day free trial · cancel any time
|
||||
|
||||
[**Start free trial →**](https://basicmemory.com?utm_source=github&utm_medium=referral&utm_campaign=readme&utm_content=quickstart)
|
||||
|
||||
</td>
|
||||
<td valign="top">
|
||||
|
||||
**2 minutes.** Install, configure your AI client, run.
|
||||
|
||||
- Free forever (AGPL-3.0)
|
||||
- All data on your disk
|
||||
- Air-gapped friendly
|
||||
- Requires Python via [`uv`](https://docs.astral.sh/uv/)
|
||||
|
||||
```bash
|
||||
uv tool install basic-memory
|
||||
```
|
||||
|
||||
[**Configure your client ↓**](#connect-your-ai-client)
|
||||
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
## What people are saying
|
||||
|
||||
> Basic Memory changed my whole relationship with LLMs. I switched from GPT
|
||||
> and Gemini to exclusively Claude and Claude Code because of this
|
||||
> integration and am completely revamping all our company's processes around
|
||||
> a Basic Memory workflow.
|
||||
>
|
||||
> — **Alex**, TrainerDay
|
||||
|
||||
> Basic Memory is the missing 'wow' factor in AI chatbots. Now I can't
|
||||
> imagine Claude or Claude Code without it.
|
||||
>
|
||||
> — **Caleb**, Caleb Picker Consulting
|
||||
|
||||
> I don't code without Basic Memory anymore. It's such a time saver to be
|
||||
> able to refer to projects I don't currently have active and keep a running
|
||||
> log of all my learnings and ProTips.
|
||||
>
|
||||
> — **@groksrc**, Developer
|
||||
|
||||
More on [basicmemory.com](https://basicmemory.com?utm_source=github&utm_medium=referral&utm_campaign=readme).
|
||||
|
||||
## Basic Memory Cloud
|
||||
|
||||
The hosted version of Basic Memory. Same product, same Markdown files, same
|
||||
MCP tools — we just host the database, run the sync, and put it on your
|
||||
phone.
|
||||
|
||||
### What you get
|
||||
|
||||
- **Every device, same brain.** Your knowledge graph on web, mobile, and
|
||||
desktop. No copy-paste between machines.
|
||||
- **Connect any MCP client.** Claude Desktop, Claude Code, Codex, Cursor,
|
||||
ChatGPT (Custom GPTs), VS Code — one-click connect from the web app.
|
||||
- **Bidirectional sync to local.** Edit on your phone, see it in Obsidian on
|
||||
your laptop. rclone-powered with conflict resolution.
|
||||
- **Snapshots and backups.** Point-in-time restore. Browse history. Never
|
||||
lose a note.
|
||||
- **No lock-in.** Your notes are plain Markdown. Export to local Markdown any
|
||||
time — same files, same format, same wikilinks. Cancel anytime, your data
|
||||
stays yours.
|
||||
|
||||
Built on WorkOS AuthKit, Neon Postgres, and Tigris S3. Platform source at
|
||||
[basic-memory-cloud](https://github.com/basicmachines-co/basic-memory-cloud).
|
||||
|
||||
### Pricing
|
||||
|
||||
**$14.25/mo, locked in for the life of your subscription** (regular price
|
||||
$19). Sign up during beta and the rate never goes up — as long as you stay
|
||||
subscribed, you keep the price. One plan, no tiers, no surprise upgrades.
|
||||
Unlimited notes, unlimited projects, every feature.
|
||||
|
||||
- 7-day free trial. Cancel any time before day 7 if it's not for you.
|
||||
- Cancel anytime after that too — export your notes whenever you want.
|
||||
- OSS users: code `BMFOSS` for another 20% off for 3 months (~$11.40/mo).
|
||||
|
||||
[**Start your 7-day free trial →**](https://basicmemory.com?utm_source=github&utm_medium=referral&utm_campaign=readme&utm_content=cloud-section)
|
||||
|
||||
## Cloud vs. local
|
||||
|
||||
| | Cloud | Local |
|
||||
|---|---|---|
|
||||
| **Setup time** | 30 seconds | 2 minutes (requires Python) |
|
||||
| **Cost** | $14.25/mo, locked for life (7-day trial) | Free |
|
||||
| **Storage** | We host (Tigris S3) | Your disk |
|
||||
| **Cross-device sync** | Built in | Manual (Git, Syncthing, etc.) |
|
||||
| **Mobile access** | Yes (web + app) | No |
|
||||
| **Air-gapped** | No | Yes |
|
||||
| **Your data stays yours** | Yes — export anytime | Yes — already there |
|
||||
| **Source code** | AGPL-3.0 | AGPL-3.0 |
|
||||
| **Snapshots & backups** | Built in | Roll your own |
|
||||
|
||||
Both paths use the same OSS engine and the same Markdown files. There's no
|
||||
lock-in either way — flip between them when your needs change.
|
||||
|
||||
## Works with the tools you already use
|
||||
|
||||
| Client | Transport | Notes |
|
||||
|---|---|---|
|
||||
| Cloud web app | https | Sign in at basicmemory.com — no install |
|
||||
| [Claude Desktop](#claude-desktop) | stdio/https | macOS / Windows / Linux |
|
||||
| [Claude Code](#claude-code) | stdio/https | `claude mcp add` |
|
||||
| [Codex](#codex-cli) | stdio/https | OpenAI's coding agent |
|
||||
| [Cursor](#cursor) | stdio/https | `.cursor/mcp.json` |
|
||||
| [VS Code](#vs-code) | stdio/https | Native MCP support |
|
||||
| [ChatGPT](#chatgpt) | https | Custom GPT actions (`search` / `fetch`) |
|
||||
| [Obsidian](#obsidian) | — | Reads/writes the same Markdown directly |
|
||||
| Anything MCP | stdio/https | If it speaks MCP, it works |
|
||||
|
||||
## Pick up where you left off
|
||||
|
||||
https://github.com/user-attachments/assets/a55d8238-8dd0-454a-be4c-8860dbbd0ddc
|
||||
|
||||
## Quick Start
|
||||
## Connect your AI client
|
||||
|
||||
```bash
|
||||
# Install with uv (recommended)
|
||||
uv tool install basic-memory
|
||||
If you went the [Cloud](#get-started) route, the web app walks you through
|
||||
client connect. The snippets below are for local installs.
|
||||
|
||||
# Configure Claude Desktop (edit ~/Library/Application Support/Claude/claude_desktop_config.json)
|
||||
# Add this to your config:
|
||||
### Claude Desktop
|
||||
|
||||
Edit `~/Library/Application Support/Claude/claude_desktop_config.json`:
|
||||
|
||||
```json
|
||||
{
|
||||
"mcpServers": {
|
||||
"basic-memory": {
|
||||
"command": "uvx",
|
||||
"args": [
|
||||
"basic-memory",
|
||||
"mcp"
|
||||
]
|
||||
"args": ["basic-memory", "mcp"]
|
||||
}
|
||||
}
|
||||
}
|
||||
# Now in Claude Desktop, you can:
|
||||
# - Write notes with "Create a note about coffee brewing methods"
|
||||
# - Read notes with "What do I know about pour over coffee?"
|
||||
# - Search with "Find information about Ethiopian beans"
|
||||
|
||||
```
|
||||
|
||||
You can view shared context via files in `~/basic-memory` (default directory location).
|
||||
Restart Claude Desktop. Notes live in `~/basic-memory` by default.
|
||||
|
||||
## Why Basic Memory?
|
||||
<details>
|
||||
<summary><b>Claude Code, Codex CLI, Cursor, VS Code, ChatGPT, Obsidian</b></summary>
|
||||
|
||||
Most LLM interactions are ephemeral - you ask a question, get an answer, and everything is forgotten. Each conversation
|
||||
starts fresh, without the context or knowledge from previous ones. Current workarounds have limitations:
|
||||
### Claude Code
|
||||
|
||||
- Chat histories capture conversations but aren't structured knowledge
|
||||
- RAG systems can query documents but don't let LLMs write back
|
||||
- Vector databases require complex setups and often live in the cloud
|
||||
- Knowledge graphs typically need specialized tools to maintain
|
||||
|
||||
Basic Memory addresses these problems with a simple approach: structured Markdown files that both humans and LLMs can
|
||||
read
|
||||
and write to. The key advantages:
|
||||
|
||||
- **Local-first:** All knowledge stays in files you control
|
||||
- **Bi-directional:** Both you and the LLM read and write to the same files
|
||||
- **Structured yet simple:** Uses familiar Markdown with semantic patterns
|
||||
- **Traversable knowledge graph:** LLMs can follow links between topics
|
||||
- **Standard formats:** Works with existing editors like Obsidian
|
||||
- **Lightweight infrastructure:** Just local files indexed in a local SQLite database
|
||||
|
||||
With Basic Memory, you can:
|
||||
|
||||
- Have conversations that build on previous knowledge
|
||||
- Create structured notes during natural conversations
|
||||
- Have conversations with LLMs that remember what you've discussed before
|
||||
- Navigate your knowledge graph semantically
|
||||
- Keep everything local and under your control
|
||||
- Use familiar tools like Obsidian to view and edit notes
|
||||
- Build a personal knowledge base that grows over time
|
||||
- Sync your knowledge to the cloud with bidirectional synchronization
|
||||
- Authenticate and manage cloud projects with subscription validation
|
||||
- Mount cloud storage for direct file access
|
||||
|
||||
## How It Works in Practice
|
||||
|
||||
Let's say you're exploring coffee brewing methods and want to capture your knowledge. Here's how it works:
|
||||
|
||||
1. Start by chatting normally:
|
||||
|
||||
```
|
||||
I've been experimenting with different coffee brewing methods. Key things I've learned:
|
||||
|
||||
- Pour over gives more clarity in flavor than French press
|
||||
- Water temperature is critical - around 205°F seems best
|
||||
- Freshly ground beans make a huge difference
|
||||
```bash
|
||||
claude mcp add basic-memory -- uvx basic-memory mcp
|
||||
```
|
||||
|
||||
... continue conversation.
|
||||
### Codex CLI
|
||||
|
||||
2. Ask the LLM to help structure this knowledge:
|
||||
Add to `~/.codex/config.toml`:
|
||||
|
||||
```
|
||||
"Let's write a note about coffee brewing methods."
|
||||
```toml
|
||||
[mcp_servers.basic-memory]
|
||||
command = "uvx"
|
||||
args = ["basic-memory", "mcp"]
|
||||
```
|
||||
|
||||
LLM creates a new Markdown file on your system (which you can see instantly in Obsidian or your editor):
|
||||
### Cursor
|
||||
|
||||
```markdown
|
||||
---
|
||||
title: Coffee Brewing Methods
|
||||
permalink: coffee-brewing-methods
|
||||
tags:
|
||||
- coffee
|
||||
- brewing
|
||||
---
|
||||
Add to `.cursor/mcp.json` (project) or `~/.cursor/mcp.json` (global):
|
||||
|
||||
# Coffee Brewing Methods
|
||||
|
||||
## Observations
|
||||
|
||||
- [method] Pour over provides more clarity and highlights subtle flavors
|
||||
- [technique] Water temperature at 205°F (96°C) extracts optimal compounds
|
||||
- [principle] Freshly ground beans preserve aromatics and flavor
|
||||
|
||||
## Relations
|
||||
|
||||
- relates_to [[Coffee Bean Origins]]
|
||||
- requires [[Proper Grinding Technique]]
|
||||
- affects [[Flavor Extraction]]
|
||||
```json
|
||||
{
|
||||
"mcpServers": {
|
||||
"basic-memory": {
|
||||
"command": "uvx",
|
||||
"args": ["basic-memory", "mcp"]
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
The note embeds semantic content and links to other topics via simple Markdown formatting.
|
||||
### VS Code
|
||||
|
||||
3. You see this file on your computer in real time in the current project directory (default `~/$HOME/basic-memory`).
|
||||
|
||||
- Realtime sync can be enabled via running `basic-memory sync --watch`
|
||||
|
||||
4. In a chat with the LLM, you can reference a topic:
|
||||
|
||||
```
|
||||
Look at `coffee-brewing-methods` for context about pour over coffee
|
||||
```
|
||||
|
||||
The LLM can now build rich context from the knowledge graph. For example:
|
||||
|
||||
```
|
||||
Following relation 'relates_to [[Coffee Bean Origins]]':
|
||||
- Found information about Ethiopian Yirgacheffe
|
||||
- Notes on Colombian beans' nutty profile
|
||||
- Altitude effects on bean characteristics
|
||||
|
||||
Following relation 'requires [[Proper Grinding Technique]]':
|
||||
- Burr vs. blade grinder comparisons
|
||||
- Grind size recommendations for different methods
|
||||
- Impact of consistent particle size on extraction
|
||||
```
|
||||
|
||||
Each related document can lead to more context, building a rich semantic understanding of your knowledge base.
|
||||
|
||||
This creates a two-way flow where:
|
||||
|
||||
- Humans write and edit Markdown files
|
||||
- LLMs read and write through the MCP protocol
|
||||
- Sync keeps everything consistent
|
||||
- All knowledge stays in local files.
|
||||
|
||||
## Technical Implementation
|
||||
|
||||
Under the hood, Basic Memory:
|
||||
|
||||
1. Stores everything in Markdown files
|
||||
2. Uses a SQLite database for searching and indexing
|
||||
3. Extracts semantic meaning from simple Markdown patterns
|
||||
- Files become `Entity` objects
|
||||
- Each `Entity` can have `Observations`, or facts associated with it
|
||||
- `Relations` connect entities together to form the knowledge graph
|
||||
4. Maintains the local knowledge graph derived from the files
|
||||
5. Provides bidirectional synchronization between files and the knowledge graph
|
||||
6. Implements the Model Context Protocol (MCP) for AI integration
|
||||
7. Exposes tools that let AI assistants traverse and manipulate the knowledge graph
|
||||
8. Uses memory:// URLs to reference entities across tools and conversations
|
||||
|
||||
The file format is just Markdown with some simple markup:
|
||||
|
||||
Each Markdown file has:
|
||||
|
||||
### Frontmatter
|
||||
|
||||
```markdown
|
||||
title: <Entity title>
|
||||
type: <The type of Entity> (e.g. note)
|
||||
permalink: <a uri slug>
|
||||
|
||||
- <optional metadata> (such as tags)
|
||||
```
|
||||
|
||||
### Observations
|
||||
|
||||
Observations are facts about a topic.
|
||||
They can be added by creating a Markdown list with a special format that can reference a `category`, `tags` using a
|
||||
"#" character, and an optional `context`.
|
||||
|
||||
Observation Markdown format:
|
||||
|
||||
```markdown
|
||||
- [category] content #tag (optional context)
|
||||
```
|
||||
|
||||
Examples of observations:
|
||||
|
||||
```markdown
|
||||
- [method] Pour over extracts more floral notes than French press
|
||||
- [tip] Grind size should be medium-fine for pour over #brewing
|
||||
- [preference] Ethiopian beans have bright, fruity flavors (especially from Yirgacheffe)
|
||||
- [fact] Lighter roasts generally contain more caffeine than dark roasts
|
||||
- [experiment] Tried 1:15 coffee-to-water ratio with good results
|
||||
- [resource] James Hoffman's V60 technique on YouTube is excellent
|
||||
- [question] Does water temperature affect extraction of different compounds differently?
|
||||
- [note] My favorite local shop uses a 30-second bloom time
|
||||
```
|
||||
|
||||
### Relations
|
||||
|
||||
Relations are links to other topics. They define how entities connect in the knowledge graph.
|
||||
|
||||
Markdown format:
|
||||
|
||||
```markdown
|
||||
- relation_type [[WikiLink]] (optional context)
|
||||
```
|
||||
|
||||
Examples of relations:
|
||||
|
||||
```markdown
|
||||
- pairs_well_with [[Chocolate Desserts]]
|
||||
- grown_in [[Ethiopia]]
|
||||
- contrasts_with [[Tea Brewing Methods]]
|
||||
- requires [[Burr Grinder]]
|
||||
- improves_with [[Fresh Beans]]
|
||||
- relates_to [[Morning Routine]]
|
||||
- inspired_by [[Japanese Coffee Culture]]
|
||||
- documented_in [[Coffee Journal]]
|
||||
```
|
||||
|
||||
## Using with VS Code
|
||||
|
||||
Add the following JSON block to your User Settings (JSON) file in VS Code. You can do this by pressing `Ctrl + Shift + P` and typing `Preferences: Open User Settings (JSON)`.
|
||||
Add to your User Settings (JSON):
|
||||
|
||||
```json
|
||||
{
|
||||
@@ -274,238 +247,301 @@ Add the following JSON block to your User Settings (JSON) file in VS Code. You c
|
||||
}
|
||||
```
|
||||
|
||||
Optionally, you can add it to a file called `.vscode/mcp.json` in your workspace. This will allow you to share the configuration with others.
|
||||
### ChatGPT
|
||||
|
||||
```json
|
||||
{
|
||||
"servers": {
|
||||
"basic-memory": {
|
||||
"command": "uvx",
|
||||
"args": ["basic-memory", "mcp"]
|
||||
}
|
||||
}
|
||||
}
|
||||
Basic Memory exposes OpenAI-compatible `search` and `fetch` tools for Custom
|
||||
GPT actions. See the [ChatGPT integration
|
||||
guide](https://docs.basicmemory.com/integrations/chatgpt/?utm_source=github&utm_medium=referral&utm_campaign=readme).
|
||||
|
||||
### Obsidian
|
||||
|
||||
No setup. Point Obsidian at `~/basic-memory` (or your project folder) and the
|
||||
same wikilinks, frontmatter, and Markdown your AI writes appear in your graph
|
||||
view. Edit either side — sync handles the rest.
|
||||
|
||||
</details>
|
||||
|
||||
Try a prompt:
|
||||
|
||||
```
|
||||
"Create a note about our project architecture decisions."
|
||||
"Find information about JWT auth in my notes."
|
||||
"What have I been working on this week?"
|
||||
```
|
||||
|
||||
You can use Basic Memory with VS Code to easily retrieve and store information while coding.
|
||||
## What's New
|
||||
|
||||
## Using with Claude Desktop
|
||||
- **Automatic updates.** Basic Memory keeps itself up to date for `uv tool`
|
||||
and Homebrew installs; `bm update` triggers a manual check.
|
||||
- **Semantic vector search.** Find notes by meaning, not just keywords.
|
||||
Hybrid full-text + vector ranking with FastEmbed embeddings, on SQLite or
|
||||
Postgres.
|
||||
- **Schema system.** Infer, validate, and diff the structure of your
|
||||
knowledge base with `schema_infer`, `schema_validate`, `schema_diff`.
|
||||
- **Per-project cloud routing.** Route individual projects through the cloud
|
||||
while others stay local, via API key (`bm project set-cloud`).
|
||||
- **Smarter editing.** `edit_note` append/prepend auto-creates notes when
|
||||
missing; `write_note` guards against accidental overwrites.
|
||||
- **Richer search results.** Matched chunk text is included so the LLM gets
|
||||
context, not just hits.
|
||||
- **FastMCP 3.0 + tool annotations.** Every tool ships with MCP behavior
|
||||
hints (`readOnlyHint`, `destructiveHint`, `idempotentHint`,
|
||||
`openWorldHint`) so agents can discover capabilities progressively at
|
||||
runtime instead of guessing or burning tokens.
|
||||
- **CLI overhaul.** `--json` output for scripting, workspace-aware commands,
|
||||
and an htop-inspired project dashboard.
|
||||
|
||||
Basic Memory is built using the MCP (Model Context Protocol) and works with the Claude desktop app (https://claude.ai/):
|
||||
Full [CHANGELOG](CHANGELOG.md) for v0.18 → v0.20.
|
||||
|
||||
1. Configure Claude Desktop to use Basic Memory:
|
||||
## Why Basic Memory
|
||||
|
||||
Edit your MCP configuration file (usually located at `~/Library/Application Support/Claude/claude_desktop_config.json`
|
||||
for OS X):
|
||||
Most LLM conversations are ephemeral. You ask a question, get an answer, then
|
||||
everything is forgotten. Workarounds have limits:
|
||||
|
||||
```json
|
||||
{
|
||||
"mcpServers": {
|
||||
"basic-memory": {
|
||||
"command": "uvx",
|
||||
"args": [
|
||||
"basic-memory",
|
||||
"mcp"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
- **Chat history** captures conversations but isn't structured knowledge.
|
||||
- **RAG** lets the LLM query your documents but not write back to them.
|
||||
- **Vector DBs** need complex infra and usually live in someone else's cloud.
|
||||
- **Knowledge graphs** need specialized tooling to maintain.
|
||||
|
||||
Basic Memory takes a simpler path: **structured Markdown files that humans
|
||||
and LLMs both read and write.**
|
||||
|
||||
- All knowledge stays in plain files you control.
|
||||
- Both sides read and write to the same files.
|
||||
- Familiar Markdown with semantic patterns — no new format to learn.
|
||||
- A traversable graph the LLM can follow link by link.
|
||||
- Works with the editors you already use (Obsidian, VS Code, anything).
|
||||
- Just files plus a local SQLite index. No servers required.
|
||||
|
||||
## How it works
|
||||
|
||||
You're chatting normally about coffee:
|
||||
|
||||
> I've been experimenting with brewing methods. Pour over gives more clarity
|
||||
> than French press, water at 205°F seems best, and freshly ground beans
|
||||
> make a huge difference.
|
||||
|
||||
Ask the LLM to capture it:
|
||||
|
||||
> "Make a note on coffee brewing methods."
|
||||
|
||||
A Markdown file appears in your project directory in real time:
|
||||
|
||||
```markdown
|
||||
---
|
||||
title: Coffee Brewing Methods
|
||||
permalink: coffee-brewing-methods
|
||||
tags: [coffee, brewing]
|
||||
---
|
||||
|
||||
# Coffee Brewing Methods
|
||||
|
||||
## Observations
|
||||
- [method] Pour over highlights subtle flavors over body
|
||||
- [technique] Water at 205°F (96°C) extracts optimal compounds
|
||||
- [principle] Freshly ground beans preserve aromatics
|
||||
|
||||
## Relations
|
||||
- relates_to [[Coffee Bean Origins]]
|
||||
- requires [[Proper Grinding Technique]]
|
||||
- affects [[Flavor Extraction]]
|
||||
```
|
||||
|
||||
If you want to use a specific project (see [Multiple Projects](#multiple-projects) below), update your Claude Desktop
|
||||
config:
|
||||
Next session, the LLM picks up the thread. It follows the relations to
|
||||
surface what you already know about Ethiopian beans and burr grinders, and
|
||||
builds on it instead of starting over. You see the same files in Obsidian or
|
||||
your editor. Edit them by hand — the AI sees your changes too.
|
||||
|
||||
```json
|
||||
{
|
||||
"mcpServers": {
|
||||
"basic-memory": {
|
||||
"command": "uvx",
|
||||
"args": [
|
||||
"basic-memory",
|
||||
"mcp",
|
||||
"--project",
|
||||
"your-project-name"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
Real two-way flow: humans edit Markdown, LLMs read/write through MCP, sync
|
||||
keeps everything consistent, and the source of truth is always your files.
|
||||
|
||||
## The Markdown format
|
||||
|
||||
Each file is an `Entity`. Entities have `Observations` (facts about them) and
|
||||
`Relations` (links to other entities). That's the whole grammar.
|
||||
|
||||
### Frontmatter
|
||||
|
||||
```markdown
|
||||
---
|
||||
title: <Entity title>
|
||||
type: note
|
||||
permalink: <uri-slug>
|
||||
tags: [optional, list]
|
||||
---
|
||||
```
|
||||
|
||||
2. Sync your knowledge:
|
||||
### Observations
|
||||
|
||||
Facts about the entity. Categories in `[brackets]`, tags with `#`, optional
|
||||
context in parens.
|
||||
|
||||
```markdown
|
||||
- [method] Pour over highlights subtle flavors
|
||||
- [tip] Grind medium-fine for V60 #brewing
|
||||
- [fact] Lighter roasts contain more caffeine than dark
|
||||
- [resource] James Hoffmann's V60 technique on YouTube
|
||||
- [question] How does temperature affect compound extraction?
|
||||
```
|
||||
|
||||
### Relations
|
||||
|
||||
Wiki-style links that form the graph. Single-token relation types, or quote
|
||||
multi-word ones.
|
||||
|
||||
```markdown
|
||||
- pairs_well_with [[Chocolate Desserts]]
|
||||
- grown_in [[Ethiopia]]
|
||||
- requires [[Burr Grinder]]
|
||||
- "pairs well with" [[Dark Chocolate]]
|
||||
```
|
||||
|
||||
Bare `- [[Target]]` and prose `- Worth checking out [[Target]]` index as
|
||||
`links_to`. Full reference in the
|
||||
[docs](https://docs.basicmemory.com/getting-started/note-formatting/?utm_source=github&utm_medium=referral&utm_campaign=readme).
|
||||
|
||||
## MCP tools
|
||||
|
||||
Basic Memory exposes these tools to any MCP client. Every tool is annotated
|
||||
with MCP behavior hints (read-only, destructive, idempotent, open-world) so
|
||||
agents can pick the right one without trial-and-error:
|
||||
|
||||
- **Content:** `write_note`, `read_note`, `edit_note`, `move_note`,
|
||||
`delete_note`, `read_content`, `view_note`
|
||||
- **Search & discovery:** `search`, `search_notes`, `recent_activity`,
|
||||
`list_directory`
|
||||
- **Knowledge graph:** `build_context` (navigates `memory://` URLs),
|
||||
`canvas` (Obsidian canvas generation)
|
||||
- **Projects:** `list_memory_projects`, `create_memory_project`,
|
||||
`get_current_project`, `sync_status`
|
||||
- **Schema:** `schema_infer`, `schema_validate`, `schema_diff`
|
||||
- **Cloud:** `cloud_info`, `release_notes`
|
||||
|
||||
All MCP tools default to text output; pass `output_format="json"` for
|
||||
structured responses. Full tool reference in the
|
||||
[docs](https://docs.basicmemory.com/?utm_source=github&utm_medium=referral&utm_campaign=readme).
|
||||
|
||||
## CLI essentials
|
||||
|
||||
```bash
|
||||
# One-time sync of local knowledge updates
|
||||
basic-memory sync
|
||||
# Projects
|
||||
basic-memory project list
|
||||
basic-memory project add research ~/research
|
||||
basic-memory project set-cloud research # route through cloud
|
||||
basic-memory project set-local research # revert
|
||||
|
||||
# Run realtime sync process (recommended)
|
||||
basic-memory sync --watch
|
||||
# Health & maintenance
|
||||
basic-memory status
|
||||
basic-memory doctor # file <-> DB consistency check
|
||||
basic-memory tool edit-note ... # CLI access to MCP tools
|
||||
basic-memory update # check for and install updates
|
||||
|
||||
# Imports
|
||||
basic-memory import claude conversations
|
||||
basic-memory import chatgpt
|
||||
basic-memory import memory-json
|
||||
```
|
||||
|
||||
3. Cloud features (optional, requires subscription):
|
||||
Routing flags (`--local` / `--cloud`) force a target when you're in mixed
|
||||
mode. Full CLI reference in the
|
||||
[docs](https://docs.basicmemory.com/guides/cli-reference/?utm_source=github&utm_medium=referral&utm_campaign=readme).
|
||||
|
||||
## Auto-updates
|
||||
|
||||
CLI installs check for updates every 24 hours by default and apply them
|
||||
silently (so the MCP server keeps responding).
|
||||
|
||||
- Supported install sources: `uv tool`, Homebrew
|
||||
- Skipped for `uvx` (ephemeral runtime managed by uv)
|
||||
- Manual: `bm update` (check + apply) or `bm update --check` (check only)
|
||||
|
||||
Disable in `~/.basic-memory/config.json`:
|
||||
|
||||
```json
|
||||
{ "auto_update": false }
|
||||
```
|
||||
|
||||
## Telemetry
|
||||
|
||||
Minimal, anonymous events to understand the CLI-to-cloud conversion funnel.
|
||||
|
||||
**What we collect:** cloud promo impressions, cloud login attempts and
|
||||
outcomes, promo opt-out events.
|
||||
|
||||
**What we don't:** file contents, note titles, knowledge base data, PII, IP
|
||||
addresses, per-command or per-tool tracking.
|
||||
|
||||
Events go to our [Umami Cloud](https://umami.is) instance (open-source,
|
||||
privacy-focused) on a background thread — never blocks the CLI.
|
||||
|
||||
Opt out:
|
||||
|
||||
```bash
|
||||
# Authenticate with cloud
|
||||
basic-memory cloud login
|
||||
|
||||
# Bidirectional sync with cloud
|
||||
basic-memory cloud sync
|
||||
|
||||
# Verify cloud integrity
|
||||
basic-memory cloud check
|
||||
|
||||
# Mount cloud storage
|
||||
basic-memory cloud mount
|
||||
export BASIC_MEMORY_NO_PROMOS=1
|
||||
```
|
||||
|
||||
4. In Claude Desktop, the LLM can now use these tools:
|
||||
|
||||
**Content Management:**
|
||||
```
|
||||
write_note(title, content, folder, tags) - Create or update notes
|
||||
read_note(identifier, page, page_size) - Read notes by title or permalink
|
||||
read_content(path) - Read raw file content (text, images, binaries)
|
||||
view_note(identifier) - View notes as formatted artifacts
|
||||
edit_note(identifier, operation, content) - Edit notes incrementally
|
||||
move_note(identifier, destination_path) - Move notes with database consistency
|
||||
delete_note(identifier) - Delete notes from knowledge base
|
||||
```
|
||||
|
||||
**Knowledge Graph Navigation:**
|
||||
```
|
||||
build_context(url, depth, timeframe) - Navigate knowledge graph via memory:// URLs
|
||||
recent_activity(type, depth, timeframe) - Find recently updated information
|
||||
list_directory(dir_name, depth) - Browse directory contents with filtering
|
||||
```
|
||||
|
||||
**Search & Discovery:**
|
||||
```
|
||||
search(query, page, page_size) - Search across your knowledge base
|
||||
```
|
||||
|
||||
**Project Management:**
|
||||
```
|
||||
list_memory_projects() - List all available projects
|
||||
create_memory_project(project_name, project_path) - Create new projects
|
||||
get_current_project() - Show current project stats
|
||||
sync_status() - Check synchronization status
|
||||
```
|
||||
|
||||
**Visualization:**
|
||||
```
|
||||
canvas(nodes, edges, title, folder) - Generate knowledge visualizations
|
||||
```
|
||||
|
||||
5. Example prompts to try:
|
||||
|
||||
```
|
||||
"Create a note about our project architecture decisions"
|
||||
"Find information about JWT authentication in my notes"
|
||||
"Create a canvas visualization of my project components"
|
||||
"Read my notes on the authentication system"
|
||||
"What have I been working on in the past week?"
|
||||
```
|
||||
|
||||
## Futher info
|
||||
|
||||
See the [Documentation](https://docs.basicmemory.com) for more info, including:
|
||||
|
||||
- [Complete User Guide](https://docs.basicmemory.com/user-guide/)
|
||||
- [CLI tools](https://docs.basicmemory.com/guides/cli-reference/)
|
||||
- [Cloud CLI and Sync](https://docs.basicmemory.com/guides/cloud-cli/)
|
||||
- [Managing multiple Projects](https://docs.basicmemory.com/guides/cli-reference/#project)
|
||||
- [Importing data from OpenAI/Claude Projects](https://docs.basicmemory.com/guides/cli-reference/#import)
|
||||
This disables promos and all telemetry.
|
||||
|
||||
## Logging
|
||||
|
||||
Basic Memory uses [Loguru](https://github.com/Delgan/loguru) for logging. The logging behavior varies by entry point:
|
||||
Basic Memory uses [Loguru](https://github.com/Delgan/loguru). Defaults vary
|
||||
by entry point:
|
||||
|
||||
| Entry Point | Default Behavior | Use Case |
|
||||
|-------------|------------------|----------|
|
||||
| CLI commands | File only | Prevents log output from interfering with command output |
|
||||
| MCP server | File only | Stdout would corrupt the JSON-RPC protocol |
|
||||
| API server | File (local) or stdout (cloud) | Docker/cloud deployments use stdout |
|
||||
| Entry point | Default | Why |
|
||||
|---|---|---|
|
||||
| CLI commands | File only | Doesn't interfere with command output |
|
||||
| MCP server | File only | Stdout would corrupt JSON-RPC |
|
||||
| API server | File (local) or stdout (cloud) | Docker/cloud uses stdout |
|
||||
|
||||
**Log file location:** `~/.basic-memory/basic-memory.log` (10MB rotation, 10 days retention)
|
||||
Log file: `~/.basic-memory/basic-memory.log` (10MB rotation, 10 days
|
||||
retention).
|
||||
|
||||
### Environment Variables
|
||||
### Environment variables
|
||||
|
||||
| Variable | Default | Description |
|
||||
|----------|---------|-------------|
|
||||
| `BASIC_MEMORY_LOG_LEVEL` | `INFO` | Log level: DEBUG, INFO, WARNING, ERROR |
|
||||
| `BASIC_MEMORY_CLOUD_MODE` | `false` | When `true`, API logs to stdout with structured context |
|
||||
|---|---|---|
|
||||
| `BASIC_MEMORY_LOG_LEVEL` | `INFO` | DEBUG / INFO / WARNING / ERROR |
|
||||
| `BASIC_MEMORY_CLOUD_MODE` | `false` | API logs to stdout with structured context |
|
||||
| `BASIC_MEMORY_FORCE_LOCAL` | `false` | Force local API routing |
|
||||
| `BASIC_MEMORY_FORCE_CLOUD` | `false` | Force cloud API routing |
|
||||
| `BASIC_MEMORY_EXPLICIT_ROUTING` | `false` | Mark route selection as explicit |
|
||||
| `BASIC_MEMORY_ENV` | `dev` | Set to `test` for test mode (stderr only) |
|
||||
|
||||
### Examples
|
||||
| `BASIC_MEMORY_NO_PROMOS` | `false` | Disable cloud promos and telemetry |
|
||||
| `BASIC_MEMORY_IMPORT_UPLOAD_MAX_BYTES` | `104857600` | Max uploaded import size |
|
||||
|
||||
```bash
|
||||
# Enable debug logging
|
||||
BASIC_MEMORY_LOG_LEVEL=DEBUG basic-memory sync
|
||||
|
||||
# View logs
|
||||
tail -f ~/.basic-memory/basic-memory.log
|
||||
|
||||
# Cloud/Docker mode (stdout logging with structured context)
|
||||
BASIC_MEMORY_CLOUD_MODE=true uvicorn basic_memory.api.app:app
|
||||
```
|
||||
|
||||
## Development
|
||||
|
||||
### Running Tests
|
||||
Basic Memory supports SQLite (default, fast, no Docker) and Postgres
|
||||
(via testcontainers — Docker required).
|
||||
|
||||
Basic Memory supports dual database backends (SQLite and Postgres). By default, tests run against SQLite. Set `BASIC_MEMORY_TEST_POSTGRES=1` to run against Postgres (uses testcontainers - Docker required).
|
||||
|
||||
**Quick Start:**
|
||||
```bash
|
||||
# Run all tests against SQLite (default, fast)
|
||||
just test-sqlite
|
||||
|
||||
# Run all tests against Postgres (uses testcontainers)
|
||||
just test-postgres
|
||||
|
||||
# Run both SQLite and Postgres tests
|
||||
just test
|
||||
```
|
||||
|
||||
**Available Test Commands:**
|
||||
|
||||
- `just test` - Run all tests against both SQLite and Postgres
|
||||
- `just test-sqlite` - Run all tests against SQLite (fast, no Docker needed)
|
||||
- `just test-postgres` - Run all tests against Postgres (uses testcontainers)
|
||||
- `just test-unit-sqlite` - Run unit tests against SQLite
|
||||
- `just test-unit-postgres` - Run unit tests against Postgres
|
||||
- `just test-int-sqlite` - Run integration tests against SQLite
|
||||
- `just test-int-postgres` - Run integration tests against Postgres
|
||||
- `just test-windows` - Run Windows-specific tests (auto-skips on other platforms)
|
||||
- `just test-benchmark` - Run performance benchmark tests
|
||||
|
||||
**Postgres Testing:**
|
||||
|
||||
Postgres tests use [testcontainers](https://testcontainers-python.readthedocs.io/) which automatically spins up a Postgres instance in Docker. No manual database setup required - just have Docker running.
|
||||
|
||||
**Test Markers:**
|
||||
|
||||
Tests use pytest markers for selective execution:
|
||||
- `windows` - Windows-specific database optimizations
|
||||
- `benchmark` - Performance tests (excluded from default runs)
|
||||
|
||||
**Other Development Commands:**
|
||||
```bash
|
||||
just install # Install with dev dependencies
|
||||
just lint # Run linting checks
|
||||
just typecheck # Run type checking
|
||||
just format # Format code with ruff
|
||||
just check # Run all quality checks
|
||||
just migration "msg" # Create database migration
|
||||
just test-sqlite # All tests, SQLite
|
||||
just test-postgres # All tests, Postgres (testcontainers)
|
||||
just test # Both backends
|
||||
just fast-check # fix/format/typecheck + impacted tests + smoke
|
||||
just doctor # File <-> DB consistency check (temp config)
|
||||
just lint
|
||||
just typecheck # Pyright (primary)
|
||||
just typecheck-ty # ty (supplemental)
|
||||
just format
|
||||
just check # All quality checks
|
||||
just migration "msg" # New Alembic migration
|
||||
```
|
||||
|
||||
See the [justfile](justfile) for the complete list of development commands.
|
||||
Tests use pytest markers: `windows`, `benchmark`, `smoke`. See
|
||||
[justfile](justfile) for the full list.
|
||||
|
||||
Contributions welcome — see [CONTRIBUTING.md](CONTRIBUTING.md).
|
||||
|
||||
## License
|
||||
|
||||
AGPL-3.0
|
||||
|
||||
Contributions are welcome. See the [Contributing](CONTRIBUTING.md) guide for info about setting up the project locally
|
||||
and submitting PRs.
|
||||
[AGPL-3.0](LICENSE).
|
||||
|
||||
## Star History
|
||||
|
||||
@@ -517,4 +553,4 @@ and submitting PRs.
|
||||
</picture>
|
||||
</a>
|
||||
|
||||
Built with ♥️ by Basic Machines
|
||||
Built with ♥️ by [Basic Machines](https://basicmachines.co?utm_source=github&utm_medium=referral&utm_campaign=readme)
|
||||
|
||||
+67
-2
@@ -8,6 +8,71 @@
|
||||
|
||||
## Reporting a Vulnerability
|
||||
|
||||
Use this section to tell people how to report a vulnerability.
|
||||
If you find a vulnerability, please contact hello@basicmachines.co.
|
||||
|
||||
If you find a vulnerability, please contact hello@basicmachines.co
|
||||
Please do not open a public GitHub issue for security vulnerabilities. We aim
|
||||
to respond within 72 hours and will coordinate a fix and disclosure timeline
|
||||
with you.
|
||||
|
||||
## Threat Model
|
||||
|
||||
Basic Memory is a local-first MCP server that reads and writes markdown files
|
||||
inside configured project directories. It runs on your machine with your user
|
||||
permissions, so local configuration deserves the same care as any other
|
||||
developer tool that can access your files.
|
||||
|
||||
### What Basic Memory Controls
|
||||
|
||||
- Filesystem-touching tools validate paths against the configured project root
|
||||
with `validate_project_path()`, resolved paths, and `Path.is_relative_to()`.
|
||||
Path traversal attempts such as `../../etc/passwd` are blocked at this layer.
|
||||
- Scan optimizations in `sync_service.py` call `find` through
|
||||
`asyncio.create_subprocess_exec()` with explicit argument lists. Project paths
|
||||
are passed as data, not interpolated into shell strings.
|
||||
- Auto-update code uses hardcoded commands, list-form arguments, and
|
||||
`stdin=DEVNULL`. User-controlled strings do not reach a shell there.
|
||||
|
||||
### MCP Client-Side Risk
|
||||
|
||||
Recent MCP ecosystem research has highlighted a client-side pattern where an
|
||||
MCP host can be configured to run arbitrary commands as "servers." That risk is
|
||||
in the host configuration, not in notes or Basic Memory tool input.
|
||||
|
||||
The recommended Basic Memory MCP configuration uses a known command with
|
||||
explicit arguments:
|
||||
|
||||
```json
|
||||
{
|
||||
"mcpServers": {
|
||||
"basic-memory": {
|
||||
"command": "uvx",
|
||||
"args": ["basic-memory", "mcp"]
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Only add MCP server entries from sources you trust. Avoid inline shell scripts
|
||||
or command strings copied from untrusted sources. Treat third-party MCP server
|
||||
configuration with the same scrutiny as any locally executed program.
|
||||
|
||||
Related ecosystem context:
|
||||
|
||||
- OX Security: The Mother of All AI Supply Chains
|
||||
- CSO Online: RCE by design: MCP architectural choice haunts AI agent ecosystem
|
||||
|
||||
### Out Of Scope
|
||||
|
||||
- Basic Memory does not execute note content as code. Notes are returned as
|
||||
data to the LLM.
|
||||
- Basic Memory does not open network ports by default. The MCP server uses
|
||||
stdio; the optional REST API is intended for localhost use.
|
||||
- Basic Memory is designed for single-user local knowledge bases and does not
|
||||
implement access controls between operating-system users.
|
||||
|
||||
## Secure Configuration Checklist
|
||||
|
||||
- MCP config `command` points to `uvx` or a trusted binary, not a shell string.
|
||||
- Project paths in Basic Memory config come from trusted local configuration.
|
||||
- If exposing the REST API, bind it only to localhost.
|
||||
- Review any third-party MCP servers before adding them to your host config.
|
||||
|
||||
@@ -1,5 +1,8 @@
|
||||
# Docker Compose configuration for Basic Memory with PostgreSQL
|
||||
# Use this for local development and testing with Postgres backend
|
||||
# Use this for local development and testing with Postgres backend.
|
||||
#
|
||||
# The Postgres backend requires the pgvector extension (semantic search).
|
||||
# This image bundles pgvector; plain postgres:17 will not work for vector search.
|
||||
#
|
||||
# Usage:
|
||||
# docker-compose -f docker-compose-postgres.yml up -d
|
||||
@@ -7,7 +10,7 @@
|
||||
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:17
|
||||
image: pgvector/pgvector:pg17
|
||||
container_name: basic-memory-postgres
|
||||
environment:
|
||||
# Local development/test credentials - NOT for production
|
||||
|
||||
+3
-1
@@ -17,7 +17,9 @@ services:
|
||||
volumes:
|
||||
|
||||
# Persistent storage for configuration and database
|
||||
- basic-memory-config:/root/.basic-memory:rw
|
||||
# Container runs as `appuser` (Dockerfile USER directive), so the CLI
|
||||
# config dir lives under /home/appuser, not /root.
|
||||
- basic-memory-config:/home/appuser/.basic-memory:rw
|
||||
|
||||
# Mount your knowledge directory (required)
|
||||
# Change './knowledge' to your actual Obsidian vault or knowledge directory
|
||||
|
||||
+44
-14
@@ -18,7 +18,7 @@ Each entrypoint uses a **composition root** pattern to manage configuration and
|
||||
A composition root is the single place in an application where dependencies are wired together. In Basic Memory, each entrypoint has its own composition root that:
|
||||
|
||||
1. Reads configuration from `ConfigManager`
|
||||
2. Resolves runtime mode (cloud/local/test)
|
||||
2. Resolves runtime mode (local/test)
|
||||
3. Creates and provides dependencies to downstream code
|
||||
|
||||
**Key principle**: Only composition roots read global configuration. All other modules receive configuration explicitly.
|
||||
@@ -52,10 +52,7 @@ class Container:
|
||||
def create(cls) -> "Container":
|
||||
"""Create container by reading ConfigManager."""
|
||||
config = ConfigManager().config
|
||||
mode = resolve_runtime_mode(
|
||||
cloud_mode_enabled=config.cloud_mode_enabled,
|
||||
is_test_env=config.is_test_env,
|
||||
)
|
||||
mode = resolve_runtime_mode(is_test_env=config.is_test_env)
|
||||
return cls(config=config, mode=mode)
|
||||
|
||||
@property
|
||||
@@ -99,17 +96,20 @@ class RuntimeMode(Enum):
|
||||
return self == RuntimeMode.TEST
|
||||
```
|
||||
|
||||
Resolution follows this precedence: **TEST > CLOUD > LOCAL**
|
||||
Resolution follows this precedence in local app flows: **TEST > LOCAL**
|
||||
|
||||
```python
|
||||
def resolve_runtime_mode(cloud_mode_enabled: bool, is_test_env: bool) -> RuntimeMode:
|
||||
def resolve_runtime_mode(is_test_env: bool) -> RuntimeMode:
|
||||
if is_test_env:
|
||||
return RuntimeMode.TEST
|
||||
if cloud_mode_enabled:
|
||||
return RuntimeMode.CLOUD
|
||||
return RuntimeMode.LOCAL
|
||||
```
|
||||
|
||||
**Note**: `RuntimeMode` determines global behavior (e.g., whether to start file sync).
|
||||
Per-project routing is orthogonal: individual projects can be set to `cloud` mode via `ProjectMode`,
|
||||
which affects client routing in `get_client(project_name=...)` without changing global runtime mode.
|
||||
`RuntimeMode.CLOUD` may remain for compatibility, but standard local runtime resolution does not select it.
|
||||
|
||||
## Dependencies Package
|
||||
|
||||
### Structure
|
||||
@@ -214,15 +214,45 @@ Example tool using typed client:
|
||||
|
||||
```python
|
||||
@mcp.tool()
|
||||
async def search_notes(query: str, project: str | None = None) -> SearchResponse:
|
||||
async with get_client() as client:
|
||||
active_project = await get_active_project(client, project)
|
||||
|
||||
async def search_notes(
|
||||
query: str,
|
||||
project: str | None = None,
|
||||
metadata_filters: dict | None = None,
|
||||
tags: list[str] | None = None,
|
||||
status: str | None = None,
|
||||
) -> SearchResponse:
|
||||
async with get_project_client(project, context) as (client, active_project):
|
||||
# Import client inside function to avoid circular imports
|
||||
from basic_memory.mcp.clients import SearchClient
|
||||
from basic_memory.schemas.search import SearchQuery
|
||||
|
||||
search_query = SearchQuery(
|
||||
text=query,
|
||||
metadata_filters=metadata_filters,
|
||||
tags=tags,
|
||||
status=status,
|
||||
)
|
||||
search_client = SearchClient(client, active_project.external_id)
|
||||
return await search_client.search(query)
|
||||
return await search_client.search(search_query.model_dump())
|
||||
```
|
||||
|
||||
### Per-Project Client Routing
|
||||
|
||||
`get_project_client()` from `mcp/project_context.py` is an async context manager that:
|
||||
1. Resolves the project name from config (no network call)
|
||||
2. Creates the correctly-routed client based on the project's mode (local ASGI or cloud HTTP with API key)
|
||||
3. Validates the project via the API
|
||||
4. Yields `(client, active_project)` tuple
|
||||
|
||||
This solves the bootstrap problem: you need the project name to choose the right client (local vs cloud), but you need the client to validate the project exists.
|
||||
|
||||
```python
|
||||
from basic_memory.mcp.project_context import get_project_client
|
||||
|
||||
async with get_project_client(project, context) as (client, active_project):
|
||||
# client is routed based on project's mode (local or cloud)
|
||||
# active_project is validated via the API
|
||||
...
|
||||
```
|
||||
|
||||
## Sync Coordination
|
||||
|
||||
@@ -0,0 +1,512 @@
|
||||
# Note Format Reference
|
||||
|
||||
Every document in Basic Memory is a plain Markdown file. Files are the source of truth — changes to files automatically update the knowledge graph in the database. You maintain complete ownership, files work with git, and knowledge persists independently of any AI conversation.
|
||||
|
||||
## Document Structure
|
||||
|
||||
A note has three parts: YAML frontmatter, content (observations), and relations.
|
||||
|
||||
```markdown
|
||||
---
|
||||
title: Coffee Brewing Methods
|
||||
type: note
|
||||
tags: [coffee, brewing]
|
||||
permalink: coffee-brewing-methods
|
||||
---
|
||||
|
||||
# Coffee Brewing Methods
|
||||
|
||||
## Observations
|
||||
- [method] Pour over provides more flavor clarity than French press
|
||||
- [technique] Water temperature at 205°F extracts optimal compounds #brewing
|
||||
- [preference] Ethiopian beans work well with lighter roasts (personal experience)
|
||||
|
||||
## Relations
|
||||
- relates_to [[Coffee Bean Origins]]
|
||||
- requires [[Proper Grinding Technique]]
|
||||
- contrasts_with [[Tea Brewing Methods]]
|
||||
```
|
||||
|
||||
The `## Observations` and `## Relations` headings are conventional but not required — the parser detects observations and relations by their syntax patterns anywhere in the document.
|
||||
|
||||
## Frontmatter
|
||||
|
||||
YAML metadata between `---` fences at the top of the file.
|
||||
|
||||
| Field | Required | Default | Description |
|
||||
|-------|----------|---------|-------------|
|
||||
| `title` | No | filename stem | Used for linking and references. Auto-set from filename if missing. |
|
||||
| `type` | No | `note` | Entity type. Used for schema resolution and filtering. |
|
||||
| `tags` | No | `[]` | List or comma-separated string. Used for organization and search. |
|
||||
| `permalink` | No | generated from title | Stable identifier. Persists even if the file moves. |
|
||||
| `schema` | No | none | Schema attachment — dict (inline), string (reference), or omitted (implicit). |
|
||||
|
||||
Custom fields are allowed. Any key not in the standard set is stored as `entity_metadata` and indexed for search and filtering.
|
||||
|
||||
```yaml
|
||||
---
|
||||
title: Paul Graham
|
||||
type: Person
|
||||
tags: [startups, essays, lisp]
|
||||
permalink: paul-graham
|
||||
status: active
|
||||
source: wikipedia
|
||||
---
|
||||
```
|
||||
|
||||
Here `status` and `source` are custom fields stored in `entity_metadata`.
|
||||
|
||||
### Frontmatter Value Handling
|
||||
|
||||
YAML automatically converts some values to native types. Basic Memory normalizes them:
|
||||
|
||||
- Date strings (`2025-10-24`) → kept as ISO format strings
|
||||
- Numbers (`1.0`) → converted to strings
|
||||
- Booleans (`true`) → converted to strings (`"True"`)
|
||||
- Lists and dicts → preserved, items normalized recursively
|
||||
|
||||
This prevents errors when downstream code expects string values.
|
||||
|
||||
## Observations
|
||||
|
||||
An observation is a categorized fact about the entity. Written as a Markdown list item.
|
||||
|
||||
**Syntax:**
|
||||
|
||||
```
|
||||
- [category] content text #tag1 #tag2 (context)
|
||||
```
|
||||
|
||||
| Part | Required | Description |
|
||||
|------|----------|-------------|
|
||||
| `[category]` | Yes | Classification in square brackets. Any text except `[]()` chars. |
|
||||
| content | Yes | The fact or statement. |
|
||||
| `#tags` | No | Inline tags. Space-separated, each starting with `#`. |
|
||||
| `(context)` | No | Parenthesized text at end of line. Supporting details or source. |
|
||||
|
||||
### Examples
|
||||
|
||||
```markdown
|
||||
- [tech] Uses SQLite for storage #database
|
||||
- [design] Follows local-first architecture #architecture
|
||||
- [decision] Selected bcrypt for passwords #security (based on OWASP audit)
|
||||
- [name] Paul Graham
|
||||
- [expertise] Startups
|
||||
- [expertise] Lisp
|
||||
- [expertise] Essay writing
|
||||
```
|
||||
|
||||
Array-like fields use repeated categories — multiple `[expertise]` observations above.
|
||||
|
||||
### What Is Not an Observation
|
||||
|
||||
The parser excludes these list item patterns:
|
||||
|
||||
| Pattern | Example | Reason |
|
||||
|---------|---------|--------|
|
||||
| Checkboxes | `- [ ] Todo item`, `- [x] Done`, `- [-] Cancelled` | Task list syntax |
|
||||
| Markdown links | `- [text](url)` | URL link syntax |
|
||||
| Bare wiki links | `- [[Target]]` | Treated as a `links_to` relation instead |
|
||||
|
||||
A list item with `#tags` but no `[category]` is still parsed — the tags are extracted and the category defaults to `Note`.
|
||||
|
||||
## Relations
|
||||
|
||||
Relations connect documents to form the knowledge graph. There are two kinds.
|
||||
|
||||
### Explicit Relations
|
||||
|
||||
Written as list items with a relation type and a `[[wiki link]]` target. Unquoted
|
||||
relation types are single tokens. Quote relation types that contain spaces.
|
||||
|
||||
**Syntax:**
|
||||
|
||||
```
|
||||
- relation_type [[Target Entity]] (context)
|
||||
- "multi word relation type" [[Target Entity]] (context)
|
||||
- 'multi word relation type' [[Target Entity]] (context)
|
||||
```
|
||||
|
||||
| Part | Required | Description |
|
||||
|------|----------|-------------|
|
||||
| `relation_type` | Yes | Single unquoted token before `[[`, or quoted text for multi-word labels. |
|
||||
| `[[Target]]` | Yes | Wiki link to the target entity. Matched by title or permalink. |
|
||||
| `(context)` | No | Parenthesized text after `]]`. Supporting details. |
|
||||
|
||||
### Examples
|
||||
|
||||
Explicit relations:
|
||||
|
||||
```markdown
|
||||
- implements [[Search Design]]
|
||||
- depends_on [[Database Schema]]
|
||||
- works_at [[Y Combinator]] (co-founder)
|
||||
- "based on" [[Customer Interview]]
|
||||
- 'in response to' [[Incident Review]]
|
||||
```
|
||||
|
||||
Bare wiki links and prose list items create implicit `links_to` relations:
|
||||
|
||||
```markdown
|
||||
- [[Some Entity]]
|
||||
- some other thing [[Some Entity]]
|
||||
```
|
||||
|
||||
Both examples above create `links_to [[Some Entity]]`. Use quotes when the words before
|
||||
`[[` are meant to be a multi-word relation type.
|
||||
|
||||
Common relation types:
|
||||
- `implements`, `depends_on`, `relates_to`, `inspired_by`
|
||||
- `extends`, `part_of`, `contains`, `pairs_with`
|
||||
- `works_at`, `authored`, `collaborated_with`
|
||||
|
||||
Any single-token text or quoted text works as a relation type. These are conventions,
|
||||
not a fixed set.
|
||||
|
||||
### Inline References
|
||||
|
||||
Wiki links appearing in regular prose create implicit `links_to` relations. This includes
|
||||
list items that do not match the explicit relation grammar above.
|
||||
|
||||
```markdown
|
||||
This builds on [[Core Design]] and uses [[Utility Functions]].
|
||||
- We should revisit [[Search Design]] after the API changes.
|
||||
```
|
||||
|
||||
This creates three relations: `links_to [[Core Design]]`, `links_to [[Utility Functions]]`,
|
||||
and `links_to [[Search Design]]`.
|
||||
|
||||
### Forward References
|
||||
|
||||
Relations can link to entities that don't exist yet. Basic Memory resolves them when the target is created.
|
||||
|
||||
## Permalinks and memory:// URLs
|
||||
|
||||
Every document has a unique **permalink** — a stable identifier derived from its title. You can set one explicitly in frontmatter, or let the system generate it.
|
||||
|
||||
```yaml
|
||||
permalink: auth-approaches-2024
|
||||
```
|
||||
|
||||
Permalinks form the basis of `memory://` URLs:
|
||||
|
||||
```
|
||||
memory://auth-approaches-2024 # By permalink
|
||||
memory://Authentication Approaches # By title (auto-resolves)
|
||||
memory://project/auth-approaches # By path
|
||||
```
|
||||
|
||||
Pattern matching is supported:
|
||||
|
||||
```
|
||||
memory://auth* # Starts with "auth"
|
||||
memory://*/approaches # Ends with "approaches"
|
||||
memory://project/*/requirements # Nested wildcard
|
||||
```
|
||||
|
||||
## Schemas
|
||||
|
||||
Schemas declare the expected structure of a note — which observation categories and relation types a well-formed note should have. They use Picoschema, a compact notation from Google's Dotprompt that fits naturally in YAML frontmatter.
|
||||
|
||||
### Picoschema Syntax
|
||||
|
||||
```yaml
|
||||
schema:
|
||||
name: string, full name # required field with description
|
||||
email?: string, contact email # ? = optional
|
||||
role?: string, job title
|
||||
works_at?: Organization, employer # capitalized type = entity reference
|
||||
tags?(array): string, categories # array of type
|
||||
status?(enum): [active, inactive] # enum with allowed values
|
||||
metadata?(object): # nested object
|
||||
updated_at?: string
|
||||
source?: string
|
||||
```
|
||||
|
||||
| Notation | Meaning | Example |
|
||||
|----------|---------|---------|
|
||||
| `field: type` | Required field | `name: string` |
|
||||
| `field?: type` | Optional field | `role?: string` |
|
||||
| `field(array): type` | Array of values | `expertise(array): string` |
|
||||
| `field?(enum): [vals]` | Enum with allowed values | `status?(enum): [active, inactive]` |
|
||||
| `field?(object):` | Nested object with sub-fields | `metadata?(object):` |
|
||||
| `, description` | Description after comma | `name: string, full name` |
|
||||
| `EntityName` | Capitalized type = entity reference | `works_at?: Organization` |
|
||||
|
||||
**Scalar types:** `string`, `integer`, `number`, `boolean`, `any`
|
||||
|
||||
Any type not in that set whose first letter is uppercase is treated as an entity reference (a relation target).
|
||||
|
||||
### Schema-to-Note Mapping
|
||||
|
||||
Schemas validate against existing observation/relation syntax. Note authors don't learn new syntax.
|
||||
|
||||
| Schema Declaration | Maps To | Example in Note |
|
||||
|--------------------|---------|-----------------|
|
||||
| `field: string` | Observation `[field] value` | `- [name] Paul Graham` |
|
||||
| `field?(array): string` | Multiple `[field]` observations | `- [expertise] Lisp` (repeated) |
|
||||
| `field?: EntityType` | Relation `field [[Target]]` | `- works_at [[Y Combinator]]` |
|
||||
| `field?(array): EntityType` | Multiple `field` relations | `- authored [[Book]]` (repeated) |
|
||||
| `tags` | Frontmatter `tags` array | `tags: [startups, essays]` |
|
||||
| `field?(enum): [vals]` | Observation `[field] value` where value is in the set | `- [status] active` |
|
||||
|
||||
Observations and relations not covered by the schema are valid — schemas describe a subset, not a straitjacket.
|
||||
|
||||
### Schema Attachment
|
||||
|
||||
Three ways to attach a schema to a note, resolved in priority order:
|
||||
|
||||
**1. Inline schema** — `schema` is a dict in frontmatter:
|
||||
|
||||
```yaml
|
||||
---
|
||||
title: Team Standup 2024-01-15
|
||||
type: meeting
|
||||
schema:
|
||||
attendees(array): string, who was there
|
||||
decisions(array): string, what was decided
|
||||
action_items(array): string, follow-ups
|
||||
blockers?(array): string, anything stuck
|
||||
---
|
||||
```
|
||||
|
||||
Good for one-off structured notes or prototyping a schema before extracting it.
|
||||
|
||||
**2. Explicit reference** — `schema` is a string naming a schema note:
|
||||
|
||||
```yaml
|
||||
---
|
||||
title: Basic Memory
|
||||
schema: SoftwareProject
|
||||
---
|
||||
```
|
||||
|
||||
or by permalink:
|
||||
|
||||
```yaml
|
||||
---
|
||||
title: LLM Memory Patterns
|
||||
schema: schema/research-project
|
||||
---
|
||||
```
|
||||
|
||||
Use when the note's `type` differs from the schema it should validate against, or when multiple schema variants exist.
|
||||
|
||||
**3. Implicit by type** — no `schema` field, resolved by matching `type`:
|
||||
|
||||
```yaml
|
||||
---
|
||||
title: Paul Graham
|
||||
type: Person
|
||||
---
|
||||
```
|
||||
|
||||
The system looks up a schema note where `entity: Person`. If found, it applies. If not, no validation occurs.
|
||||
|
||||
**4. No schema** — perfectly fine. Most notes don't need one.
|
||||
|
||||
### Schema Notes
|
||||
|
||||
A schema is itself a Basic Memory note with `type: schema`. It lives anywhere (though `schema/` is the conventional directory).
|
||||
|
||||
```yaml
|
||||
# schema/Person.md
|
||||
---
|
||||
title: Person
|
||||
type: schema
|
||||
entity: Person
|
||||
version: 1
|
||||
schema:
|
||||
name: string, full name
|
||||
role?: string, job title or position
|
||||
works_at?: Organization, employer
|
||||
expertise?(array): string, areas of knowledge
|
||||
email?: string, contact email
|
||||
settings:
|
||||
validation: warn
|
||||
---
|
||||
|
||||
# Person
|
||||
|
||||
A human individual in the knowledge graph.
|
||||
```
|
||||
|
||||
| Field | Required | Description |
|
||||
|-------|----------|-------------|
|
||||
| `type` | Yes | Must be `schema` |
|
||||
| `entity` | Yes | The entity type this schema describes (e.g., `Person`) |
|
||||
| `version` | No | Schema version number (default: `1`) |
|
||||
| `schema` | Yes | Picoschema dict defining the fields |
|
||||
| `settings.validation` | No | Validation mode (default: `warn`) |
|
||||
|
||||
Schema notes are regular notes — they show up in search, can have observations and relations, and participate in the knowledge graph.
|
||||
|
||||
### Validation Modes
|
||||
|
||||
| Mode | Behavior |
|
||||
|------|----------|
|
||||
| `warn` | Warnings in output, doesn't block (default) |
|
||||
| `strict` | Errors that block sync, for CI/CD enforcement |
|
||||
| `off` | No validation |
|
||||
|
||||
### Validation Output
|
||||
|
||||
```
|
||||
$ bm schema validate people/ada-lovelace.md
|
||||
|
||||
⚠ Person schema validation:
|
||||
- Missing required field: name (expected [name] observation)
|
||||
- Missing optional field: role
|
||||
- Missing optional field: works_at (no relation found)
|
||||
|
||||
ℹ Unmatched observations: [fact] ×2, [born] ×1
|
||||
ℹ Unmatched relations: collaborated_with
|
||||
```
|
||||
|
||||
"Unmatched" items are informational — observations and relations the schema doesn't cover.
|
||||
|
||||
### Schema Inference
|
||||
|
||||
Generate schemas from existing notes by analyzing observation and relation frequency:
|
||||
|
||||
```
|
||||
$ bm schema infer Person
|
||||
|
||||
Analyzing 30 notes with type: Person...
|
||||
|
||||
Observations found:
|
||||
[name] 30/30 100% → name: string
|
||||
[role] 27/30 90% → role?: string
|
||||
[expertise] 18/30 60% → expertise?(array): string
|
||||
[email] 8/30 27% → email?: string
|
||||
|
||||
Relations found:
|
||||
works_at 22/30 73% → works_at?: Organization
|
||||
|
||||
Suggested schema:
|
||||
name: string, full name
|
||||
role?: string, job title
|
||||
expertise?(array): string, areas of knowledge
|
||||
email?: string, contact email
|
||||
works_at?: Organization, employer
|
||||
|
||||
Save to schema/Person.md? [y/n]
|
||||
```
|
||||
|
||||
Frequency thresholds:
|
||||
- **100% present** → required field
|
||||
- **25%+ present** → optional field
|
||||
- **Below 25%** → excluded from suggestion
|
||||
|
||||
### Schema Drift Detection
|
||||
|
||||
Track how usage patterns shift over time:
|
||||
|
||||
```
|
||||
$ bm schema diff Person
|
||||
|
||||
Schema drift detected:
|
||||
|
||||
+ expertise: now in 81% of notes (was 12%)
|
||||
- department: dropped to 3% of notes
|
||||
~ works_at: cardinality changed (one → many)
|
||||
|
||||
Update schema? [y/n/review]
|
||||
```
|
||||
|
||||
## Complete Examples
|
||||
|
||||
### Simple Note (No Schema)
|
||||
|
||||
```markdown
|
||||
---
|
||||
title: Project Ideas
|
||||
type: note
|
||||
tags: [ideas, brainstorm]
|
||||
---
|
||||
|
||||
# Project Ideas
|
||||
|
||||
## Observations
|
||||
- [idea] Build a CLI tool for markdown linting #tooling
|
||||
- [idea] Create a recipe knowledge base #cooking
|
||||
- [priority] Focus on developer tools first (Q1 goal)
|
||||
|
||||
## Relations
|
||||
- inspired_by [[Developer Workflow Research]]
|
||||
- part_of [[Q1 Planning]]
|
||||
```
|
||||
|
||||
### Schema-Validated Note
|
||||
|
||||
Schema at `schema/Person.md`:
|
||||
|
||||
```yaml
|
||||
---
|
||||
title: Person
|
||||
type: schema
|
||||
entity: Person
|
||||
version: 1
|
||||
schema:
|
||||
name: string, full name
|
||||
role?: string, job title or position
|
||||
works_at?: Organization, employer
|
||||
expertise?(array): string, areas of knowledge
|
||||
email?: string, contact email
|
||||
settings:
|
||||
validation: warn
|
||||
---
|
||||
|
||||
# Person
|
||||
|
||||
A human individual in the knowledge graph.
|
||||
```
|
||||
|
||||
Note at `people/paul-graham.md`:
|
||||
|
||||
```markdown
|
||||
---
|
||||
title: Paul Graham
|
||||
type: Person
|
||||
tags: [startups, essays, lisp]
|
||||
---
|
||||
|
||||
# Paul Graham
|
||||
|
||||
## Observations
|
||||
- [name] Paul Graham
|
||||
- [role] Essayist and investor
|
||||
- [expertise] Startups
|
||||
- [expertise] Lisp
|
||||
- [expertise] Essay writing
|
||||
- [fact] Created Viaweb, the first web app
|
||||
|
||||
## Relations
|
||||
- works_at [[Y Combinator]]
|
||||
- authored [[Hackers and Painters]]
|
||||
```
|
||||
|
||||
The `[fact]` observation and `authored` relation are not in the schema — they're valid, just unmatched. The schema only checks that `[name]` exists (required) and looks for optional fields like `[role]`, `[expertise]`, and `works_at`.
|
||||
|
||||
### Inline Schema Note
|
||||
|
||||
```markdown
|
||||
---
|
||||
title: Team Standup 2024-01-15
|
||||
type: meeting
|
||||
schema:
|
||||
attendees(array): string, who was there
|
||||
decisions(array): string, what was decided
|
||||
action_items(array): string, follow-ups
|
||||
blockers?(array): string, anything stuck
|
||||
---
|
||||
|
||||
# Team Standup 2024-01-15
|
||||
|
||||
## Observations
|
||||
- [attendees] Paul
|
||||
- [attendees] Sarah
|
||||
- [decisions] Ship v2 by Friday
|
||||
- [action_items] Paul to review PR #42
|
||||
- [blockers] Waiting on API credentials
|
||||
```
|
||||
@@ -0,0 +1,147 @@
|
||||
# Simplified Local/Cloud Routing
|
||||
|
||||
## Context
|
||||
|
||||
Basic Memory now uses explicit, project-aware routing without a global cloud-mode toggle.
|
||||
Routing is determined by command-level flags and project mode, not by a global `cloud_mode` state.
|
||||
|
||||
This document is the canonical contract for local/cloud routing behavior in CLI, MCP, and API-adjacent clients.
|
||||
|
||||
## Goals
|
||||
|
||||
1. Remove global `cloud_mode` from runtime/routing semantics.
|
||||
2. Keep MCP HTTP/SSE local-only; let stdio honor per-project routing.
|
||||
3. Make CLI routing explicit and easy to reason about.
|
||||
4. Support projects that exist in both local and cloud without ambiguity.
|
||||
|
||||
## Routing Contract
|
||||
|
||||
Routing is resolved in this order:
|
||||
|
||||
1. Injected client factory (for composition/integration contexts)
|
||||
2. Explicit routing override (`--local` / `--cloud` or env vars below)
|
||||
3. Project-scoped routing (`project.mode`) when a project is known
|
||||
4. Default local routing
|
||||
|
||||
### Routing Environment Variables
|
||||
|
||||
- `BASIC_MEMORY_FORCE_LOCAL=true`: force local transport
|
||||
- `BASIC_MEMORY_FORCE_CLOUD=true`: force cloud proxy transport
|
||||
- `BASIC_MEMORY_EXPLICIT_ROUTING=true`: marks routing as explicitly chosen for this command
|
||||
|
||||
When explicit routing is active, project mode does not override the selected route.
|
||||
|
||||
## Config Semantics
|
||||
|
||||
- `project.mode` is the only config-based routing signal for project-scoped operations.
|
||||
- Legacy `cloud_mode` values may be encountered during migration/loading but are not used for routing behavior.
|
||||
- Normalization saves remove stale `cloud_mode` from `~/.basic-memory/config.json`.
|
||||
|
||||
### Example Config
|
||||
|
||||
```json
|
||||
{
|
||||
"projects": {
|
||||
"main": {
|
||||
"path": "/Users/me/basic-memory",
|
||||
"mode": "local",
|
||||
"local_sync_path": null,
|
||||
"bisync_initialized": false,
|
||||
"last_sync": null
|
||||
},
|
||||
"specs": {
|
||||
"path": "specs",
|
||||
"mode": "cloud",
|
||||
"local_sync_path": "/Users/me/dev/specs",
|
||||
"bisync_initialized": true,
|
||||
"last_sync": "2026-02-06T17:36:38.544153"
|
||||
}
|
||||
},
|
||||
"default_project": "main",
|
||||
"cloud_api_key": "bmc_abc123...",
|
||||
"cloud_host": "https://cloud.basicmemory.com"
|
||||
}
|
||||
```
|
||||
|
||||
## Cloud Commands Are Auth-Only
|
||||
|
||||
`bm cloud login`, `bm cloud logout`, and `bm cloud status` manage authentication state.
|
||||
|
||||
- `bm cloud login`
|
||||
- performs OAuth device flow
|
||||
- stores/refreshes token material
|
||||
- may verify cloud health/subscription
|
||||
- does not change routing defaults
|
||||
- `bm cloud logout`
|
||||
- removes stored OAuth session tokens
|
||||
- does not change routing defaults
|
||||
- `bm cloud status`
|
||||
- reports auth state (API key, OAuth token validity)
|
||||
- runs health checks only when credentials are available
|
||||
|
||||
## MCP Transport Routing
|
||||
|
||||
### Stdio (default)
|
||||
|
||||
`bm mcp --transport stdio` uses natural per-project routing.
|
||||
|
||||
- Local-mode projects route through the in-process ASGI transport.
|
||||
- Cloud-mode projects route to the cloud proxy with Bearer auth (API key).
|
||||
- No explicit routing env vars are injected by the CLI command.
|
||||
- Externally-set env vars are honored (e.g. `BASIC_MEMORY_FORCE_CLOUD=true` for cloud deployments).
|
||||
- Users who need all projects forced local can set `BASIC_MEMORY_FORCE_LOCAL=true` externally.
|
||||
|
||||
### HTTP and SSE Transports
|
||||
|
||||
`bm mcp --transport streamable-http` and `bm mcp --transport sse` always route locally.
|
||||
|
||||
These transports set explicit local routing (`BASIC_MEMORY_FORCE_LOCAL=true` and
|
||||
`BASIC_MEMORY_EXPLICIT_ROUTING=true`) before starting the server. This prevents cloud
|
||||
routing regardless of project mode, since HTTP/SSE serve as local API endpoints.
|
||||
|
||||
## Project List UX for Dual Presence
|
||||
|
||||
Projects may exist in both local and cloud. `bm project list` should display that clearly in one row per logical
|
||||
project identity, with explicit source/target signals.
|
||||
|
||||
Recommended display contract:
|
||||
|
||||
1. Keep one row per normalized project name/permalink.
|
||||
2. Show both local and cloud presence as separate columns/indicators.
|
||||
3. Show an explicit `MCP (stdio)` target column that always resolves to `local`.
|
||||
4. Keep CLI route semantics explicit:
|
||||
- no flags: default local for non-project commands
|
||||
- `--cloud`: force cloud
|
||||
- `--local`: force local
|
||||
|
||||
## Project LS Targeting
|
||||
|
||||
`bm project ls` should clearly identify which project instance is being listed.
|
||||
|
||||
Targeting rules:
|
||||
|
||||
1. No routing flags: list local project files.
|
||||
2. `--cloud`: list cloud project files.
|
||||
3. `--local`: list local project files (explicit override).
|
||||
4. Output should label the active target (`LOCAL` or `CLOUD`) in heading or status line.
|
||||
|
||||
## Runtime Mode
|
||||
|
||||
Runtime mode is no longer a cloud/local routing switch for local app flows.
|
||||
|
||||
- `resolve_runtime_mode(is_test_env)` resolves to:
|
||||
- `TEST` when running in test environment
|
||||
- `LOCAL` otherwise
|
||||
- `RuntimeMode.CLOUD` may remain for compatibility with existing tests/call sites but is not selected by normal local
|
||||
runtime resolution.
|
||||
|
||||
## Verification Checklist
|
||||
|
||||
1. Loading config with legacy `cloud_mode` succeeds.
|
||||
2. Saving config strips legacy `cloud_mode`.
|
||||
3. `--local/--cloud` always override per-project mode for that command.
|
||||
4. No-project + no-flags commands route local by default.
|
||||
5. `bm cloud login/logout` do not toggle routing behavior.
|
||||
6. `bm mcp` stdio routes per-project mode; HTTP/SSE remain local-forced.
|
||||
7. `bm project list` communicates dual local/cloud presence without ambiguity.
|
||||
8. `bm project ls` output identifies route target explicitly.
|
||||
@@ -91,10 +91,11 @@ SQLite Database (Index)
|
||||
# List all projects
|
||||
projects = await list_memory_projects()
|
||||
|
||||
# Response structure:
|
||||
# Response structure (each entry includes external_id you can pass as project_id):
|
||||
# [
|
||||
# {
|
||||
# "name": "main",
|
||||
# "external_id": "550e8400-e29b-41d4-a716-446655440000",
|
||||
# "path": "/Users/name/notes",
|
||||
# "is_default": True,
|
||||
# "note_count": 156,
|
||||
@@ -102,6 +103,7 @@ projects = await list_memory_projects()
|
||||
# },
|
||||
# {
|
||||
# "name": "work",
|
||||
# "external_id": "9f86d081-884c-42a3-b5e3-1c0c5b4c8e52",
|
||||
# "path": "/Users/name/work-notes",
|
||||
# "is_default": False,
|
||||
# "note_count": 89,
|
||||
@@ -164,6 +166,44 @@ active_project = "main"
|
||||
results = await search_notes(query="topic", project=active_project)
|
||||
```
|
||||
|
||||
### `project` vs `project_id`
|
||||
|
||||
Every project has two identifiers:
|
||||
|
||||
- **`project`** — human-readable name (e.g., `"main"`). Easy to use, but can collide across cloud workspaces.
|
||||
- **`project_id`** — stable `external_id` UUID. Always unambiguous; takes precedence over `project` when both are passed.
|
||||
|
||||
**When to prefer `project_id`:**
|
||||
|
||||
1. **Cloud multi-workspace setups.** If the user belongs to more than one workspace (personal + organization, or several organizations) and the same project name might exist in more than one of them, pass `project_id` to route to the exact project. Without it, name resolution falls back to the default workspace, which may not be the one the user means.
|
||||
2. **After `list_memory_projects()`.** Once you have the `external_id`, prefer using it — it's the same number of characters in JSON and saves a name-resolution round-trip.
|
||||
3. **When persisting a project choice across a long session.** UUIDs are stable; names can be renamed.
|
||||
|
||||
**When `project` (name) is fine:**
|
||||
|
||||
- Local single-workspace setups (no collision risk).
|
||||
- One-off operations where the name is clearly visible to the user (e.g., quick `search_notes(project="main", ...)`).
|
||||
- The user explicitly references a project by name in their message.
|
||||
|
||||
**Example — cloud multi-workspace pattern:**
|
||||
|
||||
```python
|
||||
# Discover and pick the right project for this user
|
||||
projects = await list_memory_projects()
|
||||
target = next(p for p in projects if p["name"] == "research" and p["workspace"]["slug"] == "acme")
|
||||
|
||||
# Use the UUID for all subsequent operations — no ambiguity
|
||||
await write_note(
|
||||
title="Meeting Notes",
|
||||
content="...",
|
||||
folder="meetings",
|
||||
project_id=target["external_id"],
|
||||
)
|
||||
results = await search_notes(query="kickoff", project_id=target["external_id"])
|
||||
```
|
||||
|
||||
**Precedence rule:** When both are passed, `project_id` wins. This lets you safely supply `project="main"` for backward compatibility while still routing precisely with `project_id`.
|
||||
|
||||
### Cross-Project Operations
|
||||
|
||||
**Some tools work across all projects when project parameter omitted:**
|
||||
@@ -427,6 +467,8 @@ await write_note(
|
||||
)
|
||||
```
|
||||
|
||||
> **Important**: `write_note` errors if the note already exists. Use `edit_note` for incremental changes, or pass `overwrite=True` to replace.
|
||||
|
||||
**Well-structured note**:
|
||||
|
||||
```python
|
||||
@@ -760,6 +802,9 @@ notes = await read_note(
|
||||
identifier="memory://specs/*",
|
||||
project="main"
|
||||
)
|
||||
|
||||
# Cross-project URL (auto-routes to the correct project)
|
||||
note = await read_note(identifier="memory://research/specs/api-design")
|
||||
```
|
||||
|
||||
```python
|
||||
@@ -1038,9 +1083,41 @@ recent_decisions = await search_notes(
|
||||
)
|
||||
```
|
||||
|
||||
**Structured frontmatter filters**:
|
||||
|
||||
```python
|
||||
# Filter by tags and status
|
||||
results = await search_notes(
|
||||
query="authentication",
|
||||
tags=["security"],
|
||||
status="in-progress",
|
||||
project="main"
|
||||
)
|
||||
|
||||
# Complex metadata filters (supports $in, $gt, $gte, $lt, $lte, $between)
|
||||
results = await search_notes(
|
||||
query="api design",
|
||||
metadata_filters={
|
||||
"type": "spec",
|
||||
"priority": {"$in": ["high", "critical"]},
|
||||
"tags": ["architecture"]
|
||||
},
|
||||
project="main"
|
||||
)
|
||||
|
||||
# Metadata-only search (no query needed)
|
||||
results = await search_notes(
|
||||
metadata_filters={"type": "spec", "status": "in-progress"},
|
||||
project="main"
|
||||
)
|
||||
```
|
||||
|
||||
### Search Types
|
||||
|
||||
**Text search (default)**:
|
||||
Available types: `"text"`, `"title"`, `"permalink"`, `"vector"`/`"semantic"`, `"hybrid"`.
|
||||
Default is `"hybrid"` when semantic search is enabled, `"text"` otherwise.
|
||||
|
||||
**Text search**:
|
||||
|
||||
```python
|
||||
# Full-text search across all content
|
||||
@@ -1051,15 +1128,50 @@ results = await search_notes(
|
||||
)
|
||||
```
|
||||
|
||||
**Semantic search**:
|
||||
**Title and permalink search**:
|
||||
|
||||
```python
|
||||
# Search by title only
|
||||
results = await search_notes(query="API Design", search_type="title", project="main")
|
||||
|
||||
# Search by permalink
|
||||
results = await search_notes(query="specs/api-design", search_type="permalink", project="main")
|
||||
```
|
||||
|
||||
**Semantic/vector search**:
|
||||
|
||||
```python
|
||||
# Semantic/vector search (if enabled)
|
||||
results = await search_notes(
|
||||
query="user login security",
|
||||
search_type="semantic",
|
||||
search_type="semantic", # or "vector"
|
||||
project="main"
|
||||
)
|
||||
|
||||
# Override similarity threshold
|
||||
results = await search_notes(
|
||||
query="user login security",
|
||||
search_type="semantic",
|
||||
min_similarity=0.5,
|
||||
project="main"
|
||||
)
|
||||
```
|
||||
|
||||
**Hybrid search** (combines text + semantic):
|
||||
|
||||
```python
|
||||
results = await search_notes(
|
||||
query="authentication best practices",
|
||||
search_type="hybrid",
|
||||
project="main"
|
||||
)
|
||||
```
|
||||
|
||||
**Tag shorthand in query**:
|
||||
|
||||
```python
|
||||
# Use tag: prefix as shorthand
|
||||
results = await search_notes(query="tag:security", project="main")
|
||||
```
|
||||
|
||||
### Search Response
|
||||
@@ -2132,6 +2244,31 @@ active_project = projects[0]["name"]
|
||||
results = await search_notes(query="test", project=active_project)
|
||||
```
|
||||
|
||||
### Note Already Exists
|
||||
|
||||
**Error**: `write_note` called for a note that already exists
|
||||
|
||||
**Solution**:
|
||||
|
||||
```python
|
||||
# Preferred: use edit_note for incremental updates
|
||||
await edit_note(
|
||||
identifier="Existing Topic",
|
||||
operation="append",
|
||||
content="\n- [update] new information",
|
||||
project="main"
|
||||
)
|
||||
|
||||
# Alternative: replace the entire note
|
||||
await write_note(
|
||||
title="Existing Topic",
|
||||
content="# Existing Topic\n...",
|
||||
folder="notes",
|
||||
overwrite=True,
|
||||
project="main"
|
||||
)
|
||||
```
|
||||
|
||||
### Entity Not Found
|
||||
|
||||
**Error**: Note doesn't exist
|
||||
@@ -2687,14 +2824,15 @@ await write_note(
|
||||
|
||||
### Content Management
|
||||
|
||||
**write_note(title, content, folder, tags, note_type, project)**
|
||||
- Create or update markdown notes
|
||||
**write_note(title, content, folder, tags, note_type, overwrite, project)**
|
||||
- Create new markdown notes (errors if note already exists unless overwrite=True)
|
||||
- Parameters:
|
||||
- `title` (required): Note title
|
||||
- `content` (required): Markdown content
|
||||
- `folder` (required): Destination folder
|
||||
- `tags` (optional): List of tags
|
||||
- `note_type` (optional): Type of note (stored in frontmatter). Can be "note", "person", "meeting", "guide", etc.
|
||||
- `overwrite` (optional): Set to True to replace an existing note (default: error if exists)
|
||||
- `project` (required unless default_project_mode): Target project
|
||||
- Returns: Created/updated entity with permalink
|
||||
- Example:
|
||||
@@ -2861,16 +2999,20 @@ contents = await list_directory(
|
||||
|
||||
### Search & Discovery
|
||||
|
||||
**search_notes(query, page, page_size, search_type, types, entity_types, after_date, project)**
|
||||
**search_notes(query, page, page_size, search_type, types, entity_types, after_date, metadata_filters, tags, status, min_similarity, project)**
|
||||
- Search across knowledge base
|
||||
- Parameters:
|
||||
- `query` (required): Search query
|
||||
- `query` (optional): Search query (not required for filter-only searches)
|
||||
- `page` (optional): Page number (default: 1)
|
||||
- `page_size` (optional): Results per page (default: 10)
|
||||
- `search_type` (optional): "text" or "semantic"
|
||||
- `search_type` (optional): "text", "title", "permalink", "vector"/"semantic", "hybrid" (default: "hybrid" when semantic enabled, "text" otherwise)
|
||||
- `types` (optional): Entity type filter
|
||||
- `entity_types` (optional): Observation category filter
|
||||
- `after_date` (optional): Date filter (ISO format)
|
||||
- `metadata_filters` (optional): Structured frontmatter filters (dict, supports `$in`, `$gt`, `$gte`, `$lt`, `$lte`, `$between` operators)
|
||||
- `tags` (optional): Frontmatter tags filter (list); also available via `tag:` query shorthand
|
||||
- `status` (optional): Frontmatter status filter (string)
|
||||
- `min_similarity` (optional): Override similarity threshold for vector/hybrid search
|
||||
- `project` (required unless default_project_mode): Target project
|
||||
- Returns: Matching entities with scores
|
||||
- Example:
|
||||
@@ -2883,6 +3025,15 @@ results = await search_notes(
|
||||
)
|
||||
```
|
||||
|
||||
**Metadata-only search (via search_notes)**
|
||||
- Use `search_notes` with `metadata_filters` and no `query` for metadata-only searches:
|
||||
```python
|
||||
results = await search_notes(
|
||||
metadata_filters={"type": "spec", "status": "in-progress"},
|
||||
project="main"
|
||||
)
|
||||
```
|
||||
|
||||
### Project Management
|
||||
|
||||
**list_memory_projects()**
|
||||
@@ -2930,6 +3081,15 @@ await delete_project(project_name="old-project")
|
||||
status = await sync_status(project="main")
|
||||
```
|
||||
|
||||
**list_workspaces()**
|
||||
- List available workspaces (cloud)
|
||||
- Parameters: None
|
||||
- Returns: List of workspaces with metadata
|
||||
- Example:
|
||||
```python
|
||||
workspaces = await list_workspaces()
|
||||
```
|
||||
|
||||
### Visualization
|
||||
|
||||
**canvas(nodes, edges, title, folder, project)**
|
||||
@@ -3198,8 +3358,8 @@ await edit_note(
|
||||
project="main"
|
||||
)
|
||||
|
||||
# Avoid: Complete rewrite
|
||||
# (unless necessary for major restructuring)
|
||||
# When full rewrite is needed, use overwrite=True
|
||||
await write_note(title="Note", content="...", folder="notes", overwrite=True)
|
||||
```
|
||||
|
||||
### 14. Tagging Strategy
|
||||
|
||||
+290
-90
@@ -5,7 +5,7 @@ The Basic Memory Cloud CLI provides seamless integration between local and cloud
|
||||
## Overview
|
||||
|
||||
The cloud CLI enables you to:
|
||||
- **Toggle cloud mode** - All regular `bm` commands work with cloud when enabled
|
||||
- **Authenticate cloud access** - OAuth/API key credentials are stored locally for cloud operations
|
||||
- **Project-scoped sync** - Each project independently manages its sync configuration
|
||||
- **Explicit operations** - Sync only what you want, when you want
|
||||
- **Bidirectional sync** - Keep local and cloud in sync with rclone bisync
|
||||
@@ -17,6 +17,8 @@ Before using Basic Memory Cloud, you need:
|
||||
|
||||
- **Active Subscription**: An active Basic Memory Cloud subscription is required to access cloud features
|
||||
- **Subscribe**: Visit [https://basicmemory.com/subscribe](https://basicmemory.com/subscribe) to sign up
|
||||
- **Optional**: Cloud is optional. Local-first open-source usage continues without cloud.
|
||||
- **OSS Discount**: Use code `{{OSS_DISCOUNT_CODE}}` for 20% off for 3 months.
|
||||
|
||||
If you attempt to log in without an active subscription, you'll receive a "Subscription Required" error with a link to subscribe.
|
||||
|
||||
@@ -38,7 +40,7 @@ If you attempt to log in without an active subscription, you'll receive a "Subsc
|
||||
|
||||
1. **Cloud-only** - Project exists on cloud, no local copy
|
||||
2. **Cloud + Local (synced)** - Project has a local working directory that syncs
|
||||
3. **Local-only** - Project exists locally (when cloud mode is disabled)
|
||||
3. **Local-only** - Project exists locally and is not routed to cloud
|
||||
|
||||
**Example:**
|
||||
|
||||
@@ -48,9 +50,9 @@ If you attempt to log in without an active subscription, you'll receive a "Subsc
|
||||
# - work: wants local sync at ~/work-notes
|
||||
# - temp: cloud-only, no local sync needed
|
||||
|
||||
bm project add research --local-path ~/Documents/research
|
||||
bm project add work --local-path ~/work-notes
|
||||
bm project add temp # No local sync
|
||||
bm project add research --cloud --local-path ~/Documents/research
|
||||
bm project add work --cloud --local-path ~/work-notes
|
||||
bm project add temp --cloud # No local sync
|
||||
|
||||
# Now you can sync individually (after initial --resync):
|
||||
bm project bisync --name research
|
||||
@@ -66,9 +68,9 @@ bm project bisync --name work
|
||||
|
||||
## Quick Start
|
||||
|
||||
### 1. Enable Cloud Mode
|
||||
### 1. Authenticate Cloud Access
|
||||
|
||||
Authenticate and enable cloud mode:
|
||||
Authenticate with cloud:
|
||||
|
||||
```bash
|
||||
bm cloud login
|
||||
@@ -76,11 +78,12 @@ bm cloud login
|
||||
|
||||
**What this does:**
|
||||
1. Opens browser to Basic Memory Cloud authentication page
|
||||
2. Stores authentication token in `~/.basic-memory/auth/token`
|
||||
3. **Enables cloud mode** - all CLI commands now work against cloud
|
||||
4. Validates your subscription status
|
||||
2. Stores authentication tokens in `~/.basic-memory/basic-memory-cloud.json`
|
||||
3. Validates your subscription status
|
||||
4. Leaves routing behavior unchanged (auth only)
|
||||
|
||||
**Result:** All `bm project`, `bm tools` commands now work with cloud.
|
||||
**Result:** Cloud credentials are available for cloud-routed commands.
|
||||
Apply OSS discount code `{{OSS_DISCOUNT_CODE}}` during checkout to receive 20% off for 3 months.
|
||||
|
||||
### 2. Set Up Sync
|
||||
|
||||
@@ -91,33 +94,38 @@ bm cloud setup
|
||||
```
|
||||
|
||||
**What this does:**
|
||||
1. Installs rclone automatically (if needed)
|
||||
1. Installs rclone with a supported package manager (if needed)
|
||||
2. Fetches your tenant information from cloud
|
||||
3. Generates scoped S3 credentials for sync
|
||||
4. Configures single rclone remote: `basic-memory-cloud`
|
||||
|
||||
**Result:** You're ready to sync projects. No sync directories created yet - those come with project setup.
|
||||
|
||||
Rclone setup uses package managers such as Homebrew, MacPorts, apt, dnf, yum, pacman,
|
||||
zypper, snap, winget, Chocolatey, or Scoop when available. It does not run remote
|
||||
install scripts with `sudo`; if no supported package manager is found, the CLI prints
|
||||
manual install instructions.
|
||||
|
||||
### 3. Add Projects with Sync
|
||||
|
||||
Create projects with optional local sync paths:
|
||||
|
||||
```bash
|
||||
# Create cloud project without local sync
|
||||
bm project add research
|
||||
bm project add research --cloud
|
||||
|
||||
# Create cloud project WITH local sync
|
||||
bm project add research --local-path ~/Documents/research
|
||||
bm project add research --cloud --local-path ~/Documents/research
|
||||
|
||||
# Or configure sync for existing project
|
||||
bm project sync-setup research ~/Documents/research
|
||||
bm cloud sync-setup research ~/Documents/research
|
||||
```
|
||||
|
||||
**What happens under the covers:**
|
||||
|
||||
When you add a project with `--local-path`:
|
||||
1. Project created on cloud at `/app/data/research`
|
||||
2. Local path stored in config: `cloud_projects.research.local_path = "~/Documents/research"`
|
||||
2. Local path stored in config for that project (`local_sync_path`)
|
||||
3. Local directory created if it doesn't exist
|
||||
4. Bisync state directory created at `~/.basic-memory/bisync-state/research/`
|
||||
|
||||
@@ -179,7 +187,8 @@ bm cloud status
|
||||
```
|
||||
|
||||
You should see:
|
||||
- `Mode: Cloud (enabled)`
|
||||
- `OAuth: token valid` (or missing/expired)
|
||||
- `API Key: configured` (or not set)
|
||||
- `Cloud instance is healthy`
|
||||
- Instructions for project sync commands
|
||||
|
||||
@@ -187,16 +196,16 @@ You should see:
|
||||
|
||||
### Understanding Project Commands
|
||||
|
||||
**Key concept:** When cloud mode is enabled, use regular `bm project` commands (not `bm cloud project`).
|
||||
**Key concept:** Use regular `bm project` commands (not `bm cloud project`).
|
||||
|
||||
```bash
|
||||
# In cloud mode:
|
||||
bm project list # Lists cloud projects
|
||||
bm project add research # Creates cloud project
|
||||
# Local route
|
||||
bm project list --local
|
||||
bm project add research ~/Documents/research
|
||||
|
||||
# In local mode:
|
||||
bm project list # Lists local projects
|
||||
bm project add research ~/Documents/research # Creates local project
|
||||
# Cloud route
|
||||
bm project list --cloud
|
||||
bm project add research --cloud
|
||||
```
|
||||
|
||||
### Creating Projects
|
||||
@@ -204,7 +213,7 @@ bm project add research ~/Documents/research # Creates local project
|
||||
**Use case 1: Cloud-only project (no local sync)**
|
||||
|
||||
```bash
|
||||
bm project add temp-notes
|
||||
bm project add temp-notes --cloud
|
||||
```
|
||||
|
||||
**What this does:**
|
||||
@@ -217,7 +226,7 @@ bm project add temp-notes
|
||||
**Use case 2: Cloud project with local sync**
|
||||
|
||||
```bash
|
||||
bm project add research --local-path ~/Documents/research
|
||||
bm project add research --cloud --local-path ~/Documents/research
|
||||
```
|
||||
|
||||
**What this does:**
|
||||
@@ -232,7 +241,7 @@ bm project add research --local-path ~/Documents/research
|
||||
|
||||
```bash
|
||||
# Project already exists on cloud
|
||||
bm project sync-setup research ~/Documents/research
|
||||
bm cloud sync-setup research ~/Documents/research
|
||||
```
|
||||
|
||||
**What this does:**
|
||||
@@ -251,11 +260,35 @@ bm project list
|
||||
```
|
||||
|
||||
**What you see:**
|
||||
- All projects in cloud (when cloud mode enabled)
|
||||
- Local projects always
|
||||
- Cloud projects when credentials are available
|
||||
- Default project marked
|
||||
- Project paths shown
|
||||
- Route-related metadata (for example, local/cloud presence and sync info)
|
||||
|
||||
**Future:** Will show sync status (synced/not synced, last sync time).
|
||||
Example shape (single row for dual-presence projects):
|
||||
|
||||
```text
|
||||
Name Path Local Path Cloud Path CLI Default MCP (stdio)
|
||||
main /basic-memory ~/basic-memory /basic-memory local local
|
||||
specs /specs ~/dev/specs /specs cloud local
|
||||
```
|
||||
|
||||
### When a Project Exists in Both Local and Cloud
|
||||
|
||||
Use routing flags to disambiguate command targets:
|
||||
|
||||
```bash
|
||||
# Force local target for this command
|
||||
bm project info main --local
|
||||
bm project ls --name main --local
|
||||
|
||||
# Force cloud target for this command
|
||||
bm project info main --cloud
|
||||
bm project ls --name main --cloud
|
||||
```
|
||||
|
||||
Default behavior for no-project, no-flag commands is local.
|
||||
For MCP stdio, routing is always local.
|
||||
|
||||
## File Synchronization
|
||||
|
||||
@@ -363,24 +396,28 @@ bm project bisync --name research --dry-run
|
||||
|
||||
**Result:** Safe preview of sync operations.
|
||||
|
||||
### Advanced: List Remote Files
|
||||
### Advanced: List Project Files by Route
|
||||
|
||||
**Use case:** See what files exist on cloud without syncing.
|
||||
**Use case:** Inspect local or cloud project files explicitly.
|
||||
|
||||
```bash
|
||||
# List all files in project
|
||||
# List local project files (default target when no route flag is given)
|
||||
bm project ls --name research
|
||||
bm project ls --name research --local
|
||||
|
||||
# List cloud project files
|
||||
bm project ls --name research --cloud
|
||||
|
||||
# List files in subdirectory
|
||||
bm project ls --name research --path subfolder
|
||||
bm project ls --name research --cloud --path subfolder
|
||||
```
|
||||
|
||||
**What happens:**
|
||||
1. Connects to cloud via rclone
|
||||
2. Lists files in remote project path
|
||||
1. Resolves route from flags (or local default when no route is given)
|
||||
2. Lists files for the chosen project instance
|
||||
3. No files transferred
|
||||
|
||||
**Result:** See cloud file listing.
|
||||
**Result:** See file listing for the target route.
|
||||
|
||||
## Multiple Projects
|
||||
|
||||
@@ -390,9 +427,9 @@ bm project ls --name research --path subfolder
|
||||
|
||||
```bash
|
||||
# Setup multiple projects
|
||||
bm project add research --local-path ~/Documents/research
|
||||
bm project add work --local-path ~/work-notes
|
||||
bm project add personal --local-path ~/personal
|
||||
bm project add research --cloud --local-path ~/Documents/research
|
||||
bm project add work --cloud --local-path ~/work-notes
|
||||
bm project add personal --cloud --local-path ~/personal
|
||||
|
||||
# Establish baselines
|
||||
bm project bisync --name research --resync
|
||||
@@ -417,12 +454,12 @@ bm project bisync --all # Coming soon
|
||||
|
||||
```bash
|
||||
# Projects with sync
|
||||
bm project add research --local-path ~/Documents/research
|
||||
bm project add work --local-path ~/work
|
||||
bm project add research --cloud --local-path ~/Documents/research
|
||||
bm project add work --cloud --local-path ~/work
|
||||
|
||||
# Cloud-only projects
|
||||
bm project add archive
|
||||
bm project add temp-notes
|
||||
bm project add archive --cloud
|
||||
bm project add temp-notes --cloud
|
||||
|
||||
# Sync only the configured ones
|
||||
bm project bisync --name research
|
||||
@@ -433,20 +470,104 @@ bm project bisync --name work
|
||||
|
||||
**Result:** Fine-grained control over what syncs.
|
||||
|
||||
## Disable Cloud Mode
|
||||
## Per-Project Cloud Routing (API Key)
|
||||
|
||||
Return to local mode:
|
||||
Route individual projects through cloud using an API key. This lets you keep some projects local while others route through cloud.
|
||||
|
||||
### Setting Up API Key Auth
|
||||
|
||||
**Option A: Create a key in the web app, then save it locally:**
|
||||
|
||||
```bash
|
||||
bm cloud set-key bmc_abc123...
|
||||
```
|
||||
|
||||
**Option B: Create a key via CLI (requires OAuth login first):**
|
||||
|
||||
```bash
|
||||
bm cloud login # One-time OAuth login
|
||||
bm cloud create-key "my-laptop" # Creates key and saves it locally
|
||||
```
|
||||
|
||||
The API key is account-level — it grants access to all your cloud projects. It's stored in `~/.basic-memory/config.json` as `cloud_api_key`.
|
||||
On POSIX systems, Basic Memory writes `~/.basic-memory/` as user-private (`0700`) and
|
||||
`config.json` as user-read/write only (`0600`). Treat this config file as a credential
|
||||
file when an API key is saved.
|
||||
|
||||
### Setting Project Modes
|
||||
|
||||
```bash
|
||||
# Route a project through cloud
|
||||
bm project set-cloud research
|
||||
|
||||
# Revert to local mode
|
||||
bm project set-local research
|
||||
|
||||
# View project modes
|
||||
bm project list
|
||||
```
|
||||
|
||||
**What happens:**
|
||||
- `set-cloud`: validates the API key exists, then sets the project mode to `cloud` in config
|
||||
- `set-local`: reverts the project to local mode (removes the mode entry from config)
|
||||
- MCP tools and CLI commands for that project will route to `cloud_host/proxy` with the API key as Bearer token
|
||||
|
||||
### How It Works
|
||||
|
||||
When an MCP tool or CLI command runs for a cloud-mode project:
|
||||
|
||||
1. `get_client(project_name="research")` checks the project's mode in config
|
||||
2. If mode is `cloud`, creates an HTTP client pointed at `cloud_host/proxy` with `Authorization: Bearer bmc_...`
|
||||
3. If mode is `local` (default), uses the in-process ASGI transport as usual
|
||||
|
||||
**Routing priority** (highest to lowest):
|
||||
1. Factory injection (cloud app, tests)
|
||||
2. Explicit route override (`--local` / `--cloud`)
|
||||
3. Per-project cloud mode (API key)
|
||||
4. Local ASGI transport (default)
|
||||
|
||||
Route override environment variables:
|
||||
- `BASIC_MEMORY_FORCE_LOCAL=true`
|
||||
- `BASIC_MEMORY_FORCE_CLOUD=true`
|
||||
- `BASIC_MEMORY_EXPLICIT_ROUTING=true`
|
||||
|
||||
No-project, no-flag CLI commands default to local routing.
|
||||
|
||||
### Configuration Example
|
||||
|
||||
```json
|
||||
{
|
||||
"projects": {
|
||||
"personal": "/Users/me/notes",
|
||||
"research": "/Users/me/research"
|
||||
},
|
||||
"project_modes": {
|
||||
"research": "cloud"
|
||||
},
|
||||
"cloud_api_key": "bmc_abc123...",
|
||||
"cloud_host": "https://cloud.basicmemory.com",
|
||||
"default_project": "personal"
|
||||
}
|
||||
```
|
||||
|
||||
In this example, `personal` stays local and `research` routes through cloud. Projects not listed in `project_modes` default to local.
|
||||
|
||||
### Sync Behavior
|
||||
|
||||
Cloud-mode projects are automatically skipped during local file sync (background sync and file watching). Their files live on the cloud instance, not locally.
|
||||
|
||||
## OAuth Logout
|
||||
|
||||
```bash
|
||||
bm cloud logout
|
||||
```
|
||||
|
||||
**What this does:**
|
||||
1. Disables cloud mode in config
|
||||
2. All commands now work locally
|
||||
3. Auth token remains (can re-enable with login)
|
||||
1. Removes stored OAuth token(s)
|
||||
2. Does not change per-project route configuration
|
||||
3. Does not change command routing defaults
|
||||
|
||||
**Result:** All `bm` commands work with local projects again.
|
||||
**Result:** OAuth session is cleared. API-key-based routing still works if `cloud_api_key` is configured.
|
||||
|
||||
## Filter Configuration
|
||||
|
||||
@@ -461,33 +582,62 @@ bm cloud logout
|
||||
**Default patterns:**
|
||||
|
||||
```gitignore
|
||||
# Version control
|
||||
.git/**
|
||||
|
||||
# Python
|
||||
__pycache__/**
|
||||
*.pyc
|
||||
.venv/**
|
||||
venv/**
|
||||
|
||||
# Node.js
|
||||
node_modules/**
|
||||
# Hidden files and directories
|
||||
.*
|
||||
|
||||
# Basic Memory internals
|
||||
memory.db/**
|
||||
memory.db-shm/**
|
||||
memory.db-wal/**
|
||||
config.json/**
|
||||
watch-status.json/**
|
||||
.bmignore.rclone/**
|
||||
*.db
|
||||
*.db-shm
|
||||
*.db-wal
|
||||
config.json
|
||||
|
||||
# Version control
|
||||
.git
|
||||
.svn
|
||||
|
||||
# Python
|
||||
__pycache__
|
||||
*.pyc
|
||||
*.pyo
|
||||
*.pyd
|
||||
.pytest_cache
|
||||
.coverage
|
||||
*.egg-info
|
||||
.tox
|
||||
.mypy_cache
|
||||
.ruff_cache
|
||||
|
||||
# Virtual environments
|
||||
.venv
|
||||
venv
|
||||
env
|
||||
.env
|
||||
|
||||
# Node.js
|
||||
node_modules
|
||||
|
||||
# Build artifacts
|
||||
build
|
||||
dist
|
||||
.cache
|
||||
|
||||
# IDE
|
||||
.idea
|
||||
.vscode
|
||||
|
||||
# OS files
|
||||
.DS_Store/**
|
||||
Thumbs.db/**
|
||||
.DS_Store
|
||||
Thumbs.db
|
||||
desktop.ini
|
||||
|
||||
# Environment files
|
||||
.env/**
|
||||
.env.local/**
|
||||
# Obsidian
|
||||
.obsidian
|
||||
|
||||
# Temporary files
|
||||
*.tmp
|
||||
*.swp
|
||||
*.swo
|
||||
*~
|
||||
```
|
||||
|
||||
**How it works:**
|
||||
@@ -496,6 +646,11 @@ Thumbs.db/**
|
||||
3. Rclone uses filters during sync
|
||||
4. Same patterns used by all projects
|
||||
|
||||
During conversion, file patterns exclude the direct match and recursive contents.
|
||||
For example, `config.json` becomes both `- config.json` and `- config.json/**`,
|
||||
while `.*` becomes both `- .*` and `- .*/**`. Directory-only patterns keep
|
||||
their trailing slash, so `cache/` becomes `- cache/` and `- cache/**`.
|
||||
|
||||
**Customizing:**
|
||||
|
||||
```bash
|
||||
@@ -503,7 +658,7 @@ Thumbs.db/**
|
||||
code ~/.basic-memory/.bmignore
|
||||
|
||||
# Add custom patterns
|
||||
echo "*.tmp/**" >> ~/.basic-memory/.bmignore
|
||||
echo "*.tmp" >> ~/.basic-memory/.bmignore
|
||||
|
||||
# Next sync uses updated patterns
|
||||
bm project bisync --name research
|
||||
@@ -511,6 +666,33 @@ bm project bisync --name research
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Rclone Setup Cannot Install Automatically
|
||||
|
||||
**Problem:** `bm cloud setup` cannot find a supported package manager, or package-manager
|
||||
installation fails.
|
||||
|
||||
**Explanation:** The CLI avoids remote privileged install scripts. It only invokes known
|
||||
package managers and otherwise asks you to install rclone manually.
|
||||
|
||||
**Solution:** Install rclone with your OS package manager, then rerun setup:
|
||||
|
||||
```bash
|
||||
# macOS
|
||||
brew install rclone
|
||||
|
||||
# Debian/Ubuntu
|
||||
sudo apt install rclone
|
||||
|
||||
# Fedora
|
||||
sudo dnf install rclone
|
||||
|
||||
# Arch
|
||||
sudo pacman -S rclone
|
||||
|
||||
# After rclone is on PATH
|
||||
bm cloud setup
|
||||
```
|
||||
|
||||
### Authentication Issues
|
||||
|
||||
**Problem:** "Authentication failed" or "Invalid token"
|
||||
@@ -626,7 +808,7 @@ bm project sync --name research
|
||||
**Solution:**
|
||||
|
||||
```bash
|
||||
bm project sync-setup research ~/Documents/research
|
||||
bm cloud sync-setup research ~/Documents/research
|
||||
bm project bisync --name research --resync
|
||||
```
|
||||
|
||||
@@ -646,37 +828,53 @@ If instance is down, wait a few minutes and retry.
|
||||
|
||||
- **Authentication**: OAuth 2.1 with PKCE flow
|
||||
- **Tokens**: Stored securely in `~/.basic-memory/basic-memory-cloud.json`
|
||||
- **API keys**: Stored in `~/.basic-memory/config.json`, which is written with private file permissions on POSIX systems
|
||||
- **Transport**: All data encrypted in transit (HTTPS)
|
||||
- **Credentials**: Scoped S3 credentials (read-write to your tenant only)
|
||||
- **Rclone setup**: Uses package managers or manual instructions; no remote privileged install-script fallback
|
||||
- **Isolation**: Your data isolated from other tenants
|
||||
- **Ignore patterns**: Sensitive files automatically excluded via `.bmignore`
|
||||
|
||||
## Command Reference
|
||||
|
||||
### Cloud Mode Management
|
||||
### Cloud Authentication
|
||||
|
||||
```bash
|
||||
bm cloud login # Authenticate and enable cloud mode
|
||||
bm cloud logout # Disable cloud mode
|
||||
bm cloud status # Check cloud mode and instance health
|
||||
bm cloud login # Authenticate and store OAuth credentials
|
||||
bm cloud logout # Remove stored OAuth credentials
|
||||
bm cloud status # Check auth state and instance health
|
||||
bm cloud promo --off # Disable CLI cloud promo notices
|
||||
```
|
||||
|
||||
### API Key Management
|
||||
|
||||
```bash
|
||||
bm cloud set-key <key> # Save a cloud API key (bmc_ prefixed)
|
||||
bm cloud create-key <name> # Create API key via cloud API (requires OAuth login)
|
||||
```
|
||||
|
||||
### Setup
|
||||
|
||||
```bash
|
||||
bm cloud setup # Install rclone and configure credentials
|
||||
bm cloud setup # Install rclone via package manager and configure credentials
|
||||
```
|
||||
|
||||
### Project Management
|
||||
|
||||
When cloud mode is enabled:
|
||||
```bash
|
||||
bm project list --local # Local project list
|
||||
bm project list --cloud # Cloud project list
|
||||
bm project add <name> --cloud # Create cloud project (no sync)
|
||||
bm project add <name> --cloud --local-path <path> # Create with local sync
|
||||
bm cloud sync-setup <name> <path> # Add sync to existing project
|
||||
bm project rm <name> # Delete project
|
||||
```
|
||||
|
||||
### Per-Project Routing
|
||||
|
||||
```bash
|
||||
bm project list # List cloud projects
|
||||
bm project add <name> # Create cloud project (no sync)
|
||||
bm project add <name> --local-path <path> # Create with local sync
|
||||
bm project sync-setup <name> <path> # Add sync to existing project
|
||||
bm project rm <name> # Delete project
|
||||
bm project set-cloud <name> # Route project through cloud (requires API key)
|
||||
bm project set-local <name> # Revert project to local mode
|
||||
```
|
||||
|
||||
### File Synchronization
|
||||
@@ -697,18 +895,20 @@ bm project bisync --name <project> --verbose
|
||||
bm project check --name <project>
|
||||
bm project check --name <project> --one-way
|
||||
|
||||
# List remote files
|
||||
bm project ls --name <project>
|
||||
bm project ls --name <project> --path <subpath>
|
||||
# List project files by route
|
||||
bm project ls --name <project> # Default target: local
|
||||
bm project ls --name <project> --local
|
||||
bm project ls --name <project> --cloud
|
||||
bm project ls --name <project> --cloud --path <subpath>
|
||||
```
|
||||
|
||||
## Summary
|
||||
|
||||
**Basic Memory Cloud uses project-scoped sync:**
|
||||
|
||||
1. **Enable cloud mode** - `bm cloud login`
|
||||
1. **Authenticate cloud access** - `bm cloud login`
|
||||
2. **Install rclone** - `bm cloud setup`
|
||||
3. **Add projects with sync** - `bm project add research --local-path ~/Documents/research`
|
||||
3. **Add projects with sync** - `bm project add research --cloud --local-path ~/Documents/research`
|
||||
4. **Preview first sync** - `bm project bisync --name research --resync --dry-run`
|
||||
5. **Establish baseline** - `bm project bisync --name research --resync`
|
||||
6. **Daily workflow** - `bm project bisync --name research`
|
||||
|
||||
@@ -0,0 +1,91 @@
|
||||
# Cloud Semantic Search Value (Customer-Facing Technical Story)
|
||||
|
||||
This document explains why teams should buy cloud semantic search even when local search exists.
|
||||
|
||||
## Core Promise
|
||||
|
||||
Markdown files remain the source of truth in both local and cloud modes.
|
||||
|
||||
- Files are portable.
|
||||
- Search indexes are derived and rebuildable.
|
||||
- You never get locked into proprietary document storage.
|
||||
|
||||
## The Customer Problem
|
||||
|
||||
Teams paying for cloud are usually not optimizing for "can this run locally." They are optimizing for:
|
||||
|
||||
- finding the right note the first time,
|
||||
- keeping retrieval quality high as note volume grows,
|
||||
- avoiding search slowdowns while content is actively changing,
|
||||
- getting consistent results across users, agents, and sessions.
|
||||
|
||||
## Why Cloud Is the Aspirin
|
||||
|
||||
Cloud semantic search is the immediate pain reliever because it fixes the problems users feel right now.
|
||||
|
||||
### 1) Better hit rate on real queries
|
||||
|
||||
Cloud uses stronger managed embeddings than the default local model, which improves semantic recall for paraphrases and vague questions.
|
||||
|
||||
Customer outcome:
|
||||
|
||||
- fewer "I know this exists but search missed it" moments,
|
||||
- less query rewording,
|
||||
- faster time to answer.
|
||||
|
||||
### 2) Better behavior under active workloads
|
||||
|
||||
Cloud indexing runs out of band in workers, so indexing does not compete with interactive read/write traffic.
|
||||
|
||||
Customer outcome:
|
||||
|
||||
- stable search responsiveness during heavy updates,
|
||||
- fresher semantic results shortly after edits,
|
||||
- less user-visible performance variance.
|
||||
|
||||
### 3) Better consistency for shared knowledge
|
||||
|
||||
Cloud retrieval runs against a centralized tenant index, so teams and agents resolve against the same semantic state.
|
||||
|
||||
Customer outcome:
|
||||
|
||||
- fewer "works on my machine" search differences,
|
||||
- more predictable agent behavior across environments,
|
||||
- easier cross-user collaboration on large knowledge bases.
|
||||
|
||||
### 4) Better quality at higher scale
|
||||
|
||||
With Postgres + `pgvector` per tenant, cloud can sustain larger note collections and higher query volumes than typical local setups.
|
||||
|
||||
Customer outcome:
|
||||
|
||||
- confidence as repositories grow to tens of thousands of notes,
|
||||
- less need for user-side tuning,
|
||||
- fewer quality regressions as usage increases.
|
||||
|
||||
## Local Is the Vitamin
|
||||
|
||||
Local semantic search still matters and should stay strong.
|
||||
|
||||
- offline use,
|
||||
- privacy-first operation,
|
||||
- no cloud dependency,
|
||||
- user-controlled runtime.
|
||||
|
||||
It compounds long-term ownership and resilience, but does not remove the immediate pain points cloud solves for teams at scale.
|
||||
|
||||
## Recommended Messaging
|
||||
|
||||
One-liner:
|
||||
|
||||
"Cloud semantic search is the aspirin: it fixes retrieval quality and performance pain now. Local semantic search is the vitamin: it builds long-term control and resilience."
|
||||
|
||||
Long form:
|
||||
|
||||
"Basic Memory keeps markdown as the source of truth everywhere. Local gives privacy and offline control. Cloud adds immediate, measurable improvements in search quality, consistency, and responsiveness for teams and agents running at scale."
|
||||
|
||||
## Packaging Guidance
|
||||
|
||||
- Base: local FTS plus optional local semantic search.
|
||||
- Cloud value: higher semantic quality, stable performance under load, and consistent team-wide retrieval.
|
||||
- Keep interfaces pluggable (`EmbeddingProvider`, vector backend protocol) so implementation can evolve without changing user workflows.
|
||||
@@ -0,0 +1,499 @@
|
||||
# Logfire Instrumentation Strategy
|
||||
|
||||
## Why
|
||||
|
||||
We want Logfire in Basic Memory for two specific use cases:
|
||||
|
||||
1. Local development and performance investigation
|
||||
2. Cloud deployments where Basic Memory runs inside Basic Memory Cloud
|
||||
|
||||
This instrumentation must be:
|
||||
|
||||
- Disabled by default
|
||||
- Useful when enabled
|
||||
- Safe for local-first users
|
||||
- Searchable in Logfire over time
|
||||
|
||||
The previous integration added telemetry, but it leaned too much on generic framework instrumentation. That created noisy spans with weak names and made the trace view harder to navigate. This strategy favors manual instrumentation around Basic Memory's real units of work.
|
||||
|
||||
## Core Principles
|
||||
|
||||
### 1. Default-off
|
||||
|
||||
Basic Memory should ship with Logfire disabled unless the operator explicitly enables it.
|
||||
|
||||
That means:
|
||||
|
||||
- no required token for normal local usage
|
||||
- no surprise outbound telemetry
|
||||
- no behavior change for existing users
|
||||
|
||||
### 2. Manual spans over automatic framework spans
|
||||
|
||||
We should not rely on broad auto-instrumentation for FastAPI, MCP, SQLAlchemy, or HTTP as the primary experience.
|
||||
|
||||
Why:
|
||||
|
||||
- auto-generated span names are often generic
|
||||
- routes and middleware produce too many low-signal spans
|
||||
- it becomes harder to answer product questions like "why was `write_note` slow?" or "where did sync time go?"
|
||||
|
||||
The preferred model is:
|
||||
|
||||
- one meaningful root span per high-level operation
|
||||
- a small number of child spans for important phases
|
||||
- optional targeted instrumentation only where it adds clear value
|
||||
|
||||
### 3. Logs must live inside traces
|
||||
|
||||
Basic Memory already uses `loguru` pervasively. The Logfire integration should preserve that and make those logs visible inside the active trace/span context.
|
||||
|
||||
If traces exist but the logs are detached from them, the integration is not doing its job.
|
||||
|
||||
### 4. Stable names, selective attributes
|
||||
|
||||
Span names should describe the operation class, not the specific input.
|
||||
|
||||
Good:
|
||||
|
||||
- `mcp.tool.write_note`
|
||||
- `sync.project.scan`
|
||||
- `search.execute`
|
||||
- `routing.resolve_project`
|
||||
|
||||
Bad:
|
||||
|
||||
- `Searching for "foo bar baz"`
|
||||
- `POST /v2/projects/123/search/`
|
||||
- `write note to /specs/api.md`
|
||||
|
||||
Dynamic values belong in attributes, not in the span name.
|
||||
|
||||
## What We Should Not Do
|
||||
|
||||
### Avoid broad FastAPI auto-instrumentation
|
||||
|
||||
We should not turn on `instrument_fastapi()` and treat that as the main telemetry story.
|
||||
|
||||
It may still be useful in narrowly scoped debugging, but it should not define the production trace shape. The meaningful root spans should come from Basic Memory's own entrypoints and service boundaries.
|
||||
|
||||
### Avoid per-file spans by default
|
||||
|
||||
`sync` can process many files. A span per file will explode trace cardinality and make performance views noisy.
|
||||
|
||||
Default behavior should be:
|
||||
|
||||
- one span for the project sync
|
||||
- child spans for scan, move handling, delete handling, markdown sync batch, relation resolution, embedding sync, watermark update
|
||||
- per-file spans only for failures or very slow outliers
|
||||
|
||||
### Avoid high-cardinality attributes on every span
|
||||
|
||||
Do not attach large or highly variable values everywhere:
|
||||
|
||||
- raw note content
|
||||
- file bodies
|
||||
- long search text
|
||||
- arbitrary metadata blobs
|
||||
- unique IDs that make every span shape distinct
|
||||
|
||||
Prefer compact, queryable attributes:
|
||||
|
||||
- `project_name`
|
||||
- `workspace_id`
|
||||
- `route_mode`
|
||||
- `scan_type`
|
||||
- `file_count`
|
||||
- `result_count`
|
||||
- `search_type`
|
||||
- `retrieval_mode`
|
||||
- `duration_ms`
|
||||
|
||||
## Proposed Architecture
|
||||
|
||||
Add a dedicated telemetry module in core Basic Memory, separate from logging setup.
|
||||
|
||||
Suggested shape:
|
||||
|
||||
```python
|
||||
# basic_memory/telemetry.py
|
||||
|
||||
def configure_telemetry(service_name: str, *, enable_logfire: bool) -> None: ...
|
||||
def telemetry_enabled() -> bool: ...
|
||||
def span(name: str, **attrs): ...
|
||||
def bind_telemetry_context(**attrs): ...
|
||||
```
|
||||
|
||||
This module should:
|
||||
|
||||
- configure Logfire only when explicitly enabled
|
||||
- set up the Logfire `loguru` handler
|
||||
- expose lightweight helpers so application code does not import `logfire` directly everywhere
|
||||
- degrade cleanly to no-op behavior when disabled
|
||||
|
||||
This keeps the rest of the codebase readable and makes it easy to reason about what telemetry is doing.
|
||||
|
||||
## Logging Integration Strategy
|
||||
|
||||
### Goal
|
||||
|
||||
When a span is active, logs emitted through `loguru` during that operation should show up in the same trace.
|
||||
|
||||
### Preferred design
|
||||
|
||||
1. Configure Logfire once in the telemetry bootstrap
|
||||
2. Add the Logfire `loguru` handler to the existing `loguru` configuration
|
||||
3. At operation boundaries, bind stable contextual fields with `loguru`
|
||||
4. Let logs emitted inside the span inherit the active trace context
|
||||
|
||||
### Context to bind
|
||||
|
||||
Bind only the fields that help correlate work across the system:
|
||||
|
||||
- `service_name`
|
||||
- `entrypoint`
|
||||
- `project_name`
|
||||
- `workspace_id`
|
||||
- `route_mode`
|
||||
- `tool_name`
|
||||
- `command_name`
|
||||
|
||||
This binding should happen at the root of an operation, not deep in leaf functions.
|
||||
|
||||
### Important nuance
|
||||
|
||||
We should not try to encode the entire trace model into logger extras. The logger context should be a human-meaningful slice of the active operation. Trace linkage comes from the active Logfire/OpenTelemetry context; logger extras are there to improve searchability and readability.
|
||||
|
||||
## Span Model
|
||||
|
||||
### Root spans
|
||||
|
||||
Each user-visible or system-visible operation should get one root span.
|
||||
|
||||
Examples:
|
||||
|
||||
- `cli.command.status`
|
||||
- `cli.command.project_sync`
|
||||
- `api.request.search`
|
||||
- `mcp.tool.write_note`
|
||||
- `mcp.tool.read_note`
|
||||
- `mcp.tool.search_notes`
|
||||
- `sync.project.run`
|
||||
- `db.semantic_backfill`
|
||||
|
||||
### Child spans
|
||||
|
||||
Child spans should represent real phases whose duration we care about.
|
||||
|
||||
Examples:
|
||||
|
||||
- `routing.client_session`
|
||||
- `routing.resolve_project`
|
||||
- `routing.resolve_workspace`
|
||||
- `api.search.execute`
|
||||
- `sync.project.scan`
|
||||
- `sync.project.detect_moves`
|
||||
- `sync.project.apply_changes`
|
||||
- `sync.project.resolve_relations`
|
||||
- `sync.project.sync_embeddings`
|
||||
- `sync.file.markdown`
|
||||
- `sync.file.regular`
|
||||
- `search.execute`
|
||||
- `search.relaxed_fts_retry`
|
||||
- `db.init`
|
||||
- `db.migrate`
|
||||
|
||||
### Span naming rules
|
||||
|
||||
- Use dot-separated names
|
||||
- Start with subsystem
|
||||
- Keep the verb at the end
|
||||
- Keep names stable across runs
|
||||
- Never include request-specific text in the span name
|
||||
|
||||
## Attribute Taxonomy
|
||||
|
||||
### Required attributes on root spans
|
||||
|
||||
Every root span should have a small common set:
|
||||
|
||||
- `service_name`
|
||||
- `entrypoint`
|
||||
- `project_name` when applicable
|
||||
- `workspace_id` when applicable
|
||||
- `route_mode` with values like `local_asgi`, `cloud_proxy`, `factory`
|
||||
|
||||
### Operation-specific attributes
|
||||
|
||||
Examples:
|
||||
|
||||
For search:
|
||||
|
||||
- `search_type`
|
||||
- `retrieval_mode`
|
||||
- `page`
|
||||
- `page_size`
|
||||
- `result_count`
|
||||
- `fallback_used`
|
||||
|
||||
For sync:
|
||||
|
||||
- `scan_type`
|
||||
- `force_full`
|
||||
- `new_count`
|
||||
- `modified_count`
|
||||
- `deleted_count`
|
||||
- `move_count`
|
||||
- `skipped_count`
|
||||
- `embeddings_enabled`
|
||||
|
||||
For note operations:
|
||||
|
||||
- `tool_name`
|
||||
- `note_type`
|
||||
- `directory`
|
||||
- `overwrite`
|
||||
- `output_format`
|
||||
|
||||
### Attributes to avoid by default
|
||||
|
||||
- full `query.text`
|
||||
- full note titles if they create privacy or cardinality issues
|
||||
- file content
|
||||
- raw frontmatter
|
||||
- raw HTTP bodies
|
||||
|
||||
If we need richer payloads for a local debugging session, that should be an explicit temporary mode, not the default telemetry shape.
|
||||
|
||||
## Instrumentation Plan By Layer
|
||||
|
||||
### 1. Entrypoints
|
||||
|
||||
Instrument these first:
|
||||
|
||||
- `cli.app` callback and major commands
|
||||
- API lifespan and selected routers
|
||||
- MCP server lifespan
|
||||
- MCP tool entrypoints
|
||||
|
||||
Why:
|
||||
|
||||
- this establishes clean root spans
|
||||
- it gives us trace boundaries that match how users think about the product
|
||||
|
||||
### 2. Routing and context resolution
|
||||
|
||||
Instrument:
|
||||
|
||||
- client routing decisions
|
||||
- workspace resolution
|
||||
- project resolution
|
||||
- default-project fallback
|
||||
|
||||
Why:
|
||||
|
||||
- Basic Memory has local/cloud/per-project routing logic
|
||||
- when something is slow or surprising, we need to know which path was taken
|
||||
|
||||
### 3. Sync and indexing
|
||||
|
||||
This is the highest-value area to instrument deeply.
|
||||
|
||||
Instrument:
|
||||
|
||||
- sync root
|
||||
- scan strategy decision
|
||||
- filesystem scan
|
||||
- move detection
|
||||
- delete handling
|
||||
- markdown sync phase
|
||||
- relation resolution
|
||||
- vector embedding sync
|
||||
- scan watermark update
|
||||
|
||||
Why:
|
||||
|
||||
- this is where performance work will happen
|
||||
- cloud and local both benefit from this visibility
|
||||
|
||||
### 4. Search
|
||||
|
||||
Instrument:
|
||||
|
||||
- search execution
|
||||
- retrieval mode
|
||||
- relaxed FTS fallback
|
||||
- result shaping
|
||||
|
||||
Why:
|
||||
|
||||
- search is user-facing and latency-sensitive
|
||||
- hybrid/vector/FTS paths need to be distinguishable
|
||||
|
||||
### 5. Database and initialization
|
||||
|
||||
Instrument selectively:
|
||||
|
||||
- DB init
|
||||
- migrations
|
||||
- semantic backfill
|
||||
- connection mode selection
|
||||
|
||||
Avoid full automatic SQL span firehose by default.
|
||||
|
||||
## Recommended Rollout Phases
|
||||
|
||||
## Task List
|
||||
|
||||
- [x] Phase 1: Bootstrap and config gating
|
||||
- [x] Phase 2: Root spans for entrypoints and primary operations
|
||||
- [x] Phase 3: Child spans for sync, search, and routing
|
||||
- [x] Phase 4: Failure-focused detail and final verification
|
||||
- [x] Phase 5: Loguru context binding and scoped context inheritance
|
||||
|
||||
## Recommended Rollout Phases
|
||||
|
||||
### Phase 1: Bootstrap and config gating
|
||||
|
||||
Add:
|
||||
|
||||
- telemetry bootstrap module
|
||||
- config/env gating
|
||||
- `loguru` + Logfire handler integration
|
||||
|
||||
This gives immediate value with low noise.
|
||||
|
||||
### Phase 2: Root spans for entrypoints and primary operations
|
||||
|
||||
Add:
|
||||
|
||||
- root spans for CLI, API, MCP, and main MCP tools
|
||||
- stable root attributes for project, workspace, route mode, and operation type
|
||||
|
||||
This gives us clean top-level traces that match how users think about the product.
|
||||
|
||||
### Phase 3: Child spans for sync, search, and routing
|
||||
|
||||
Add child spans to:
|
||||
|
||||
- sync
|
||||
- search
|
||||
- routing
|
||||
|
||||
This is the main performance-investigation layer.
|
||||
|
||||
### Phase 4: Failure-focused detail
|
||||
|
||||
Add selective deeper spans/log enrichment for:
|
||||
|
||||
- sync failures
|
||||
- relation resolution failures
|
||||
- slow file operations
|
||||
- cloud routing/auth failures
|
||||
|
||||
This keeps normal traces clean while improving debuggability.
|
||||
|
||||
### Phase 5: Loguru context binding and scoped context inheritance
|
||||
|
||||
Add:
|
||||
|
||||
- context-local telemetry state in `basic_memory.telemetry`
|
||||
- a shared `scope(...)` helper that opens a span and binds stable logger context together
|
||||
- context inheritance for routing, sync, and search so downstream `loguru` logs carry the active operation fields
|
||||
|
||||
This makes the trace view and the log stream tell the same story without forcing logger rewrites across the codebase.
|
||||
|
||||
## Local Dev Playbook
|
||||
|
||||
The fastest way to sanity-check the current trace shape is:
|
||||
|
||||
```bash
|
||||
LOGFIRE_TOKEN=lf_... just telemetry-smoke
|
||||
```
|
||||
|
||||
What this does:
|
||||
|
||||
- creates an isolated temp home, config dir, and project path
|
||||
- enables Logfire for the run
|
||||
- automatically exports to Logfire when `LOGFIRE_TOKEN` is present
|
||||
- defaults `BASIC_MEMORY_SEMANTIC_SEARCH_ENABLED=false` so the smoke run stays fast and trace-friendly
|
||||
- disables promo telemetry so the trace is about Basic Memory work, not analytics noise
|
||||
- runs a small CLI workflow:
|
||||
- `project add`
|
||||
- `tool write-note`
|
||||
- `tool read-note`
|
||||
- `tool edit-note`
|
||||
- `tool build-context`
|
||||
- `tool search-notes`
|
||||
- `doctor`
|
||||
|
||||
If you want to exercise the instrumentation without exporting anything upstream:
|
||||
|
||||
```bash
|
||||
BASIC_MEMORY_LOGFIRE_SEND_TO_LOGFIRE=false just telemetry-smoke
|
||||
```
|
||||
|
||||
If you want the smoke run to include vector or hybrid retrieval spans too:
|
||||
|
||||
```bash
|
||||
LOGFIRE_TOKEN=lf_... BASIC_MEMORY_SEMANTIC_SEARCH_ENABLED=true just telemetry-smoke
|
||||
```
|
||||
|
||||
The recipe sets `BASIC_MEMORY_LOGFIRE_ENVIRONMENT=telemetry-smoke` by default so these traces are easy to isolate in Logfire. Override it if you want the smoke traces grouped under a different environment name.
|
||||
|
||||
### What to look for
|
||||
|
||||
You should see a small set of comparable root spans rather than a framework-generated span forest:
|
||||
|
||||
- `cli.command.project`
|
||||
- `cli.command.tool`
|
||||
- `mcp.tool.write_note`
|
||||
- `mcp.tool.read_note`
|
||||
- `mcp.tool.edit_note`
|
||||
- `mcp.tool.build_context`
|
||||
- `mcp.tool.search_notes`
|
||||
- `sync.project.run`
|
||||
|
||||
You should also see correlated logs under those traces with stable fields like:
|
||||
|
||||
- `project_name`
|
||||
- `route_mode`
|
||||
- `tool_name`
|
||||
- `entrypoint`
|
||||
|
||||
### Expected nuance
|
||||
|
||||
`doctor` creates its own temporary project on purpose. That means the sync trace will usually show a different project name than the `telemetry-smoke` write/search traces. That is fine for smoke testing because the goal is to confirm:
|
||||
|
||||
- root span names are meaningful
|
||||
- scoped logs stay attached to the active trace
|
||||
- routing, tool, search, and sync phases are easy to distinguish
|
||||
|
||||
## Validation Checklist
|
||||
|
||||
We should consider the integration successful when the following are true:
|
||||
|
||||
1. With telemetry disabled, Basic Memory behaves exactly as it does today.
|
||||
2. With telemetry enabled, one user action produces one obvious root span.
|
||||
3. Logs emitted during that action are visible inside the same trace.
|
||||
4. A search in Logfire for `mcp.tool.write_note` or `sync.project.run` returns comparable spans across runs.
|
||||
5. Trace views show phase timing clearly without drowning in framework noise.
|
||||
6. Sensitive payloads are not captured by default.
|
||||
|
||||
## Immediate Implementation Direction
|
||||
|
||||
When we start coding, the first pass should be:
|
||||
|
||||
1. Add `basic_memory.telemetry`
|
||||
2. Add config/env switches for `enabled`, `send_to_logfire`, and service name
|
||||
3. Wire telemetry bootstrap into CLI, API, and MCP entrypoints
|
||||
4. Configure `loguru` to emit to both existing sinks and the Logfire handler when enabled
|
||||
5. Add manual root spans around:
|
||||
- CLI commands
|
||||
- API request handlers we care about
|
||||
- MCP tool entrypoints
|
||||
- sync root
|
||||
- search root
|
||||
6. Add child spans to the sync and routing phases only after the root span model feels clean
|
||||
|
||||
That gives us a strong foundation without repeating the earlier "turn on instrumentation everywhere" approach.
|
||||
@@ -0,0 +1,138 @@
|
||||
# MCP UI Bakeoff - Instructions & Test Plan
|
||||
|
||||
Last updated: 2026-02-02
|
||||
|
||||
## Scope
|
||||
|
||||
Compare three presentation paths for Basic Memory MCP tools:
|
||||
|
||||
1. **Tool‑UI (React)** via MCP App resources.
|
||||
2. **MCP‑UI Python SDK** embedded UI resources (legacy host path).
|
||||
3. **ASCII/ANSI** output for TUI clients.
|
||||
|
||||
This doc is the running instruction set and test plan. Update as implementation progresses.
|
||||
|
||||
---
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- Repo: `basic-memory` (worktree: `basic-memory-mcp-ui-poc`)
|
||||
- Node for tool‑ui build (already used for POC)
|
||||
- Python 3.12+ with `uv`
|
||||
|
||||
Optional (for MCP‑UI Python SDK path):
|
||||
|
||||
- Local repo: `/Users/phernandez/dev/mcp-ui`
|
||||
- Install the server SDK into the Basic Memory venv:
|
||||
- `uv pip install -e /Users/phernandez/dev/mcp-ui/sdks/python/server`
|
||||
|
||||
---
|
||||
|
||||
## Build / Refresh Steps
|
||||
|
||||
### Tool‑UI React bundle
|
||||
|
||||
```bash
|
||||
cd ui/tool-ui-react
|
||||
npm install
|
||||
npm run build
|
||||
```
|
||||
|
||||
This regenerates:
|
||||
|
||||
- `src/basic_memory/mcp/ui/html/search-results-tool-ui.html`
|
||||
- `src/basic_memory/mcp/ui/html/note-preview-tool-ui.html`
|
||||
|
||||
---
|
||||
|
||||
## How to Run the MCP Server
|
||||
|
||||
```bash
|
||||
basic-memory mcp --transport stdio
|
||||
```
|
||||
|
||||
Optional to pick UI variant for MCP App resources:
|
||||
|
||||
```bash
|
||||
export BASIC_MEMORY_MCP_UI_VARIANT=tool-ui # or vanilla | mcp-ui
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Test Cases
|
||||
|
||||
### 1) MCP App Resource UI (tool‑ui / vanilla / mcp‑ui)
|
||||
|
||||
Tools:
|
||||
- `search_notes`
|
||||
- `read_note`
|
||||
|
||||
Expect:
|
||||
- Tool meta points to `ui://basic-memory/search-results` and `ui://basic-memory/note-preview`
|
||||
- Resource content differs by `BASIC_MEMORY_MCP_UI_VARIANT`
|
||||
- Variant‑specific URIs also available:
|
||||
- `ui://basic-memory/search-results/vanilla`
|
||||
- `ui://basic-memory/search-results/tool-ui`
|
||||
- `ui://basic-memory/search-results/mcp-ui`
|
||||
- `ui://basic-memory/note-preview/vanilla`
|
||||
- `ui://basic-memory/note-preview/tool-ui`
|
||||
- `ui://basic-memory/note-preview/mcp-ui`
|
||||
|
||||
Manual check:
|
||||
- Trigger tool in MCP‑App‑capable host and confirm UI renders.
|
||||
|
||||
---
|
||||
|
||||
### 2) Text / JSON Output Modes
|
||||
|
||||
Tools:
|
||||
- `search_notes(output_format="text" | "json")`
|
||||
- `read_note(output_format="text" | "json")`
|
||||
- `write_note(output_format="text" | "json")`
|
||||
- `edit_note(output_format="text" | "json")`
|
||||
- `recent_activity(output_format="text" | "json")`
|
||||
- `list_memory_projects(output_format="text" | "json")`
|
||||
- `create_memory_project(output_format="text" | "json")`
|
||||
- `delete_note(output_format="text" | "json")`
|
||||
- `move_note(output_format="text" | "json")`
|
||||
- `build_context(output_format="json" | "text")`
|
||||
|
||||
Expect:
|
||||
- `text` mode preserves existing human-readable responses.
|
||||
- `json` mode returns structured dict/list payloads for machine-readable clients.
|
||||
|
||||
Automated:
|
||||
- `uv run pytest test-int/mcp/test_output_format_json_integration.py`
|
||||
|
||||
---
|
||||
|
||||
### 3) MCP‑UI Python SDK (embedded UI resource)
|
||||
|
||||
Tools (embedded resource responses):
|
||||
- `search_notes_ui` (MCP‑UI SDK)
|
||||
- `read_note_ui` (MCP‑UI SDK)
|
||||
|
||||
Expected output:
|
||||
- Tool response content contains an EmbeddedResource (`type: "resource"`)
|
||||
- `mimeType` is `text/html`
|
||||
- `_meta` includes:
|
||||
- `mcpui.dev/ui-preferred-frame-size`
|
||||
- `mcpui.dev/ui-initial-render-data`
|
||||
|
||||
Manual check:
|
||||
- Render tool responses using `UIResourceRenderer` (legacy host flow).
|
||||
|
||||
Automated (if SDK installed):
|
||||
- `uv run pytest test-int/mcp/test_ui_sdk_integration.py`
|
||||
|
||||
---
|
||||
|
||||
## Bakeoff Notes Template
|
||||
|
||||
Fill in after running:
|
||||
|
||||
- Tool‑UI (React): __
|
||||
- MCP‑UI SDK (embedded): __
|
||||
- Text/JSON modes: __
|
||||
|
||||
Decision + rationale: __
|
||||
@@ -0,0 +1,260 @@
|
||||
# Metadata Search Reference
|
||||
|
||||
Basic Memory automatically indexes custom frontmatter fields so you can query them with structured filters. Any YAML key in a note's frontmatter beyond the standard set (`title`, `type`, `tags`, `permalink`, `schema`) is stored as `entity_metadata` and becomes searchable.
|
||||
|
||||
## Querying with `search_notes`
|
||||
|
||||
`search_notes` is the single search tool for all queries — text, metadata filters, or both. The `query` parameter is optional, so you can use metadata filters alone without passing an empty string.
|
||||
|
||||
## Filter Syntax
|
||||
|
||||
Filters are a JSON dictionary where each key targets a frontmatter field and the value specifies the match condition. Multiple keys combine with **AND** logic — every filter must match.
|
||||
|
||||
### Equality
|
||||
|
||||
Match a single value exactly.
|
||||
|
||||
```json
|
||||
{"status": "active"}
|
||||
```
|
||||
|
||||
Finds notes whose frontmatter contains `status: active`.
|
||||
|
||||
### Array Contains (all)
|
||||
|
||||
Pass a list to require **all** listed values to be present in the field.
|
||||
|
||||
```json
|
||||
{"tags": ["security", "oauth"]}
|
||||
```
|
||||
|
||||
Finds notes tagged with both `security` and `oauth`.
|
||||
|
||||
### `$in` (any of)
|
||||
|
||||
Match if the field equals **any** value in the list.
|
||||
|
||||
```json
|
||||
{"priority": {"$in": ["high", "critical"]}}
|
||||
```
|
||||
|
||||
### `$gt`, `$gte`, `$lt`, `$lte`
|
||||
|
||||
Numeric and text comparisons. Numeric values use numeric comparison; strings use lexicographic comparison.
|
||||
|
||||
```json
|
||||
{"confidence": {"$gt": 0.7}}
|
||||
{"score": {"$lte": 100}}
|
||||
```
|
||||
|
||||
### `$between`
|
||||
|
||||
Range filter (inclusive). Takes a `[min, max]` pair.
|
||||
|
||||
```json
|
||||
{"score": {"$between": [0.3, 0.8]}}
|
||||
```
|
||||
|
||||
### Nested Access (dot notation)
|
||||
|
||||
Access nested frontmatter values using dots.
|
||||
|
||||
```json
|
||||
{"schema.version": "2"}
|
||||
```
|
||||
|
||||
This queries the `version` key inside a `schema` object in frontmatter.
|
||||
|
||||
### Summary Table
|
||||
|
||||
| Operator | Syntax | Example |
|
||||
|----------|--------|---------|
|
||||
| Equality | `{"field": "value"}` | `{"status": "active"}` |
|
||||
| Array contains (all) | `{"field": ["a", "b"]}` | `{"tags": ["security", "oauth"]}` |
|
||||
| `$in` (any of) | `{"field": {"$in": [...]}}` | `{"priority": {"$in": ["high", "critical"]}}` |
|
||||
| `$gt` / `$gte` | `{"field": {"$gt": N}}` | `{"confidence": {"$gt": 0.7}}` |
|
||||
| `$lt` / `$lte` | `{"field": {"$lt": N}}` | `{"score": {"$lt": 0.5}}` |
|
||||
| `$between` | `{"field": {"$between": [min, max]}}` | `{"score": {"$between": [0.3, 0.8]}}` |
|
||||
| Nested access | `{"a.b": "value"}` | `{"schema.version": "2"}` |
|
||||
|
||||
**Key rules:**
|
||||
- Filter keys must match `[A-Za-z0-9_-]+` (dots separate nesting levels).
|
||||
- Each operator dict must contain exactly one operator.
|
||||
- `$in` and array-contains require non-empty lists.
|
||||
- `$between` requires exactly two values `[min, max]`.
|
||||
|
||||
## MCP Tool — `search_notes`
|
||||
|
||||
`search_notes` is the single search tool for text queries, metadata filters, or both. The `query` parameter is optional.
|
||||
|
||||
**Relevant parameters:**
|
||||
|
||||
| Parameter | Type | Description |
|
||||
|-----------|------|-------------|
|
||||
| `query` | string (optional) | Text search query. Omit for filter-only searches. |
|
||||
| `metadata_filters` | dict | Structured filter dict (see syntax above) |
|
||||
| `tags` | list[str] | Convenience shorthand — merged into `metadata_filters["tags"]` |
|
||||
| `status` | string | Convenience shorthand — merged into `metadata_filters["status"]` |
|
||||
|
||||
**Merging rules:** `tags` and `status` are convenience shortcuts. They are merged into `metadata_filters` using `setdefault` — if the same key already exists in `metadata_filters`, the explicit filter wins.
|
||||
|
||||
**Examples:**
|
||||
|
||||
```python
|
||||
# Text search filtered by metadata
|
||||
await search_notes("authentication", metadata_filters={"status": "draft"})
|
||||
|
||||
# Filter-only search (no query needed)
|
||||
await search_notes(metadata_filters={"type": "spec"})
|
||||
|
||||
# Combine text, tags shortcut, and metadata
|
||||
await search_notes(
|
||||
"oauth flow",
|
||||
tags=["security"],
|
||||
metadata_filters={"confidence": {"$gt": 0.7}},
|
||||
)
|
||||
|
||||
# Convenience shortcuts
|
||||
await search_notes("planning", status="active")
|
||||
await search_notes(tags=["tier1", "alpha"])
|
||||
```
|
||||
|
||||
## Tag Search Shortcuts
|
||||
|
||||
The `tag:` prefix in a search query is a shorthand for tag-based metadata filtering. When `search_notes` receives a query starting with `tag:`, it converts the query into a `tags` filter and clears the text query.
|
||||
|
||||
```python
|
||||
# These are equivalent:
|
||||
await search_notes("tag:tier1")
|
||||
await search_notes("", tags=["tier1"])
|
||||
|
||||
# Multiple tags (comma or space separated) — all must be present:
|
||||
await search_notes("tag:tier1,alpha")
|
||||
await search_notes("tag:tier1 alpha")
|
||||
```
|
||||
|
||||
## CLI Access
|
||||
|
||||
The `bm tool search-notes` command exposes metadata filtering via `--meta` and `--filter` flags.
|
||||
|
||||
### `--meta` — simple key=value filters
|
||||
|
||||
Repeatable flag for equality filters on frontmatter fields.
|
||||
|
||||
```bash
|
||||
# Single filter
|
||||
bm tool search-notes "my query" --meta status=draft
|
||||
|
||||
# Multiple filters (AND logic)
|
||||
bm tool search-notes "" --meta status=active --meta priority=high
|
||||
```
|
||||
|
||||
### `--filter` — advanced JSON filters
|
||||
|
||||
Pass a full JSON filter dictionary for operator-based queries.
|
||||
|
||||
```bash
|
||||
# Range filter
|
||||
bm tool search-notes "" --filter '{"score": {"$between": [0.3, 0.8]}}'
|
||||
|
||||
# $in filter
|
||||
bm tool search-notes "" --filter '{"priority": {"$in": ["high", "critical"]}}'
|
||||
```
|
||||
|
||||
### `--tag` and `--status` — convenience shortcuts
|
||||
|
||||
```bash
|
||||
bm tool search-notes "query" --tag security --tag oauth
|
||||
bm tool search-notes "" --status draft
|
||||
```
|
||||
|
||||
### Combined example
|
||||
|
||||
```bash
|
||||
bm tool search-notes "authentication" --tag security --meta status=draft --type spec
|
||||
```
|
||||
|
||||
## Practical Examples
|
||||
|
||||
### Example notes with custom frontmatter
|
||||
|
||||
**`specs/auth-design.md`:**
|
||||
|
||||
```markdown
|
||||
---
|
||||
title: Auth Design
|
||||
type: spec
|
||||
tags: [security, oauth]
|
||||
status: in-progress
|
||||
priority: high
|
||||
confidence: 0.85
|
||||
---
|
||||
|
||||
# Auth Design
|
||||
|
||||
## Observations
|
||||
- [decision] Use OAuth 2.1 with PKCE for all client types #security
|
||||
- [requirement] Token refresh must be transparent to the user
|
||||
|
||||
## Relations
|
||||
- implements [[Security Requirements]]
|
||||
```
|
||||
|
||||
**`specs/search-redesign.md`:**
|
||||
|
||||
```markdown
|
||||
---
|
||||
title: Search Redesign
|
||||
type: spec
|
||||
tags: [search, performance]
|
||||
status: draft
|
||||
priority: medium
|
||||
confidence: 0.6
|
||||
---
|
||||
|
||||
# Search Redesign
|
||||
|
||||
## Observations
|
||||
- [goal] Sub-100ms search response times #performance
|
||||
- [approach] Hybrid FTS + vector retrieval
|
||||
|
||||
## Relations
|
||||
- depends_on [[Database Schema]]
|
||||
```
|
||||
|
||||
### Queries that find them
|
||||
|
||||
```python
|
||||
# Find all in-progress specs
|
||||
await search_notes(metadata_filters={"status": "in-progress", "type": "spec"})
|
||||
# → Auth Design
|
||||
|
||||
# Find high-confidence specs
|
||||
await search_notes(metadata_filters={"confidence": {"$gt": 0.7}})
|
||||
# → Auth Design (confidence: 0.85)
|
||||
|
||||
# Find specs with priority high or medium
|
||||
await search_notes(metadata_filters={"priority": {"$in": ["high", "medium"]}})
|
||||
# → Auth Design, Search Redesign
|
||||
|
||||
# Find specs in a confidence range
|
||||
await search_notes(metadata_filters={"confidence": {"$between": [0.5, 0.9]}})
|
||||
# → Auth Design (0.85), Search Redesign (0.6)
|
||||
|
||||
# Find notes tagged with security
|
||||
await search_notes("tag:security")
|
||||
# → Auth Design
|
||||
|
||||
# Combined: text search + metadata filter
|
||||
await search_notes("OAuth", metadata_filters={"status": "in-progress"})
|
||||
# → Auth Design
|
||||
```
|
||||
|
||||
### CLI equivalents
|
||||
|
||||
```bash
|
||||
bm tool search-notes "" --meta status=in-progress --type spec
|
||||
bm tool search-notes "" --filter '{"confidence": {"$gt": 0.7}}'
|
||||
bm tool search-notes "OAuth" --meta status=in-progress
|
||||
bm tool search-notes --tag security
|
||||
```
|
||||
@@ -0,0 +1,344 @@
|
||||
# Post-v0.18.0 Test Plan and Acceptance Criteria
|
||||
|
||||
## Goal
|
||||
|
||||
Define a complete validation plan for all major features merged after `v0.18.0`, combining:
|
||||
|
||||
- Coverage-gap-driven automated tests
|
||||
- Real MCP server integration tests (no mocks for target flows)
|
||||
- Manual MCP verification via LLM-driven tool calls
|
||||
|
||||
This plan is based on commits in `v0.18.0..HEAD` and the latest `just check` coverage output.
|
||||
|
||||
## Scope Window
|
||||
|
||||
- Start tag: `v0.18.0` (2026-01-28)
|
||||
- End: current `main`
|
||||
- Change volume: 12 feature commits + 14 bug-fix commits (+ release chores/hotfixes)
|
||||
|
||||
## Execution Strategy
|
||||
|
||||
1. Stabilize all feature-level acceptance criteria in automated tests first.
|
||||
2. Add black-box MCP integration tests for semantic search + schema (real server startup).
|
||||
3. Run manual MCP tool-call verification to confirm real UX and routing behavior.
|
||||
4. Re-run full gate: `just check` + targeted integration packs.
|
||||
|
||||
## Global Quality Gates
|
||||
|
||||
- Feature criteria below must all pass.
|
||||
- No regressions in existing suites.
|
||||
- Coverage improves in targeted low-coverage feature modules.
|
||||
- SQLite and Postgres parity for search/semantic features.
|
||||
|
||||
## Priority Coverage Gaps (from latest run)
|
||||
|
||||
These are the most important post-`v0.18.0` feature modules currently under-covered:
|
||||
|
||||
- `src/basic_memory/mcp/tools/schema.py` (27%)
|
||||
- `src/basic_memory/mcp/clients/schema.py` (36%)
|
||||
- `src/basic_memory/mcp/tools/ui_sdk.py` (43%)
|
||||
- `src/basic_memory/mcp/tools/search.py` (73%)
|
||||
- `src/basic_memory/repository/postgres_search_repository.py` (63%)
|
||||
- `src/basic_memory/mcp/async_client.py` (82%)
|
||||
- `src/basic_memory/api/v2/routers/schema_router.py` (80%)
|
||||
|
||||
## Feature Acceptance Criteria and Test Plan
|
||||
|
||||
### 1) Schema System (`c97733d`) — DONE
|
||||
|
||||
### Acceptance criteria
|
||||
|
||||
- `schema_validate`, `schema_infer`, and `schema_diff` produce consistent outcomes across CLI/API/MCP for the same fixture set.
|
||||
- Strict validation fails deterministically on required-field/type violations.
|
||||
- Validation warnings are stable and machine-readable in non-strict mode.
|
||||
- Inference output is deterministic for unchanged input corpus.
|
||||
- Drift diff output is deterministic and identifies missing/extra/type-mismatch fields correctly.
|
||||
|
||||
### Existing coverage anchor points
|
||||
|
||||
- `tests/schema/*`
|
||||
- `tests/api/v2/test_schema_router.py`
|
||||
- `test-int/test_schema/*`
|
||||
|
||||
### Gaps to close — DONE
|
||||
|
||||
- ~~MCP schema tool branches (`src/basic_memory/mcp/tools/schema.py`)~~ — 18 tests in `tests/mcp/test_tool_schema.py`
|
||||
- ~~MCP schema client behavior (`src/basic_memory/mcp/clients/schema.py`)~~ — `tests/mcp/test_client_schema.py`
|
||||
- ~~Schema router error-path branches (`src/basic_memory/api/v2/routers/schema_router.py`)~~ — `tests/api/v2/test_schema_router.py`
|
||||
|
||||
### Planned additions — DONE
|
||||
|
||||
- ~~Add MCP tool tests for `schema_validate` strict + non-strict result shapes.~~ **DONE**
|
||||
- ~~Add MCP tool tests for `schema_infer` with explicit `entity_type` and inferred type fallback.~~ **DONE**
|
||||
- ~~Add MCP tool tests for `schema_diff` empty-diff and non-empty-diff paths.~~ **DONE**
|
||||
- ~~Add API tests for schema router invalid payload/edge error handling.~~ **DONE**
|
||||
- Add integration test that starts MCP server and calls schema tools end-to-end on fixture notes. — deferred to backlog item 4.
|
||||
|
||||
### 2) Semantic Search (`0777879`, `1428d18`, `344e651`) — DONE
|
||||
|
||||
### Acceptance criteria
|
||||
|
||||
- `search_type=text|vector|hybrid` returns expected ranked results on canonical semantic corpus.
|
||||
- Missing semantic dependencies fail fast with actionable install guidance.
|
||||
- Reindex and provider/model changes produce valid vectors without dimension mismatch.
|
||||
- SQLite and Postgres produce equivalent behavior for semantic modes on the same dataset.
|
||||
- Generated-column migration path is valid on SQLite environments in use.
|
||||
|
||||
### Existing coverage anchor points
|
||||
|
||||
- `tests/repository/test_sqlite_vector_search_repository.py`
|
||||
- `tests/repository/test_postgres_search_repository.py`
|
||||
- `tests/services/test_semantic_search.py`
|
||||
- `tests/mcp/test_tool_search.py`
|
||||
- `test-int/test_search_performance_benchmark.py`
|
||||
|
||||
### Gaps to close — DONE
|
||||
|
||||
- ~~Uncovered Postgres vector/hybrid branches~~ — 20 tests in `tests/repository/test_postgres_search_repository_unit.py` + 5 integration tests in `test-int/semantic/test_semantic_coverage.py`
|
||||
- ~~MCP search semantic/output branches~~ — expanded `tests/mcp/test_tool_search.py`
|
||||
|
||||
### Planned additions — DONE
|
||||
|
||||
- ~~Expand Postgres repository tests for vector query composition edge cases.~~ **DONE**
|
||||
- ~~Expand Postgres repository tests for hybrid fusion ranking and pagination branches.~~ **DONE**
|
||||
- ~~Expand Postgres repository tests for embedding/provider error handling branches.~~ **DONE**
|
||||
- ~~Expand MCP search tool tests for vector/hybrid output formatting branches.~~ **DONE**
|
||||
- ~~Expand MCP search tool tests for semantic-disabled and missing-dependency failures.~~ **DONE**
|
||||
- Add MCP integration tests that start server and execute semantic `search_notes` tool calls. — deferred to backlog item 4.
|
||||
|
||||
### Semantic search quality benchmarks (NEW)
|
||||
|
||||
Full benchmark suite in `test-int/semantic/` covering 5 backend×provider combinations:
|
||||
- `sqlite-fts`, `sqlite-fastembed`, `postgres-fts`, `postgres-fastembed`, `postgres-openai`
|
||||
- Quality metrics: hit@1, recall@5, MRR@10 with per-query timing
|
||||
- Realistic corpus with cross-topic vocabulary overlap (240 notes, 4 topics)
|
||||
- Rich CLI viewer: `just semantic-report`
|
||||
- JSON artifact output: `just test-semantic-report`
|
||||
|
||||
Key finding: **FastEmbed (384-d local ONNX) matches or exceeds OpenAI (1536-d) quality at 30x lower latency.** Recommending FastEmbed as default for both local and cloud deployments.
|
||||
|
||||
### 3) Per-Project Local/Cloud Routing + API Key Auth (`d84708c`, `ed94877`, `312662f`) — DONE
|
||||
|
||||
### Acceptance criteria
|
||||
|
||||
- Project mode (`local`/`cloud`) persists and displays correctly.
|
||||
- Routing selects ASGI for local projects and HTTP+Bearer for cloud projects.
|
||||
- Cloud project without key fails with explicit remediation (`cloud set-key`/`cloud create-key`).
|
||||
- Resolution precedence is correct (factory > force-local > per-project cloud > global fallback > local).
|
||||
- Watch/sync only run for local projects.
|
||||
|
||||
### Existing coverage anchor points
|
||||
|
||||
- `tests/mcp/test_async_client_modes.py`
|
||||
- `tests/cli/test_project_set_cloud_local.py`
|
||||
- `tests/mcp/test_project_context.py`
|
||||
- `tests/test_project_resolver.py`
|
||||
- `tests/sync/test_watch_service_reload.py`
|
||||
|
||||
### Gaps to close — DONE
|
||||
|
||||
- ~~Cloud routing branch gaps in `src/basic_memory/mcp/async_client.py`~~ — expanded `tests/mcp/test_async_client_modes.py`
|
||||
|
||||
### Planned additions — DONE
|
||||
|
||||
- ~~Add branch-focused tests for all unresolved routing branches in `get_client()`.~~ **DONE**
|
||||
- Add MCP integration scenario with mixed local/cloud project config — deferred to backlog item 4.
|
||||
|
||||
### 4) Project-Prefixed Permalinks + Memory URL Routing (`545804f`) — DONE
|
||||
|
||||
### Acceptance criteria
|
||||
|
||||
- Project-prefixed permalinks are generated consistently on create/update/import flows.
|
||||
- Memory URLs resolve to the correct project/entity even with duplicate note titles.
|
||||
- `read_note`, `search`, `build_context`, write/edit/move flows preserve project identity correctly.
|
||||
- Link resolution remains correct for context-aware wikilinks.
|
||||
|
||||
### Existing coverage anchor points
|
||||
|
||||
- `tests/utils/test_permalink_formatting.py`
|
||||
- `tests/mcp/test_tool_read_note.py`
|
||||
- `tests/mcp/test_tool_search.py`
|
||||
- `tests/services/test_context_service.py`
|
||||
- `test-int/mcp/test_read_note_integration.py`
|
||||
|
||||
### Gaps to close
|
||||
|
||||
- No major coverage alarm in report, but keep as regression-critical due broad impact surface.
|
||||
|
||||
### Planned additions — DONE
|
||||
|
||||
- ~~Add one integration test with colliding titles across two projects and assert URL routing invariants.~~ **DONE** — `test-int/mcp/test_permalink_collision_integration.py` (2 tests: collision across projects + memory:// URL routing with project prefix)
|
||||
|
||||
### 5) MCP UI Variants + TUI Output (`8bc03d1`) — DONE
|
||||
|
||||
### Acceptance criteria
|
||||
|
||||
- UI resource variant selection (`tool-ui`, `vanilla`, `mcp-ui`) follows env configuration.
|
||||
- `search_notes` and `read_note` expose expected resource metadata for UI hosts.
|
||||
- `ascii`/`ansi` outputs are deterministic and stable for terminal clients.
|
||||
|
||||
### Existing coverage anchor points
|
||||
|
||||
- `tests/mcp/test_tool_contracts.py`
|
||||
- `test-int/mcp/test_output_format_json_integration.py`
|
||||
- `test-int/mcp/test_ui_sdk_integration.py`
|
||||
|
||||
### Gaps to close — DONE
|
||||
|
||||
- ~~`src/basic_memory/mcp/tools/ui_sdk.py` branch coverage~~ — `tests/mcp/test_ui_sdk.py`
|
||||
- ~~`src/basic_memory/mcp/ui/sdk.py` and `src/basic_memory/mcp/ui/templates.py` branch coverage~~ — `tests/mcp/test_ui_templates.py` + `tests/mcp/test_ui_resources.py`
|
||||
|
||||
### Planned additions — DONE
|
||||
|
||||
- ~~Add unit tests for UI SDK metadata generation and template selection branches.~~ **DONE** — 31 tests
|
||||
- ~~Add integration assertion for variant-specific resource URIs and metadata payload shape.~~ **DONE**
|
||||
|
||||
### 6) Watch Command (`8df88e4`) — DONE
|
||||
|
||||
### Acceptance criteria
|
||||
|
||||
- `basic-memory watch` starts and processes create/update/delete events.
|
||||
- Watch restart/reload path does not duplicate watchers.
|
||||
- Cloud-mode projects are excluded from active watcher set.
|
||||
|
||||
### Existing coverage anchor points
|
||||
|
||||
- `tests/cli/test_watch.py`
|
||||
- `tests/sync/test_coordinator.py`
|
||||
- `tests/sync/test_watch_service_reload.py`
|
||||
|
||||
### Planned additions — DONE
|
||||
|
||||
- ~~Add one stress-style integration test for rapid file changes and watcher stability.~~ **DONE** — `tests/sync/test_watch_service_stress.py` (3 tests: 50-file batch, mixed add/modify/delete batch, rapid modifications to same file)
|
||||
|
||||
### 7) CLI JSON Output (`a47c9c0`) — DONE
|
||||
|
||||
### Acceptance criteria
|
||||
|
||||
- `--format json` returns valid JSON with stable keys for success paths.
|
||||
- Error paths also return JSON-shaped output with correct non-zero exits.
|
||||
- Default human output remains unchanged.
|
||||
|
||||
### Existing coverage anchor points
|
||||
|
||||
- `tests/cli/test_cli_tool_json_output.py`
|
||||
- `test-int/cli/test_cli_tool_json_integration.py`
|
||||
|
||||
### Planned additions — DONE
|
||||
|
||||
- ~~Add one failure-path integration test per high-use tool command.~~ **DONE** — `test-int/cli/test_cli_tool_json_failure_integration.py` (4 tests: read-note not found, write-note missing content, write→read roundtrip, recent-activity empty project)
|
||||
|
||||
### 8) Search/Edit and Metadata Fixes (`530cbac`, `f1d50c2`, `8838571`, `009e849`) — DONE
|
||||
|
||||
### Acceptance criteria
|
||||
|
||||
- Metadata filters produce consistent results on SQLite and Postgres.
|
||||
- `tag:` shorthand works alone and with mixed query terms.
|
||||
- Fast write/edit paths preserve `external_id` and metadata integrity.
|
||||
|
||||
### Existing coverage anchor points
|
||||
|
||||
- `tests/repository/test_metadata_filters.py`
|
||||
- `tests/repository/test_search_repository.py`
|
||||
- `tests/services/test_search_service.py`
|
||||
|
||||
### Planned additions — DONE
|
||||
|
||||
- ~~Add Postgres-specific metadata filter edge-case tests to mirror SQLite assertions exactly.~~ **DONE** — `tests/repository/test_metadata_filters_edge_cases.py` (6 tests: missing field, AND logic, contains single-element array, nested path missing intermediate, $gte/$lte boundaries, $between inclusive — all pass on both SQLite and Postgres)
|
||||
|
||||
### 9) Compatibility and Hotfix Regression Pack (`c46d7a6`, `a0e754b`, `343a6e1`, `24ca5f6`, `e3ced49`, `8489a3d`, `b609c4e`, `f6e0a5b`, `7624a20`)
|
||||
|
||||
### Acceptance criteria
|
||||
|
||||
- Legacy endpoints required by older CLI versions function without `405` (`GET /projects/projects`, `POST /projects/projects`, `POST /projects/config/sync`).
|
||||
- Entity creation conflicts map to conflict status (not 500).
|
||||
- `recent_activity` prompt defaults are correct.
|
||||
- No spurious `metadata: {}` in serialized frontmatter.
|
||||
- Tigris/rclone uses global consistency headers for all transaction types.
|
||||
- `bm --version` fast path avoids heavy import path and remains responsive.
|
||||
- Default SQLite DB path is isolated by config dir.
|
||||
|
||||
### Gaps to close
|
||||
|
||||
- ~~Commits with no direct tests added (`c46d7a6`, `344e651`, `f6e0a5b`) need explicit regression tests.~~ **DONE**
|
||||
|
||||
### Planned additions — DONE
|
||||
|
||||
- ~~Add API compat test covering all legacy endpoint methods and payloads.~~ **DONE** — `test_legacy_v1_add_project_endpoint`, `test_legacy_v1_sync_config_endpoint`
|
||||
- ~~Add CLI fast-path test for `--version` import behavior/performance guard.~~ **DONE** — `test_bm_version_does_not_import_heavy_modules`
|
||||
- ~~Add empty metadata serialization regression test.~~ **DONE** — `test_schema_to_markdown_empty_metadata_no_metadata_key`
|
||||
- Add migration safety test for SQLite generated columns (`VIRTUAL` expectation) — deferred, low risk.
|
||||
|
||||
## MCP Manual Verification Plan (LLM Tool Calls)
|
||||
|
||||
Run after automated tests pass.
|
||||
|
||||
### Setup
|
||||
|
||||
- Start MCP server: `basic-memory mcp --transport stdio`
|
||||
- Use an MCP-capable client and issue tool calls directly.
|
||||
|
||||
### Manual scenarios
|
||||
|
||||
- Schema: call `schema_validate`, `schema_infer`, and `schema_diff` on known fixtures.
|
||||
- Schema: verify error and success payloads match acceptance criteria.
|
||||
- Semantic search: call `search_notes` with `search_type=text|vector|hybrid`.
|
||||
- Semantic search: verify ranking relevance on semantic fixture queries.
|
||||
- Routing: call tools with explicit project on mixed local/cloud setup.
|
||||
- Routing: verify success/failure paths with and without API key.
|
||||
- Permalink routing: read/write/search notes across projects with colliding titles.
|
||||
- Permalink routing: verify memory URL routing correctness.
|
||||
- UI/TUI: call `search_notes` and `read_note` with UI variants and `output_format=text|json`.
|
||||
- UI/TUI: verify payload/resource format and metadata completeness.
|
||||
|
||||
## Implementation Backlog (Ordered)
|
||||
|
||||
1. ~~Fill schema MCP/client/router coverage gaps.~~ **DONE** — 18 tests in `test_tool_schema.py` + `test_client_schema.py`
|
||||
2. ~~Fill semantic search MCP + Postgres repository gaps.~~ **DONE** — 20 tests in `test_postgres_search_repository_unit.py` + `test_tool_search.py`
|
||||
3. ~~Add compatibility regression tests (legacy endpoints, migration, version fast path).~~ **DONE** — 5 tests across 3 files (see below)
|
||||
4. ~~Add feature-level integration tests (permalinks, watch, CLI JSON, metadata filters).~~ **DONE** — 15 tests across 4 files (see items 4, 6, 7, 8 above)
|
||||
5. ~~Expand UI SDK and template branch tests.~~ **DONE** — 31 tests in `test_ui_templates.py` + `test_ui_sdk.py` + `test_ui_resources.py`
|
||||
6. ~~Run full gate and capture results in a short release readiness summary.~~ **DONE** — see results below
|
||||
|
||||
### Full Gate Results (`just check`)
|
||||
|
||||
| Phase | Result |
|
||||
|-------|--------|
|
||||
| lint | PASS |
|
||||
| format | PASS |
|
||||
| typecheck | PASS |
|
||||
| Unit tests (SQLite) | 1788 passed, 15 skipped |
|
||||
| Integration tests (SQLite) | 243 passed, 4 skipped, 10 deselected |
|
||||
| Unit tests (Postgres) | 1760 passed, 28 skipped |
|
||||
| Integration tests (Postgres) | 234 passed, 13 skipped, 10 deselected |
|
||||
|
||||
**0 failures. 10 deselected = semantic benchmark tests (run separately via `just test-semantic`).**
|
||||
|
||||
### Item 3 Details — Compatibility Regression Tests
|
||||
|
||||
| Test | File | What it covers |
|
||||
|------|------|----------------|
|
||||
| `test_legacy_v1_add_project_endpoint` | `tests/api/v2/test_project_router.py` | POST `/projects/projects` legacy route reachable (idempotent path) |
|
||||
| `test_legacy_v1_sync_config_endpoint` | `tests/api/v2/test_project_router.py` | POST `/projects/config/sync` legacy route reachable |
|
||||
| `test_bm_version_does_not_import_heavy_modules` | `tests/cli/test_cli_exit.py` | `bm --version` fast path does not load `basic_memory.mcp` |
|
||||
| `test_schema_to_markdown_empty_metadata_no_metadata_key` | `tests/markdown/test_entity_parser_error_handling.py` | `schema_to_markdown()` with `entity_metadata={}` emits no `metadata:` key |
|
||||
| `test_legacy_v1_list_projects_endpoint` | `tests/api/v2/test_project_router.py` | (pre-existing) GET `/projects/projects` legacy route |
|
||||
|
||||
**Suite totals after item 3: 1764 passed, 15 skipped, 0 failures.**
|
||||
|
||||
## Suggested Commands
|
||||
|
||||
- Full suite: `just check`
|
||||
- Fast loop: `just fast-check`
|
||||
- E2E consistency: `just doctor`
|
||||
- SQLite focused: `just test-sqlite`
|
||||
- Postgres focused: `just test-postgres`
|
||||
- Schema integration: `pytest test-int/test_schema -q`
|
||||
- Semantic + repo focus: `pytest tests/repository/test_postgres_search_repository.py tests/mcp/test_tool_search.py tests/services/test_semantic_search.py -q`
|
||||
- MCP integration focus: `pytest test-int/mcp -q`
|
||||
|
||||
## Exit Criteria for This Plan
|
||||
|
||||
- All feature acceptance criteria above are validated.
|
||||
- All identified high-priority coverage gaps are addressed or explicitly documented as intentional.
|
||||
- Manual MCP verification scenarios complete with no P0/P1 findings.
|
||||
@@ -0,0 +1,318 @@
|
||||
# v0.19.0 Release Notes
|
||||
|
||||
## Overview
|
||||
|
||||
v0.19.0 is a major release that introduces semantic vector search, a schema validation system,
|
||||
project-prefixed permalinks, per-project cloud routing, and a significant upgrade to FastMCP 3.0.
|
||||
It includes 90+ commits since v0.18.0 spanning new features, architectural improvements, and
|
||||
stability fixes across both SQLite and Postgres backends.
|
||||
|
||||
---
|
||||
|
||||
## Major Features
|
||||
|
||||
### Semantic Vector Search
|
||||
|
||||
Full vector and hybrid search for SQLite (via sqlite-vec) and Postgres (via pgvector).
|
||||
|
||||
- **Hybrid search mode** combines full-text search (FTS) with vector similarity for best results
|
||||
- **Score-based fusion** replaces RRF for hybrid ranking — `max(vec, fts) + 0.3 * min(vec, fts)` preserves dominant signals and rewards dual-source agreement (#577)
|
||||
- **Default search mode** is now `hybrid` when semantic search is enabled, `text` when disabled
|
||||
- Embedding providers: FastEmbed (local, default) or OpenAI API
|
||||
- Configurable similarity threshold via `semantic_min_similarity` (default 0.55)
|
||||
- Per-query `min_similarity` override on `search_notes` tool
|
||||
- Auto-backfill: existing entities get embeddings generated on first startup
|
||||
- Backend-specific distance-to-similarity conversion (cosine for SQLite, inner product for Postgres)
|
||||
- FTS fallback: if semantic dependencies are missing, search gracefully degrades to text-only
|
||||
- sqlite-vec knn `k` parameter capped at 4096 to prevent backend errors
|
||||
|
||||
**Configuration:**
|
||||
```json
|
||||
{
|
||||
"semantic_search_enabled": true,
|
||||
"semantic_embedding_provider": "fastembed",
|
||||
"semantic_embedding_model": "bge-small-en-v1.5",
|
||||
"semantic_min_similarity": 0.55
|
||||
}
|
||||
```
|
||||
|
||||
**Usage:**
|
||||
```
|
||||
search_notes("machine learning concepts", search_type="hybrid")
|
||||
search_notes("similar to my notes on coffee", search_type="vector")
|
||||
search_notes("exact phrase match", search_type="text")
|
||||
search_notes("broad search", min_similarity=0.3) # lower threshold for more results
|
||||
```
|
||||
|
||||
### Schema System
|
||||
|
||||
Validate note structure against user-defined schemas with frontmatter-based rules.
|
||||
|
||||
- Define schemas as YAML in note frontmatter with field types, required fields, and constraints
|
||||
- Frontmatter validation during sync — malformed notes get clear error messages
|
||||
- Schema inference from existing notes to bootstrap schemas from your content
|
||||
- Schema diff to compare two schemas and see changes
|
||||
- Available via MCP tools and CLI
|
||||
|
||||
### Project-Prefixed Permalinks
|
||||
|
||||
Permalinks now include the project name for unambiguous cross-project references.
|
||||
|
||||
- Memory URLs like `memory://project-name/folder/note` route to the correct project
|
||||
- Existing non-prefixed permalinks continue to work (backwards compatible)
|
||||
- Controlled by `permalinks_include_project` config (default: true)
|
||||
- `build_context` and `search_notes` auto-detect project from URL prefix
|
||||
|
||||
### Per-Project Cloud Routing
|
||||
|
||||
Individual projects can be routed through the cloud while others stay local.
|
||||
|
||||
- Set a project to cloud mode: `bm project set-cloud research`
|
||||
- Revert to local: `bm project set-local research`
|
||||
- Uses API key authentication: `bm cloud set-key bmc_abc123...`
|
||||
- MCP tools automatically route based on each project's mode
|
||||
- Local MCP server (`bm mcp`) still uses local routing for all projects by default
|
||||
- `--local` and `--cloud` CLI flags override per-command
|
||||
|
||||
### Workspace Selection
|
||||
|
||||
Cloud projects can target specific workspaces for multi-tenant environments.
|
||||
|
||||
- `workspace` parameter on MCP tools for explicit workspace targeting
|
||||
- CLI workspace-aware project listing with `bm project list`
|
||||
- Spinner feedback while fetching cloud projects
|
||||
|
||||
---
|
||||
|
||||
## New Tools and Capabilities
|
||||
|
||||
### Dashboard (`bm project info`)
|
||||
|
||||
`bm project info` now displays an htop-inspired compact dashboard with:
|
||||
|
||||
- Horizontal bar charts for note types (top 5)
|
||||
- Embedding coverage bar with Unicode block characters
|
||||
- Colored status dots for at-a-glance health
|
||||
- `EmbeddingStatus` schema and `get_embedding_status()` service method for programmatic access
|
||||
|
||||
### Unified Metadata Search
|
||||
|
||||
`search_by_metadata` has been merged into `search_notes` — one tool for all searches.
|
||||
`query` is now optional, so you can search purely by frontmatter metadata.
|
||||
|
||||
```
|
||||
search_notes(metadata_filters={"status": "in-progress"})
|
||||
search_notes(metadata_filters={"tags": ["security", "oauth"]})
|
||||
search_notes(metadata_filters={"priority": {"$in": ["high", "critical"]}})
|
||||
search_notes(metadata_filters={"schema.confidence": {"$gt": 0.7}})
|
||||
search_notes(tags=["security"]) # convenience shorthand
|
||||
search_notes(status="draft") # convenience shorthand
|
||||
```
|
||||
|
||||
### JSON Output Mode
|
||||
|
||||
All MCP tools now support `output_format="json"` for machine-readable responses.
|
||||
|
||||
- Default remains `"text"` for human-readable output (no breaking changes)
|
||||
- `build_context` defaults to `"json"` with slimmed payloads (redundant fields stripped)
|
||||
- CLI tool commands support `--format json` flag
|
||||
|
||||
### `tag:` Search Shorthand
|
||||
|
||||
Search by tag using convenient shorthand syntax.
|
||||
|
||||
```
|
||||
search_notes("tag:security")
|
||||
search_notes("tag:coffee AND tag:brewing")
|
||||
```
|
||||
|
||||
### Entity User Tracking
|
||||
|
||||
Entities now track `created_by` and `last_updated_by` fields for attribution.
|
||||
|
||||
### Improved Search Result Content (#609)
|
||||
|
||||
Search results now surface more relevant context:
|
||||
|
||||
- `matched_chunk_text` populated for FTS-only hybrid results (no more fallback to truncated content)
|
||||
- `TOP_CHUNKS_PER_RESULT` increased from 3 to 5, catching answers deeper in large notes (~2700 → ~4500 chars)
|
||||
- `CONTENT_DISPLAY_LIMIT` doubled from 2000 to 4000 chars for results without matched chunks
|
||||
|
||||
### `write_note` Overwrite Guard (#632)
|
||||
|
||||
`write_note` is now non-idempotent by default. If a note already exists, the tool returns an
|
||||
error instead of silently overwriting. Pass `overwrite=True` to replace, or use `edit_note`
|
||||
for incremental updates. Config option `write_note_overwrite_default` restores the old upsert
|
||||
behavior.
|
||||
|
||||
---
|
||||
|
||||
## Architecture Changes
|
||||
|
||||
### Score-Based Hybrid Fusion (#577)
|
||||
|
||||
RRF (Reciprocal Rank Fusion) compressed all fused scores to ~0.016, destroying ranking
|
||||
differentiation. The new formula `max(vec, fts) + FUSION_BONUS * min(vec, fts)` preserves
|
||||
dominant signals and rewards dual-source agreement. Zero-score results now produce zero
|
||||
fused score instead of receiving a 0.1 weight floor.
|
||||
|
||||
### FastMCP 3.0 Upgrade
|
||||
|
||||
Upgraded from FastMCP 2.12.3 to 3.0.1.
|
||||
|
||||
- Tool annotations (`readOnlyHint`, `openWorldHint`) for better client integration
|
||||
- Improved MCP protocol compliance
|
||||
- Better error handling and context management
|
||||
|
||||
### Prompts Call MCP Tools Directly
|
||||
|
||||
MCP prompts (`search`, `continue_conversation`) now call MCP tools directly instead of
|
||||
going through API endpoints. This fixes empty results in discovery mode and ensures prompts
|
||||
use the same resolution logic as tools (including LinkResolver fallback).
|
||||
|
||||
### build_context LinkResolver Fallback
|
||||
|
||||
`build_context` now falls back to LinkResolver when an exact permalink lookup returns empty.
|
||||
This uses the same 7-strategy resolution pipeline as `read_note`, so callers no longer get
|
||||
empty results for valid note identifiers that don't match exact permalinks.
|
||||
|
||||
### Sync Handles Semantic Dependency Errors Gracefully
|
||||
|
||||
When sqlite-vec or another embedding provider is unavailable, `sync_file` now catches
|
||||
`SemanticDependenciesMissingError` separately. The entity is created and FTS-indexed
|
||||
successfully — only vector embeddings are skipped, with a clear warning:
|
||||
|
||||
```
|
||||
WARNING: Semantic search dependencies missing — vector embeddings skipped for path=note.md.
|
||||
Run 'bm reindex --embeddings' after resolving the dependency issue.
|
||||
```
|
||||
|
||||
### Unified Project Path
|
||||
|
||||
Cloud projects with bisync now store the local filesystem path in `path` (not the Docker
|
||||
container path). Config migration automatically promotes `local_sync_path` → `path` for
|
||||
existing configs.
|
||||
|
||||
---
|
||||
|
||||
## CLI Improvements
|
||||
|
||||
### Status and Doctor Default to Local Routing
|
||||
|
||||
`bm status` and `bm doctor` now default to local routing since they scan the local filesystem.
|
||||
Previously, cloud-mode projects would route these commands to the cloud API, which returned
|
||||
Docker-internal paths that don't exist locally.
|
||||
|
||||
### `--format json` for CLI Tool Commands
|
||||
|
||||
All `bm tool` subcommands support `--format json` for machine-readable output, enabling
|
||||
integration with scripts and plugins.
|
||||
|
||||
### `--json` for Top-Level CLI Commands
|
||||
|
||||
Five additional CLI commands now support `--json` for machine-readable output:
|
||||
|
||||
- `bm status --json` — sync report with new/modified/deleted/moved files and skipped files
|
||||
- `bm project list --json` — structured project list with name, paths, routing mode, and defaults
|
||||
- `bm schema validate --json` — validation report with per-note pass/fail, warnings, and errors
|
||||
- `bm schema infer --json` — field frequency analysis and suggested schema definition
|
||||
- `bm schema diff --json` — drift report with new fields, dropped fields, and cardinality changes
|
||||
|
||||
This complements the existing `bm project info --json` and `bm tool --format json` support,
|
||||
making all major CLI commands scriptable for CI pipelines and automation.
|
||||
|
||||
### Cloud Promo and Analytics
|
||||
|
||||
- Cloud promo panel shown on first run or version bump with OSS discount code
|
||||
- Anonymous usage telemetry via Umami Cloud (promo/login funnel events only)
|
||||
- Opt out with `BASIC_MEMORY_NO_PROMOS=1`
|
||||
- No PII, no file contents, no per-command tracking
|
||||
- See [Telemetry](https://github.com/basicmachines-co/basic-memory#telemetry) in README
|
||||
|
||||
---
|
||||
|
||||
## Bug Fixes
|
||||
|
||||
- **#577**: RRF fusion compressed all hybrid scores to ~0.016, destroying ranking differentiation
|
||||
- **#582**: build_context returns empty results on valid note identifiers
|
||||
- **#575**: Remove hardcoded "main" default from default_project
|
||||
- **#595**: recent_activity dedup and pagination across MCP tools
|
||||
- **#593**: Backend-specific distance-to-similarity conversion
|
||||
- **#592**: Strip NUL bytes from content before PostgreSQL search indexing
|
||||
- **#562**: Use VIRTUAL instead of STORED columns in SQLite migration
|
||||
- **#558**: Add X-Tigris-Consistent headers to all rclone commands
|
||||
- **#541**: Handle EntityCreationError as conflict
|
||||
- **#536**: Stabilize metadata filters on Postgres
|
||||
- **#533**: Fix recent_activity prompt defaults
|
||||
- **#530**: Prevent spurious `metadata: {}` in frontmatter output
|
||||
- **#601**: Return matched chunk text in search results
|
||||
- **#606**: Accept `null` for `expected_replacements` in `edit_note`
|
||||
- **#579, #607**: Guard against closed streams in promo panel and missing vector tables on shutdown
|
||||
- **#609**: FTS-only hybrid results missing `matched_chunk_text`; content limits too conservative
|
||||
- **#631**: `build_context` related_results schema validation failure — replaced fragile `_slim_context()` stripping with Pydantic `exclude=True` field config
|
||||
- **#630**: Skip workspace resolution when client factory is active — prevents 401 errors in cloud MCP server mode
|
||||
- **#30**: `tag:` prefix query fails with hybrid search — moved tag prefix parsing to MCP tool level so it works with all search modes
|
||||
- **#31**: `search_notes` returns cluttered observation/relation-level results — now defaults to entity-level results
|
||||
- **#28**: `schema_infer` and `schema_diff` return raw Pydantic models as "undefined" in LLM output — added markdown formatters
|
||||
- Fix `schema_validate` identifier resolution (now uses LinkResolver) and text rendering (markdown formatter)
|
||||
- **#634**: `schema_validate` and `schema_diff` use stale database metadata instead of reading schema definitions from file — now reads frontmatter directly from the file with fallback to database metadata
|
||||
- Fix `Post(**metadata)` crash when frontmatter contains `content` or `handler` keys
|
||||
- Fix list-valued frontmatter fields (`title`, `type`) crashing on `.strip()` — now coerced to strings
|
||||
- Cap sqlite-vec knn `k` parameter at 4096 to prevent backend errors
|
||||
- Parameterize SQL queries in search repository type filters
|
||||
- Double-default display in project list
|
||||
- `ensure_frontmatter_on_sync` default changed to `True`
|
||||
- Status/doctor commands fail with cloud-mode projects (Docker path error)
|
||||
- Prompts return "0 projects" in discovery mode
|
||||
|
||||
---
|
||||
|
||||
## Security
|
||||
|
||||
- Upgrade `cryptography` for CVE advisory
|
||||
- Upgrade `python-multipart` for security advisory
|
||||
|
||||
---
|
||||
|
||||
## Internal / Developer
|
||||
|
||||
- **#598**: Upgrade FastMCP 2.12.3 → 3.0.1 with tool annotations
|
||||
- **#594**: Add `ty` as supplemental type checker
|
||||
- **#538**: Add fast feedback loop tooling (`just fast-check`, `just doctor`, `just testmon`)
|
||||
- **#600**: Rename `entity_type` to `note_type` for consistency
|
||||
- **#596**: Fix CLI runtime defects and audit regressions
|
||||
- CLI refactoring and workspace-aware cloud project listing
|
||||
- Split and speed up PR test matrix in CI
|
||||
- Fix CI: collect coverage from test jobs instead of re-running all tests
|
||||
- Create `search_vector_chunks` in test fixtures for Postgres compatibility
|
||||
|
||||
---
|
||||
|
||||
## Configuration Changes
|
||||
|
||||
| Setting | Old Default | New Default | Notes |
|
||||
|---------|-------------|-------------|-------|
|
||||
| `semantic_search_enabled` | `false` | `true` | Semantic search on by default |
|
||||
| `ensure_frontmatter_on_sync` | `false` | `true` | Frontmatter added during sync |
|
||||
| `permalinks_include_project` | `false` | `true` | Project prefix in permalinks |
|
||||
|
||||
---
|
||||
|
||||
## Upgrade Notes
|
||||
|
||||
- **Semantic search dependencies** are now included by default. If sqlite-vec fails to load,
|
||||
search gracefully falls back to FTS. Run `bm reindex --embeddings` to generate embeddings
|
||||
for existing content.
|
||||
- **Hybrid search scoring** has changed from RRF to score-based fusion. Search result ordering
|
||||
may differ — results should be more accurate with better score differentiation.
|
||||
- **`search_by_metadata`** is removed as a standalone tool. Use `search_notes` with
|
||||
`metadata_filters` instead (same parameters, same behavior).
|
||||
- **Project-prefixed permalinks** are enabled by default. Existing notes keep their current
|
||||
permalinks until modified. Set `permalinks_include_project: false` to disable.
|
||||
- **Frontmatter on sync** is now enabled by default. Files without frontmatter will have it
|
||||
added on next sync. Set `ensure_frontmatter_on_sync: false` to preserve old behavior.
|
||||
- **Config migration** runs automatically for cloud projects with bisync — `local_sync_path`
|
||||
is promoted to `path` so filesystem operations work correctly.
|
||||
- **`write_note` is no longer idempotent** — calls to `write_note` for existing notes now
|
||||
return an error unless `overwrite=True` is passed. Use `edit_note` for incremental changes,
|
||||
or set `write_note_overwrite_default: true` in config to restore the old behavior.
|
||||
@@ -0,0 +1,209 @@
|
||||
# Semantic Search Manual Test Log
|
||||
|
||||
## Overview
|
||||
|
||||
Manual test session for semantic (vector) search on the main project.
|
||||
- Date: 2026-02-15
|
||||
- Database: ~/.basic-memory/memory.db (SQLite)
|
||||
- Entities: 456 embedded, 2714 vector chunks
|
||||
- Search index: 2390 FTS entries
|
||||
- Embedding model: default (384-dim, sqlite-vec)
|
||||
|
||||
## Test Plan
|
||||
|
||||
1. **Search Type Routing** — verify vector/hybrid/text dispatch, invalid search_type handling
|
||||
2. **Conceptual Queries** — natural language where vector should beat FTS
|
||||
3. **Keyword Queries** — exact terms where FTS should be strong
|
||||
4. **Hybrid Ranking** — queries where both FTS and vector contribute
|
||||
5. **Result Types** — entities, observations, relations in vector results
|
||||
6. **Filters + Vector** — combine vector with types/entity_types/after_date
|
||||
7. **Edge Cases** — short queries, long queries, empty, special chars, no-match
|
||||
8. **Pagination** — page > 1, page_size respected
|
||||
|
||||
---
|
||||
|
||||
## Test Results
|
||||
|
||||
### Test 1: Search Type Routing
|
||||
|
||||
#### 1a: search_type="semantic" (invalid value)
|
||||
- **Input:** query="how does the knowledge graph work", search_type="semantic"
|
||||
- **Expected:** error or explicit fallback
|
||||
- **Actual:** Silently falls through to text search (else branch in search.py:430)
|
||||
- **Verdict:** BUG — should either be a recognized alias for "vector" or return an error
|
||||
|
||||
#### 1b: search_type="vector"
|
||||
- **Input:** query="keeping AI context between sessions", search_type="vector"
|
||||
- **Actual:** 5 results, scores ~0.58-0.59, found "Maintaining context across conversation boundaries" observation
|
||||
- **Verdict:** PASS
|
||||
|
||||
#### 1c: search_type="text" with conceptual query
|
||||
- **Input:** query="keeping AI context between sessions", search_type="text"
|
||||
- **Actual:** 0 results (no exact keyword match)
|
||||
- **Verdict:** PASS (expected — FTS requires token overlap)
|
||||
|
||||
#### 1d: search_type="hybrid" with conceptual query
|
||||
- **Input:** query="keeping AI context between sessions", search_type="hybrid"
|
||||
- **Actual:** 5 results, same ranking as vector (FTS contributed nothing here)
|
||||
- **Verdict:** PASS
|
||||
|
||||
#### 1e: search_type="text" with keyword query
|
||||
- **Input:** query="OAuth authentication", search_type="text"
|
||||
- **Actual:** 3 results — AUTH.md Supabase OAuth, OAuth Rip-and-Replace, OAuth Integration Analysis
|
||||
- **Verdict:** PASS
|
||||
|
||||
#### 1f: search_type="vector" with keyword query
|
||||
- **Input:** query="OAuth authentication", search_type="vector"
|
||||
- **Actual:** Same top results as text (keyword-rich content also scores well in vector space)
|
||||
- **Verdict:** PASS
|
||||
|
||||
---
|
||||
|
||||
### Test 2: Conceptual Queries (vector advantage)
|
||||
|
||||
#### 2a: Natural language question
|
||||
- **Input:** query="why do AI assistants forget things", search_type="vector"
|
||||
- **Actual:** 5 results — Manual Testing Session, "Balance security and usability" observation, "Tools should match thought patterns" observation. Scores ~0.56-0.57
|
||||
- **Vector advantage:** Found conceptually related content despite no exact keyword overlap
|
||||
- **Verdict:** PASS
|
||||
|
||||
#### 2b: Same query, text search
|
||||
- **Input:** query="why do AI assistants forget things", search_type="text"
|
||||
- **Actual:** 1 result — "What is Basic Memory?" (likely matched on "AI" token)
|
||||
- **Verdict:** PASS (demonstrates vector advantage — text barely matched)
|
||||
|
||||
#### 2c: Domain concept with no jargon
|
||||
- **Input:** query="pricing strategy for cloud product", search_type="vector"
|
||||
- **Actual:** 3 results — SPEC-16 MCP Cloud Service Consolidation, knowledge architecture observation, Visual Knowledge Spaces relation. Scores ~0.56-0.57
|
||||
- **Verdict:** PASS (found cloud-related content conceptually)
|
||||
|
||||
#### 2d: Technical concept, long query
|
||||
- **Input:** query="SQLite performance optimization WAL mode concurrent writes", search_type="vector"
|
||||
- **Actual:** 3 results — SPEC-11 API Performance Optimization, Real-Time Updates with WebSockets, marketing status update. Scores ~0.55-0.58
|
||||
- **Verdict:** PASS (found performance-related content)
|
||||
|
||||
---
|
||||
|
||||
### Test 3: Keyword Queries (FTS strength)
|
||||
|
||||
#### 3a: Exact term match — "OAuth authentication"
|
||||
- **Text:** 3 results with high relevance (exact matches in titles)
|
||||
- **Vector:** Same top results (keyword overlap helps vector too)
|
||||
- **Verdict:** PASS — FTS and vector converge on keyword-rich queries
|
||||
|
||||
#### 3b: "OAuth" single keyword, hybrid mode
|
||||
- **Input:** query="OAuth", search_type="hybrid"
|
||||
- **Actual:** 5 results — Basic Memory Coding Guide, AI Collaboration Examples, SPEC-18, daily note, Manual Testing Session. FTS + vector blended. Scores ~0.016-0.032
|
||||
- **Note:** Top hybrid result is "Basic Memory Coding Guide" not an OAuth-specific doc — suggests hybrid scoring may dilute strong FTS matches
|
||||
- **Verdict:** PASS but hybrid ranking questionable for single-keyword queries
|
||||
|
||||
---
|
||||
|
||||
### Test 4: Hybrid Ranking
|
||||
|
||||
#### 4a: Hybrid vs vector on "OAuth authentication"
|
||||
- **Hybrid with entity_types=["entity"]:** 5 results — RLS Implementation Lessons, Cloud Readiness Assessment, AUTH.md OAuth, Core Service Implementation, OAuth Rip-and-Replace. Scores ~0.016-0.023
|
||||
- **Vector with entity_types=["entity"]:** 5 results — Core Service Implementation, SPEC-13 CLI Auth, Coding Guide, Authentication Service, ADR Production Auth. Scores ~0.55-0.60
|
||||
- **Observation:** Hybrid surfaces different top results than vector-only. Hybrid found RLS and Cloud Readiness docs that vector didn't prioritize. Different ranking is expected from RRF fusion.
|
||||
- **Verdict:** PASS — hybrid produces meaningfully different ranking
|
||||
|
||||
---
|
||||
|
||||
### Test 5: Result Types
|
||||
|
||||
#### 5a: Vector returns all result types
|
||||
- **Input:** query="keeping AI context between sessions", search_type="vector"
|
||||
- **Entities:** SPEC-18 AI Memory Management Tool (type=entity)
|
||||
- **Relations:** Prompt Builder integrates_with (type=relation)
|
||||
- **Observations:** "Translation layer is key" (type=observation), "Maintaining context across conversation boundaries" (type=observation)
|
||||
- **Verdict:** PASS — all three types appear in vector results
|
||||
|
||||
#### 5b: Observations carry metadata
|
||||
- **Observation result:** category="challenge", content="Maintaining context across conversation boundaries", from_entity="research/ai-knowledge-management-research"
|
||||
- **Verdict:** PASS — category, content, from_entity, tags all present
|
||||
|
||||
#### 5c: Relations carry link info
|
||||
- **Relation result:** relation_type="integrates_with", from_entity="development/features/prompt-builder...", to_entity (present but truncated in some)
|
||||
- **Verdict:** PASS — relation metadata present
|
||||
|
||||
---
|
||||
|
||||
### Test 6: Filters + Vector Search
|
||||
|
||||
#### 6a: entity_types=["entity"] with vector
|
||||
- **Input:** query="OAuth authentication", search_type="vector", entity_types=["entity"]
|
||||
- **Actual:** 5 results, all type="entity" (Core Service Implementation, SPEC-13, Coding Guide, Authentication Service, ADR Auth)
|
||||
- **Verdict:** PASS — filter correctly restricts to entities only
|
||||
|
||||
#### 6b: types=["note"] with vector
|
||||
- **Input:** query="OAuth authentication", search_type="vector", types=["note"]
|
||||
- **Actual:** Same 5 results (all have entity_type="note" in metadata)
|
||||
- **Verdict:** PASS — types filter works with vector search
|
||||
|
||||
#### 6c: after_date with vector
|
||||
- **Input:** query="OAuth authentication", search_type="vector", after_date="2025-06-01"
|
||||
- **Actual:** 3 results — Core Service Implementation, Cloud Web App analysis observation, SPEC-13. Filtered out older OAuth docs.
|
||||
- **Verdict:** PASS — date filter applied correctly
|
||||
|
||||
#### 6d: entity_types=["entity"] with hybrid
|
||||
- **Input:** query="OAuth authentication", search_type="hybrid", entity_types=["entity"]
|
||||
- **Actual:** 5 results, all type="entity" — RLS lessons, Cloud Readiness, AUTH.md OAuth, Core Service, OAuth Rip-and-Replace
|
||||
- **Verdict:** PASS — filter works with hybrid mode too
|
||||
|
||||
#### 6e: types=["entity"] with vector (WRONG filter name)
|
||||
- **Input:** query="OAuth authentication", search_type="vector", types=["entity"]
|
||||
- **Actual:** 0 results
|
||||
- **Note:** `types` filters by entity_type metadata (e.g., "note", "person"), NOT by SearchItemType. Using types=["entity"] looks for entity_type="entity" which few/no notes have. This is a UX confusion point — the param names are ambiguous.
|
||||
- **Verdict:** PASS (correct behavior) but USABILITY ISSUE — easy to confuse types vs entity_types
|
||||
|
||||
---
|
||||
|
||||
### Test 7: Edge Cases
|
||||
|
||||
#### 7a: Single character query
|
||||
- **Input:** query="x", search_type="vector"
|
||||
- **Actual:** 3 results — "Self-contained application bundle" observation, Non-Markdown File Support relation, quick-win-tools entity. Scores ~0.57-0.59
|
||||
- **Note:** Single character still produces an embedding and returns results. Quality is low/random as expected.
|
||||
- **Verdict:** PASS (no crash, returns results)
|
||||
|
||||
#### 7b: Whitespace-only query
|
||||
- **Input:** query=" ", search_type="vector"
|
||||
- **Actual:** 0 results
|
||||
- **Verdict:** PASS (handled gracefully — _check_vector_eligible strips and rejects empty)
|
||||
|
||||
#### 7c: Query with no relevant content
|
||||
- **Input:** query="quantum computing blockchain", search_type="vector"
|
||||
- **Actual:** 3 results — Inter-Agent Communication relation, Self-contained bundle observation, JSON-LD interop observation. Scores ~0.54
|
||||
- **Note:** Still returns results because vector search always finds nearest neighbors. Scores are lower (~0.54) than relevant queries (~0.58-0.60). No relevance threshold applied.
|
||||
- **Verdict:** PASS (expected behavior) but NOTE — no relevance cutoff means irrelevant queries always return something
|
||||
|
||||
---
|
||||
|
||||
### Test 8: Pagination
|
||||
|
||||
#### 8a: Vector search page 2
|
||||
- **Input:** query="keeping AI context between sessions", search_type="vector", page=2, page_size=3
|
||||
- **Actual:** 3 results on page 2, current_page=2. Different results from page 1. Top: "Maintaining context across conversation boundaries" observation (score 0.587)
|
||||
- **Note:** Interestingly, page 2 had a higher-scoring result than some page 1 results. This may indicate pagination doesn't sort globally — it might be paginating within a pre-scored set.
|
||||
- **Verdict:** PASS (pagination works) but POSSIBLE ISSUE — result ordering across pages needs investigation
|
||||
|
||||
---
|
||||
|
||||
## Summary
|
||||
|
||||
### Passing Tests: 20/21
|
||||
|
||||
### Bugs Found
|
||||
1. **search_type="semantic" silently falls through** (Test 1a) — Invalid search_type values fall to the `else` branch and default to text search without any warning. Should either alias "semantic" to "vector" or raise an error.
|
||||
|
||||
### Usability Issues
|
||||
2. **types vs entity_types confusion** (Test 6e) — `types` filters by entity_type metadata (note, person, etc.) while `entity_types` filters by SearchItemType (entity, observation, relation). The naming is ambiguous and easy to mix up.
|
||||
3. **No relevance threshold** (Test 7c) — Vector search always returns nearest neighbors even for completely irrelevant queries. Consider adding a minimum score threshold or at least documenting expected score ranges.
|
||||
4. **Hybrid ranking for single keywords** (Test 3b) — Hybrid mode on simple keyword queries produced less intuitive rankings than pure FTS or pure vector. The RRF fusion may dilute strong FTS signals.
|
||||
|
||||
### Observations
|
||||
- Vector search successfully finds conceptually related content that FTS misses entirely
|
||||
- Score ranges: relevant queries ~0.56-0.60, irrelevant queries ~0.54 (narrow spread)
|
||||
- All three result types (entity, observation, relation) appear correctly in vector results
|
||||
- Filters (entity_types, types, after_date) all work correctly with vector and hybrid modes
|
||||
- Pagination works but cross-page ordering may need investigation
|
||||
@@ -0,0 +1,271 @@
|
||||
# Semantic Search
|
||||
|
||||
This guide covers Basic Memory's semantic (vector) search feature, which adds meaning-based retrieval alongside the existing full-text search.
|
||||
|
||||
## Overview
|
||||
|
||||
Basic Memory's search supports both full-text search (FTS) and semantic retrieval. Semantic search adds vector embeddings that capture the *meaning* of your content, enabling:
|
||||
|
||||
- **Paraphrase matching**: Find "authentication flow" when searching for "login process"
|
||||
- **Conceptual queries**: Search for "ways to improve performance" and find notes about caching, indexing, and optimization
|
||||
- **Hybrid retrieval**: Combine the precision of keyword search with the recall of semantic similarity
|
||||
|
||||
Semantic search is enabled by default when semantic dependencies are available at runtime. It works on both SQLite (local) and Postgres (cloud) backends.
|
||||
|
||||
## Installation
|
||||
|
||||
Semantic search dependencies (fastembed, sqlite-vec, openai) are included in the default `basic-memory` install.
|
||||
|
||||
```bash
|
||||
pip install basic-memory
|
||||
```
|
||||
|
||||
You can always override with `BASIC_MEMORY_SEMANTIC_SEARCH_ENABLED=true|false`.
|
||||
|
||||
### Platform Compatibility
|
||||
|
||||
| Platform | FastEmbed (local) | OpenAI (API) |
|
||||
|---|---|---|
|
||||
| macOS ARM64 (Apple Silicon) | Yes | Yes |
|
||||
| macOS x86_64 (Intel Mac) | No — see workaround below | Yes |
|
||||
| Linux x86_64 | Yes | Yes |
|
||||
| Linux ARM64 | Yes | Yes |
|
||||
| Windows x86_64 | Yes | Yes |
|
||||
|
||||
#### Intel Mac Workaround
|
||||
|
||||
The default install includes FastEmbed, which depends on ONNX Runtime. ONNX Runtime dropped Intel Mac (x86_64) wheels starting in v1.24, so install with a compatible ONNX Runtime pin first:
|
||||
|
||||
```bash
|
||||
pip install basic-memory 'onnxruntime<1.24'
|
||||
```
|
||||
|
||||
After installation, Intel Mac users have two runtime options:
|
||||
|
||||
**Option 1: Use OpenAI embeddings (recommended)**
|
||||
|
||||
```bash
|
||||
export BASIC_MEMORY_SEMANTIC_SEARCH_ENABLED=true
|
||||
export BASIC_MEMORY_SEMANTIC_EMBEDDING_PROVIDER=openai
|
||||
export OPENAI_API_KEY=sk-...
|
||||
```
|
||||
|
||||
**Option 2: Use FastEmbed locally**
|
||||
|
||||
Keep the same pinned installation and use FastEmbed (default provider):
|
||||
|
||||
```bash
|
||||
export BASIC_MEMORY_SEMANTIC_SEARCH_ENABLED=true
|
||||
export BASIC_MEMORY_SEMANTIC_EMBEDDING_PROVIDER=fastembed
|
||||
```
|
||||
|
||||
## Quick Start
|
||||
|
||||
1. Install Basic Memory:
|
||||
|
||||
```bash
|
||||
pip install basic-memory
|
||||
```
|
||||
|
||||
2. (Optional) Explicitly enable semantic search:
|
||||
|
||||
```bash
|
||||
export BASIC_MEMORY_SEMANTIC_SEARCH_ENABLED=true
|
||||
```
|
||||
|
||||
3. Build vector embeddings for your existing content:
|
||||
|
||||
```bash
|
||||
bm reindex --embeddings
|
||||
```
|
||||
|
||||
4. Search using semantic modes:
|
||||
|
||||
```python
|
||||
# Pure vector similarity
|
||||
search_notes("login process", search_type="vector")
|
||||
|
||||
# Hybrid: combines FTS precision with vector recall (recommended)
|
||||
search_notes("login process", search_type="hybrid")
|
||||
|
||||
# Explicit full-text search
|
||||
search_notes("login process", search_type="text")
|
||||
```
|
||||
|
||||
## Configuration Reference
|
||||
|
||||
All settings are fields on `BasicMemoryConfig` and can be set via environment variables (prefixed with `BASIC_MEMORY_`).
|
||||
|
||||
| Config Field | Env Var | Default | Description |
|
||||
|---|---|---|---|
|
||||
| `semantic_search_enabled` | `BASIC_MEMORY_SEMANTIC_SEARCH_ENABLED` | Auto (`true` when semantic deps are available) | Enable semantic search. Required before vector/hybrid modes work. |
|
||||
| `semantic_embedding_provider` | `BASIC_MEMORY_SEMANTIC_EMBEDDING_PROVIDER` | `"fastembed"` | Embedding provider: `"fastembed"` (local) or `"openai"` (API). |
|
||||
| `semantic_embedding_model` | `BASIC_MEMORY_SEMANTIC_EMBEDDING_MODEL` | `"bge-small-en-v1.5"` | Model identifier. Auto-adjusted per provider if left at default. |
|
||||
| `semantic_embedding_dimensions` | `BASIC_MEMORY_SEMANTIC_EMBEDDING_DIMENSIONS` | Auto-detected | Vector dimensions. 384 for FastEmbed, 1536 for OpenAI. Override only if using a non-default model. |
|
||||
| `semantic_embedding_batch_size` | `BASIC_MEMORY_SEMANTIC_EMBEDDING_BATCH_SIZE` | `64` | Number of texts to embed per batch. |
|
||||
| `semantic_vector_k` | `BASIC_MEMORY_SEMANTIC_VECTOR_K` | `100` | Candidate count for vector nearest-neighbour retrieval. Higher values improve recall at the cost of latency. |
|
||||
|
||||
## Embedding Providers
|
||||
|
||||
### FastEmbed (default)
|
||||
|
||||
FastEmbed runs entirely locally using ONNX models — no API key, no network calls, no cost.
|
||||
|
||||
- **Model**: `BAAI/bge-small-en-v1.5`
|
||||
- **Dimensions**: 384
|
||||
- **Tradeoff**: Smaller model, fast inference, good quality for most use cases
|
||||
|
||||
```bash
|
||||
# Install basic-memory and enable semantic search
|
||||
pip install basic-memory
|
||||
export BASIC_MEMORY_SEMANTIC_SEARCH_ENABLED=true
|
||||
```
|
||||
|
||||
### OpenAI
|
||||
|
||||
Uses OpenAI's embeddings API for higher-dimensional vectors. Requires an API key.
|
||||
|
||||
- **Model**: `text-embedding-3-small`
|
||||
- **Dimensions**: 1536
|
||||
- **Tradeoff**: Higher quality embeddings, requires API calls and an OpenAI key
|
||||
|
||||
```bash
|
||||
export BASIC_MEMORY_SEMANTIC_SEARCH_ENABLED=true
|
||||
export BASIC_MEMORY_SEMANTIC_EMBEDDING_PROVIDER=openai
|
||||
export OPENAI_API_KEY=sk-...
|
||||
```
|
||||
|
||||
When switching from FastEmbed to OpenAI (or vice versa), you must rebuild embeddings since the vector dimensions differ:
|
||||
|
||||
```bash
|
||||
bm reindex --embeddings
|
||||
```
|
||||
|
||||
## Search Modes
|
||||
|
||||
### `text` (default)
|
||||
|
||||
Full-text keyword search using FTS5 (SQLite) or tsvector (Postgres). Supports boolean operators (`AND`, `OR`, `NOT`), phrase matching, and prefix wildcards.
|
||||
|
||||
```python
|
||||
search_notes("project AND planning", search_type="text")
|
||||
```
|
||||
|
||||
This is the existing default and does not require semantic search to be enabled.
|
||||
|
||||
### `vector`
|
||||
|
||||
Pure semantic similarity search. Embeds your query and finds the nearest content vectors. Good for conceptual or paraphrase queries where exact keywords may not appear in the content.
|
||||
|
||||
```python
|
||||
search_notes("how to speed up the app", search_type="vector")
|
||||
```
|
||||
|
||||
Returns results ranked by cosine similarity. Individual observations and relations surface as first-class results, not collapsed into parent entities.
|
||||
|
||||
### `hybrid`
|
||||
|
||||
Combines FTS and vector results using score-based fusion. This is generally the best mode when you want both keyword precision and semantic recall.
|
||||
|
||||
```python
|
||||
search_notes("authentication security", search_type="hybrid")
|
||||
```
|
||||
|
||||
Score-based fusion uses the formula `max(vec, fts) + bonus * min(vec, fts)` to preserve the dominant signal while rewarding results found by both methods.
|
||||
|
||||
### When to Use Which
|
||||
|
||||
| Mode | Best For |
|
||||
|---|---|
|
||||
| `text` | Exact keyword matching, boolean queries, tag/category searches |
|
||||
| `vector` | Conceptual queries, paraphrase matching, exploratory searches |
|
||||
| `hybrid` | General-purpose search combining precision and recall |
|
||||
|
||||
## The Reindex Command
|
||||
|
||||
The `bm reindex` command rebuilds search indexes without dropping the database.
|
||||
|
||||
```bash
|
||||
# Rebuild everything (FTS + embeddings if semantic is enabled)
|
||||
bm reindex
|
||||
|
||||
# Only rebuild vector embeddings
|
||||
bm reindex --embeddings
|
||||
|
||||
# Only rebuild the full-text search index
|
||||
bm reindex --search
|
||||
|
||||
# Target a specific project
|
||||
bm reindex -p my-project
|
||||
```
|
||||
|
||||
### When You Need to Reindex
|
||||
|
||||
- **Upgrade note**: Migration now performs a one-time automatic embedding backfill on upgrade.
|
||||
- **Manual enable case**: If you explicitly had `semantic_search_enabled=false` and then turn it on
|
||||
- **Provider change**: After switching between `fastembed` and `openai`
|
||||
- **Model change**: After changing `semantic_embedding_model`
|
||||
- **Dimension change**: After changing `semantic_embedding_dimensions`
|
||||
|
||||
The reindex command shows progress with embedded/skipped/error counts:
|
||||
|
||||
```
|
||||
Project: main
|
||||
Building vector embeddings...
|
||||
✓ Embeddings complete: 142 entities embedded, 0 skipped, 0 errors
|
||||
|
||||
Reindex complete!
|
||||
```
|
||||
|
||||
## How It Works
|
||||
|
||||
### Chunking
|
||||
|
||||
Each entity in the search index is split into semantic chunks before embedding:
|
||||
|
||||
- **Headers**: Markdown headers (`#`, `##`, etc.) start new chunks
|
||||
- **Bullets**: Each bullet item (`-`, `*`) becomes its own chunk for granular fact retrieval
|
||||
- **Prose sections**: Non-bullet text is merged up to ~900 characters per chunk
|
||||
- **Long sections**: Oversized content is split with ~120 character overlap to preserve context at boundaries
|
||||
|
||||
Each search index item type (entity, observation, relation) is chunked independently, so observations and relations are embeddable as discrete facts.
|
||||
|
||||
### Deduplication
|
||||
|
||||
Each chunk has a `source_hash` (SHA-256 of the chunk text). On re-sync, unchanged chunks skip re-embedding entirely. This makes incremental updates fast — only modified content triggers API calls or model inference.
|
||||
|
||||
### Hybrid Fusion
|
||||
|
||||
Hybrid search uses score-based fusion to merge FTS and vector results:
|
||||
|
||||
1. Run FTS search to get keyword-ranked results; normalize scores to [0, 1]
|
||||
2. Run vector search to get similarity-ranked results (already [0, 1])
|
||||
3. For each result, compute: `fused = max(vec_score, fts_score) + 0.3 * min(vec_score, fts_score)`
|
||||
4. Sort by fused score
|
||||
|
||||
The dominant signal (whichever source scored higher) is preserved, and dual-source agreement adds a bonus. Unlike rank-based fusion, this approach retains score magnitude — a strong vector match stays strong even without an FTS hit.
|
||||
|
||||
### Observation-Level Results
|
||||
|
||||
Vector and hybrid modes return individual observations and relations as first-class search results, not just parent entities. This means a search for "water temperature for brewing" can surface the specific observation about 205°F without returning the entire "Coffee Brewing Methods" entity.
|
||||
|
||||
## Database Backends
|
||||
|
||||
### SQLite (local)
|
||||
|
||||
- **Vector storage**: [sqlite-vec](https://github.com/asg017/sqlite-vec) virtual table
|
||||
- **Table creation**: At runtime when semantic search is first used — no migration needed
|
||||
- **Embedding table**: `search_vector_embeddings` using `vec0(embedding float[N])` where N is the configured dimensions
|
||||
- **Chunk metadata**: `search_vector_chunks` table stores chunk text, keys, and source hashes
|
||||
|
||||
The sqlite-vec extension is loaded per-connection. Vector tables are created lazily on first use.
|
||||
|
||||
### Postgres (cloud)
|
||||
|
||||
- **Vector storage**: [pgvector](https://github.com/pgvector/pgvector) with HNSW indexing
|
||||
- **Local Docker**: use `docker-compose-postgres.yml` (`pgvector/pgvector:pg17`). Plain `postgres:17` lacks the extension; run `CREATE EXTENSION IF NOT EXISTS vector;` on any external instance before first migration.
|
||||
- **Chunk metadata table**: Created via Alembic migration (`search_vector_chunks` with `BIGSERIAL` primary key)
|
||||
- **Embedding table**: `search_vector_embeddings` created at runtime (dimension-dependent, same pattern as SQLite)
|
||||
- **Index**: HNSW index on the embedding column for fast approximate nearest-neighbour queries
|
||||
|
||||
The Alembic migration creates the dimension-independent chunks table. The embeddings table and HNSW index are deferred to runtime because they depend on the configured vector dimensions.
|
||||
@@ -0,0 +1,225 @@
|
||||
# SPEC-LOCAL-PLUS-PUBLISH: Local+ Published Notes and Privacy Tiers
|
||||
|
||||
**Status:** Draft
|
||||
**Date:** 2026-02-14
|
||||
**Owner:** Basic Memory
|
||||
|
||||
## Summary
|
||||
|
||||
Add a paid Local+ feature that lets users publish selected notes to shareable URLs while keeping the
|
||||
main knowledge base local-first. Use this as a product wedge for users who do not want full cloud
|
||||
hosting but do want collaboration and distribution features.
|
||||
|
||||
This spec also captures a practical position on "zero knowledge" for Local+.
|
||||
|
||||
## Context
|
||||
|
||||
Basic Memory already has strong local-first primitives and optional cloud routing/sync. A recurring
|
||||
request is:
|
||||
|
||||
- keep knowledge local by default,
|
||||
- pay for selective value-add,
|
||||
- share specific outputs externally.
|
||||
|
||||
Published Notes fits this model: explicit per-note opt-in, reversible, and easy to understand.
|
||||
|
||||
## Goals
|
||||
|
||||
1. Provide an Obsidian Publish-style sharing experience for selected notes.
|
||||
2. Keep local markdown files as source of truth.
|
||||
3. Make sharing compatible with current cloud/auth/billing primitives.
|
||||
4. Define clear Local+ packaging that does not degrade OSS local workflows.
|
||||
5. Document zero-knowledge constraints so product decisions are explicit.
|
||||
|
||||
## Non-Goals
|
||||
|
||||
1. Full hosted editing for all notes (Cloud Full remains separate).
|
||||
2. Public website builder/CMS features.
|
||||
3. Strict cryptographic zero-knowledge server processing for MCP/search in v1.
|
||||
|
||||
## Local+ Feature Catalog (Sellable)
|
||||
|
||||
Core Local+ candidates:
|
||||
|
||||
1. Published Notes (share URL, revoke, expiry, password).
|
||||
2. Snapshot Time Machine (point-in-time restore for local projects).
|
||||
3. Recovery Drill Reports (automated restore verification).
|
||||
4. Device/API Key Governance (per-device keys, revocation, audit trail).
|
||||
5. BYO Storage Orchestration (managed setup for user-owned object storage).
|
||||
6. Semantic Boost Add-on (higher quality retrieval options while files remain source-of-truth).
|
||||
|
||||
Team-oriented add-ons:
|
||||
|
||||
1. Team-owned shared links and domain branding.
|
||||
2. Role-based publish permissions.
|
||||
3. Shared workspace policies for what can be published.
|
||||
|
||||
## Proposed MVP: Published Notes
|
||||
|
||||
### User Experience
|
||||
|
||||
Per note actions:
|
||||
|
||||
1. Publish.
|
||||
2. Unpublish.
|
||||
3. Copy URL.
|
||||
4. Regenerate URL.
|
||||
5. Set visibility and controls.
|
||||
|
||||
Controls:
|
||||
|
||||
1. Visibility: `unlisted` (default) or `public`.
|
||||
2. Optional password gate.
|
||||
3. Optional expiration datetime.
|
||||
4. Optional "disable indexing" flag for public mode.
|
||||
|
||||
Behavior:
|
||||
|
||||
1. Source note remains local markdown.
|
||||
2. Publish is explicit opt-in per note.
|
||||
3. Unpublish removes public access immediately.
|
||||
4. Republish creates a new URL token unless user chooses to keep current URL.
|
||||
|
||||
### URL Model
|
||||
|
||||
1. Unlisted share URL: high-entropy token path.
|
||||
2. Public URL: slug path (optional, later phase).
|
||||
3. Team plans can support custom domain mapping in later phase.
|
||||
|
||||
### Content Model
|
||||
|
||||
v1 published page includes:
|
||||
|
||||
1. Rendered markdown body.
|
||||
2. Optional metadata (title, updated_at).
|
||||
|
||||
v1 excludes:
|
||||
|
||||
1. Full graph traversal expansion.
|
||||
2. Related note auto-discovery on public pages.
|
||||
|
||||
### Sync Model
|
||||
|
||||
1. Local file remains canonical.
|
||||
2. Publish stores a rendered snapshot plus metadata in cloud.
|
||||
3. Update path:
|
||||
- manual "update published version", or
|
||||
- optional auto-update on note change (plan-gated).
|
||||
|
||||
## Architecture (v1)
|
||||
|
||||
### High-Level Flow
|
||||
|
||||
1. Client selects a note to publish.
|
||||
2. Client sends publish request with note identifier and policy.
|
||||
3. Service resolves note content (local sync artifact or explicit upload payload).
|
||||
4. Service stores published artifact and returns share URL.
|
||||
|
||||
### Data Model
|
||||
|
||||
`published_notes`
|
||||
|
||||
1. `id` (uuid)
|
||||
2. `tenant_id` or `workspace_id`
|
||||
3. `project_id`
|
||||
4. `entity_permalink` (or stable external_id)
|
||||
5. `share_token` (hashed in DB)
|
||||
6. `visibility` (`unlisted`|`public`)
|
||||
7. `password_hash` (nullable)
|
||||
8. `expires_at` (nullable)
|
||||
9. `is_active`
|
||||
10. `published_content` (rendered snapshot or reference)
|
||||
11. `published_at`
|
||||
12. `updated_at`
|
||||
|
||||
### API Shape (Draft)
|
||||
|
||||
1. `POST /api/published-notes`
|
||||
2. `GET /api/published-notes`
|
||||
3. `GET /api/published-notes/{id}`
|
||||
4. `PATCH /api/published-notes/{id}`
|
||||
5. `DELETE /api/published-notes/{id}` (unpublish)
|
||||
6. `POST /api/published-notes/{id}/regenerate-url`
|
||||
7. `GET /p/{token}` (public resolver)
|
||||
|
||||
### CLI Shape (Draft)
|
||||
|
||||
1. `bm cloud publish <identifier>`
|
||||
2. `bm cloud publish list`
|
||||
3. `bm cloud publish update <id>`
|
||||
4. `bm cloud publish unpublish <id>`
|
||||
5. `bm cloud publish rotate-url <id>`
|
||||
|
||||
### Security
|
||||
|
||||
1. Default to unlisted URLs.
|
||||
2. Store only hashed share tokens.
|
||||
3. Passwords hashed server-side.
|
||||
4. Enforce expiration at request time.
|
||||
5. Log publish/unpublish/rotate events for auditability.
|
||||
|
||||
## Packaging and Pricing Direction
|
||||
|
||||
Suggested split:
|
||||
|
||||
1. OSS Local: no publish URLs.
|
||||
2. Local+ Solo: publish URLs + snapshots + recovery.
|
||||
3. Local+ Team: solo features + team governance and branding.
|
||||
4. Cloud Full: hosted app + full cloud workflows.
|
||||
|
||||
Key message:
|
||||
"Keep everything local. Publish only what you choose."
|
||||
|
||||
## Rollout Plan
|
||||
|
||||
1. Phase 1: Unlisted publish URLs + unpublish + regenerate URL.
|
||||
2. Phase 2: Password/expiry controls.
|
||||
3. Phase 3: Auto-update on note change and basic analytics.
|
||||
4. Phase 4: Team branding/domains/policies.
|
||||
|
||||
## Zero-Knowledge Position
|
||||
|
||||
### Strict Zero-Knowledge Definition
|
||||
|
||||
Strict zero-knowledge means the server cannot decrypt note content at all.
|
||||
|
||||
### Why This Conflicts with MCP and Search
|
||||
|
||||
If server cannot decrypt:
|
||||
|
||||
1. MCP tool execution against cloud content cannot read/write semantic content.
|
||||
2. Full-text search cannot index plaintext content.
|
||||
3. Semantic/vector search cannot generate or query embeddings on plaintext.
|
||||
4. Server-side relation resolution and context building become severely limited.
|
||||
|
||||
This matches earlier findings: strict zero-knowledge materially handicaps MCP-driven behavior and
|
||||
search quality.
|
||||
|
||||
### Viable Alternatives (Not Strict Zero-Knowledge)
|
||||
|
||||
1. Encryption at rest/in transit with server-side decrypt in trusted runtime.
|
||||
- Preserves MCP/search quality.
|
||||
- Not zero-knowledge cryptographically.
|
||||
|
||||
2. Client-side retrieval mode.
|
||||
- Keep MCP/search local; cloud is sync/share/backup relay.
|
||||
- Best for privacy-first users.
|
||||
- Requires local agent availability for advanced retrieval.
|
||||
|
||||
3. Limited encrypted indexing.
|
||||
- Blind indexes for exact keywords only.
|
||||
- No high-quality semantic search.
|
||||
- Usually poor UX for natural-language memory recall.
|
||||
|
||||
### Recommendation
|
||||
|
||||
For Local+:
|
||||
|
||||
1. Do not promise strict zero-knowledge for cloud MCP/search paths.
|
||||
2. Offer a privacy-first local mode where advanced retrieval stays local.
|
||||
3. Clearly label tradeoffs:
|
||||
- "Local private mode" (best privacy, best local retrieval).
|
||||
- "Cloud-assisted mode" (best cross-device/MCP consistency, trusted-runtime decrypt).
|
||||
|
||||
This keeps messaging honest and avoids repeating the known incompatibility.
|
||||
|
||||
@@ -0,0 +1,368 @@
|
||||
# SPEC-SCHEMA-IMPL: Schema System Implementation Plan
|
||||
|
||||
**Status:** Draft
|
||||
**Created:** 2025-02-06
|
||||
**Branch:** `feature/schema-system`
|
||||
**Depends on:** [SPEC-SCHEMA](SPEC-SCHEMA.md)
|
||||
|
||||
## Overview
|
||||
|
||||
Implementation plan for the Basic Memory Schema System. The system is entirely programmatic —
|
||||
no LLM agent runtime or API key required. The LLM already in the user's session (Claude Code,
|
||||
Claude Desktop, etc.) provides the intelligence layer by reading schema notes via existing
|
||||
MCP tools.
|
||||
|
||||
## Architecture
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────────────┐
|
||||
│ Entry Points │
|
||||
│ CLI (bm schema ...) │ MCP (schema_validate) │
|
||||
└──────────┬────────────┴──────────┬──────────────┘
|
||||
│ │
|
||||
▼ ▼
|
||||
┌─────────────────────────────────────────────────┐
|
||||
│ Schema Service Layer │
|
||||
│ resolve_schema · validate · infer · diff │
|
||||
└──────────┬────────────────────────┬──────────────┘
|
||||
│ │
|
||||
▼ ▼
|
||||
┌──────────────────────┐ ┌────────────────────────┐
|
||||
│ Picoschema Parser │ │ Note/Entity Access │
|
||||
│ YAML → SchemaModel │ │ (existing repository) │
|
||||
└──────────────────────┘ └────────────────────────┘
|
||||
```
|
||||
|
||||
No new database tables. Schemas are notes with `type: schema` — they're already indexed.
|
||||
Validation reads observations and relations from existing data.
|
||||
|
||||
## Components
|
||||
|
||||
### 1. Picoschema Parser
|
||||
|
||||
**Location:** `src/basic_memory/schema/parser.py`
|
||||
|
||||
Parses Picoschema YAML into an internal representation.
|
||||
|
||||
```python
|
||||
@dataclass
|
||||
class SchemaField:
|
||||
name: str
|
||||
type: str # string, integer, number, boolean, any, or EntityName
|
||||
required: bool # True unless field name ends with ?
|
||||
is_array: bool # True if (array) notation
|
||||
is_enum: bool # True if (enum) notation
|
||||
enum_values: list[str] # Populated for enums
|
||||
description: str | None # Text after comma
|
||||
is_entity_ref: bool # True if type is capitalized (entity reference)
|
||||
children: list[SchemaField] # For (object) types
|
||||
|
||||
|
||||
@dataclass
|
||||
class SchemaDefinition:
|
||||
entity: str # The entity type this schema describes
|
||||
version: int # Schema version
|
||||
fields: list[SchemaField] # Parsed fields
|
||||
validation_mode: str # "warn" | "strict" | "off"
|
||||
frontmatter_fields: list[SchemaField] # From settings.frontmatter (default: [])
|
||||
|
||||
|
||||
def parse_picoschema(yaml_dict: dict) -> list[SchemaField]:
|
||||
"""Parse a Picoschema YAML dict into a list of SchemaField objects."""
|
||||
|
||||
|
||||
def parse_schema_note(frontmatter: dict) -> SchemaDefinition:
|
||||
"""Parse a full schema note's frontmatter into a SchemaDefinition."""
|
||||
```
|
||||
|
||||
**Input/Output:**
|
||||
```yaml
|
||||
# Input (YAML dict from frontmatter)
|
||||
schema:
|
||||
name: string, full name
|
||||
role?: string, job title
|
||||
works_at?: Organization, employer
|
||||
expertise?(array): string, areas of knowledge
|
||||
```
|
||||
|
||||
```python
|
||||
# Output
|
||||
[
|
||||
SchemaField(name="name", type="string", required=True, description="full name", ...),
|
||||
SchemaField(name="role", type="string", required=False, description="job title", ...),
|
||||
SchemaField(name="works_at", type="Organization", required=False, is_entity_ref=True, ...),
|
||||
SchemaField(name="expertise", type="string", required=False, is_array=True, ...),
|
||||
]
|
||||
```
|
||||
|
||||
### 2. Schema Resolver
|
||||
|
||||
**Location:** `src/basic_memory/schema/resolver.py`
|
||||
|
||||
Finds the applicable schema for a note using the resolution order.
|
||||
|
||||
```python
|
||||
async def resolve_schema(
|
||||
note_frontmatter: dict,
|
||||
search_fn: Callable, # injected search capability
|
||||
) -> SchemaDefinition | None:
|
||||
"""Resolve schema for a note.
|
||||
|
||||
Resolution order:
|
||||
1. Inline schema (frontmatter['schema'] is a dict)
|
||||
2. Explicit reference (frontmatter['schema'] is a string)
|
||||
3. Implicit by type (frontmatter['type'] → schema note with matching entity)
|
||||
4. No schema (returns None)
|
||||
"""
|
||||
```
|
||||
|
||||
### 3. Schema Validator
|
||||
|
||||
**Location:** `src/basic_memory/schema/validator.py`
|
||||
|
||||
Validates a note's observations and relations against a resolved schema.
|
||||
|
||||
```python
|
||||
@dataclass
|
||||
class FieldResult:
|
||||
field: SchemaField
|
||||
status: str # "present" | "missing" | "type_mismatch"
|
||||
values: list[str] # Matched observation values or relation targets
|
||||
message: str | None # Human-readable detail
|
||||
|
||||
|
||||
@dataclass
|
||||
class ValidationResult:
|
||||
note_identifier: str
|
||||
schema_entity: str
|
||||
passed: bool # True if no errors (warnings are OK)
|
||||
field_results: list[FieldResult]
|
||||
unmatched_observations: dict[str, int] # category → count
|
||||
unmatched_relations: list[str] # relation types not in schema
|
||||
warnings: list[str]
|
||||
errors: list[str]
|
||||
|
||||
|
||||
async def validate_note(
|
||||
note: Note,
|
||||
schema: SchemaDefinition,
|
||||
frontmatter: dict | None = None,
|
||||
) -> ValidationResult:
|
||||
"""Validate a note against a schema definition.
|
||||
|
||||
Mapping rules:
|
||||
- field: string → observation [field] exists
|
||||
- field?(array): type → multiple [field] observations
|
||||
- field?: EntityType → relation 'field [[...]]' exists
|
||||
- field?(enum): [v] → observation [field] value ∈ enum values
|
||||
- settings.frontmatter field → frontmatter key presence/value
|
||||
"""
|
||||
```
|
||||
|
||||
### 4. Schema Inference Engine
|
||||
|
||||
**Location:** `src/basic_memory/schema/inference.py`
|
||||
|
||||
Analyzes notes of a given type and suggests a schema based on usage frequency.
|
||||
|
||||
```python
|
||||
@dataclass
|
||||
class FieldFrequency:
|
||||
name: str
|
||||
source: str # "observation" | "relation"
|
||||
count: int # notes containing this field
|
||||
total: int # total notes analyzed
|
||||
percentage: float
|
||||
sample_values: list[str] # representative values
|
||||
is_array: bool # True if typically appears multiple times per note
|
||||
target_type: str | None # For relations, the most common target entity type
|
||||
|
||||
|
||||
@dataclass
|
||||
class InferenceResult:
|
||||
entity_type: str
|
||||
notes_analyzed: int
|
||||
field_frequencies: list[FieldFrequency]
|
||||
suggested_schema: dict # Ready-to-use Picoschema YAML dict
|
||||
suggested_required: list[str]
|
||||
suggested_optional: list[str]
|
||||
excluded: list[str] # Below threshold
|
||||
|
||||
|
||||
async def infer_schema(
|
||||
entity_type: str,
|
||||
notes: list[Note],
|
||||
required_threshold: float = 0.95, # 95%+ = required
|
||||
optional_threshold: float = 0.25, # 25%+ = optional
|
||||
) -> InferenceResult:
|
||||
"""Analyze notes and suggest a Picoschema definition."""
|
||||
```
|
||||
|
||||
### 5. Schema Diff
|
||||
|
||||
**Location:** `src/basic_memory/schema/diff.py`
|
||||
|
||||
Compares current note usage against an existing schema definition.
|
||||
|
||||
```python
|
||||
@dataclass
|
||||
class SchemaDrift:
|
||||
new_fields: list[FieldFrequency] # Fields not in schema but common in notes
|
||||
dropped_fields: list[FieldFrequency] # Fields in schema but rare in notes
|
||||
cardinality_changes: list[str] # one → many or many → one
|
||||
type_mismatches: list[str] # observation values don't match declared type
|
||||
|
||||
|
||||
async def diff_schema(
|
||||
schema: SchemaDefinition,
|
||||
notes: list[Note],
|
||||
) -> SchemaDrift:
|
||||
"""Compare a schema against actual note usage to detect drift."""
|
||||
```
|
||||
|
||||
## Entry Points
|
||||
|
||||
### CLI Commands
|
||||
|
||||
**Location:** `src/basic_memory/cli/schema.py`
|
||||
|
||||
```python
|
||||
import typer
|
||||
|
||||
schema_app = typer.Typer(name="schema", help="Schema management commands")
|
||||
|
||||
@schema_app.command()
|
||||
async def validate(
|
||||
target: str = typer.Argument(None, help="Note path or entity type"),
|
||||
strict: bool = typer.Option(False, help="Override to strict mode"),
|
||||
):
|
||||
"""Validate notes against their schemas."""
|
||||
|
||||
@schema_app.command()
|
||||
async def infer(
|
||||
entity_type: str = typer.Argument(..., help="Entity type to analyze"),
|
||||
threshold: float = typer.Option(0.25, help="Minimum frequency for optional fields"),
|
||||
save: bool = typer.Option(False, help="Save to schema/ directory"),
|
||||
):
|
||||
"""Infer schema from existing notes of a type."""
|
||||
|
||||
@schema_app.command()
|
||||
async def diff(
|
||||
entity_type: str = typer.Argument(..., help="Entity type to diff"),
|
||||
):
|
||||
"""Show drift between schema and actual usage."""
|
||||
```
|
||||
|
||||
Registered as subcommand: `bm schema validate`, `bm schema infer`, `bm schema diff`.
|
||||
|
||||
### MCP Tools
|
||||
|
||||
**Location:** `src/basic_memory/mcp/tools/schema.py`
|
||||
|
||||
```python
|
||||
@mcp_tool
|
||||
async def schema_validate(
|
||||
entity_type: str | None = None,
|
||||
identifier: str | None = None,
|
||||
project: str | None = None,
|
||||
) -> str:
|
||||
"""Validate notes against their resolved schema."""
|
||||
|
||||
@mcp_tool
|
||||
async def schema_infer(
|
||||
entity_type: str,
|
||||
threshold: float = 0.25,
|
||||
project: str | None = None,
|
||||
) -> str:
|
||||
"""Analyze existing notes and suggest a schema definition."""
|
||||
```
|
||||
|
||||
### API Endpoints
|
||||
|
||||
**Location:** `src/basic_memory/api/schema_router.py`
|
||||
|
||||
```python
|
||||
router = APIRouter(prefix="/schema", tags=["schema"])
|
||||
|
||||
@router.post("/validate")
|
||||
async def validate_schema(...) -> ValidationReport: ...
|
||||
|
||||
@router.post("/infer")
|
||||
async def infer_schema(...) -> InferenceResult: ...
|
||||
|
||||
@router.get("/diff/{entity_type}")
|
||||
async def diff_schema(...) -> SchemaDrift: ...
|
||||
```
|
||||
|
||||
MCP tools call these endpoints via the typed client pattern (consistent with existing
|
||||
architecture).
|
||||
|
||||
## Implementation Phases
|
||||
|
||||
### Phase 1: Parser + Resolver
|
||||
|
||||
Build the foundation — can parse Picoschema and find schemas for notes.
|
||||
|
||||
**Deliverables:**
|
||||
- `schema/parser.py` — Picoschema YAML → `SchemaDefinition`
|
||||
- `schema/resolver.py` — Resolution order (inline → explicit ref → implicit by type → none)
|
||||
- Unit tests for all Picoschema syntax variations
|
||||
- Unit tests for resolution order
|
||||
|
||||
**No external dependencies.** Pure Python parsing of YAML dicts. Can develop and test
|
||||
in isolation.
|
||||
|
||||
### Phase 2: Validator
|
||||
|
||||
Connect schemas to notes and produce validation results.
|
||||
|
||||
**Deliverables:**
|
||||
- `schema/validator.py` — Validate note observations/relations against schema fields
|
||||
- API endpoint: `POST /schema/validate`
|
||||
- MCP tool: `schema_validate`
|
||||
- CLI command: `bm schema validate`
|
||||
- Integration tests with real notes and schemas
|
||||
|
||||
**Depends on:** Phase 1 (parser + resolver)
|
||||
|
||||
### Phase 3: Inference
|
||||
|
||||
Analyze existing notes to suggest schemas.
|
||||
|
||||
**Deliverables:**
|
||||
- `schema/inference.py` — Frequency analysis across notes of a type
|
||||
- API endpoint: `POST /schema/infer`
|
||||
- MCP tool: `schema_infer`
|
||||
- CLI command: `bm schema infer`
|
||||
- Option to save inferred schema as a note via `write_note`
|
||||
|
||||
**Depends on:** Phase 1 (parser for output format)
|
||||
|
||||
### Phase 4: Diff
|
||||
|
||||
Compare schemas against current usage.
|
||||
|
||||
**Deliverables:**
|
||||
- `schema/diff.py` — Drift detection between schema and actual notes
|
||||
- API endpoint: `GET /schema/diff/{entity_type}`
|
||||
- CLI command: `bm schema diff`
|
||||
|
||||
**Depends on:** Phase 1 (parser), Phase 3 (inference, for frequency analysis)
|
||||
|
||||
## Testing Strategy
|
||||
|
||||
- **Unit tests** (`tests/schema/`): Parser edge cases, resolution logic, validation mapping,
|
||||
inference thresholds
|
||||
- **Integration tests** (`test-int/schema/`): End-to-end with real markdown files, schema notes
|
||||
on disk, CLI invocation
|
||||
- Coverage target: 100% (consistent with project standard)
|
||||
|
||||
## What This Does NOT Include
|
||||
|
||||
- No new database tables or migrations
|
||||
- No new markdown syntax (schemas validate existing observations/relations)
|
||||
- No LLM agent runtime or API key management
|
||||
- No hook integration (deferred)
|
||||
- No schema composition/inheritance (deferred)
|
||||
- No OWL/RDF export (deferred)
|
||||
- No built-in templates (deferred)
|
||||
@@ -0,0 +1,492 @@
|
||||
# SPEC-SCHEMA: Basic Memory Schema System
|
||||
|
||||
**Status:** Draft
|
||||
**Created:** 2025-02-06
|
||||
**Branch:** `feature/schema-system`
|
||||
|
||||
## Summary
|
||||
|
||||
A schema system for Basic Memory that uses [Picoschema](https://genkit.dev/docs/dotprompt/)
|
||||
syntax in YAML frontmatter. Schemas validate notes against their existing observation/relation
|
||||
structure — no new data model, no migration, just a declarative lens over what's already there.
|
||||
|
||||
## Core Principles
|
||||
|
||||
1. **Schemas are just notes** — A schema is a note with `type: schema`, lives anywhere
|
||||
2. **Use prior art** — Picoschema syntax in YAML frontmatter, no custom notation
|
||||
3. **Validation maps to existing format** — Observations and relations, not a parallel data model
|
||||
4. **Validation is soft** — Warnings by default, not blocking errors
|
||||
5. **Inference over prescription** — Schemas describe reality, emerge from usage
|
||||
6. **No built-in agent** — Programmatic core; the LLM already in the session provides intelligence
|
||||
|
||||
## Picoschema Syntax
|
||||
|
||||
Picoschema is a compact schema notation from Google's Dotprompt that fits naturally in YAML
|
||||
frontmatter.
|
||||
|
||||
### Supported Types
|
||||
|
||||
| Type | Description |
|
||||
|------|-------------|
|
||||
| `string` | Text value |
|
||||
| `integer` | Whole number |
|
||||
| `number` | Decimal number |
|
||||
| `boolean` | True/false |
|
||||
| `any` | Any scalar type |
|
||||
| `EntityName` | Reference to another entity (capitalized = entity reference) |
|
||||
|
||||
### Syntax Rules
|
||||
|
||||
```yaml
|
||||
schema:
|
||||
name: string, full name # required field with description
|
||||
email?: string, contact email # ? = optional
|
||||
role?: string, job title
|
||||
works_at?: Organization, employer # capitalized type = entity reference
|
||||
tags?(array): string, categories # array of type
|
||||
status?(enum): [active, inactive] # enum with allowed values
|
||||
metadata?(object): # nested object
|
||||
updated_at?: string
|
||||
source?: string
|
||||
```
|
||||
|
||||
- `field: type` — required field
|
||||
- `field?: type` — optional field
|
||||
- `field(array): type` — array of values
|
||||
- `field?(enum): [values]` — enumeration
|
||||
- `field?(object):` — nested object with sub-fields
|
||||
- `, description` — description after comma
|
||||
- `EntityName` as type (capitalized) — reference to another entity
|
||||
|
||||
## Schema-to-Note Mapping
|
||||
|
||||
Schemas validate against the existing Basic Memory note format. No new syntax for note
|
||||
authors to learn.
|
||||
|
||||
### Mapping Rules
|
||||
|
||||
| Schema Declaration | Grounded In | Example Match |
|
||||
|--------------------|-------------|---------------|
|
||||
| `field: string` | Observation `[field] value` | `- [name] Paul Graham` |
|
||||
| `field?(array): string` | Multiple `[field]` observations | `- [expertise] Lisp` (×N) |
|
||||
| `field?: EntityType` | Relation `field [[Target]]` | `- works_at [[Y Combinator]]` |
|
||||
| `field?(array): EntityType` | Multiple `field` relations | `- authored [[Book]]` (×N) |
|
||||
| `tags` | Frontmatter `tags` array | `tags: [startups, essays]` |
|
||||
| `field?(enum): [values]` | Observation `[field] value` where value ∈ set | `- [status] active` |
|
||||
| `settings.frontmatter` field | Frontmatter key presence/value | `tags: [python, ai]` |
|
||||
|
||||
### Key Insight
|
||||
|
||||
Schemas don't introduce a new way to store data. They describe the patterns already present
|
||||
in observations and relations. A note doesn't have to change how it's written — the schema
|
||||
just says "a good Person note has a `[name]` observation and a `works_at` relation."
|
||||
|
||||
## Schema Definition
|
||||
|
||||
### As a Dedicated Schema Note
|
||||
|
||||
```yaml
|
||||
# schema/Person.md
|
||||
---
|
||||
title: Person
|
||||
type: schema
|
||||
entity: Person
|
||||
version: 1
|
||||
schema:
|
||||
name: string, full name
|
||||
email?: string, contact email
|
||||
role?: string, job title
|
||||
works_at?: Organization, employer
|
||||
expertise?(array): string, areas of knowledge
|
||||
settings:
|
||||
validation: warn # warn | strict | off
|
||||
frontmatter:
|
||||
tags?(array): string, note categories
|
||||
status?(enum): [draft, review, published]
|
||||
---
|
||||
|
||||
# Person
|
||||
|
||||
A human individual in the knowledge graph.
|
||||
|
||||
Any documentation about this entity type goes here as prose.
|
||||
```
|
||||
|
||||
Schema notes are regular Basic Memory notes. They show up in search, can have their own
|
||||
observations and relations, and can be organized in any folder (though `schema/` is
|
||||
the suggested convention).
|
||||
|
||||
### Inline Schema in a Note
|
||||
|
||||
Notes can carry their own schema directly:
|
||||
|
||||
```yaml
|
||||
# meetings/2024-01-15-standup.md
|
||||
---
|
||||
title: Team Standup 2024-01-15
|
||||
type: meeting
|
||||
schema:
|
||||
attendees(array): string, who was there
|
||||
decisions(array): string, what was decided
|
||||
action_items(array): string, follow-ups
|
||||
blockers?(array): string, anything stuck
|
||||
---
|
||||
|
||||
# Team Standup 2024-01-15
|
||||
|
||||
## Observations
|
||||
- [attendees] Paul
|
||||
- [attendees] Sarah
|
||||
- [decisions] Ship v2 by Friday
|
||||
- [action_items] Paul to review PR #42
|
||||
- [blockers] Waiting on API credentials
|
||||
```
|
||||
|
||||
Good for one-off structured notes or prototyping a schema before extracting it.
|
||||
|
||||
### Explicit Schema Reference
|
||||
|
||||
A note can reference a schema by entity name or permalink:
|
||||
|
||||
```yaml
|
||||
# projects/basic-memory.md
|
||||
---
|
||||
title: Basic Memory
|
||||
schema: SoftwareProject # by entity name
|
||||
---
|
||||
|
||||
# research/llm-memory-patterns.md
|
||||
---
|
||||
title: LLM Memory Patterns
|
||||
schema: schema/research-project # by permalink
|
||||
---
|
||||
```
|
||||
|
||||
Use cases:
|
||||
- Note's `type` differs from the schema it should validate against
|
||||
- Multiple schema variants exist for the same domain
|
||||
- Applying structure to existing notes without changing their type
|
||||
|
||||
## Schema Resolution
|
||||
|
||||
When validating a note, schemas resolve in priority order:
|
||||
|
||||
```
|
||||
1. Inline schema → schema: { ... } (dict in frontmatter)
|
||||
2. Explicit ref → schema: Person (string in frontmatter)
|
||||
3. Implicit by type → type: Person (lookup schema note with entity: Person)
|
||||
4. No schema → no validation (perfectly fine)
|
||||
```
|
||||
|
||||
```python
|
||||
async def resolve_schema(note: Note) -> Schema | None:
|
||||
schema_value = note.frontmatter.get('schema')
|
||||
|
||||
# 1. Inline schema (dict)
|
||||
if isinstance(schema_value, dict):
|
||||
return parse_picoschema(schema_value)
|
||||
|
||||
# 2. Explicit reference (string)
|
||||
if isinstance(schema_value, str):
|
||||
schema_note = await find_schema_note(schema_value)
|
||||
if schema_note:
|
||||
return parse_picoschema(schema_note.frontmatter['schema'])
|
||||
|
||||
# 3. Implicit by type
|
||||
note_type = note.frontmatter.get('type')
|
||||
if note_type:
|
||||
results = await search_notes(f"type:schema entity:{note_type}")
|
||||
if results:
|
||||
return parse_picoschema(results[0].frontmatter['schema'])
|
||||
|
||||
# 4. No schema
|
||||
return None
|
||||
```
|
||||
|
||||
## Validation
|
||||
|
||||
### Modes
|
||||
|
||||
Configured in the schema's `settings.validation`:
|
||||
|
||||
| Mode | Behavior |
|
||||
|------|----------|
|
||||
| `off` | No validation |
|
||||
| `warn` | Warnings in output, doesn't block (default) |
|
||||
| `strict` | Errors that block sync, for CI/CD enforcement |
|
||||
|
||||
### Validation Output
|
||||
|
||||
For a note missing required fields:
|
||||
|
||||
```
|
||||
$ bm schema validate people/ada-lovelace.md
|
||||
|
||||
⚠ Person schema validation:
|
||||
- Missing required field: name (expected [name] observation)
|
||||
- Missing optional field: role
|
||||
- Missing optional field: works_at (no relation found)
|
||||
|
||||
ℹ Unmatched observations: [fact] ×2, [born] ×1
|
||||
ℹ Unmatched relations: collaborated_with
|
||||
```
|
||||
|
||||
"Unmatched" items are informational — observations and relations the schema doesn't cover.
|
||||
They're valid. Schemas are a subset, not a straitjacket.
|
||||
|
||||
### Frontmatter Validation
|
||||
|
||||
Schema notes can declare validation rules for frontmatter keys under `settings.frontmatter`
|
||||
using the same Picoschema syntax as the `schema` block:
|
||||
|
||||
```yaml
|
||||
settings:
|
||||
validation: warn
|
||||
frontmatter:
|
||||
tags?(array): string
|
||||
status?(enum): [draft, review, published]
|
||||
```
|
||||
|
||||
- Frontmatter rules use the same Picoschema key syntax (`?` for optional, `(enum)`, `(array)`)
|
||||
- Only available on schema notes (inline schemas skip frontmatter validation)
|
||||
- Checks key presence (required vs optional) and enum value membership
|
||||
- Unmatched frontmatter keys not in the schema are silently ignored
|
||||
- Missing required frontmatter keys produce a warning (or error in strict mode)
|
||||
|
||||
Example output for a missing required frontmatter key:
|
||||
|
||||
```
|
||||
⚠ Person schema validation:
|
||||
- Missing required frontmatter key: status
|
||||
```
|
||||
|
||||
### Batch Validation
|
||||
|
||||
```
|
||||
$ bm schema validate Person
|
||||
|
||||
Validating 30 notes against Person schema...
|
||||
|
||||
✓ people/paul-graham.md — all fields present
|
||||
✓ people/rich-hickey.md — all fields present
|
||||
⚠ people/ada-lovelace.md — missing: name
|
||||
⚠ people/alan-kay.md — missing: name, role
|
||||
✓ people/linus-torvalds.md — all fields present
|
||||
...
|
||||
|
||||
Summary: 22/30 valid, 8 warnings, 0 errors
|
||||
```
|
||||
|
||||
## Emerging Schemas
|
||||
|
||||
### The Problem with Traditional Schemas
|
||||
|
||||
Most schema systems require: define schema → create conforming content → fight the schema
|
||||
when reality doesn't match. This is backwards. Knowledge grows organically.
|
||||
|
||||
### The Basic Memory Approach
|
||||
|
||||
```
|
||||
Write notes freely → Patterns emerge → Crystallize into schema → Validate future notes
|
||||
```
|
||||
|
||||
### Schema Inference
|
||||
|
||||
Generate schemas from existing notes by analyzing observation and relation frequency:
|
||||
|
||||
```
|
||||
$ bm schema infer Person
|
||||
|
||||
Analyzing 30 notes with type: Person...
|
||||
|
||||
Observations found:
|
||||
[name] 30/30 100% → name: string
|
||||
[role] 27/30 90% → role?: string
|
||||
[fact] 25/30 83% (generic — no single field)
|
||||
[expertise] 18/30 60% → expertise?(array): string
|
||||
[email] 8/30 27% → email?: string
|
||||
[born] 6/30 20% (below threshold)
|
||||
|
||||
Relations found:
|
||||
works_at 22/30 73% → works_at?: Organization
|
||||
authored 11/30 37% → authored?(array): string
|
||||
|
||||
Suggested schema:
|
||||
name: string, full name
|
||||
role?: string, job title
|
||||
expertise?(array): string, areas of knowledge
|
||||
email?: string, contact email
|
||||
works_at?: Organization, employer
|
||||
|
||||
Save to schema/Person.md? [y/n]
|
||||
```
|
||||
|
||||
Frequency thresholds:
|
||||
- 100% present → required field
|
||||
- 25%+ present → optional field
|
||||
- Below 25% → excluded from suggestion (but noted)
|
||||
|
||||
### Schema Drift Detection
|
||||
|
||||
Track how usage patterns shift over time:
|
||||
|
||||
```
|
||||
$ bm schema diff Person
|
||||
|
||||
Schema drift detected:
|
||||
|
||||
+ expertise: now in 81% of notes (was 12%)
|
||||
- department: dropped to 3% of notes
|
||||
~ works_at: cardinality changed (one → many)
|
||||
|
||||
Update schema? [y/n/review]
|
||||
```
|
||||
|
||||
## LLM Integration (AI Guidance)
|
||||
|
||||
No agent runtime or API key required. The LLM already in the session uses schemas as
|
||||
context for note creation.
|
||||
|
||||
### Flow
|
||||
|
||||
1. User asks LLM to "write a note about Rich Hickey"
|
||||
2. LLM determines `type: Person` is appropriate
|
||||
3. LLM calls `search_notes("type:schema entity:Person")` → finds schema
|
||||
4. LLM reads schema fields: required `name`, optional `role`, `works_at`, `expertise`
|
||||
5. LLM calls `write_note` with observations and relations that satisfy the schema
|
||||
|
||||
The schema acts as a creation template. The LLM knows what a "complete" note looks like
|
||||
without any custom agent infrastructure.
|
||||
|
||||
### MCP Tools
|
||||
|
||||
```python
|
||||
@mcp_tool
|
||||
async def schema_validate(
|
||||
entity_type: str | None = None,
|
||||
identifier: str | None = None,
|
||||
project: str | None = None,
|
||||
) -> ValidationReport:
|
||||
"""Validate notes against their resolved schema.
|
||||
|
||||
Validates a specific note (by identifier) or all notes of a given type.
|
||||
Returns warnings/errors based on the schema's validation mode.
|
||||
"""
|
||||
|
||||
@mcp_tool
|
||||
async def schema_infer(
|
||||
entity_type: str,
|
||||
threshold: float = 0.25,
|
||||
project: str | None = None,
|
||||
) -> SuggestedSchema:
|
||||
"""Analyze existing notes and suggest a schema definition.
|
||||
|
||||
Examines observation categories and relation types across all notes
|
||||
of the given type. Returns frequency analysis and suggested Picoschema.
|
||||
"""
|
||||
```
|
||||
|
||||
## CLI Commands
|
||||
|
||||
```bash
|
||||
# Validate a specific note
|
||||
bm schema validate people/ada-lovelace.md
|
||||
|
||||
# Validate all notes of a type
|
||||
bm schema validate Person
|
||||
|
||||
# Validate everything with a schema
|
||||
bm schema validate
|
||||
|
||||
# Infer schema from existing notes
|
||||
bm schema infer Person
|
||||
|
||||
# Show schema drift from current definition
|
||||
bm schema diff Person
|
||||
|
||||
# List all schema notes
|
||||
bm search "type:schema"
|
||||
```
|
||||
|
||||
## Examples
|
||||
|
||||
### Complete Person Workflow
|
||||
|
||||
**Schema:**
|
||||
```yaml
|
||||
# schema/Person.md
|
||||
---
|
||||
title: Person
|
||||
type: schema
|
||||
entity: Person
|
||||
version: 1
|
||||
schema:
|
||||
name: string, full name
|
||||
role?: string, job title or position
|
||||
works_at?: Organization, employer
|
||||
expertise?(array): string, areas of knowledge
|
||||
email?: string, contact email
|
||||
settings:
|
||||
validation: warn
|
||||
---
|
||||
|
||||
# Person
|
||||
|
||||
A human individual in the knowledge graph.
|
||||
```
|
||||
|
||||
**Valid note:**
|
||||
```yaml
|
||||
# people/paul-graham.md
|
||||
---
|
||||
title: Paul Graham
|
||||
type: Person
|
||||
tags: [startups, essays, lisp]
|
||||
---
|
||||
|
||||
# Paul Graham
|
||||
|
||||
## Observations
|
||||
- [name] Paul Graham
|
||||
- [role] Essayist and investor
|
||||
- [expertise] Startups
|
||||
- [expertise] Lisp
|
||||
- [expertise] Essay writing
|
||||
- [fact] Created Viaweb, the first web app
|
||||
|
||||
## Relations
|
||||
- works_at [[Y Combinator]]
|
||||
- authored [[Hackers and Painters]]
|
||||
```
|
||||
|
||||
**Note with warnings:**
|
||||
```yaml
|
||||
# people/ada-lovelace.md
|
||||
---
|
||||
title: Ada Lovelace
|
||||
type: Person
|
||||
---
|
||||
|
||||
# Ada Lovelace
|
||||
|
||||
## Observations
|
||||
- [fact] Wrote the first computer program
|
||||
- [born] 1815
|
||||
|
||||
## Relations
|
||||
- collaborated_with [[Charles Babbage]]
|
||||
```
|
||||
|
||||
Validation: warns about missing required `[name]` observation. Everything else is optional
|
||||
or unmatched (which is fine).
|
||||
|
||||
## Future Considerations (Deferred)
|
||||
|
||||
These are interesting but out of scope for the initial implementation:
|
||||
|
||||
- **Multiple schema inheritance** — `schema: [Person, Author]`
|
||||
- **Hook integration** — Pre-write validation via the hooks system
|
||||
- **OWL/RDF export** — `bm schema export --format owl`
|
||||
- **SPARQL queries** — Schema-aware graph queries
|
||||
- **Built-in templates** — `bm schema use gtd`, `bm schema use zettelkasten`
|
||||
- **Schema versioning/migration** — Tracking breaking changes across versions
|
||||
@@ -2,7 +2,6 @@
|
||||
|
||||
# Install dependencies
|
||||
install:
|
||||
uv pip install -e ".[dev]"
|
||||
uv sync
|
||||
@echo ""
|
||||
@echo "💡 Remember to activate the virtual environment by running: source .venv/bin/activate"
|
||||
@@ -43,9 +42,9 @@ test-unit-sqlite:
|
||||
test-unit-postgres:
|
||||
BASIC_MEMORY_ENV=test BASIC_MEMORY_TEST_POSTGRES=1 uv run pytest -p pytest_mock -v --no-cov tests
|
||||
|
||||
# Run integration tests against SQLite
|
||||
# Run integration tests against SQLite (excludes semantic benchmarks — use just test-semantic)
|
||||
test-int-sqlite:
|
||||
uv run pytest -p pytest_mock -v --no-cov test-int
|
||||
BASIC_MEMORY_ENV=test uv run pytest -p pytest_mock -v --no-cov -m "not semantic" test-int
|
||||
|
||||
# Run integration tests against Postgres
|
||||
# Note: Uses timeout due to FastMCP Client + asyncpg cleanup hang (tests pass, process hangs on exit)
|
||||
@@ -56,12 +55,28 @@ test-int-postgres:
|
||||
# Use gtimeout (macOS/Homebrew) or timeout (Linux)
|
||||
TIMEOUT_CMD=$(command -v gtimeout || command -v timeout || echo "")
|
||||
if [[ -n "$TIMEOUT_CMD" ]]; then
|
||||
$TIMEOUT_CMD --signal=KILL 600 bash -c 'BASIC_MEMORY_TEST_POSTGRES=1 uv run pytest -p pytest_mock -v --no-cov test-int' || test $? -eq 137
|
||||
$TIMEOUT_CMD --signal=KILL 600 bash -c 'BASIC_MEMORY_ENV=test BASIC_MEMORY_TEST_POSTGRES=1 uv run pytest -p pytest_mock -v --no-cov -m "not semantic" test-int' || test $? -eq 137
|
||||
else
|
||||
echo "⚠️ No timeout command found, running without timeout..."
|
||||
BASIC_MEMORY_TEST_POSTGRES=1 uv run pytest -p pytest_mock -v --no-cov test-int
|
||||
BASIC_MEMORY_ENV=test BASIC_MEMORY_TEST_POSTGRES=1 uv run pytest -p pytest_mock -v --no-cov -m "not semantic" test-int
|
||||
fi
|
||||
|
||||
# Run tests impacted by recent changes (requires pytest-testmon)
|
||||
# Pass paths or node ids after `just testmon` to limit the candidate set further.
|
||||
testmon *args:
|
||||
BASIC_MEMORY_ENV=test uv run pytest -p pytest_mock -v --no-cov --testmon {{args}}
|
||||
|
||||
# Run MCP smoke test (fast end-to-end loop)
|
||||
test-smoke:
|
||||
BASIC_MEMORY_ENV=test uv run pytest -p pytest_mock -v --no-cov -m smoke test-int/mcp/test_smoke_integration.py
|
||||
|
||||
# Fast local loop: lint, format, typecheck, impacted tests via pytest-testmon
|
||||
fast-check:
|
||||
just fix
|
||||
just format
|
||||
just typecheck
|
||||
just testmon
|
||||
|
||||
# Reset Postgres test database (drops and recreates schema)
|
||||
# Useful when Alembic migration state gets out of sync during development
|
||||
# Uses credentials from docker-compose-postgres.yml
|
||||
@@ -83,18 +98,43 @@ postgres-migrate:
|
||||
# These tests verify Windows-specific database optimizations (locking mode, NullPool)
|
||||
# Will be skipped automatically on non-Windows platforms
|
||||
test-windows:
|
||||
uv run pytest -p pytest_mock -v --no-cov -m windows tests test-int
|
||||
BASIC_MEMORY_ENV=test uv run pytest -p pytest_mock -v --no-cov -m windows tests test-int
|
||||
|
||||
# Run benchmark tests only (performance testing)
|
||||
# These are slow tests that measure sync performance with various file counts
|
||||
# Excluded from default test runs to keep CI fast
|
||||
test-benchmark:
|
||||
uv run pytest -p pytest_mock -v --no-cov -m benchmark tests test-int
|
||||
BASIC_MEMORY_ENV=test uv run pytest -p pytest_mock -v --no-cov -m benchmark tests test-int
|
||||
|
||||
# Run semantic search quality benchmarks (all combos)
|
||||
test-semantic:
|
||||
BASIC_MEMORY_ENV=test uv run pytest -p pytest_mock -v --no-cov -m semantic test-int/semantic/
|
||||
|
||||
# Run semantic benchmarks with JSON artifact output, then show report
|
||||
test-semantic-report:
|
||||
BASIC_MEMORY_ENV=test BASIC_MEMORY_BENCHMARK_OUTPUT=.benchmarks/semantic-quality.jsonl uv run pytest -p pytest_mock -v -s --no-cov -m semantic test-int/semantic/
|
||||
uv run python test-int/semantic/report.py .benchmarks/semantic-quality.jsonl
|
||||
|
||||
# Run semantic benchmarks (Postgres combos only)
|
||||
test-semantic-postgres:
|
||||
BASIC_MEMORY_ENV=test uv run pytest -p pytest_mock -v --no-cov -m semantic -k postgres test-int/semantic/
|
||||
|
||||
# View semantic benchmark results (rich formatted table)
|
||||
# Usage: just semantic-report [--filter-combo sqlite] [--filter-suite paraphrase] [--sort-by avg_latency_ms]
|
||||
semantic-report *args:
|
||||
uv run python test-int/semantic/report.py .benchmarks/semantic-quality.jsonl {{args}}
|
||||
|
||||
# Compare two search benchmark JSONL outputs
|
||||
# Usage:
|
||||
# just benchmark-compare .benchmarks/search-baseline.jsonl .benchmarks/search-candidate.jsonl
|
||||
# just benchmark-compare .benchmarks/search-baseline.jsonl .benchmarks/search-candidate.jsonl --format markdown --show-missing
|
||||
benchmark-compare baseline candidate *args:
|
||||
uv run python test-int/compare_search_benchmarks.py "{{baseline}}" "{{candidate}}" --format table {{args}}
|
||||
|
||||
# Run all tests including Windows, Postgres, and Benchmarks (for CI/comprehensive testing)
|
||||
# Use this before releasing to ensure everything works across all backends and platforms
|
||||
test-all:
|
||||
uv run pytest -p pytest_mock -v --no-cov tests test-int
|
||||
BASIC_MEMORY_ENV=test uv run pytest -p pytest_mock -v --no-cov tests test-int
|
||||
|
||||
# Generate HTML coverage report
|
||||
coverage:
|
||||
@@ -130,10 +170,18 @@ lint: fix
|
||||
fix:
|
||||
uv run ruff check --fix --unsafe-fixes src tests test-int
|
||||
|
||||
# Type check code
|
||||
# Type check code (ty)
|
||||
typecheck:
|
||||
uv run ty check src tests test-int
|
||||
|
||||
# Type check code (pyright)
|
||||
typecheck-pyright:
|
||||
uv run pyright
|
||||
|
||||
# Type check code (ty)
|
||||
typecheck-ty:
|
||||
just typecheck
|
||||
|
||||
# Clean build artifacts and cache files
|
||||
clean:
|
||||
find . -type f -name '*.pyc' -delete
|
||||
@@ -149,6 +197,63 @@ format:
|
||||
run-inspector:
|
||||
npx @modelcontextprotocol/inspector
|
||||
|
||||
# Run doctor checks in an isolated temp home/config
|
||||
doctor:
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
TMP_HOME=$(mktemp -d)
|
||||
TMP_CONFIG=$(mktemp -d)
|
||||
HOME="$TMP_HOME" \
|
||||
BASIC_MEMORY_ENV=test \
|
||||
BASIC_MEMORY_HOME="$TMP_HOME/basic-memory" \
|
||||
BASIC_MEMORY_CONFIG_DIR="$TMP_CONFIG" \
|
||||
./.venv/bin/python -m basic_memory.cli.main doctor --local
|
||||
|
||||
# Run an isolated Logfire smoke workflow for local trace inspection
|
||||
telemetry-smoke:
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
TMP_HOME=$(mktemp -d)
|
||||
TMP_CONFIG=$(mktemp -d)
|
||||
TMP_PROJECT=$(mktemp -d)
|
||||
export HOME="$TMP_HOME"
|
||||
export BASIC_MEMORY_ENV="${BASIC_MEMORY_ENV:-dev}"
|
||||
export BASIC_MEMORY_HOME="$TMP_PROJECT/home-root"
|
||||
export BASIC_MEMORY_CONFIG_DIR="$TMP_CONFIG"
|
||||
export BASIC_MEMORY_NO_PROMOS=1
|
||||
export BASIC_MEMORY_LOG_LEVEL="${BASIC_MEMORY_LOG_LEVEL:-INFO}"
|
||||
export BASIC_MEMORY_SEMANTIC_SEARCH_ENABLED="${BASIC_MEMORY_SEMANTIC_SEARCH_ENABLED:-false}"
|
||||
export BASIC_MEMORY_LOGFIRE_ENABLED="${BASIC_MEMORY_LOGFIRE_ENABLED:-true}"
|
||||
export BASIC_MEMORY_LOGFIRE_ENVIRONMENT="${BASIC_MEMORY_LOGFIRE_ENVIRONMENT:-telemetry-smoke}"
|
||||
if [[ -z "${BASIC_MEMORY_LOGFIRE_SEND_TO_LOGFIRE:-}" ]]; then
|
||||
if [[ -n "${LOGFIRE_TOKEN:-}" ]]; then
|
||||
export BASIC_MEMORY_LOGFIRE_SEND_TO_LOGFIRE=true
|
||||
else
|
||||
export BASIC_MEMORY_LOGFIRE_SEND_TO_LOGFIRE=false
|
||||
fi
|
||||
fi
|
||||
mkdir -p "$BASIC_MEMORY_HOME"
|
||||
echo "Telemetry smoke setup:"
|
||||
echo " logfire_enabled=$BASIC_MEMORY_LOGFIRE_ENABLED"
|
||||
echo " send_to_logfire=$BASIC_MEMORY_LOGFIRE_SEND_TO_LOGFIRE"
|
||||
echo " log_level=$BASIC_MEMORY_LOG_LEVEL"
|
||||
echo " semantic_search_enabled=$BASIC_MEMORY_SEMANTIC_SEARCH_ENABLED"
|
||||
echo " logfire_environment=$BASIC_MEMORY_LOGFIRE_ENVIRONMENT"
|
||||
echo " project_path=$TMP_PROJECT"
|
||||
./.venv/bin/python -m basic_memory.cli.main project add telemetry-smoke "$TMP_PROJECT" --default --local
|
||||
./.venv/bin/python -m basic_memory.cli.main tool write-note --title "Telemetry Smoke" --folder notes --content "hello from smoke" --project telemetry-smoke --local
|
||||
./.venv/bin/python -m basic_memory.cli.main tool read-note notes/telemetry-smoke --project telemetry-smoke --local
|
||||
./.venv/bin/python -m basic_memory.cli.main tool edit-note notes/telemetry-smoke --operation append --content $'\n\nsmoke edit line' --project telemetry-smoke --local
|
||||
./.venv/bin/python -m basic_memory.cli.main tool build-context notes/telemetry-smoke --project telemetry-smoke --local --page-size 5 --max-related 5
|
||||
./.venv/bin/python -m basic_memory.cli.main tool search-notes telemetry --project telemetry-smoke --local
|
||||
./.venv/bin/python -m basic_memory.cli.main doctor --local
|
||||
echo ""
|
||||
echo "Telemetry smoke complete."
|
||||
echo "Search Logfire for:"
|
||||
echo " service_name: basic-memory-cli"
|
||||
echo " environment: $BASIC_MEMORY_LOGFIRE_ENVIRONMENT"
|
||||
echo " span names: mcp.tool.write_note, mcp.tool.read_note, mcp.tool.edit_note, mcp.tool.build_context, mcp.tool.search_notes, sync.project.run"
|
||||
|
||||
|
||||
# Update all dependencies to latest versions
|
||||
update-deps:
|
||||
@@ -157,6 +262,9 @@ update-deps:
|
||||
# Run all code quality checks and tests
|
||||
check: lint format typecheck test
|
||||
|
||||
# Run all code quality checks and all test suites, including semantic benchmarks
|
||||
check-all: lint format typecheck test test-semantic
|
||||
|
||||
# Generate Alembic migration with descriptive message
|
||||
migration message:
|
||||
cd src/basic_memory/alembic && alembic revision --autogenerate -m "{{message}}"
|
||||
|
||||
+17
-1
@@ -54,6 +54,22 @@ Or for a one-time sync:
|
||||
basic-memory sync
|
||||
```
|
||||
|
||||
### 4. Updating Basic Memory
|
||||
|
||||
Basic Memory supports automatic updates by default for `uv tool` and Homebrew installs.
|
||||
|
||||
For manual checks and upgrades:
|
||||
|
||||
```bash
|
||||
# Check now and install if supported
|
||||
bm update
|
||||
|
||||
# Check only, do not install
|
||||
bm update --check
|
||||
```
|
||||
|
||||
To disable automatic updates, set `"auto_update": false` in `~/.basic-memory/config.json`.
|
||||
|
||||
## Configuration Options
|
||||
|
||||
### Custom Directory
|
||||
@@ -125,4 +141,4 @@ If you encounter issues:
|
||||
cat ~/.basic-memory/basic-memory.log
|
||||
```
|
||||
|
||||
For more detailed information, refer to the [full documentation](https://memory.basicmachines.co/).
|
||||
For more detailed information, refer to the [full documentation](https://docs.basicmemory.com/).
|
||||
|
||||
+23
-4
@@ -25,15 +25,16 @@ dependencies = [
|
||||
"unidecode>=1.3.8",
|
||||
"dateparser>=1.2.0",
|
||||
"watchfiles>=1.0.4",
|
||||
"fastapi[standard]>=0.115.8",
|
||||
"fastapi[standard]>=0.136.1",
|
||||
"alembic>=1.14.1",
|
||||
"pillow>=11.1.0",
|
||||
"pybars3>=0.9.7",
|
||||
"fastmcp==2.12.3", # Pinned - 2.14.x breaks MCP tools visibility (issue #463)
|
||||
# Keep FastMCP pinned until each minor upgrade passes the MCP transport matrix.
|
||||
"fastmcp==3.3.1",
|
||||
"pyjwt>=2.10.1",
|
||||
"python-dotenv>=1.1.0",
|
||||
"pytest-aio>=1.9.0",
|
||||
"aiofiles>=24.1.0", # Optional observability (disabled by default via config)
|
||||
"aiofiles>=24.1.0",
|
||||
"asyncpg>=0.30.0",
|
||||
"nest-asyncio>=1.6.0", # For Alembic migrations with Postgres
|
||||
"pytest-asyncio>=1.2.0",
|
||||
@@ -44,9 +45,13 @@ dependencies = [
|
||||
"sniffio>=1.3.1",
|
||||
"anyio>=4.10.0",
|
||||
"httpx>=0.28.0",
|
||||
"fastembed>=0.7.4",
|
||||
"sqlite-vec>=0.1.6",
|
||||
"openai>=1.100.2",
|
||||
"logfire>=4.19.0",
|
||||
"psutil>=5.9.0",
|
||||
]
|
||||
|
||||
|
||||
[project.urls]
|
||||
Homepage = "https://github.com/basicmachines-co/basic-memory"
|
||||
Repository = "https://github.com/basicmachines-co/basic-memory"
|
||||
@@ -66,11 +71,19 @@ addopts = "--cov=basic_memory --cov-report term-missing"
|
||||
testpaths = ["tests", "test-int"]
|
||||
asyncio_mode = "strict"
|
||||
asyncio_default_fixture_loop_scope = "function"
|
||||
filterwarnings = [
|
||||
"ignore:The @wait_container_is_ready decorator is deprecated.*:DeprecationWarning:testcontainers\\.core\\.waiting_utils",
|
||||
"ignore:The default datetime adapter is deprecated as of Python 3\\.12.*:DeprecationWarning:aiosqlite\\.core",
|
||||
"ignore:codecs\\.open\\(\\) is deprecated\\. Use open\\(\\) instead\\.:DeprecationWarning:frontmatter",
|
||||
"ignore:Parsing dates involving a day of month without a year specified is ambiguous.*:DeprecationWarning:dateparser\\.utils\\.strptime",
|
||||
]
|
||||
markers = [
|
||||
"benchmark: Performance benchmark tests (deselect with '-m \"not benchmark\"')",
|
||||
"slow: Slow-running tests (deselect with '-m \"not slow\"')",
|
||||
"postgres: Tests that run against Postgres backend (deselect with '-m \"not postgres\"')",
|
||||
"windows: Windows-specific tests (deselect with '-m \"not windows\"')",
|
||||
"smoke: Fast end-to-end smoke tests for MCP flows",
|
||||
"semantic: Tests requiring semantic dependencies (fastembed, sqlite-vec, openai)",
|
||||
]
|
||||
|
||||
[tool.ruff]
|
||||
@@ -79,6 +92,7 @@ target-version = "py312"
|
||||
|
||||
[dependency-groups]
|
||||
dev = [
|
||||
"logfire>=4.19.0",
|
||||
"gevent>=24.11.1",
|
||||
"icecream>=2.1.3",
|
||||
"pytest>=8.3.4",
|
||||
@@ -91,6 +105,10 @@ dev = [
|
||||
"testcontainers[postgres]>=4.0.0",
|
||||
"psycopg>=3.2.0",
|
||||
"pyright>=1.1.408",
|
||||
"pytest-testmon>=2.2.0",
|
||||
"ty>=0.0.18",
|
||||
"cst-lsp>=0.1.3",
|
||||
"libcst>=1.8.6",
|
||||
]
|
||||
|
||||
[tool.hatch.version]
|
||||
@@ -109,6 +127,7 @@ ignore = ["test/"]
|
||||
defineConstant = { DEBUG = true }
|
||||
reportMissingImports = "error"
|
||||
reportMissingTypeStubs = false
|
||||
reportUnusedImport = "none"
|
||||
pythonVersion = "3.12"
|
||||
|
||||
|
||||
|
||||
+2
-2
@@ -6,12 +6,12 @@
|
||||
"url": "https://github.com/basicmachines-co/basic-memory.git",
|
||||
"source": "github"
|
||||
},
|
||||
"version": "0.17.9",
|
||||
"version": "0.21.1",
|
||||
"packages": [
|
||||
{
|
||||
"registryType": "pypi",
|
||||
"identifier": "basic-memory",
|
||||
"version": "0.17.9",
|
||||
"version": "0.21.1",
|
||||
"runtimeHint": "uvx",
|
||||
"runtimeArguments": [
|
||||
{"type": "positional", "value": "basic-memory"},
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
{
|
||||
"version": 1,
|
||||
"skills": {
|
||||
"instrumentation": {
|
||||
"source": "pydantic/skills",
|
||||
"sourceType": "github",
|
||||
"computedHash": "0727bffc6a92fdeaf675ae5796ae25341e193327e8c95cd06b188dc4a0a4e62e"
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,7 +1,7 @@
|
||||
"""basic-memory - Local-first knowledge management combining Zettelkasten with knowledge graphs"""
|
||||
|
||||
# Package version - updated by release automation
|
||||
__version__ = "0.17.9"
|
||||
__version__ = "0.21.1"
|
||||
|
||||
# API version for FastAPI - independent of package version
|
||||
__api_version__ = "v0"
|
||||
|
||||
@@ -66,7 +66,7 @@ target_metadata = Base.metadata
|
||||
|
||||
|
||||
# Add this function to tell Alembic what to include/exclude
|
||||
def include_object(object, name, type_, reflected, compare_to):
|
||||
def include_object(obj, name, type_, reflected, compare_to):
|
||||
# Ignore SQLite FTS tables
|
||||
if type_ == "table" and name.startswith("search_index"):
|
||||
return False
|
||||
@@ -118,6 +118,54 @@ async def run_async_migrations(connectable):
|
||||
await connectable.dispose()
|
||||
|
||||
|
||||
def _run_async_migrations_with_asyncio_run(connectable) -> None:
|
||||
"""Run async migrations with asyncio.run while closing failed coroutines.
|
||||
|
||||
Trigger: asyncio.run() may reject execution when another event loop is already active.
|
||||
Why: Python raises before awaiting the coroutine, which otherwise leaks a
|
||||
RuntimeWarning about an un-awaited coroutine.
|
||||
Outcome: close the pending coroutine before bubbling the RuntimeError to the
|
||||
fallback path.
|
||||
"""
|
||||
migration_coro = run_async_migrations(connectable)
|
||||
try:
|
||||
asyncio.run(migration_coro)
|
||||
except RuntimeError:
|
||||
migration_coro.close()
|
||||
raise
|
||||
|
||||
|
||||
def _run_async_migrations_in_thread(connectable) -> None:
|
||||
"""Run async migrations in a dedicated thread with its own event loop."""
|
||||
import concurrent.futures
|
||||
|
||||
def run_in_thread():
|
||||
"""Run async migrations in a new event loop in a separate thread."""
|
||||
new_loop = asyncio.new_event_loop()
|
||||
asyncio.set_event_loop(new_loop)
|
||||
try:
|
||||
new_loop.run_until_complete(run_async_migrations(connectable))
|
||||
finally:
|
||||
new_loop.close()
|
||||
|
||||
with concurrent.futures.ThreadPoolExecutor() as executor:
|
||||
future = executor.submit(run_in_thread)
|
||||
future.result() # Wait for completion and re-raise any exceptions
|
||||
|
||||
|
||||
def _run_async_engine_migrations(connectable) -> None:
|
||||
"""Run async-engine migrations with a running-loop fallback."""
|
||||
try:
|
||||
_run_async_migrations_with_asyncio_run(connectable)
|
||||
except RuntimeError as e:
|
||||
if "cannot be called from a running event loop" in str(e):
|
||||
# We're in a running event loop (likely uvloop or Python 3.14+ tests).
|
||||
# Switch to a dedicated thread so Alembic can finish without nesting loops.
|
||||
_run_async_migrations_in_thread(connectable)
|
||||
else:
|
||||
raise
|
||||
|
||||
|
||||
def run_migrations_online() -> None:
|
||||
"""Run migrations in 'online' mode.
|
||||
|
||||
@@ -148,30 +196,10 @@ def run_migrations_online() -> None:
|
||||
|
||||
# Handle async engines (PostgreSQL with asyncpg)
|
||||
if isinstance(connectable, AsyncEngine):
|
||||
# Try to run async migrations
|
||||
# nest_asyncio allows asyncio.run() from within event loops, but doesn't work with uvloop
|
||||
try:
|
||||
asyncio.run(run_async_migrations(connectable))
|
||||
except RuntimeError as e:
|
||||
if "cannot be called from a running event loop" in str(e):
|
||||
# We're in a running event loop (likely uvloop) - need to use a different approach
|
||||
# Create a new thread to run the async migrations
|
||||
import concurrent.futures
|
||||
|
||||
def run_in_thread():
|
||||
"""Run async migrations in a new event loop in a separate thread."""
|
||||
new_loop = asyncio.new_event_loop()
|
||||
asyncio.set_event_loop(new_loop)
|
||||
try:
|
||||
new_loop.run_until_complete(run_async_migrations(connectable))
|
||||
finally:
|
||||
new_loop.close()
|
||||
|
||||
with concurrent.futures.ThreadPoolExecutor() as executor:
|
||||
future = executor.submit(run_in_thread)
|
||||
future.result() # Wait for completion and re-raise any exceptions
|
||||
else:
|
||||
raise
|
||||
# Trigger: async engines need Alembic work to cross the sync/async boundary.
|
||||
# Why: most callers can use asyncio.run(), but running-loop contexts need a thread fallback.
|
||||
# Outcome: migrations complete without leaking un-awaited coroutines.
|
||||
_run_async_engine_migrations(connectable)
|
||||
else:
|
||||
# Handle sync engines (SQLite) or sync connections
|
||||
if hasattr(connectable, "connect"):
|
||||
|
||||
@@ -0,0 +1,154 @@
|
||||
"""Add structured metadata indexes for entity frontmatter
|
||||
|
||||
Revision ID: d7e8f9a0b1c2
|
||||
Revises: g9a0b3c4d5e6
|
||||
Create Date: 2026-01-31 12:00:00.000000
|
||||
|
||||
"""
|
||||
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy import text
|
||||
|
||||
|
||||
def column_exists(connection, table: str, column: str) -> bool:
|
||||
"""Check if a column exists in a table (idempotent migration support)."""
|
||||
if connection.dialect.name == "postgresql":
|
||||
result = connection.execute(
|
||||
text(
|
||||
"SELECT 1 FROM information_schema.columns "
|
||||
"WHERE table_name = :table AND column_name = :column"
|
||||
),
|
||||
{"table": table, "column": column},
|
||||
)
|
||||
return result.fetchone() is not None
|
||||
# SQLite
|
||||
result = connection.execute(text(f"PRAGMA table_info({table})"))
|
||||
columns = [row[1] for row in result]
|
||||
return column in columns
|
||||
|
||||
|
||||
def index_exists(connection, index_name: str) -> bool:
|
||||
"""Check if an index exists (idempotent migration support)."""
|
||||
if connection.dialect.name == "postgresql":
|
||||
result = connection.execute(
|
||||
text("SELECT 1 FROM pg_indexes WHERE indexname = :index_name"),
|
||||
{"index_name": index_name},
|
||||
)
|
||||
return result.fetchone() is not None
|
||||
# SQLite
|
||||
result = connection.execute(
|
||||
text("SELECT 1 FROM sqlite_master WHERE type='index' AND name = :index_name"),
|
||||
{"index_name": index_name},
|
||||
)
|
||||
return result.fetchone() is not None
|
||||
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = "d7e8f9a0b1c2"
|
||||
down_revision: Union[str, None] = "6830751f5fb6"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Add JSONB/GiN indexes for Postgres and generated columns for SQLite."""
|
||||
connection = op.get_bind()
|
||||
dialect = connection.dialect.name
|
||||
|
||||
if dialect == "postgresql":
|
||||
# Ensure JSONB for efficient indexing
|
||||
result = connection.execute(
|
||||
text(
|
||||
"SELECT data_type FROM information_schema.columns "
|
||||
"WHERE table_name = 'entity' AND column_name = 'entity_metadata'"
|
||||
)
|
||||
).fetchone()
|
||||
if result and result[0] != "jsonb":
|
||||
op.execute(
|
||||
"ALTER TABLE entity ALTER COLUMN entity_metadata "
|
||||
"TYPE jsonb USING entity_metadata::jsonb"
|
||||
)
|
||||
|
||||
# General JSONB GIN index
|
||||
op.execute(
|
||||
"CREATE INDEX IF NOT EXISTS idx_entity_metadata_gin "
|
||||
"ON entity USING GIN (entity_metadata jsonb_path_ops)"
|
||||
)
|
||||
|
||||
# Common field indexes
|
||||
op.execute(
|
||||
"CREATE INDEX IF NOT EXISTS idx_entity_tags_json "
|
||||
"ON entity USING GIN ((entity_metadata -> 'tags'))"
|
||||
)
|
||||
op.execute(
|
||||
"CREATE INDEX IF NOT EXISTS idx_entity_frontmatter_type "
|
||||
"ON entity ((entity_metadata ->> 'type'))"
|
||||
)
|
||||
op.execute(
|
||||
"CREATE INDEX IF NOT EXISTS idx_entity_frontmatter_status "
|
||||
"ON entity ((entity_metadata ->> 'status'))"
|
||||
)
|
||||
return
|
||||
|
||||
# SQLite: add generated columns for common frontmatter fields
|
||||
# Constraint: SQLite ALTER TABLE ADD COLUMN only supports VIRTUAL generated columns,
|
||||
# not STORED. json_extract is deterministic so VIRTUAL columns can still be indexed.
|
||||
if not column_exists(connection, "entity", "tags_json"):
|
||||
op.add_column(
|
||||
"entity",
|
||||
sa.Column(
|
||||
"tags_json",
|
||||
sa.Text(),
|
||||
sa.Computed("json_extract(entity_metadata, '$.tags')", persisted=False),
|
||||
),
|
||||
)
|
||||
if not column_exists(connection, "entity", "frontmatter_status"):
|
||||
op.add_column(
|
||||
"entity",
|
||||
sa.Column(
|
||||
"frontmatter_status",
|
||||
sa.Text(),
|
||||
sa.Computed("json_extract(entity_metadata, '$.status')", persisted=False),
|
||||
),
|
||||
)
|
||||
if not column_exists(connection, "entity", "frontmatter_type"):
|
||||
op.add_column(
|
||||
"entity",
|
||||
sa.Column(
|
||||
"frontmatter_type",
|
||||
sa.Text(),
|
||||
sa.Computed("json_extract(entity_metadata, '$.type')", persisted=False),
|
||||
),
|
||||
)
|
||||
|
||||
# Index generated columns
|
||||
if not index_exists(connection, "idx_entity_tags_json"):
|
||||
op.create_index("idx_entity_tags_json", "entity", ["tags_json"])
|
||||
if not index_exists(connection, "idx_entity_frontmatter_status"):
|
||||
op.create_index("idx_entity_frontmatter_status", "entity", ["frontmatter_status"])
|
||||
if not index_exists(connection, "idx_entity_frontmatter_type"):
|
||||
op.create_index("idx_entity_frontmatter_type", "entity", ["frontmatter_type"])
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Best-effort downgrade (drop indexes, revert JSONB on Postgres)."""
|
||||
connection = op.get_bind()
|
||||
dialect = connection.dialect.name
|
||||
|
||||
if dialect == "postgresql":
|
||||
op.execute("DROP INDEX IF EXISTS idx_entity_frontmatter_status")
|
||||
op.execute("DROP INDEX IF EXISTS idx_entity_frontmatter_type")
|
||||
op.execute("DROP INDEX IF EXISTS idx_entity_tags_json")
|
||||
op.execute("DROP INDEX IF EXISTS idx_entity_metadata_gin")
|
||||
op.execute(
|
||||
"ALTER TABLE entity ALTER COLUMN entity_metadata TYPE json USING entity_metadata::json"
|
||||
)
|
||||
return
|
||||
|
||||
# SQLite: drop indexes (dropping generated columns requires table rebuild)
|
||||
op.execute("DROP INDEX IF EXISTS idx_entity_frontmatter_status")
|
||||
op.execute("DROP INDEX IF EXISTS idx_entity_frontmatter_type")
|
||||
op.execute("DROP INDEX IF EXISTS idx_entity_tags_json")
|
||||
@@ -0,0 +1,68 @@
|
||||
"""Add Postgres semantic vector search tables (pgvector-aware, optional)
|
||||
|
||||
Revision ID: h1b2c3d4e5f6
|
||||
Revises: d7e8f9a0b1c2
|
||||
Create Date: 2026-02-07 00:00:00.000000
|
||||
|
||||
"""
|
||||
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = "h1b2c3d4e5f6"
|
||||
down_revision: Union[str, None] = "d7e8f9a0b1c2"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Create Postgres vector chunk metadata table.
|
||||
|
||||
Trigger: database backend is PostgreSQL.
|
||||
Why: search_vector_chunks stores text metadata with no vector-dimension
|
||||
dependency, so it's safe in a migration. search_vector_embeddings (which
|
||||
requires pgvector and a provider-specific dimension) is created at runtime
|
||||
by PostgresSearchRepository._ensure_vector_tables(), mirroring the SQLite
|
||||
pattern where vector tables are created dynamically.
|
||||
Outcome: creates the dimension-independent chunks table. The embeddings
|
||||
table + HNSW index are deferred to runtime.
|
||||
"""
|
||||
connection = op.get_bind()
|
||||
if connection.dialect.name != "postgresql":
|
||||
return
|
||||
|
||||
op.execute(
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS search_vector_chunks (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
entity_id INTEGER NOT NULL,
|
||||
project_id INTEGER NOT NULL,
|
||||
chunk_key TEXT NOT NULL,
|
||||
chunk_text TEXT NOT NULL,
|
||||
source_hash TEXT NOT NULL,
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
UNIQUE (project_id, entity_id, chunk_key)
|
||||
)
|
||||
"""
|
||||
)
|
||||
op.execute(
|
||||
"""
|
||||
CREATE INDEX IF NOT EXISTS idx_search_vector_chunks_project_entity
|
||||
ON search_vector_chunks (project_id, entity_id)
|
||||
"""
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Remove Postgres vector chunk/embedding tables.
|
||||
|
||||
Does not drop pgvector extension because other schema objects may depend on it.
|
||||
"""
|
||||
connection = op.get_bind()
|
||||
if connection.dialect.name != "postgresql":
|
||||
return
|
||||
|
||||
op.execute("DROP TABLE IF EXISTS search_vector_embeddings")
|
||||
op.execute("DROP TABLE IF EXISTS search_vector_chunks")
|
||||
@@ -0,0 +1,29 @@
|
||||
"""Trigger automatic semantic embedding backfill during migration.
|
||||
|
||||
Revision ID: i2c3d4e5f6g7
|
||||
Revises: h1b2c3d4e5f6
|
||||
Create Date: 2026-02-19 00:00:00.000000
|
||||
|
||||
"""
|
||||
|
||||
from typing import Sequence, Union
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = "i2c3d4e5f6g7"
|
||||
down_revision: Union[str, None] = "h1b2c3d4e5f6"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""No schema change.
|
||||
|
||||
Trigger: this revision is newly applied.
|
||||
Why: db.run_migrations() detects this revision transition and runs the existing
|
||||
sync_entity_vectors() pipeline to backfill semantic embeddings automatically.
|
||||
Outcome: users no longer need to run `bm reindex --embeddings` after upgrading.
|
||||
"""
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""No-op downgrade."""
|
||||
@@ -0,0 +1,164 @@
|
||||
"""Rename entity_type column to note_type
|
||||
|
||||
Revision ID: j3d4e5f6g7h8
|
||||
Revises: i2c3d4e5f6g7
|
||||
Create Date: 2026-02-22 12:00:00.000000
|
||||
|
||||
"""
|
||||
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
from sqlalchemy import text
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = "j3d4e5f6g7h8"
|
||||
down_revision: Union[str, None] = "i2c3d4e5f6g7"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def table_exists(connection, table_name: str) -> bool:
|
||||
"""Check if a table exists (idempotent migration support)."""
|
||||
if connection.dialect.name == "postgresql":
|
||||
result = connection.execute(
|
||||
text("SELECT 1 FROM information_schema.tables WHERE table_name = :table_name"),
|
||||
{"table_name": table_name},
|
||||
)
|
||||
return result.fetchone() is not None
|
||||
# SQLite
|
||||
result = connection.execute(
|
||||
text("SELECT 1 FROM sqlite_master WHERE type='table' AND name = :table_name"),
|
||||
{"table_name": table_name},
|
||||
)
|
||||
return result.fetchone() is not None
|
||||
|
||||
|
||||
def index_exists(connection, index_name: str) -> bool:
|
||||
"""Check if an index exists (idempotent migration support)."""
|
||||
if connection.dialect.name == "postgresql":
|
||||
result = connection.execute(
|
||||
text("SELECT 1 FROM pg_indexes WHERE indexname = :index_name"),
|
||||
{"index_name": index_name},
|
||||
)
|
||||
return result.fetchone() is not None
|
||||
# SQLite
|
||||
result = connection.execute(
|
||||
text("SELECT 1 FROM sqlite_master WHERE type='index' AND name = :index_name"),
|
||||
{"index_name": index_name},
|
||||
)
|
||||
return result.fetchone() is not None
|
||||
|
||||
|
||||
def column_exists(connection, table: str, column: str) -> bool:
|
||||
"""Check if a column exists in a table (idempotent migration support)."""
|
||||
if connection.dialect.name == "postgresql":
|
||||
result = connection.execute(
|
||||
text(
|
||||
"SELECT 1 FROM information_schema.columns "
|
||||
"WHERE table_name = :table AND column_name = :column"
|
||||
),
|
||||
{"table": table, "column": column},
|
||||
)
|
||||
return result.fetchone() is not None
|
||||
# SQLite
|
||||
result = connection.execute(text(f"PRAGMA table_info({table})"))
|
||||
columns = [row[1] for row in result]
|
||||
return column in columns
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Rename entity_type → note_type on the entity table."""
|
||||
connection = op.get_bind()
|
||||
dialect = connection.dialect.name
|
||||
|
||||
# Skip if already migrated (idempotent)
|
||||
if column_exists(connection, "entity", "note_type"):
|
||||
return
|
||||
|
||||
if dialect == "postgresql":
|
||||
# Postgres supports direct column rename
|
||||
op.execute("ALTER TABLE entity RENAME COLUMN entity_type TO note_type")
|
||||
|
||||
# Recreate the index with new name
|
||||
op.execute("DROP INDEX IF EXISTS ix_entity_type")
|
||||
op.execute("CREATE INDEX ix_note_type ON entity (note_type)")
|
||||
else:
|
||||
# SQLite 3.25.0+ supports ALTER TABLE RENAME COLUMN directly.
|
||||
# Avoids batch_alter_table which fails on tables with generated columns
|
||||
# (duplicate column name error when recreating the table).
|
||||
op.execute("ALTER TABLE entity RENAME COLUMN entity_type TO note_type")
|
||||
|
||||
# Recreate the index with new name
|
||||
if index_exists(connection, "ix_entity_type"):
|
||||
op.drop_index("ix_entity_type", table_name="entity")
|
||||
op.create_index("ix_note_type", "entity", ["note_type"])
|
||||
|
||||
# Update search index metadata: rename entity_type → note_type in JSON
|
||||
# This updates the stored metadata so search results use the new field name
|
||||
# Guard: search_index may not exist on a fresh DB (created by an earlier migration)
|
||||
if not table_exists(connection, "search_index"):
|
||||
return
|
||||
|
||||
if dialect == "postgresql":
|
||||
op.execute(
|
||||
text("""
|
||||
UPDATE search_index
|
||||
SET metadata = metadata - 'entity_type' || jsonb_build_object('note_type', metadata->'entity_type')
|
||||
WHERE metadata ? 'entity_type'
|
||||
""")
|
||||
)
|
||||
else:
|
||||
op.execute(
|
||||
text("""
|
||||
UPDATE search_index
|
||||
SET metadata = json_set(
|
||||
json_remove(metadata, '$.entity_type'),
|
||||
'$.note_type',
|
||||
json_extract(metadata, '$.entity_type')
|
||||
)
|
||||
WHERE json_extract(metadata, '$.entity_type') IS NOT NULL
|
||||
""")
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Rename note_type → entity_type on the entity table."""
|
||||
connection = op.get_bind()
|
||||
dialect = connection.dialect.name
|
||||
|
||||
if dialect == "postgresql":
|
||||
op.execute("ALTER TABLE entity RENAME COLUMN note_type TO entity_type")
|
||||
op.execute("DROP INDEX IF EXISTS ix_note_type")
|
||||
op.execute("CREATE INDEX ix_entity_type ON entity (entity_type)")
|
||||
else:
|
||||
op.execute("ALTER TABLE entity RENAME COLUMN note_type TO entity_type")
|
||||
|
||||
if index_exists(connection, "ix_note_type"):
|
||||
op.drop_index("ix_note_type", table_name="entity")
|
||||
op.create_index("ix_entity_type", "entity", ["entity_type"])
|
||||
|
||||
# Revert search index metadata
|
||||
if not table_exists(connection, "search_index"):
|
||||
return
|
||||
|
||||
if dialect == "postgresql":
|
||||
op.execute(
|
||||
text("""
|
||||
UPDATE search_index
|
||||
SET metadata = metadata - 'note_type' || jsonb_build_object('entity_type', metadata->'note_type')
|
||||
WHERE metadata ? 'note_type'
|
||||
""")
|
||||
)
|
||||
else:
|
||||
op.execute(
|
||||
text("""
|
||||
UPDATE search_index
|
||||
SET metadata = json_set(
|
||||
json_remove(metadata, '$.note_type'),
|
||||
'$.entity_type',
|
||||
json_extract(metadata, '$.note_type')
|
||||
)
|
||||
WHERE json_extract(metadata, '$.note_type') IS NOT NULL
|
||||
""")
|
||||
)
|
||||
+74
@@ -0,0 +1,74 @@
|
||||
"""Add created_by and last_updated_by columns to entity table.
|
||||
|
||||
Revision ID: k4e5f6g7h8i9
|
||||
Revises: j3d4e5f6g7h8
|
||||
Create Date: 2026-02-23 00:00:00.000000
|
||||
|
||||
These columns track which cloud user created and last modified each entity.
|
||||
Both are nullable — NULL for local/CLI usage and existing entities.
|
||||
"""
|
||||
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy import text
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = "k4e5f6g7h8i9"
|
||||
down_revision: Union[str, None] = "j3d4e5f6g7h8"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def column_exists(connection, table: str, column: str) -> bool:
|
||||
"""Check if a column exists in a table (idempotent migration support)."""
|
||||
if connection.dialect.name == "postgresql":
|
||||
result = connection.execute(
|
||||
text(
|
||||
"SELECT 1 FROM information_schema.columns "
|
||||
"WHERE table_name = :table AND column_name = :column"
|
||||
),
|
||||
{"table": table, "column": column},
|
||||
)
|
||||
return result.fetchone() is not None
|
||||
else:
|
||||
# SQLite
|
||||
result = connection.execute(text(f"PRAGMA table_info({table})"))
|
||||
columns = [row[1] for row in result]
|
||||
return column in columns
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Add created_by and last_updated_by columns to entity table.
|
||||
|
||||
Both columns are nullable strings that store cloud user_profile_id UUIDs.
|
||||
No data backfill — existing rows get NULL.
|
||||
"""
|
||||
connection = op.get_bind()
|
||||
|
||||
if not column_exists(connection, "entity", "created_by"):
|
||||
op.add_column("entity", sa.Column("created_by", sa.String(), nullable=True))
|
||||
|
||||
if not column_exists(connection, "entity", "last_updated_by"):
|
||||
op.add_column("entity", sa.Column("last_updated_by", sa.String(), nullable=True))
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Remove created_by and last_updated_by columns from entity table."""
|
||||
connection = op.get_bind()
|
||||
dialect = connection.dialect.name
|
||||
|
||||
if column_exists(connection, "entity", "last_updated_by"):
|
||||
if dialect == "postgresql":
|
||||
op.drop_column("entity", "last_updated_by")
|
||||
else:
|
||||
with op.batch_alter_table("entity") as batch_op:
|
||||
batch_op.drop_column("last_updated_by")
|
||||
|
||||
if column_exists(connection, "entity", "created_by"):
|
||||
if dialect == "postgresql":
|
||||
op.drop_column("entity", "created_by")
|
||||
else:
|
||||
with op.batch_alter_table("entity") as batch_op:
|
||||
batch_op.drop_column("created_by")
|
||||
@@ -0,0 +1,65 @@
|
||||
"""Add note_content table
|
||||
|
||||
Revision ID: l5g6h7i8j9k0
|
||||
Revises: k4e5f6g7h8i9
|
||||
Create Date: 2026-04-04 12:00:00.000000
|
||||
|
||||
"""
|
||||
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = "l5g6h7i8j9k0"
|
||||
down_revision: Union[str, None] = "k4e5f6g7h8i9"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Create note_content for materialized note content and sync state."""
|
||||
op.create_table(
|
||||
"note_content",
|
||||
sa.Column("entity_id", sa.Integer(), nullable=False),
|
||||
sa.Column("project_id", sa.Integer(), nullable=False),
|
||||
sa.Column("external_id", sa.String(), nullable=False),
|
||||
sa.Column("file_path", sa.String(), nullable=False),
|
||||
sa.Column("markdown_content", sa.Text(), nullable=False),
|
||||
sa.Column("db_version", sa.BigInteger(), nullable=False),
|
||||
sa.Column("db_checksum", sa.String(), nullable=False),
|
||||
sa.Column("file_version", sa.BigInteger(), nullable=True),
|
||||
sa.Column("file_checksum", sa.String(), nullable=True),
|
||||
sa.Column("file_write_status", sa.String(), nullable=False),
|
||||
sa.Column("last_source", sa.String(), nullable=True),
|
||||
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("file_updated_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("last_materialization_error", sa.Text(), nullable=True),
|
||||
sa.Column("last_materialization_attempt_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.CheckConstraint(
|
||||
"file_write_status IN ("
|
||||
"'pending', "
|
||||
"'writing', "
|
||||
"'synced', "
|
||||
"'failed', "
|
||||
"'external_change_detected'"
|
||||
")",
|
||||
name="ck_note_content_file_write_status",
|
||||
),
|
||||
sa.ForeignKeyConstraint(["entity_id"], ["entity.id"], ondelete="CASCADE"),
|
||||
sa.ForeignKeyConstraint(["project_id"], ["project.id"], ondelete="CASCADE"),
|
||||
sa.PrimaryKeyConstraint("entity_id"),
|
||||
)
|
||||
op.create_index("ix_note_content_project_id", "note_content", ["project_id"], unique=False)
|
||||
op.create_index("ix_note_content_file_path", "note_content", ["file_path"], unique=False)
|
||||
op.create_index("ix_note_content_external_id", "note_content", ["external_id"], unique=True)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Drop note_content and its supporting indexes."""
|
||||
op.drop_index("ix_note_content_external_id", table_name="note_content")
|
||||
op.drop_index("ix_note_content_file_path", table_name="note_content")
|
||||
op.drop_index("ix_note_content_project_id", table_name="note_content")
|
||||
op.drop_table("note_content")
|
||||
@@ -0,0 +1,84 @@
|
||||
"""Persist vector sync fingerprints on chunk metadata.
|
||||
|
||||
Revision ID: m6h7i8j9k0l1
|
||||
Revises: l5g6h7i8j9k0
|
||||
Create Date: 2026-04-07 00:00:00.000000
|
||||
|
||||
"""
|
||||
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = "m6h7i8j9k0l1"
|
||||
down_revision: Union[str, None] = "l5g6h7i8j9k0"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Add entity fingerprint + embedding model metadata to Postgres chunk rows.
|
||||
|
||||
Trigger: vector sync now fast-skips unchanged entities using persisted
|
||||
semantic fingerprints.
|
||||
Why: chunk rows already own the per-entity derived metadata we diff against,
|
||||
so persisting the fingerprint on that table avoids a second sync-state table.
|
||||
Outcome: existing rows get empty-string placeholders and will be refreshed on
|
||||
the next vector sync before they become eligible for skip checks.
|
||||
"""
|
||||
connection = op.get_bind()
|
||||
if connection.dialect.name != "postgresql":
|
||||
return
|
||||
|
||||
op.execute(
|
||||
"""
|
||||
ALTER TABLE search_vector_chunks
|
||||
ADD COLUMN IF NOT EXISTS entity_fingerprint TEXT
|
||||
"""
|
||||
)
|
||||
op.execute(
|
||||
"""
|
||||
ALTER TABLE search_vector_chunks
|
||||
ADD COLUMN IF NOT EXISTS embedding_model TEXT
|
||||
"""
|
||||
)
|
||||
op.execute(
|
||||
"""
|
||||
UPDATE search_vector_chunks
|
||||
SET entity_fingerprint = COALESCE(entity_fingerprint, ''),
|
||||
embedding_model = COALESCE(embedding_model, '')
|
||||
"""
|
||||
)
|
||||
op.execute(
|
||||
"""
|
||||
ALTER TABLE search_vector_chunks
|
||||
ALTER COLUMN entity_fingerprint SET NOT NULL
|
||||
"""
|
||||
)
|
||||
op.execute(
|
||||
"""
|
||||
ALTER TABLE search_vector_chunks
|
||||
ALTER COLUMN embedding_model SET NOT NULL
|
||||
"""
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Remove vector sync fingerprint columns from Postgres chunk rows."""
|
||||
connection = op.get_bind()
|
||||
if connection.dialect.name != "postgresql":
|
||||
return
|
||||
|
||||
op.execute(
|
||||
"""
|
||||
ALTER TABLE search_vector_chunks
|
||||
DROP COLUMN IF EXISTS embedding_model
|
||||
"""
|
||||
)
|
||||
op.execute(
|
||||
"""
|
||||
ALTER TABLE search_vector_chunks
|
||||
DROP COLUMN IF EXISTS entity_fingerprint
|
||||
"""
|
||||
)
|
||||
@@ -0,0 +1,86 @@
|
||||
"""Remove orphaned search rows whose project was already deleted.
|
||||
|
||||
Revision ID: n7i8j9k0l1m2
|
||||
Revises: m6h7i8j9k0l1
|
||||
Create Date: 2026-05-15 18:30:00.000000
|
||||
|
||||
"""
|
||||
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
from sqlalchemy import inspect
|
||||
|
||||
|
||||
revision: str = "n7i8j9k0l1m2"
|
||||
down_revision: Union[str, None] = "m6h7i8j9k0l1"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def _table_exists(connection, table_name: str) -> bool:
|
||||
"""Inspector-based table check, dialect agnostic.
|
||||
|
||||
Trigger: SQLite creates search_index as an FTS5 virtual table at runtime
|
||||
via SearchRepository.init_search_index, not through Alembic, so fresh
|
||||
installs hit this migration before the table exists.
|
||||
Why: a blind DELETE against a missing table fails the whole upgrade.
|
||||
Outcome: callers skip the sweep when the table isn't present yet — the
|
||||
runtime-created table on a fresh DB has no orphans to clean.
|
||||
"""
|
||||
return table_name in inspect(connection).get_table_names()
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Purge orphaned search rows left over from prior project deletions.
|
||||
|
||||
Trigger: project deletion on SQLite never removed the derived FTS rows,
|
||||
because the FTS5 virtual table can't carry a foreign key. The leak shows
|
||||
up in two shapes:
|
||||
1. project_id no longer exists in `project` (deleted project, id never
|
||||
reused).
|
||||
2. project_id still exists but `entity_id` no longer exists in `entity`
|
||||
— auto-increment handed the id to a brand-new project and the FTS
|
||||
rows from the deleted predecessor masquerade as the new tenant's data.
|
||||
Why: search_index.project_id is the only scope predicate the search
|
||||
repository applies, so leftover rows surface under the wrong project on
|
||||
every search.
|
||||
Outcome: a one-time sweep deletes both shapes, from the FTS index and
|
||||
from search_vector_chunks. Postgres already cascaded on FK delete, so
|
||||
these statements are no-ops there.
|
||||
"""
|
||||
connection = op.get_bind()
|
||||
|
||||
if _table_exists(connection, "search_index"):
|
||||
op.execute(
|
||||
"""
|
||||
DELETE FROM search_index
|
||||
WHERE project_id NOT IN (SELECT id FROM project)
|
||||
"""
|
||||
)
|
||||
op.execute(
|
||||
"""
|
||||
DELETE FROM search_index
|
||||
WHERE entity_id IS NOT NULL
|
||||
AND entity_id NOT IN (SELECT id FROM entity)
|
||||
"""
|
||||
)
|
||||
|
||||
if _table_exists(connection, "search_vector_chunks"):
|
||||
op.execute(
|
||||
"""
|
||||
DELETE FROM search_vector_chunks
|
||||
WHERE project_id NOT IN (SELECT id FROM project)
|
||||
"""
|
||||
)
|
||||
op.execute(
|
||||
"""
|
||||
DELETE FROM search_vector_chunks
|
||||
WHERE entity_id NOT IN (SELECT id FROM entity)
|
||||
"""
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""No-op: orphan rows cannot be reconstructed."""
|
||||
pass
|
||||
+107
-41
@@ -2,23 +2,14 @@
|
||||
|
||||
from contextlib import asynccontextmanager
|
||||
|
||||
from fastapi import FastAPI, HTTPException
|
||||
from fastapi import FastAPI, HTTPException, Request
|
||||
from fastapi.exception_handlers import http_exception_handler
|
||||
from fastapi.responses import JSONResponse
|
||||
from fastapi.routing import APIRouter
|
||||
from loguru import logger
|
||||
|
||||
from basic_memory import __version__ as version
|
||||
from basic_memory.api.container import ApiContainer, set_container
|
||||
from basic_memory.api.routers import (
|
||||
directory_router,
|
||||
importer_router,
|
||||
knowledge,
|
||||
management,
|
||||
memory,
|
||||
project,
|
||||
resource,
|
||||
search,
|
||||
prompt_router,
|
||||
)
|
||||
from basic_memory.api.v2.routers import (
|
||||
knowledge_router as v2_knowledge,
|
||||
project_router as v2_project,
|
||||
@@ -28,9 +19,23 @@ from basic_memory.api.v2.routers import (
|
||||
directory_router as v2_directory,
|
||||
prompt_router as v2_prompt,
|
||||
importer_router as v2_importer,
|
||||
schema_router as v2_schema,
|
||||
)
|
||||
from basic_memory.api.v2.routers.project_router import (
|
||||
add_project,
|
||||
list_projects,
|
||||
synchronize_projects,
|
||||
)
|
||||
import logfire
|
||||
from basic_memory.config import init_api_logging
|
||||
from basic_memory.services.exceptions import EntityAlreadyExistsError
|
||||
from basic_memory.services.initialization import initialize_app
|
||||
from basic_memory.workspace_context import (
|
||||
WORKSPACE_SLUG_HEADER,
|
||||
WORKSPACE_TYPE_HEADER,
|
||||
workspace_permalink_context_validation_error,
|
||||
workspace_permalink_context,
|
||||
)
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
@@ -46,30 +51,39 @@ async def lifespan(app: FastAPI): # pragma: no cover
|
||||
set_container(container)
|
||||
app.state.container = container
|
||||
|
||||
logger.info(f"Starting Basic Memory API (mode={container.mode.name})")
|
||||
with logfire.span(
|
||||
"api.lifecycle.startup",
|
||||
entrypoint="api",
|
||||
mode=container.mode.name.lower(),
|
||||
):
|
||||
logger.info(f"Starting Basic Memory API (mode={container.mode.name})")
|
||||
|
||||
await initialize_app(container.config)
|
||||
await initialize_app(container.config)
|
||||
|
||||
# Cache database connections in app state for performance
|
||||
logger.info("Initializing database and caching connections...")
|
||||
engine, session_maker = await container.init_database()
|
||||
app.state.engine = engine
|
||||
app.state.session_maker = session_maker
|
||||
logger.info("Database connections cached in app state")
|
||||
# Cache database connections in app state for performance
|
||||
logger.info("Initializing database and caching connections...")
|
||||
engine, session_maker = await container.init_database()
|
||||
app.state.engine = engine
|
||||
app.state.session_maker = session_maker
|
||||
logger.info("Database connections cached in app state")
|
||||
|
||||
# Create and start sync coordinator (lifecycle centralized in coordinator)
|
||||
sync_coordinator = container.create_sync_coordinator()
|
||||
await sync_coordinator.start()
|
||||
app.state.sync_coordinator = sync_coordinator
|
||||
# Create and start sync coordinator (lifecycle centralized in coordinator)
|
||||
sync_coordinator = container.create_sync_coordinator()
|
||||
await sync_coordinator.start()
|
||||
app.state.sync_coordinator = sync_coordinator
|
||||
|
||||
# Proceed with startup
|
||||
yield
|
||||
|
||||
# Shutdown - coordinator handles clean task cancellation
|
||||
logger.info("Shutting down Basic Memory API")
|
||||
await sync_coordinator.stop()
|
||||
|
||||
await container.shutdown_database()
|
||||
with logfire.span(
|
||||
"api.lifecycle.shutdown",
|
||||
entrypoint="api",
|
||||
mode=container.mode.name.lower(),
|
||||
):
|
||||
logger.info("Shutting down Basic Memory API")
|
||||
await sync_coordinator.stop()
|
||||
await container.shutdown_database()
|
||||
|
||||
|
||||
# Initialize FastAPI app
|
||||
@@ -80,6 +94,32 @@ app = FastAPI(
|
||||
lifespan=lifespan,
|
||||
)
|
||||
|
||||
|
||||
@app.middleware("http")
|
||||
async def workspace_permalink_context_middleware(request: Request, call_next):
|
||||
"""Populate workspace permalink context from request headers."""
|
||||
workspace_slug = request.headers.get(WORKSPACE_SLUG_HEADER)
|
||||
workspace_type = request.headers.get(WORKSPACE_TYPE_HEADER)
|
||||
|
||||
validation_error = workspace_permalink_context_validation_error(workspace_slug, workspace_type)
|
||||
if validation_error is not None:
|
||||
return JSONResponse(
|
||||
status_code=400,
|
||||
content={"detail": validation_error},
|
||||
)
|
||||
|
||||
if not workspace_slug:
|
||||
return await call_next(request)
|
||||
|
||||
# ContextVar state remains active across the awaited downstream handler while
|
||||
# this context manager is open, so entity creation can see request metadata.
|
||||
with workspace_permalink_context(
|
||||
workspace_slug=workspace_slug,
|
||||
workspace_type=workspace_type,
|
||||
):
|
||||
return await call_next(request)
|
||||
|
||||
|
||||
# Include v2 routers FIRST (more specific paths must match before /{project} catch-all)
|
||||
app.include_router(v2_knowledge, prefix="/v2/projects/{project_id}")
|
||||
app.include_router(v2_memory, prefix="/v2/projects/{project_id}")
|
||||
@@ -88,21 +128,44 @@ app.include_router(v2_resource, prefix="/v2/projects/{project_id}")
|
||||
app.include_router(v2_directory, prefix="/v2/projects/{project_id}")
|
||||
app.include_router(v2_prompt, prefix="/v2/projects/{project_id}")
|
||||
app.include_router(v2_importer, prefix="/v2/projects/{project_id}")
|
||||
app.include_router(v2_schema, prefix="/v2/projects/{project_id}")
|
||||
app.include_router(v2_project, prefix="/v2")
|
||||
|
||||
# Include v1 routers (/{project} is a catch-all, must come after specific prefixes)
|
||||
app.include_router(knowledge.router, prefix="/{project}")
|
||||
app.include_router(memory.router, prefix="/{project}")
|
||||
app.include_router(resource.router, prefix="/{project}")
|
||||
app.include_router(search.router, prefix="/{project}")
|
||||
app.include_router(project.project_router, prefix="/{project}")
|
||||
app.include_router(directory_router.router, prefix="/{project}")
|
||||
app.include_router(prompt_router.router, prefix="/{project}")
|
||||
app.include_router(importer_router.router, prefix="/{project}")
|
||||
# Legacy web app proxy paths (compat with /proxy/projects/projects)
|
||||
app.include_router(v2_project, prefix="/proxy/projects")
|
||||
|
||||
# Project resource router works across projects
|
||||
app.include_router(project.project_resource_router)
|
||||
app.include_router(management.router)
|
||||
# Legacy v1 compat: older CLI versions (v0.18.0 and earlier) call /projects/...
|
||||
# Using router mount causes 307 redirect which proxy doesn't follow, so add explicit routes
|
||||
legacy_router = APIRouter(tags=["legacy"])
|
||||
legacy_router.add_api_route("/projects/projects", list_projects, methods=["GET"])
|
||||
legacy_router.add_api_route("/projects/projects", add_project, methods=["POST"])
|
||||
legacy_router.add_api_route("/projects/config/sync", synchronize_projects, methods=["POST"])
|
||||
app.include_router(legacy_router)
|
||||
|
||||
# V2 routers are the only public API surface
|
||||
|
||||
|
||||
@app.exception_handler(EntityAlreadyExistsError)
|
||||
async def entity_already_exists_error_handler(request: Request, exc: EntityAlreadyExistsError):
|
||||
"""Handle entity creation conflicts (e.g., file already exists).
|
||||
|
||||
This is expected behavior when users try to create notes that exist,
|
||||
so log at INFO level instead of ERROR.
|
||||
"""
|
||||
logger.info(
|
||||
"Entity already exists",
|
||||
url=str(request.url),
|
||||
method=request.method,
|
||||
path=request.url.path,
|
||||
error=str(exc),
|
||||
)
|
||||
return await http_exception_handler(
|
||||
request,
|
||||
HTTPException(
|
||||
status_code=409,
|
||||
detail="Note already exists. Use edit_note to modify it, or delete it first.",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@app.exception_handler(Exception)
|
||||
@@ -116,4 +179,7 @@ async def exception_handler(request, exc): # pragma: no cover
|
||||
error_type=type(exc).__name__,
|
||||
error=str(exc),
|
||||
)
|
||||
return await http_exception_handler(request, HTTPException(status_code=500, detail=str(exc)))
|
||||
return await http_exception_handler(
|
||||
request,
|
||||
HTTPException(status_code=500, detail="Internal server error"),
|
||||
)
|
||||
|
||||
@@ -47,7 +47,6 @@ class ApiContainer:
|
||||
"""
|
||||
config = ConfigManager().config
|
||||
mode = resolve_runtime_mode(
|
||||
cloud_mode_enabled=config.cloud_mode_enabled,
|
||||
is_test_env=config.is_test_env,
|
||||
)
|
||||
return cls(config=config, mode=mode)
|
||||
|
||||
@@ -1,11 +0,0 @@
|
||||
"""API routers."""
|
||||
|
||||
from . import knowledge_router as knowledge
|
||||
from . import management_router as management
|
||||
from . import memory_router as memory
|
||||
from . import project_router as project
|
||||
from . import resource_router as resource
|
||||
from . import search_router as search
|
||||
from . import prompt_router as prompt
|
||||
|
||||
__all__ = ["knowledge", "management", "memory", "project", "resource", "search", "prompt"]
|
||||
@@ -1,84 +0,0 @@
|
||||
"""Router for directory tree operations."""
|
||||
|
||||
from typing import List, Optional
|
||||
|
||||
from fastapi import APIRouter, Query
|
||||
|
||||
from basic_memory.deps import DirectoryServiceDep, ProjectIdDep
|
||||
from basic_memory.schemas.directory import DirectoryNode
|
||||
|
||||
router = APIRouter(prefix="/directory", tags=["directory"])
|
||||
|
||||
|
||||
@router.get("/tree", response_model=DirectoryNode, response_model_exclude_none=True)
|
||||
async def get_directory_tree(
|
||||
directory_service: DirectoryServiceDep,
|
||||
project_id: ProjectIdDep,
|
||||
):
|
||||
"""Get hierarchical directory structure from the knowledge base.
|
||||
|
||||
Args:
|
||||
directory_service: Service for directory operations
|
||||
project_id: ID of the current project
|
||||
|
||||
Returns:
|
||||
DirectoryNode representing the root of the hierarchical tree structure
|
||||
"""
|
||||
# Get a hierarchical directory tree for the specific project
|
||||
tree = await directory_service.get_directory_tree()
|
||||
|
||||
# Return the hierarchical tree
|
||||
return tree
|
||||
|
||||
|
||||
@router.get("/structure", response_model=DirectoryNode, response_model_exclude_none=True)
|
||||
async def get_directory_structure(
|
||||
directory_service: DirectoryServiceDep,
|
||||
project_id: ProjectIdDep,
|
||||
):
|
||||
"""Get folder structure for navigation (no files).
|
||||
|
||||
Optimized endpoint for folder tree navigation. Returns only directory nodes
|
||||
without file metadata. For full tree with files, use /directory/tree.
|
||||
|
||||
Args:
|
||||
directory_service: Service for directory operations
|
||||
project_id: ID of the current project
|
||||
|
||||
Returns:
|
||||
DirectoryNode tree containing only folders (type="directory")
|
||||
"""
|
||||
structure = await directory_service.get_directory_structure()
|
||||
return structure
|
||||
|
||||
|
||||
@router.get("/list", response_model=List[DirectoryNode], response_model_exclude_none=True)
|
||||
async def list_directory(
|
||||
directory_service: DirectoryServiceDep,
|
||||
project_id: ProjectIdDep,
|
||||
dir_name: str = Query("/", description="Directory path to list"),
|
||||
depth: int = Query(1, ge=1, le=10, description="Recursion depth (1-10)"),
|
||||
file_name_glob: Optional[str] = Query(
|
||||
None, description="Glob pattern for filtering file names"
|
||||
),
|
||||
):
|
||||
"""List directory contents with filtering and depth control.
|
||||
|
||||
Args:
|
||||
directory_service: Service for directory operations
|
||||
project_id: ID of the current project
|
||||
dir_name: Directory path to list (default: root "/")
|
||||
depth: Recursion depth (1-10, default: 1 for immediate children only)
|
||||
file_name_glob: Optional glob pattern for filtering file names (e.g., "*.md", "*meeting*")
|
||||
|
||||
Returns:
|
||||
List of DirectoryNode objects matching the criteria
|
||||
"""
|
||||
# Get directory listing with filtering
|
||||
nodes = await directory_service.list_directory(
|
||||
dir_name=dir_name,
|
||||
depth=depth,
|
||||
file_name_glob=file_name_glob,
|
||||
)
|
||||
|
||||
return nodes
|
||||
@@ -1,152 +0,0 @@
|
||||
"""Import router for Basic Memory API."""
|
||||
|
||||
import json
|
||||
import logging
|
||||
|
||||
from fastapi import APIRouter, Form, HTTPException, UploadFile, status
|
||||
|
||||
from basic_memory.deps import (
|
||||
ChatGPTImporterDep,
|
||||
ClaudeConversationsImporterDep,
|
||||
ClaudeProjectsImporterDep,
|
||||
MemoryJsonImporterDep,
|
||||
)
|
||||
from basic_memory.importers import Importer
|
||||
from basic_memory.schemas.importer import (
|
||||
ChatImportResult,
|
||||
EntityImportResult,
|
||||
ProjectImportResult,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/import", tags=["import"])
|
||||
|
||||
|
||||
@router.post("/chatgpt", response_model=ChatImportResult)
|
||||
async def import_chatgpt(
|
||||
importer: ChatGPTImporterDep,
|
||||
file: UploadFile,
|
||||
folder: str = Form("conversations"),
|
||||
) -> ChatImportResult:
|
||||
"""Import conversations from ChatGPT JSON export.
|
||||
|
||||
Args:
|
||||
file: The ChatGPT conversations.json file.
|
||||
folder: The folder to place the files in.
|
||||
markdown_processor: MarkdownProcessor instance.
|
||||
|
||||
Returns:
|
||||
ChatImportResult with import statistics.
|
||||
|
||||
Raises:
|
||||
HTTPException: If import fails.
|
||||
"""
|
||||
return await import_file(importer, file, folder)
|
||||
|
||||
|
||||
@router.post("/claude/conversations", response_model=ChatImportResult)
|
||||
async def import_claude_conversations(
|
||||
importer: ClaudeConversationsImporterDep,
|
||||
file: UploadFile,
|
||||
folder: str = Form("conversations"),
|
||||
) -> ChatImportResult:
|
||||
"""Import conversations from Claude conversations.json export.
|
||||
|
||||
Args:
|
||||
file: The Claude conversations.json file.
|
||||
folder: The folder to place the files in.
|
||||
markdown_processor: MarkdownProcessor instance.
|
||||
|
||||
Returns:
|
||||
ChatImportResult with import statistics.
|
||||
|
||||
Raises:
|
||||
HTTPException: If import fails.
|
||||
"""
|
||||
return await import_file(importer, file, folder)
|
||||
|
||||
|
||||
@router.post("/claude/projects", response_model=ProjectImportResult)
|
||||
async def import_claude_projects(
|
||||
importer: ClaudeProjectsImporterDep,
|
||||
file: UploadFile,
|
||||
folder: str = Form("projects"),
|
||||
) -> ProjectImportResult:
|
||||
"""Import projects from Claude projects.json export.
|
||||
|
||||
Args:
|
||||
file: The Claude projects.json file.
|
||||
base_folder: The base folder to place the files in.
|
||||
markdown_processor: MarkdownProcessor instance.
|
||||
|
||||
Returns:
|
||||
ProjectImportResult with import statistics.
|
||||
|
||||
Raises:
|
||||
HTTPException: If import fails.
|
||||
"""
|
||||
return await import_file(importer, file, folder)
|
||||
|
||||
|
||||
@router.post("/memory-json", response_model=EntityImportResult)
|
||||
async def import_memory_json(
|
||||
importer: MemoryJsonImporterDep,
|
||||
file: UploadFile,
|
||||
folder: str = Form("conversations"),
|
||||
) -> EntityImportResult:
|
||||
"""Import entities and relations from a memory.json file.
|
||||
|
||||
Args:
|
||||
file: The memory.json file.
|
||||
destination_folder: Optional destination folder within the project.
|
||||
markdown_processor: MarkdownProcessor instance.
|
||||
|
||||
Returns:
|
||||
EntityImportResult with import statistics.
|
||||
|
||||
Raises:
|
||||
HTTPException: If import fails.
|
||||
"""
|
||||
try:
|
||||
file_data = []
|
||||
file_bytes = await file.read()
|
||||
file_str = file_bytes.decode("utf-8")
|
||||
for line in file_str.splitlines():
|
||||
json_data = json.loads(line)
|
||||
file_data.append(json_data)
|
||||
|
||||
result = await importer.import_data(file_data, folder)
|
||||
if not result.success: # pragma: no cover
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail=result.error_message or "Import failed",
|
||||
)
|
||||
except Exception as e:
|
||||
logger.exception("Import failed")
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail=f"Import failed: {str(e)}",
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
async def import_file(importer: Importer, file: UploadFile, destination_folder: str):
|
||||
try:
|
||||
# Process file
|
||||
json_data = json.load(file.file)
|
||||
result = await importer.import_data(json_data, destination_folder)
|
||||
if not result.success: # pragma: no cover
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail=result.error_message or "Import failed",
|
||||
)
|
||||
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
logger.exception("Import failed")
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail=f"Import failed: {str(e)}",
|
||||
)
|
||||
@@ -1,319 +0,0 @@
|
||||
"""Router for knowledge graph operations.
|
||||
|
||||
⚠️ DEPRECATED: This v1 API is deprecated and will be removed on June 30, 2026.
|
||||
Please migrate to /v2/{project}/knowledge endpoints which use entity IDs instead
|
||||
of path-based identifiers for improved performance and stability.
|
||||
|
||||
Migration guide: See docs/migration/v1-to-v2.md
|
||||
"""
|
||||
|
||||
from typing import Annotated
|
||||
|
||||
from fastapi import APIRouter, HTTPException, BackgroundTasks, Depends, Query, Response
|
||||
from loguru import logger
|
||||
|
||||
from basic_memory.deps import (
|
||||
EntityServiceDep,
|
||||
get_search_service,
|
||||
SearchServiceDep,
|
||||
LinkResolverDep,
|
||||
ProjectPathDep,
|
||||
FileServiceDep,
|
||||
ProjectConfigDep,
|
||||
AppConfigDep,
|
||||
SyncServiceDep,
|
||||
)
|
||||
from basic_memory.schemas import (
|
||||
EntityListResponse,
|
||||
EntityResponse,
|
||||
DeleteEntitiesResponse,
|
||||
DeleteEntitiesRequest,
|
||||
)
|
||||
from basic_memory.schemas.request import EditEntityRequest, MoveEntityRequest
|
||||
from basic_memory.schemas.base import Permalink, Entity
|
||||
|
||||
router = APIRouter(
|
||||
prefix="/knowledge",
|
||||
tags=["knowledge"],
|
||||
deprecated=True, # Marks entire router as deprecated in OpenAPI docs
|
||||
)
|
||||
|
||||
|
||||
async def resolve_relations_background(sync_service, entity_id: int, entity_permalink: str) -> None:
|
||||
"""Background task to resolve relations for a specific entity.
|
||||
|
||||
This runs asynchronously after the API response is sent, preventing
|
||||
long delays when creating entities with many relations.
|
||||
"""
|
||||
try:
|
||||
# Only resolve relations for the newly created entity
|
||||
await sync_service.resolve_relations(entity_id=entity_id)
|
||||
logger.debug(
|
||||
f"Background: Resolved relations for entity {entity_permalink} (id={entity_id})"
|
||||
)
|
||||
except Exception as e: # pragma: no cover
|
||||
# Log but don't fail - this is a background task.
|
||||
# Avoid forcing synthetic failures just for coverage.
|
||||
logger.warning( # pragma: no cover
|
||||
f"Background: Failed to resolve relations for entity {entity_permalink}: {e}"
|
||||
)
|
||||
|
||||
|
||||
## Create endpoints
|
||||
|
||||
|
||||
@router.post("/entities", response_model=EntityResponse)
|
||||
async def create_entity(
|
||||
data: Entity,
|
||||
background_tasks: BackgroundTasks,
|
||||
entity_service: EntityServiceDep,
|
||||
search_service: SearchServiceDep,
|
||||
) -> EntityResponse:
|
||||
"""Create an entity."""
|
||||
logger.info(
|
||||
"API request", endpoint="create_entity", entity_type=data.entity_type, title=data.title
|
||||
)
|
||||
|
||||
entity = await entity_service.create_entity(data)
|
||||
|
||||
# reindex
|
||||
await search_service.index_entity(entity, background_tasks=background_tasks)
|
||||
result = EntityResponse.model_validate(entity)
|
||||
|
||||
logger.info(
|
||||
f"API response: endpoint='create_entity' title={result.title}, permalink={result.permalink}, status_code=201"
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
@router.put("/entities/{permalink:path}", response_model=EntityResponse)
|
||||
async def create_or_update_entity(
|
||||
project: ProjectPathDep,
|
||||
permalink: Permalink,
|
||||
data: Entity,
|
||||
response: Response,
|
||||
background_tasks: BackgroundTasks,
|
||||
entity_service: EntityServiceDep,
|
||||
search_service: SearchServiceDep,
|
||||
file_service: FileServiceDep,
|
||||
sync_service: SyncServiceDep,
|
||||
) -> EntityResponse:
|
||||
"""Create or update an entity. If entity exists, it will be updated, otherwise created."""
|
||||
logger.info(
|
||||
f"API request: create_or_update_entity for {project=}, {permalink=}, {data.entity_type=}, {data.title=}"
|
||||
)
|
||||
|
||||
# Validate permalink matches
|
||||
if data.permalink != permalink:
|
||||
logger.warning(
|
||||
f"API validation error: creating/updating entity with permalink mismatch - url={permalink}, data={data.permalink}",
|
||||
)
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"Entity permalink {data.permalink} must match URL path: '{permalink}'",
|
||||
)
|
||||
|
||||
# Try create_or_update operation
|
||||
entity, created = await entity_service.create_or_update_entity(data)
|
||||
response.status_code = 201 if created else 200
|
||||
|
||||
# reindex
|
||||
await search_service.index_entity(entity, background_tasks=background_tasks)
|
||||
|
||||
# Schedule relation resolution as a background task for new entities
|
||||
# This prevents blocking the API response while resolving potentially many relations
|
||||
if created:
|
||||
background_tasks.add_task(
|
||||
resolve_relations_background, sync_service, entity.id, entity.permalink or ""
|
||||
)
|
||||
|
||||
result = EntityResponse.model_validate(entity)
|
||||
|
||||
logger.info(
|
||||
f"API response: {result.title=}, {result.permalink=}, {created=}, status_code={response.status_code}"
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
@router.patch("/entities/{identifier:path}", response_model=EntityResponse)
|
||||
async def edit_entity(
|
||||
identifier: str,
|
||||
data: EditEntityRequest,
|
||||
background_tasks: BackgroundTasks,
|
||||
entity_service: EntityServiceDep,
|
||||
search_service: SearchServiceDep,
|
||||
) -> EntityResponse:
|
||||
"""Edit an existing entity using various operations like append, prepend, find_replace, or replace_section.
|
||||
|
||||
This endpoint allows for targeted edits without requiring the full entity content.
|
||||
"""
|
||||
logger.info(
|
||||
f"API request: endpoint='edit_entity', identifier='{identifier}', operation='{data.operation}'"
|
||||
)
|
||||
|
||||
try:
|
||||
# Edit the entity using the service
|
||||
entity = await entity_service.edit_entity(
|
||||
identifier=identifier,
|
||||
operation=data.operation,
|
||||
content=data.content,
|
||||
section=data.section,
|
||||
find_text=data.find_text,
|
||||
expected_replacements=data.expected_replacements,
|
||||
)
|
||||
|
||||
# Reindex the updated entity
|
||||
await search_service.index_entity(entity, background_tasks=background_tasks)
|
||||
|
||||
# Return the updated entity response
|
||||
result = EntityResponse.model_validate(entity)
|
||||
|
||||
logger.info(
|
||||
"API response",
|
||||
endpoint="edit_entity",
|
||||
identifier=identifier,
|
||||
operation=data.operation,
|
||||
permalink=result.permalink,
|
||||
status_code=200,
|
||||
)
|
||||
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error editing entity: {e}")
|
||||
raise HTTPException(status_code=400, detail=str(e))
|
||||
|
||||
|
||||
@router.post("/move")
|
||||
async def move_entity(
|
||||
data: MoveEntityRequest,
|
||||
background_tasks: BackgroundTasks,
|
||||
entity_service: EntityServiceDep,
|
||||
project_config: ProjectConfigDep,
|
||||
app_config: AppConfigDep,
|
||||
search_service: SearchServiceDep,
|
||||
) -> EntityResponse:
|
||||
"""Move an entity to a new file location with project consistency.
|
||||
|
||||
This endpoint moves a note to a different path while maintaining project
|
||||
consistency and optionally updating permalinks based on configuration.
|
||||
"""
|
||||
logger.info(
|
||||
f"API request: endpoint='move_entity', identifier='{data.identifier}', destination='{data.destination_path}'"
|
||||
)
|
||||
|
||||
try:
|
||||
# Move the entity using the service
|
||||
moved_entity = await entity_service.move_entity(
|
||||
identifier=data.identifier,
|
||||
destination_path=data.destination_path,
|
||||
project_config=project_config,
|
||||
app_config=app_config,
|
||||
)
|
||||
|
||||
# Get the moved entity to reindex it
|
||||
entity = await entity_service.link_resolver.resolve_link(data.destination_path)
|
||||
if entity:
|
||||
await search_service.index_entity(entity, background_tasks=background_tasks)
|
||||
|
||||
logger.info(
|
||||
"API response",
|
||||
endpoint="move_entity",
|
||||
identifier=data.identifier,
|
||||
destination=data.destination_path,
|
||||
status_code=200,
|
||||
)
|
||||
result = EntityResponse.model_validate(moved_entity)
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error moving entity: {e}")
|
||||
raise HTTPException(status_code=400, detail=str(e))
|
||||
|
||||
|
||||
## Read endpoints
|
||||
|
||||
|
||||
@router.get("/entities/{identifier:path}", response_model=EntityResponse)
|
||||
async def get_entity(
|
||||
entity_service: EntityServiceDep,
|
||||
link_resolver: LinkResolverDep,
|
||||
identifier: str,
|
||||
) -> EntityResponse:
|
||||
"""Get a specific entity by file path or permalink..
|
||||
|
||||
Args:
|
||||
identifier: Entity file path or permalink
|
||||
:param entity_service: EntityService
|
||||
:param link_resolver: LinkResolver
|
||||
"""
|
||||
logger.info(f"request: get_entity with identifier={identifier}")
|
||||
entity = await link_resolver.resolve_link(identifier)
|
||||
if not entity:
|
||||
raise HTTPException(status_code=404, detail=f"Entity {identifier} not found")
|
||||
|
||||
result = EntityResponse.model_validate(entity)
|
||||
return result
|
||||
|
||||
|
||||
@router.get("/entities", response_model=EntityListResponse)
|
||||
async def get_entities(
|
||||
entity_service: EntityServiceDep,
|
||||
permalink: Annotated[list[str] | None, Query()] = None,
|
||||
) -> EntityListResponse:
|
||||
"""Open specific entities"""
|
||||
logger.info(f"request: get_entities with permalinks={permalink}")
|
||||
|
||||
entities = await entity_service.get_entities_by_permalinks(permalink) if permalink else []
|
||||
result = EntityListResponse(
|
||||
entities=[EntityResponse.model_validate(entity) for entity in entities]
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
## Delete endpoints
|
||||
|
||||
|
||||
@router.delete("/entities/{identifier:path}", response_model=DeleteEntitiesResponse)
|
||||
async def delete_entity(
|
||||
identifier: str,
|
||||
background_tasks: BackgroundTasks,
|
||||
entity_service: EntityServiceDep,
|
||||
link_resolver: LinkResolverDep,
|
||||
search_service=Depends(get_search_service),
|
||||
) -> DeleteEntitiesResponse:
|
||||
"""Delete a single entity and remove from search index."""
|
||||
logger.info(f"request: delete_entity with identifier={identifier}")
|
||||
|
||||
entity = await link_resolver.resolve_link(identifier)
|
||||
if entity is None:
|
||||
return DeleteEntitiesResponse(deleted=False)
|
||||
|
||||
# Delete the entity
|
||||
deleted = await entity_service.delete_entity(entity.permalink or entity.id)
|
||||
|
||||
# Remove from search index (entity, observations, and relations)
|
||||
background_tasks.add_task(search_service.handle_delete, entity)
|
||||
|
||||
result = DeleteEntitiesResponse(deleted=deleted)
|
||||
return result
|
||||
|
||||
|
||||
@router.post("/entities/delete", response_model=DeleteEntitiesResponse)
|
||||
async def delete_entities(
|
||||
data: DeleteEntitiesRequest,
|
||||
background_tasks: BackgroundTasks,
|
||||
entity_service: EntityServiceDep,
|
||||
search_service=Depends(get_search_service),
|
||||
) -> DeleteEntitiesResponse:
|
||||
"""Delete entities and remove from search index."""
|
||||
logger.info(f"request: delete_entities with data={data}")
|
||||
deleted = False
|
||||
|
||||
# Remove each deleted entity from search index
|
||||
for permalink in data.permalinks:
|
||||
deleted = await entity_service.delete_entity(permalink)
|
||||
background_tasks.add_task(search_service.delete_by_permalink, permalink)
|
||||
|
||||
result = DeleteEntitiesResponse(deleted=deleted)
|
||||
return result
|
||||
@@ -1,80 +0,0 @@
|
||||
"""Management router for basic-memory API."""
|
||||
|
||||
import asyncio
|
||||
|
||||
from fastapi import APIRouter, Request
|
||||
from loguru import logger
|
||||
from pydantic import BaseModel
|
||||
|
||||
from basic_memory.config import ConfigManager
|
||||
from basic_memory.deps import SyncServiceDep, ProjectRepositoryDep
|
||||
|
||||
router = APIRouter(prefix="/management", tags=["management"])
|
||||
|
||||
|
||||
class WatchStatusResponse(BaseModel):
|
||||
"""Response model for watch status."""
|
||||
|
||||
running: bool
|
||||
"""Whether the watch service is currently running."""
|
||||
|
||||
|
||||
@router.get("/watch/status", response_model=WatchStatusResponse)
|
||||
async def get_watch_status(request: Request) -> WatchStatusResponse:
|
||||
"""Get the current status of the watch service."""
|
||||
return WatchStatusResponse(
|
||||
running=request.app.state.watch_task is not None and not request.app.state.watch_task.done()
|
||||
)
|
||||
|
||||
|
||||
@router.post("/watch/start", response_model=WatchStatusResponse)
|
||||
async def start_watch_service(
|
||||
request: Request, project_repository: ProjectRepositoryDep, sync_service: SyncServiceDep
|
||||
) -> WatchStatusResponse:
|
||||
"""Start the watch service if it's not already running."""
|
||||
|
||||
# needed because of circular imports from sync -> app
|
||||
from basic_memory.sync import WatchService
|
||||
from basic_memory.sync.background_sync import create_background_sync_task
|
||||
|
||||
if request.app.state.watch_task is not None and not request.app.state.watch_task.done():
|
||||
# Watch service is already running
|
||||
return WatchStatusResponse(running=True)
|
||||
|
||||
app_config = ConfigManager().config
|
||||
|
||||
# Create and start a new watch service
|
||||
logger.info("Starting watch service via management API")
|
||||
|
||||
# Get services needed for the watch task
|
||||
watch_service = WatchService(
|
||||
app_config=app_config,
|
||||
project_repository=project_repository,
|
||||
)
|
||||
|
||||
# Create and store the task
|
||||
watch_task = create_background_sync_task(sync_service, watch_service)
|
||||
request.app.state.watch_task = watch_task
|
||||
|
||||
return WatchStatusResponse(running=True)
|
||||
|
||||
|
||||
@router.post("/watch/stop", response_model=WatchStatusResponse)
|
||||
async def stop_watch_service(request: Request) -> WatchStatusResponse: # pragma: no cover
|
||||
"""Stop the watch service if it's running."""
|
||||
if request.app.state.watch_task is None or request.app.state.watch_task.done():
|
||||
# Watch service is not running
|
||||
return WatchStatusResponse(running=False)
|
||||
|
||||
# Cancel the running task
|
||||
logger.info("Stopping watch service via management API")
|
||||
request.app.state.watch_task.cancel()
|
||||
|
||||
# Wait for it to be properly cancelled
|
||||
try:
|
||||
await request.app.state.watch_task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
|
||||
request.app.state.watch_task = None
|
||||
return WatchStatusResponse(running=False)
|
||||
@@ -1,90 +0,0 @@
|
||||
"""Routes for memory:// URI operations."""
|
||||
|
||||
from typing import Annotated, Optional
|
||||
|
||||
from fastapi import APIRouter, Query
|
||||
from loguru import logger
|
||||
|
||||
from basic_memory.deps import ContextServiceDep, EntityRepositoryDep
|
||||
from basic_memory.schemas.base import TimeFrame, parse_timeframe
|
||||
from basic_memory.schemas.memory import (
|
||||
GraphContext,
|
||||
normalize_memory_url,
|
||||
)
|
||||
from basic_memory.schemas.search import SearchItemType
|
||||
from basic_memory.api.routers.utils import to_graph_context
|
||||
|
||||
router = APIRouter(prefix="/memory", tags=["memory"])
|
||||
|
||||
|
||||
@router.get("/recent", response_model=GraphContext)
|
||||
async def recent(
|
||||
context_service: ContextServiceDep,
|
||||
entity_repository: EntityRepositoryDep,
|
||||
type: Annotated[list[SearchItemType] | None, Query()] = None,
|
||||
depth: int = 1,
|
||||
timeframe: TimeFrame = "7d",
|
||||
page: int = 1,
|
||||
page_size: int = 10,
|
||||
max_related: int = 10,
|
||||
) -> GraphContext:
|
||||
# return all types by default
|
||||
types = (
|
||||
[SearchItemType.ENTITY, SearchItemType.RELATION, SearchItemType.OBSERVATION]
|
||||
if not type
|
||||
else type
|
||||
)
|
||||
|
||||
logger.debug(
|
||||
f"Getting recent context: `{types}` depth: `{depth}` timeframe: `{timeframe}` page: `{page}` page_size: `{page_size}` max_related: `{max_related}`"
|
||||
)
|
||||
# Parse timeframe
|
||||
since = parse_timeframe(timeframe)
|
||||
limit = page_size
|
||||
offset = (page - 1) * page_size
|
||||
|
||||
# Build context
|
||||
context = await context_service.build_context(
|
||||
types=types, depth=depth, since=since, limit=limit, offset=offset, max_related=max_related
|
||||
)
|
||||
recent_context = await to_graph_context(
|
||||
context, entity_repository=entity_repository, page=page, page_size=page_size
|
||||
)
|
||||
logger.debug(f"Recent context: {recent_context.model_dump_json()}")
|
||||
return recent_context
|
||||
|
||||
|
||||
# get_memory_context needs to be declared last so other paths can match
|
||||
|
||||
|
||||
@router.get("/{uri:path}", response_model=GraphContext)
|
||||
async def get_memory_context(
|
||||
context_service: ContextServiceDep,
|
||||
entity_repository: EntityRepositoryDep,
|
||||
uri: str,
|
||||
depth: int = 1,
|
||||
timeframe: Optional[TimeFrame] = None,
|
||||
page: int = 1,
|
||||
page_size: int = 10,
|
||||
max_related: int = 10,
|
||||
) -> GraphContext:
|
||||
"""Get rich context from memory:// URI."""
|
||||
# add the project name from the config to the url as the "host
|
||||
# Parse URI
|
||||
logger.debug(
|
||||
f"Getting context for URI: `{uri}` depth: `{depth}` timeframe: `{timeframe}` page: `{page}` page_size: `{page_size}` max_related: `{max_related}`"
|
||||
)
|
||||
memory_url = normalize_memory_url(uri)
|
||||
|
||||
# Parse timeframe
|
||||
since = parse_timeframe(timeframe) if timeframe else None
|
||||
limit = page_size
|
||||
offset = (page - 1) * page_size
|
||||
|
||||
# Build context
|
||||
context = await context_service.build_context(
|
||||
memory_url, depth=depth, since=since, limit=limit, offset=offset, max_related=max_related
|
||||
)
|
||||
return await to_graph_context(
|
||||
context, entity_repository=entity_repository, page=page, page_size=page_size
|
||||
)
|
||||
@@ -1,472 +0,0 @@
|
||||
"""Router for project management."""
|
||||
|
||||
import os
|
||||
from fastapi import APIRouter, HTTPException, Path, Body, BackgroundTasks, Response, Query
|
||||
from typing import Optional
|
||||
from loguru import logger
|
||||
|
||||
from basic_memory.deps import (
|
||||
ProjectConfigDep,
|
||||
ProjectServiceDep,
|
||||
ProjectPathDep,
|
||||
SyncServiceDep,
|
||||
)
|
||||
from basic_memory.schemas import ProjectInfoResponse, SyncReportResponse
|
||||
from basic_memory.schemas.project_info import (
|
||||
ProjectList,
|
||||
ProjectItem,
|
||||
ProjectInfoRequest,
|
||||
ProjectStatusResponse,
|
||||
)
|
||||
from basic_memory.utils import normalize_project_path
|
||||
|
||||
# Router for resources in a specific project
|
||||
# The ProjectPathDep is used in the path as a prefix, so the request path is like /{project}/project/info
|
||||
project_router = APIRouter(prefix="/project", tags=["project"])
|
||||
|
||||
# Router for managing project resources
|
||||
project_resource_router = APIRouter(prefix="/projects", tags=["project_management"])
|
||||
|
||||
|
||||
@project_router.get("/info", response_model=ProjectInfoResponse)
|
||||
async def get_project_info(
|
||||
project_service: ProjectServiceDep,
|
||||
project: ProjectPathDep,
|
||||
) -> ProjectInfoResponse:
|
||||
"""Get comprehensive information about the specified Basic Memory project."""
|
||||
return await project_service.get_project_info(project)
|
||||
|
||||
|
||||
@project_router.get("/item", response_model=ProjectItem)
|
||||
async def get_project(
|
||||
project_service: ProjectServiceDep,
|
||||
project: ProjectPathDep,
|
||||
) -> ProjectItem:
|
||||
"""Get bassic info about the specified Basic Memory project."""
|
||||
found_project = await project_service.get_project(project)
|
||||
if not found_project:
|
||||
raise HTTPException(
|
||||
status_code=404, detail=f"Project: '{project}' does not exist"
|
||||
) # pragma: no cover
|
||||
|
||||
return ProjectItem(
|
||||
id=found_project.id,
|
||||
external_id=found_project.external_id,
|
||||
name=found_project.name,
|
||||
path=normalize_project_path(found_project.path),
|
||||
is_default=found_project.is_default or False,
|
||||
)
|
||||
|
||||
|
||||
# Update a project
|
||||
@project_router.patch("/{name}", response_model=ProjectStatusResponse)
|
||||
async def update_project(
|
||||
project_service: ProjectServiceDep,
|
||||
name: str = Path(..., description="Name of the project to update"),
|
||||
path: Optional[str] = Body(None, description="New absolute path for the project"),
|
||||
is_active: Optional[bool] = Body(None, description="Status of the project (active/inactive)"),
|
||||
) -> ProjectStatusResponse:
|
||||
"""Update a project's information in configuration and database.
|
||||
|
||||
Args:
|
||||
name: The name of the project to update
|
||||
path: Optional new absolute path for the project
|
||||
is_active: Optional status update for the project
|
||||
|
||||
Returns:
|
||||
Response confirming the project was updated
|
||||
"""
|
||||
try:
|
||||
# Validate that path is absolute if provided
|
||||
if path and not os.path.isabs(path):
|
||||
raise HTTPException(status_code=400, detail="Path must be absolute")
|
||||
|
||||
# Get original project info for the response
|
||||
old_project = await project_service.get_project(name)
|
||||
if not old_project:
|
||||
raise HTTPException(
|
||||
status_code=400, detail=f"Project '{name}' not found in configuration"
|
||||
)
|
||||
|
||||
old_project_info = ProjectItem(
|
||||
id=old_project.id,
|
||||
external_id=old_project.external_id,
|
||||
name=old_project.name,
|
||||
path=old_project.path,
|
||||
is_default=old_project.is_default or False,
|
||||
)
|
||||
|
||||
if path:
|
||||
await project_service.move_project(name, path)
|
||||
elif is_active is not None:
|
||||
await project_service.update_project(name, is_active=is_active)
|
||||
|
||||
# Get updated project info
|
||||
updated_project = await project_service.get_project(name)
|
||||
if not updated_project:
|
||||
raise HTTPException( # pragma: no cover
|
||||
status_code=404, detail=f"Project '{name}' not found after update"
|
||||
)
|
||||
|
||||
return ProjectStatusResponse(
|
||||
message=f"Project '{name}' updated successfully",
|
||||
status="success",
|
||||
default=(name == project_service.default_project),
|
||||
old_project=old_project_info,
|
||||
new_project=ProjectItem(
|
||||
id=updated_project.id,
|
||||
external_id=updated_project.external_id,
|
||||
name=updated_project.name,
|
||||
path=updated_project.path,
|
||||
is_default=updated_project.is_default or False,
|
||||
),
|
||||
)
|
||||
except ValueError as e:
|
||||
raise HTTPException(status_code=400, detail=str(e)) # pragma: no cover
|
||||
|
||||
|
||||
# Sync project filesystem
|
||||
@project_router.post("/sync")
|
||||
async def sync_project(
|
||||
background_tasks: BackgroundTasks,
|
||||
sync_service: SyncServiceDep,
|
||||
project_config: ProjectConfigDep,
|
||||
force_full: bool = Query(
|
||||
False, description="Force full scan, bypassing watermark optimization"
|
||||
),
|
||||
run_in_background: bool = Query(True, description="Run in background"),
|
||||
):
|
||||
"""Force project filesystem sync to database.
|
||||
|
||||
Scans the project directory and updates the database with any new or modified files.
|
||||
|
||||
Args:
|
||||
background_tasks: FastAPI background tasks
|
||||
sync_service: Sync service for this project
|
||||
project_config: Project configuration
|
||||
force_full: If True, force a full scan even if watermark exists
|
||||
run_in_background: If True, run sync in background and return immediately
|
||||
|
||||
Returns:
|
||||
Response confirming sync was initiated (background) or SyncReportResponse (foreground)
|
||||
"""
|
||||
if run_in_background:
|
||||
background_tasks.add_task(
|
||||
sync_service.sync, project_config.home, project_config.name, force_full=force_full
|
||||
)
|
||||
logger.info(
|
||||
f"Filesystem sync initiated for project: {project_config.name} (force_full={force_full})"
|
||||
)
|
||||
|
||||
return {
|
||||
"status": "sync_started",
|
||||
"message": f"Filesystem sync initiated for project '{project_config.name}'",
|
||||
}
|
||||
else:
|
||||
report = await sync_service.sync(
|
||||
project_config.home, project_config.name, force_full=force_full
|
||||
)
|
||||
logger.info(
|
||||
f"Filesystem sync completed for project: {project_config.name} (force_full={force_full})"
|
||||
)
|
||||
return SyncReportResponse.from_sync_report(report)
|
||||
|
||||
|
||||
@project_router.post("/status", response_model=SyncReportResponse)
|
||||
async def project_sync_status(
|
||||
sync_service: SyncServiceDep,
|
||||
project_config: ProjectConfigDep,
|
||||
) -> SyncReportResponse:
|
||||
"""Scan directory for changes compared to database state.
|
||||
|
||||
Args:
|
||||
sync_service: Sync service for this project
|
||||
project_config: Project configuration
|
||||
|
||||
Returns:
|
||||
Scan report with details on files that need syncing
|
||||
"""
|
||||
logger.info(f"Scanning filesystem for project: {project_config.name}") # pragma: no cover
|
||||
sync_report = await sync_service.scan(project_config.home) # pragma: no cover
|
||||
|
||||
return SyncReportResponse.from_sync_report(sync_report) # pragma: no cover
|
||||
|
||||
|
||||
# List all available projects
|
||||
@project_resource_router.get("/projects", response_model=ProjectList)
|
||||
async def list_projects(
|
||||
project_service: ProjectServiceDep,
|
||||
) -> ProjectList:
|
||||
"""List all configured projects.
|
||||
|
||||
Returns:
|
||||
A list of all projects with metadata
|
||||
"""
|
||||
projects = await project_service.list_projects()
|
||||
default_project = project_service.default_project
|
||||
|
||||
project_items = [
|
||||
ProjectItem(
|
||||
id=project.id,
|
||||
external_id=project.external_id,
|
||||
name=project.name,
|
||||
path=normalize_project_path(project.path),
|
||||
is_default=project.is_default or False,
|
||||
)
|
||||
for project in projects
|
||||
]
|
||||
|
||||
return ProjectList(
|
||||
projects=project_items,
|
||||
default_project=default_project,
|
||||
)
|
||||
|
||||
|
||||
# Add a new project
|
||||
@project_resource_router.post("/projects", response_model=ProjectStatusResponse, status_code=201)
|
||||
async def add_project(
|
||||
response: Response,
|
||||
project_data: ProjectInfoRequest,
|
||||
project_service: ProjectServiceDep,
|
||||
) -> ProjectStatusResponse:
|
||||
"""Add a new project to configuration and database.
|
||||
|
||||
Args:
|
||||
project_data: The project name and path, with option to set as default
|
||||
|
||||
Returns:
|
||||
Response confirming the project was added
|
||||
"""
|
||||
# Check if project already exists before attempting to add
|
||||
existing_project = await project_service.get_project(project_data.name)
|
||||
if existing_project:
|
||||
# Project exists - check if paths match for true idempotency
|
||||
# Normalize paths for comparison (resolve symlinks, etc.)
|
||||
from pathlib import Path
|
||||
|
||||
requested_path = Path(project_data.path).resolve()
|
||||
existing_path = Path(existing_project.path).resolve()
|
||||
|
||||
if requested_path == existing_path:
|
||||
# Same name, same path - return 200 OK (idempotent)
|
||||
response.status_code = 200
|
||||
return ProjectStatusResponse( # pyright: ignore [reportCallIssue]
|
||||
message=f"Project '{project_data.name}' already exists",
|
||||
status="success",
|
||||
default=existing_project.is_default or False,
|
||||
new_project=ProjectItem(
|
||||
id=existing_project.id,
|
||||
external_id=existing_project.external_id,
|
||||
name=existing_project.name,
|
||||
path=existing_project.path,
|
||||
is_default=existing_project.is_default or False,
|
||||
),
|
||||
)
|
||||
else:
|
||||
# Same name, different path - this is an error
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"Project '{project_data.name}' already exists with different path. Existing: {existing_project.path}, Requested: {project_data.path}",
|
||||
)
|
||||
|
||||
try: # pragma: no cover
|
||||
# The service layer now handles cloud mode validation and path sanitization
|
||||
await project_service.add_project(
|
||||
project_data.name, project_data.path, set_default=project_data.set_default
|
||||
)
|
||||
|
||||
# Fetch the newly created project to get its ID
|
||||
new_project = await project_service.get_project(project_data.name)
|
||||
if not new_project:
|
||||
raise HTTPException(status_code=500, detail="Failed to retrieve newly created project")
|
||||
|
||||
return ProjectStatusResponse( # pyright: ignore [reportCallIssue]
|
||||
message=f"Project '{new_project.name}' added successfully",
|
||||
status="success",
|
||||
default=project_data.set_default,
|
||||
new_project=ProjectItem(
|
||||
id=new_project.id,
|
||||
external_id=new_project.external_id,
|
||||
name=new_project.name,
|
||||
path=new_project.path,
|
||||
is_default=new_project.is_default or False,
|
||||
),
|
||||
)
|
||||
except ValueError as e: # pragma: no cover
|
||||
raise HTTPException(status_code=400, detail=str(e))
|
||||
|
||||
|
||||
# Remove a project
|
||||
@project_resource_router.delete("/{name}", response_model=ProjectStatusResponse)
|
||||
async def remove_project(
|
||||
project_service: ProjectServiceDep,
|
||||
name: str = Path(..., description="Name of the project to remove"),
|
||||
delete_notes: bool = Query(
|
||||
False, description="If True, delete project directory from filesystem"
|
||||
),
|
||||
) -> ProjectStatusResponse:
|
||||
"""Remove a project from configuration and database.
|
||||
|
||||
Args:
|
||||
name: The name of the project to remove
|
||||
delete_notes: If True, delete the project directory from the filesystem
|
||||
|
||||
Returns:
|
||||
Response confirming the project was removed
|
||||
"""
|
||||
try:
|
||||
old_project = await project_service.get_project(name)
|
||||
if not old_project: # pragma: no cover
|
||||
raise HTTPException(
|
||||
status_code=404, detail=f"Project: '{name}' does not exist"
|
||||
) # pragma: no cover
|
||||
|
||||
# Check if trying to delete the default project
|
||||
# In cloud mode, database is source of truth; in local mode, check config
|
||||
config_default = project_service.default_project
|
||||
db_default = await project_service.repository.get_default_project()
|
||||
|
||||
# Use database default if available, otherwise fall back to config default
|
||||
default_project_name = db_default.name if db_default else config_default
|
||||
|
||||
if name == default_project_name:
|
||||
available_projects = await project_service.list_projects()
|
||||
other_projects = [p.name for p in available_projects if p.name != name]
|
||||
detail = f"Cannot delete default project '{name}'. "
|
||||
if other_projects:
|
||||
detail += (
|
||||
f"Set another project as default first. Available: {', '.join(other_projects)}"
|
||||
)
|
||||
else:
|
||||
detail += "This is the only project in your configuration."
|
||||
raise HTTPException(status_code=400, detail=detail)
|
||||
|
||||
await project_service.remove_project(name, delete_notes=delete_notes)
|
||||
|
||||
return ProjectStatusResponse(
|
||||
message=f"Project '{old_project.name}' removed successfully",
|
||||
status="success",
|
||||
default=False,
|
||||
old_project=ProjectItem(
|
||||
id=old_project.id,
|
||||
external_id=old_project.external_id,
|
||||
name=old_project.name,
|
||||
path=old_project.path,
|
||||
is_default=old_project.is_default or False,
|
||||
),
|
||||
new_project=None,
|
||||
)
|
||||
except ValueError as e: # pragma: no cover
|
||||
raise HTTPException(status_code=400, detail=str(e))
|
||||
|
||||
|
||||
# Set a project as default
|
||||
@project_resource_router.put("/{name}/default", response_model=ProjectStatusResponse)
|
||||
async def set_default_project(
|
||||
project_service: ProjectServiceDep,
|
||||
name: str = Path(..., description="Name of the project to set as default"),
|
||||
) -> ProjectStatusResponse:
|
||||
"""Set a project as the default project.
|
||||
|
||||
Args:
|
||||
name: The name of the project to set as default
|
||||
|
||||
Returns:
|
||||
Response confirming the project was set as default
|
||||
"""
|
||||
try:
|
||||
# Get the old default project
|
||||
default_name = project_service.default_project
|
||||
default_project = await project_service.get_project(default_name)
|
||||
if not default_project: # pragma: no cover
|
||||
raise HTTPException( # pragma: no cover
|
||||
status_code=404, detail=f"Default Project: '{default_name}' does not exist"
|
||||
)
|
||||
|
||||
# get the new project
|
||||
new_default_project = await project_service.get_project(name)
|
||||
if not new_default_project: # pragma: no cover
|
||||
raise HTTPException(
|
||||
status_code=404, detail=f"Project: '{name}' does not exist"
|
||||
) # pragma: no cover
|
||||
|
||||
await project_service.set_default_project(name)
|
||||
|
||||
return ProjectStatusResponse(
|
||||
message=f"Project '{name}' set as default successfully",
|
||||
status="success",
|
||||
default=True,
|
||||
old_project=ProjectItem(
|
||||
id=default_project.id,
|
||||
external_id=default_project.external_id,
|
||||
name=default_name,
|
||||
path=default_project.path,
|
||||
is_default=False,
|
||||
),
|
||||
new_project=ProjectItem(
|
||||
id=new_default_project.id,
|
||||
external_id=new_default_project.external_id,
|
||||
name=name,
|
||||
path=new_default_project.path,
|
||||
is_default=True,
|
||||
),
|
||||
)
|
||||
except ValueError as e: # pragma: no cover
|
||||
raise HTTPException(status_code=400, detail=str(e))
|
||||
|
||||
|
||||
# Get the default project
|
||||
@project_resource_router.get("/default", response_model=ProjectItem)
|
||||
async def get_default_project(
|
||||
project_service: ProjectServiceDep,
|
||||
) -> ProjectItem:
|
||||
"""Get the default project.
|
||||
|
||||
Returns:
|
||||
Response with project default information
|
||||
"""
|
||||
# Get the default project
|
||||
# In cloud mode, database is source of truth; in local mode, check config
|
||||
config_default = project_service.default_project
|
||||
db_default = await project_service.repository.get_default_project()
|
||||
|
||||
# Use database default if available, otherwise fall back to config default
|
||||
default_name = db_default.name if db_default else config_default
|
||||
default_project = await project_service.get_project(default_name)
|
||||
if not default_project: # pragma: no cover
|
||||
raise HTTPException( # pragma: no cover
|
||||
status_code=404, detail=f"Default Project: '{default_name}' does not exist"
|
||||
)
|
||||
|
||||
return ProjectItem(
|
||||
id=default_project.id,
|
||||
external_id=default_project.external_id,
|
||||
name=default_project.name,
|
||||
path=default_project.path,
|
||||
is_default=True,
|
||||
)
|
||||
|
||||
|
||||
# Synchronize projects between config and database
|
||||
@project_resource_router.post("/config/sync", response_model=ProjectStatusResponse)
|
||||
async def synchronize_projects(
|
||||
project_service: ProjectServiceDep,
|
||||
) -> ProjectStatusResponse:
|
||||
"""Synchronize projects between configuration file and database.
|
||||
|
||||
Ensures that all projects in the configuration file exist in the database
|
||||
and vice versa.
|
||||
|
||||
Returns:
|
||||
Response confirming synchronization was completed
|
||||
"""
|
||||
try: # pragma: no cover
|
||||
await project_service.synchronize_projects()
|
||||
|
||||
return ProjectStatusResponse( # pyright: ignore [reportCallIssue]
|
||||
message="Projects synchronized successfully between configuration and database",
|
||||
status="success",
|
||||
default=False,
|
||||
)
|
||||
except ValueError as e: # pragma: no cover
|
||||
raise HTTPException(status_code=400, detail=str(e))
|
||||
@@ -1,260 +0,0 @@
|
||||
"""Router for prompt-related operations.
|
||||
|
||||
This router is responsible for rendering various prompts using Handlebars templates.
|
||||
It centralizes all prompt formatting logic that was previously in the MCP prompts.
|
||||
"""
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from fastapi import APIRouter, HTTPException, status
|
||||
from loguru import logger
|
||||
|
||||
from basic_memory.api.routers.utils import to_graph_context, to_search_results
|
||||
from basic_memory.api.template_loader import template_loader
|
||||
from basic_memory.schemas.base import parse_timeframe
|
||||
from basic_memory.deps import (
|
||||
ContextServiceDep,
|
||||
EntityRepositoryDep,
|
||||
SearchServiceDep,
|
||||
EntityServiceDep,
|
||||
)
|
||||
from basic_memory.schemas.prompt import (
|
||||
ContinueConversationRequest,
|
||||
SearchPromptRequest,
|
||||
PromptResponse,
|
||||
PromptMetadata,
|
||||
)
|
||||
from basic_memory.schemas.search import SearchItemType, SearchQuery
|
||||
|
||||
router = APIRouter(prefix="/prompt", tags=["prompt"])
|
||||
|
||||
|
||||
@router.post("/continue-conversation", response_model=PromptResponse)
|
||||
async def continue_conversation(
|
||||
search_service: SearchServiceDep,
|
||||
entity_service: EntityServiceDep,
|
||||
context_service: ContextServiceDep,
|
||||
entity_repository: EntityRepositoryDep,
|
||||
request: ContinueConversationRequest,
|
||||
) -> PromptResponse:
|
||||
"""Generate a prompt for continuing a conversation.
|
||||
|
||||
This endpoint takes a topic and/or timeframe and generates a prompt with
|
||||
relevant context from the knowledge base.
|
||||
|
||||
Args:
|
||||
request: The request parameters
|
||||
|
||||
Returns:
|
||||
Formatted continuation prompt with context
|
||||
"""
|
||||
logger.info(
|
||||
f"Generating continue conversation prompt, topic: {request.topic}, timeframe: {request.timeframe}"
|
||||
)
|
||||
|
||||
since = parse_timeframe(request.timeframe) if request.timeframe else None
|
||||
|
||||
# Initialize search results
|
||||
search_results = []
|
||||
|
||||
# Get data needed for template
|
||||
if request.topic:
|
||||
query = SearchQuery(text=request.topic, after_date=request.timeframe)
|
||||
results = await search_service.search(query, limit=request.search_items_limit)
|
||||
search_results = await to_search_results(entity_service, results)
|
||||
|
||||
# Build context from results
|
||||
all_hierarchical_results = []
|
||||
for result in search_results:
|
||||
if hasattr(result, "permalink") and result.permalink:
|
||||
# Get hierarchical context using the new dataclass-based approach
|
||||
context_result = await context_service.build_context(
|
||||
result.permalink,
|
||||
depth=request.depth,
|
||||
since=since,
|
||||
max_related=request.related_items_limit,
|
||||
include_observations=True, # Include observations for entities
|
||||
)
|
||||
|
||||
# Process results into the schema format
|
||||
graph_context = await to_graph_context(
|
||||
context_result, entity_repository=entity_repository
|
||||
)
|
||||
|
||||
# Add results to our collection (limit to top results for each permalink)
|
||||
if graph_context.results:
|
||||
all_hierarchical_results.extend(graph_context.results[:3])
|
||||
|
||||
# Limit to a reasonable number of total results
|
||||
all_hierarchical_results = all_hierarchical_results[:10]
|
||||
|
||||
template_context = {
|
||||
"topic": request.topic,
|
||||
"timeframe": request.timeframe,
|
||||
"hierarchical_results": all_hierarchical_results,
|
||||
"has_results": len(all_hierarchical_results) > 0,
|
||||
}
|
||||
else:
|
||||
# If no topic, get recent activity
|
||||
context_result = await context_service.build_context(
|
||||
types=[SearchItemType.ENTITY],
|
||||
depth=request.depth,
|
||||
since=since,
|
||||
max_related=request.related_items_limit,
|
||||
include_observations=True,
|
||||
)
|
||||
recent_context = await to_graph_context(context_result, entity_repository=entity_repository)
|
||||
|
||||
hierarchical_results = recent_context.results[:5] # Limit to top 5 recent items
|
||||
|
||||
template_context = {
|
||||
"topic": f"Recent Activity from ({request.timeframe})",
|
||||
"timeframe": request.timeframe,
|
||||
"hierarchical_results": hierarchical_results,
|
||||
"has_results": len(hierarchical_results) > 0,
|
||||
}
|
||||
|
||||
try:
|
||||
# Render template
|
||||
rendered_prompt = await template_loader.render(
|
||||
"prompts/continue_conversation.hbs", template_context
|
||||
)
|
||||
|
||||
# Calculate metadata
|
||||
# Count items of different types
|
||||
observation_count = 0
|
||||
relation_count = 0
|
||||
entity_count = 0
|
||||
|
||||
# Get the hierarchical results from the template context
|
||||
hierarchical_results_for_count = template_context.get("hierarchical_results", [])
|
||||
|
||||
# For topic-based search
|
||||
if request.topic:
|
||||
for item in hierarchical_results_for_count:
|
||||
if hasattr(item, "observations"):
|
||||
observation_count += len(item.observations) if item.observations else 0
|
||||
|
||||
if hasattr(item, "related_results"):
|
||||
for related in item.related_results or []:
|
||||
if hasattr(related, "type"):
|
||||
if related.type == "relation":
|
||||
relation_count += 1
|
||||
elif related.type == "entity": # pragma: no cover
|
||||
entity_count += 1 # pragma: no cover
|
||||
# For recent activity
|
||||
else:
|
||||
for item in hierarchical_results_for_count:
|
||||
if hasattr(item, "observations"):
|
||||
observation_count += len(item.observations) if item.observations else 0
|
||||
|
||||
if hasattr(item, "related_results"):
|
||||
for related in item.related_results or []:
|
||||
if hasattr(related, "type"):
|
||||
if related.type == "relation":
|
||||
relation_count += 1
|
||||
elif related.type == "entity": # pragma: no cover
|
||||
entity_count += 1 # pragma: no cover
|
||||
|
||||
# Build metadata
|
||||
metadata = {
|
||||
"query": request.topic,
|
||||
"timeframe": request.timeframe,
|
||||
"search_count": len(search_results)
|
||||
if request.topic
|
||||
else 0, # Original search results count
|
||||
"context_count": len(hierarchical_results_for_count),
|
||||
"observation_count": observation_count,
|
||||
"relation_count": relation_count,
|
||||
"total_items": (
|
||||
len(hierarchical_results_for_count)
|
||||
+ observation_count
|
||||
+ relation_count
|
||||
+ entity_count
|
||||
),
|
||||
"search_limit": request.search_items_limit,
|
||||
"context_depth": request.depth,
|
||||
"related_limit": request.related_items_limit,
|
||||
"generated_at": datetime.now(timezone.utc).isoformat(),
|
||||
}
|
||||
|
||||
prompt_metadata = PromptMetadata(**metadata)
|
||||
|
||||
return PromptResponse(
|
||||
prompt=rendered_prompt, context=template_context, metadata=prompt_metadata
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Error rendering continue conversation template: {e}")
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail=f"Error rendering prompt template: {str(e)}",
|
||||
)
|
||||
|
||||
|
||||
@router.post("/search", response_model=PromptResponse)
|
||||
async def search_prompt(
|
||||
search_service: SearchServiceDep,
|
||||
entity_service: EntityServiceDep,
|
||||
request: SearchPromptRequest,
|
||||
page: int = 1,
|
||||
page_size: int = 10,
|
||||
) -> PromptResponse:
|
||||
"""Generate a prompt for search results.
|
||||
|
||||
This endpoint takes a search query and formats the results into a helpful
|
||||
prompt with context and suggestions.
|
||||
|
||||
Args:
|
||||
request: The search parameters
|
||||
page: The page number for pagination
|
||||
page_size: The number of results per page, defaults to 10
|
||||
|
||||
Returns:
|
||||
Formatted search results prompt with context
|
||||
"""
|
||||
logger.info(f"Generating search prompt, query: {request.query}, timeframe: {request.timeframe}")
|
||||
|
||||
limit = page_size
|
||||
offset = (page - 1) * page_size
|
||||
|
||||
query = SearchQuery(text=request.query, after_date=request.timeframe)
|
||||
results = await search_service.search(query, limit=limit, offset=offset)
|
||||
search_results = await to_search_results(entity_service, results)
|
||||
|
||||
template_context = {
|
||||
"query": request.query,
|
||||
"timeframe": request.timeframe,
|
||||
"results": search_results,
|
||||
"has_results": len(search_results) > 0,
|
||||
"result_count": len(search_results),
|
||||
}
|
||||
|
||||
try:
|
||||
# Render template
|
||||
rendered_prompt = await template_loader.render("prompts/search.hbs", template_context)
|
||||
|
||||
# Build metadata
|
||||
metadata = {
|
||||
"query": request.query,
|
||||
"timeframe": request.timeframe,
|
||||
"search_count": len(search_results),
|
||||
"context_count": len(search_results),
|
||||
"observation_count": 0, # Search results don't include observations
|
||||
"relation_count": 0, # Search results don't include relations
|
||||
"total_items": len(search_results),
|
||||
"search_limit": limit,
|
||||
"context_depth": 0, # No context depth for basic search
|
||||
"related_limit": 0, # No related items for basic search
|
||||
"generated_at": datetime.now(timezone.utc).isoformat(),
|
||||
}
|
||||
|
||||
prompt_metadata = PromptMetadata(**metadata)
|
||||
|
||||
return PromptResponse(
|
||||
prompt=rendered_prompt, context=template_context, metadata=prompt_metadata
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Error rendering search template: {e}")
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail=f"Error rendering prompt template: {str(e)}",
|
||||
)
|
||||
@@ -1,252 +0,0 @@
|
||||
"""Routes for getting entity content."""
|
||||
|
||||
import tempfile
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from typing import Annotated, Union
|
||||
|
||||
from fastapi import APIRouter, HTTPException, BackgroundTasks, Body, Response
|
||||
from fastapi.responses import FileResponse, JSONResponse
|
||||
from loguru import logger
|
||||
|
||||
from basic_memory.deps import (
|
||||
ProjectConfigDep,
|
||||
LinkResolverDep,
|
||||
SearchServiceDep,
|
||||
EntityServiceDep,
|
||||
FileServiceDep,
|
||||
EntityRepositoryDep,
|
||||
)
|
||||
from basic_memory.repository.search_repository import SearchIndexRow
|
||||
from basic_memory.schemas.memory import normalize_memory_url
|
||||
from basic_memory.schemas.search import SearchQuery, SearchItemType
|
||||
from basic_memory.models.knowledge import Entity as EntityModel
|
||||
from datetime import datetime
|
||||
|
||||
router = APIRouter(prefix="/resource", tags=["resources"])
|
||||
|
||||
|
||||
def _mtime_to_datetime(entity: EntityModel) -> datetime:
|
||||
"""Convert entity mtime (file modification time) to datetime.
|
||||
|
||||
Returns the file's actual modification time, falling back to updated_at
|
||||
if mtime is not available.
|
||||
"""
|
||||
if entity.mtime: # pragma: no cover
|
||||
return datetime.fromtimestamp(entity.mtime).astimezone() # pragma: no cover
|
||||
return entity.updated_at
|
||||
|
||||
|
||||
def get_entity_ids(item: SearchIndexRow) -> set[int]:
|
||||
match item.type:
|
||||
case SearchItemType.ENTITY:
|
||||
return {item.id}
|
||||
case SearchItemType.OBSERVATION:
|
||||
return {item.entity_id} # pyright: ignore [reportReturnType]
|
||||
case SearchItemType.RELATION:
|
||||
from_entity = item.from_id
|
||||
to_entity = item.to_id # pyright: ignore [reportReturnType]
|
||||
return {from_entity, to_entity} if to_entity else {from_entity} # pyright: ignore [reportReturnType]
|
||||
case _: # pragma: no cover
|
||||
raise ValueError(f"Unexpected type: {item.type}")
|
||||
|
||||
|
||||
@router.get("/{identifier:path}", response_model=None)
|
||||
async def get_resource_content(
|
||||
config: ProjectConfigDep,
|
||||
link_resolver: LinkResolverDep,
|
||||
search_service: SearchServiceDep,
|
||||
entity_service: EntityServiceDep,
|
||||
file_service: FileServiceDep,
|
||||
background_tasks: BackgroundTasks,
|
||||
identifier: str,
|
||||
page: int = 1,
|
||||
page_size: int = 10,
|
||||
) -> Union[Response, FileResponse]:
|
||||
"""Get resource content by identifier: name or permalink."""
|
||||
logger.debug(f"Getting content for: {identifier}")
|
||||
|
||||
# Find single entity by permalink
|
||||
entity = await link_resolver.resolve_link(identifier)
|
||||
results = [entity] if entity else []
|
||||
|
||||
# pagination for multiple results
|
||||
limit = page_size
|
||||
offset = (page - 1) * page_size
|
||||
|
||||
# search using the identifier as a permalink
|
||||
if not results:
|
||||
# if the identifier contains a wildcard, use GLOB search
|
||||
query = (
|
||||
SearchQuery(permalink_match=identifier)
|
||||
if "*" in identifier
|
||||
else SearchQuery(permalink=identifier)
|
||||
)
|
||||
search_results = await search_service.search(query, limit, offset)
|
||||
if not search_results:
|
||||
raise HTTPException(status_code=404, detail=f"Resource not found: {identifier}")
|
||||
|
||||
# get the deduplicated entities related to the search results
|
||||
entity_ids = {id for result in search_results for id in get_entity_ids(result)}
|
||||
results = await entity_service.get_entities_by_id(list(entity_ids))
|
||||
|
||||
# return single response
|
||||
if len(results) == 1:
|
||||
entity = results[0]
|
||||
# Check file exists via file_service (for cloud compatibility)
|
||||
if not await file_service.exists(entity.file_path):
|
||||
raise HTTPException(
|
||||
status_code=404,
|
||||
detail=f"File not found: {entity.file_path}",
|
||||
)
|
||||
# Read content via file_service as bytes (works with both local and S3)
|
||||
content = await file_service.read_file_bytes(entity.file_path)
|
||||
content_type = file_service.content_type(entity.file_path)
|
||||
return Response(content=content, media_type=content_type)
|
||||
|
||||
# for multiple files, initialize a temporary file for writing the results
|
||||
with tempfile.NamedTemporaryFile(delete=False, mode="w", suffix=".md") as tmp_file:
|
||||
temp_file_path = tmp_file.name
|
||||
|
||||
for result in results:
|
||||
# Read content for each entity
|
||||
content = await file_service.read_entity_content(result)
|
||||
memory_url = normalize_memory_url(result.permalink)
|
||||
modified_date = _mtime_to_datetime(result).isoformat()
|
||||
checksum = result.checksum[:8] if result.checksum else ""
|
||||
|
||||
# Prepare the delimited content
|
||||
response_content = f"--- {memory_url} {modified_date} {checksum}\n"
|
||||
response_content += f"\n{content}\n"
|
||||
response_content += "\n"
|
||||
|
||||
# Write content directly to the temporary file in append mode
|
||||
tmp_file.write(response_content)
|
||||
|
||||
# Ensure all content is written to disk
|
||||
tmp_file.flush()
|
||||
|
||||
# Schedule the temporary file to be deleted after the response
|
||||
background_tasks.add_task(cleanup_temp_file, temp_file_path)
|
||||
|
||||
# Return the file response
|
||||
return FileResponse(path=temp_file_path)
|
||||
|
||||
|
||||
def cleanup_temp_file(file_path: str):
|
||||
"""Delete the temporary file."""
|
||||
try:
|
||||
Path(file_path).unlink() # Deletes the file
|
||||
logger.debug(f"Temporary file deleted: {file_path}")
|
||||
except Exception as e: # pragma: no cover
|
||||
logger.error(f"Error deleting temporary file {file_path}: {e}")
|
||||
|
||||
|
||||
@router.put("/{file_path:path}")
|
||||
async def write_resource(
|
||||
config: ProjectConfigDep,
|
||||
file_service: FileServiceDep,
|
||||
entity_repository: EntityRepositoryDep,
|
||||
search_service: SearchServiceDep,
|
||||
file_path: str,
|
||||
content: Annotated[str, Body()],
|
||||
) -> JSONResponse:
|
||||
"""Write content to a file in the project.
|
||||
|
||||
This endpoint allows writing content directly to a file in the project.
|
||||
Also creates an entity record and indexes the file for search.
|
||||
|
||||
Args:
|
||||
file_path: Path to write to, relative to project root
|
||||
request: Contains the content to write
|
||||
|
||||
Returns:
|
||||
JSON response with file information
|
||||
"""
|
||||
try:
|
||||
# Get content from request body
|
||||
|
||||
# Defensive type checking: ensure content is a string
|
||||
# FastAPI should validate this, but if a dict somehow gets through
|
||||
# (e.g., via JSON body parsing), we need to catch it here
|
||||
if isinstance(content, dict):
|
||||
logger.error( # pragma: no cover
|
||||
f"Error writing resource {file_path}: "
|
||||
f"content is a dict, expected string. Keys: {list(content.keys())}"
|
||||
)
|
||||
raise HTTPException( # pragma: no cover
|
||||
status_code=400,
|
||||
detail="content must be a string, not a dict. "
|
||||
"Ensure request body is sent as raw string content, not JSON object.",
|
||||
)
|
||||
|
||||
# Ensure it's UTF-8 string content
|
||||
if isinstance(content, bytes): # pragma: no cover
|
||||
content_str = content.decode("utf-8")
|
||||
else:
|
||||
content_str = str(content)
|
||||
|
||||
# Cloud compatibility: do not assume a local filesystem path structure.
|
||||
# Delegate directory creation + writes to the configured FileService (local or S3).
|
||||
await file_service.ensure_directory(Path(file_path).parent)
|
||||
checksum = await file_service.write_file(file_path, content_str)
|
||||
|
||||
# Get file info
|
||||
file_metadata = await file_service.get_file_metadata(file_path)
|
||||
|
||||
# Determine file details
|
||||
file_name = Path(file_path).name
|
||||
content_type = file_service.content_type(file_path)
|
||||
|
||||
entity_type = "canvas" if file_path.endswith(".canvas") else "file"
|
||||
|
||||
# Check if entity already exists
|
||||
existing_entity = await entity_repository.get_by_file_path(file_path)
|
||||
|
||||
if existing_entity:
|
||||
# Update existing entity
|
||||
entity = await entity_repository.update(
|
||||
existing_entity.id,
|
||||
{
|
||||
"title": file_name,
|
||||
"entity_type": entity_type,
|
||||
"content_type": content_type,
|
||||
"file_path": file_path,
|
||||
"checksum": checksum,
|
||||
"updated_at": file_metadata.modified_at,
|
||||
},
|
||||
)
|
||||
status_code = 200
|
||||
else:
|
||||
# Create a new entity model
|
||||
# Explicitly set external_id to ensure NOT NULL constraint is satisfied (fixes #512)
|
||||
entity = EntityModel(
|
||||
external_id=str(uuid.uuid4()),
|
||||
title=file_name,
|
||||
entity_type=entity_type,
|
||||
content_type=content_type,
|
||||
file_path=file_path,
|
||||
checksum=checksum,
|
||||
created_at=file_metadata.created_at,
|
||||
updated_at=file_metadata.modified_at,
|
||||
)
|
||||
entity = await entity_repository.add(entity)
|
||||
status_code = 201
|
||||
|
||||
# Index the file for search
|
||||
await search_service.index_entity(entity) # pyright: ignore
|
||||
|
||||
# Return success response
|
||||
return JSONResponse(
|
||||
status_code=status_code,
|
||||
content={
|
||||
"file_path": file_path,
|
||||
"checksum": checksum,
|
||||
"size": file_metadata.size,
|
||||
"created_at": file_metadata.created_at.timestamp(),
|
||||
"modified_at": file_metadata.modified_at.timestamp(),
|
||||
},
|
||||
)
|
||||
except Exception as e: # pragma: no cover
|
||||
logger.error(f"Error writing resource {file_path}: {e}")
|
||||
raise HTTPException(status_code=500, detail=f"Failed to write resource: {str(e)}")
|
||||
@@ -1,36 +0,0 @@
|
||||
"""Router for search operations."""
|
||||
|
||||
from fastapi import APIRouter, BackgroundTasks
|
||||
|
||||
from basic_memory.api.routers.utils import to_search_results
|
||||
from basic_memory.schemas.search import SearchQuery, SearchResponse
|
||||
from basic_memory.deps import SearchServiceDep, EntityServiceDep
|
||||
|
||||
router = APIRouter(prefix="/search", tags=["search"])
|
||||
|
||||
|
||||
@router.post("/", response_model=SearchResponse)
|
||||
async def search(
|
||||
query: SearchQuery,
|
||||
search_service: SearchServiceDep,
|
||||
entity_service: EntityServiceDep,
|
||||
page: int = 1,
|
||||
page_size: int = 10,
|
||||
):
|
||||
"""Search across all knowledge and documents."""
|
||||
limit = page_size
|
||||
offset = (page - 1) * page_size
|
||||
results = await search_service.search(query, limit=limit, offset=offset)
|
||||
search_results = await to_search_results(entity_service, results)
|
||||
return SearchResponse(
|
||||
results=search_results,
|
||||
current_page=page,
|
||||
page_size=page_size,
|
||||
)
|
||||
|
||||
|
||||
@router.post("/reindex")
|
||||
async def reindex(background_tasks: BackgroundTasks, search_service: SearchServiceDep):
|
||||
"""Recreate and populate the search index."""
|
||||
await search_service.reindex_all(background_tasks=background_tasks)
|
||||
return {"status": "ok", "message": "Reindex initiated"}
|
||||
@@ -1,169 +0,0 @@
|
||||
from typing import Optional, List
|
||||
|
||||
from basic_memory.repository import EntityRepository
|
||||
from basic_memory.repository.search_repository import SearchIndexRow
|
||||
from basic_memory.schemas.memory import (
|
||||
EntitySummary,
|
||||
ObservationSummary,
|
||||
RelationSummary,
|
||||
MemoryMetadata,
|
||||
GraphContext,
|
||||
ContextResult,
|
||||
)
|
||||
from basic_memory.schemas.search import SearchItemType, SearchResult
|
||||
from basic_memory.services import EntityService
|
||||
from basic_memory.services.context_service import (
|
||||
ContextResultRow,
|
||||
ContextResult as ServiceContextResult,
|
||||
)
|
||||
|
||||
|
||||
async def to_graph_context(
|
||||
context_result: ServiceContextResult,
|
||||
entity_repository: EntityRepository,
|
||||
page: Optional[int] = None,
|
||||
page_size: Optional[int] = None,
|
||||
):
|
||||
# First pass: collect all entity IDs needed for relations
|
||||
entity_ids_needed: set[int] = set()
|
||||
for context_item in context_result.results:
|
||||
for item in (
|
||||
[context_item.primary_result] + context_item.observations + context_item.related_results
|
||||
):
|
||||
if item.type == SearchItemType.RELATION:
|
||||
if item.from_id: # pyright: ignore
|
||||
entity_ids_needed.add(item.from_id) # pyright: ignore
|
||||
if item.to_id:
|
||||
entity_ids_needed.add(item.to_id)
|
||||
|
||||
# Batch fetch all entities at once
|
||||
entity_lookup: dict[int, str] = {}
|
||||
if entity_ids_needed:
|
||||
entities = await entity_repository.find_by_ids(list(entity_ids_needed))
|
||||
entity_lookup = {e.id: e.title for e in entities}
|
||||
|
||||
# Helper function to convert items to summaries
|
||||
def to_summary(item: SearchIndexRow | ContextResultRow):
|
||||
match item.type:
|
||||
case SearchItemType.ENTITY:
|
||||
return EntitySummary(
|
||||
entity_id=item.id,
|
||||
title=item.title, # pyright: ignore
|
||||
permalink=item.permalink,
|
||||
content=item.content,
|
||||
file_path=item.file_path,
|
||||
created_at=item.created_at,
|
||||
)
|
||||
case SearchItemType.OBSERVATION:
|
||||
return ObservationSummary(
|
||||
observation_id=item.id,
|
||||
entity_id=item.entity_id, # pyright: ignore
|
||||
title=item.title, # pyright: ignore
|
||||
file_path=item.file_path,
|
||||
category=item.category, # pyright: ignore
|
||||
content=item.content, # pyright: ignore
|
||||
permalink=item.permalink, # pyright: ignore
|
||||
created_at=item.created_at,
|
||||
)
|
||||
case SearchItemType.RELATION:
|
||||
from_title = entity_lookup.get(item.from_id) if item.from_id else None # pyright: ignore
|
||||
to_title = entity_lookup.get(item.to_id) if item.to_id else None
|
||||
return RelationSummary(
|
||||
relation_id=item.id,
|
||||
entity_id=item.entity_id, # pyright: ignore
|
||||
title=item.title, # pyright: ignore
|
||||
file_path=item.file_path,
|
||||
permalink=item.permalink, # pyright: ignore
|
||||
relation_type=item.relation_type, # pyright: ignore
|
||||
from_entity=from_title,
|
||||
from_entity_id=item.from_id, # pyright: ignore
|
||||
to_entity=to_title,
|
||||
to_entity_id=item.to_id,
|
||||
created_at=item.created_at,
|
||||
)
|
||||
case _: # pragma: no cover
|
||||
raise ValueError(f"Unexpected type: {item.type}")
|
||||
|
||||
# Process the hierarchical results
|
||||
hierarchical_results = []
|
||||
for context_item in context_result.results:
|
||||
# Process primary result
|
||||
primary_result = to_summary(context_item.primary_result)
|
||||
|
||||
# Process observations (always ObservationSummary, validated by context_service)
|
||||
observations = [to_summary(obs) for obs in context_item.observations]
|
||||
|
||||
# Process related results
|
||||
related = [to_summary(rel) for rel in context_item.related_results]
|
||||
|
||||
# Add to hierarchical results
|
||||
hierarchical_results.append(
|
||||
ContextResult(
|
||||
primary_result=primary_result,
|
||||
observations=observations, # pyright: ignore[reportArgumentType]
|
||||
related_results=related,
|
||||
)
|
||||
)
|
||||
|
||||
# Create schema metadata from service metadata
|
||||
metadata = MemoryMetadata(
|
||||
uri=context_result.metadata.uri,
|
||||
types=context_result.metadata.types,
|
||||
depth=context_result.metadata.depth,
|
||||
timeframe=context_result.metadata.timeframe,
|
||||
generated_at=context_result.metadata.generated_at,
|
||||
primary_count=context_result.metadata.primary_count,
|
||||
related_count=context_result.metadata.related_count,
|
||||
total_results=context_result.metadata.primary_count + context_result.metadata.related_count,
|
||||
total_relations=context_result.metadata.total_relations,
|
||||
total_observations=context_result.metadata.total_observations,
|
||||
)
|
||||
|
||||
# Return new GraphContext with just hierarchical results
|
||||
return GraphContext(
|
||||
results=hierarchical_results,
|
||||
metadata=metadata,
|
||||
page=page,
|
||||
page_size=page_size,
|
||||
)
|
||||
|
||||
|
||||
async def to_search_results(entity_service: EntityService, results: List[SearchIndexRow]):
|
||||
search_results = []
|
||||
for r in results:
|
||||
entities = await entity_service.get_entities_by_id([r.entity_id, r.from_id, r.to_id]) # pyright: ignore
|
||||
|
||||
# Determine which IDs to set based on type
|
||||
entity_id = None
|
||||
observation_id = None
|
||||
relation_id = None
|
||||
|
||||
if r.type == SearchItemType.ENTITY:
|
||||
entity_id = r.id
|
||||
elif r.type == SearchItemType.OBSERVATION:
|
||||
observation_id = r.id
|
||||
entity_id = r.entity_id # Parent entity
|
||||
elif r.type == SearchItemType.RELATION:
|
||||
relation_id = r.id
|
||||
entity_id = r.entity_id # Parent entity
|
||||
|
||||
search_results.append(
|
||||
SearchResult(
|
||||
title=r.title, # pyright: ignore
|
||||
type=r.type, # pyright: ignore
|
||||
permalink=r.permalink,
|
||||
score=r.score, # pyright: ignore
|
||||
entity=entities[0].permalink if entities else None,
|
||||
content=r.content,
|
||||
file_path=r.file_path,
|
||||
metadata=r.metadata,
|
||||
entity_id=entity_id,
|
||||
observation_id=observation_id,
|
||||
relation_id=relation_id,
|
||||
category=r.category,
|
||||
from_entity=entities[0].permalink if entities else None,
|
||||
to_entity=entities[1].permalink if len(entities) > 1 else None,
|
||||
relation_type=r.relation_type,
|
||||
)
|
||||
)
|
||||
return search_results
|
||||
@@ -8,6 +8,7 @@ from basic_memory.api.v2.routers.resource_router import router as resource_route
|
||||
from basic_memory.api.v2.routers.directory_router import router as directory_router
|
||||
from basic_memory.api.v2.routers.prompt_router import router as prompt_router
|
||||
from basic_memory.api.v2.routers.importer_router import router as importer_router
|
||||
from basic_memory.api.v2.routers.schema_router import router as schema_router
|
||||
|
||||
__all__ = [
|
||||
"knowledge_router",
|
||||
@@ -18,4 +19,5 @@ __all__ = [
|
||||
"directory_router",
|
||||
"prompt_router",
|
||||
"importer_router",
|
||||
"schema_router",
|
||||
]
|
||||
|
||||
@@ -10,6 +10,7 @@ import logging
|
||||
from fastapi import APIRouter, Form, HTTPException, UploadFile, status, Path
|
||||
|
||||
from basic_memory.deps import (
|
||||
AppConfigDep,
|
||||
ChatGPTImporterV2ExternalDep,
|
||||
ClaudeConversationsImporterV2ExternalDep,
|
||||
ClaudeProjectsImporterV2ExternalDep,
|
||||
@@ -27,19 +28,31 @@ logger = logging.getLogger(__name__)
|
||||
router = APIRouter(prefix="/import", tags=["import-v2"])
|
||||
|
||||
|
||||
async def read_import_upload(file: UploadFile, max_bytes: int) -> bytes:
|
||||
"""Read an import upload with a hard cap before JSON parsing."""
|
||||
content = await file.read(max_bytes + 1)
|
||||
if len(content) > max_bytes:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_413_CONTENT_TOO_LARGE,
|
||||
detail=f"Import file exceeds maximum size of {max_bytes} bytes.",
|
||||
)
|
||||
return content
|
||||
|
||||
|
||||
@router.post("/chatgpt", response_model=ChatImportResult)
|
||||
async def import_chatgpt(
|
||||
importer: ChatGPTImporterV2ExternalDep,
|
||||
config: AppConfigDep,
|
||||
file: UploadFile,
|
||||
project_id: str = Path(..., description="Project external UUID"),
|
||||
folder: str = Form("conversations"),
|
||||
directory: str = Form("conversations"),
|
||||
) -> ChatImportResult:
|
||||
"""Import conversations from ChatGPT JSON export.
|
||||
|
||||
Args:
|
||||
project_id: Project external UUID from URL path
|
||||
file: The ChatGPT conversations.json file.
|
||||
folder: The folder to place the files in.
|
||||
directory: The directory to place the files in.
|
||||
importer: ChatGPT importer instance.
|
||||
|
||||
Returns:
|
||||
@@ -49,22 +62,23 @@ async def import_chatgpt(
|
||||
HTTPException: If import fails.
|
||||
"""
|
||||
logger.info(f"V2 Importing ChatGPT conversations for project {project_id}")
|
||||
return await import_file(importer, file, folder)
|
||||
return await import_file(importer, file, directory, config.import_upload_max_bytes)
|
||||
|
||||
|
||||
@router.post("/claude/conversations", response_model=ChatImportResult)
|
||||
async def import_claude_conversations(
|
||||
importer: ClaudeConversationsImporterV2ExternalDep,
|
||||
config: AppConfigDep,
|
||||
file: UploadFile,
|
||||
project_id: str = Path(..., description="Project external UUID"),
|
||||
folder: str = Form("conversations"),
|
||||
directory: str = Form("conversations"),
|
||||
) -> ChatImportResult:
|
||||
"""Import conversations from Claude conversations.json export.
|
||||
|
||||
Args:
|
||||
project_id: Project external UUID from URL path
|
||||
file: The Claude conversations.json file.
|
||||
folder: The folder to place the files in.
|
||||
directory: The directory to place the files in.
|
||||
importer: Claude conversations importer instance.
|
||||
|
||||
Returns:
|
||||
@@ -74,22 +88,23 @@ async def import_claude_conversations(
|
||||
HTTPException: If import fails.
|
||||
"""
|
||||
logger.info(f"V2 Importing Claude conversations for project {project_id}")
|
||||
return await import_file(importer, file, folder)
|
||||
return await import_file(importer, file, directory, config.import_upload_max_bytes)
|
||||
|
||||
|
||||
@router.post("/claude/projects", response_model=ProjectImportResult)
|
||||
async def import_claude_projects(
|
||||
importer: ClaudeProjectsImporterV2ExternalDep,
|
||||
config: AppConfigDep,
|
||||
file: UploadFile,
|
||||
project_id: str = Path(..., description="Project external UUID"),
|
||||
folder: str = Form("projects"),
|
||||
directory: str = Form("projects"),
|
||||
) -> ProjectImportResult:
|
||||
"""Import projects from Claude projects.json export.
|
||||
|
||||
Args:
|
||||
project_id: Project external UUID from URL path
|
||||
file: The Claude projects.json file.
|
||||
folder: The base folder to place the files in.
|
||||
directory: The base directory to place the files in.
|
||||
importer: Claude projects importer instance.
|
||||
|
||||
Returns:
|
||||
@@ -99,22 +114,23 @@ async def import_claude_projects(
|
||||
HTTPException: If import fails.
|
||||
"""
|
||||
logger.info(f"V2 Importing Claude projects for project {project_id}")
|
||||
return await import_file(importer, file, folder)
|
||||
return await import_file(importer, file, directory, config.import_upload_max_bytes)
|
||||
|
||||
|
||||
@router.post("/memory-json", response_model=EntityImportResult)
|
||||
async def import_memory_json(
|
||||
importer: MemoryJsonImporterV2ExternalDep,
|
||||
config: AppConfigDep,
|
||||
file: UploadFile,
|
||||
project_id: str = Path(..., description="Project external UUID"),
|
||||
folder: str = Form("conversations"),
|
||||
directory: str = Form("conversations"),
|
||||
) -> EntityImportResult:
|
||||
"""Import entities and relations from a memory.json file.
|
||||
|
||||
Args:
|
||||
project_id: Project external UUID from URL path
|
||||
file: The memory.json file.
|
||||
folder: Optional destination folder within the project.
|
||||
directory: Optional destination directory within the project.
|
||||
importer: Memory JSON importer instance.
|
||||
|
||||
Returns:
|
||||
@@ -126,18 +142,20 @@ async def import_memory_json(
|
||||
logger.info(f"V2 Importing memory.json for project {project_id}")
|
||||
try:
|
||||
file_data = []
|
||||
file_bytes = await file.read()
|
||||
file_bytes = await read_import_upload(file, config.import_upload_max_bytes)
|
||||
file_str = file_bytes.decode("utf-8")
|
||||
for line in file_str.splitlines():
|
||||
json_data = json.loads(line)
|
||||
file_data.append(json_data)
|
||||
|
||||
result = await importer.import_data(file_data, folder)
|
||||
result = await importer.import_data(file_data, directory)
|
||||
if not result.success: # pragma: no cover
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail=result.error_message or "Import failed",
|
||||
)
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.exception("V2 Import failed")
|
||||
raise HTTPException(
|
||||
@@ -147,13 +165,16 @@ async def import_memory_json(
|
||||
return result
|
||||
|
||||
|
||||
async def import_file(importer: Importer, file: UploadFile, destination_folder: str):
|
||||
async def import_file(
|
||||
importer: Importer, file: UploadFile, destination_directory: str, max_bytes: int
|
||||
):
|
||||
"""Helper function to import a file using an importer instance.
|
||||
|
||||
Args:
|
||||
importer: The importer instance to use
|
||||
file: The file to import
|
||||
destination_folder: Destination folder for imported content
|
||||
destination_directory: Destination directory for imported content
|
||||
max_bytes: Maximum upload size in bytes; raises HTTP 413 if exceeded
|
||||
|
||||
Returns:
|
||||
Import result from the importer
|
||||
@@ -163,8 +184,9 @@ async def import_file(importer: Importer, file: UploadFile, destination_folder:
|
||||
"""
|
||||
try:
|
||||
# Process file
|
||||
json_data = json.load(file.file)
|
||||
result = await importer.import_data(json_data, destination_folder)
|
||||
upload_bytes = await read_import_upload(file, max_bytes)
|
||||
json_data = json.loads(upload_bytes)
|
||||
result = await importer.import_data(json_data, destination_directory)
|
||||
if not result.success: # pragma: no cover
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
@@ -173,6 +195,8 @@ async def import_file(importer: Importer, file: UploadFile, destination_folder:
|
||||
|
||||
return result
|
||||
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.exception("V2 Import failed")
|
||||
raise HTTPException(
|
||||
|
||||
@@ -10,18 +10,20 @@ Key improvements:
|
||||
- Simplified caching strategies
|
||||
"""
|
||||
|
||||
from fastapi import APIRouter, HTTPException, BackgroundTasks, Depends, Response, Path
|
||||
from fastapi import APIRouter, HTTPException, Response, Path
|
||||
from loguru import logger
|
||||
|
||||
import logfire
|
||||
from basic_memory.deps import (
|
||||
EntityServiceV2ExternalDep,
|
||||
SearchServiceV2ExternalDep,
|
||||
LinkResolverV2ExternalDep,
|
||||
ProjectConfigV2ExternalDep,
|
||||
AppConfigDep,
|
||||
SyncServiceV2ExternalDep,
|
||||
EntityRepositoryV2ExternalDep,
|
||||
RelationRepositoryV2ExternalDep,
|
||||
ProjectExternalIdPathDep,
|
||||
TaskSchedulerDep,
|
||||
)
|
||||
from basic_memory.schemas import DeleteEntitiesResponse
|
||||
from basic_memory.schemas.base import Entity
|
||||
@@ -30,29 +32,115 @@ from basic_memory.schemas.v2 import (
|
||||
EntityResolveRequest,
|
||||
EntityResolveResponse,
|
||||
EntityResponseV2,
|
||||
GraphEdge,
|
||||
GraphNode,
|
||||
GraphResponse,
|
||||
MoveEntityRequestV2,
|
||||
MoveDirectoryRequestV2,
|
||||
DeleteDirectoryRequestV2,
|
||||
OrphanEntitiesResponse,
|
||||
)
|
||||
from basic_memory.schemas.response import DirectoryMoveResult, DirectoryDeleteResult
|
||||
|
||||
router = APIRouter(prefix="/knowledge", tags=["knowledge-v2"])
|
||||
|
||||
|
||||
async def resolve_relations_background(sync_service, entity_id: int, entity_permalink: str) -> None:
|
||||
"""Background task to resolve relations for a specific entity.
|
||||
def _schedule_vector_sync_if_enabled(
|
||||
*,
|
||||
task_scheduler,
|
||||
app_config,
|
||||
entity_id: int,
|
||||
project_id: int,
|
||||
) -> None:
|
||||
"""Schedule out-of-band vector sync only when semantic search is enabled."""
|
||||
if app_config.semantic_search_enabled:
|
||||
task_scheduler.schedule(
|
||||
"sync_entity_vectors",
|
||||
entity_id=entity_id,
|
||||
project_id=project_id,
|
||||
)
|
||||
|
||||
This runs asynchronously after the API response is sent, preventing
|
||||
long delays when creating entities with many relations.
|
||||
|
||||
## Graph endpoint
|
||||
|
||||
|
||||
@router.get("/graph", response_model=GraphResponse)
|
||||
async def get_graph(
|
||||
project_id: ProjectExternalIdPathDep,
|
||||
entity_repository: EntityRepositoryV2ExternalDep,
|
||||
relation_repository: RelationRepositoryV2ExternalDep,
|
||||
) -> GraphResponse:
|
||||
"""Return all entities and resolved relations for knowledge graph visualization.
|
||||
|
||||
Returns a flat node/edge structure optimized for rendering with graph libraries.
|
||||
Only includes resolved relations (where to_id is not null).
|
||||
"""
|
||||
try: # pragma: no cover
|
||||
# Only resolve relations for the newly created entity
|
||||
await sync_service.resolve_relations(entity_id=entity_id) # pragma: no cover
|
||||
logger.debug( # pragma: no cover
|
||||
f"Background: Resolved relations for entity {entity_permalink} (id={entity_id})"
|
||||
)
|
||||
except Exception as e: # pragma: no cover
|
||||
# Log but don't fail - this is a background task
|
||||
logger.warning( # pragma: no cover
|
||||
f"Background: Failed to resolve relations for entity {entity_permalink}: {e}"
|
||||
)
|
||||
with logfire.span(
|
||||
"api.request.knowledge.get_graph",
|
||||
entrypoint="api",
|
||||
domain="knowledge",
|
||||
action="get_graph",
|
||||
):
|
||||
logger.info("API v2 request: get_graph")
|
||||
|
||||
# Fetch all entities for this project
|
||||
entities = await entity_repository.find_all(use_load_options=False)
|
||||
nodes = [
|
||||
GraphNode(
|
||||
external_id=entity.external_id,
|
||||
title=entity.title,
|
||||
note_type=entity.note_type,
|
||||
file_path=entity.file_path,
|
||||
)
|
||||
for entity in entities
|
||||
]
|
||||
|
||||
# Fetch all resolved relations (to_id is not null) with eager-loaded entities
|
||||
relations = await relation_repository.find_all()
|
||||
edges = [
|
||||
GraphEdge(
|
||||
from_id=relation.from_entity.external_id,
|
||||
to_id=relation.to_entity.external_id,
|
||||
relation_type=relation.relation_type,
|
||||
)
|
||||
for relation in relations
|
||||
if relation.to_entity is not None
|
||||
]
|
||||
|
||||
logger.info(f"API v2 response: graph with {len(nodes)} nodes and {len(edges)} edges")
|
||||
return GraphResponse(nodes=nodes, edges=edges)
|
||||
|
||||
|
||||
## Orphan entities endpoint
|
||||
|
||||
|
||||
@router.get("/orphans", response_model=OrphanEntitiesResponse)
|
||||
async def get_orphan_entities(
|
||||
project_id: ProjectExternalIdPathDep,
|
||||
entity_repository: EntityRepositoryV2ExternalDep,
|
||||
) -> OrphanEntitiesResponse:
|
||||
"""Return entities that have no incoming or outgoing relations."""
|
||||
with logfire.span(
|
||||
"api.request.knowledge.get_orphans",
|
||||
entrypoint="api",
|
||||
domain="knowledge",
|
||||
action="get_orphans",
|
||||
):
|
||||
logger.info("API v2 request: get_orphan_entities")
|
||||
|
||||
entities = await entity_repository.find_without_relations()
|
||||
nodes = [
|
||||
GraphNode(
|
||||
external_id=entity.external_id,
|
||||
title=entity.title,
|
||||
note_type=entity.note_type,
|
||||
file_path=entity.file_path,
|
||||
)
|
||||
for entity in entities
|
||||
]
|
||||
|
||||
logger.info(f"API v2 response: {len(nodes)} orphan entities")
|
||||
return OrphanEntitiesResponse(entities=nodes, total=len(nodes))
|
||||
|
||||
|
||||
## Resolution endpoint
|
||||
@@ -93,43 +181,48 @@ async def resolve_identifier(
|
||||
"resolution_method": "permalink"
|
||||
}
|
||||
"""
|
||||
logger.info(f"API v2 request: resolve_identifier for '{data.identifier}'")
|
||||
with logfire.span(
|
||||
"api.request.knowledge.resolve_entity",
|
||||
entrypoint="api",
|
||||
domain="knowledge",
|
||||
action="resolve_entity",
|
||||
):
|
||||
logger.info(f"API v2 request: resolve_identifier for '{data.identifier}'")
|
||||
|
||||
# Try to resolve by external_id first
|
||||
entity = await entity_repository.get_by_external_id(data.identifier)
|
||||
resolution_method = "external_id" if entity else "search"
|
||||
entity = await entity_repository.get_by_external_id(data.identifier)
|
||||
resolution_method = "external_id" if entity else "search"
|
||||
|
||||
# If not found by external_id, try other resolution methods
|
||||
if not entity:
|
||||
entity = await link_resolver.resolve_link(data.identifier)
|
||||
if entity:
|
||||
# Determine resolution method
|
||||
if entity.permalink == data.identifier:
|
||||
resolution_method = "permalink"
|
||||
elif entity.title == data.identifier:
|
||||
resolution_method = "title"
|
||||
elif entity.file_path == data.identifier:
|
||||
resolution_method = "path"
|
||||
else:
|
||||
resolution_method = "search"
|
||||
if not entity:
|
||||
entity = await link_resolver.resolve_link(
|
||||
data.identifier, source_path=data.source_path, strict=data.strict
|
||||
)
|
||||
if entity:
|
||||
if entity.permalink == data.identifier:
|
||||
resolution_method = "permalink"
|
||||
elif entity.title == data.identifier:
|
||||
resolution_method = "title"
|
||||
elif entity.file_path == data.identifier:
|
||||
resolution_method = "path"
|
||||
else:
|
||||
resolution_method = "search"
|
||||
|
||||
if not entity:
|
||||
raise HTTPException(status_code=404, detail=f"Entity not found: '{data.identifier}'")
|
||||
if not entity:
|
||||
raise HTTPException(status_code=404, detail=f"Entity not found: '{data.identifier}'")
|
||||
|
||||
result = EntityResolveResponse(
|
||||
external_id=entity.external_id,
|
||||
entity_id=entity.id,
|
||||
permalink=entity.permalink,
|
||||
file_path=entity.file_path,
|
||||
title=entity.title,
|
||||
resolution_method=resolution_method,
|
||||
)
|
||||
result = EntityResolveResponse(
|
||||
external_id=entity.external_id,
|
||||
entity_id=entity.id,
|
||||
permalink=entity.permalink,
|
||||
file_path=entity.file_path,
|
||||
title=entity.title,
|
||||
resolution_method=resolution_method,
|
||||
)
|
||||
|
||||
logger.info(
|
||||
f"API v2 response: resolved '{data.identifier}' to external_id={result.external_id} via {resolution_method}"
|
||||
)
|
||||
logger.debug(
|
||||
f"API v2 response: resolved '{data.identifier}' to external_id={result.external_id} via {resolution_method}"
|
||||
)
|
||||
|
||||
return result
|
||||
return result
|
||||
|
||||
|
||||
## Read endpoints
|
||||
@@ -155,18 +248,24 @@ async def get_entity_by_id(
|
||||
Raises:
|
||||
HTTPException: 404 if entity not found
|
||||
"""
|
||||
logger.info(f"API v2 request: get_entity_by_id entity_id={entity_id}")
|
||||
with logfire.span(
|
||||
"api.request.knowledge.get_entity",
|
||||
entrypoint="api",
|
||||
domain="knowledge",
|
||||
action="get_entity",
|
||||
):
|
||||
logger.info(f"API v2 request: get_entity_by_id entity_id={entity_id}")
|
||||
|
||||
entity = await entity_repository.get_by_external_id(entity_id)
|
||||
if not entity:
|
||||
raise HTTPException(
|
||||
status_code=404, detail=f"Entity with external_id '{entity_id}' not found"
|
||||
)
|
||||
entity = await entity_repository.get_by_external_id(entity_id)
|
||||
if not entity:
|
||||
raise HTTPException(
|
||||
status_code=404, detail=f"Entity with external_id '{entity_id}' not found"
|
||||
)
|
||||
|
||||
result = EntityResponseV2.model_validate(entity)
|
||||
logger.info(f"API v2 response: external_id={entity_id}, title='{result.title}'")
|
||||
result = EntityResponseV2.model_validate(entity)
|
||||
logger.info(f"API v2 response: external_id={entity_id}, title='{result.title}'")
|
||||
|
||||
return result
|
||||
return result
|
||||
|
||||
|
||||
## Create endpoints
|
||||
@@ -176,9 +275,10 @@ async def get_entity_by_id(
|
||||
async def create_entity(
|
||||
project_id: ProjectExternalIdPathDep,
|
||||
data: Entity,
|
||||
background_tasks: BackgroundTasks,
|
||||
entity_service: EntityServiceV2ExternalDep,
|
||||
search_service: SearchServiceV2ExternalDep,
|
||||
task_scheduler: TaskSchedulerDep,
|
||||
app_config: AppConfigDep,
|
||||
) -> EntityResponseV2:
|
||||
"""Create a new entity.
|
||||
|
||||
@@ -186,22 +286,38 @@ async def create_entity(
|
||||
data: Entity data to create
|
||||
|
||||
Returns:
|
||||
Created entity with generated external_id (UUID)
|
||||
Created entity with generated external_id (UUID) and file content
|
||||
"""
|
||||
logger.info(
|
||||
"API v2 request", endpoint="create_entity", entity_type=data.entity_type, title=data.title
|
||||
)
|
||||
with logfire.span(
|
||||
"api.request.knowledge.create_entity",
|
||||
entrypoint="api",
|
||||
domain="knowledge",
|
||||
action="create_entity",
|
||||
):
|
||||
logger.info(
|
||||
"API v2 request", endpoint="create_entity", note_type=data.note_type, title=data.title
|
||||
)
|
||||
|
||||
entity = await entity_service.create_entity(data)
|
||||
# Note writes are now internally consistent before the response returns. We only leave
|
||||
# truly derived work, like semantic vectors, on the async scheduler.
|
||||
write_result = await entity_service.create_entity_with_content(data)
|
||||
entity = write_result.entity
|
||||
await search_service.index_entity(entity, content=write_result.search_content)
|
||||
_schedule_vector_sync_if_enabled(
|
||||
task_scheduler=task_scheduler,
|
||||
app_config=app_config,
|
||||
entity_id=entity.id,
|
||||
project_id=project_id,
|
||||
)
|
||||
|
||||
# reindex
|
||||
await search_service.index_entity(entity, background_tasks=background_tasks)
|
||||
result = EntityResponseV2.model_validate(entity)
|
||||
result = EntityResponseV2.model_validate(entity)
|
||||
# The write service already returns the canonical markdown accepted for this request.
|
||||
result = result.model_copy(update={"content": write_result.content})
|
||||
|
||||
logger.info(
|
||||
f"API v2 response: endpoint='create_entity' external_id={entity.external_id}, title={result.title}, permalink={result.permalink}, status_code=201"
|
||||
)
|
||||
return result
|
||||
logger.info(
|
||||
f"API v2 response: endpoint='create_entity' external_id={entity.external_id}, title={result.title}, permalink={result.permalink}, status_code=201"
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
## Update endpoints
|
||||
@@ -211,12 +327,12 @@ async def create_entity(
|
||||
async def update_entity_by_id(
|
||||
data: Entity,
|
||||
response: Response,
|
||||
background_tasks: BackgroundTasks,
|
||||
project_id: ProjectExternalIdPathDep,
|
||||
entity_service: EntityServiceV2ExternalDep,
|
||||
search_service: SearchServiceV2ExternalDep,
|
||||
sync_service: SyncServiceV2ExternalDep,
|
||||
entity_repository: EntityRepositoryV2ExternalDep,
|
||||
task_scheduler: TaskSchedulerDep,
|
||||
app_config: AppConfigDep,
|
||||
entity_id: str = Path(..., description="Entity external ID (UUID)"),
|
||||
) -> EntityResponseV2:
|
||||
"""Update an entity by external ID.
|
||||
@@ -228,43 +344,66 @@ async def update_entity_by_id(
|
||||
data: Updated entity data
|
||||
|
||||
Returns:
|
||||
Updated entity
|
||||
Updated entity with file content
|
||||
"""
|
||||
logger.info(f"API v2 request: update_entity_by_id entity_id={entity_id}")
|
||||
with logfire.span(
|
||||
"api.request.knowledge.update_entity",
|
||||
entrypoint="api",
|
||||
domain="knowledge",
|
||||
action="update_entity",
|
||||
):
|
||||
logger.info(f"API v2 request: update_entity_by_id entity_id={entity_id}")
|
||||
|
||||
# Check if entity exists
|
||||
existing = await entity_repository.get_by_external_id(entity_id)
|
||||
created = existing is None
|
||||
existing = await entity_repository.get_by_external_id(entity_id)
|
||||
created = existing is None
|
||||
|
||||
# Perform update or create
|
||||
entity, _ = await entity_service.create_or_update_entity(data)
|
||||
response.status_code = 201 if created else 200
|
||||
if existing:
|
||||
write_result = await entity_service.update_entity_with_content(existing, data)
|
||||
entity = write_result.entity
|
||||
response.status_code = 200
|
||||
else:
|
||||
write_result = await entity_service.create_entity_with_content(data)
|
||||
entity = write_result.entity
|
||||
if entity.external_id != entity_id:
|
||||
entity = await entity_repository.update(
|
||||
entity.id,
|
||||
{"external_id": entity_id},
|
||||
)
|
||||
# external_id fixup only changes the DB row. The file content is unchanged,
|
||||
# so the markdown captured during the write remains valid downstream.
|
||||
if not entity:
|
||||
raise HTTPException(
|
||||
status_code=404,
|
||||
detail=f"Entity with external_id '{entity_id}' not found",
|
||||
)
|
||||
response.status_code = 201
|
||||
|
||||
# reindex
|
||||
await search_service.index_entity(entity, background_tasks=background_tasks)
|
||||
|
||||
# Schedule relation resolution for new entities
|
||||
if created:
|
||||
background_tasks.add_task( # pragma: no cover
|
||||
resolve_relations_background, sync_service, entity.id, entity.permalink or ""
|
||||
await search_service.index_entity(entity, content=write_result.search_content)
|
||||
_schedule_vector_sync_if_enabled(
|
||||
task_scheduler=task_scheduler,
|
||||
app_config=app_config,
|
||||
entity_id=entity.id,
|
||||
project_id=project_id,
|
||||
)
|
||||
|
||||
result = EntityResponseV2.model_validate(entity)
|
||||
result = EntityResponseV2.model_validate(entity)
|
||||
result = result.model_copy(update={"content": write_result.content})
|
||||
|
||||
logger.info(
|
||||
f"API v2 response: external_id={entity_id}, created={created}, status_code={response.status_code}"
|
||||
)
|
||||
return result
|
||||
logger.info(
|
||||
f"API v2 response: external_id={entity_id}, created={created}, status_code={response.status_code}"
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
@router.patch("/entities/{entity_id}", response_model=EntityResponseV2)
|
||||
async def edit_entity_by_id(
|
||||
data: EditEntityRequest,
|
||||
background_tasks: BackgroundTasks,
|
||||
project_id: ProjectExternalIdPathDep,
|
||||
entity_service: EntityServiceV2ExternalDep,
|
||||
search_service: SearchServiceV2ExternalDep,
|
||||
entity_repository: EntityRepositoryV2ExternalDep,
|
||||
task_scheduler: TaskSchedulerDep,
|
||||
app_config: AppConfigDep,
|
||||
entity_id: str = Path(..., description="Entity external ID (UUID)"),
|
||||
) -> EntityResponseV2:
|
||||
"""Edit an existing entity by external ID using operations like append, prepend, etc.
|
||||
@@ -274,48 +413,58 @@ async def edit_entity_by_id(
|
||||
data: Edit operation details
|
||||
|
||||
Returns:
|
||||
Updated entity
|
||||
Updated entity with file content
|
||||
|
||||
Raises:
|
||||
HTTPException: 404 if entity not found, 400 if edit fails
|
||||
"""
|
||||
logger.info(
|
||||
f"API v2 request: edit_entity_by_id entity_id={entity_id}, operation='{data.operation}'"
|
||||
)
|
||||
|
||||
# Verify entity exists
|
||||
entity = await entity_repository.get_by_external_id(entity_id)
|
||||
if not entity: # pragma: no cover
|
||||
raise HTTPException(
|
||||
status_code=404, detail=f"Entity with external_id '{entity_id}' not found"
|
||||
)
|
||||
|
||||
try:
|
||||
# Edit using the entity's permalink or path
|
||||
identifier = entity.permalink or entity.file_path
|
||||
updated_entity = await entity_service.edit_entity(
|
||||
identifier=identifier,
|
||||
operation=data.operation,
|
||||
content=data.content,
|
||||
section=data.section,
|
||||
find_text=data.find_text,
|
||||
expected_replacements=data.expected_replacements,
|
||||
)
|
||||
|
||||
# Reindex
|
||||
await search_service.index_entity(updated_entity, background_tasks=background_tasks)
|
||||
|
||||
result = EntityResponseV2.model_validate(updated_entity)
|
||||
|
||||
with logfire.span(
|
||||
"api.request.knowledge.edit_entity",
|
||||
entrypoint="api",
|
||||
domain="knowledge",
|
||||
action="edit_entity",
|
||||
):
|
||||
logger.info(
|
||||
f"API v2 response: external_id={entity_id}, operation='{data.operation}', status_code=200"
|
||||
f"API v2 request: edit_entity_by_id entity_id={entity_id}, operation='{data.operation}'"
|
||||
)
|
||||
|
||||
return result
|
||||
entity = await entity_repository.get_by_external_id(entity_id)
|
||||
if not entity: # pragma: no cover
|
||||
raise HTTPException(
|
||||
status_code=404, detail=f"Entity with external_id '{entity_id}' not found"
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error editing entity {entity_id}: {e}")
|
||||
raise HTTPException(status_code=400, detail=str(e))
|
||||
try:
|
||||
identifier = entity.permalink or entity.file_path
|
||||
write_result = await entity_service.edit_entity_with_content(
|
||||
identifier=identifier,
|
||||
operation=data.operation,
|
||||
content=data.content,
|
||||
section=data.section,
|
||||
find_text=data.find_text,
|
||||
expected_replacements=data.expected_replacements,
|
||||
)
|
||||
updated_entity = write_result.entity
|
||||
await search_service.index_entity(updated_entity, content=write_result.search_content)
|
||||
_schedule_vector_sync_if_enabled(
|
||||
task_scheduler=task_scheduler,
|
||||
app_config=app_config,
|
||||
entity_id=updated_entity.id,
|
||||
project_id=project_id,
|
||||
)
|
||||
|
||||
result = EntityResponseV2.model_validate(updated_entity)
|
||||
result = result.model_copy(update={"content": write_result.content})
|
||||
|
||||
logger.info(
|
||||
f"API v2 response: external_id={entity_id}, operation='{data.operation}', status_code=200"
|
||||
)
|
||||
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error editing entity {entity_id}: {e}")
|
||||
raise HTTPException(status_code=400, detail=str(e))
|
||||
|
||||
|
||||
## Delete endpoints
|
||||
@@ -323,12 +472,10 @@ async def edit_entity_by_id(
|
||||
|
||||
@router.delete("/entities/{entity_id}", response_model=DeleteEntitiesResponse)
|
||||
async def delete_entity_by_id(
|
||||
background_tasks: BackgroundTasks,
|
||||
project_id: ProjectExternalIdPathDep,
|
||||
entity_service: EntityServiceV2ExternalDep,
|
||||
entity_repository: EntityRepositoryV2ExternalDep,
|
||||
entity_id: str = Path(..., description="Entity external ID (UUID)"),
|
||||
search_service=Depends(lambda: None), # Optional for now
|
||||
) -> DeleteEntitiesResponse:
|
||||
"""Delete an entity by external ID.
|
||||
|
||||
@@ -340,23 +487,25 @@ async def delete_entity_by_id(
|
||||
|
||||
Note: Returns deleted=False if entity doesn't exist (idempotent)
|
||||
"""
|
||||
logger.info(f"API v2 request: delete_entity_by_id entity_id={entity_id}")
|
||||
with logfire.span(
|
||||
"api.request.knowledge.delete_entity",
|
||||
entrypoint="api",
|
||||
domain="knowledge",
|
||||
action="delete_entity",
|
||||
):
|
||||
logger.info(f"API v2 request: delete_entity_by_id entity_id={entity_id}")
|
||||
|
||||
entity = await entity_repository.get_by_external_id(entity_id)
|
||||
if entity is None:
|
||||
logger.info(f"API v2 response: external_id={entity_id} not found, deleted=False")
|
||||
return DeleteEntitiesResponse(deleted=False)
|
||||
entity = await entity_repository.get_by_external_id(entity_id)
|
||||
if entity is None:
|
||||
logger.info(f"API v2 response: external_id={entity_id} not found, deleted=False")
|
||||
return DeleteEntitiesResponse(deleted=False)
|
||||
|
||||
# Delete the entity using internal ID
|
||||
deleted = await entity_service.delete_entity(entity.id)
|
||||
# Delete the entity using internal ID
|
||||
deleted = await entity_service.delete_entity(entity.id)
|
||||
|
||||
# Remove from search index if search service available
|
||||
if search_service:
|
||||
background_tasks.add_task(search_service.handle_delete, entity) # pragma: no cover
|
||||
logger.info(f"API v2 response: external_id={entity_id}, deleted={deleted}")
|
||||
|
||||
logger.info(f"API v2 response: external_id={entity_id}, deleted={deleted}")
|
||||
|
||||
return DeleteEntitiesResponse(deleted=deleted)
|
||||
return DeleteEntitiesResponse(deleted=deleted)
|
||||
|
||||
|
||||
## Move endpoint
|
||||
@@ -365,13 +514,13 @@ async def delete_entity_by_id(
|
||||
@router.put("/entities/{entity_id}/move", response_model=EntityResponseV2)
|
||||
async def move_entity(
|
||||
data: MoveEntityRequestV2,
|
||||
background_tasks: BackgroundTasks,
|
||||
project_id: ProjectExternalIdPathDep,
|
||||
entity_service: EntityServiceV2ExternalDep,
|
||||
entity_repository: EntityRepositoryV2ExternalDep,
|
||||
project_config: ProjectConfigV2ExternalDep,
|
||||
app_config: AppConfigDep,
|
||||
search_service: SearchServiceV2ExternalDep,
|
||||
task_scheduler: TaskSchedulerDep,
|
||||
entity_id: str = Path(..., description="Entity external ID (UUID)"),
|
||||
) -> EntityResponseV2:
|
||||
"""Move an entity to a new file location.
|
||||
@@ -387,39 +536,170 @@ async def move_entity(
|
||||
Returns:
|
||||
Updated entity with new file path
|
||||
"""
|
||||
logger.info(
|
||||
f"API v2 request: move_entity entity_id={entity_id}, destination='{data.destination_path}'"
|
||||
)
|
||||
|
||||
try:
|
||||
# First, get the entity by external_id to verify it exists
|
||||
entity = await entity_repository.get_by_external_id(entity_id)
|
||||
if not entity: # pragma: no cover
|
||||
raise HTTPException(
|
||||
status_code=404, detail=f"Entity with external_id '{entity_id}' not found"
|
||||
)
|
||||
|
||||
# Move the entity using its current file path as identifier
|
||||
moved_entity = await entity_service.move_entity(
|
||||
identifier=entity.file_path, # Use file path for resolution
|
||||
destination_path=data.destination_path,
|
||||
project_config=project_config,
|
||||
app_config=app_config,
|
||||
with logfire.span(
|
||||
"api.request.knowledge.move_entity",
|
||||
entrypoint="api",
|
||||
domain="knowledge",
|
||||
action="move_entity",
|
||||
):
|
||||
logger.info(
|
||||
f"API v2 request: move_entity entity_id={entity_id}, destination='{data.destination_path}'"
|
||||
)
|
||||
|
||||
# Reindex at new location
|
||||
reindexed_entity = await entity_service.link_resolver.resolve_link(data.destination_path)
|
||||
if reindexed_entity:
|
||||
await search_service.index_entity(reindexed_entity, background_tasks=background_tasks)
|
||||
try:
|
||||
# First, get the entity by external_id to verify it exists
|
||||
entity = await entity_repository.get_by_external_id(entity_id)
|
||||
if not entity: # pragma: no cover
|
||||
raise HTTPException(
|
||||
status_code=404, detail=f"Entity with external_id '{entity_id}' not found"
|
||||
)
|
||||
|
||||
result = EntityResponseV2.model_validate(moved_entity)
|
||||
# Move the entity using its current file path as identifier
|
||||
moved_entity = await entity_service.move_entity(
|
||||
identifier=entity.file_path, # Use file path for resolution
|
||||
destination_path=data.destination_path,
|
||||
project_config=project_config,
|
||||
app_config=app_config,
|
||||
)
|
||||
|
||||
logger.info(f"API v2 response: moved external_id={entity_id} to '{data.destination_path}'")
|
||||
# Reindex at new location
|
||||
reindexed_entity = await entity_service.link_resolver.resolve_link(
|
||||
data.destination_path
|
||||
)
|
||||
if reindexed_entity:
|
||||
await search_service.index_entity(reindexed_entity)
|
||||
_schedule_vector_sync_if_enabled(
|
||||
task_scheduler=task_scheduler,
|
||||
app_config=app_config,
|
||||
entity_id=reindexed_entity.id,
|
||||
project_id=project_id,
|
||||
)
|
||||
|
||||
return result
|
||||
result = EntityResponseV2.model_validate(moved_entity)
|
||||
|
||||
except HTTPException: # pragma: no cover
|
||||
raise # pragma: no cover
|
||||
except Exception as e:
|
||||
logger.error(f"Error moving entity: {e}")
|
||||
raise HTTPException(status_code=400, detail=str(e))
|
||||
logger.info(
|
||||
f"API v2 response: moved external_id={entity_id} to '{data.destination_path}'"
|
||||
)
|
||||
|
||||
return result
|
||||
|
||||
except HTTPException: # pragma: no cover
|
||||
raise # pragma: no cover
|
||||
except Exception as e:
|
||||
logger.error(f"Error moving entity: {e}")
|
||||
raise HTTPException(status_code=400, detail=str(e))
|
||||
|
||||
|
||||
## Move directory endpoint
|
||||
|
||||
|
||||
@router.post("/move-directory", response_model=DirectoryMoveResult)
|
||||
async def move_directory(
|
||||
data: MoveDirectoryRequestV2,
|
||||
project_id: ProjectExternalIdPathDep,
|
||||
entity_service: EntityServiceV2ExternalDep,
|
||||
project_config: ProjectConfigV2ExternalDep,
|
||||
app_config: AppConfigDep,
|
||||
search_service: SearchServiceV2ExternalDep,
|
||||
task_scheduler: TaskSchedulerDep,
|
||||
) -> DirectoryMoveResult:
|
||||
"""Move all entities in a directory to a new location.
|
||||
|
||||
V2 API uses project external_id in the URL path for stable references.
|
||||
Moves all files within a source directory to a destination directory,
|
||||
updating database records and optionally updating permalinks.
|
||||
|
||||
Args:
|
||||
project_id: Project external ID from URL path
|
||||
data: Move request with source and destination directories
|
||||
|
||||
Returns:
|
||||
DirectoryMoveResult with counts and details of moved files
|
||||
"""
|
||||
with logfire.span(
|
||||
"api.request.knowledge.move_directory",
|
||||
entrypoint="api",
|
||||
domain="knowledge",
|
||||
action="move_directory",
|
||||
):
|
||||
logger.info(
|
||||
f"API v2 request: move_directory source='{data.source_directory}', destination='{data.destination_directory}'"
|
||||
)
|
||||
|
||||
try:
|
||||
# Move the directory using the service
|
||||
result = await entity_service.move_directory(
|
||||
source_directory=data.source_directory,
|
||||
destination_directory=data.destination_directory,
|
||||
project_config=project_config,
|
||||
app_config=app_config,
|
||||
)
|
||||
|
||||
# Reindex moved entities
|
||||
for file_path in result.moved_files:
|
||||
entity = await entity_service.link_resolver.resolve_link(file_path)
|
||||
if entity:
|
||||
await search_service.index_entity(entity)
|
||||
_schedule_vector_sync_if_enabled(
|
||||
task_scheduler=task_scheduler,
|
||||
app_config=app_config,
|
||||
entity_id=entity.id,
|
||||
project_id=project_id,
|
||||
)
|
||||
|
||||
logger.info(
|
||||
f"API v2 response: move_directory "
|
||||
f"total={result.total_files}, success={result.successful_moves}, failed={result.failed_moves}"
|
||||
)
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error moving directory: {e}")
|
||||
raise HTTPException(status_code=400, detail=str(e))
|
||||
|
||||
|
||||
## Delete directory endpoint
|
||||
|
||||
|
||||
@router.post("/delete-directory", response_model=DirectoryDeleteResult)
|
||||
async def delete_directory(
|
||||
data: DeleteDirectoryRequestV2,
|
||||
project_id: ProjectExternalIdPathDep,
|
||||
entity_service: EntityServiceV2ExternalDep,
|
||||
) -> DirectoryDeleteResult:
|
||||
"""Delete all entities in a directory.
|
||||
|
||||
V2 API uses project external_id in the URL path for stable references.
|
||||
Deletes all files within a directory, updating database records and
|
||||
removing files from the filesystem.
|
||||
|
||||
Args:
|
||||
project_id: Project external ID from URL path
|
||||
data: Delete request with directory path
|
||||
|
||||
Returns:
|
||||
DirectoryDeleteResult with counts and details of deleted files
|
||||
"""
|
||||
with logfire.span(
|
||||
"api.request.knowledge.delete_directory",
|
||||
entrypoint="api",
|
||||
domain="knowledge",
|
||||
action="delete_directory",
|
||||
):
|
||||
logger.info(f"API v2 request: delete_directory directory='{data.directory}'")
|
||||
|
||||
try:
|
||||
# Delete the directory using the service
|
||||
result = await entity_service.delete_directory(
|
||||
directory=data.directory,
|
||||
)
|
||||
|
||||
logger.info(
|
||||
f"API v2 response: delete_directory "
|
||||
f"total={result.total_files}, success={result.successful_deletes}, failed={result.failed_deletes}"
|
||||
)
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error deleting directory: {e}")
|
||||
raise HTTPException(status_code=400, detail=str(e))
|
||||
|
||||
@@ -9,6 +9,7 @@ from typing import Annotated, Optional
|
||||
from fastapi import APIRouter, Query, Path
|
||||
from loguru import logger
|
||||
|
||||
import logfire
|
||||
from basic_memory.deps import ContextServiceV2ExternalDep, EntityRepositoryV2ExternalDep
|
||||
from basic_memory.schemas.base import TimeFrame, parse_timeframe
|
||||
from basic_memory.schemas.memory import (
|
||||
@@ -16,7 +17,7 @@ from basic_memory.schemas.memory import (
|
||||
normalize_memory_url,
|
||||
)
|
||||
from basic_memory.schemas.search import SearchItemType
|
||||
from basic_memory.api.routers.utils import to_graph_context
|
||||
from basic_memory.api.v2.utils import to_graph_context
|
||||
|
||||
# Note: No prefix here - it's added during registration as /v2/{project_id}/memory
|
||||
router = APIRouter(tags=["memory"])
|
||||
@@ -50,30 +51,55 @@ async def recent(
|
||||
Returns:
|
||||
GraphContext with recent activity and related entities
|
||||
"""
|
||||
# return all types by default
|
||||
types = (
|
||||
[SearchItemType.ENTITY, SearchItemType.RELATION, SearchItemType.OBSERVATION]
|
||||
if not type
|
||||
else type
|
||||
)
|
||||
with logfire.span(
|
||||
"api.request.memory.recent_activity",
|
||||
entrypoint="api",
|
||||
domain="memory",
|
||||
action="recent_activity",
|
||||
page=page,
|
||||
page_size=page_size,
|
||||
):
|
||||
types = (
|
||||
[SearchItemType.ENTITY, SearchItemType.RELATION, SearchItemType.OBSERVATION]
|
||||
if not type
|
||||
else type
|
||||
)
|
||||
|
||||
logger.debug(
|
||||
f"V2 Getting recent context for project {project_id}: `{types}` depth: `{depth}` timeframe: `{timeframe}` page: `{page}` page_size: `{page_size}` max_related: `{max_related}`"
|
||||
)
|
||||
# Parse timeframe
|
||||
since = parse_timeframe(timeframe)
|
||||
limit = page_size
|
||||
offset = (page - 1) * page_size
|
||||
logger.debug(
|
||||
f"V2 Getting recent context for project {project_id}: `{types}` depth: `{depth}` timeframe: `{timeframe}` page: `{page}` page_size: `{page_size}` max_related: `{max_related}`"
|
||||
)
|
||||
since = parse_timeframe(timeframe)
|
||||
limit = page_size
|
||||
offset = (page - 1) * page_size
|
||||
|
||||
# Build context
|
||||
context = await context_service.build_context(
|
||||
types=types, depth=depth, since=since, limit=limit, offset=offset, max_related=max_related
|
||||
)
|
||||
recent_context = await to_graph_context(
|
||||
context, entity_repository=entity_repository, page=page, page_size=page_size
|
||||
)
|
||||
logger.debug(f"V2 Recent context: {recent_context.model_dump_json()}")
|
||||
return recent_context
|
||||
with logfire.span(
|
||||
"api.memory.recent_activity.build_context",
|
||||
domain="memory",
|
||||
action="recent_activity",
|
||||
phase="build_context",
|
||||
page=page,
|
||||
page_size=page_size,
|
||||
):
|
||||
context = await context_service.build_context(
|
||||
types=types,
|
||||
depth=depth,
|
||||
since=since,
|
||||
limit=limit,
|
||||
offset=offset,
|
||||
max_related=max_related,
|
||||
)
|
||||
with logfire.span(
|
||||
"api.memory.recent_activity.shape_response",
|
||||
domain="memory",
|
||||
action="recent_activity",
|
||||
phase="shape_response",
|
||||
result_count=len(context.results),
|
||||
):
|
||||
recent_context = await to_graph_context(
|
||||
context, entity_repository=entity_repository, page=page, page_size=page_size
|
||||
)
|
||||
logger.debug(f"V2 Recent context: {recent_context.model_dump_json()}")
|
||||
return recent_context
|
||||
|
||||
|
||||
# get_memory_context needs to be declared last so other paths can match
|
||||
@@ -111,20 +137,46 @@ async def get_memory_context(
|
||||
Returns:
|
||||
GraphContext with the entity and its related context
|
||||
"""
|
||||
logger.debug(
|
||||
f"V2 Getting context for project {project_id}, URI: `{uri}` depth: `{depth}` timeframe: `{timeframe}` page: `{page}` page_size: `{page_size}` max_related: `{max_related}`"
|
||||
)
|
||||
memory_url = normalize_memory_url(uri)
|
||||
with logfire.span(
|
||||
"api.request.memory.build_context",
|
||||
entrypoint="api",
|
||||
domain="memory",
|
||||
action="build_context",
|
||||
page=page,
|
||||
page_size=page_size,
|
||||
):
|
||||
logger.debug(
|
||||
f"V2 Getting context for project {project_id}, URI: `{uri}` depth: `{depth}` timeframe: `{timeframe}` page: `{page}` page_size: `{page_size}` max_related: `{max_related}`"
|
||||
)
|
||||
memory_url = normalize_memory_url(uri)
|
||||
|
||||
# Parse timeframe
|
||||
since = parse_timeframe(timeframe) if timeframe else None
|
||||
limit = page_size
|
||||
offset = (page - 1) * page_size
|
||||
since = parse_timeframe(timeframe) if timeframe else None
|
||||
limit = page_size
|
||||
offset = (page - 1) * page_size
|
||||
|
||||
# Build context
|
||||
context = await context_service.build_context(
|
||||
memory_url, depth=depth, since=since, limit=limit, offset=offset, max_related=max_related
|
||||
)
|
||||
return await to_graph_context(
|
||||
context, entity_repository=entity_repository, page=page, page_size=page_size
|
||||
)
|
||||
with logfire.span(
|
||||
"api.memory.build_context.build_context",
|
||||
domain="memory",
|
||||
action="build_context",
|
||||
phase="build_context",
|
||||
page=page,
|
||||
page_size=page_size,
|
||||
):
|
||||
context = await context_service.build_context(
|
||||
memory_url,
|
||||
depth=depth,
|
||||
since=since,
|
||||
limit=limit,
|
||||
offset=offset,
|
||||
max_related=max_related,
|
||||
)
|
||||
with logfire.span(
|
||||
"api.memory.build_context.shape_response",
|
||||
domain="memory",
|
||||
action="build_context",
|
||||
phase="shape_response",
|
||||
result_count=len(context.results),
|
||||
):
|
||||
return await to_graph_context(
|
||||
context, entity_repository=entity_repository, page=page, page_size=page_size
|
||||
)
|
||||
|
||||
@@ -11,7 +11,7 @@ Key improvements:
|
||||
"""
|
||||
|
||||
import os
|
||||
from typing import Optional
|
||||
from typing import Literal, Optional
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Body, Query, Path
|
||||
from loguru import logger
|
||||
@@ -19,15 +19,259 @@ from loguru import logger
|
||||
from basic_memory.deps import (
|
||||
ProjectServiceDep,
|
||||
ProjectRepositoryDep,
|
||||
ProjectConfigV2ExternalDep,
|
||||
SyncServiceV2ExternalDep,
|
||||
TaskSchedulerDep,
|
||||
ProjectExternalIdPathDep,
|
||||
)
|
||||
from basic_memory.schemas import SyncReportResponse
|
||||
from basic_memory.models import Project
|
||||
from basic_memory.repository.project_repository import ProjectRepository
|
||||
from basic_memory.schemas.project_info import (
|
||||
ProjectItem,
|
||||
ProjectList,
|
||||
ProjectInfoRequest,
|
||||
ProjectInfoResponse,
|
||||
ProjectStatusResponse,
|
||||
)
|
||||
from basic_memory.schemas.v2 import ProjectResolveRequest, ProjectResolveResponse
|
||||
from basic_memory.utils import normalize_project_path, generate_permalink
|
||||
|
||||
router = APIRouter(prefix="/projects", tags=["project_management-v2"])
|
||||
ProjectResolveMethod = Literal["external_id", "name", "permalink"]
|
||||
|
||||
|
||||
def _split_qualified_project_identifier(identifier: str) -> tuple[str | None, str]:
|
||||
"""Split ``<workspace>/<project>`` identifiers while preserving plain project names."""
|
||||
cleaned = identifier.strip()
|
||||
if "/" not in cleaned:
|
||||
return None, cleaned
|
||||
|
||||
workspace_identifier, project_identifier = cleaned.split("/", 1)
|
||||
if not workspace_identifier or not project_identifier:
|
||||
return None, cleaned
|
||||
return workspace_identifier, project_identifier
|
||||
|
||||
|
||||
async def _resolve_project_identifier_candidate(
|
||||
project_repository: ProjectRepository,
|
||||
identifier: str,
|
||||
) -> tuple[Project | None, ProjectResolveMethod]:
|
||||
"""Resolve one project identifier candidate and report the matching method."""
|
||||
identifier_permalink = generate_permalink(identifier)
|
||||
|
||||
project = await project_repository.get_by_external_id(identifier)
|
||||
if project:
|
||||
return project, "external_id"
|
||||
|
||||
project = await project_repository.get_by_permalink(identifier_permalink)
|
||||
if project:
|
||||
return project, "permalink"
|
||||
|
||||
project = await project_repository.get_by_name_case_insensitive(identifier)
|
||||
if project:
|
||||
return project, "name" # pragma: no cover
|
||||
|
||||
return None, "name"
|
||||
|
||||
|
||||
async def _resolve_project_identifier(
|
||||
project_repository: ProjectRepository,
|
||||
identifier: str,
|
||||
) -> tuple[Project | None, ProjectResolveMethod]:
|
||||
"""Resolve exact identifiers first, then accepted workspace-qualified forms."""
|
||||
project, resolution_method = await _resolve_project_identifier_candidate(
|
||||
project_repository,
|
||||
identifier,
|
||||
)
|
||||
if project:
|
||||
return project, resolution_method
|
||||
|
||||
workspace_identifier, project_identifier = _split_qualified_project_identifier(identifier)
|
||||
if workspace_identifier is None:
|
||||
return None, resolution_method
|
||||
|
||||
# Trigger: an MCP disambiguation error suggested ``workspace/project``.
|
||||
# Why: request routing already selected the workspace/tenant; this endpoint
|
||||
# only needs the project segment to validate the active project.
|
||||
# Outcome: models can follow the hint verbatim instead of looping on a 404.
|
||||
project, resolution_method = await _resolve_project_identifier_candidate(
|
||||
project_repository,
|
||||
project_identifier,
|
||||
)
|
||||
if project:
|
||||
return project, resolution_method
|
||||
|
||||
return None, resolution_method
|
||||
|
||||
|
||||
@router.get("/", response_model=ProjectList)
|
||||
async def list_projects(
|
||||
project_service: ProjectServiceDep,
|
||||
) -> ProjectList:
|
||||
"""List all configured projects.
|
||||
|
||||
Returns:
|
||||
A list of all projects with metadata
|
||||
"""
|
||||
projects = await project_service.list_projects()
|
||||
default_project = await project_service.get_default_project_name()
|
||||
|
||||
project_items = [
|
||||
ProjectItem(
|
||||
id=project.id,
|
||||
external_id=project.external_id,
|
||||
name=project.name,
|
||||
path=normalize_project_path(project.path),
|
||||
is_default=project.is_default or False,
|
||||
)
|
||||
for project in projects
|
||||
]
|
||||
|
||||
return ProjectList(
|
||||
projects=project_items,
|
||||
default_project=default_project,
|
||||
)
|
||||
|
||||
|
||||
@router.post("/", response_model=ProjectStatusResponse, status_code=201)
|
||||
async def add_project(
|
||||
project_data: ProjectInfoRequest,
|
||||
project_service: ProjectServiceDep,
|
||||
) -> ProjectStatusResponse:
|
||||
"""Add a new project to configuration and database.
|
||||
|
||||
Args:
|
||||
project_data: The project name and path, with option to set as default
|
||||
|
||||
Returns:
|
||||
Response confirming the project was added
|
||||
"""
|
||||
# Check if project already exists before attempting to add
|
||||
existing_project = await project_service.get_project(project_data.name)
|
||||
if existing_project:
|
||||
# Project exists - check if paths match for true idempotency
|
||||
# Normalize paths for comparison (resolve symlinks, etc.)
|
||||
requested_path = os.path.abspath(os.path.expanduser(project_data.path))
|
||||
existing_path = os.path.abspath(os.path.expanduser(existing_project.path))
|
||||
|
||||
if requested_path == existing_path:
|
||||
# Same name, same path - return 200 OK (idempotent)
|
||||
return ProjectStatusResponse( # pyright: ignore [reportCallIssue]
|
||||
message=f"Project '{project_data.name}' already exists",
|
||||
status="success",
|
||||
default=existing_project.is_default or False,
|
||||
new_project=ProjectItem(
|
||||
id=existing_project.id,
|
||||
external_id=existing_project.external_id,
|
||||
name=existing_project.name,
|
||||
path=existing_project.path,
|
||||
is_default=existing_project.is_default or False,
|
||||
),
|
||||
)
|
||||
else:
|
||||
# Same name, different path - this is an error
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=(
|
||||
f"Project '{project_data.name}' already exists with different path. "
|
||||
f"Existing: {existing_project.path}, Requested: {project_data.path}"
|
||||
),
|
||||
)
|
||||
|
||||
try: # pragma: no cover
|
||||
# The service layer handles cloud mode validation and path sanitization
|
||||
await project_service.add_project(
|
||||
project_data.name, project_data.path, set_default=project_data.set_default
|
||||
)
|
||||
|
||||
# Fetch the newly created project to get its ID
|
||||
new_project = await project_service.get_project(project_data.name)
|
||||
if not new_project:
|
||||
raise HTTPException(status_code=500, detail="Failed to retrieve newly created project")
|
||||
|
||||
return ProjectStatusResponse( # pyright: ignore [reportCallIssue]
|
||||
message=f"Project '{new_project.name}' added successfully",
|
||||
status="success",
|
||||
default=project_data.set_default,
|
||||
new_project=ProjectItem(
|
||||
id=new_project.id,
|
||||
external_id=new_project.external_id,
|
||||
name=new_project.name,
|
||||
path=new_project.path,
|
||||
is_default=new_project.is_default or False,
|
||||
),
|
||||
)
|
||||
except ValueError as e: # pragma: no cover
|
||||
raise HTTPException(status_code=400, detail=str(e))
|
||||
|
||||
|
||||
@router.post("/config/sync", response_model=ProjectStatusResponse)
|
||||
async def synchronize_projects(
|
||||
project_service: ProjectServiceDep,
|
||||
) -> ProjectStatusResponse:
|
||||
"""Synchronize projects between configuration file and database."""
|
||||
try: # pragma: no cover
|
||||
await project_service.synchronize_projects()
|
||||
|
||||
return ProjectStatusResponse( # pyright: ignore [reportCallIssue]
|
||||
message="Projects synchronized successfully between configuration and database",
|
||||
status="success",
|
||||
default=False,
|
||||
)
|
||||
except ValueError as e: # pragma: no cover
|
||||
raise HTTPException(status_code=400, detail=str(e))
|
||||
|
||||
|
||||
@router.post("/{project_id}/sync")
|
||||
async def sync_project(
|
||||
sync_service: SyncServiceV2ExternalDep,
|
||||
project_config: ProjectConfigV2ExternalDep,
|
||||
task_scheduler: TaskSchedulerDep,
|
||||
project_internal_id: ProjectExternalIdPathDep,
|
||||
force_full: bool = Query(
|
||||
False, description="Force full scan, bypassing watermark optimization"
|
||||
),
|
||||
run_in_background: bool = Query(True, description="Run in background"),
|
||||
):
|
||||
"""Force project filesystem sync to database."""
|
||||
if run_in_background:
|
||||
task_scheduler.schedule(
|
||||
"sync_project",
|
||||
project_id=project_internal_id,
|
||||
force_full=force_full,
|
||||
)
|
||||
logger.info(
|
||||
f"Filesystem sync initiated for project: {project_config.name} (force_full={force_full})"
|
||||
)
|
||||
|
||||
return {
|
||||
"status": "sync_started",
|
||||
"message": f"Filesystem sync initiated for project '{project_config.name}'",
|
||||
}
|
||||
|
||||
report = await sync_service.sync(
|
||||
project_config.home, project_config.name, force_full=force_full
|
||||
)
|
||||
logger.info(
|
||||
f"Filesystem sync completed for project: {project_config.name} (force_full={force_full})"
|
||||
)
|
||||
return SyncReportResponse.from_sync_report(report)
|
||||
|
||||
|
||||
@router.post("/{project_id}/status", response_model=SyncReportResponse)
|
||||
async def get_project_status(
|
||||
sync_service: SyncServiceV2ExternalDep,
|
||||
project_config: ProjectConfigV2ExternalDep,
|
||||
project_id: str = Path(..., description="Project external ID (UUID)"),
|
||||
force_full: bool = Query(
|
||||
False, description="Force full scan, bypassing watermark optimization"
|
||||
),
|
||||
) -> SyncReportResponse:
|
||||
"""Get sync status of files vs database for a project."""
|
||||
logger.info(f"API v2 request: get_project_status for project_id={project_id}")
|
||||
report = await sync_service.scan(project_config.home, force_full=force_full)
|
||||
return SyncReportResponse.from_sync_report(report)
|
||||
|
||||
|
||||
@router.post("/resolve", response_model=ProjectResolveResponse)
|
||||
@@ -70,28 +314,10 @@ async def resolve_project_identifier(
|
||||
"""
|
||||
logger.info(f"API v2 request: resolve_project_identifier for '{data.identifier}'")
|
||||
|
||||
# Generate permalink for comparison
|
||||
identifier_permalink = generate_permalink(data.identifier)
|
||||
|
||||
resolution_method = "name"
|
||||
project = None
|
||||
|
||||
# Try external_id first (UUID format)
|
||||
project = await project_repository.get_by_external_id(data.identifier)
|
||||
if project:
|
||||
resolution_method = "external_id"
|
||||
|
||||
# If not found by external_id, try by permalink (exact match)
|
||||
if not project:
|
||||
project = await project_repository.get_by_permalink(identifier_permalink)
|
||||
if project:
|
||||
resolution_method = "permalink"
|
||||
|
||||
# If not found by permalink, try case-insensitive name search
|
||||
if not project:
|
||||
project = await project_repository.get_by_name_case_insensitive(data.identifier)
|
||||
if project:
|
||||
resolution_method = "name" # pragma: no cover
|
||||
project, resolution_method = await _resolve_project_identifier(
|
||||
project_repository,
|
||||
data.identifier,
|
||||
)
|
||||
|
||||
if not project:
|
||||
raise HTTPException(status_code=404, detail=f"Project not found: '{data.identifier}'")
|
||||
@@ -147,6 +373,22 @@ async def get_project_by_id(
|
||||
)
|
||||
|
||||
|
||||
@router.get("/{project_id}/info", response_model=ProjectInfoResponse)
|
||||
async def get_project_info_by_id(
|
||||
project_service: ProjectServiceDep,
|
||||
project_repository: ProjectRepositoryDep,
|
||||
project_id: str = Path(..., description="Project external ID (UUID)"),
|
||||
) -> ProjectInfoResponse:
|
||||
"""Get detailed project information by external ID."""
|
||||
logger.info(f"API v2 request: get_project_info_by_id for project_id={project_id}")
|
||||
project = await project_repository.get_by_external_id(project_id)
|
||||
if not project:
|
||||
raise HTTPException(
|
||||
status_code=404, detail=f"Project with external_id '{project_id}' not found"
|
||||
)
|
||||
return await project_service.get_project_info(project.name)
|
||||
|
||||
|
||||
@router.patch("/{project_id}", response_model=ProjectStatusResponse)
|
||||
async def update_project_by_id(
|
||||
project_service: ProjectServiceDep,
|
||||
|
||||
@@ -6,10 +6,11 @@ have entity IDs in URLs - they generate formatted prompts from queries.
|
||||
"""
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
from fastapi import APIRouter, HTTPException, status, Path
|
||||
from loguru import logger
|
||||
|
||||
from basic_memory.api.routers.utils import to_graph_context, to_search_results
|
||||
from basic_memory.api.v2.utils import to_graph_context, to_search_results
|
||||
from basic_memory.api.template_loader import template_loader
|
||||
from basic_memory.schemas.base import parse_timeframe
|
||||
from basic_memory.deps import (
|
||||
@@ -59,6 +60,7 @@ async def continue_conversation(
|
||||
|
||||
# Initialize search results
|
||||
search_results = []
|
||||
hierarchical_results_for_count = []
|
||||
|
||||
# Get data needed for template
|
||||
if request.topic:
|
||||
@@ -91,7 +93,8 @@ async def continue_conversation(
|
||||
# Limit to a reasonable number of total results
|
||||
all_hierarchical_results = all_hierarchical_results[:10]
|
||||
|
||||
template_context = {
|
||||
hierarchical_results_for_count = all_hierarchical_results
|
||||
template_context: dict[str, Any] = {
|
||||
"topic": request.topic,
|
||||
"timeframe": request.timeframe,
|
||||
"hierarchical_results": all_hierarchical_results,
|
||||
@@ -110,6 +113,7 @@ async def continue_conversation(
|
||||
|
||||
hierarchical_results = recent_context.results[:5] # Limit to top 5 recent items
|
||||
|
||||
hierarchical_results_for_count = hierarchical_results
|
||||
template_context = {
|
||||
"topic": f"Recent Activity from ({request.timeframe})",
|
||||
"timeframe": request.timeframe,
|
||||
@@ -129,9 +133,6 @@ async def continue_conversation(
|
||||
relation_count = 0
|
||||
entity_count = 0
|
||||
|
||||
# Get the hierarchical results from the template context
|
||||
hierarchical_results_for_count = template_context.get("hierarchical_results", [])
|
||||
|
||||
# For topic-based search
|
||||
if request.topic:
|
||||
for item in hierarchical_results_for_count:
|
||||
@@ -159,29 +160,24 @@ async def continue_conversation(
|
||||
elif related.type == "entity": # pragma: no cover
|
||||
entity_count += 1 # pragma: no cover
|
||||
|
||||
# Build metadata
|
||||
metadata = {
|
||||
"query": request.topic,
|
||||
"timeframe": request.timeframe,
|
||||
"search_count": len(search_results)
|
||||
if request.topic
|
||||
else 0, # Original search results count
|
||||
"context_count": len(hierarchical_results_for_count),
|
||||
"observation_count": observation_count,
|
||||
"relation_count": relation_count,
|
||||
"total_items": (
|
||||
prompt_metadata = PromptMetadata(
|
||||
query=request.topic,
|
||||
timeframe=request.timeframe,
|
||||
search_count=len(search_results) if request.topic else 0,
|
||||
context_count=len(hierarchical_results_for_count),
|
||||
observation_count=observation_count,
|
||||
relation_count=relation_count,
|
||||
total_items=(
|
||||
len(hierarchical_results_for_count)
|
||||
+ observation_count
|
||||
+ relation_count
|
||||
+ entity_count
|
||||
),
|
||||
"search_limit": request.search_items_limit,
|
||||
"context_depth": request.depth,
|
||||
"related_limit": request.related_items_limit,
|
||||
"generated_at": datetime.now(timezone.utc).isoformat(),
|
||||
}
|
||||
|
||||
prompt_metadata = PromptMetadata(**metadata)
|
||||
search_limit=request.search_items_limit,
|
||||
context_depth=request.depth,
|
||||
related_limit=request.related_items_limit,
|
||||
generated_at=datetime.now(timezone.utc).isoformat(),
|
||||
)
|
||||
|
||||
return PromptResponse(
|
||||
prompt=rendered_prompt, context=template_context, metadata=prompt_metadata
|
||||
@@ -229,7 +225,7 @@ async def search_prompt(
|
||||
results = await search_service.search(query, limit=limit, offset=offset)
|
||||
search_results = await to_search_results(entity_service, results)
|
||||
|
||||
template_context = {
|
||||
template_context: dict[str, Any] = {
|
||||
"query": request.query,
|
||||
"timeframe": request.timeframe,
|
||||
"results": search_results,
|
||||
@@ -241,22 +237,19 @@ async def search_prompt(
|
||||
# Render template
|
||||
rendered_prompt = await template_loader.render("prompts/search.hbs", template_context)
|
||||
|
||||
# Build metadata
|
||||
metadata = {
|
||||
"query": request.query,
|
||||
"timeframe": request.timeframe,
|
||||
"search_count": len(search_results),
|
||||
"context_count": len(search_results),
|
||||
"observation_count": 0, # Search results don't include observations
|
||||
"relation_count": 0, # Search results don't include relations
|
||||
"total_items": len(search_results),
|
||||
"search_limit": limit,
|
||||
"context_depth": 0, # No context depth for basic search
|
||||
"related_limit": 0, # No related items for basic search
|
||||
"generated_at": datetime.now(timezone.utc).isoformat(),
|
||||
}
|
||||
|
||||
prompt_metadata = PromptMetadata(**metadata)
|
||||
prompt_metadata = PromptMetadata(
|
||||
query=request.query,
|
||||
timeframe=request.timeframe,
|
||||
search_count=len(search_results),
|
||||
context_count=len(search_results),
|
||||
observation_count=0,
|
||||
relation_count=0,
|
||||
total_items=len(search_results),
|
||||
search_limit=limit,
|
||||
context_depth=0,
|
||||
related_limit=0,
|
||||
generated_at=datetime.now(timezone.utc).isoformat(),
|
||||
)
|
||||
|
||||
return PromptResponse(
|
||||
prompt=rendered_prompt, context=template_context, metadata=prompt_metadata
|
||||
|
||||
@@ -15,6 +15,7 @@ from pathlib import Path as PathLib
|
||||
from fastapi import APIRouter, HTTPException, Response, Path
|
||||
from loguru import logger
|
||||
|
||||
import logfire
|
||||
from basic_memory.deps import (
|
||||
ProjectConfigV2ExternalDep,
|
||||
FileServiceV2ExternalDep,
|
||||
@@ -55,36 +56,62 @@ async def get_resource_content(
|
||||
Raises:
|
||||
HTTPException: 404 if entity or file not found
|
||||
"""
|
||||
logger.debug(f"V2 Getting content for project {project_id}, entity_id: {entity_id}")
|
||||
with logfire.span(
|
||||
"api.request.resource.get_content",
|
||||
entrypoint="api",
|
||||
domain="resource",
|
||||
action="get_content",
|
||||
):
|
||||
logger.debug(f"V2 Getting content for project {project_id}, entity_id: {entity_id}")
|
||||
|
||||
# Get entity by external_id
|
||||
entity = await entity_repository.get_by_external_id(entity_id)
|
||||
if not entity:
|
||||
raise HTTPException(status_code=404, detail=f"Entity {entity_id} not found")
|
||||
with logfire.span(
|
||||
"api.resource.get_content.load_entity",
|
||||
domain="resource",
|
||||
action="get_content",
|
||||
phase="load_entity",
|
||||
):
|
||||
entity = await entity_repository.get_by_external_id(entity_id)
|
||||
if not entity:
|
||||
raise HTTPException(status_code=404, detail=f"Entity {entity_id} not found")
|
||||
|
||||
# Validate entity file path to prevent path traversal
|
||||
project_path = PathLib(config.home)
|
||||
if not validate_project_path(entity.file_path, project_path):
|
||||
logger.error( # pragma: no cover
|
||||
f"Invalid file path in entity {entity.id}: {entity.file_path}"
|
||||
)
|
||||
raise HTTPException( # pragma: no cover
|
||||
status_code=500,
|
||||
detail="Entity contains invalid file path",
|
||||
)
|
||||
with logfire.span(
|
||||
"api.resource.get_content.validate_path",
|
||||
domain="resource",
|
||||
action="get_content",
|
||||
phase="validate_path",
|
||||
):
|
||||
project_path = PathLib(config.home)
|
||||
if not validate_project_path(entity.file_path, project_path):
|
||||
logger.error( # pragma: no cover
|
||||
f"Invalid file path in entity {entity.id}: {entity.file_path}"
|
||||
)
|
||||
raise HTTPException( # pragma: no cover
|
||||
status_code=500,
|
||||
detail="Entity contains invalid file path",
|
||||
)
|
||||
|
||||
# Check file exists via file_service (for cloud compatibility)
|
||||
if not await file_service.exists(entity.file_path):
|
||||
raise HTTPException( # pragma: no cover
|
||||
status_code=404,
|
||||
detail=f"File not found: {entity.file_path}",
|
||||
)
|
||||
with logfire.span(
|
||||
"api.resource.get_content.ensure_exists",
|
||||
domain="resource",
|
||||
action="get_content",
|
||||
phase="ensure_exists",
|
||||
):
|
||||
if not await file_service.exists(entity.file_path):
|
||||
raise HTTPException( # pragma: no cover
|
||||
status_code=404,
|
||||
detail=f"File not found: {entity.file_path}",
|
||||
)
|
||||
|
||||
# Read content via file_service as bytes (works with both local and S3)
|
||||
content = await file_service.read_file_bytes(entity.file_path)
|
||||
content_type = file_service.content_type(entity.file_path)
|
||||
with logfire.span(
|
||||
"api.resource.get_content.read_content",
|
||||
domain="resource",
|
||||
action="get_content",
|
||||
phase="read_content",
|
||||
):
|
||||
content = await file_service.read_file_bytes(entity.file_path)
|
||||
content_type = file_service.content_type(entity.file_path)
|
||||
|
||||
return Response(content=content, media_type=content_type)
|
||||
return Response(content=content, media_type=content_type)
|
||||
|
||||
|
||||
@router.post("", response_model=ResourceResponse)
|
||||
@@ -112,74 +139,94 @@ async def create_resource(
|
||||
Raises:
|
||||
HTTPException: 400 for invalid file paths, 409 if file already exists
|
||||
"""
|
||||
try:
|
||||
# Validate path to prevent path traversal attacks
|
||||
project_path = PathLib(config.home)
|
||||
if not validate_project_path(data.file_path, project_path):
|
||||
logger.warning(
|
||||
f"Invalid file path attempted: {data.file_path} in project {config.name}"
|
||||
with logfire.span(
|
||||
"api.request.resource.create",
|
||||
entrypoint="api",
|
||||
domain="resource",
|
||||
action="create",
|
||||
):
|
||||
try:
|
||||
# Validate path to prevent path traversal attacks
|
||||
project_path = PathLib(config.home)
|
||||
if not validate_project_path(data.file_path, project_path):
|
||||
logger.warning(
|
||||
f"Invalid file path attempted: {data.file_path} in project {config.name}"
|
||||
)
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"Invalid file path: {data.file_path}. "
|
||||
"Path must be relative and stay within project boundaries.",
|
||||
)
|
||||
|
||||
existing_entity = await entity_repository.get_by_file_path(data.file_path)
|
||||
if existing_entity:
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail=f"Resource already exists at {data.file_path} with entity_id {existing_entity.external_id}. "
|
||||
f"Use PUT /resource/{existing_entity.external_id} to update it.",
|
||||
)
|
||||
|
||||
with logfire.span(
|
||||
"api.resource.create.write_file",
|
||||
domain="resource",
|
||||
action="create",
|
||||
phase="write_file",
|
||||
):
|
||||
await file_service.ensure_directory(PathLib(data.file_path).parent)
|
||||
checksum = await file_service.write_file(data.file_path, data.content)
|
||||
|
||||
with logfire.span(
|
||||
"api.resource.create.read_metadata",
|
||||
domain="resource",
|
||||
action="create",
|
||||
phase="read_metadata",
|
||||
):
|
||||
file_metadata = await file_service.get_file_metadata(data.file_path)
|
||||
|
||||
file_name = PathLib(data.file_path).name
|
||||
content_type = file_service.content_type(data.file_path)
|
||||
note_type = "canvas" if data.file_path.endswith(".canvas") else "file"
|
||||
|
||||
entity = EntityModel(
|
||||
external_id=str(uuid.uuid4()),
|
||||
title=file_name,
|
||||
note_type=note_type,
|
||||
content_type=content_type,
|
||||
file_path=data.file_path,
|
||||
checksum=checksum,
|
||||
created_at=file_metadata.created_at,
|
||||
updated_at=file_metadata.modified_at,
|
||||
)
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"Invalid file path: {data.file_path}. "
|
||||
"Path must be relative and stay within project boundaries.",
|
||||
with logfire.span(
|
||||
"api.resource.create.upsert_entity",
|
||||
domain="resource",
|
||||
action="create",
|
||||
phase="upsert_entity",
|
||||
):
|
||||
entity = await entity_repository.add(entity)
|
||||
|
||||
with logfire.span(
|
||||
"api.resource.create.search_index",
|
||||
domain="resource",
|
||||
action="create",
|
||||
phase="search_index",
|
||||
):
|
||||
await search_service.index_entity(entity)
|
||||
|
||||
return ResourceResponse(
|
||||
entity_id=entity.id,
|
||||
external_id=entity.external_id,
|
||||
file_path=data.file_path,
|
||||
checksum=checksum,
|
||||
size=file_metadata.size,
|
||||
created_at=file_metadata.created_at.timestamp(),
|
||||
modified_at=file_metadata.modified_at.timestamp(),
|
||||
)
|
||||
|
||||
# Check if entity already exists
|
||||
existing_entity = await entity_repository.get_by_file_path(data.file_path)
|
||||
if existing_entity:
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail=f"Resource already exists at {data.file_path} with entity_id {existing_entity.external_id}. "
|
||||
f"Use PUT /resource/{existing_entity.external_id} to update it.",
|
||||
)
|
||||
|
||||
# Cloud compatibility: avoid assuming a local filesystem path.
|
||||
# Delegate directory creation + writes to FileService (local or S3).
|
||||
await file_service.ensure_directory(PathLib(data.file_path).parent)
|
||||
checksum = await file_service.write_file(data.file_path, data.content)
|
||||
|
||||
# Get file info
|
||||
file_metadata = await file_service.get_file_metadata(data.file_path)
|
||||
|
||||
# Determine file details
|
||||
file_name = PathLib(data.file_path).name
|
||||
content_type = file_service.content_type(data.file_path)
|
||||
entity_type = "canvas" if data.file_path.endswith(".canvas") else "file"
|
||||
|
||||
# Create a new entity model
|
||||
# Explicitly set external_id to ensure NOT NULL constraint is satisfied (fixes #512)
|
||||
entity = EntityModel(
|
||||
external_id=str(uuid.uuid4()),
|
||||
title=file_name,
|
||||
entity_type=entity_type,
|
||||
content_type=content_type,
|
||||
file_path=data.file_path,
|
||||
checksum=checksum,
|
||||
created_at=file_metadata.created_at,
|
||||
updated_at=file_metadata.modified_at,
|
||||
)
|
||||
entity = await entity_repository.add(entity)
|
||||
|
||||
# Index the file for search
|
||||
await search_service.index_entity(entity) # pyright: ignore
|
||||
|
||||
# Return success response
|
||||
return ResourceResponse(
|
||||
entity_id=entity.id,
|
||||
external_id=entity.external_id,
|
||||
file_path=data.file_path,
|
||||
checksum=checksum,
|
||||
size=file_metadata.size,
|
||||
created_at=file_metadata.created_at.timestamp(),
|
||||
modified_at=file_metadata.modified_at.timestamp(),
|
||||
)
|
||||
except HTTPException:
|
||||
# Re-raise HTTP exceptions without wrapping
|
||||
raise
|
||||
except Exception as e: # pragma: no cover
|
||||
logger.error(f"Error creating resource {data.file_path}: {e}")
|
||||
raise HTTPException(status_code=500, detail=f"Failed to create resource: {str(e)}")
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e: # pragma: no cover
|
||||
logger.error(f"Error creating resource {data.file_path}: {e}")
|
||||
raise HTTPException(status_code=500, detail=f"Failed to create resource: {str(e)}")
|
||||
|
||||
|
||||
@router.put("/{entity_id}", response_model=ResourceResponse)
|
||||
@@ -211,79 +258,96 @@ async def update_resource(
|
||||
Raises:
|
||||
HTTPException: 404 if entity not found, 400 for invalid paths
|
||||
"""
|
||||
try:
|
||||
# Get existing entity by external_id
|
||||
entity = await entity_repository.get_by_external_id(entity_id)
|
||||
if not entity:
|
||||
raise HTTPException(status_code=404, detail=f"Entity {entity_id} not found")
|
||||
with logfire.span(
|
||||
"api.request.resource.update",
|
||||
entrypoint="api",
|
||||
domain="resource",
|
||||
action="update",
|
||||
):
|
||||
try:
|
||||
entity = await entity_repository.get_by_external_id(entity_id)
|
||||
if not entity:
|
||||
raise HTTPException(status_code=404, detail=f"Entity {entity_id} not found")
|
||||
|
||||
# Determine target file path
|
||||
target_file_path = data.file_path if data.file_path else entity.file_path
|
||||
target_file_path = data.file_path if data.file_path else entity.file_path
|
||||
|
||||
# Validate path to prevent path traversal attacks
|
||||
project_path = PathLib(config.home)
|
||||
if not validate_project_path(target_file_path, project_path):
|
||||
logger.warning(
|
||||
f"Invalid file path attempted: {target_file_path} in project {config.name}"
|
||||
project_path = PathLib(config.home)
|
||||
if not validate_project_path(target_file_path, project_path):
|
||||
logger.warning(
|
||||
f"Invalid file path attempted: {target_file_path} in project {config.name}"
|
||||
)
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"Invalid file path: {target_file_path}. "
|
||||
"Path must be relative and stay within project boundaries.",
|
||||
)
|
||||
|
||||
with logfire.span(
|
||||
"api.resource.update.write_file",
|
||||
domain="resource",
|
||||
action="update",
|
||||
phase="write_file",
|
||||
):
|
||||
if data.file_path and data.file_path != entity.file_path:
|
||||
await file_service.ensure_directory(PathLib(target_file_path).parent)
|
||||
if await file_service.exists(entity.file_path):
|
||||
await file_service.delete_file(entity.file_path)
|
||||
else:
|
||||
await file_service.ensure_directory(PathLib(target_file_path).parent)
|
||||
|
||||
checksum = await file_service.write_file(target_file_path, data.content)
|
||||
|
||||
with logfire.span(
|
||||
"api.resource.update.read_metadata",
|
||||
domain="resource",
|
||||
action="update",
|
||||
phase="read_metadata",
|
||||
):
|
||||
file_metadata = await file_service.get_file_metadata(target_file_path)
|
||||
|
||||
file_name = PathLib(target_file_path).name
|
||||
content_type = file_service.content_type(target_file_path)
|
||||
note_type = "canvas" if target_file_path.endswith(".canvas") else "file"
|
||||
|
||||
with logfire.span(
|
||||
"api.resource.update.update_entity",
|
||||
domain="resource",
|
||||
action="update",
|
||||
phase="update_entity",
|
||||
):
|
||||
updated_entity = await entity_repository.update(
|
||||
entity.id,
|
||||
{
|
||||
"title": file_name,
|
||||
"note_type": note_type,
|
||||
"content_type": content_type,
|
||||
"file_path": target_file_path,
|
||||
"checksum": checksum,
|
||||
"updated_at": file_metadata.modified_at,
|
||||
},
|
||||
)
|
||||
if updated_entity is None:
|
||||
raise HTTPException(status_code=404, detail=f"Entity {entity_id} not found")
|
||||
|
||||
with logfire.span(
|
||||
"api.resource.update.search_index",
|
||||
domain="resource",
|
||||
action="update",
|
||||
phase="search_index",
|
||||
):
|
||||
await search_service.index_entity(updated_entity)
|
||||
|
||||
return ResourceResponse(
|
||||
entity_id=entity.id,
|
||||
external_id=entity.external_id,
|
||||
file_path=target_file_path,
|
||||
checksum=checksum,
|
||||
size=file_metadata.size,
|
||||
created_at=file_metadata.created_at.timestamp(),
|
||||
modified_at=file_metadata.modified_at.timestamp(),
|
||||
)
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"Invalid file path: {target_file_path}. "
|
||||
"Path must be relative and stay within project boundaries.",
|
||||
)
|
||||
|
||||
# If moving file, handle the move
|
||||
if data.file_path and data.file_path != entity.file_path:
|
||||
# Ensure new parent directory exists (no-op for S3)
|
||||
await file_service.ensure_directory(PathLib(target_file_path).parent)
|
||||
|
||||
# If old file exists, remove it via file_service (for cloud compatibility)
|
||||
if await file_service.exists(entity.file_path):
|
||||
await file_service.delete_file(entity.file_path)
|
||||
else:
|
||||
# Ensure directory exists for in-place update
|
||||
await file_service.ensure_directory(PathLib(target_file_path).parent)
|
||||
|
||||
# Write content to target file
|
||||
checksum = await file_service.write_file(target_file_path, data.content)
|
||||
|
||||
# Get file info
|
||||
file_metadata = await file_service.get_file_metadata(target_file_path)
|
||||
|
||||
# Determine file details
|
||||
file_name = PathLib(target_file_path).name
|
||||
content_type = file_service.content_type(target_file_path)
|
||||
entity_type = "canvas" if target_file_path.endswith(".canvas") else "file"
|
||||
|
||||
# Update entity using internal ID
|
||||
updated_entity = await entity_repository.update(
|
||||
entity.id,
|
||||
{
|
||||
"title": file_name,
|
||||
"entity_type": entity_type,
|
||||
"content_type": content_type,
|
||||
"file_path": target_file_path,
|
||||
"checksum": checksum,
|
||||
"updated_at": file_metadata.modified_at,
|
||||
},
|
||||
)
|
||||
|
||||
# Index the updated file for search
|
||||
await search_service.index_entity(updated_entity) # pyright: ignore
|
||||
|
||||
# Return success response
|
||||
return ResourceResponse(
|
||||
entity_id=entity.id,
|
||||
external_id=entity.external_id,
|
||||
file_path=target_file_path,
|
||||
checksum=checksum,
|
||||
size=file_metadata.size,
|
||||
created_at=file_metadata.created_at.timestamp(),
|
||||
modified_at=file_metadata.modified_at.timestamp(),
|
||||
)
|
||||
except HTTPException:
|
||||
# Re-raise HTTP exceptions without wrapping
|
||||
raise
|
||||
except Exception as e: # pragma: no cover
|
||||
logger.error(f"Error updating resource {entity_id}: {e}")
|
||||
raise HTTPException(status_code=500, detail=f"Failed to update resource: {str(e)}")
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e: # pragma: no cover
|
||||
logger.error(f"Error updating resource {entity_id}: {e}")
|
||||
raise HTTPException(status_code=500, detail=f"Failed to update resource: {str(e)}")
|
||||
|
||||
@@ -0,0 +1,412 @@
|
||||
"""V2 router for schema operations.
|
||||
|
||||
Provides endpoints for schema validation, inference, and drift detection.
|
||||
The schema system validates notes against Picoschema definitions without
|
||||
introducing any new data model -- it works entirely with existing
|
||||
observations and relations.
|
||||
|
||||
Flow: Entity loaded with eager observations/relations -> convert to tuples -> core functions.
|
||||
"""
|
||||
|
||||
from pathlib import Path as FilePath
|
||||
|
||||
import frontmatter
|
||||
from fastapi import APIRouter, Path, Query
|
||||
from loguru import logger
|
||||
|
||||
from basic_memory.deps import (
|
||||
EntityRepositoryV2ExternalDep,
|
||||
FileServiceV2ExternalDep,
|
||||
LinkResolverV2ExternalDep,
|
||||
)
|
||||
from basic_memory.models.knowledge import Entity
|
||||
from basic_memory.schemas.schema import (
|
||||
ValidationReport,
|
||||
InferenceReport,
|
||||
DriftReport,
|
||||
NoteValidationResponse,
|
||||
FieldResultResponse,
|
||||
FieldFrequencyResponse,
|
||||
DriftFieldResponse,
|
||||
)
|
||||
from basic_memory.schema.resolver import resolve_schema
|
||||
from basic_memory.schema.validator import validate_note
|
||||
from basic_memory.schema.inference import infer_schema, NoteData, ObservationData, RelationData
|
||||
from basic_memory.schema.diff import diff_schema
|
||||
from basic_memory.utils import generate_permalink
|
||||
|
||||
# Note: No prefix here -- it's added during registration as /v2/{project_id}/schema
|
||||
router = APIRouter(tags=["schema"])
|
||||
|
||||
|
||||
# --- ORM to core data conversion ---
|
||||
|
||||
|
||||
def _entity_observations(entity: Entity) -> list[ObservationData]:
|
||||
"""Extract ObservationData from an entity's observations."""
|
||||
return [ObservationData(obs.category, obs.content) for obs in entity.observations]
|
||||
|
||||
|
||||
def _entity_relations(entity: Entity) -> list[RelationData]:
|
||||
"""Extract RelationData from an entity's outgoing relations.
|
||||
|
||||
Carries the target entity's type on each relation so the inference engine
|
||||
can suggest correct types (e.g. works_at -> Organization, not the source type).
|
||||
"""
|
||||
return [
|
||||
RelationData(
|
||||
relation_type=rel.relation_type,
|
||||
target_name=rel.to_name,
|
||||
target_note_type=rel.to_entity.note_type if rel.to_entity else None,
|
||||
)
|
||||
for rel in entity.outgoing_relations
|
||||
]
|
||||
|
||||
|
||||
def _entity_to_note_data(entity: Entity) -> NoteData:
|
||||
"""Convert an ORM Entity to a NoteData for inference/diff analysis."""
|
||||
return NoteData(
|
||||
identifier=entity.permalink or entity.file_path,
|
||||
observations=_entity_observations(entity),
|
||||
relations=_entity_relations(entity),
|
||||
)
|
||||
|
||||
|
||||
def _entity_frontmatter(entity: Entity) -> dict:
|
||||
"""Build a frontmatter dict from an entity's database metadata.
|
||||
|
||||
Used for the notes being validated — their type and schema ref are
|
||||
unlikely to change between syncs.
|
||||
"""
|
||||
fm = dict(entity.entity_metadata) if entity.entity_metadata else {}
|
||||
if entity.note_type:
|
||||
fm.setdefault("type", entity.note_type)
|
||||
return fm
|
||||
|
||||
|
||||
async def _schema_frontmatter_from_file(
|
||||
file_service: FileServiceV2ExternalDep,
|
||||
entity: Entity,
|
||||
) -> dict:
|
||||
"""Read a schema entity's frontmatter directly from its file.
|
||||
|
||||
Schema definitions (field declarations, validation mode) are the source
|
||||
of truth for validation. Reading from the file ensures schema-validate
|
||||
always uses the latest settings, even when the file watcher hasn't
|
||||
synced changes to entity_metadata in the database.
|
||||
"""
|
||||
try:
|
||||
content = await file_service.read_file_content(entity.file_path)
|
||||
post = frontmatter.loads(content)
|
||||
metadata = dict(post.metadata)
|
||||
|
||||
# Trigger: file is mid-edit and missing required schema fields
|
||||
# Why: parse_schema_note() raises ValueError for missing entity/schema,
|
||||
# which would turn validation into a 500 response
|
||||
# Outcome: fall back to last-known-good database metadata
|
||||
if not metadata.get("entity") or not isinstance(metadata.get("schema"), dict):
|
||||
logger.warning(
|
||||
"Schema file has incomplete frontmatter, falling back to database metadata",
|
||||
file_path=entity.file_path,
|
||||
)
|
||||
return _entity_frontmatter(entity)
|
||||
|
||||
return metadata
|
||||
except Exception:
|
||||
# Trigger: file is missing, unreadable, or has malformed frontmatter
|
||||
# Why: fall back to database metadata rather than failing validation entirely
|
||||
# Outcome: behaves like before this change — uses potentially stale data
|
||||
logger.warning(
|
||||
"Failed to read schema file, falling back to database metadata",
|
||||
file_path=entity.file_path,
|
||||
)
|
||||
return _entity_frontmatter(entity)
|
||||
|
||||
|
||||
# --- Validation ---
|
||||
|
||||
|
||||
@router.post("/schema/validate", response_model=ValidationReport)
|
||||
async def validate_schema(
|
||||
entity_repository: EntityRepositoryV2ExternalDep,
|
||||
file_service: FileServiceV2ExternalDep,
|
||||
link_resolver: LinkResolverV2ExternalDep,
|
||||
project_id: str = Path(..., description="Project external UUID"),
|
||||
note_type: str | None = Query(None, description="Note type to validate"),
|
||||
identifier: str | None = Query(None, description="Specific note identifier"),
|
||||
):
|
||||
"""Validate notes against their resolved schemas.
|
||||
|
||||
Validates a specific note (by identifier) or all notes of a given type.
|
||||
Returns warnings/errors based on the schema's validation mode.
|
||||
|
||||
Schema definitions are read directly from their files to ensure the
|
||||
latest settings (validation mode, field declarations) are always used,
|
||||
even when file changes haven't been synced to the database yet.
|
||||
"""
|
||||
results: list[NoteValidationResponse] = []
|
||||
|
||||
# --- Single note validation ---
|
||||
if identifier:
|
||||
# Resolve identifier flexibly (permalink, title, path, fuzzy)
|
||||
# to match how read_note and other tools resolve identifiers
|
||||
entity = await link_resolver.resolve_link(identifier)
|
||||
if not entity:
|
||||
return ValidationReport(note_type=note_type, total_notes=0, total_entities=0)
|
||||
|
||||
frontmatter = _entity_frontmatter(entity)
|
||||
schema_ref = frontmatter.get("schema")
|
||||
|
||||
async def search_fn(query: str) -> list[dict]:
|
||||
entities = await _find_schema_entities(
|
||||
entity_repository,
|
||||
query,
|
||||
allow_reference_match=isinstance(schema_ref, str) and query == schema_ref,
|
||||
)
|
||||
return [await _schema_frontmatter_from_file(file_service, e) for e in entities]
|
||||
|
||||
schema_def = await resolve_schema(frontmatter, search_fn)
|
||||
if schema_def:
|
||||
result = validate_note(
|
||||
entity.title or entity.permalink or identifier,
|
||||
schema_def,
|
||||
_entity_observations(entity),
|
||||
_entity_relations(entity),
|
||||
frontmatter=frontmatter,
|
||||
)
|
||||
results.append(_to_note_validation_response(result))
|
||||
|
||||
return ValidationReport(
|
||||
note_type=note_type or entity.note_type,
|
||||
total_notes=len(results),
|
||||
total_entities=1,
|
||||
valid_count=1 if (results and results[0].passed) else 0,
|
||||
warning_count=sum(len(r.warnings) for r in results),
|
||||
error_count=sum(len(r.errors) for r in results),
|
||||
results=results,
|
||||
)
|
||||
|
||||
# --- Batch validation by note type ---
|
||||
entities = await _find_by_note_type(entity_repository, note_type) if note_type else []
|
||||
|
||||
for entity in entities:
|
||||
frontmatter = _entity_frontmatter(entity)
|
||||
schema_ref = frontmatter.get("schema")
|
||||
|
||||
async def search_fn(query: str) -> list[dict]:
|
||||
entities = await _find_schema_entities(
|
||||
entity_repository,
|
||||
query,
|
||||
allow_reference_match=isinstance(schema_ref, str) and query == schema_ref,
|
||||
)
|
||||
return [await _schema_frontmatter_from_file(file_service, e) for e in entities]
|
||||
|
||||
schema_def = await resolve_schema(frontmatter, search_fn)
|
||||
if schema_def:
|
||||
result = validate_note(
|
||||
entity.title or entity.permalink or entity.file_path,
|
||||
schema_def,
|
||||
_entity_observations(entity),
|
||||
_entity_relations(entity),
|
||||
frontmatter=frontmatter,
|
||||
)
|
||||
results.append(_to_note_validation_response(result))
|
||||
|
||||
valid = sum(1 for r in results if r.passed)
|
||||
return ValidationReport(
|
||||
note_type=note_type,
|
||||
total_notes=len(results),
|
||||
total_entities=len(entities),
|
||||
valid_count=valid,
|
||||
warning_count=sum(len(r.warnings) for r in results),
|
||||
error_count=sum(len(r.errors) for r in results),
|
||||
results=results,
|
||||
)
|
||||
|
||||
|
||||
# --- Inference ---
|
||||
|
||||
|
||||
@router.post("/schema/infer", response_model=InferenceReport)
|
||||
async def infer_schema_endpoint(
|
||||
entity_repository: EntityRepositoryV2ExternalDep,
|
||||
project_id: str = Path(..., description="Project external UUID"),
|
||||
note_type: str = Query(..., description="Note type to analyze"),
|
||||
threshold: float = Query(0.25, description="Minimum frequency for optional fields"),
|
||||
):
|
||||
"""Infer a schema from existing notes of a given type.
|
||||
|
||||
Examines observation categories and relation types across all notes
|
||||
of the given type. Returns frequency analysis and suggested Picoschema.
|
||||
"""
|
||||
entities = await _find_by_note_type(entity_repository, note_type)
|
||||
notes_data = [_entity_to_note_data(entity) for entity in entities]
|
||||
|
||||
result = infer_schema(note_type, notes_data, optional_threshold=threshold)
|
||||
|
||||
return InferenceReport(
|
||||
note_type=result.note_type,
|
||||
notes_analyzed=result.notes_analyzed,
|
||||
field_frequencies=[
|
||||
FieldFrequencyResponse(
|
||||
name=f.name,
|
||||
source=f.source,
|
||||
count=f.count,
|
||||
total=f.total,
|
||||
percentage=f.percentage,
|
||||
sample_values=f.sample_values,
|
||||
is_array=f.is_array,
|
||||
target_type=f.target_type,
|
||||
)
|
||||
for f in result.field_frequencies
|
||||
],
|
||||
suggested_schema=result.suggested_schema,
|
||||
suggested_required=result.suggested_required,
|
||||
suggested_optional=result.suggested_optional,
|
||||
excluded=result.excluded,
|
||||
)
|
||||
|
||||
|
||||
# --- Drift Detection ---
|
||||
|
||||
|
||||
@router.get("/schema/diff/{note_type}", response_model=DriftReport)
|
||||
async def diff_schema_endpoint(
|
||||
entity_repository: EntityRepositoryV2ExternalDep,
|
||||
file_service: FileServiceV2ExternalDep,
|
||||
note_type: str = Path(..., description="Note type to check for drift"),
|
||||
project_id: str = Path(..., description="Project external UUID"),
|
||||
):
|
||||
"""Show drift between a schema definition and actual note usage.
|
||||
|
||||
Compares the existing schema for an entity type against how notes
|
||||
of that type are actually structured. Identifies new fields, dropped
|
||||
fields, and cardinality changes.
|
||||
"""
|
||||
|
||||
async def search_fn(query: str) -> list[dict]:
|
||||
entities = await _find_schema_entities(entity_repository, query)
|
||||
return [await _schema_frontmatter_from_file(file_service, e) for e in entities]
|
||||
|
||||
# Resolve schema by note type
|
||||
schema_frontmatter = {"type": note_type}
|
||||
schema_def = await resolve_schema(schema_frontmatter, search_fn)
|
||||
|
||||
if not schema_def:
|
||||
return DriftReport(note_type=note_type, schema_found=False)
|
||||
|
||||
# Collect all notes of this type
|
||||
entities = await _find_by_note_type(entity_repository, note_type)
|
||||
notes_data = [_entity_to_note_data(entity) for entity in entities]
|
||||
|
||||
result = diff_schema(schema_def, notes_data)
|
||||
|
||||
return DriftReport(
|
||||
note_type=note_type,
|
||||
new_fields=[
|
||||
DriftFieldResponse(
|
||||
name=f.name,
|
||||
source=f.source,
|
||||
count=f.count,
|
||||
total=f.total,
|
||||
percentage=f.percentage,
|
||||
)
|
||||
for f in result.new_fields
|
||||
],
|
||||
dropped_fields=[
|
||||
DriftFieldResponse(
|
||||
name=f.name,
|
||||
source=f.source,
|
||||
count=f.count,
|
||||
total=f.total,
|
||||
percentage=f.percentage,
|
||||
)
|
||||
for f in result.dropped_fields
|
||||
],
|
||||
cardinality_changes=result.cardinality_changes,
|
||||
)
|
||||
|
||||
|
||||
# --- Helpers ---
|
||||
|
||||
|
||||
async def _find_by_note_type(
|
||||
entity_repository: EntityRepositoryV2ExternalDep,
|
||||
note_type: str,
|
||||
) -> list[Entity]:
|
||||
"""Find all entities of a given type using the repository's select pattern."""
|
||||
query = entity_repository.select().where(Entity.note_type == note_type)
|
||||
result = await entity_repository.execute_query(query)
|
||||
return list(result.scalars().all())
|
||||
|
||||
|
||||
async def _find_schema_entities(
|
||||
entity_repository: EntityRepositoryV2ExternalDep,
|
||||
target_note_type: str,
|
||||
*,
|
||||
allow_reference_match: bool = False,
|
||||
) -> list[Entity]:
|
||||
"""Find schema entities for resolver lookups.
|
||||
|
||||
Resolution strategy:
|
||||
1) Always try exact entity_metadata['entity'] match (for implicit type lookup
|
||||
and explicit references that use entity names)
|
||||
2) Only when allow_reference_match=True and no entity match was found, try
|
||||
exact reference matching by title/permalink (explicit schema references)
|
||||
"""
|
||||
query = entity_repository.select().where(Entity.note_type == "schema")
|
||||
result = await entity_repository.execute_query(query)
|
||||
entities = list(result.scalars().all())
|
||||
|
||||
normalized_target = generate_permalink(target_note_type)
|
||||
|
||||
entity_matches = [
|
||||
e
|
||||
for e in entities
|
||||
if e.entity_metadata
|
||||
and isinstance(e.entity_metadata.get("entity"), str)
|
||||
and generate_permalink(e.entity_metadata["entity"]) == normalized_target
|
||||
]
|
||||
if entity_matches:
|
||||
return entity_matches
|
||||
|
||||
if not allow_reference_match:
|
||||
return []
|
||||
|
||||
reference_matches: list[Entity] = []
|
||||
for entity in entities:
|
||||
candidate_refs: list[str] = []
|
||||
if entity.title:
|
||||
candidate_refs.append(entity.title)
|
||||
if entity.permalink:
|
||||
candidate_refs.append(entity.permalink)
|
||||
candidate_refs.append(FilePath(entity.permalink).name)
|
||||
|
||||
if any(generate_permalink(ref) == normalized_target for ref in candidate_refs):
|
||||
reference_matches.append(entity)
|
||||
|
||||
return reference_matches
|
||||
|
||||
|
||||
def _to_note_validation_response(result) -> NoteValidationResponse:
|
||||
"""Convert a core ValidationResult to a Pydantic response model."""
|
||||
return NoteValidationResponse(
|
||||
note_identifier=result.note_identifier,
|
||||
schema_entity=result.schema_entity,
|
||||
passed=result.passed,
|
||||
field_results=[
|
||||
FieldResultResponse(
|
||||
field_name=fr.field.name,
|
||||
field_type=fr.field.type,
|
||||
required=fr.field.required,
|
||||
status=fr.status,
|
||||
values=fr.values,
|
||||
message=fr.message,
|
||||
)
|
||||
for fr in result.field_results
|
||||
],
|
||||
unmatched_observations=result.unmatched_observations,
|
||||
unmatched_relations=result.unmatched_relations,
|
||||
warnings=result.warnings,
|
||||
errors=result.errors,
|
||||
)
|
||||
@@ -4,11 +4,23 @@ This router uses external_id UUIDs for stable, API-friendly routing.
|
||||
V1 uses string-based project names which are less efficient and less stable.
|
||||
"""
|
||||
|
||||
from fastapi import APIRouter, BackgroundTasks, Path
|
||||
import asyncio
|
||||
|
||||
from basic_memory.api.routers.utils import to_search_results
|
||||
from basic_memory.schemas.search import SearchQuery, SearchResponse
|
||||
from basic_memory.deps import SearchServiceV2ExternalDep, EntityServiceV2ExternalDep
|
||||
from fastapi import APIRouter, HTTPException, Path
|
||||
|
||||
import logfire
|
||||
from basic_memory.api.v2.utils import to_search_results
|
||||
from basic_memory.repository.semantic_errors import (
|
||||
SemanticDependenciesMissingError,
|
||||
SemanticSearchDisabledError,
|
||||
)
|
||||
from basic_memory.schemas.search import SearchQuery, SearchResponse, SearchRetrievalMode
|
||||
from basic_memory.deps import (
|
||||
SearchServiceV2ExternalDep,
|
||||
EntityServiceV2ExternalDep,
|
||||
TaskSchedulerDep,
|
||||
ProjectExternalIdPathDep,
|
||||
)
|
||||
|
||||
# Note: No prefix here - it's added during registration as /v2/{project_id}/search
|
||||
router = APIRouter(tags=["search"])
|
||||
@@ -38,22 +50,93 @@ async def search(
|
||||
Returns:
|
||||
SearchResponse with paginated search results
|
||||
"""
|
||||
limit = page_size
|
||||
offset = (page - 1) * page_size
|
||||
results = await search_service.search(query, limit=limit, offset=offset)
|
||||
search_results = await to_search_results(entity_service, results)
|
||||
return SearchResponse(
|
||||
results=search_results,
|
||||
current_page=page,
|
||||
with logfire.span(
|
||||
"api.request.search",
|
||||
entrypoint="api",
|
||||
domain="search",
|
||||
action="search",
|
||||
page=page,
|
||||
page_size=page_size,
|
||||
)
|
||||
retrieval_mode=query.retrieval_mode.value,
|
||||
has_query=bool(
|
||||
(query.text and query.text.strip())
|
||||
or query.title
|
||||
or query.permalink
|
||||
or query.permalink_match
|
||||
),
|
||||
has_filters=bool(query.note_types or query.entity_types or query.metadata_filters),
|
||||
):
|
||||
offset = (page - 1) * page_size
|
||||
exact_count_available = query.retrieval_mode == SearchRetrievalMode.FTS
|
||||
try:
|
||||
with logfire.span(
|
||||
"api.search.search.execute_query",
|
||||
domain="search",
|
||||
action="search",
|
||||
phase="execute_query",
|
||||
page=page,
|
||||
page_size=page_size,
|
||||
):
|
||||
if exact_count_available:
|
||||
results, total = await asyncio.gather(
|
||||
search_service.search(query, limit=page_size, offset=offset),
|
||||
search_service.count(query),
|
||||
)
|
||||
else:
|
||||
results = await search_service.search(query, limit=page_size + 1, offset=offset)
|
||||
total = 0
|
||||
except SemanticSearchDisabledError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
except SemanticDependenciesMissingError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
|
||||
with logfire.span(
|
||||
"api.search.search.paginate_results",
|
||||
domain="search",
|
||||
action="search",
|
||||
phase="paginate_results",
|
||||
result_count=len(results),
|
||||
):
|
||||
if exact_count_available:
|
||||
has_more = offset + len(results) < total
|
||||
else:
|
||||
# Trigger: semantic modes would need another vector/hybrid retrieval to count.
|
||||
# Why: search requests should not pay for a second semantic pass.
|
||||
# Outcome: preserve probe pagination for semantic search and leave total at 0.
|
||||
has_more = len(results) > page_size
|
||||
if has_more:
|
||||
results = results[:page_size]
|
||||
|
||||
with logfire.span(
|
||||
"api.search.search.hydrate_results",
|
||||
domain="search",
|
||||
action="search",
|
||||
phase="hydrate_results",
|
||||
result_count=len(results),
|
||||
):
|
||||
search_results = await to_search_results(entity_service, results)
|
||||
with logfire.span(
|
||||
"api.search.search.build_response",
|
||||
domain="search",
|
||||
action="search",
|
||||
phase="build_response",
|
||||
result_count=len(search_results),
|
||||
):
|
||||
return SearchResponse(
|
||||
results=search_results,
|
||||
current_page=page,
|
||||
page_size=page_size,
|
||||
total=total,
|
||||
has_more=has_more,
|
||||
)
|
||||
|
||||
|
||||
@router.post("/search/reindex")
|
||||
async def reindex(
|
||||
background_tasks: BackgroundTasks,
|
||||
search_service: SearchServiceV2ExternalDep,
|
||||
project_id: str = Path(..., description="Project external UUID"),
|
||||
task_scheduler: TaskSchedulerDep,
|
||||
project_id: ProjectExternalIdPathDep,
|
||||
):
|
||||
"""Recreate and populate the search index for a project.
|
||||
|
||||
@@ -63,11 +146,10 @@ async def reindex(
|
||||
|
||||
Args:
|
||||
project_id: Project external UUID from URL path
|
||||
background_tasks: FastAPI background tasks handler
|
||||
search_service: Search service scoped to project
|
||||
task_scheduler: Task scheduler for background work
|
||||
|
||||
Returns:
|
||||
Status message indicating reindex has been initiated
|
||||
"""
|
||||
await search_service.reindex_all(background_tasks=background_tasks)
|
||||
task_scheduler.schedule("reindex_project", project_id=project_id)
|
||||
return {"status": "ok", "message": "Reindex initiated"}
|
||||
|
||||
@@ -0,0 +1,278 @@
|
||||
from typing import Any, Protocol, Optional, List, Sequence
|
||||
|
||||
import logfire
|
||||
from basic_memory.repository.search_repository import SearchIndexRow
|
||||
from basic_memory.schemas.memory import (
|
||||
EntitySummary,
|
||||
ObservationSummary,
|
||||
RelationSummary,
|
||||
MemoryMetadata,
|
||||
GraphContext,
|
||||
ContextResult,
|
||||
)
|
||||
from basic_memory.schemas.search import SearchItemType, SearchResult
|
||||
from basic_memory.services.context_service import (
|
||||
ContextResultRow,
|
||||
ContextResult as ServiceContextResult,
|
||||
)
|
||||
|
||||
|
||||
class EntityBatchLookup(Protocol):
|
||||
async def find_by_ids_for_hydration(self, ids: List[int]) -> Sequence[Any]: ...
|
||||
|
||||
|
||||
class EntityServiceBatchLookup(Protocol):
|
||||
async def get_entities_by_id(self, ids: List[int]) -> Sequence[Any]: ...
|
||||
|
||||
|
||||
def _required_str(value: str | None, field_name: str) -> str:
|
||||
"""Return a required search field or fail before producing invalid response data."""
|
||||
if value is None:
|
||||
raise ValueError(f"Search result is missing required field: {field_name}")
|
||||
return value
|
||||
|
||||
|
||||
def _search_item_type(value: str | SearchItemType) -> SearchItemType:
|
||||
"""Normalize repository row type strings into the public search enum."""
|
||||
return value if isinstance(value, SearchItemType) else SearchItemType(value)
|
||||
|
||||
|
||||
async def to_graph_context(
|
||||
context_result: ServiceContextResult,
|
||||
entity_repository: EntityBatchLookup,
|
||||
page: Optional[int] = None,
|
||||
page_size: Optional[int] = None,
|
||||
) -> GraphContext:
|
||||
with logfire.span(
|
||||
"memory.hydrate_context",
|
||||
domain="memory",
|
||||
action="build_context",
|
||||
phase="hydrate_context",
|
||||
page=page,
|
||||
page_size=page_size,
|
||||
result_count=len(context_result.results),
|
||||
):
|
||||
# First pass: collect all entity IDs needed for external_id lookup
|
||||
# This includes: entity primary results, observation parent entities, relation from/to entities
|
||||
entity_ids_needed: set[int] = set()
|
||||
for context_item in context_result.results:
|
||||
for item in (
|
||||
[context_item.primary_result]
|
||||
+ context_item.observations
|
||||
+ context_item.related_results
|
||||
):
|
||||
item_type = _search_item_type(item.type)
|
||||
if item_type == SearchItemType.ENTITY:
|
||||
# Entity's own ID for its external_id
|
||||
entity_ids_needed.add(item.id)
|
||||
elif item_type == SearchItemType.OBSERVATION:
|
||||
# Parent entity ID for entity_external_id
|
||||
if item.entity_id:
|
||||
entity_ids_needed.add(item.entity_id)
|
||||
elif item_type == SearchItemType.RELATION:
|
||||
# Source and target entity IDs for external_ids
|
||||
if item.from_id:
|
||||
entity_ids_needed.add(item.from_id)
|
||||
if item.to_id:
|
||||
entity_ids_needed.add(item.to_id)
|
||||
|
||||
# Batch fetch just the entity fields needed to shape the response.
|
||||
entity_title_lookup: dict[int, str] = {}
|
||||
entity_external_id_lookup: dict[int, str] = {}
|
||||
if entity_ids_needed:
|
||||
with logfire.span(
|
||||
"memory.hydrate_context.lookup_entities",
|
||||
domain="memory",
|
||||
action="build_context",
|
||||
phase="lookup_entities",
|
||||
result_count=len(entity_ids_needed),
|
||||
):
|
||||
entities = await entity_repository.find_by_ids_for_hydration(
|
||||
list(entity_ids_needed)
|
||||
)
|
||||
for e in entities:
|
||||
entity_title_lookup[e.id] = e.title
|
||||
entity_external_id_lookup[e.id] = e.external_id
|
||||
|
||||
# Helper function to convert items to summaries
|
||||
def to_summary(
|
||||
item: SearchIndexRow | ContextResultRow,
|
||||
) -> EntitySummary | ObservationSummary | RelationSummary:
|
||||
item_type = _search_item_type(item.type)
|
||||
match item_type:
|
||||
case SearchItemType.ENTITY:
|
||||
return EntitySummary(
|
||||
external_id=entity_external_id_lookup.get(item.id, ""),
|
||||
entity_id=item.id,
|
||||
title=_required_str(item.title, "title"),
|
||||
permalink=item.permalink,
|
||||
content=item.content,
|
||||
file_path=_required_str(item.file_path, "file_path"),
|
||||
created_at=item.created_at,
|
||||
)
|
||||
case SearchItemType.OBSERVATION:
|
||||
entity_ext_id = None
|
||||
entity_title = None
|
||||
if item.entity_id:
|
||||
entity_ext_id = entity_external_id_lookup.get(item.entity_id)
|
||||
entity_title = entity_title_lookup.get(item.entity_id)
|
||||
return ObservationSummary(
|
||||
observation_id=item.id,
|
||||
entity_id=item.entity_id,
|
||||
entity_external_id=entity_ext_id,
|
||||
title=entity_title,
|
||||
file_path=_required_str(item.file_path, "file_path"),
|
||||
category=_required_str(item.category, "category"),
|
||||
content=_required_str(item.content, "content"),
|
||||
permalink=_required_str(item.permalink, "permalink"),
|
||||
created_at=item.created_at,
|
||||
)
|
||||
case SearchItemType.RELATION:
|
||||
from_title = entity_title_lookup.get(item.from_id) if item.from_id else None
|
||||
to_title = entity_title_lookup.get(item.to_id) if item.to_id else None
|
||||
from_ext_id = (
|
||||
entity_external_id_lookup.get(item.from_id) if item.from_id else None
|
||||
)
|
||||
to_ext_id = entity_external_id_lookup.get(item.to_id) if item.to_id else None
|
||||
return RelationSummary(
|
||||
relation_id=item.id,
|
||||
entity_id=item.entity_id,
|
||||
title=_required_str(item.title, "title"),
|
||||
file_path=_required_str(item.file_path, "file_path"),
|
||||
permalink=_required_str(item.permalink, "permalink"),
|
||||
relation_type=_required_str(item.relation_type, "relation_type"),
|
||||
from_entity=from_title,
|
||||
from_entity_id=item.from_id,
|
||||
from_entity_external_id=from_ext_id,
|
||||
to_entity=to_title,
|
||||
to_entity_id=item.to_id,
|
||||
to_entity_external_id=to_ext_id,
|
||||
created_at=item.created_at,
|
||||
)
|
||||
|
||||
with logfire.span(
|
||||
"memory.hydrate_context.shape_results",
|
||||
domain="memory",
|
||||
action="build_context",
|
||||
phase="shape_results",
|
||||
result_count=len(context_result.results),
|
||||
):
|
||||
hierarchical_results = []
|
||||
for context_item in context_result.results:
|
||||
primary_result = to_summary(context_item.primary_result)
|
||||
observations = [
|
||||
summary
|
||||
for summary in (to_summary(obs) for obs in context_item.observations)
|
||||
if isinstance(summary, ObservationSummary)
|
||||
]
|
||||
related = [to_summary(rel) for rel in context_item.related_results]
|
||||
hierarchical_results.append(
|
||||
ContextResult(
|
||||
primary_result=primary_result,
|
||||
observations=observations,
|
||||
related_results=related,
|
||||
)
|
||||
)
|
||||
|
||||
metadata = MemoryMetadata(
|
||||
uri=context_result.metadata.uri,
|
||||
types=context_result.metadata.types,
|
||||
depth=context_result.metadata.depth,
|
||||
timeframe=context_result.metadata.timeframe,
|
||||
generated_at=context_result.metadata.generated_at,
|
||||
primary_count=context_result.metadata.primary_count,
|
||||
related_count=context_result.metadata.related_count,
|
||||
total_results=context_result.metadata.primary_count
|
||||
+ context_result.metadata.related_count,
|
||||
total_relations=context_result.metadata.total_relations,
|
||||
total_observations=context_result.metadata.total_observations,
|
||||
)
|
||||
|
||||
return GraphContext(
|
||||
results=hierarchical_results,
|
||||
metadata=metadata,
|
||||
page=page,
|
||||
page_size=page_size,
|
||||
has_more=context_result.metadata.has_more,
|
||||
)
|
||||
|
||||
|
||||
async def to_search_results(
|
||||
entity_service: EntityServiceBatchLookup, results: List[SearchIndexRow]
|
||||
) -> list[SearchResult]:
|
||||
with logfire.span(
|
||||
"search.hydrate_results",
|
||||
domain="search",
|
||||
action="search",
|
||||
phase="hydrate_results",
|
||||
result_count=len(results),
|
||||
):
|
||||
# Collect all unique entity IDs across all results in a single pass
|
||||
# This avoids N+1 queries — one batch fetch instead of one per result
|
||||
all_entity_ids: set[int] = set()
|
||||
for result in results:
|
||||
for eid in (result.entity_id, result.from_id, result.to_id):
|
||||
if eid is not None:
|
||||
all_entity_ids.add(eid)
|
||||
|
||||
# Single batch fetch for all entities
|
||||
entities_by_id: dict[int, Any] = {}
|
||||
with logfire.span(
|
||||
"search.hydrate_results.fetch_entities",
|
||||
domain="search",
|
||||
action="search",
|
||||
phase="fetch_entities",
|
||||
result_count=len(all_entity_ids),
|
||||
):
|
||||
if all_entity_ids:
|
||||
entities = await entity_service.get_entities_by_id(list(all_entity_ids))
|
||||
entities_by_id = {e.id: e for e in entities}
|
||||
|
||||
search_results = []
|
||||
with logfire.span(
|
||||
"search.hydrate_results.shape_results",
|
||||
domain="search",
|
||||
action="search",
|
||||
phase="shape_results",
|
||||
result_count=len(results),
|
||||
):
|
||||
for result in results:
|
||||
entity_id = None
|
||||
observation_id = None
|
||||
relation_id = None
|
||||
|
||||
if result.type == SearchItemType.ENTITY:
|
||||
entity_id = result.id
|
||||
elif result.type == SearchItemType.OBSERVATION:
|
||||
observation_id = result.id
|
||||
entity_id = result.entity_id
|
||||
elif result.type == SearchItemType.RELATION:
|
||||
relation_id = result.id
|
||||
entity_id = result.entity_id
|
||||
|
||||
# Look up entities by their specific IDs
|
||||
parent_entity = entities_by_id.get(result.entity_id) if result.entity_id else None
|
||||
from_entity = entities_by_id.get(result.from_id) if result.from_id else None
|
||||
to_entity = entities_by_id.get(result.to_id) if result.to_id else None
|
||||
|
||||
search_results.append(
|
||||
SearchResult(
|
||||
title=_required_str(result.title, "title"),
|
||||
type=_search_item_type(result.type),
|
||||
permalink=result.permalink,
|
||||
score=result.score if result.score is not None else 0.0,
|
||||
entity=parent_entity.permalink if parent_entity else None,
|
||||
content=result.content,
|
||||
matched_chunk=result.matched_chunk_text,
|
||||
file_path=_required_str(result.file_path, "file_path"),
|
||||
metadata=result.metadata,
|
||||
entity_id=entity_id,
|
||||
observation_id=observation_id,
|
||||
relation_id=relation_id,
|
||||
category=result.category,
|
||||
from_entity=from_entity.permalink if from_entity else None,
|
||||
to_entity=to_entity.permalink if to_entity else None,
|
||||
relation_type=result.relation_type,
|
||||
)
|
||||
)
|
||||
return search_results
|
||||
@@ -0,0 +1,114 @@
|
||||
"""Lightweight CLI analytics via Umami event collector.
|
||||
|
||||
Sends anonymous, non-blocking usage events to help understand how the
|
||||
CLI-to-cloud conversion funnel performs. No PII, no fingerprinting,
|
||||
no cookies. Respects the same opt-out mechanisms as promo messaging.
|
||||
|
||||
Events are fire-and-forget — analytics never blocks or breaks the CLI.
|
||||
|
||||
Defaults point to the Basic Memory Umami Cloud instance. Override via:
|
||||
BASIC_MEMORY_UMAMI_HOST — Custom Umami instance URL
|
||||
BASIC_MEMORY_UMAMI_SITE_ID — Custom Website ID
|
||||
Opt out entirely with BASIC_MEMORY_NO_PROMOS=1.
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import threading
|
||||
import urllib.request
|
||||
from typing import Optional
|
||||
|
||||
import basic_memory
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Configuration — defaults baked in, overridable via environment
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_DEFAULT_UMAMI_HOST = "https://api-gateway.umami.dev"
|
||||
_DEFAULT_UMAMI_SITE_ID = "f6479898-ebaf-4e60-bce2-6dc60a3f6c5c"
|
||||
|
||||
|
||||
def _umami_host() -> Optional[str]:
|
||||
return os.getenv("BASIC_MEMORY_UMAMI_HOST", "").strip() or _DEFAULT_UMAMI_HOST
|
||||
|
||||
|
||||
def _umami_site_id() -> Optional[str]:
|
||||
return os.getenv("BASIC_MEMORY_UMAMI_SITE_ID", "").strip() or _DEFAULT_UMAMI_SITE_ID
|
||||
|
||||
|
||||
def _analytics_disabled() -> bool:
|
||||
"""True when analytics should not fire."""
|
||||
value = os.getenv("BASIC_MEMORY_NO_PROMOS", "").strip().lower()
|
||||
return value in {"1", "true", "yes"}
|
||||
|
||||
|
||||
def _is_configured() -> bool:
|
||||
"""True when both host and site ID are available."""
|
||||
return _umami_host() is not None and _umami_site_id() is not None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Public API
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Well-known event names for the promo/cloud funnel
|
||||
EVENT_PROMO_SHOWN = "cli-promo-shown"
|
||||
EVENT_PROMO_OPTED_OUT = "cli-promo-opted-out"
|
||||
EVENT_CLOUD_LOGIN_STARTED = "cli-cloud-login-started"
|
||||
EVENT_CLOUD_LOGIN_SUCCESS = "cli-cloud-login-success"
|
||||
EVENT_CLOUD_LOGIN_SUB_REQUIRED = "cli-cloud-login-sub-required"
|
||||
|
||||
|
||||
def track(event_name: str, data: Optional[dict] = None) -> None:
|
||||
"""Send an analytics event to Umami. Non-blocking, silent on failure.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
event_name:
|
||||
Short kebab-case name (e.g. "cli-promo-shown").
|
||||
data:
|
||||
Optional dict of event properties (all values should be strings/numbers).
|
||||
"""
|
||||
if _analytics_disabled() or not _is_configured():
|
||||
return
|
||||
|
||||
host = _umami_host()
|
||||
site_id = _umami_site_id()
|
||||
|
||||
# Umami v2 /api/send requires "type" at top level alongside "payload"
|
||||
payload = {
|
||||
"type": "event",
|
||||
"payload": {
|
||||
"hostname": "cli.basicmemory.com",
|
||||
"language": "en",
|
||||
"url": f"/cli/{event_name}",
|
||||
"website": site_id,
|
||||
"name": event_name,
|
||||
"data": {
|
||||
"version": basic_memory.__version__,
|
||||
**(data or {}),
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
def _send():
|
||||
try:
|
||||
req = urllib.request.Request(
|
||||
f"{host}/api/send",
|
||||
data=json.dumps(payload).encode("utf-8"),
|
||||
headers={
|
||||
"Content-Type": "application/json",
|
||||
# Umami's bot detection rejects non-browser User-Agents
|
||||
"User-Agent": "Mozilla/5.0 (compatible; BasicMemoryCLI/"
|
||||
f"{basic_memory.__version__})",
|
||||
},
|
||||
)
|
||||
urllib.request.urlopen(req, timeout=3)
|
||||
except Exception:
|
||||
pass # Never break the CLI for analytics
|
||||
|
||||
# Non-daemon so the process waits for the request to complete.
|
||||
# The 3s urllib timeout caps the worst-case exit delay.
|
||||
t = threading.Thread(target=_send)
|
||||
t.start()
|
||||
@@ -8,8 +8,11 @@ from typing import Optional # noqa: E402
|
||||
|
||||
import typer # noqa: E402
|
||||
|
||||
from basic_memory.cli.auto_update import maybe_run_periodic_auto_update # noqa: E402
|
||||
from basic_memory.cli.container import CliContainer, set_container # noqa: E402
|
||||
from basic_memory.cli.promo import maybe_show_cloud_promo, maybe_show_init_line # noqa: E402
|
||||
from basic_memory.config import init_cli_logging # noqa: E402
|
||||
import logfire # noqa: E402
|
||||
|
||||
|
||||
def version_callback(value: bool) -> None:
|
||||
@@ -40,17 +43,50 @@ def app_callback(
|
||||
|
||||
# Initialize logging for CLI (file only, no stdout)
|
||||
init_cli_logging()
|
||||
command_name = ctx.invoked_subcommand or "root"
|
||||
ctx.with_resource(
|
||||
logfire.span(
|
||||
f"cli.command.{command_name}",
|
||||
entrypoint="cli",
|
||||
command_name=command_name,
|
||||
)
|
||||
)
|
||||
|
||||
# --- Composition Root ---
|
||||
# Create container and read config (single point of config access)
|
||||
container = CliContainer.create()
|
||||
set_container(container)
|
||||
|
||||
# Trigger: first-run init confirmation before command output.
|
||||
# Why: informational "initialized" message belongs above command results, not in the upsell panel.
|
||||
# Outcome: one-time plain line printed before the subcommand runs.
|
||||
maybe_show_init_line(ctx.invoked_subcommand)
|
||||
|
||||
# Trigger: register post-command messaging callbacks.
|
||||
# Why: informational/promo/update output belongs below command results.
|
||||
# Outcome: command output remains primary, with optional follow-up notices afterwards.
|
||||
def _post_command_messages() -> None:
|
||||
maybe_show_cloud_promo(ctx.invoked_subcommand)
|
||||
maybe_run_periodic_auto_update(ctx.invoked_subcommand)
|
||||
|
||||
ctx.call_on_close(_post_command_messages)
|
||||
|
||||
# Run initialization for commands that don't use the API
|
||||
# Skip for 'mcp' command - it has its own lifespan that handles initialization
|
||||
# Skip for API-using commands (status, sync, etc.) - they handle initialization via deps.py
|
||||
# Skip for 'reset' command - it manages its own database lifecycle
|
||||
skip_init_commands = {"mcp", "status", "sync", "project", "tool", "reset"}
|
||||
skip_init_commands = {
|
||||
"doctor",
|
||||
"mcp",
|
||||
"status",
|
||||
"sync",
|
||||
"project",
|
||||
"tool",
|
||||
"reset",
|
||||
"reindex",
|
||||
"update",
|
||||
"watch",
|
||||
}
|
||||
if (
|
||||
not version
|
||||
and ctx.invoked_subcommand is not None
|
||||
|
||||
@@ -0,0 +1,389 @@
|
||||
"""Automatic update checks and upgrades for the Basic Memory CLI."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timedelta
|
||||
from enum import Enum
|
||||
|
||||
from loguru import logger
|
||||
from packaging.version import InvalidVersion, Version
|
||||
from rich.console import Console
|
||||
|
||||
import basic_memory
|
||||
from basic_memory.config import ConfigManager
|
||||
|
||||
PACKAGE_NAME = "basic-memory"
|
||||
PYPI_JSON_URL = "https://pypi.org/pypi/basic-memory/json"
|
||||
|
||||
PYPI_TIMEOUT_SECONDS = 5
|
||||
BREW_OUTDATED_TIMEOUT_SECONDS = 60
|
||||
UV_UPGRADE_TIMEOUT_SECONDS = 180
|
||||
BREW_UPGRADE_TIMEOUT_SECONDS = 600
|
||||
|
||||
|
||||
class InstallSource(str, Enum):
|
||||
"""How the running CLI appears to have been installed."""
|
||||
|
||||
HOMEBREW = "homebrew"
|
||||
UV_TOOL = "uv_tool"
|
||||
UVX = "uvx"
|
||||
UNKNOWN = "unknown"
|
||||
|
||||
|
||||
class AutoUpdateStatus(str, Enum):
|
||||
"""Result classification for update checks and installs."""
|
||||
|
||||
SKIPPED = "skipped"
|
||||
UP_TO_DATE = "up_to_date"
|
||||
UPDATE_AVAILABLE = "update_available"
|
||||
UPDATED = "updated"
|
||||
FAILED = "failed"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class AutoUpdateResult:
|
||||
"""Structured result for update checks/install attempts."""
|
||||
|
||||
status: AutoUpdateStatus
|
||||
source: InstallSource
|
||||
checked: bool
|
||||
update_available: bool
|
||||
updated: bool
|
||||
latest_version: str | None = None
|
||||
message: str | None = None
|
||||
error: str | None = None
|
||||
restart_recommended: bool = False
|
||||
|
||||
|
||||
def detect_install_source(executable: str | None = None) -> InstallSource:
|
||||
"""Infer installation source from the active interpreter path."""
|
||||
active_executable = executable or sys.executable
|
||||
normalized = active_executable.lower().replace("\\", "/")
|
||||
|
||||
if "cellar/basic-memory" in normalized:
|
||||
return InstallSource.HOMEBREW
|
||||
if "uv/tools/basic-memory" in normalized:
|
||||
return InstallSource.UV_TOOL
|
||||
if "/uv/archive-" in normalized:
|
||||
return InstallSource.UVX
|
||||
return InstallSource.UNKNOWN
|
||||
|
||||
|
||||
def _is_interactive_session() -> bool:
|
||||
"""Return whether stdin/stdout are interactive terminals."""
|
||||
try:
|
||||
return sys.stdin.isatty() and sys.stdout.isatty()
|
||||
except ValueError:
|
||||
# Trigger: stdin/stdout may be closed during transport teardown.
|
||||
# Why: isatty() raises ValueError on closed descriptors.
|
||||
# Outcome: treat as non-interactive and suppress periodic output.
|
||||
return False
|
||||
|
||||
|
||||
def _run_subprocess(
|
||||
command: list[str],
|
||||
*,
|
||||
timeout_seconds: int,
|
||||
silent: bool,
|
||||
capture_output: bool,
|
||||
) -> subprocess.CompletedProcess[str]:
|
||||
"""Run a subprocess with explicit stdio behavior for protocol safety."""
|
||||
# Trigger: silent operation (MCP/background) with no need for subprocess output.
|
||||
# Why: prevent protocol/terminal pollution from child process output.
|
||||
# Outcome: stdout/stderr are discarded unless explicit capture is requested.
|
||||
use_devnull = silent and not capture_output
|
||||
stdout_target = subprocess.DEVNULL if use_devnull else subprocess.PIPE
|
||||
stderr_target = subprocess.DEVNULL if use_devnull else subprocess.PIPE
|
||||
|
||||
return subprocess.run(
|
||||
command,
|
||||
stdin=subprocess.DEVNULL,
|
||||
stdout=stdout_target,
|
||||
stderr=stderr_target,
|
||||
text=True,
|
||||
timeout=timeout_seconds,
|
||||
check=False,
|
||||
)
|
||||
|
||||
|
||||
def _version_from_pypi() -> str:
|
||||
"""Fetch the latest published package version from PyPI."""
|
||||
request = urllib.request.Request(
|
||||
PYPI_JSON_URL,
|
||||
headers={"User-Agent": f"basic-memory-cli/{basic_memory.__version__}"},
|
||||
)
|
||||
with urllib.request.urlopen(request, timeout=PYPI_TIMEOUT_SECONDS) as response:
|
||||
payload = json.loads(response.read().decode("utf-8"))
|
||||
latest = payload.get("info", {}).get("version")
|
||||
if not latest:
|
||||
raise RuntimeError("PyPI JSON response did not include info.version")
|
||||
return str(latest)
|
||||
|
||||
|
||||
def _check_homebrew_update_available(silent: bool) -> tuple[bool, str | None]:
|
||||
"""Check whether Homebrew reports an outdated basic-memory formula."""
|
||||
result = _run_subprocess(
|
||||
["brew", "outdated", "--quiet", PACKAGE_NAME],
|
||||
timeout_seconds=BREW_OUTDATED_TIMEOUT_SECONDS,
|
||||
silent=silent,
|
||||
capture_output=True,
|
||||
)
|
||||
# Trigger: brew outdated exits 1 when the formula IS outdated (with name on stdout).
|
||||
# Why: non-zero exit here means "outdated", not "error".
|
||||
# Outcome: check stdout for the package name to determine outdated status.
|
||||
stdout = (result.stdout or "").strip()
|
||||
is_outdated = PACKAGE_NAME in stdout
|
||||
return is_outdated, None
|
||||
|
||||
|
||||
def _check_pypi_update_available() -> tuple[bool, str]:
|
||||
"""Compare installed package version with PyPI latest version."""
|
||||
latest = _version_from_pypi()
|
||||
try:
|
||||
current_version = Version(basic_memory.__version__)
|
||||
latest_version = Version(latest)
|
||||
except InvalidVersion as exc:
|
||||
raise RuntimeError(
|
||||
f"Could not compare versions (current={basic_memory.__version__}, latest={latest})"
|
||||
) from exc
|
||||
|
||||
return latest_version > current_version, latest
|
||||
|
||||
|
||||
def _manual_update_hint(source: InstallSource) -> str:
|
||||
"""Return manager-appropriate manual update instructions."""
|
||||
if source == InstallSource.UV_TOOL:
|
||||
return "Run `uv tool upgrade basic-memory`."
|
||||
if source == InstallSource.HOMEBREW:
|
||||
return "Run `brew upgrade basic-memory`."
|
||||
return (
|
||||
"Automatic install is not supported for this environment. "
|
||||
"Update with your package manager (for pip: `python3 -m pip install -U basic-memory`)."
|
||||
)
|
||||
|
||||
|
||||
def _save_last_checked_timestamp(config_manager: ConfigManager, checked_at: datetime) -> None:
|
||||
"""Persist the timestamp for the most recent attempted update check."""
|
||||
config = config_manager.load_config()
|
||||
config.auto_update_last_checked_at = checked_at
|
||||
config_manager.save_config(config)
|
||||
|
||||
|
||||
def run_auto_update(
|
||||
*,
|
||||
force: bool = False,
|
||||
check_only: bool = False,
|
||||
silent: bool = False,
|
||||
config_manager: ConfigManager | None = None,
|
||||
now: datetime | None = None,
|
||||
executable: str | None = None,
|
||||
) -> AutoUpdateResult:
|
||||
"""Run update check/install flow and return a structured result."""
|
||||
manager = config_manager or ConfigManager()
|
||||
config = manager.load_config()
|
||||
source = detect_install_source(executable)
|
||||
checked_at = now or datetime.now()
|
||||
|
||||
if source == InstallSource.UVX:
|
||||
return AutoUpdateResult(
|
||||
status=AutoUpdateStatus.SKIPPED,
|
||||
source=source,
|
||||
checked=False,
|
||||
update_available=False,
|
||||
updated=False,
|
||||
message="uvx runtime detected; updates are managed by uvx cache resolution.",
|
||||
)
|
||||
|
||||
if not force and not config.auto_update:
|
||||
return AutoUpdateResult(
|
||||
status=AutoUpdateStatus.SKIPPED,
|
||||
source=source,
|
||||
checked=False,
|
||||
update_available=False,
|
||||
updated=False,
|
||||
message="Auto-update is disabled in config.",
|
||||
)
|
||||
|
||||
if not force and config.auto_update_last_checked_at is not None:
|
||||
try:
|
||||
elapsed = checked_at - config.auto_update_last_checked_at
|
||||
except TypeError:
|
||||
# Trigger: mixed naive/aware datetimes from manual config edits.
|
||||
# Why: datetime subtraction fails for mixed tz-awareness.
|
||||
# Outcome: ignore the gate once and continue with a forced check path.
|
||||
logger.warning("Auto-update interval gate skipped due to incompatible timestamp format")
|
||||
else:
|
||||
if elapsed < timedelta(seconds=config.update_check_interval):
|
||||
return AutoUpdateResult(
|
||||
status=AutoUpdateStatus.SKIPPED,
|
||||
source=source,
|
||||
checked=False,
|
||||
update_available=False,
|
||||
updated=False,
|
||||
message="Update check interval has not elapsed.",
|
||||
)
|
||||
|
||||
try:
|
||||
# --- Availability check ---
|
||||
latest_version: str | None = None
|
||||
if source == InstallSource.HOMEBREW:
|
||||
update_available, latest_version = _check_homebrew_update_available(silent=silent)
|
||||
else:
|
||||
update_available, latest_version = _check_pypi_update_available()
|
||||
|
||||
if not update_available:
|
||||
return AutoUpdateResult(
|
||||
status=AutoUpdateStatus.UP_TO_DATE,
|
||||
source=source,
|
||||
checked=True,
|
||||
update_available=False,
|
||||
updated=False,
|
||||
latest_version=latest_version,
|
||||
message=f"Basic Memory is up to date ({basic_memory.__version__}).",
|
||||
)
|
||||
|
||||
if check_only:
|
||||
return AutoUpdateResult(
|
||||
status=AutoUpdateStatus.UPDATE_AVAILABLE,
|
||||
source=source,
|
||||
checked=True,
|
||||
update_available=True,
|
||||
updated=False,
|
||||
latest_version=latest_version,
|
||||
message=(
|
||||
f"Update available (latest: {latest_version or 'unknown'}). "
|
||||
f"{_manual_update_hint(source)}"
|
||||
),
|
||||
)
|
||||
|
||||
if source == InstallSource.UNKNOWN:
|
||||
return AutoUpdateResult(
|
||||
status=AutoUpdateStatus.UPDATE_AVAILABLE,
|
||||
source=source,
|
||||
checked=True,
|
||||
update_available=True,
|
||||
updated=False,
|
||||
latest_version=latest_version,
|
||||
message=(
|
||||
f"Update available (latest: {latest_version or 'unknown'}). "
|
||||
f"{_manual_update_hint(source)}"
|
||||
),
|
||||
)
|
||||
|
||||
# --- Automatic install ---
|
||||
command = (
|
||||
["uv", "tool", "upgrade", PACKAGE_NAME]
|
||||
if source == InstallSource.UV_TOOL
|
||||
else ["brew", "upgrade", PACKAGE_NAME]
|
||||
)
|
||||
timeout = (
|
||||
UV_UPGRADE_TIMEOUT_SECONDS
|
||||
if source == InstallSource.UV_TOOL
|
||||
else BREW_UPGRADE_TIMEOUT_SECONDS
|
||||
)
|
||||
|
||||
install_result = _run_subprocess(
|
||||
command,
|
||||
timeout_seconds=timeout,
|
||||
silent=silent,
|
||||
capture_output=not silent,
|
||||
)
|
||||
if install_result.returncode != 0:
|
||||
stderr = (install_result.stderr or "").strip() if install_result.stderr else ""
|
||||
stdout = (install_result.stdout or "").strip() if install_result.stdout else ""
|
||||
detail = stderr or stdout or "update command failed"
|
||||
return AutoUpdateResult(
|
||||
status=AutoUpdateStatus.FAILED,
|
||||
source=source,
|
||||
checked=True,
|
||||
update_available=True,
|
||||
updated=False,
|
||||
latest_version=latest_version,
|
||||
message="Automatic update failed.",
|
||||
error=detail,
|
||||
)
|
||||
|
||||
return AutoUpdateResult(
|
||||
status=AutoUpdateStatus.UPDATED,
|
||||
source=source,
|
||||
checked=True,
|
||||
update_available=True,
|
||||
updated=True,
|
||||
latest_version=latest_version,
|
||||
message=(
|
||||
"Basic Memory was updated successfully. "
|
||||
"Restart running sessions to use the new version."
|
||||
),
|
||||
restart_recommended=True,
|
||||
)
|
||||
|
||||
except (
|
||||
RuntimeError,
|
||||
urllib.error.URLError,
|
||||
ValueError,
|
||||
TimeoutError,
|
||||
subprocess.SubprocessError,
|
||||
OSError,
|
||||
) as exc:
|
||||
logger.warning(f"Auto-update check failed: {exc}")
|
||||
return AutoUpdateResult(
|
||||
status=AutoUpdateStatus.FAILED,
|
||||
source=source,
|
||||
checked=True,
|
||||
update_available=False,
|
||||
updated=False,
|
||||
message="Automatic update check failed.",
|
||||
error=str(exc),
|
||||
)
|
||||
finally:
|
||||
# Trigger: we attempted a check path (including failures).
|
||||
# Why: repeated failing checks on every command create noise and unnecessary network load.
|
||||
# Outcome: next periodic check is gated by update_check_interval.
|
||||
try:
|
||||
_save_last_checked_timestamp(manager, checked_at)
|
||||
except Exception as exc: # pragma: no cover
|
||||
logger.warning(f"Failed to persist auto-update timestamp: {exc}")
|
||||
|
||||
|
||||
def maybe_run_periodic_auto_update(
|
||||
invoked_subcommand: str | None,
|
||||
*,
|
||||
config_manager: ConfigManager | None = None,
|
||||
is_interactive: bool | None = None,
|
||||
console: Console | None = None,
|
||||
) -> AutoUpdateResult | None:
|
||||
"""Run a periodic auto-update check for interactive CLI sessions."""
|
||||
interactive = _is_interactive_session() if is_interactive is None else is_interactive
|
||||
if not interactive:
|
||||
return None
|
||||
if invoked_subcommand in {None, "mcp", "update"}:
|
||||
return None
|
||||
|
||||
result = run_auto_update(
|
||||
force=False,
|
||||
check_only=False,
|
||||
silent=False,
|
||||
config_manager=config_manager,
|
||||
)
|
||||
|
||||
if result.status in {
|
||||
AutoUpdateStatus.UPDATE_AVAILABLE,
|
||||
AutoUpdateStatus.UPDATED,
|
||||
AutoUpdateStatus.FAILED,
|
||||
}:
|
||||
out = console or Console()
|
||||
if result.status == AutoUpdateStatus.UPDATED:
|
||||
out.print(f"[green]{result.message}[/green]")
|
||||
elif result.status == AutoUpdateStatus.FAILED:
|
||||
error_detail = f" {result.error}" if result.error else ""
|
||||
out.print(f"[yellow]{result.message}{error_detail}[/yellow]")
|
||||
elif result.message:
|
||||
out.print(f"[cyan]{result.message}[/cyan]")
|
||||
|
||||
return result
|
||||
@@ -1,17 +1,29 @@
|
||||
"""CLI commands for basic-memory."""
|
||||
|
||||
from . import status, db, import_memory_json, mcp, import_claude_conversations
|
||||
from . import import_claude_projects, import_chatgpt, tool, project, format
|
||||
from . import status, db, doctor, import_memory_json, mcp, import_claude_conversations, orphans
|
||||
from . import (
|
||||
import_claude_projects,
|
||||
import_chatgpt,
|
||||
tool,
|
||||
project,
|
||||
format,
|
||||
schema,
|
||||
update,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"status",
|
||||
"db",
|
||||
"doctor",
|
||||
"import_memory_json",
|
||||
"mcp",
|
||||
"import_claude_conversations",
|
||||
"orphans",
|
||||
"import_claude_projects",
|
||||
"import_chatgpt",
|
||||
"tool",
|
||||
"project",
|
||||
"format",
|
||||
"schema",
|
||||
"update",
|
||||
]
|
||||
|
||||
@@ -6,11 +6,14 @@ from basic_memory.cli.app import cloud_app
|
||||
from basic_memory.cli.commands.cloud.core_commands import * # noqa: F401,F403
|
||||
from basic_memory.cli.commands.cloud.api_client import get_authenticated_headers, get_cloud_config # noqa: F401
|
||||
from basic_memory.cli.commands.cloud.upload_command import * # noqa: F401,F403
|
||||
from basic_memory.cli.commands.cloud.project_sync import * # noqa: F401,F403
|
||||
|
||||
# Register snapshot sub-command group
|
||||
from basic_memory.cli.commands.cloud.snapshot import snapshot_app
|
||||
from basic_memory.cli.commands.cloud.workspace import workspace_app
|
||||
|
||||
cloud_app.add_typer(snapshot_app, name="snapshot")
|
||||
cloud_app.add_typer(workspace_app, name="workspace")
|
||||
|
||||
# Register restore command (directly on cloud_app via decorator)
|
||||
from basic_memory.cli.commands.cloud.restore import restore # noqa: F401, E402
|
||||
|
||||
@@ -45,14 +45,26 @@ def get_cloud_config() -> tuple[str, str, str]:
|
||||
|
||||
async def get_authenticated_headers(auth: CLIAuth | None = None) -> dict[str, str]:
|
||||
"""
|
||||
Get authentication headers with JWT token.
|
||||
handles jwt refresh if needed.
|
||||
Get authentication headers for cloud API requests.
|
||||
|
||||
Credential priority mirrors async_client._resolve_cloud_token():
|
||||
1. API key (config.cloud_api_key) — fast, no refresh needed
|
||||
2. OAuth token via CLIAuth — handles JWT refresh automatically
|
||||
"""
|
||||
# --- API key (preferred) ---
|
||||
config_manager = ConfigManager()
|
||||
api_key = config_manager.config.cloud_api_key
|
||||
if api_key:
|
||||
return {"Authorization": f"Bearer {api_key}"}
|
||||
|
||||
# --- OAuth fallback ---
|
||||
client_id, domain, _ = get_cloud_config()
|
||||
auth_obj = auth or CLIAuth(client_id=client_id, authkit_domain=domain)
|
||||
token = await auth_obj.get_valid_token()
|
||||
if not token:
|
||||
console.print("[red]Not authenticated. Please run 'basic-memory cloud login' first.[/red]")
|
||||
console.print(
|
||||
"[red]Not authenticated. Run 'bm cloud set-key <key>' or 'bm cloud login' first.[/red]"
|
||||
)
|
||||
raise typer.Exit(1)
|
||||
|
||||
return {"Authorization": f"Bearer {token}"}
|
||||
@@ -87,41 +99,39 @@ async def make_api_request(
|
||||
response = await client.request(method=method, url=url, headers=headers, json=json_data)
|
||||
response.raise_for_status()
|
||||
return response
|
||||
except httpx.HTTPStatusError as e:
|
||||
response = e.response
|
||||
|
||||
# Try to parse error detail from response
|
||||
error_detail = None
|
||||
try:
|
||||
error_detail = response.json()
|
||||
except Exception:
|
||||
# If JSON parsing fails, we'll handle it as a generic error
|
||||
pass
|
||||
|
||||
# Check for subscription_required error (403)
|
||||
if response.status_code == 403 and isinstance(error_detail, dict):
|
||||
# Handle both FastAPI HTTPException format (nested under "detail")
|
||||
# and direct format
|
||||
detail_obj = error_detail.get("detail", error_detail)
|
||||
if (
|
||||
isinstance(detail_obj, dict)
|
||||
and detail_obj.get("error") == "subscription_required"
|
||||
):
|
||||
message = detail_obj.get("message", "Active subscription required")
|
||||
subscribe_url = detail_obj.get(
|
||||
"subscribe_url", "https://basicmemory.com/subscribe"
|
||||
)
|
||||
raise SubscriptionRequiredError(
|
||||
message=message, subscribe_url=subscribe_url
|
||||
) from e
|
||||
|
||||
# Raise generic CloudAPIError with status code and detail
|
||||
raise CloudAPIError(
|
||||
f"API request failed: {e}",
|
||||
status_code=response.status_code,
|
||||
detail=error_detail if isinstance(error_detail, dict) else {},
|
||||
) from e
|
||||
except httpx.HTTPError as e:
|
||||
# Check if this is a response error with response details
|
||||
if hasattr(e, "response") and e.response is not None: # pyright: ignore [reportAttributeAccessIssue]
|
||||
response = e.response # type: ignore
|
||||
|
||||
# Try to parse error detail from response
|
||||
error_detail = None
|
||||
try:
|
||||
error_detail = response.json()
|
||||
except Exception:
|
||||
# If JSON parsing fails, we'll handle it as a generic error
|
||||
pass
|
||||
|
||||
# Check for subscription_required error (403)
|
||||
if response.status_code == 403 and isinstance(error_detail, dict):
|
||||
# Handle both FastAPI HTTPException format (nested under "detail")
|
||||
# and direct format
|
||||
detail_obj = error_detail.get("detail", error_detail)
|
||||
if (
|
||||
isinstance(detail_obj, dict)
|
||||
and detail_obj.get("error") == "subscription_required"
|
||||
):
|
||||
message = detail_obj.get("message", "Active subscription required")
|
||||
subscribe_url = detail_obj.get(
|
||||
"subscribe_url", "https://basicmemory.com/subscribe"
|
||||
)
|
||||
raise SubscriptionRequiredError(
|
||||
message=message, subscribe_url=subscribe_url
|
||||
) from e
|
||||
|
||||
# Raise generic CloudAPIError with status code and detail
|
||||
raise CloudAPIError(
|
||||
f"API request failed: {e}",
|
||||
status_code=response.status_code,
|
||||
detail=error_detail if isinstance(error_detail, dict) else {},
|
||||
) from e
|
||||
|
||||
raise CloudAPIError(f"API request failed: {e}") from e
|
||||
|
||||
@@ -14,6 +14,24 @@ class BisyncError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def _rclone_exclude_filters(pattern: str) -> list[str]:
|
||||
"""Return rclone exclude filters for a gitignore-style pattern."""
|
||||
if pattern.endswith("/"):
|
||||
# Trigger: gitignore-style patterns ending in / are directory-only rules.
|
||||
# Why: stripping the slash would also exclude a same-named file.
|
||||
# Outcome: rclone keeps the directory rule and excludes recursive contents.
|
||||
return [f"- {pattern}", f"- {pattern}**"]
|
||||
|
||||
path_pattern = pattern.removesuffix("/**")
|
||||
|
||||
# Trigger: rclone treats a directory contents filter separately from the
|
||||
# directory/file path itself.
|
||||
# Why: files like config.json and directory markers like .obsidian must both
|
||||
# be excluded, along with anything below matching directories.
|
||||
# Outcome: every ignore pattern excludes the direct match and recursive children.
|
||||
return [f"- {path_pattern}", f"- {path_pattern}/**"]
|
||||
|
||||
|
||||
async def get_mount_info() -> TenantMountInfo:
|
||||
"""Get current tenant information from cloud API."""
|
||||
try:
|
||||
@@ -78,19 +96,11 @@ def convert_bmignore_to_rclone_filters() -> Path:
|
||||
patterns.append(line)
|
||||
continue
|
||||
|
||||
# Convert gitignore pattern to rclone filter syntax
|
||||
# gitignore: node_modules → rclone: - node_modules/**
|
||||
# gitignore: *.pyc → rclone: - *.pyc
|
||||
if "*" in line:
|
||||
# Pattern already has wildcard, just add exclude prefix
|
||||
patterns.append(f"- {line}")
|
||||
else:
|
||||
# Directory pattern - add /** for recursive exclude
|
||||
patterns.append(f"- {line}/**")
|
||||
patterns.extend(_rclone_exclude_filters(line))
|
||||
|
||||
except Exception:
|
||||
# If we can't read the file, create a minimal filter
|
||||
patterns = ["# Error reading .bmignore, using minimal filters", "- .git/**"]
|
||||
patterns = ["# Error reading .bmignore, using minimal filters", "- .git", "- .git/**"]
|
||||
|
||||
# Write rclone filter file
|
||||
rclone_filter_path.write_text("\n".join(patterns) + "\n")
|
||||
|
||||
@@ -2,10 +2,12 @@
|
||||
|
||||
from basic_memory.cli.commands.cloud.api_client import make_api_request
|
||||
from basic_memory.config import ConfigManager
|
||||
from basic_memory.mcp.async_client import resolve_configured_workspace
|
||||
from basic_memory.schemas.cloud import (
|
||||
CloudProjectList,
|
||||
CloudProjectCreateRequest,
|
||||
CloudProjectCreateResponse,
|
||||
ProjectVisibility,
|
||||
)
|
||||
from basic_memory.utils import generate_permalink
|
||||
|
||||
@@ -16,12 +18,33 @@ class CloudUtilsError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def _workspace_headers(
|
||||
*,
|
||||
project_name: str | None = None,
|
||||
workspace: str | None = None,
|
||||
) -> dict[str, str]:
|
||||
"""Build optional workspace headers using the CLI config resolution chain."""
|
||||
resolved_workspace = resolve_configured_workspace(
|
||||
project_name=project_name,
|
||||
workspace=workspace,
|
||||
)
|
||||
if resolved_workspace is None:
|
||||
return {}
|
||||
return {"X-Workspace-ID": resolved_workspace}
|
||||
|
||||
|
||||
async def fetch_cloud_projects(
|
||||
*,
|
||||
project_name: str | None = None,
|
||||
workspace: str | None = None,
|
||||
api_request=make_api_request,
|
||||
) -> CloudProjectList:
|
||||
"""Fetch list of projects from cloud API.
|
||||
|
||||
Args:
|
||||
project_name: Optional project name for workspace resolution
|
||||
workspace: Cloud workspace tenant_id to list projects from
|
||||
|
||||
Returns:
|
||||
CloudProjectList with projects from cloud
|
||||
"""
|
||||
@@ -30,7 +53,11 @@ async def fetch_cloud_projects(
|
||||
config = config_manager.config
|
||||
host_url = config.cloud_host.rstrip("/")
|
||||
|
||||
response = await api_request(method="GET", url=f"{host_url}/proxy/projects/projects")
|
||||
response = await api_request(
|
||||
method="GET",
|
||||
url=f"{host_url}/proxy/v2/projects/",
|
||||
headers=_workspace_headers(project_name=project_name, workspace=workspace),
|
||||
)
|
||||
|
||||
return CloudProjectList.model_validate(response.json())
|
||||
except Exception as e:
|
||||
@@ -40,12 +67,16 @@ async def fetch_cloud_projects(
|
||||
async def create_cloud_project(
|
||||
project_name: str,
|
||||
*,
|
||||
workspace: str | None = None,
|
||||
visibility: ProjectVisibility = "workspace",
|
||||
api_request=make_api_request,
|
||||
) -> CloudProjectCreateResponse:
|
||||
"""Create a new project on cloud.
|
||||
|
||||
Args:
|
||||
project_name: Name of project to create
|
||||
workspace: Optional workspace override for tenant-scoped project creation
|
||||
visibility: Visibility for the created cloud project
|
||||
|
||||
Returns:
|
||||
CloudProjectCreateResponse with project details from API
|
||||
@@ -62,12 +93,16 @@ async def create_cloud_project(
|
||||
name=project_name,
|
||||
path=project_path,
|
||||
set_default=False,
|
||||
visibility=visibility,
|
||||
)
|
||||
|
||||
response = await api_request(
|
||||
method="POST",
|
||||
url=f"{host_url}/proxy/projects/projects",
|
||||
headers={"Content-Type": "application/json"},
|
||||
url=f"{host_url}/proxy/v2/projects/",
|
||||
headers={
|
||||
"Content-Type": "application/json",
|
||||
**_workspace_headers(project_name=project_name, workspace=workspace),
|
||||
},
|
||||
json_data=project_data.model_dump(),
|
||||
)
|
||||
|
||||
@@ -81,28 +116,38 @@ async def sync_project(project_name: str, force_full: bool = False) -> None:
|
||||
|
||||
Args:
|
||||
project_name: Name of project to sync
|
||||
force_full: If True, force a full scan bypassing watermark optimization
|
||||
force_full: ignored, kept for backwards compatibility
|
||||
"""
|
||||
try:
|
||||
from basic_memory.cli.commands.command_utils import run_sync
|
||||
|
||||
await run_sync(project=project_name, force_full=force_full)
|
||||
await run_sync(project=project_name)
|
||||
except Exception as e:
|
||||
raise CloudUtilsError(f"Failed to sync project '{project_name}': {e}") from e
|
||||
|
||||
|
||||
async def project_exists(project_name: str, *, api_request=make_api_request) -> bool:
|
||||
async def project_exists(
|
||||
project_name: str,
|
||||
*,
|
||||
workspace: str | None = None,
|
||||
api_request=make_api_request,
|
||||
) -> bool:
|
||||
"""Check if a project exists on cloud.
|
||||
|
||||
Args:
|
||||
project_name: Name of project to check
|
||||
workspace: Optional workspace override for tenant-scoped project lookup
|
||||
|
||||
Returns:
|
||||
True if project exists, False otherwise
|
||||
|
||||
Raises:
|
||||
CloudUtilsError: If the project list cannot be fetched from cloud
|
||||
"""
|
||||
try:
|
||||
projects = await fetch_cloud_projects(api_request=api_request)
|
||||
project_names = {p.name for p in projects.projects}
|
||||
return project_name in project_names
|
||||
except Exception:
|
||||
return False
|
||||
projects = await fetch_cloud_projects(
|
||||
project_name=project_name,
|
||||
workspace=workspace,
|
||||
api_request=api_request,
|
||||
)
|
||||
project_names = {p.name for p in projects.projects}
|
||||
return project_name in project_names
|
||||
|
||||
@@ -6,6 +6,14 @@ from rich.console import Console
|
||||
from basic_memory.cli.app import cloud_app
|
||||
from basic_memory.cli.commands.command_utils import run_with_cleanup
|
||||
from basic_memory.cli.auth import CLIAuth
|
||||
from basic_memory.cli.analytics import (
|
||||
track,
|
||||
EVENT_CLOUD_LOGIN_STARTED,
|
||||
EVENT_CLOUD_LOGIN_SUCCESS,
|
||||
EVENT_CLOUD_LOGIN_SUB_REQUIRED,
|
||||
EVENT_PROMO_OPTED_OUT,
|
||||
)
|
||||
from basic_memory.cli.promo import OSS_DISCOUNT_CODE
|
||||
from basic_memory.config import ConfigManager
|
||||
from basic_memory.cli.commands.cloud.api_client import (
|
||||
CloudAPIError,
|
||||
@@ -29,9 +37,10 @@ console = Console()
|
||||
|
||||
@cloud_app.command()
|
||||
def login():
|
||||
"""Authenticate with WorkOS using OAuth Device Authorization flow and enable cloud mode."""
|
||||
"""Authenticate with WorkOS using OAuth Device Authorization flow."""
|
||||
|
||||
async def _login():
|
||||
track(EVENT_CLOUD_LOGIN_STARTED)
|
||||
client_id, domain, host_url = get_cloud_config()
|
||||
auth = CLIAuth(client_id=client_id, authkit_domain=domain)
|
||||
|
||||
@@ -45,18 +54,17 @@ def login():
|
||||
console.print("[dim]Verifying subscription access...[/dim]")
|
||||
await make_api_request("GET", f"{host_url.rstrip('/')}/proxy/health")
|
||||
|
||||
# Enable cloud mode after successful login and subscription validation
|
||||
config_manager = ConfigManager()
|
||||
config = config_manager.load_config()
|
||||
config.cloud_mode = True
|
||||
config_manager.save_config(config)
|
||||
|
||||
console.print("[green]Cloud mode enabled[/green]")
|
||||
console.print(f"[dim]All CLI commands now work against {host_url}[/dim]")
|
||||
track(EVENT_CLOUD_LOGIN_SUCCESS)
|
||||
console.print("[green]Cloud authentication successful[/green]")
|
||||
console.print(f"[dim]Cloud host ready: {host_url}[/dim]")
|
||||
|
||||
except SubscriptionRequiredError as e:
|
||||
track(EVENT_CLOUD_LOGIN_SUB_REQUIRED)
|
||||
console.print("\n[red]Subscription Required[/red]\n")
|
||||
console.print(f"[yellow]{e.args[0]}[/yellow]\n")
|
||||
console.print(
|
||||
f"OSS discount code: [bold]{OSS_DISCOUNT_CODE}[/bold] (20% off for 3 months)\n"
|
||||
)
|
||||
console.print(f"Subscribe at: [blue underline]{e.subscribe_url}[/blue underline]\n")
|
||||
console.print(
|
||||
"[dim]Once you have an active subscription, run [bold]bm cloud login[/bold] again.[/dim]"
|
||||
@@ -68,71 +76,68 @@ def login():
|
||||
|
||||
@cloud_app.command()
|
||||
def logout():
|
||||
"""Disable cloud mode and return to local mode."""
|
||||
|
||||
# Disable cloud mode
|
||||
"""Remove stored OAuth tokens and clear cached workspace selection."""
|
||||
config_manager = ConfigManager()
|
||||
config = config_manager.load_config()
|
||||
config.cloud_mode = False
|
||||
config_manager.save_config(config)
|
||||
config = config_manager.config
|
||||
auth = CLIAuth(client_id=config.cloud_client_id, authkit_domain=config.cloud_domain)
|
||||
auth.logout()
|
||||
|
||||
console.print("[green]Cloud mode disabled[/green]")
|
||||
console.print("[dim]All CLI commands now work locally[/dim]")
|
||||
# Trigger: ending a session must invalidate the cached workspace.
|
||||
# Why: a follow-up `bm cloud login` (often as a different user, or returning
|
||||
# from an org workspace to personal) inherits the previous selection
|
||||
# and silently routes everything through the wrong tenant. See #755.
|
||||
# Outcome: re-login starts from a clean slate; the user picks again via
|
||||
# `bm cloud workspace set-default` or per-project --workspace.
|
||||
if config.default_workspace is not None:
|
||||
config.default_workspace = None
|
||||
config_manager.save_config(config)
|
||||
|
||||
console.print("[dim]API key (if configured) remains available for cloud project routing.[/dim]")
|
||||
|
||||
|
||||
@cloud_app.command("status")
|
||||
def status() -> None:
|
||||
"""Check cloud mode status and cloud instance health."""
|
||||
# Check cloud mode
|
||||
"""Check cloud authentication and connection status."""
|
||||
config_manager = ConfigManager()
|
||||
config = config_manager.load_config()
|
||||
auth = CLIAuth(client_id=config.cloud_client_id, authkit_domain=config.cloud_domain)
|
||||
tokens = auth.load_tokens()
|
||||
|
||||
console.print("[bold blue]Cloud Mode Status[/bold blue]")
|
||||
if config.cloud_mode:
|
||||
console.print(" Mode: [green]Cloud (enabled)[/green]")
|
||||
console.print(f" Host: {config.cloud_host}")
|
||||
console.print(" [dim]All CLI commands work against cloud[/dim]")
|
||||
else:
|
||||
console.print(" Mode: [yellow]Local (disabled)[/yellow]")
|
||||
console.print(" [dim]All CLI commands work locally[/dim]")
|
||||
console.print("\n[dim]To enable cloud mode, run: bm cloud login[/dim]")
|
||||
console.print("[bold blue]Cloud Status[/bold blue]")
|
||||
console.print(f" Host: {config.cloud_host}")
|
||||
console.print(
|
||||
f" API Key: {'[green]configured[/green]' if config.cloud_api_key else '[yellow]not set[/yellow]'}"
|
||||
)
|
||||
|
||||
oauth_status = "[yellow]not logged in[/yellow]"
|
||||
if tokens:
|
||||
if auth.is_token_valid(tokens):
|
||||
oauth_status = "[green]token valid[/green]"
|
||||
else:
|
||||
oauth_status = "[yellow]token expired[/yellow]"
|
||||
console.print(f" OAuth: {oauth_status}")
|
||||
|
||||
has_credentials = bool(config.cloud_api_key) or tokens is not None
|
||||
if not has_credentials:
|
||||
console.print(
|
||||
"\n[dim]No cloud credentials found. Run: bm cloud login or bm cloud api-key save <key>[/dim]"
|
||||
)
|
||||
return
|
||||
|
||||
# Get cloud configuration
|
||||
# Quick connection check — just verify we can reach the cloud
|
||||
_, _, host_url = get_cloud_config()
|
||||
host_url = host_url.rstrip("/")
|
||||
|
||||
# Prepare headers
|
||||
headers = {}
|
||||
|
||||
try:
|
||||
console.print("\n[blue]Checking cloud instance health...[/blue]")
|
||||
|
||||
# Make API request to check health
|
||||
response = run_with_cleanup(
|
||||
make_api_request(method="GET", url=f"{host_url}/proxy/health", headers=headers)
|
||||
run_with_cleanup(make_api_request(method="GET", url=f"{host_url}/proxy/health"))
|
||||
console.print("\n[green]Cloud connected[/green]")
|
||||
except CloudAPIError:
|
||||
console.print("\n[yellow]Cloud not connected[/yellow]")
|
||||
console.print(
|
||||
"[dim]Try re-authenticating with 'bm cloud login' or 'bm cloud api-key save'.[/dim]"
|
||||
)
|
||||
|
||||
health_data = response.json()
|
||||
|
||||
console.print("[green]Cloud instance is healthy[/green]")
|
||||
|
||||
# Display status details
|
||||
if "status" in health_data:
|
||||
console.print(f" Status: {health_data['status']}")
|
||||
if "version" in health_data:
|
||||
console.print(f" Version: {health_data['version']}")
|
||||
if "timestamp" in health_data:
|
||||
console.print(f" Timestamp: {health_data['timestamp']}")
|
||||
|
||||
console.print("\n[dim]To sync projects, use: bm project bisync --name <project>[/dim]")
|
||||
|
||||
except CloudAPIError as e:
|
||||
console.print(f"[red]Error checking cloud health: {e}[/red]")
|
||||
raise typer.Exit(1)
|
||||
except Exception as e:
|
||||
console.print(f"[red]Unexpected error: {e}[/red]")
|
||||
raise typer.Exit(1)
|
||||
except Exception:
|
||||
console.print("\n[yellow]Cloud not connected[/yellow]")
|
||||
|
||||
|
||||
@cloud_app.command("setup")
|
||||
@@ -140,7 +145,7 @@ def setup() -> None:
|
||||
"""Set up cloud sync by installing rclone and configuring credentials.
|
||||
|
||||
After setup, use project commands for syncing:
|
||||
bm project add <name> <path> --local-path ~/projects/<name>
|
||||
bm project add <name> --cloud --local-path ~/projects/<name>
|
||||
bm project bisync --name <name> --resync # First time
|
||||
bm project bisync --name <name> # Subsequent syncs
|
||||
"""
|
||||
@@ -172,9 +177,9 @@ def setup() -> None:
|
||||
console.print("\n[bold green]Cloud setup completed successfully![/bold green]")
|
||||
console.print("\n[bold]Next steps:[/bold]")
|
||||
console.print("1. Add a project with local sync path:")
|
||||
console.print(" bm project add research --local-path ~/Documents/research")
|
||||
console.print(" bm project add research --cloud --local-path ~/Documents/research")
|
||||
console.print("\n Or configure sync for an existing project:")
|
||||
console.print(" bm project sync-setup research ~/Documents/research")
|
||||
console.print(" bm cloud sync-setup research ~/Documents/research")
|
||||
console.print("\n2. Preview the initial sync (recommended):")
|
||||
console.print(" bm project bisync --name research --resync --dry-run")
|
||||
console.print("\n3. If all looks good, run the actual sync:")
|
||||
@@ -191,3 +196,98 @@ def setup() -> None:
|
||||
except Exception as e:
|
||||
console.print(f"\n[red]Unexpected error during setup: {e}[/red]")
|
||||
raise typer.Exit(1)
|
||||
|
||||
|
||||
@cloud_app.command("promo")
|
||||
def promo(enabled: bool = typer.Option(True, "--on/--off", help="Enable or disable CLI promos.")):
|
||||
"""Enable or disable CLI cloud promo messages."""
|
||||
config_manager = ConfigManager()
|
||||
config = config_manager.load_config()
|
||||
config.cloud_promo_opt_out = not enabled
|
||||
config_manager.save_config(config)
|
||||
|
||||
if enabled:
|
||||
console.print("[green]Cloud promo messages enabled[/green]")
|
||||
else:
|
||||
track(EVENT_PROMO_OPTED_OUT)
|
||||
console.print("[yellow]Cloud promo messages disabled[/yellow]")
|
||||
|
||||
|
||||
# --- API key management subcommand group ---
|
||||
|
||||
api_key_app = typer.Typer(help="Manage cloud API keys")
|
||||
cloud_app.add_typer(api_key_app, name="api-key")
|
||||
|
||||
|
||||
@api_key_app.command("save")
|
||||
def api_key_save(
|
||||
api_key: str = typer.Argument(..., help="API key (bmc_ prefixed) for cloud access"),
|
||||
) -> None:
|
||||
"""Save an existing API key to local config.
|
||||
|
||||
Use when you already have an API key (e.g., from the web app).
|
||||
|
||||
Example:
|
||||
bm cloud api-key save bmc_abc123...
|
||||
"""
|
||||
if not api_key.startswith("bmc_"):
|
||||
console.print("[red]Error: API key must start with 'bmc_'[/red]")
|
||||
raise typer.Exit(1)
|
||||
|
||||
config_manager = ConfigManager()
|
||||
config = config_manager.load_config()
|
||||
config.cloud_api_key = api_key
|
||||
config_manager.save_config(config)
|
||||
|
||||
console.print("[green]API key saved[/green]")
|
||||
console.print("[dim]Projects set to cloud mode will use this key for authentication[/dim]")
|
||||
console.print("[dim]Set a project to cloud mode: bm project set-cloud <name>[/dim]")
|
||||
|
||||
|
||||
@api_key_app.command("create")
|
||||
def api_key_create(
|
||||
name: str = typer.Argument(..., help="Human-readable name for the API key"),
|
||||
) -> None:
|
||||
"""Create a new API key via the cloud API and save it locally.
|
||||
|
||||
Requires active OAuth session (run 'bm cloud login' first).
|
||||
|
||||
Example:
|
||||
bm cloud api-key create "my-laptop"
|
||||
"""
|
||||
|
||||
async def _create_key():
|
||||
_, _, host_url = get_cloud_config()
|
||||
host_url = host_url.rstrip("/")
|
||||
|
||||
console.print(f"[dim]Creating API key '{name}'...[/dim]")
|
||||
response = await make_api_request(
|
||||
method="POST",
|
||||
url=f"{host_url}/api/keys",
|
||||
json_data={"name": name},
|
||||
)
|
||||
|
||||
key_data = response.json()
|
||||
api_key = key_data.get("key")
|
||||
if not api_key:
|
||||
console.print("[red]Error: No key returned from API[/red]")
|
||||
raise typer.Exit(1)
|
||||
|
||||
# Save to config
|
||||
config_manager = ConfigManager()
|
||||
config = config_manager.load_config()
|
||||
config.cloud_api_key = api_key
|
||||
config_manager.save_config(config)
|
||||
|
||||
console.print(f"[green]API key '{name}' created and saved[/green]")
|
||||
console.print("[dim]Projects set to cloud mode will use this key for authentication[/dim]")
|
||||
console.print("[dim]Set a project to cloud mode: bm project set-cloud <name>[/dim]")
|
||||
|
||||
try:
|
||||
run_with_cleanup(_create_key())
|
||||
except CloudAPIError as e:
|
||||
console.print(f"[red]Error creating API key: {e}[/red]")
|
||||
raise typer.Exit(1)
|
||||
except Exception as e:
|
||||
console.print(f"[red]Unexpected error: {e}[/red]")
|
||||
raise typer.Exit(1)
|
||||
|
||||
@@ -0,0 +1,343 @@
|
||||
"""Cloud sync commands for Basic Memory projects.
|
||||
|
||||
Commands for syncing, bisyncing, and checking integrity between local and cloud
|
||||
project instances. These were previously in project.py but belong here since
|
||||
they are cloud-specific operations.
|
||||
"""
|
||||
|
||||
import os
|
||||
from datetime import datetime
|
||||
|
||||
import typer
|
||||
from rich.console import Console
|
||||
|
||||
from basic_memory.cli.app import cloud_app
|
||||
from basic_memory.cli.commands.cloud.bisync_commands import get_mount_info
|
||||
from basic_memory.cli.commands.cloud.rclone_commands import (
|
||||
RcloneError,
|
||||
SyncProject,
|
||||
get_project_bisync_state,
|
||||
project_bisync,
|
||||
project_check,
|
||||
project_sync,
|
||||
)
|
||||
from basic_memory.cli.commands.command_utils import run_with_cleanup
|
||||
from basic_memory.cli.commands.routing import force_routing
|
||||
from basic_memory.config import ConfigManager, ProjectEntry
|
||||
from basic_memory.mcp.async_client import get_client
|
||||
from basic_memory.mcp.clients import ProjectClient
|
||||
from basic_memory.schemas.project_info import ProjectItem
|
||||
from basic_memory.utils import generate_permalink, normalize_project_path
|
||||
|
||||
console = Console()
|
||||
|
||||
|
||||
# --- Shared helpers ---
|
||||
|
||||
|
||||
def _has_cloud_credentials(config) -> bool:
|
||||
"""Return whether cloud credentials are available (API key or OAuth token)."""
|
||||
from basic_memory.config import has_cloud_credentials
|
||||
|
||||
return has_cloud_credentials(config)
|
||||
|
||||
|
||||
def _require_cloud_credentials(config) -> None:
|
||||
"""Exit with actionable guidance when cloud credentials are missing."""
|
||||
if _has_cloud_credentials(config):
|
||||
return
|
||||
|
||||
console.print("[red]Error: cloud credentials are required for this command[/red]")
|
||||
console.print("[dim]Run 'bm cloud login' or 'bm cloud api-key save <key>' first[/dim]")
|
||||
raise typer.Exit(1)
|
||||
|
||||
|
||||
async def _get_cloud_project(name: str) -> ProjectItem | None:
|
||||
"""Fetch a project by name from the cloud API."""
|
||||
async with get_client(project_name=name) as client:
|
||||
projects_list = await ProjectClient(client).list_projects()
|
||||
for proj in projects_list.projects:
|
||||
if generate_permalink(proj.name) == generate_permalink(name):
|
||||
return proj
|
||||
return None
|
||||
|
||||
|
||||
def _get_sync_project(
|
||||
name: str, config, project_data: ProjectItem
|
||||
) -> tuple[SyncProject, str | None]:
|
||||
"""Build a SyncProject and resolve local_sync_path from config.
|
||||
|
||||
Returns (sync_project, local_sync_path). Exits if no local_sync_path configured.
|
||||
"""
|
||||
sync_entry = config.projects.get(name)
|
||||
# Support both new (path) and legacy (local_sync_path) configs
|
||||
local_sync_path = (sync_entry.local_sync_path or sync_entry.path) if sync_entry else None
|
||||
|
||||
if not local_sync_path or not os.path.isabs(local_sync_path):
|
||||
console.print(f"[red]Error: Project '{name}' has no local sync path configured[/red]")
|
||||
console.print(f"\nConfigure sync with: bm cloud sync-setup {name} ~/path/to/local")
|
||||
raise typer.Exit(1)
|
||||
|
||||
sync_project = SyncProject(
|
||||
name=project_data.name,
|
||||
path=normalize_project_path(project_data.path),
|
||||
local_sync_path=local_sync_path,
|
||||
)
|
||||
return sync_project, local_sync_path
|
||||
|
||||
|
||||
# --- Commands ---
|
||||
|
||||
|
||||
@cloud_app.command("sync")
|
||||
def sync_project_command(
|
||||
name: str = typer.Option(..., "--name", help="Project name to sync"),
|
||||
dry_run: bool = typer.Option(False, "--dry-run", help="Preview changes without syncing"),
|
||||
verbose: bool = typer.Option(False, "--verbose", "-v", help="Show detailed output"),
|
||||
) -> None:
|
||||
"""One-way sync: local -> cloud (make cloud identical to local).
|
||||
|
||||
Example:
|
||||
bm cloud sync --name research
|
||||
bm cloud sync --name research --dry-run
|
||||
"""
|
||||
config = ConfigManager().config
|
||||
_require_cloud_credentials(config)
|
||||
|
||||
try:
|
||||
# Get tenant info for bucket name
|
||||
tenant_info = run_with_cleanup(get_mount_info())
|
||||
bucket_name = tenant_info.bucket_name
|
||||
|
||||
# Get project info
|
||||
with force_routing(cloud=True):
|
||||
project_data = run_with_cleanup(_get_cloud_project(name))
|
||||
if not project_data:
|
||||
console.print(f"[red]Error: Project '{name}' not found[/red]")
|
||||
raise typer.Exit(1)
|
||||
|
||||
sync_project, local_sync_path = _get_sync_project(name, config, project_data)
|
||||
|
||||
# Run sync
|
||||
console.print(f"[blue]Syncing {name} (local -> cloud)...[/blue]")
|
||||
success = project_sync(sync_project, bucket_name, dry_run=dry_run, verbose=verbose)
|
||||
|
||||
if success:
|
||||
console.print(f"[green]{name} synced successfully[/green]")
|
||||
else:
|
||||
console.print(f"[red]{name} sync failed[/red]")
|
||||
raise typer.Exit(1)
|
||||
|
||||
except RcloneError as e:
|
||||
console.print(f"[red]Sync error: {e}[/red]")
|
||||
raise typer.Exit(1)
|
||||
except Exception as e:
|
||||
console.print(f"[red]Error: {e}[/red]")
|
||||
raise typer.Exit(1)
|
||||
|
||||
|
||||
@cloud_app.command("bisync")
|
||||
def bisync_project_command(
|
||||
name: str = typer.Option(..., "--name", help="Project name to bisync"),
|
||||
dry_run: bool = typer.Option(False, "--dry-run", help="Preview changes without syncing"),
|
||||
resync: bool = typer.Option(False, "--resync", help="Force new baseline"),
|
||||
verbose: bool = typer.Option(False, "--verbose", "-v", help="Show detailed output"),
|
||||
) -> None:
|
||||
"""Two-way sync: local <-> cloud (bidirectional sync).
|
||||
|
||||
Examples:
|
||||
bm cloud bisync --name research --resync # First time
|
||||
bm cloud bisync --name research # Subsequent syncs
|
||||
bm cloud bisync --name research --dry-run # Preview changes
|
||||
"""
|
||||
config = ConfigManager().config
|
||||
_require_cloud_credentials(config)
|
||||
|
||||
try:
|
||||
# Get tenant info for bucket name
|
||||
tenant_info = run_with_cleanup(get_mount_info())
|
||||
bucket_name = tenant_info.bucket_name
|
||||
|
||||
# Get project info
|
||||
with force_routing(cloud=True):
|
||||
project_data = run_with_cleanup(_get_cloud_project(name))
|
||||
if not project_data:
|
||||
console.print(f"[red]Error: Project '{name}' not found[/red]")
|
||||
raise typer.Exit(1)
|
||||
|
||||
sync_project, local_sync_path = _get_sync_project(name, config, project_data)
|
||||
|
||||
# Run bisync
|
||||
console.print(f"[blue]Bisync {name} (local <-> cloud)...[/blue]")
|
||||
success = project_bisync(
|
||||
sync_project, bucket_name, dry_run=dry_run, resync=resync, verbose=verbose
|
||||
)
|
||||
|
||||
if success:
|
||||
console.print(f"[green]{name} bisync completed successfully[/green]")
|
||||
|
||||
# Update config — sync_entry is guaranteed non-None because
|
||||
# _get_sync_project validated local_sync_path (which comes from sync_entry)
|
||||
sync_entry = config.projects.get(name)
|
||||
if sync_entry is None:
|
||||
raise RuntimeError(
|
||||
f"Sync entry for project '{name}' unexpectedly missing after validation"
|
||||
)
|
||||
sync_entry.last_sync = datetime.now()
|
||||
sync_entry.bisync_initialized = True
|
||||
ConfigManager().save_config(config)
|
||||
else:
|
||||
console.print(f"[red]{name} bisync failed[/red]")
|
||||
raise typer.Exit(1)
|
||||
|
||||
except RcloneError as e:
|
||||
console.print(f"[red]Bisync error: {e}[/red]")
|
||||
raise typer.Exit(1)
|
||||
except Exception as e:
|
||||
console.print(f"[red]Error: {e}[/red]")
|
||||
raise typer.Exit(1)
|
||||
|
||||
|
||||
@cloud_app.command("check")
|
||||
def check_project_command(
|
||||
name: str = typer.Option(..., "--name", help="Project name to check"),
|
||||
one_way: bool = typer.Option(False, "--one-way", help="Check one direction only (faster)"),
|
||||
) -> None:
|
||||
"""Verify file integrity between local and cloud.
|
||||
|
||||
Example:
|
||||
bm cloud check --name research
|
||||
"""
|
||||
config = ConfigManager().config
|
||||
_require_cloud_credentials(config)
|
||||
|
||||
try:
|
||||
# Get tenant info for bucket name
|
||||
tenant_info = run_with_cleanup(get_mount_info())
|
||||
bucket_name = tenant_info.bucket_name
|
||||
|
||||
# Get project info
|
||||
with force_routing(cloud=True):
|
||||
project_data = run_with_cleanup(_get_cloud_project(name))
|
||||
if not project_data:
|
||||
console.print(f"[red]Error: Project '{name}' not found[/red]")
|
||||
raise typer.Exit(1)
|
||||
|
||||
sync_project, local_sync_path = _get_sync_project(name, config, project_data)
|
||||
|
||||
# Run check
|
||||
console.print(f"[blue]Checking {name} integrity...[/blue]")
|
||||
match = project_check(sync_project, bucket_name, one_way=one_way)
|
||||
|
||||
if match:
|
||||
console.print(f"[green]{name} files match[/green]")
|
||||
else:
|
||||
console.print(f"[yellow]!{name} has differences[/yellow]")
|
||||
|
||||
except RcloneError as e:
|
||||
console.print(f"[red]Check error: {e}[/red]")
|
||||
raise typer.Exit(1)
|
||||
except Exception as e:
|
||||
console.print(f"[red]Error: {e}[/red]")
|
||||
raise typer.Exit(1)
|
||||
|
||||
|
||||
@cloud_app.command("bisync-reset")
|
||||
def bisync_reset(
|
||||
name: str = typer.Argument(..., help="Project name to reset bisync state for"),
|
||||
) -> None:
|
||||
"""Clear bisync state for a project.
|
||||
|
||||
This removes the bisync metadata files, forcing a fresh --resync on next bisync.
|
||||
Useful when bisync gets into an inconsistent state or when remote path changes.
|
||||
"""
|
||||
import shutil
|
||||
|
||||
try:
|
||||
state_path = get_project_bisync_state(name)
|
||||
|
||||
if not state_path.exists():
|
||||
console.print(f"[yellow]No bisync state found for project '{name}'[/yellow]")
|
||||
return
|
||||
|
||||
# Remove the entire state directory
|
||||
shutil.rmtree(state_path)
|
||||
console.print(f"[green]Cleared bisync state for project '{name}'[/green]")
|
||||
console.print("\nNext steps:")
|
||||
console.print(f" 1. Preview: bm cloud bisync --name {name} --resync --dry-run")
|
||||
console.print(f" 2. Sync: bm cloud bisync --name {name} --resync")
|
||||
|
||||
except Exception as e:
|
||||
console.print(f"[red]Error clearing bisync state: {str(e)}[/red]")
|
||||
raise typer.Exit(1)
|
||||
|
||||
|
||||
@cloud_app.command("sync-setup")
|
||||
def setup_project_sync(
|
||||
name: str = typer.Argument(..., help="Project name"),
|
||||
local_path: str = typer.Argument(..., help="Local sync directory"),
|
||||
) -> None:
|
||||
"""Configure local sync for an existing cloud project.
|
||||
|
||||
Example:
|
||||
bm cloud sync-setup research ~/Documents/research
|
||||
"""
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
config_manager = ConfigManager()
|
||||
config = config_manager.config
|
||||
_require_cloud_credentials(config)
|
||||
|
||||
async def _verify_project_exists():
|
||||
"""Verify the project exists on cloud by listing all projects."""
|
||||
async with get_client(project_name=name) as client:
|
||||
projects_list = await ProjectClient(client).list_projects()
|
||||
project_names = [p.name for p in projects_list.projects]
|
||||
if name not in project_names:
|
||||
raise ValueError(f"Project '{name}' not found on cloud")
|
||||
return True
|
||||
|
||||
try:
|
||||
# Verify project exists on cloud
|
||||
with force_routing(cloud=True):
|
||||
run_with_cleanup(_verify_project_exists())
|
||||
|
||||
# Resolve and create local path
|
||||
resolved_path = Path(os.path.abspath(os.path.expanduser(local_path)))
|
||||
resolved_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Update project entry with sync path — path is always the local directory
|
||||
entry = config.projects.get(name)
|
||||
if entry:
|
||||
entry.path = resolved_path.as_posix()
|
||||
entry.local_sync_path = resolved_path.as_posix()
|
||||
entry.bisync_initialized = False
|
||||
entry.last_sync = None
|
||||
else:
|
||||
config.projects[name] = ProjectEntry(
|
||||
path=resolved_path.as_posix(),
|
||||
local_sync_path=resolved_path.as_posix(),
|
||||
)
|
||||
config_manager.save_config(config)
|
||||
|
||||
# Create the project in the local DB so the MCP server can immediately use it
|
||||
async def _create_local_project():
|
||||
async with get_client() as client:
|
||||
data = {"name": name, "path": resolved_path.as_posix(), "set_default": False}
|
||||
return await ProjectClient(client).create_project(data)
|
||||
|
||||
with force_routing(local=True):
|
||||
try:
|
||||
run_with_cleanup(_create_local_project())
|
||||
except Exception:
|
||||
pass # Project may already exist locally; reconcile on next startup
|
||||
|
||||
console.print(f"[green]Sync configured for project '{name}'[/green]")
|
||||
console.print(f"\nLocal sync path: {resolved_path}")
|
||||
console.print("\nNext steps:")
|
||||
console.print(f" 1. Preview: bm cloud bisync --name {name} --resync --dry-run")
|
||||
console.print(f" 2. Sync: bm cloud bisync --name {name} --resync")
|
||||
except Exception as e:
|
||||
console.print(f"[red]Error configuring sync: {str(e)}[/red]")
|
||||
raise typer.Exit(1)
|
||||
@@ -20,6 +20,7 @@ from loguru import logger
|
||||
from rich.console import Console
|
||||
|
||||
from basic_memory.cli.commands.cloud.rclone_installer import is_rclone_installed
|
||||
from basic_memory.config import resolve_data_dir
|
||||
from basic_memory.utils import normalize_project_path
|
||||
|
||||
console = Console()
|
||||
@@ -27,6 +28,16 @@ console = Console()
|
||||
# Minimum rclone version for --create-empty-src-dirs support
|
||||
MIN_RCLONE_VERSION_EMPTY_DIRS = (1, 64, 0)
|
||||
|
||||
# Tigris edge caching returns stale data for users outside the origin region (iad).
|
||||
# --header is rclone's global flag that applies to ALL HTTP transactions (list, download,
|
||||
# upload). This is critical because bisync starts with S3 ListObjectsV2, which is neither
|
||||
# a download nor upload — so --header-download/--header-upload would miss list requests.
|
||||
# See: https://www.tigrisdata.com/docs/objects/consistency/
|
||||
TIGRIS_CONSISTENCY_HEADERS = [
|
||||
"--header",
|
||||
"X-Tigris-Consistent: true",
|
||||
]
|
||||
|
||||
|
||||
class RunResult(Protocol):
|
||||
returncode: int
|
||||
@@ -128,13 +139,16 @@ def get_bmignore_filter_path() -> Path:
|
||||
def get_project_bisync_state(project_name: str) -> Path:
|
||||
"""Get path to project's bisync state directory.
|
||||
|
||||
Honors ``BASIC_MEMORY_CONFIG_DIR`` so isolated instances each keep their
|
||||
own bisync state alongside their config.
|
||||
|
||||
Args:
|
||||
project_name: Name of the project
|
||||
|
||||
Returns:
|
||||
Path to bisync state directory for this project
|
||||
"""
|
||||
return Path.home() / ".basic-memory" / "bisync-state" / project_name
|
||||
return resolve_data_dir() / "bisync-state" / project_name
|
||||
|
||||
|
||||
def bisync_initialized(project_name: str) -> bool:
|
||||
@@ -210,8 +224,12 @@ def project_sync(
|
||||
"sync",
|
||||
str(local_path),
|
||||
remote_path,
|
||||
*TIGRIS_CONSISTENCY_HEADERS,
|
||||
"--filter-from",
|
||||
str(filter_path),
|
||||
# Prevent NUL byte padding on virtual filesystems (e.g. Google Drive File Stream)
|
||||
# See: rclone/rclone#6801
|
||||
"--local-no-preallocate",
|
||||
]
|
||||
|
||||
if verbose:
|
||||
@@ -279,6 +297,7 @@ def project_bisync(
|
||||
"bisync",
|
||||
str(local_path),
|
||||
remote_path,
|
||||
*TIGRIS_CONSISTENCY_HEADERS,
|
||||
"--resilient",
|
||||
"--conflict-resolve=newer",
|
||||
"--max-delete=25",
|
||||
@@ -287,6 +306,9 @@ def project_bisync(
|
||||
str(filter_path),
|
||||
"--workdir",
|
||||
str(state_path),
|
||||
# Prevent NUL byte padding on virtual filesystems (e.g. Google Drive File Stream)
|
||||
# See: rclone/rclone#6801
|
||||
"--local-no-preallocate",
|
||||
]
|
||||
|
||||
# Add --create-empty-src-dirs if rclone version supports it (v1.64+)
|
||||
@@ -354,6 +376,7 @@ def project_check(
|
||||
"check",
|
||||
str(local_path),
|
||||
remote_path,
|
||||
*TIGRIS_CONSISTENCY_HEADERS,
|
||||
"--filter-from",
|
||||
str(filter_path),
|
||||
]
|
||||
@@ -393,6 +416,6 @@ def project_ls(
|
||||
if path:
|
||||
remote_path = f"{remote_path}/{path}"
|
||||
|
||||
cmd = ["rclone", "ls", remote_path]
|
||||
cmd = ["rclone", "ls", *TIGRIS_CONSISTENCY_HEADERS, remote_path]
|
||||
result = run(cmd, capture_output=True, text=True, check=True)
|
||||
return result.stdout.splitlines()
|
||||
|
||||
@@ -4,7 +4,7 @@ import os
|
||||
import platform
|
||||
import shutil
|
||||
import subprocess
|
||||
from typing import Optional
|
||||
from typing import Any, Optional, cast
|
||||
|
||||
from rich.console import Console
|
||||
|
||||
@@ -53,7 +53,9 @@ def run_command(command: list[str], check: bool = True) -> subprocess.CompletedP
|
||||
|
||||
|
||||
def install_rclone_macos() -> None:
|
||||
"""Install rclone on macOS using Homebrew or official script."""
|
||||
"""Install rclone on macOS using package managers."""
|
||||
install_errors: list[str] = []
|
||||
|
||||
# Try Homebrew first
|
||||
if shutil.which("brew"):
|
||||
try:
|
||||
@@ -61,35 +63,37 @@ def install_rclone_macos() -> None:
|
||||
run_command(["brew", "install", "rclone"])
|
||||
console.print("[green]rclone installed via Homebrew[/green]")
|
||||
return
|
||||
except RcloneInstallError:
|
||||
console.print(
|
||||
"[yellow]Homebrew installation failed, trying official script...[/yellow]"
|
||||
)
|
||||
except RcloneInstallError as exc:
|
||||
install_errors.append(f"Homebrew failed: {exc}")
|
||||
console.print("[yellow]Homebrew installation failed, trying MacPorts...[/yellow]")
|
||||
|
||||
# Fallback to official script
|
||||
console.print("[blue]Installing rclone via official script...[/blue]")
|
||||
try:
|
||||
run_command(["sh", "-c", "curl https://rclone.org/install.sh | sudo bash"])
|
||||
console.print("[green]rclone installed via official script[/green]")
|
||||
except RcloneInstallError:
|
||||
raise RcloneInstallError(
|
||||
"Failed to install rclone. Please install manually: brew install rclone"
|
||||
)
|
||||
if shutil.which("port"):
|
||||
try:
|
||||
console.print("[blue]Installing rclone via MacPorts...[/blue]")
|
||||
run_command(["sudo", "port", "install", "rclone"])
|
||||
console.print("[green]rclone installed via MacPorts[/green]")
|
||||
return
|
||||
except RcloneInstallError as exc:
|
||||
install_errors.append(f"MacPorts failed: {exc}")
|
||||
console.print("[yellow]MacPorts installation failed[/yellow]")
|
||||
|
||||
details = "\n".join(f"- {error}" for error in install_errors)
|
||||
if details:
|
||||
details = f"\n\nAttempts:\n{details}"
|
||||
raise RcloneInstallError(
|
||||
"Could not install rclone automatically with an available package manager.\n\n"
|
||||
"Install rclone manually with one of:\n"
|
||||
" brew install rclone\n"
|
||||
" sudo port install rclone\n"
|
||||
" Download from https://rclone.org/downloads/ and add rclone to PATH"
|
||||
f"{details}"
|
||||
)
|
||||
|
||||
|
||||
def install_rclone_linux() -> None:
|
||||
"""Install rclone on Linux using package managers or official script."""
|
||||
# Try snap first (most universal)
|
||||
if shutil.which("snap"):
|
||||
try:
|
||||
console.print("[blue]Installing rclone via snap...[/blue]")
|
||||
run_command(["sudo", "snap", "install", "rclone"])
|
||||
console.print("[green]rclone installed via snap[/green]")
|
||||
return
|
||||
except RcloneInstallError:
|
||||
console.print("[yellow]Snap installation failed, trying apt...[/yellow]")
|
||||
"""Install rclone on Linux using package managers."""
|
||||
install_errors: list[str] = []
|
||||
|
||||
# Try apt (Debian/Ubuntu)
|
||||
if shutil.which("apt"):
|
||||
try:
|
||||
console.print("[blue]Installing rclone via apt...[/blue]")
|
||||
@@ -97,18 +101,75 @@ def install_rclone_linux() -> None:
|
||||
run_command(["sudo", "apt", "install", "-y", "rclone"])
|
||||
console.print("[green]rclone installed via apt[/green]")
|
||||
return
|
||||
except RcloneInstallError:
|
||||
console.print("[yellow]apt installation failed, trying official script...[/yellow]")
|
||||
except RcloneInstallError as exc:
|
||||
install_errors.append(f"apt failed: {exc}")
|
||||
console.print("[yellow]apt installation failed, trying dnf...[/yellow]")
|
||||
|
||||
# Fallback to official script
|
||||
console.print("[blue]Installing rclone via official script...[/blue]")
|
||||
try:
|
||||
run_command(["sh", "-c", "curl https://rclone.org/install.sh | sudo bash"])
|
||||
console.print("[green]rclone installed via official script[/green]")
|
||||
except RcloneInstallError:
|
||||
raise RcloneInstallError(
|
||||
"Failed to install rclone. Please install manually: sudo snap install rclone"
|
||||
)
|
||||
if shutil.which("dnf"):
|
||||
try:
|
||||
console.print("[blue]Installing rclone via dnf...[/blue]")
|
||||
run_command(["sudo", "dnf", "install", "-y", "rclone"])
|
||||
console.print("[green]rclone installed via dnf[/green]")
|
||||
return
|
||||
except RcloneInstallError as exc:
|
||||
install_errors.append(f"dnf failed: {exc}")
|
||||
console.print("[yellow]dnf installation failed, trying yum...[/yellow]")
|
||||
|
||||
if shutil.which("yum"):
|
||||
try:
|
||||
console.print("[blue]Installing rclone via yum...[/blue]")
|
||||
run_command(["sudo", "yum", "install", "-y", "rclone"])
|
||||
console.print("[green]rclone installed via yum[/green]")
|
||||
return
|
||||
except RcloneInstallError as exc:
|
||||
install_errors.append(f"yum failed: {exc}")
|
||||
console.print("[yellow]yum installation failed, trying pacman...[/yellow]")
|
||||
|
||||
if shutil.which("pacman"):
|
||||
try:
|
||||
console.print("[blue]Installing rclone via pacman...[/blue]")
|
||||
run_command(["sudo", "pacman", "-S", "--noconfirm", "rclone"])
|
||||
console.print("[green]rclone installed via pacman[/green]")
|
||||
return
|
||||
except RcloneInstallError as exc:
|
||||
install_errors.append(f"pacman failed: {exc}")
|
||||
console.print("[yellow]pacman installation failed, trying zypper...[/yellow]")
|
||||
|
||||
if shutil.which("zypper"):
|
||||
try:
|
||||
console.print("[blue]Installing rclone via zypper...[/blue]")
|
||||
run_command(["sudo", "zypper", "--non-interactive", "install", "rclone"])
|
||||
console.print("[green]rclone installed via zypper[/green]")
|
||||
return
|
||||
except RcloneInstallError as exc:
|
||||
install_errors.append(f"zypper failed: {exc}")
|
||||
console.print("[yellow]zypper installation failed, trying snap...[/yellow]")
|
||||
|
||||
if shutil.which("snap"):
|
||||
try:
|
||||
console.print("[blue]Installing rclone via snap...[/blue]")
|
||||
run_command(["sudo", "snap", "install", "rclone"])
|
||||
console.print("[green]rclone installed via snap[/green]")
|
||||
return
|
||||
except RcloneInstallError as exc:
|
||||
install_errors.append(f"snap failed: {exc}")
|
||||
console.print("[yellow]snap installation failed[/yellow]")
|
||||
|
||||
details = "\n".join(f"- {error}" for error in install_errors)
|
||||
if details:
|
||||
details = f"\n\nAttempts:\n{details}"
|
||||
raise RcloneInstallError(
|
||||
"Could not install rclone automatically with an available package manager.\n\n"
|
||||
"Install rclone manually with one of your OS package managers, for example:\n"
|
||||
" sudo apt install rclone\n"
|
||||
" sudo dnf install rclone\n"
|
||||
" sudo yum install rclone\n"
|
||||
" sudo pacman -S rclone\n"
|
||||
" sudo zypper install rclone\n"
|
||||
" sudo snap install rclone\n"
|
||||
"Or download from https://rclone.org/downloads/ and add rclone to PATH"
|
||||
f"{details}"
|
||||
)
|
||||
|
||||
|
||||
def install_rclone_windows() -> None:
|
||||
@@ -209,27 +270,38 @@ def refresh_windows_path() -> None:
|
||||
if platform.system().lower() != "windows":
|
||||
return
|
||||
|
||||
# Importing here after performing platform detection. Also note that we have to ignore pylance/pyright
|
||||
# warnings about winreg attributes so that "errors" don't appear on non-Windows platforms.
|
||||
# Importing here after performing platform detection. Non-Windows type checkers may still
|
||||
# resolve a stub without registry members, so keep this platform-only module dynamic here.
|
||||
import winreg
|
||||
|
||||
winreg_module = cast(Any, winreg)
|
||||
user_key_path = r"Environment"
|
||||
system_key_path = r"System\CurrentControlSet\Control\Session Manager\Environment"
|
||||
new_path = ""
|
||||
|
||||
# Read user PATH
|
||||
try:
|
||||
reg_key = winreg.OpenKey(winreg.HKEY_CURRENT_USER, user_key_path, 0, winreg.KEY_READ) # type: ignore[reportAttributeAccessIssue]
|
||||
user_path, _ = winreg.QueryValueEx(reg_key, "PATH") # type: ignore[reportAttributeAccessIssue]
|
||||
winreg.CloseKey(reg_key) # type: ignore[reportAttributeAccessIssue]
|
||||
reg_key = winreg_module.OpenKey(
|
||||
winreg_module.HKEY_CURRENT_USER,
|
||||
user_key_path,
|
||||
0,
|
||||
winreg_module.KEY_READ,
|
||||
)
|
||||
user_path, _ = winreg_module.QueryValueEx(reg_key, "PATH")
|
||||
winreg_module.CloseKey(reg_key)
|
||||
except Exception:
|
||||
user_path = ""
|
||||
|
||||
# Read system PATH
|
||||
try:
|
||||
reg_key = winreg.OpenKey(winreg.HKEY_LOCAL_MACHINE, system_key_path, 0, winreg.KEY_READ) # type: ignore[reportAttributeAccessIssue]
|
||||
system_path, _ = winreg.QueryValueEx(reg_key, "PATH") # type: ignore[reportAttributeAccessIssue]
|
||||
winreg.CloseKey(reg_key) # type: ignore[reportAttributeAccessIssue]
|
||||
reg_key = winreg_module.OpenKey(
|
||||
winreg_module.HKEY_LOCAL_MACHINE,
|
||||
system_key_path,
|
||||
0,
|
||||
winreg_module.KEY_READ,
|
||||
)
|
||||
system_path, _ = winreg_module.QueryValueEx(reg_key, "PATH")
|
||||
winreg_module.CloseKey(reg_key)
|
||||
except Exception:
|
||||
system_path = ""
|
||||
|
||||
|
||||
@@ -10,7 +10,6 @@ import httpx
|
||||
|
||||
from basic_memory.ignore_utils import load_gitignore_patterns, should_ignore_path
|
||||
from basic_memory.mcp.async_client import get_client
|
||||
from basic_memory.mcp.tools.utils import call_put
|
||||
|
||||
# Archive file extensions that should be skipped during upload
|
||||
ARCHIVE_EXTENSIONS = {".zip", ".tar", ".gz", ".bz2", ".xz", ".7z", ".rar", ".tgz", ".tbz2"}
|
||||
@@ -24,7 +23,7 @@ async def upload_path(
|
||||
dry_run: bool = False,
|
||||
*,
|
||||
client_cm_factory: Callable[[], AbstractAsyncContextManager[httpx.AsyncClient]] | None = None,
|
||||
put_func=call_put,
|
||||
put_func: Callable | None = None,
|
||||
) -> bool:
|
||||
"""
|
||||
Upload a file or directory to cloud project via WebDAV.
|
||||
@@ -117,9 +116,20 @@ async def upload_path(
|
||||
|
||||
# Upload via HTTP PUT to WebDAV endpoint with mtime header
|
||||
# Using X-OC-Mtime (ownCloud/Nextcloud standard)
|
||||
response = await put_func(
|
||||
client, remote_path, content=content, headers={"X-OC-Mtime": str(mtime)}
|
||||
)
|
||||
if put_func is not None:
|
||||
# Test injection path
|
||||
response = await put_func(
|
||||
client,
|
||||
remote_path,
|
||||
content=content,
|
||||
headers={"X-OC-Mtime": str(mtime)},
|
||||
)
|
||||
else:
|
||||
response = await client.put(
|
||||
remote_path,
|
||||
content=content,
|
||||
headers={"X-OC-Mtime": str(mtime)},
|
||||
)
|
||||
response.raise_for_status()
|
||||
|
||||
# Format total size based on magnitude
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
"""Upload CLI commands for basic-memory projects."""
|
||||
|
||||
from functools import partial
|
||||
from pathlib import Path
|
||||
|
||||
import typer
|
||||
@@ -8,11 +9,16 @@ from rich.console import Console
|
||||
from basic_memory.cli.app import cloud_app
|
||||
from basic_memory.cli.commands.command_utils import run_with_cleanup
|
||||
from basic_memory.cli.commands.cloud.cloud_utils import (
|
||||
CloudUtilsError,
|
||||
create_cloud_project,
|
||||
project_exists,
|
||||
sync_project,
|
||||
)
|
||||
from basic_memory.cli.commands.cloud.upload import upload_path
|
||||
from basic_memory.mcp.async_client import (
|
||||
get_cloud_control_plane_client,
|
||||
resolve_configured_workspace,
|
||||
)
|
||||
|
||||
console = Console()
|
||||
|
||||
@@ -72,12 +78,20 @@ def upload(
|
||||
"""
|
||||
|
||||
async def _upload():
|
||||
resolved_workspace = resolve_configured_workspace(project_name=project)
|
||||
|
||||
try:
|
||||
project_already_exists = await project_exists(project, workspace=resolved_workspace)
|
||||
except CloudUtilsError as e:
|
||||
console.print(f"[red]Failed to check cloud project '{project}': {e}[/red]")
|
||||
raise typer.Exit(1)
|
||||
|
||||
# Check if project exists
|
||||
if not await project_exists(project):
|
||||
if not project_already_exists:
|
||||
if create_project:
|
||||
console.print(f"[blue]Creating cloud project '{project}'...[/blue]")
|
||||
try:
|
||||
await create_cloud_project(project)
|
||||
await create_cloud_project(project, workspace=resolved_workspace)
|
||||
console.print(f"[green]Created project '{project}'[/green]")
|
||||
except Exception as e:
|
||||
console.print(f"[red]Failed to create project: {e}[/red]")
|
||||
@@ -86,12 +100,14 @@ def upload(
|
||||
console.print(
|
||||
f"[red]Project '{project}' does not exist.[/red]\n"
|
||||
f"[yellow]Options:[/yellow]\n"
|
||||
f" 1. Create it first: bm project add {project}\n"
|
||||
f" 1. Create it first: bm project add {project} --cloud\n"
|
||||
f" 2. Use --create-project flag to create automatically"
|
||||
)
|
||||
raise typer.Exit(1)
|
||||
|
||||
# Perform upload (or dry run)
|
||||
if resolved_workspace:
|
||||
console.print(f"[dim]Using workspace: {resolved_workspace}[/dim]")
|
||||
if dry_run:
|
||||
console.print(
|
||||
f"[yellow]DRY RUN: Showing what would be uploaded to '{project}'[/yellow]"
|
||||
@@ -100,7 +116,15 @@ def upload(
|
||||
console.print(f"[blue]Uploading {path} to project '{project}'...[/blue]")
|
||||
|
||||
success = await upload_path(
|
||||
path, project, verbose=verbose, use_gitignore=not no_gitignore, dry_run=dry_run
|
||||
path,
|
||||
project,
|
||||
verbose=verbose,
|
||||
use_gitignore=not no_gitignore,
|
||||
dry_run=dry_run,
|
||||
client_cm_factory=partial(
|
||||
get_cloud_control_plane_client,
|
||||
workspace=resolved_workspace,
|
||||
),
|
||||
)
|
||||
if not success:
|
||||
console.print("[red]Upload failed[/red]")
|
||||
@@ -111,12 +135,14 @@ def upload(
|
||||
else:
|
||||
console.print(f"[green]Successfully uploaded to '{project}'[/green]")
|
||||
|
||||
# Sync project if requested (skip on dry run)
|
||||
# Force full scan after bisync to ensure database is up-to-date with synced files
|
||||
# Sync project if requested (skip on dry run).
|
||||
# Trigger: upload adds new files the watcher has not observed locally.
|
||||
# Why: force_full ensures those freshly uploaded files are indexed immediately.
|
||||
# Outcome: upload keeps its eager reindex while sync/bisync stay incremental.
|
||||
if sync and not dry_run:
|
||||
console.print(f"[blue]Syncing project '{project}'...[/blue]")
|
||||
try:
|
||||
await sync_project(project, force_full=True)
|
||||
await sync_project(project)
|
||||
except Exception as e:
|
||||
console.print(f"[yellow]Warning: Sync failed: {e}[/yellow]")
|
||||
console.print("[dim]Files uploaded but may not be indexed yet[/dim]")
|
||||
|
||||
@@ -0,0 +1,118 @@
|
||||
"""Workspace commands for Basic Memory cloud workspaces."""
|
||||
|
||||
import typer
|
||||
from rich.console import Console
|
||||
from rich.table import Table
|
||||
|
||||
from basic_memory.cli.commands.command_utils import run_with_cleanup
|
||||
from basic_memory.config import ConfigManager
|
||||
from basic_memory.mcp.project_context import get_available_workspaces
|
||||
from basic_memory.schemas.cloud import (
|
||||
format_workspace_choices,
|
||||
format_workspace_selection_choices,
|
||||
workspace_matches_identifier,
|
||||
)
|
||||
|
||||
console = Console()
|
||||
|
||||
workspace_app = typer.Typer(help="Manage cloud workspaces")
|
||||
|
||||
|
||||
@workspace_app.command("list")
|
||||
def list_workspaces() -> None:
|
||||
"""List cloud workspaces available to the current OAuth session."""
|
||||
|
||||
async def _list():
|
||||
return await get_available_workspaces()
|
||||
|
||||
try:
|
||||
workspaces = run_with_cleanup(_list())
|
||||
except RuntimeError as exc:
|
||||
console.print(f"[red]Error: {exc}[/red]")
|
||||
raise typer.Exit(1)
|
||||
except Exception as exc: # pragma: no cover
|
||||
console.print(f"[red]Error listing workspaces: {exc}[/red]")
|
||||
raise typer.Exit(1)
|
||||
|
||||
if not workspaces:
|
||||
console.print("[yellow]No accessible workspaces found.[/yellow]")
|
||||
return
|
||||
|
||||
config = ConfigManager().config
|
||||
default_ws = config.default_workspace
|
||||
|
||||
table = Table(title="Available Workspaces")
|
||||
table.add_column("Name", style="cyan")
|
||||
table.add_column("Type", style="blue")
|
||||
table.add_column("Role", style="green")
|
||||
table.add_column("Tenant ID", style="yellow")
|
||||
table.add_column("Default", style="magenta")
|
||||
|
||||
for workspace in workspaces:
|
||||
is_default = "[X]" if workspace.tenant_id == default_ws else ""
|
||||
table.add_row(
|
||||
workspace.name,
|
||||
workspace.workspace_type,
|
||||
workspace.role,
|
||||
workspace.tenant_id,
|
||||
is_default,
|
||||
)
|
||||
|
||||
console.print(table)
|
||||
|
||||
|
||||
@workspace_app.command("set-default")
|
||||
def set_default_workspace(
|
||||
identifier: str = typer.Argument(
|
||||
...,
|
||||
help="Workspace name, slug, type, or tenant_id to set as default",
|
||||
),
|
||||
) -> None:
|
||||
"""Set the default cloud workspace.
|
||||
|
||||
The default workspace is used as fallback when no per-project workspace
|
||||
is configured. Resolves the identifier against available workspaces.
|
||||
|
||||
Examples:
|
||||
bm cloud workspace set-default Personal
|
||||
bm cloud workspace set-default organization
|
||||
bm cloud workspace set-default 11111111-1111-1111-1111-111111111111
|
||||
"""
|
||||
|
||||
async def _list():
|
||||
return await get_available_workspaces()
|
||||
|
||||
try:
|
||||
workspaces = run_with_cleanup(_list())
|
||||
except RuntimeError as exc:
|
||||
console.print(f"[red]Error: {exc}[/red]")
|
||||
raise typer.Exit(1)
|
||||
|
||||
if not workspaces:
|
||||
console.print("[yellow]No accessible workspaces found.[/yellow]")
|
||||
raise typer.Exit(1)
|
||||
|
||||
matches = [ws for ws in workspaces if workspace_matches_identifier(ws, identifier)]
|
||||
|
||||
if not matches:
|
||||
console.print(f"[red]Error: Workspace '{identifier}' not found[/red]")
|
||||
console.print(f"[dim]Available:\n{format_workspace_choices(workspaces)}[/dim]")
|
||||
raise typer.Exit(1)
|
||||
|
||||
if len(matches) > 1:
|
||||
console.print(f"[red]Error: Workspace '{identifier}' matches multiple workspaces.[/red]")
|
||||
console.print(
|
||||
"[dim]Choose one of these matching workspaces by slug:\n"
|
||||
f"{format_workspace_selection_choices(matches)}[/dim]"
|
||||
)
|
||||
raise typer.Exit(1)
|
||||
|
||||
selected = matches[0]
|
||||
config_manager = ConfigManager()
|
||||
config = config_manager.config
|
||||
config.default_workspace = selected.tenant_id
|
||||
config_manager.save_config(config)
|
||||
|
||||
console.print(
|
||||
f"[green]Default workspace set to '{selected.name}' ({selected.tenant_id})[/green]"
|
||||
)
|
||||
@@ -9,11 +9,10 @@ import typer
|
||||
from rich.console import Console
|
||||
|
||||
from basic_memory import db
|
||||
from basic_memory.config import ConfigManager
|
||||
from basic_memory.mcp.async_client import get_client
|
||||
|
||||
from basic_memory.mcp.tools.utils import call_post, call_get
|
||||
from basic_memory.mcp.clients import ProjectClient
|
||||
from basic_memory.mcp.project_context import get_active_project
|
||||
from basic_memory.schemas import ProjectInfoResponse
|
||||
|
||||
console = Console()
|
||||
|
||||
@@ -55,19 +54,18 @@ async def run_sync(
|
||||
run_in_background: If True, return immediately; if False, wait for completion
|
||||
"""
|
||||
|
||||
# Resolve default project so get_client() can route per-project
|
||||
project = project or ConfigManager().default_project
|
||||
|
||||
try:
|
||||
async with get_client() as client:
|
||||
async with get_client(project_name=project) as client:
|
||||
project_item = await get_active_project(client, project, None)
|
||||
url = f"{project_item.project_url}/project/sync"
|
||||
params = []
|
||||
if force_full:
|
||||
params.append("force_full=true")
|
||||
if not run_in_background:
|
||||
params.append("run_in_background=false")
|
||||
if params:
|
||||
url += "?" + "&".join(params)
|
||||
response = await call_post(client, url)
|
||||
data = response.json()
|
||||
project_client = ProjectClient(client)
|
||||
data = await project_client.sync(
|
||||
project_item.external_id,
|
||||
force_full=force_full,
|
||||
run_in_background=run_in_background,
|
||||
)
|
||||
# Background mode returns {"message": "..."}, foreground returns SyncReportResponse
|
||||
if "message" in data:
|
||||
console.print(f"[green]{data['message']}[/green]")
|
||||
@@ -88,12 +86,24 @@ async def run_sync(
|
||||
|
||||
async def get_project_info(project: str):
|
||||
"""Get project information via API endpoint."""
|
||||
|
||||
try:
|
||||
async with get_client() as client:
|
||||
async with get_client(project_name=project) as client:
|
||||
project_item = await get_active_project(client, project, None)
|
||||
response = await call_get(client, f"{project_item.project_url}/project/info")
|
||||
return ProjectInfoResponse.model_validate(response.json())
|
||||
return await ProjectClient(client).get_info(project_item.external_id)
|
||||
except (ToolError, ValueError) as e:
|
||||
console.print(f"[red]Sync failed: {e}[/red]")
|
||||
error_text = str(e)
|
||||
if "internal proxy error" in error_text.lower() and "not found in configuration" in (
|
||||
error_text.lower()
|
||||
):
|
||||
console.print(
|
||||
"[red]Project info failed: cloud returned an internal configuration error for "
|
||||
"this project.[/red]"
|
||||
)
|
||||
console.print(
|
||||
"[yellow]This is a cloud backend issue for detailed info lookups. "
|
||||
"Use `bm project list --cloud` for project metadata until the service is updated."
|
||||
"[/yellow]"
|
||||
)
|
||||
else:
|
||||
console.print(f"[red]Project info failed: {e}[/red]")
|
||||
raise typer.Exit(1)
|
||||
|
||||
@@ -1,16 +1,21 @@
|
||||
"""Database management commands."""
|
||||
|
||||
from pathlib import Path
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path, PurePosixPath, PureWindowsPath
|
||||
|
||||
import psutil
|
||||
import typer
|
||||
from loguru import logger
|
||||
from rich.console import Console
|
||||
from rich.progress import Progress, SpinnerColumn, TextColumn, BarColumn, TaskProgressColumn
|
||||
from sqlalchemy.exc import OperationalError
|
||||
|
||||
from basic_memory import db
|
||||
from basic_memory.cli.app import app
|
||||
from basic_memory.cli.commands.command_utils import run_with_cleanup
|
||||
from basic_memory.config import ConfigManager
|
||||
from basic_memory.config import ConfigManager, ProjectMode
|
||||
from basic_memory.indexing import IndexProgress
|
||||
from basic_memory.repository import ProjectRepository
|
||||
from basic_memory.services.initialization import reconcile_projects_with_config
|
||||
from basic_memory.sync.sync_service import get_sync_service
|
||||
@@ -18,6 +23,136 @@ from basic_memory.sync.sync_service import get_sync_service
|
||||
console = Console()
|
||||
|
||||
|
||||
def _is_basic_memory_mcp(cmdline: list[str]) -> bool:
|
||||
"""Heuristic: does this argv represent a `basic-memory mcp` server?
|
||||
|
||||
The MCP server can be launched any of:
|
||||
basic-memory mcp
|
||||
bm mcp # entrypoint alias from pyproject.toml
|
||||
python -m basic_memory.cli.main mcp # module form
|
||||
uv run basic-memory mcp / uv run bm mcp # uv wrappers
|
||||
/abs/path/to/{bm,basic-memory}[.exe] mcp
|
||||
|
||||
A reliable match needs both signals:
|
||||
1. "mcp" appears as an exact argv token (not "mcp-foo").
|
||||
2. Some argv token names the basic-memory entrypoint — either by
|
||||
hyphen/underscore form, or as a `bm` script (covers `/usr/local/bin/bm`,
|
||||
`bm.exe`, etc. via Path.stem).
|
||||
"""
|
||||
if "mcp" not in cmdline:
|
||||
return False
|
||||
for arg in cmdline:
|
||||
if "basic-memory" in arg or "basic_memory" in arg:
|
||||
return True
|
||||
# Try both POSIX and Windows path interpretations so a test on
|
||||
# macOS still recognizes `C:\\...\\bm.exe`, and a real Windows
|
||||
# run still recognizes `/usr/local/bin/bm`. Path() alone uses
|
||||
# the host OS, which gives wrong stems for foreign separators.
|
||||
if PurePosixPath(arg).stem == "bm" or PureWindowsPath(arg).stem == "bm":
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _find_live_mcp_processes() -> list[tuple[int, str]]:
|
||||
"""Return (pid, joined_cmdline) for live `basic-memory mcp` processes.
|
||||
|
||||
Why this exists (issue #765):
|
||||
On POSIX, `Path.unlink()` removes the directory entry but the inode
|
||||
survives as long as any process holds the file open. A `bm reset`
|
||||
run while Claude Desktop (or another MCP client) is alive will
|
||||
therefore "succeed" — but the still-running MCP keeps reading the
|
||||
old, now-invisible memory.db inode and returns phantom rows. On
|
||||
Windows the OS naturally raises PermissionError on `unlink()`, so
|
||||
the bug is POSIX-specific. We detect proactively to give the same
|
||||
error experience on every platform before doing damage.
|
||||
|
||||
The current process is excluded so this can be called from inside a
|
||||
`bm reset` invocation. NoSuchProcess / AccessDenied are swallowed
|
||||
because process tables race with the scan and we don't want a
|
||||
transient permission error to mask a real zombie.
|
||||
"""
|
||||
me = os.getpid()
|
||||
matches: list[tuple[int, str]] = []
|
||||
for proc in psutil.process_iter(["pid", "cmdline"]):
|
||||
try:
|
||||
pid = proc.info.get("pid")
|
||||
if pid is None or pid == me:
|
||||
continue
|
||||
cmdline = proc.info.get("cmdline") or []
|
||||
if not cmdline:
|
||||
continue
|
||||
if _is_basic_memory_mcp(cmdline):
|
||||
matches.append((pid, " ".join(cmdline)))
|
||||
except (psutil.NoSuchProcess, psutil.AccessDenied):
|
||||
continue
|
||||
return matches
|
||||
|
||||
|
||||
def _abort_if_mcp_processes_alive() -> None:
|
||||
"""Refuse `bm reset` while basic-memory MCP processes are still running.
|
||||
|
||||
See _find_live_mcp_processes for the underlying POSIX-vs-Windows
|
||||
rationale. Prints a per-PID list and platform-appropriate cleanup
|
||||
instructions, then exits non-zero so destructive work never starts.
|
||||
"""
|
||||
zombies = _find_live_mcp_processes()
|
||||
if not zombies:
|
||||
return
|
||||
|
||||
console.print("[red]Refusing to reset:[/red] basic-memory MCP processes are still running.")
|
||||
console.print(
|
||||
"[yellow]On macOS/Linux these would keep reading the deleted memory.db inode "
|
||||
"and return phantom search results (see #765).[/yellow]"
|
||||
)
|
||||
for pid, cmd in zombies:
|
||||
console.print(f" PID {pid}: {cmd}")
|
||||
console.print("\n[bold]How to clean up:[/bold]")
|
||||
console.print(" 1. Quit Claude Desktop and any other MCP clients.")
|
||||
if os.name == "nt":
|
||||
console.print(
|
||||
" 2. Verify nothing remains: "
|
||||
"[green]Get-CimInstance Win32_Process | "
|
||||
"Where-Object {$_.CommandLine -like '*basic-memory*mcp*'}[/green]"
|
||||
)
|
||||
else:
|
||||
console.print(" 2. Verify nothing remains: [green]pgrep -fa 'basic-memory mcp'[/green]")
|
||||
console.print(" 3. Re-run [green]bm reset[/green].")
|
||||
raise typer.Exit(1)
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class EmbeddingProgress:
|
||||
"""Typed CLI progress payload for embedding backfills."""
|
||||
|
||||
entity_id: int
|
||||
completed: int
|
||||
total: int
|
||||
|
||||
|
||||
def _format_eta(seconds: float | None) -> str:
|
||||
"""Render a compact ETA string for CLI progress descriptions."""
|
||||
if seconds is None:
|
||||
return "--:--"
|
||||
|
||||
whole_seconds = max(int(seconds), 0)
|
||||
minutes, remaining_seconds = divmod(whole_seconds, 60)
|
||||
hours, remaining_minutes = divmod(minutes, 60)
|
||||
if hours:
|
||||
return f"{hours:d}:{remaining_minutes:02d}:{remaining_seconds:02d}"
|
||||
return f"{remaining_minutes:02d}:{remaining_seconds:02d}"
|
||||
|
||||
|
||||
def _format_index_progress(progress: IndexProgress) -> str:
|
||||
"""Render typed index progress as a compact Rich task description."""
|
||||
files_per_minute = int(progress.files_per_minute) if progress.files_per_minute else 0
|
||||
return (
|
||||
" Indexing files... "
|
||||
f"{progress.files_processed}/{progress.files_total} files | "
|
||||
f"{progress.batches_completed}/{progress.batches_total} batches | "
|
||||
f"{files_per_minute}/min | ETA {_format_eta(progress.eta_seconds)}"
|
||||
)
|
||||
|
||||
|
||||
async def _reindex_projects(app_config):
|
||||
"""Reindex all projects in a single async context.
|
||||
|
||||
@@ -50,6 +185,16 @@ async def _reindex_projects(app_config):
|
||||
@app.command()
|
||||
def reset(
|
||||
reindex: bool = typer.Option(False, "--reindex", help="Rebuild db index from filesystem"),
|
||||
force: bool = typer.Option(
|
||||
False,
|
||||
"--force",
|
||||
help=(
|
||||
"Skip the pre-flight check that refuses to reset while "
|
||||
"basic-memory MCP processes are running. Use only in "
|
||||
"automated workflows where you've already ensured no MCP "
|
||||
"clients are attached to the database."
|
||||
),
|
||||
),
|
||||
): # pragma: no cover
|
||||
"""Reset database (drop all tables and recreate)."""
|
||||
console.print(
|
||||
@@ -58,6 +203,14 @@ def reset(
|
||||
"Use [green]bm reset --reindex[/green] to automatically rebuild the index afterward."
|
||||
)
|
||||
if typer.confirm("Reset the database index?"):
|
||||
# Pre-flight: refuse to proceed if MCP processes still hold the DB
|
||||
# file open. POSIX would silently let us unlink the inode while
|
||||
# they keep reading it; Windows would error here anyway. See
|
||||
# _find_live_mcp_processes for the full story. --force is the
|
||||
# documented escape hatch for scripted/CI runs.
|
||||
if not force:
|
||||
_abort_if_mcp_processes_alive()
|
||||
|
||||
logger.info("Resetting database...")
|
||||
config_manager = ConfigManager()
|
||||
app_config = config_manager.config
|
||||
@@ -103,3 +256,195 @@ def reset(
|
||||
# ensures db.shutdown_db() is called even if _reindex_projects changes
|
||||
run_with_cleanup(_reindex_projects(app_config))
|
||||
console.print("[green]Reindex complete[/green]")
|
||||
|
||||
|
||||
@app.command()
|
||||
def reindex(
|
||||
embeddings: bool = typer.Option(
|
||||
False, "--embeddings", "-e", help="Rebuild vector embeddings (requires semantic search)"
|
||||
),
|
||||
search: bool = typer.Option(False, "--search", "-s", help="Rebuild full-text search index"),
|
||||
full: bool = typer.Option(
|
||||
False,
|
||||
"--full",
|
||||
help="Force a full filesystem scan and file reindex instead of the default incremental scan",
|
||||
),
|
||||
project: str = typer.Option(
|
||||
None, "--project", "-p", help="Reindex a specific project (default: all)"
|
||||
),
|
||||
): # pragma: no cover
|
||||
"""Rebuild search indexes and/or vector embeddings without dropping the database.
|
||||
|
||||
By default runs incremental search + embeddings (if semantic search is enabled).
|
||||
Use --full to bypass incremental scan optimization, rebuild all file-backed search rows,
|
||||
and re-embed all eligible notes.
|
||||
Use --search or --embeddings to rebuild only one side.
|
||||
|
||||
Examples:
|
||||
bm reindex # Incremental search + embeddings
|
||||
bm reindex --full # Full search + full re-embed
|
||||
bm reindex --embeddings # Only rebuild vector embeddings
|
||||
bm reindex --search # Only rebuild FTS index
|
||||
bm reindex --full --search # Full search only
|
||||
bm reindex --full --embeddings # Full re-embed only
|
||||
bm reindex -p claw --full # Full reindex for only the 'claw' project
|
||||
"""
|
||||
# If neither flag is set, do both
|
||||
if not embeddings and not search:
|
||||
embeddings = True
|
||||
search = True
|
||||
|
||||
config_manager = ConfigManager()
|
||||
app_config = config_manager.config
|
||||
|
||||
if embeddings and not app_config.semantic_search_enabled:
|
||||
console.print(
|
||||
"[yellow]Semantic search is not enabled.[/yellow] "
|
||||
"Set [cyan]semantic_search_enabled: true[/cyan] in config to use embeddings."
|
||||
)
|
||||
embeddings = False
|
||||
if not search:
|
||||
raise typer.Exit(0)
|
||||
|
||||
run_with_cleanup(
|
||||
_reindex(app_config, search=search, embeddings=embeddings, full=full, project=project)
|
||||
)
|
||||
|
||||
|
||||
async def _reindex(
|
||||
app_config,
|
||||
*,
|
||||
search: bool,
|
||||
embeddings: bool,
|
||||
full: bool,
|
||||
project: str | None,
|
||||
):
|
||||
"""Run reindex operations."""
|
||||
from basic_memory.repository import EntityRepository
|
||||
from basic_memory.repository.search_repository import create_search_repository
|
||||
from basic_memory.services.search_service import SearchService
|
||||
from basic_memory.services.file_service import FileService
|
||||
from basic_memory.markdown.markdown_processor import MarkdownProcessor
|
||||
from basic_memory.markdown.entity_parser import EntityParser
|
||||
|
||||
try:
|
||||
await reconcile_projects_with_config(app_config)
|
||||
|
||||
_, session_maker = await db.get_or_create_db(
|
||||
db_path=app_config.database_path,
|
||||
db_type=db.DatabaseType.FILESYSTEM,
|
||||
)
|
||||
project_repository = ProjectRepository(session_maker)
|
||||
projects = await project_repository.get_active_projects()
|
||||
|
||||
if project:
|
||||
projects = [p for p in projects if p.name == project]
|
||||
if not projects:
|
||||
# Check if it's a cloud-only project — those can't be reindexed locally
|
||||
project_mode = app_config.get_project_mode(project)
|
||||
if project_mode == ProjectMode.CLOUD:
|
||||
console.print(
|
||||
f"[yellow]Project '{project}' is a cloud project.[/yellow]\n"
|
||||
"Reindexing is a local operation — cloud projects are "
|
||||
"indexed on the server."
|
||||
)
|
||||
else:
|
||||
console.print(f"[red]Project '{project}' not found.[/red]")
|
||||
raise typer.Exit(1)
|
||||
|
||||
for proj in projects:
|
||||
console.print(f"\n[bold]Project: [cyan]{proj.name}[/cyan][/bold]")
|
||||
|
||||
if search:
|
||||
search_mode_label = "full scan" if full else "incremental scan"
|
||||
console.print(
|
||||
f" Rebuilding full-text search index ([cyan]{search_mode_label}[/cyan])..."
|
||||
)
|
||||
sync_service = await get_sync_service(proj)
|
||||
sync_dir = Path(proj.path)
|
||||
with Progress(
|
||||
SpinnerColumn(),
|
||||
TextColumn("[progress.description]{task.description}"),
|
||||
BarColumn(),
|
||||
TaskProgressColumn(),
|
||||
console=console,
|
||||
) as progress:
|
||||
task = progress.add_task(" Indexing files... scanning changes", total=1)
|
||||
|
||||
async def on_index_progress(update: IndexProgress) -> None:
|
||||
total = update.files_total or 1
|
||||
completed = update.files_processed if update.files_total else 1
|
||||
progress.update(
|
||||
task,
|
||||
description=_format_index_progress(update),
|
||||
total=total,
|
||||
completed=min(completed, total),
|
||||
)
|
||||
|
||||
await sync_service.sync(
|
||||
sync_dir,
|
||||
project_name=proj.name,
|
||||
force_full=full,
|
||||
sync_embeddings=False,
|
||||
progress_callback=on_index_progress,
|
||||
)
|
||||
progress.update(task, completed=progress.tasks[task].total or 1)
|
||||
|
||||
console.print(" [green]done[/green] Full-text search index rebuilt")
|
||||
|
||||
if embeddings:
|
||||
embedding_mode_label = "full rebuild" if full else "incremental sync"
|
||||
console.print(
|
||||
f" Building vector embeddings ([cyan]{embedding_mode_label}[/cyan])..."
|
||||
)
|
||||
entity_repository = EntityRepository(session_maker, project_id=proj.id)
|
||||
search_repository = create_search_repository(
|
||||
session_maker, project_id=proj.id, app_config=app_config
|
||||
)
|
||||
project_path = Path(proj.path)
|
||||
entity_parser = EntityParser(project_path)
|
||||
markdown_processor = MarkdownProcessor(entity_parser, app_config=app_config)
|
||||
file_service = FileService(project_path, markdown_processor, app_config=app_config)
|
||||
search_service = SearchService(search_repository, entity_repository, file_service)
|
||||
|
||||
with Progress(
|
||||
SpinnerColumn(),
|
||||
TextColumn("[progress.description]{task.description}"),
|
||||
BarColumn(),
|
||||
TaskProgressColumn(),
|
||||
console=console,
|
||||
) as progress:
|
||||
task = progress.add_task(" Embedding entities...", total=None)
|
||||
|
||||
def on_progress(entity_id, index, total):
|
||||
embedding_progress = EmbeddingProgress(
|
||||
entity_id=entity_id,
|
||||
completed=index,
|
||||
total=total,
|
||||
)
|
||||
# Trigger: repository progress now reports terminal entity completion.
|
||||
# Why: operators need to see finished embedding work rather than
|
||||
# entities merely entering prepare.
|
||||
# Outcome: the CLI bar advances steadily with real completed work.
|
||||
progress.update(
|
||||
task,
|
||||
total=embedding_progress.total,
|
||||
completed=embedding_progress.completed,
|
||||
)
|
||||
|
||||
stats = await search_service.reindex_vectors(
|
||||
progress_callback=on_progress,
|
||||
force_full=full,
|
||||
)
|
||||
progress.update(task, completed=stats["total_entities"])
|
||||
|
||||
console.print(
|
||||
f" [green]done[/green] Embeddings complete: "
|
||||
f"{stats['embedded']} entities embedded, "
|
||||
f"{stats['skipped']} skipped, "
|
||||
f"{stats['errors']} errors"
|
||||
)
|
||||
|
||||
console.print("\n[green]Reindex complete![/green]")
|
||||
finally:
|
||||
await db.shutdown_db()
|
||||
|
||||
@@ -0,0 +1,156 @@
|
||||
"""Doctor command for local consistency checks."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import tempfile
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
from loguru import logger
|
||||
from mcp.server.fastmcp.exceptions import ToolError
|
||||
from rich.console import Console
|
||||
import typer
|
||||
|
||||
from basic_memory.cli.app import app
|
||||
from basic_memory.cli.commands.command_utils import run_with_cleanup
|
||||
from basic_memory.cli.commands.routing import force_routing, validate_routing_flags
|
||||
from basic_memory.markdown.entity_parser import EntityParser
|
||||
from basic_memory.markdown.markdown_processor import MarkdownProcessor
|
||||
from basic_memory.markdown.schemas import EntityFrontmatter, EntityMarkdown
|
||||
from basic_memory.mcp.async_client import get_client
|
||||
from basic_memory.mcp.clients import KnowledgeClient, ProjectClient, SearchClient
|
||||
from basic_memory.schemas.base import Entity
|
||||
from basic_memory.schemas.project_info import ProjectInfoRequest
|
||||
from basic_memory.schemas.search import SearchQuery
|
||||
from basic_memory.schemas import SyncReportResponse
|
||||
|
||||
console = Console()
|
||||
|
||||
|
||||
async def run_doctor() -> None:
|
||||
"""Run local consistency checks for file <-> database flows."""
|
||||
console.print("[blue]Running Basic Memory doctor checks...[/blue]")
|
||||
|
||||
project_name = f"doctor-{uuid.uuid4().hex[:8]}"
|
||||
api_note_title = "Doctor API Note"
|
||||
manual_note_title = "Doctor Manual Note"
|
||||
manual_permalink = "doctor/manual-note"
|
||||
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
temp_path = Path(temp_dir)
|
||||
|
||||
async with get_client() as client:
|
||||
project_client = ProjectClient(client)
|
||||
project_request = ProjectInfoRequest(
|
||||
name=project_name,
|
||||
path=str(temp_path),
|
||||
set_default=False,
|
||||
)
|
||||
|
||||
project_id: str | None = None
|
||||
|
||||
try:
|
||||
status = await project_client.create_project(project_request.model_dump())
|
||||
if not status.new_project:
|
||||
raise ValueError("Failed to create doctor project")
|
||||
project_id = status.new_project.external_id
|
||||
# Use the resolved path from the server — when project_root is configured,
|
||||
# the actual project directory differs from the requested temp_path
|
||||
project_path = Path(status.new_project.path)
|
||||
console.print(f"[green]OK[/green] Created doctor project: {project_name}")
|
||||
|
||||
# --- DB -> File: create an entity via API ---
|
||||
knowledge_client = KnowledgeClient(client, project_id)
|
||||
api_note = Entity(
|
||||
title=api_note_title,
|
||||
directory="doctor",
|
||||
note_type="note",
|
||||
content_type="text/markdown",
|
||||
content=f"# {api_note_title}\n\n- [note] API to file check",
|
||||
entity_metadata={"tags": ["doctor"]},
|
||||
)
|
||||
api_result = await knowledge_client.create_entity(api_note.model_dump())
|
||||
|
||||
api_file = project_path / api_result.file_path
|
||||
if not api_file.exists():
|
||||
raise ValueError(f"API note file missing: {api_result.file_path}")
|
||||
|
||||
api_text = api_file.read_text(encoding="utf-8")
|
||||
if api_note_title not in api_text:
|
||||
raise ValueError("API note content missing from file")
|
||||
|
||||
console.print("[green]OK[/green] API write created file")
|
||||
|
||||
# --- File -> DB: write markdown file directly, then sync ---
|
||||
parser = EntityParser(project_path)
|
||||
processor = MarkdownProcessor(parser)
|
||||
manual_markdown = EntityMarkdown(
|
||||
frontmatter=EntityFrontmatter(
|
||||
metadata={
|
||||
"title": manual_note_title,
|
||||
"type": "note",
|
||||
"permalink": manual_permalink,
|
||||
"tags": ["doctor"],
|
||||
}
|
||||
),
|
||||
content=f"# {manual_note_title}\n\n- [note] File to DB check",
|
||||
)
|
||||
|
||||
manual_path = project_path / "doctor" / "manual-note.md"
|
||||
await processor.write_file(manual_path, manual_markdown)
|
||||
console.print("[green]OK[/green] Manual file written")
|
||||
|
||||
sync_data = await project_client.sync(
|
||||
project_id, force_full=False, run_in_background=False
|
||||
)
|
||||
sync_report = SyncReportResponse.model_validate(sync_data)
|
||||
if sync_report.total == 0:
|
||||
raise ValueError("Sync did not detect any changes")
|
||||
|
||||
console.print("[green]OK[/green] Sync indexed manual file")
|
||||
|
||||
search_client = SearchClient(client, project_id)
|
||||
search_query = SearchQuery(title=manual_note_title)
|
||||
search_results = await search_client.search(
|
||||
search_query.model_dump(), page=1, page_size=5
|
||||
)
|
||||
if not any(result.title == manual_note_title for result in search_results.results):
|
||||
raise ValueError("Manual note not found in search index")
|
||||
|
||||
console.print("[green]OK[/green] Search confirmed manual file")
|
||||
|
||||
status_report = await project_client.get_status(project_id)
|
||||
if status_report.total != 0:
|
||||
raise ValueError("Project status not clean after sync")
|
||||
|
||||
console.print("[green]OK[/green] Status clean after sync")
|
||||
|
||||
finally:
|
||||
if project_id:
|
||||
await project_client.delete_project(project_id)
|
||||
|
||||
console.print("[green]Doctor checks passed.[/green]")
|
||||
|
||||
|
||||
@app.command()
|
||||
def doctor(
|
||||
local: bool = typer.Option(
|
||||
False, "--local", help="Force local API routing (ignore cloud mode)"
|
||||
),
|
||||
cloud: bool = typer.Option(False, "--cloud", help="Force cloud API routing"),
|
||||
) -> None:
|
||||
"""Run local consistency checks to verify file/database sync."""
|
||||
try:
|
||||
validate_routing_flags(local, cloud)
|
||||
# Doctor runs local filesystem checks — always default to local routing
|
||||
if not local and not cloud:
|
||||
local = True
|
||||
with force_routing(local=local, cloud=cloud):
|
||||
run_with_cleanup(run_doctor())
|
||||
except (ToolError, ValueError) as e:
|
||||
console.print(f"[red]Doctor failed: {e}[/red]")
|
||||
raise typer.Exit(code=1)
|
||||
except Exception as e:
|
||||
logger.error(f"Doctor failed: {e}")
|
||||
typer.echo(f"Doctor failed: {e}", err=True)
|
||||
raise typer.Exit(code=1) # pragma: no cover
|
||||
@@ -183,10 +183,10 @@ def format(
|
||||
By default, formats all .md, .json, and .canvas files in the current project.
|
||||
|
||||
Examples:
|
||||
basic-memory format # Format all files in current project
|
||||
basic-memory format --project research # Format files in specific project
|
||||
basic-memory format notes/meeting.md # Format a specific file
|
||||
basic-memory format notes/ # Format all files in directory
|
||||
bm format # Format all files in current project
|
||||
bm format --project research # Format files in specific project
|
||||
bm format notes/meeting.md # Format a specific file
|
||||
bm format notes/ # Format all files in directory
|
||||
"""
|
||||
try:
|
||||
run_with_cleanup(run_format(path, project))
|
||||
|
||||
@@ -44,7 +44,7 @@ def import_chatgpt(
|
||||
2. Convert them to linear markdown conversations
|
||||
3. Save as clean, readable markdown files
|
||||
|
||||
After importing, run 'basic-memory sync' to index the new files.
|
||||
After importing, run 'bm reindex --search' to index the new files.
|
||||
"""
|
||||
|
||||
try:
|
||||
@@ -60,7 +60,9 @@ def import_chatgpt(
|
||||
console.print(f"\nImporting chats from {conversations_json}...writing to {base_path}")
|
||||
|
||||
# Create importer and run import
|
||||
importer = ChatGPTImporter(config.home, markdown_processor, file_service)
|
||||
importer = ChatGPTImporter(
|
||||
config.home, markdown_processor, file_service, project_name=config.name
|
||||
)
|
||||
with conversations_json.open("r", encoding="utf-8") as file:
|
||||
json_data = json.load(file)
|
||||
result = run_with_cleanup(importer.import_data(json_data, folder))
|
||||
@@ -79,7 +81,7 @@ def import_chatgpt(
|
||||
)
|
||||
)
|
||||
|
||||
console.print("\nRun 'basic-memory sync' to index the new files.")
|
||||
console.print("\nRun 'bm reindex --search' to index the new files.")
|
||||
|
||||
except Exception as e:
|
||||
logger.error("Import failed")
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user