mirror of
https://github.com/basicmachines-co/basic-memory
synced 2026-06-21 13:47:35 +00:00
Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 46b372c3e1 |
+68
-14
@@ -92,7 +92,7 @@ jobs:
|
||||
run: |
|
||||
uv pip install -e ".[dev]"
|
||||
|
||||
- name: Run tests
|
||||
- name: Run tests (SQLite Unit)
|
||||
run: |
|
||||
just test-unit-sqlite
|
||||
|
||||
@@ -139,7 +139,7 @@ jobs:
|
||||
run: |
|
||||
uv pip install -e ".[dev]"
|
||||
|
||||
- name: Run tests
|
||||
- name: Run tests (SQLite Integration)
|
||||
run: |
|
||||
just test-int-sqlite
|
||||
|
||||
@@ -150,10 +150,7 @@ jobs:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- python-version: "3.12"
|
||||
- python-version: "3.13"
|
||||
- python-version: "3.14"
|
||||
python-version: [ "3.12", "3.13", "3.14" ]
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
# Note: No services section needed - testcontainers handles Postgres in Docker
|
||||
@@ -183,7 +180,7 @@ jobs:
|
||||
run: |
|
||||
uv pip install -e ".[dev]"
|
||||
|
||||
- name: Run tests
|
||||
- name: Run tests (Postgres Unit)
|
||||
run: |
|
||||
just test-unit-postgres
|
||||
|
||||
@@ -194,10 +191,7 @@ jobs:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- python-version: "3.12"
|
||||
- python-version: "3.13"
|
||||
- python-version: "3.14"
|
||||
python-version: [ "3.12", "3.13", "3.14" ]
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
# Note: No services section needed - testcontainers handles Postgres in Docker
|
||||
@@ -227,7 +221,7 @@ jobs:
|
||||
run: |
|
||||
uv pip install -e ".[dev]"
|
||||
|
||||
- name: Run tests
|
||||
- name: Run tests (Postgres Integration)
|
||||
run: |
|
||||
just test-int-postgres
|
||||
|
||||
@@ -260,8 +254,68 @@ jobs:
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
uv pip install -e ".[dev]"
|
||||
uv pip install -e ".[dev,semantic]"
|
||||
|
||||
- name: Run tests
|
||||
- name: Run tests (Semantic)
|
||||
run: |
|
||||
just test-semantic
|
||||
|
||||
coverage:
|
||||
name: Coverage Summary (combined, Python 3.12)
|
||||
timeout-minutes: 60
|
||||
needs:
|
||||
- static-checks
|
||||
- test-sqlite-unit
|
||||
- test-sqlite-integration
|
||||
- test-postgres-unit
|
||||
- test-postgres-integration
|
||||
- test-semantic
|
||||
if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: true
|
||||
|
||||
- name: Set up Python 3.12
|
||||
uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: "3.12"
|
||||
cache: "pip"
|
||||
|
||||
- name: Install uv
|
||||
run: |
|
||||
pip install uv
|
||||
|
||||
- uses: extractions/setup-just@v3
|
||||
|
||||
- name: Create virtual env
|
||||
run: |
|
||||
uv venv
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
uv pip install -e ".[dev,semantic]"
|
||||
|
||||
- name: Run combined coverage (SQLite + Postgres)
|
||||
run: |
|
||||
just coverage
|
||||
|
||||
- name: Add coverage report to job summary
|
||||
if: always()
|
||||
run: |
|
||||
{
|
||||
echo "## Coverage"
|
||||
echo ""
|
||||
echo '```'
|
||||
uv run coverage report -m
|
||||
echo '```'
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
- name: Upload HTML coverage report
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: htmlcov
|
||||
path: htmlcov/
|
||||
|
||||
@@ -31,7 +31,6 @@ See the [README.md](README.md) file for a project overview.
|
||||
- Run benchmarks: `pytest test-int/test_sync_performance_benchmark.py -v -m "benchmark and not slow"`
|
||||
- Lint: `just lint` or `ruff check . --fix`
|
||||
- Type check: `just typecheck` or `uv run pyright`
|
||||
- Type check (supplemental): `just typecheck-ty` or `uv run ty check src/`
|
||||
- Format: `just format` or `uv run ruff format .`
|
||||
- Run all code checks: `just check` (runs lint, format, typecheck, test)
|
||||
- Create db migration: `just migration "Your migration message"`
|
||||
|
||||
+6
-168
@@ -2,182 +2,20 @@
|
||||
|
||||
## Unreleased
|
||||
|
||||
## v0.19.2 (2026-03-09)
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- **#657**: Coerce string params to list/dict in MCP tools
|
||||
- MCP clients that serialize `list`/`dict` arguments as JSON strings no longer fail Pydantic validation
|
||||
- Adds `BeforeValidator` coercion to `search_notes` (`entity_types`, `note_types`, `tags`, `metadata_filters`), `write_note` (`metadata`), and `canvas` (`nodes`, `edges`)
|
||||
- **#655**: Handle SQLite and Windows semantic search regressions
|
||||
- Fix embedding status query for non-semantic SQLite databases
|
||||
- Windows-safe log file rotation with per-process log filenames
|
||||
- Robust `setup_logging` that handles all environments cleanly
|
||||
|
||||
## v0.19.1 (2026-03-08)
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- **#649**: Enforce strict entity resolution in destructive MCP tools (`edit_note`, `move_note`, `delete_note`)
|
||||
- Prevents fuzzy-match fallback from silently editing/moving/deleting the wrong note
|
||||
- DST-related timeframe validation fix (round instead of truncate days)
|
||||
|
||||
### Features
|
||||
|
||||
- **#648**: Add `insert_before_section` and `insert_after_section` edit operations
|
||||
- Add `GET /knowledge/graph` endpoint for full graph visualization
|
||||
|
||||
### Dependencies
|
||||
|
||||
- Bump authlib from 1.6.6 to 1.6.7
|
||||
|
||||
## v0.19.0 (2026-03-07)
|
||||
|
||||
### Highlights
|
||||
|
||||
- **Semantic vector search** for SQLite and Postgres with FastEmbed embeddings
|
||||
- **Schema system** for validating and inferring knowledge base structure
|
||||
- **Per-project cloud routing** with API key authentication
|
||||
- **Upgraded to FastMCP 3.0** with tool annotations
|
||||
- **CLI overhaul** with JSON output, workspace awareness, and project dashboard
|
||||
|
||||
### Features
|
||||
|
||||
- **#550**: Add semantic vector search for SQLite and Postgres
|
||||
- FastEmbed-based embeddings with automatic backfill
|
||||
- Hybrid search combining full-text and vector similarity
|
||||
- Score-based fusion replacing RRF for better ranking
|
||||
- `min_similarity` override for tuning search precision
|
||||
- Semantic dependencies are now default, with optional extras fallback
|
||||
|
||||
- **#549**: Schema system for Basic Memory
|
||||
- `schema_infer` — infer schema from existing notes
|
||||
- `schema_validate` — validate notes against a schema definition
|
||||
- `schema_diff` — compare schemas across projects
|
||||
- Frontmatter validation support (#597)
|
||||
- Read schema definitions from file instead of stale DB metadata (#635)
|
||||
|
||||
- **#555**: Per-project local/cloud routing with API key auth
|
||||
- Individual projects route through cloud while others stay local
|
||||
- `basic-memory cloud set-key` and `basic-memory project set-cloud/set-local`
|
||||
- Stdio MCP honors per-project cloud routing (#590)
|
||||
|
||||
- **#598**: Upgrade FastMCP 2.12.3 to 3.0.1 with tool annotations
|
||||
|
||||
- **#585**: Add JSON output mode for MCP tools (default text)
|
||||
- `--json` output for CLI commands for scripting and CI
|
||||
|
||||
- **#576**: Add workspace selection flow for MCP and CLI
|
||||
- Workspace-aware cloud project listing
|
||||
- CLI refactoring for workspace support
|
||||
|
||||
- **#544**: Project-prefixed permalinks and memory URL routing
|
||||
|
||||
- **#632**: Add overwrite guard to `write_note` tool
|
||||
|
||||
- **#614**: `edit_note` append/prepend auto-creates note if not found
|
||||
|
||||
- **#609**: Richer content context in search results
|
||||
- Return matched chunk text in search results (#601)
|
||||
- Improved content hit rate
|
||||
|
||||
- **#602**: Add `created_by` and `last_updated_by` user tracking to Entity
|
||||
|
||||
- **#600**: Rename `entity_type` to `note_type` across codebase
|
||||
|
||||
- **#574**: Add `display_name` and `is_private` to ProjectItem
|
||||
|
||||
- **#569**: Expose `external_id` in EntityResponse and link resolver
|
||||
|
||||
- **#567**: Isolate default SQLite DB by config dir
|
||||
|
||||
- **#560**: Enable `default_project_mode` by default
|
||||
|
||||
- **#559**: Add `basic-memory watch` CLI command
|
||||
|
||||
- **#546**: Add cloud discovery touchpoints to CLI and MCP
|
||||
|
||||
- **#572**: CLI analytics via Umami event collector
|
||||
|
||||
- Replace project info with htop-inspired dashboard
|
||||
|
||||
- Merge `search_by_metadata` into `search_notes` with optional query
|
||||
|
||||
- Add `--strip-frontmatter` to `basic-memory tool read-note`
|
||||
- Default behavior is unchanged: `content` still includes raw markdown with frontmatter.
|
||||
- With `--strip-frontmatter`, both text and JSON modes return body-only markdown content.
|
||||
- JSON output now includes an additive `frontmatter` field with parsed YAML metadata (or `null`
|
||||
when no valid opening frontmatter block exists).
|
||||
|
||||
- Add `destination_folder` parameter to `move_note` tool
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- **#644**: Fix default project resolution in cloud mode
|
||||
- ChatGPT search/fetch tools broken in cloud mode
|
||||
- `resolve_project_parameter` falls back to projects API
|
||||
|
||||
- **#638**: Restore API backward compatibility for v0.18.x clients
|
||||
|
||||
- **#637**: Create backup before config migration overwrites old format
|
||||
|
||||
- **#636**: `list_workspaces` bypasses factory pattern on cloud MCP server
|
||||
|
||||
- **#631**: `build_context` related_results schema validation failure
|
||||
|
||||
- **#613**: Reduce excessive log volume by demoting per-request noise to DEBUG
|
||||
|
||||
- **#612**: Handle quoted picoschema enum strings in YAML frontmatter
|
||||
|
||||
- **#607**: Guard against closed streams in promo and missing vector tables
|
||||
|
||||
- **#606**: Accept null for `expected_replacements` in `edit_note`
|
||||
|
||||
- **#595**: `recent_activity` dedup and pagination across MCP tools
|
||||
|
||||
- **#593**: Backend-specific distance-to-similarity conversion
|
||||
|
||||
- **#582**: Use LinkResolver fallback in `build_context` for flexible identifier matching
|
||||
|
||||
- **#577**: Replace RRF with score-based fusion in hybrid search
|
||||
|
||||
- **#575**: Remove hardcoded "main" default from `default_project`
|
||||
|
||||
- **#534**: Speed up `bm --version` startup
|
||||
|
||||
- Fix semantic embeddings not generated on fresh DB or upgrade
|
||||
|
||||
- Clarify `search_notes` parameter naming and fix `note_types` case sensitivity
|
||||
|
||||
- Parse `tag:` prefix at MCP tool level to avoid hybrid search failure
|
||||
|
||||
- Cap sqlite-vec knn k parameter at 4096 limit
|
||||
|
||||
- Parameterize SQL queries in search repository type filters
|
||||
|
||||
- Coerce list frontmatter values to strings for title and type fields
|
||||
|
||||
- Avoid `Post(**metadata)` crash when frontmatter contains 'content' or 'handler' keys
|
||||
|
||||
- Upgrade cryptography and python-multipart for security advisories
|
||||
|
||||
### Internal
|
||||
|
||||
- **#594**: Add `ty` as supplemental type checker
|
||||
- Batched vector sync orchestration across repositories
|
||||
- FastEmbed parallel guardrails and provider caching
|
||||
- Improved cloud CLI status and error messages
|
||||
- CI coverage and Postgres test fixes
|
||||
|
||||
## v0.18.5 (2026-02-13)
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- Strip NUL bytes from content before PostgreSQL search indexing
|
||||
([`ec9b2c4`](https://github.com/basicmachines-co/basic-memory/commit/ec9b2c4))
|
||||
|
||||
## v0.18.4 (2026-02-12)
|
||||
## v0.18.3 (2026-02-12)
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- Use global `--header` flag for Tigris consistency on all rclone transactions
|
||||
([`0eae0e1`](https://github.com/basicmachines-co/basic-memory/commit/0eae0e1))
|
||||
([`7fcf587`](https://github.com/basicmachines-co/basic-memory/commit/7fcf587))
|
||||
- `--header-download` / `--header-upload` only apply to GET/PUT requests, missing S3
|
||||
ListObjectsV2 calls that bisync issues first. Non-US users saw stale edge-cached metadata.
|
||||
- `--header` applies to ALL HTTP transactions (list, download, upload), fixing bisync for
|
||||
|
||||
@@ -11,9 +11,9 @@
|
||||
|
||||
- **Cross-device and multi-platform support is here.** Your knowledge graph now works on desktop, web, and mobile.
|
||||
- **Cloud is optional.** The local-first open-source workflow continues as always.
|
||||
- **OSS discount:** use code `BMFOSS` for 20% off for 3 months.
|
||||
- **OSS discount:** use code `{{OSS_DISCOUNT_CODE}}` for 20% off for 3 months.
|
||||
|
||||
[Sign up now →](https://basicmemory.com?utm_source=github&utm_medium=referral&utm_campaign=readme)
|
||||
[Sign up now →](https://basicmemory.com)
|
||||
|
||||
with a 7 day free trial
|
||||
|
||||
@@ -23,21 +23,8 @@ Basic Memory lets you build persistent knowledge through natural conversations w
|
||||
Claude, while keeping everything in simple Markdown files on your computer. It uses the Model Context Protocol (MCP) to
|
||||
enable any compatible LLM to read and write to your local knowledge base.
|
||||
|
||||
## What's New in v0.19.0
|
||||
|
||||
- **Semantic Vector Search** — find notes by meaning, not just keywords. Combines full-text and vector similarity for hybrid search with FastEmbed embeddings.
|
||||
- **Schema System** — infer, validate, and diff the structure of your knowledge base with `schema_infer`, `schema_validate`, and `schema_diff` tools.
|
||||
- **Per-Project Cloud Routing** — route individual projects through the cloud while others stay local, using API key authentication (`basic-memory project set-cloud`).
|
||||
- **FastMCP 3.0** — upgraded to FastMCP 3.0 with tool annotations for better client integration.
|
||||
- **CLI Overhaul** — JSON output mode (`--json`) for scripting, workspace-aware commands, and an htop-inspired project dashboard.
|
||||
- **Smarter Editing** — `edit_note` append/prepend auto-creates notes if they don't exist; `write_note` has an overwrite guard to prevent accidental data loss.
|
||||
- **Richer Search Results** — matched chunk text returned in search results for better context.
|
||||
|
||||
See the full [CHANGELOG](CHANGELOG.md) for details.
|
||||
|
||||
- Website: [basicmemory.com](https://basicmemory.com?utm_source=github&utm_medium=referral&utm_campaign=readme)
|
||||
- Documentation: [docs.basicmemory.com](https://docs.basicmemory.com?utm_source=github&utm_medium=referral&utm_campaign=readme)
|
||||
- Community: [Discord](https://discord.gg/tyvKNccgqN?utm_source=github&utm_medium=referral&utm_campaign=readme)
|
||||
- Website: https://basicmemory.com
|
||||
- Documentation: https://docs.basicmemory.com
|
||||
|
||||
## Pick up your conversation right where you left off
|
||||
|
||||
@@ -409,8 +396,7 @@ basic-memory project ls --name main --cloud
|
||||
|
||||
No-flag behavior defaults to local when no project context is present.
|
||||
|
||||
The local MCP server routes per transport: `--transport stdio` honors per-project routing
|
||||
(local or cloud), while `--transport streamable-http` and `--transport sse` always route locally.
|
||||
The local MCP server (`basic-memory mcp`) always uses local routing (including `--transport stdio`).
|
||||
|
||||
**CLI Note Editing (`tool edit-note`):**
|
||||
|
||||
@@ -451,7 +437,8 @@ list_directory(dir_name, depth) - Browse directory contents with filtering
|
||||
**Search & Discovery:**
|
||||
```
|
||||
search(query, page, page_size) - Search across your knowledge base
|
||||
search_notes(query, page, page_size, search_type, types, entity_types, after_date, metadata_filters, tags, status, project) - Search with filters (query is optional for filter-only searches)
|
||||
search_notes(query, page, page_size, search_type, types, entity_types, after_date, metadata_filters, tags, status, project) - Search with filters
|
||||
search_by_metadata(filters, limit, offset, project) - Structured frontmatter search
|
||||
```
|
||||
|
||||
**Project Management:**
|
||||
@@ -488,38 +475,13 @@ canvas(nodes, edges, title, folder) - Generate knowledge visualizations
|
||||
|
||||
## Futher info
|
||||
|
||||
See the [Documentation](https://docs.basicmemory.com?utm_source=github&utm_medium=referral&utm_campaign=readme) for more info, including:
|
||||
See the [Documentation](https://docs.basicmemory.com) for more info, including:
|
||||
|
||||
- [Complete User Guide](https://docs.basicmemory.com/user-guide/?utm_source=github&utm_medium=referral&utm_campaign=readme)
|
||||
- [CLI tools](https://docs.basicmemory.com/guides/cli-reference/?utm_source=github&utm_medium=referral&utm_campaign=readme)
|
||||
- [Cloud CLI and Sync](https://docs.basicmemory.com/guides/cloud-cli/?utm_source=github&utm_medium=referral&utm_campaign=readme)
|
||||
- [Managing multiple Projects](https://docs.basicmemory.com/guides/cli-reference/?utm_source=github&utm_medium=referral&utm_campaign=readme#project)
|
||||
- [Importing data from OpenAI/Claude Projects](https://docs.basicmemory.com/guides/cli-reference/?utm_source=github&utm_medium=referral&utm_campaign=readme#import)
|
||||
|
||||
## Telemetry
|
||||
|
||||
Basic Memory collects anonymous, minimal usage events to understand how the CLI-to-cloud conversion funnel performs. This helps us prioritize features and improve the product.
|
||||
|
||||
**What we collect:**
|
||||
- Cloud promo impressions (when the promo banner is shown)
|
||||
- Cloud login attempts and outcomes
|
||||
- Promo opt-out events
|
||||
|
||||
**What we do NOT collect:**
|
||||
- No file contents, note titles, or knowledge base data
|
||||
- No personally identifiable information (PII)
|
||||
- No IP address tracking or fingerprinting
|
||||
- No per-command or per-tool-call tracking
|
||||
|
||||
Events are sent to our [Umami Cloud](https://umami.is) instance, an open-source, privacy-focused analytics platform. Events are fire-and-forget on a background thread — analytics never blocks or slows the CLI.
|
||||
|
||||
**Opt out** by setting the environment variable:
|
||||
|
||||
```bash
|
||||
export BASIC_MEMORY_NO_PROMOS=1
|
||||
```
|
||||
|
||||
This disables both promo messages and all telemetry events.
|
||||
- [Complete User Guide](https://docs.basicmemory.com/user-guide/)
|
||||
- [CLI tools](https://docs.basicmemory.com/guides/cli-reference/)
|
||||
- [Cloud CLI and Sync](https://docs.basicmemory.com/guides/cloud-cli/)
|
||||
- [Managing multiple Projects](https://docs.basicmemory.com/guides/cli-reference/#project)
|
||||
- [Importing data from OpenAI/Claude Projects](https://docs.basicmemory.com/guides/cli-reference/#import)
|
||||
|
||||
## Logging
|
||||
|
||||
@@ -543,7 +505,6 @@ Basic Memory uses [Loguru](https://github.com/Delgan/loguru) for logging. The lo
|
||||
| `BASIC_MEMORY_FORCE_CLOUD` | `false` | When `true`, forces cloud API routing |
|
||||
| `BASIC_MEMORY_EXPLICIT_ROUTING` | `false` | When `true`, marks route selection as explicit (`--local`/`--cloud`) |
|
||||
| `BASIC_MEMORY_ENV` | `dev` | Set to `test` for test mode (stderr only) |
|
||||
| `BASIC_MEMORY_NO_PROMOS` | `false` | When `true`, disables cloud promo messages and telemetry |
|
||||
|
||||
### Examples
|
||||
|
||||
@@ -610,7 +571,6 @@ Tests use pytest markers for selective execution:
|
||||
just install # Install with dev dependencies
|
||||
just lint # Run linting checks
|
||||
just typecheck # Run type checking
|
||||
just typecheck-ty # Run ty type checking (incremental supplement to pyright)
|
||||
just format # Format code with ruff
|
||||
just fast-check # Fast local loop (fix/format/typecheck + testmon + smoke)
|
||||
just doctor # Local consistency check (temp config)
|
||||
@@ -618,11 +578,6 @@ just check # Run all quality checks
|
||||
just migration "msg" # Create database migration
|
||||
```
|
||||
|
||||
**Type Checking Strategy:**
|
||||
- `just typecheck` (Pyright) remains the primary, blocking type checker.
|
||||
- `just typecheck-ty` (Astral `ty`) is available as a supplemental checker while rules are adopted incrementally.
|
||||
- We recommend running both locally while reducing `ty` diagnostics over time.
|
||||
|
||||
**Local Consistency Check:**
|
||||
```bash
|
||||
basic-memory doctor # Verifies file <-> database sync in a temp project
|
||||
@@ -647,4 +602,4 @@ and submitting PRs.
|
||||
</picture>
|
||||
</a>
|
||||
|
||||
Built with ♥️ by [Basic Machines](https://basicmachines.co?utm_source=github&utm_medium=referral&utm_campaign=readme)
|
||||
Built with ♥️ by Basic Machines
|
||||
|
||||
@@ -10,7 +10,7 @@ This document is the canonical contract for local/cloud routing behavior in CLI,
|
||||
## Goals
|
||||
|
||||
1. Remove global `cloud_mode` from runtime/routing semantics.
|
||||
2. Keep MCP HTTP/SSE local-only; let stdio honor per-project routing.
|
||||
2. Keep MCP stdio local-only and predictable.
|
||||
3. Make CLI routing explicit and easy to reason about.
|
||||
4. Support projects that exist in both local and cloud without ambiguity.
|
||||
|
||||
@@ -45,14 +45,14 @@ When explicit routing is active, project mode does not override the selected rou
|
||||
"main": {
|
||||
"path": "/Users/me/basic-memory",
|
||||
"mode": "local",
|
||||
"local_sync_path": null,
|
||||
"cloud_sync_path": null,
|
||||
"bisync_initialized": false,
|
||||
"last_sync": null
|
||||
},
|
||||
"specs": {
|
||||
"path": "specs",
|
||||
"mode": "cloud",
|
||||
"local_sync_path": "/Users/me/dev/specs",
|
||||
"cloud_sync_path": "/Users/me/dev/specs",
|
||||
"bisync_initialized": true,
|
||||
"last_sync": "2026-02-06T17:36:38.544153"
|
||||
}
|
||||
@@ -79,25 +79,13 @@ When explicit routing is active, project mode does not override the selected rou
|
||||
- reports auth state (API key, OAuth token validity)
|
||||
- runs health checks only when credentials are available
|
||||
|
||||
## MCP Transport Routing
|
||||
## MCP Stdio Local Guarantee
|
||||
|
||||
### Stdio (default)
|
||||
`bm mcp --transport stdio` always routes locally.
|
||||
|
||||
`bm mcp --transport stdio` uses natural per-project routing.
|
||||
|
||||
- Local-mode projects route through the in-process ASGI transport.
|
||||
- Cloud-mode projects route to the cloud proxy with Bearer auth (API key).
|
||||
- No explicit routing env vars are injected by the CLI command.
|
||||
- Externally-set env vars are honored (e.g. `BASIC_MEMORY_FORCE_CLOUD=true` for cloud deployments).
|
||||
- Users who need all projects forced local can set `BASIC_MEMORY_FORCE_LOCAL=true` externally.
|
||||
|
||||
### HTTP and SSE Transports
|
||||
|
||||
`bm mcp --transport streamable-http` and `bm mcp --transport sse` always route locally.
|
||||
|
||||
These transports set explicit local routing (`BASIC_MEMORY_FORCE_LOCAL=true` and
|
||||
`BASIC_MEMORY_EXPLICIT_ROUTING=true`) before starting the server. This prevents cloud
|
||||
routing regardless of project mode, since HTTP/SSE serve as local API endpoints.
|
||||
The command sets explicit local routing (`BASIC_MEMORY_FORCE_LOCAL=true` and
|
||||
`BASIC_MEMORY_EXPLICIT_ROUTING=true`) before starting the server. This prevents cloud routing for stdio MCP,
|
||||
even if the selected project has `mode: cloud`.
|
||||
|
||||
## Project List UX for Dual Presence
|
||||
|
||||
@@ -142,6 +130,6 @@ Runtime mode is no longer a cloud/local routing switch for local app flows.
|
||||
3. `--local/--cloud` always override per-project mode for that command.
|
||||
4. No-project + no-flags commands route local by default.
|
||||
5. `bm cloud login/logout` do not toggle routing behavior.
|
||||
6. `bm mcp` stdio routes per-project mode; HTTP/SSE remain local-forced.
|
||||
6. `bm mcp` remains local-only in stdio mode.
|
||||
7. `bm project list` communicates dual local/cloud presence without ambiguity.
|
||||
8. `bm project ls` output identifies route target explicitly.
|
||||
|
||||
@@ -427,8 +427,6 @@ await write_note(
|
||||
)
|
||||
```
|
||||
|
||||
> **Important**: `write_note` errors if the note already exists. Use `edit_note` for incremental changes, or pass `overwrite=True` to replace.
|
||||
|
||||
**Well-structured note**:
|
||||
|
||||
```python
|
||||
@@ -762,9 +760,6 @@ notes = await read_note(
|
||||
identifier="memory://specs/*",
|
||||
project="main"
|
||||
)
|
||||
|
||||
# Cross-project URL (auto-routes to the correct project)
|
||||
note = await read_note(identifier="memory://research/specs/api-design")
|
||||
```
|
||||
|
||||
```python
|
||||
@@ -1065,19 +1060,16 @@ results = await search_notes(
|
||||
project="main"
|
||||
)
|
||||
|
||||
# Metadata-only search (no query needed)
|
||||
results = await search_notes(
|
||||
metadata_filters={"type": "spec", "status": "in-progress"},
|
||||
# Metadata-only search
|
||||
results = await search_by_metadata(
|
||||
filters={"type": "spec", "status": "in-progress"},
|
||||
project="main"
|
||||
)
|
||||
```
|
||||
|
||||
### Search Types
|
||||
|
||||
Available types: `"text"`, `"title"`, `"permalink"`, `"vector"`/`"semantic"`, `"hybrid"`.
|
||||
Default is `"hybrid"` when semantic search is enabled, `"text"` otherwise.
|
||||
|
||||
**Text search**:
|
||||
**Text search (default)**:
|
||||
|
||||
```python
|
||||
# Full-text search across all content
|
||||
@@ -1088,52 +1080,17 @@ results = await search_notes(
|
||||
)
|
||||
```
|
||||
|
||||
**Title and permalink search**:
|
||||
|
||||
```python
|
||||
# Search by title only
|
||||
results = await search_notes(query="API Design", search_type="title", project="main")
|
||||
|
||||
# Search by permalink
|
||||
results = await search_notes(query="specs/api-design", search_type="permalink", project="main")
|
||||
```
|
||||
|
||||
**Semantic/vector search**:
|
||||
**Semantic search**:
|
||||
|
||||
```python
|
||||
# Semantic/vector search (if enabled)
|
||||
results = await search_notes(
|
||||
query="user login security",
|
||||
search_type="semantic", # or "vector"
|
||||
project="main"
|
||||
)
|
||||
|
||||
# Override similarity threshold
|
||||
results = await search_notes(
|
||||
query="user login security",
|
||||
search_type="semantic",
|
||||
min_similarity=0.5,
|
||||
project="main"
|
||||
)
|
||||
```
|
||||
|
||||
**Hybrid search** (combines text + semantic):
|
||||
|
||||
```python
|
||||
results = await search_notes(
|
||||
query="authentication best practices",
|
||||
search_type="hybrid",
|
||||
project="main"
|
||||
)
|
||||
```
|
||||
|
||||
**Tag shorthand in query**:
|
||||
|
||||
```python
|
||||
# Use tag: prefix as shorthand
|
||||
results = await search_notes(query="tag:security", project="main")
|
||||
```
|
||||
|
||||
### Search Response
|
||||
|
||||
**Result structure**:
|
||||
@@ -2204,31 +2161,6 @@ active_project = projects[0]["name"]
|
||||
results = await search_notes(query="test", project=active_project)
|
||||
```
|
||||
|
||||
### Note Already Exists
|
||||
|
||||
**Error**: `write_note` called for a note that already exists
|
||||
|
||||
**Solution**:
|
||||
|
||||
```python
|
||||
# Preferred: use edit_note for incremental updates
|
||||
await edit_note(
|
||||
identifier="Existing Topic",
|
||||
operation="append",
|
||||
content="\n- [update] new information",
|
||||
project="main"
|
||||
)
|
||||
|
||||
# Alternative: replace the entire note
|
||||
await write_note(
|
||||
title="Existing Topic",
|
||||
content="# Existing Topic\n...",
|
||||
folder="notes",
|
||||
overwrite=True,
|
||||
project="main"
|
||||
)
|
||||
```
|
||||
|
||||
### Entity Not Found
|
||||
|
||||
**Error**: Note doesn't exist
|
||||
@@ -2784,15 +2716,14 @@ await write_note(
|
||||
|
||||
### Content Management
|
||||
|
||||
**write_note(title, content, folder, tags, note_type, overwrite, project)**
|
||||
- Create new markdown notes (errors if note already exists unless overwrite=True)
|
||||
**write_note(title, content, folder, tags, note_type, project)**
|
||||
- Create or update markdown notes
|
||||
- Parameters:
|
||||
- `title` (required): Note title
|
||||
- `content` (required): Markdown content
|
||||
- `folder` (required): Destination folder
|
||||
- `tags` (optional): List of tags
|
||||
- `note_type` (optional): Type of note (stored in frontmatter). Can be "note", "person", "meeting", "guide", etc.
|
||||
- `overwrite` (optional): Set to True to replace an existing note (default: error if exists)
|
||||
- `project` (required unless default_project_mode): Target project
|
||||
- Returns: Created/updated entity with permalink
|
||||
- Example:
|
||||
@@ -2959,20 +2890,19 @@ contents = await list_directory(
|
||||
|
||||
### Search & Discovery
|
||||
|
||||
**search_notes(query, page, page_size, search_type, types, entity_types, after_date, metadata_filters, tags, status, min_similarity, project)**
|
||||
**search_notes(query, page, page_size, search_type, types, entity_types, after_date, metadata_filters, tags, status, project)**
|
||||
- Search across knowledge base
|
||||
- Parameters:
|
||||
- `query` (optional): Search query (not required for filter-only searches)
|
||||
- `query` (required): Search query
|
||||
- `page` (optional): Page number (default: 1)
|
||||
- `page_size` (optional): Results per page (default: 10)
|
||||
- `search_type` (optional): "text", "title", "permalink", "vector"/"semantic", "hybrid" (default: "hybrid" when semantic enabled, "text" otherwise)
|
||||
- `search_type` (optional): "text" or "semantic"
|
||||
- `types` (optional): Entity type filter
|
||||
- `entity_types` (optional): Observation category filter
|
||||
- `after_date` (optional): Date filter (ISO format)
|
||||
- `metadata_filters` (optional): Structured frontmatter filters (dict, supports `$in`, `$gt`, `$gte`, `$lt`, `$lte`, `$between` operators)
|
||||
- `tags` (optional): Frontmatter tags filter (list); also available via `tag:` query shorthand
|
||||
- `metadata_filters` (optional): Structured frontmatter filters (dict)
|
||||
- `tags` (optional): Frontmatter tags filter (list)
|
||||
- `status` (optional): Frontmatter status filter (string)
|
||||
- `min_similarity` (optional): Override similarity threshold for vector/hybrid search
|
||||
- `project` (required unless default_project_mode): Target project
|
||||
- Returns: Matching entities with scores
|
||||
- Example:
|
||||
@@ -2985,11 +2915,18 @@ results = await search_notes(
|
||||
)
|
||||
```
|
||||
|
||||
**Metadata-only search (via search_notes)**
|
||||
- Use `search_notes` with `metadata_filters` and no `query` for metadata-only searches:
|
||||
**search_by_metadata(filters, limit, offset, project)**
|
||||
- Metadata-only search using structured frontmatter
|
||||
- Parameters:
|
||||
- `filters` (required): Dict of field -> value (supports $in, $gt/$gte/$lt/$lte, $between)
|
||||
- `limit` (optional): Max results (default: 20)
|
||||
- `offset` (optional): Pagination offset (default: 0)
|
||||
- `project` (required unless default_project_mode): Target project
|
||||
- Returns: Matching entities
|
||||
- Example:
|
||||
```python
|
||||
results = await search_notes(
|
||||
metadata_filters={"type": "spec", "status": "in-progress"},
|
||||
results = await search_by_metadata(
|
||||
filters={"type": "spec", "status": "in-progress"},
|
||||
project="main"
|
||||
)
|
||||
```
|
||||
@@ -3041,15 +2978,6 @@ await delete_project(project_name="old-project")
|
||||
status = await sync_status(project="main")
|
||||
```
|
||||
|
||||
**list_workspaces()**
|
||||
- List available workspaces (cloud)
|
||||
- Parameters: None
|
||||
- Returns: List of workspaces with metadata
|
||||
- Example:
|
||||
```python
|
||||
workspaces = await list_workspaces()
|
||||
```
|
||||
|
||||
### Visualization
|
||||
|
||||
**canvas(nodes, edges, title, folder, project)**
|
||||
@@ -3318,8 +3246,8 @@ await edit_note(
|
||||
project="main"
|
||||
)
|
||||
|
||||
# When full rewrite is needed, use overwrite=True
|
||||
await write_note(title="Note", content="...", folder="notes", overwrite=True)
|
||||
# Avoid: Complete rewrite
|
||||
# (unless necessary for major restructuring)
|
||||
```
|
||||
|
||||
### 14. Tagging Strategy
|
||||
|
||||
+1
-1
@@ -120,7 +120,7 @@ bm project sync-setup research ~/Documents/research
|
||||
|
||||
When you add a project with `--local-path`:
|
||||
1. Project created on cloud at `/app/data/research`
|
||||
2. Local path stored in config for that project (`local_sync_path`)
|
||||
2. Local path stored in config for that project (`cloud_sync_path`)
|
||||
3. Local directory created if it doesn't exist
|
||||
4. Bisync state directory created at `~/.basic-memory/bisync-state/research/`
|
||||
|
||||
|
||||
@@ -1,260 +0,0 @@
|
||||
# Metadata Search Reference
|
||||
|
||||
Basic Memory automatically indexes custom frontmatter fields so you can query them with structured filters. Any YAML key in a note's frontmatter beyond the standard set (`title`, `type`, `tags`, `permalink`, `schema`) is stored as `entity_metadata` and becomes searchable.
|
||||
|
||||
## Querying with `search_notes`
|
||||
|
||||
`search_notes` is the single search tool for all queries — text, metadata filters, or both. The `query` parameter is optional, so you can use metadata filters alone without passing an empty string.
|
||||
|
||||
## Filter Syntax
|
||||
|
||||
Filters are a JSON dictionary where each key targets a frontmatter field and the value specifies the match condition. Multiple keys combine with **AND** logic — every filter must match.
|
||||
|
||||
### Equality
|
||||
|
||||
Match a single value exactly.
|
||||
|
||||
```json
|
||||
{"status": "active"}
|
||||
```
|
||||
|
||||
Finds notes whose frontmatter contains `status: active`.
|
||||
|
||||
### Array Contains (all)
|
||||
|
||||
Pass a list to require **all** listed values to be present in the field.
|
||||
|
||||
```json
|
||||
{"tags": ["security", "oauth"]}
|
||||
```
|
||||
|
||||
Finds notes tagged with both `security` and `oauth`.
|
||||
|
||||
### `$in` (any of)
|
||||
|
||||
Match if the field equals **any** value in the list.
|
||||
|
||||
```json
|
||||
{"priority": {"$in": ["high", "critical"]}}
|
||||
```
|
||||
|
||||
### `$gt`, `$gte`, `$lt`, `$lte`
|
||||
|
||||
Numeric and text comparisons. Numeric values use numeric comparison; strings use lexicographic comparison.
|
||||
|
||||
```json
|
||||
{"confidence": {"$gt": 0.7}}
|
||||
{"score": {"$lte": 100}}
|
||||
```
|
||||
|
||||
### `$between`
|
||||
|
||||
Range filter (inclusive). Takes a `[min, max]` pair.
|
||||
|
||||
```json
|
||||
{"score": {"$between": [0.3, 0.8]}}
|
||||
```
|
||||
|
||||
### Nested Access (dot notation)
|
||||
|
||||
Access nested frontmatter values using dots.
|
||||
|
||||
```json
|
||||
{"schema.version": "2"}
|
||||
```
|
||||
|
||||
This queries the `version` key inside a `schema` object in frontmatter.
|
||||
|
||||
### Summary Table
|
||||
|
||||
| Operator | Syntax | Example |
|
||||
|----------|--------|---------|
|
||||
| Equality | `{"field": "value"}` | `{"status": "active"}` |
|
||||
| Array contains (all) | `{"field": ["a", "b"]}` | `{"tags": ["security", "oauth"]}` |
|
||||
| `$in` (any of) | `{"field": {"$in": [...]}}` | `{"priority": {"$in": ["high", "critical"]}}` |
|
||||
| `$gt` / `$gte` | `{"field": {"$gt": N}}` | `{"confidence": {"$gt": 0.7}}` |
|
||||
| `$lt` / `$lte` | `{"field": {"$lt": N}}` | `{"score": {"$lt": 0.5}}` |
|
||||
| `$between` | `{"field": {"$between": [min, max]}}` | `{"score": {"$between": [0.3, 0.8]}}` |
|
||||
| Nested access | `{"a.b": "value"}` | `{"schema.version": "2"}` |
|
||||
|
||||
**Key rules:**
|
||||
- Filter keys must match `[A-Za-z0-9_-]+` (dots separate nesting levels).
|
||||
- Each operator dict must contain exactly one operator.
|
||||
- `$in` and array-contains require non-empty lists.
|
||||
- `$between` requires exactly two values `[min, max]`.
|
||||
|
||||
## MCP Tool — `search_notes`
|
||||
|
||||
`search_notes` is the single search tool for text queries, metadata filters, or both. The `query` parameter is optional.
|
||||
|
||||
**Relevant parameters:**
|
||||
|
||||
| Parameter | Type | Description |
|
||||
|-----------|------|-------------|
|
||||
| `query` | string (optional) | Text search query. Omit for filter-only searches. |
|
||||
| `metadata_filters` | dict | Structured filter dict (see syntax above) |
|
||||
| `tags` | list[str] | Convenience shorthand — merged into `metadata_filters["tags"]` |
|
||||
| `status` | string | Convenience shorthand — merged into `metadata_filters["status"]` |
|
||||
|
||||
**Merging rules:** `tags` and `status` are convenience shortcuts. They are merged into `metadata_filters` using `setdefault` — if the same key already exists in `metadata_filters`, the explicit filter wins.
|
||||
|
||||
**Examples:**
|
||||
|
||||
```python
|
||||
# Text search filtered by metadata
|
||||
await search_notes("authentication", metadata_filters={"status": "draft"})
|
||||
|
||||
# Filter-only search (no query needed)
|
||||
await search_notes(metadata_filters={"type": "spec"})
|
||||
|
||||
# Combine text, tags shortcut, and metadata
|
||||
await search_notes(
|
||||
"oauth flow",
|
||||
tags=["security"],
|
||||
metadata_filters={"confidence": {"$gt": 0.7}},
|
||||
)
|
||||
|
||||
# Convenience shortcuts
|
||||
await search_notes("planning", status="active")
|
||||
await search_notes(tags=["tier1", "alpha"])
|
||||
```
|
||||
|
||||
## Tag Search Shortcuts
|
||||
|
||||
The `tag:` prefix in a search query is a shorthand for tag-based metadata filtering. When `search_notes` receives a query starting with `tag:`, it converts the query into a `tags` filter and clears the text query.
|
||||
|
||||
```python
|
||||
# These are equivalent:
|
||||
await search_notes("tag:tier1")
|
||||
await search_notes("", tags=["tier1"])
|
||||
|
||||
# Multiple tags (comma or space separated) — all must be present:
|
||||
await search_notes("tag:tier1,alpha")
|
||||
await search_notes("tag:tier1 alpha")
|
||||
```
|
||||
|
||||
## CLI Access
|
||||
|
||||
The `bm tool search-notes` command exposes metadata filtering via `--meta` and `--filter` flags.
|
||||
|
||||
### `--meta` — simple key=value filters
|
||||
|
||||
Repeatable flag for equality filters on frontmatter fields.
|
||||
|
||||
```bash
|
||||
# Single filter
|
||||
bm tool search-notes "my query" --meta status=draft
|
||||
|
||||
# Multiple filters (AND logic)
|
||||
bm tool search-notes "" --meta status=active --meta priority=high
|
||||
```
|
||||
|
||||
### `--filter` — advanced JSON filters
|
||||
|
||||
Pass a full JSON filter dictionary for operator-based queries.
|
||||
|
||||
```bash
|
||||
# Range filter
|
||||
bm tool search-notes "" --filter '{"score": {"$between": [0.3, 0.8]}}'
|
||||
|
||||
# $in filter
|
||||
bm tool search-notes "" --filter '{"priority": {"$in": ["high", "critical"]}}'
|
||||
```
|
||||
|
||||
### `--tag` and `--status` — convenience shortcuts
|
||||
|
||||
```bash
|
||||
bm tool search-notes "query" --tag security --tag oauth
|
||||
bm tool search-notes "" --status draft
|
||||
```
|
||||
|
||||
### Combined example
|
||||
|
||||
```bash
|
||||
bm tool search-notes "authentication" --tag security --meta status=draft --type spec
|
||||
```
|
||||
|
||||
## Practical Examples
|
||||
|
||||
### Example notes with custom frontmatter
|
||||
|
||||
**`specs/auth-design.md`:**
|
||||
|
||||
```markdown
|
||||
---
|
||||
title: Auth Design
|
||||
type: spec
|
||||
tags: [security, oauth]
|
||||
status: in-progress
|
||||
priority: high
|
||||
confidence: 0.85
|
||||
---
|
||||
|
||||
# Auth Design
|
||||
|
||||
## Observations
|
||||
- [decision] Use OAuth 2.1 with PKCE for all client types #security
|
||||
- [requirement] Token refresh must be transparent to the user
|
||||
|
||||
## Relations
|
||||
- implements [[Security Requirements]]
|
||||
```
|
||||
|
||||
**`specs/search-redesign.md`:**
|
||||
|
||||
```markdown
|
||||
---
|
||||
title: Search Redesign
|
||||
type: spec
|
||||
tags: [search, performance]
|
||||
status: draft
|
||||
priority: medium
|
||||
confidence: 0.6
|
||||
---
|
||||
|
||||
# Search Redesign
|
||||
|
||||
## Observations
|
||||
- [goal] Sub-100ms search response times #performance
|
||||
- [approach] Hybrid FTS + vector retrieval
|
||||
|
||||
## Relations
|
||||
- depends_on [[Database Schema]]
|
||||
```
|
||||
|
||||
### Queries that find them
|
||||
|
||||
```python
|
||||
# Find all in-progress specs
|
||||
await search_notes(metadata_filters={"status": "in-progress", "type": "spec"})
|
||||
# → Auth Design
|
||||
|
||||
# Find high-confidence specs
|
||||
await search_notes(metadata_filters={"confidence": {"$gt": 0.7}})
|
||||
# → Auth Design (confidence: 0.85)
|
||||
|
||||
# Find specs with priority high or medium
|
||||
await search_notes(metadata_filters={"priority": {"$in": ["high", "medium"]}})
|
||||
# → Auth Design, Search Redesign
|
||||
|
||||
# Find specs in a confidence range
|
||||
await search_notes(metadata_filters={"confidence": {"$between": [0.5, 0.9]}})
|
||||
# → Auth Design (0.85), Search Redesign (0.6)
|
||||
|
||||
# Find notes tagged with security
|
||||
await search_notes("tag:security")
|
||||
# → Auth Design
|
||||
|
||||
# Combined: text search + metadata filter
|
||||
await search_notes("OAuth", metadata_filters={"status": "in-progress"})
|
||||
# → Auth Design
|
||||
```
|
||||
|
||||
### CLI equivalents
|
||||
|
||||
```bash
|
||||
bm tool search-notes "" --meta status=in-progress --type spec
|
||||
bm tool search-notes "" --filter '{"confidence": {"$gt": 0.7}}'
|
||||
bm tool search-notes "OAuth" --meta status=in-progress
|
||||
bm tool search-notes --tag security
|
||||
```
|
||||
@@ -79,7 +79,7 @@ These are the most important post-`v0.18.0` feature modules currently under-cove
|
||||
### Acceptance criteria
|
||||
|
||||
- `search_type=text|vector|hybrid` returns expected ranked results on canonical semantic corpus.
|
||||
- Missing semantic dependencies fail fast with actionable install guidance.
|
||||
- Missing semantic extras fail fast with actionable install guidance.
|
||||
- Reindex and provider/model changes produce valid vectors without dimension mismatch.
|
||||
- SQLite and Postgres produce equivalent behavior for semantic modes on the same dataset.
|
||||
- Generated-column migration path is valid on SQLite environments in use.
|
||||
|
||||
@@ -1,318 +0,0 @@
|
||||
# v0.19.0 Release Notes
|
||||
|
||||
## Overview
|
||||
|
||||
v0.19.0 is a major release that introduces semantic vector search, a schema validation system,
|
||||
project-prefixed permalinks, per-project cloud routing, and a significant upgrade to FastMCP 3.0.
|
||||
It includes 90+ commits since v0.18.0 spanning new features, architectural improvements, and
|
||||
stability fixes across both SQLite and Postgres backends.
|
||||
|
||||
---
|
||||
|
||||
## Major Features
|
||||
|
||||
### Semantic Vector Search
|
||||
|
||||
Full vector and hybrid search for SQLite (via sqlite-vec) and Postgres (via pgvector).
|
||||
|
||||
- **Hybrid search mode** combines full-text search (FTS) with vector similarity for best results
|
||||
- **Score-based fusion** replaces RRF for hybrid ranking — `max(vec, fts) + 0.3 * min(vec, fts)` preserves dominant signals and rewards dual-source agreement (#577)
|
||||
- **Default search mode** is now `hybrid` when semantic search is enabled, `text` when disabled
|
||||
- Embedding providers: FastEmbed (local, default) or OpenAI API
|
||||
- Configurable similarity threshold via `semantic_min_similarity` (default 0.55)
|
||||
- Per-query `min_similarity` override on `search_notes` tool
|
||||
- Auto-backfill: existing entities get embeddings generated on first startup
|
||||
- Backend-specific distance-to-similarity conversion (cosine for SQLite, inner product for Postgres)
|
||||
- FTS fallback: if semantic dependencies are missing, search gracefully degrades to text-only
|
||||
- sqlite-vec knn `k` parameter capped at 4096 to prevent backend errors
|
||||
|
||||
**Configuration:**
|
||||
```json
|
||||
{
|
||||
"semantic_search_enabled": true,
|
||||
"semantic_embedding_provider": "fastembed",
|
||||
"semantic_embedding_model": "bge-small-en-v1.5",
|
||||
"semantic_min_similarity": 0.55
|
||||
}
|
||||
```
|
||||
|
||||
**Usage:**
|
||||
```
|
||||
search_notes("machine learning concepts", search_type="hybrid")
|
||||
search_notes("similar to my notes on coffee", search_type="vector")
|
||||
search_notes("exact phrase match", search_type="text")
|
||||
search_notes("broad search", min_similarity=0.3) # lower threshold for more results
|
||||
```
|
||||
|
||||
### Schema System
|
||||
|
||||
Validate note structure against user-defined schemas with frontmatter-based rules.
|
||||
|
||||
- Define schemas as YAML in note frontmatter with field types, required fields, and constraints
|
||||
- Frontmatter validation during sync — malformed notes get clear error messages
|
||||
- Schema inference from existing notes to bootstrap schemas from your content
|
||||
- Schema diff to compare two schemas and see changes
|
||||
- Available via MCP tools and CLI
|
||||
|
||||
### Project-Prefixed Permalinks
|
||||
|
||||
Permalinks now include the project name for unambiguous cross-project references.
|
||||
|
||||
- Memory URLs like `memory://project-name/folder/note` route to the correct project
|
||||
- Existing non-prefixed permalinks continue to work (backwards compatible)
|
||||
- Controlled by `permalinks_include_project` config (default: true)
|
||||
- `build_context` and `search_notes` auto-detect project from URL prefix
|
||||
|
||||
### Per-Project Cloud Routing
|
||||
|
||||
Individual projects can be routed through the cloud while others stay local.
|
||||
|
||||
- Set a project to cloud mode: `bm project set-cloud research`
|
||||
- Revert to local: `bm project set-local research`
|
||||
- Uses API key authentication: `bm cloud set-key bmc_abc123...`
|
||||
- MCP tools automatically route based on each project's mode
|
||||
- Local MCP server (`bm mcp`) still uses local routing for all projects by default
|
||||
- `--local` and `--cloud` CLI flags override per-command
|
||||
|
||||
### Workspace Selection
|
||||
|
||||
Cloud projects can target specific workspaces for multi-tenant environments.
|
||||
|
||||
- `workspace` parameter on MCP tools for explicit workspace targeting
|
||||
- CLI workspace-aware project listing with `bm project list`
|
||||
- Spinner feedback while fetching cloud projects
|
||||
|
||||
---
|
||||
|
||||
## New Tools and Capabilities
|
||||
|
||||
### Dashboard (`bm project info`)
|
||||
|
||||
`bm project info` now displays an htop-inspired compact dashboard with:
|
||||
|
||||
- Horizontal bar charts for note types (top 5)
|
||||
- Embedding coverage bar with Unicode block characters
|
||||
- Colored status dots for at-a-glance health
|
||||
- `EmbeddingStatus` schema and `get_embedding_status()` service method for programmatic access
|
||||
|
||||
### Unified Metadata Search
|
||||
|
||||
`search_by_metadata` has been merged into `search_notes` — one tool for all searches.
|
||||
`query` is now optional, so you can search purely by frontmatter metadata.
|
||||
|
||||
```
|
||||
search_notes(metadata_filters={"status": "in-progress"})
|
||||
search_notes(metadata_filters={"tags": ["security", "oauth"]})
|
||||
search_notes(metadata_filters={"priority": {"$in": ["high", "critical"]}})
|
||||
search_notes(metadata_filters={"schema.confidence": {"$gt": 0.7}})
|
||||
search_notes(tags=["security"]) # convenience shorthand
|
||||
search_notes(status="draft") # convenience shorthand
|
||||
```
|
||||
|
||||
### JSON Output Mode
|
||||
|
||||
All MCP tools now support `output_format="json"` for machine-readable responses.
|
||||
|
||||
- Default remains `"text"` for human-readable output (no breaking changes)
|
||||
- `build_context` defaults to `"json"` with slimmed payloads (redundant fields stripped)
|
||||
- CLI tool commands support `--format json` flag
|
||||
|
||||
### `tag:` Search Shorthand
|
||||
|
||||
Search by tag using convenient shorthand syntax.
|
||||
|
||||
```
|
||||
search_notes("tag:security")
|
||||
search_notes("tag:coffee AND tag:brewing")
|
||||
```
|
||||
|
||||
### Entity User Tracking
|
||||
|
||||
Entities now track `created_by` and `last_updated_by` fields for attribution.
|
||||
|
||||
### Improved Search Result Content (#609)
|
||||
|
||||
Search results now surface more relevant context:
|
||||
|
||||
- `matched_chunk_text` populated for FTS-only hybrid results (no more fallback to truncated content)
|
||||
- `TOP_CHUNKS_PER_RESULT` increased from 3 to 5, catching answers deeper in large notes (~2700 → ~4500 chars)
|
||||
- `CONTENT_DISPLAY_LIMIT` doubled from 2000 to 4000 chars for results without matched chunks
|
||||
|
||||
### `write_note` Overwrite Guard (#632)
|
||||
|
||||
`write_note` is now non-idempotent by default. If a note already exists, the tool returns an
|
||||
error instead of silently overwriting. Pass `overwrite=True` to replace, or use `edit_note`
|
||||
for incremental updates. Config option `write_note_overwrite_default` restores the old upsert
|
||||
behavior.
|
||||
|
||||
---
|
||||
|
||||
## Architecture Changes
|
||||
|
||||
### Score-Based Hybrid Fusion (#577)
|
||||
|
||||
RRF (Reciprocal Rank Fusion) compressed all fused scores to ~0.016, destroying ranking
|
||||
differentiation. The new formula `max(vec, fts) + FUSION_BONUS * min(vec, fts)` preserves
|
||||
dominant signals and rewards dual-source agreement. Zero-score results now produce zero
|
||||
fused score instead of receiving a 0.1 weight floor.
|
||||
|
||||
### FastMCP 3.0 Upgrade
|
||||
|
||||
Upgraded from FastMCP 2.12.3 to 3.0.1.
|
||||
|
||||
- Tool annotations (`readOnlyHint`, `openWorldHint`) for better client integration
|
||||
- Improved MCP protocol compliance
|
||||
- Better error handling and context management
|
||||
|
||||
### Prompts Call MCP Tools Directly
|
||||
|
||||
MCP prompts (`search`, `continue_conversation`) now call MCP tools directly instead of
|
||||
going through API endpoints. This fixes empty results in discovery mode and ensures prompts
|
||||
use the same resolution logic as tools (including LinkResolver fallback).
|
||||
|
||||
### build_context LinkResolver Fallback
|
||||
|
||||
`build_context` now falls back to LinkResolver when an exact permalink lookup returns empty.
|
||||
This uses the same 7-strategy resolution pipeline as `read_note`, so callers no longer get
|
||||
empty results for valid note identifiers that don't match exact permalinks.
|
||||
|
||||
### Sync Handles Semantic Dependency Errors Gracefully
|
||||
|
||||
When sqlite-vec or another embedding provider is unavailable, `sync_file` now catches
|
||||
`SemanticDependenciesMissingError` separately. The entity is created and FTS-indexed
|
||||
successfully — only vector embeddings are skipped, with a clear warning:
|
||||
|
||||
```
|
||||
WARNING: Semantic search dependencies missing — vector embeddings skipped for path=note.md.
|
||||
Run 'bm reindex --embeddings' after resolving the dependency issue.
|
||||
```
|
||||
|
||||
### Unified Project Path
|
||||
|
||||
Cloud projects with bisync now store the local filesystem path in `path` (not the Docker
|
||||
container path). Config migration automatically promotes `local_sync_path` → `path` for
|
||||
existing configs.
|
||||
|
||||
---
|
||||
|
||||
## CLI Improvements
|
||||
|
||||
### Status and Doctor Default to Local Routing
|
||||
|
||||
`bm status` and `bm doctor` now default to local routing since they scan the local filesystem.
|
||||
Previously, cloud-mode projects would route these commands to the cloud API, which returned
|
||||
Docker-internal paths that don't exist locally.
|
||||
|
||||
### `--format json` for CLI Tool Commands
|
||||
|
||||
All `bm tool` subcommands support `--format json` for machine-readable output, enabling
|
||||
integration with scripts and plugins.
|
||||
|
||||
### `--json` for Top-Level CLI Commands
|
||||
|
||||
Five additional CLI commands now support `--json` for machine-readable output:
|
||||
|
||||
- `bm status --json` — sync report with new/modified/deleted/moved files and skipped files
|
||||
- `bm project list --json` — structured project list with name, paths, routing mode, and defaults
|
||||
- `bm schema validate --json` — validation report with per-note pass/fail, warnings, and errors
|
||||
- `bm schema infer --json` — field frequency analysis and suggested schema definition
|
||||
- `bm schema diff --json` — drift report with new fields, dropped fields, and cardinality changes
|
||||
|
||||
This complements the existing `bm project info --json` and `bm tool --format json` support,
|
||||
making all major CLI commands scriptable for CI pipelines and automation.
|
||||
|
||||
### Cloud Promo and Analytics
|
||||
|
||||
- Cloud promo panel shown on first run or version bump with OSS discount code
|
||||
- Anonymous usage telemetry via Umami Cloud (promo/login funnel events only)
|
||||
- Opt out with `BASIC_MEMORY_NO_PROMOS=1`
|
||||
- No PII, no file contents, no per-command tracking
|
||||
- See [Telemetry](https://github.com/basicmachines-co/basic-memory#telemetry) in README
|
||||
|
||||
---
|
||||
|
||||
## Bug Fixes
|
||||
|
||||
- **#577**: RRF fusion compressed all hybrid scores to ~0.016, destroying ranking differentiation
|
||||
- **#582**: build_context returns empty results on valid note identifiers
|
||||
- **#575**: Remove hardcoded "main" default from default_project
|
||||
- **#595**: recent_activity dedup and pagination across MCP tools
|
||||
- **#593**: Backend-specific distance-to-similarity conversion
|
||||
- **#592**: Strip NUL bytes from content before PostgreSQL search indexing
|
||||
- **#562**: Use VIRTUAL instead of STORED columns in SQLite migration
|
||||
- **#558**: Add X-Tigris-Consistent headers to all rclone commands
|
||||
- **#541**: Handle EntityCreationError as conflict
|
||||
- **#536**: Stabilize metadata filters on Postgres
|
||||
- **#533**: Fix recent_activity prompt defaults
|
||||
- **#530**: Prevent spurious `metadata: {}` in frontmatter output
|
||||
- **#601**: Return matched chunk text in search results
|
||||
- **#606**: Accept `null` for `expected_replacements` in `edit_note`
|
||||
- **#579, #607**: Guard against closed streams in promo panel and missing vector tables on shutdown
|
||||
- **#609**: FTS-only hybrid results missing `matched_chunk_text`; content limits too conservative
|
||||
- **#631**: `build_context` related_results schema validation failure — replaced fragile `_slim_context()` stripping with Pydantic `exclude=True` field config
|
||||
- **#630**: Skip workspace resolution when client factory is active — prevents 401 errors in cloud MCP server mode
|
||||
- **#30**: `tag:` prefix query fails with hybrid search — moved tag prefix parsing to MCP tool level so it works with all search modes
|
||||
- **#31**: `search_notes` returns cluttered observation/relation-level results — now defaults to entity-level results
|
||||
- **#28**: `schema_infer` and `schema_diff` return raw Pydantic models as "undefined" in LLM output — added markdown formatters
|
||||
- Fix `schema_validate` identifier resolution (now uses LinkResolver) and text rendering (markdown formatter)
|
||||
- **#634**: `schema_validate` and `schema_diff` use stale database metadata instead of reading schema definitions from file — now reads frontmatter directly from the file with fallback to database metadata
|
||||
- Fix `Post(**metadata)` crash when frontmatter contains `content` or `handler` keys
|
||||
- Fix list-valued frontmatter fields (`title`, `type`) crashing on `.strip()` — now coerced to strings
|
||||
- Cap sqlite-vec knn `k` parameter at 4096 to prevent backend errors
|
||||
- Parameterize SQL queries in search repository type filters
|
||||
- Double-default display in project list
|
||||
- `ensure_frontmatter_on_sync` default changed to `True`
|
||||
- Status/doctor commands fail with cloud-mode projects (Docker path error)
|
||||
- Prompts return "0 projects" in discovery mode
|
||||
|
||||
---
|
||||
|
||||
## Security
|
||||
|
||||
- Upgrade `cryptography` for CVE advisory
|
||||
- Upgrade `python-multipart` for security advisory
|
||||
|
||||
---
|
||||
|
||||
## Internal / Developer
|
||||
|
||||
- **#598**: Upgrade FastMCP 2.12.3 → 3.0.1 with tool annotations
|
||||
- **#594**: Add `ty` as supplemental type checker
|
||||
- **#538**: Add fast feedback loop tooling (`just fast-check`, `just doctor`, `just testmon`)
|
||||
- **#600**: Rename `entity_type` to `note_type` for consistency
|
||||
- **#596**: Fix CLI runtime defects and audit regressions
|
||||
- CLI refactoring and workspace-aware cloud project listing
|
||||
- Split and speed up PR test matrix in CI
|
||||
- Fix CI: collect coverage from test jobs instead of re-running all tests
|
||||
- Create `search_vector_chunks` in test fixtures for Postgres compatibility
|
||||
|
||||
---
|
||||
|
||||
## Configuration Changes
|
||||
|
||||
| Setting | Old Default | New Default | Notes |
|
||||
|---------|-------------|-------------|-------|
|
||||
| `semantic_search_enabled` | `false` | `true` | Semantic search on by default |
|
||||
| `ensure_frontmatter_on_sync` | `false` | `true` | Frontmatter added during sync |
|
||||
| `permalinks_include_project` | `false` | `true` | Project prefix in permalinks |
|
||||
|
||||
---
|
||||
|
||||
## Upgrade Notes
|
||||
|
||||
- **Semantic search dependencies** are now included by default. If sqlite-vec fails to load,
|
||||
search gracefully falls back to FTS. Run `bm reindex --embeddings` to generate embeddings
|
||||
for existing content.
|
||||
- **Hybrid search scoring** has changed from RRF to score-based fusion. Search result ordering
|
||||
may differ — results should be more accurate with better score differentiation.
|
||||
- **`search_by_metadata`** is removed as a standalone tool. Use `search_notes` with
|
||||
`metadata_filters` instead (same parameters, same behavior).
|
||||
- **Project-prefixed permalinks** are enabled by default. Existing notes keep their current
|
||||
permalinks until modified. Set `permalinks_include_project: false` to disable.
|
||||
- **Frontmatter on sync** is now enabled by default. Files without frontmatter will have it
|
||||
added on next sync. Set `ensure_frontmatter_on_sync: false` to preserve old behavior.
|
||||
- **Config migration** runs automatically for cloud projects with bisync — `local_sync_path`
|
||||
is promoted to `path` so filesystem operations work correctly.
|
||||
- **`write_note` is no longer idempotent** — calls to `write_note` for existing notes now
|
||||
return an error unless `overwrite=True` is passed. Use `edit_note` for incremental changes,
|
||||
or set `write_note_overwrite_default: true` in config to restore the old behavior.
|
||||
+28
-33
@@ -1,26 +1,26 @@
|
||||
# Semantic Search
|
||||
|
||||
This guide covers Basic Memory's semantic (vector) search feature, which adds meaning-based retrieval alongside the existing full-text search.
|
||||
This guide covers Basic Memory's optional semantic (vector) search feature, which adds meaning-based retrieval alongside the existing full-text search.
|
||||
|
||||
## Overview
|
||||
|
||||
Basic Memory's search supports both full-text search (FTS) and semantic retrieval. Semantic search adds vector embeddings that capture the *meaning* of your content, enabling:
|
||||
Basic Memory's default search uses full-text search (FTS) — keyword matching with boolean operators. Semantic search adds vector embeddings that capture the *meaning* of your content, enabling:
|
||||
|
||||
- **Paraphrase matching**: Find "authentication flow" when searching for "login process"
|
||||
- **Conceptual queries**: Search for "ways to improve performance" and find notes about caching, indexing, and optimization
|
||||
- **Hybrid retrieval**: Combine the precision of keyword search with the recall of semantic similarity
|
||||
|
||||
Semantic search is enabled by default when semantic dependencies are available at runtime. It works on both SQLite (local) and Postgres (cloud) backends.
|
||||
Semantic search is **opt-in** — existing behavior is completely unchanged unless you enable it. It works on both SQLite (local) and Postgres (cloud) backends.
|
||||
|
||||
## Installation
|
||||
|
||||
Semantic search dependencies (fastembed, sqlite-vec, openai) are included in the default `basic-memory` install.
|
||||
Semantic search dependencies (fastembed, sqlite-vec, openai) are **optional extras** — they are not installed with the base `basic-memory` package. Install them with:
|
||||
|
||||
```bash
|
||||
pip install basic-memory
|
||||
pip install 'basic-memory[semantic]'
|
||||
```
|
||||
|
||||
You can always override with `BASIC_MEMORY_SEMANTIC_SEARCH_ENABLED=true|false`.
|
||||
This keeps the base install lightweight and avoids platform-specific issues with ONNX Runtime wheels.
|
||||
|
||||
### Platform Compatibility
|
||||
|
||||
@@ -34,40 +34,36 @@ You can always override with `BASIC_MEMORY_SEMANTIC_SEARCH_ENABLED=true|false`.
|
||||
|
||||
#### Intel Mac Workaround
|
||||
|
||||
The default install includes FastEmbed, which depends on ONNX Runtime. ONNX Runtime dropped Intel Mac (x86_64) wheels starting in v1.24, so install with a compatible ONNX Runtime pin first:
|
||||
|
||||
```bash
|
||||
pip install basic-memory 'onnxruntime<1.24'
|
||||
```
|
||||
|
||||
After installation, Intel Mac users have two runtime options:
|
||||
The default FastEmbed provider uses ONNX Runtime, which dropped Intel Mac (x86_64) wheels starting in v1.24. Intel Mac users have two options:
|
||||
|
||||
**Option 1: Use OpenAI embeddings (recommended)**
|
||||
|
||||
Install only the OpenAI dependency manually — no ONNX Runtime or FastEmbed needed:
|
||||
|
||||
```bash
|
||||
pip install openai sqlite-vec
|
||||
export BASIC_MEMORY_SEMANTIC_SEARCH_ENABLED=true
|
||||
export BASIC_MEMORY_SEMANTIC_EMBEDDING_PROVIDER=openai
|
||||
export OPENAI_API_KEY=sk-...
|
||||
```
|
||||
|
||||
**Option 2: Use FastEmbed locally**
|
||||
**Option 2: Pin an older ONNX Runtime**
|
||||
|
||||
Keep the same pinned installation and use FastEmbed (default provider):
|
||||
FastEmbed's ONNX Runtime dependency is unpinned, so you can constrain it to an older version that still ships Intel Mac wheels by passing both requirements in the same install command:
|
||||
|
||||
```bash
|
||||
export BASIC_MEMORY_SEMANTIC_SEARCH_ENABLED=true
|
||||
export BASIC_MEMORY_SEMANTIC_EMBEDDING_PROVIDER=fastembed
|
||||
pip install 'basic-memory[semantic]' 'onnxruntime<1.24'
|
||||
```
|
||||
|
||||
## Quick Start
|
||||
|
||||
1. Install Basic Memory:
|
||||
1. Install semantic extras:
|
||||
|
||||
```bash
|
||||
pip install basic-memory
|
||||
pip install 'basic-memory[semantic]'
|
||||
```
|
||||
|
||||
2. (Optional) Explicitly enable semantic search:
|
||||
2. Enable semantic search:
|
||||
|
||||
```bash
|
||||
export BASIC_MEMORY_SEMANTIC_SEARCH_ENABLED=true
|
||||
@@ -88,7 +84,7 @@ search_notes("login process", search_type="vector")
|
||||
# Hybrid: combines FTS precision with vector recall (recommended)
|
||||
search_notes("login process", search_type="hybrid")
|
||||
|
||||
# Explicit full-text search
|
||||
# Traditional full-text search (still the default)
|
||||
search_notes("login process", search_type="text")
|
||||
```
|
||||
|
||||
@@ -98,7 +94,7 @@ All settings are fields on `BasicMemoryConfig` and can be set via environment va
|
||||
|
||||
| Config Field | Env Var | Default | Description |
|
||||
|---|---|---|---|
|
||||
| `semantic_search_enabled` | `BASIC_MEMORY_SEMANTIC_SEARCH_ENABLED` | Auto (`true` when semantic deps are available) | Enable semantic search. Required before vector/hybrid modes work. |
|
||||
| `semantic_search_enabled` | `BASIC_MEMORY_SEMANTIC_SEARCH_ENABLED` | `false` | Enable semantic search. Required before vector/hybrid modes work. |
|
||||
| `semantic_embedding_provider` | `BASIC_MEMORY_SEMANTIC_EMBEDDING_PROVIDER` | `"fastembed"` | Embedding provider: `"fastembed"` (local) or `"openai"` (API). |
|
||||
| `semantic_embedding_model` | `BASIC_MEMORY_SEMANTIC_EMBEDDING_MODEL` | `"bge-small-en-v1.5"` | Model identifier. Auto-adjusted per provider if left at default. |
|
||||
| `semantic_embedding_dimensions` | `BASIC_MEMORY_SEMANTIC_EMBEDDING_DIMENSIONS` | Auto-detected | Vector dimensions. 384 for FastEmbed, 1536 for OpenAI. Override only if using a non-default model. |
|
||||
@@ -116,8 +112,8 @@ FastEmbed runs entirely locally using ONNX models — no API key, no network cal
|
||||
- **Tradeoff**: Smaller model, fast inference, good quality for most use cases
|
||||
|
||||
```bash
|
||||
# Install basic-memory and enable semantic search
|
||||
pip install basic-memory
|
||||
# Install semantic extras and enable
|
||||
pip install 'basic-memory[semantic]'
|
||||
export BASIC_MEMORY_SEMANTIC_SEARCH_ENABLED=true
|
||||
```
|
||||
|
||||
@@ -165,13 +161,13 @@ Returns results ranked by cosine similarity. Individual observations and relatio
|
||||
|
||||
### `hybrid`
|
||||
|
||||
Combines FTS and vector results using score-based fusion. This is generally the best mode when you want both keyword precision and semantic recall.
|
||||
Combines FTS and vector results using reciprocal rank fusion (RRF). This is generally the best mode when you want both keyword precision and semantic recall.
|
||||
|
||||
```python
|
||||
search_notes("authentication security", search_type="hybrid")
|
||||
```
|
||||
|
||||
Score-based fusion uses the formula `max(vec, fts) + bonus * min(vec, fts)` to preserve the dominant signal while rewarding results found by both methods.
|
||||
RRF merges the two ranked lists so that items appearing in both get a score boost, while items found by only one method still appear.
|
||||
|
||||
### When to Use Which
|
||||
|
||||
@@ -201,8 +197,7 @@ bm reindex -p my-project
|
||||
|
||||
### When You Need to Reindex
|
||||
|
||||
- **Upgrade note**: Migration now performs a one-time automatic embedding backfill on upgrade.
|
||||
- **Manual enable case**: If you explicitly had `semantic_search_enabled=false` and then turn it on
|
||||
- **First enable**: After turning on `semantic_search_enabled` for the first time
|
||||
- **Provider change**: After switching between `fastembed` and `openai`
|
||||
- **Model change**: After changing `semantic_embedding_model`
|
||||
- **Dimension change**: After changing `semantic_embedding_dimensions`
|
||||
@@ -236,14 +231,14 @@ Each chunk has a `source_hash` (SHA-256 of the chunk text). On re-sync, unchange
|
||||
|
||||
### Hybrid Fusion
|
||||
|
||||
Hybrid search uses score-based fusion to merge FTS and vector results:
|
||||
Hybrid search uses reciprocal rank fusion (RRF) to merge FTS and vector results:
|
||||
|
||||
1. Run FTS search to get keyword-ranked results; normalize scores to [0, 1]
|
||||
2. Run vector search to get similarity-ranked results (already [0, 1])
|
||||
3. For each result, compute: `fused = max(vec_score, fts_score) + 0.3 * min(vec_score, fts_score)`
|
||||
1. Run FTS search to get keyword-ranked results
|
||||
2. Run vector search to get similarity-ranked results
|
||||
3. For each result, compute: `score = 1/(k + fts_rank) + 1/(k + vector_rank)` where `k = 60`
|
||||
4. Sort by fused score
|
||||
|
||||
The dominant signal (whichever source scored higher) is preserved, and dual-source agreement adds a bonus. Unlike rank-based fusion, this approach retains score magnitude — a strong vector match stays strong even without an FTS hit.
|
||||
Items found by both methods get a natural score boost. Items found by only one method still appear but rank lower.
|
||||
|
||||
### Observation-Level Results
|
||||
|
||||
|
||||
@@ -64,7 +64,6 @@ class SchemaDefinition:
|
||||
version: int # Schema version
|
||||
fields: list[SchemaField] # Parsed fields
|
||||
validation_mode: str # "warn" | "strict" | "off"
|
||||
frontmatter_fields: list[SchemaField] # From settings.frontmatter (default: [])
|
||||
|
||||
|
||||
def parse_picoschema(yaml_dict: dict) -> list[SchemaField]:
|
||||
@@ -146,16 +145,14 @@ class ValidationResult:
|
||||
async def validate_note(
|
||||
note: Note,
|
||||
schema: SchemaDefinition,
|
||||
frontmatter: dict | None = None,
|
||||
) -> ValidationResult:
|
||||
"""Validate a note against a schema definition.
|
||||
|
||||
Mapping rules:
|
||||
- field: string → observation [field] exists
|
||||
- field?(array): type → multiple [field] observations
|
||||
- field?: EntityType → relation 'field [[...]]' exists
|
||||
- field?(enum): [v] → observation [field] value ∈ enum values
|
||||
- settings.frontmatter field → frontmatter key presence/value
|
||||
- field: string → observation [field] exists
|
||||
- field?(array): type → multiple [field] observations
|
||||
- field?: EntityType → relation 'field [[...]]' exists
|
||||
- field?(enum): [v] → observation [field] value ∈ enum values
|
||||
"""
|
||||
```
|
||||
|
||||
|
||||
@@ -73,7 +73,6 @@ authors to learn.
|
||||
| `field?(array): EntityType` | Multiple `field` relations | `- authored [[Book]]` (×N) |
|
||||
| `tags` | Frontmatter `tags` array | `tags: [startups, essays]` |
|
||||
| `field?(enum): [values]` | Observation `[field] value` where value ∈ set | `- [status] active` |
|
||||
| `settings.frontmatter` field | Frontmatter key presence/value | `tags: [python, ai]` |
|
||||
|
||||
### Key Insight
|
||||
|
||||
@@ -100,9 +99,6 @@ schema:
|
||||
expertise?(array): string, areas of knowledge
|
||||
settings:
|
||||
validation: warn # warn | strict | off
|
||||
frontmatter:
|
||||
tags?(array): string, note categories
|
||||
status?(enum): [draft, review, published]
|
||||
---
|
||||
|
||||
# Person
|
||||
@@ -234,32 +230,6 @@ $ bm schema validate people/ada-lovelace.md
|
||||
"Unmatched" items are informational — observations and relations the schema doesn't cover.
|
||||
They're valid. Schemas are a subset, not a straitjacket.
|
||||
|
||||
### Frontmatter Validation
|
||||
|
||||
Schema notes can declare validation rules for frontmatter keys under `settings.frontmatter`
|
||||
using the same Picoschema syntax as the `schema` block:
|
||||
|
||||
```yaml
|
||||
settings:
|
||||
validation: warn
|
||||
frontmatter:
|
||||
tags?(array): string
|
||||
status?(enum): [draft, review, published]
|
||||
```
|
||||
|
||||
- Frontmatter rules use the same Picoschema key syntax (`?` for optional, `(enum)`, `(array)`)
|
||||
- Only available on schema notes (inline schemas skip frontmatter validation)
|
||||
- Checks key presence (required vs optional) and enum value membership
|
||||
- Unmatched frontmatter keys not in the schema are silently ignored
|
||||
- Missing required frontmatter keys produce a warning (or error in strict mode)
|
||||
|
||||
Example output for a missing required frontmatter key:
|
||||
|
||||
```
|
||||
⚠ Person schema validation:
|
||||
- Missing required frontmatter key: status
|
||||
```
|
||||
|
||||
### Batch Validation
|
||||
|
||||
```
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
# Install dependencies
|
||||
install:
|
||||
uv sync
|
||||
uv sync --extra semantic
|
||||
@echo ""
|
||||
@echo "💡 Remember to activate the virtual environment by running: source .venv/bin/activate"
|
||||
|
||||
@@ -170,14 +170,10 @@ lint: fix
|
||||
fix:
|
||||
uv run ruff check --fix --unsafe-fixes src tests test-int
|
||||
|
||||
# Type check code (pyright)
|
||||
# Type check code
|
||||
typecheck:
|
||||
uv run pyright
|
||||
|
||||
# Type check code (ty)
|
||||
typecheck-ty:
|
||||
uv run ty check src/
|
||||
|
||||
# Clean build artifacts and cache files
|
||||
clean:
|
||||
find . -type f -name '*.pyc' -delete
|
||||
|
||||
+6
-6
@@ -29,7 +29,7 @@ dependencies = [
|
||||
"alembic>=1.14.1",
|
||||
"pillow>=11.1.0",
|
||||
"pybars3>=0.9.7",
|
||||
"fastmcp>=3.0.1,<4",
|
||||
"fastmcp==2.12.3", # Pinned - 2.14.x breaks MCP tools visibility (issue #463)
|
||||
"pyjwt>=2.10.1",
|
||||
"python-dotenv>=1.1.0",
|
||||
"pytest-aio>=1.9.0",
|
||||
@@ -44,6 +44,10 @@ dependencies = [
|
||||
"sniffio>=1.3.1",
|
||||
"anyio>=4.10.0",
|
||||
"httpx>=0.28.0",
|
||||
]
|
||||
|
||||
[project.optional-dependencies]
|
||||
semantic = [
|
||||
"fastembed>=0.7.4",
|
||||
"sqlite-vec>=0.1.6",
|
||||
"openai>=1.100.2",
|
||||
@@ -74,7 +78,7 @@ markers = [
|
||||
"postgres: Tests that run against Postgres backend (deselect with '-m \"not postgres\"')",
|
||||
"windows: Windows-specific tests (deselect with '-m \"not windows\"')",
|
||||
"smoke: Fast end-to-end smoke tests for MCP flows",
|
||||
"semantic: Tests requiring semantic dependencies (fastembed, sqlite-vec, openai)",
|
||||
"semantic: Tests requiring [semantic] extras (fastembed, sqlite-vec, openai)",
|
||||
]
|
||||
|
||||
[tool.ruff]
|
||||
@@ -96,9 +100,6 @@ dev = [
|
||||
"psycopg>=3.2.0",
|
||||
"pyright>=1.1.408",
|
||||
"pytest-testmon>=2.2.0",
|
||||
"ty>=0.0.18",
|
||||
"cst-lsp>=0.1.3",
|
||||
"libcst>=1.8.6",
|
||||
]
|
||||
|
||||
[tool.hatch.version]
|
||||
@@ -117,7 +118,6 @@ ignore = ["test/"]
|
||||
defineConstant = { DEBUG = true }
|
||||
reportMissingImports = "error"
|
||||
reportMissingTypeStubs = false
|
||||
reportUnusedImport = "none"
|
||||
pythonVersion = "3.12"
|
||||
|
||||
|
||||
|
||||
+2
-2
@@ -6,12 +6,12 @@
|
||||
"url": "https://github.com/basicmachines-co/basic-memory.git",
|
||||
"source": "github"
|
||||
},
|
||||
"version": "0.19.2",
|
||||
"version": "0.18.3",
|
||||
"packages": [
|
||||
{
|
||||
"registryType": "pypi",
|
||||
"identifier": "basic-memory",
|
||||
"version": "0.19.2",
|
||||
"version": "0.18.3",
|
||||
"runtimeHint": "uvx",
|
||||
"runtimeArguments": [
|
||||
{"type": "positional", "value": "basic-memory"},
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
"""basic-memory - Local-first knowledge management combining Zettelkasten with knowledge graphs"""
|
||||
|
||||
# Package version - updated by release automation
|
||||
__version__ = "0.19.2"
|
||||
__version__ = "0.18.3"
|
||||
|
||||
# API version for FastAPI - independent of package version
|
||||
__api_version__ = "v0"
|
||||
|
||||
@@ -1,29 +0,0 @@
|
||||
"""Trigger automatic semantic embedding backfill during migration.
|
||||
|
||||
Revision ID: i2c3d4e5f6g7
|
||||
Revises: h1b2c3d4e5f6
|
||||
Create Date: 2026-02-19 00:00:00.000000
|
||||
|
||||
"""
|
||||
|
||||
from typing import Sequence, Union
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = "i2c3d4e5f6g7"
|
||||
down_revision: Union[str, None] = "h1b2c3d4e5f6"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""No schema change.
|
||||
|
||||
Trigger: this revision is newly applied.
|
||||
Why: db.run_migrations() detects this revision transition and runs the existing
|
||||
sync_entity_vectors() pipeline to backfill semantic embeddings automatically.
|
||||
Outcome: users no longer need to run `bm reindex --embeddings` after upgrading.
|
||||
"""
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""No-op downgrade."""
|
||||
@@ -1,164 +0,0 @@
|
||||
"""Rename entity_type column to note_type
|
||||
|
||||
Revision ID: j3d4e5f6g7h8
|
||||
Revises: i2c3d4e5f6g7
|
||||
Create Date: 2026-02-22 12:00:00.000000
|
||||
|
||||
"""
|
||||
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
from sqlalchemy import text
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = "j3d4e5f6g7h8"
|
||||
down_revision: Union[str, None] = "i2c3d4e5f6g7"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def table_exists(connection, table_name: str) -> bool:
|
||||
"""Check if a table exists (idempotent migration support)."""
|
||||
if connection.dialect.name == "postgresql":
|
||||
result = connection.execute(
|
||||
text("SELECT 1 FROM information_schema.tables WHERE table_name = :table_name"),
|
||||
{"table_name": table_name},
|
||||
)
|
||||
return result.fetchone() is not None
|
||||
# SQLite
|
||||
result = connection.execute(
|
||||
text("SELECT 1 FROM sqlite_master WHERE type='table' AND name = :table_name"),
|
||||
{"table_name": table_name},
|
||||
)
|
||||
return result.fetchone() is not None
|
||||
|
||||
|
||||
def index_exists(connection, index_name: str) -> bool:
|
||||
"""Check if an index exists (idempotent migration support)."""
|
||||
if connection.dialect.name == "postgresql":
|
||||
result = connection.execute(
|
||||
text("SELECT 1 FROM pg_indexes WHERE indexname = :index_name"),
|
||||
{"index_name": index_name},
|
||||
)
|
||||
return result.fetchone() is not None
|
||||
# SQLite
|
||||
result = connection.execute(
|
||||
text("SELECT 1 FROM sqlite_master WHERE type='index' AND name = :index_name"),
|
||||
{"index_name": index_name},
|
||||
)
|
||||
return result.fetchone() is not None
|
||||
|
||||
|
||||
def column_exists(connection, table: str, column: str) -> bool:
|
||||
"""Check if a column exists in a table (idempotent migration support)."""
|
||||
if connection.dialect.name == "postgresql":
|
||||
result = connection.execute(
|
||||
text(
|
||||
"SELECT 1 FROM information_schema.columns "
|
||||
"WHERE table_name = :table AND column_name = :column"
|
||||
),
|
||||
{"table": table, "column": column},
|
||||
)
|
||||
return result.fetchone() is not None
|
||||
# SQLite
|
||||
result = connection.execute(text(f"PRAGMA table_info({table})"))
|
||||
columns = [row[1] for row in result]
|
||||
return column in columns
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Rename entity_type → note_type on the entity table."""
|
||||
connection = op.get_bind()
|
||||
dialect = connection.dialect.name
|
||||
|
||||
# Skip if already migrated (idempotent)
|
||||
if column_exists(connection, "entity", "note_type"):
|
||||
return
|
||||
|
||||
if dialect == "postgresql":
|
||||
# Postgres supports direct column rename
|
||||
op.execute("ALTER TABLE entity RENAME COLUMN entity_type TO note_type")
|
||||
|
||||
# Recreate the index with new name
|
||||
op.execute("DROP INDEX IF EXISTS ix_entity_type")
|
||||
op.execute("CREATE INDEX ix_note_type ON entity (note_type)")
|
||||
else:
|
||||
# SQLite 3.25.0+ supports ALTER TABLE RENAME COLUMN directly.
|
||||
# Avoids batch_alter_table which fails on tables with generated columns
|
||||
# (duplicate column name error when recreating the table).
|
||||
op.execute("ALTER TABLE entity RENAME COLUMN entity_type TO note_type")
|
||||
|
||||
# Recreate the index with new name
|
||||
if index_exists(connection, "ix_entity_type"):
|
||||
op.drop_index("ix_entity_type", table_name="entity")
|
||||
op.create_index("ix_note_type", "entity", ["note_type"])
|
||||
|
||||
# Update search index metadata: rename entity_type → note_type in JSON
|
||||
# This updates the stored metadata so search results use the new field name
|
||||
# Guard: search_index may not exist on a fresh DB (created by an earlier migration)
|
||||
if not table_exists(connection, "search_index"):
|
||||
return
|
||||
|
||||
if dialect == "postgresql":
|
||||
op.execute(
|
||||
text("""
|
||||
UPDATE search_index
|
||||
SET metadata = metadata - 'entity_type' || jsonb_build_object('note_type', metadata->'entity_type')
|
||||
WHERE metadata ? 'entity_type'
|
||||
""")
|
||||
)
|
||||
else:
|
||||
op.execute(
|
||||
text("""
|
||||
UPDATE search_index
|
||||
SET metadata = json_set(
|
||||
json_remove(metadata, '$.entity_type'),
|
||||
'$.note_type',
|
||||
json_extract(metadata, '$.entity_type')
|
||||
)
|
||||
WHERE json_extract(metadata, '$.entity_type') IS NOT NULL
|
||||
""")
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Rename note_type → entity_type on the entity table."""
|
||||
connection = op.get_bind()
|
||||
dialect = connection.dialect.name
|
||||
|
||||
if dialect == "postgresql":
|
||||
op.execute("ALTER TABLE entity RENAME COLUMN note_type TO entity_type")
|
||||
op.execute("DROP INDEX IF EXISTS ix_note_type")
|
||||
op.execute("CREATE INDEX ix_entity_type ON entity (entity_type)")
|
||||
else:
|
||||
op.execute("ALTER TABLE entity RENAME COLUMN note_type TO entity_type")
|
||||
|
||||
if index_exists(connection, "ix_note_type"):
|
||||
op.drop_index("ix_note_type", table_name="entity")
|
||||
op.create_index("ix_entity_type", "entity", ["entity_type"])
|
||||
|
||||
# Revert search index metadata
|
||||
if not table_exists(connection, "search_index"):
|
||||
return
|
||||
|
||||
if dialect == "postgresql":
|
||||
op.execute(
|
||||
text("""
|
||||
UPDATE search_index
|
||||
SET metadata = metadata - 'note_type' || jsonb_build_object('entity_type', metadata->'note_type')
|
||||
WHERE metadata ? 'note_type'
|
||||
""")
|
||||
)
|
||||
else:
|
||||
op.execute(
|
||||
text("""
|
||||
UPDATE search_index
|
||||
SET metadata = json_set(
|
||||
json_remove(metadata, '$.note_type'),
|
||||
'$.entity_type',
|
||||
json_extract(metadata, '$.note_type')
|
||||
)
|
||||
WHERE json_extract(metadata, '$.note_type') IS NOT NULL
|
||||
""")
|
||||
)
|
||||
-74
@@ -1,74 +0,0 @@
|
||||
"""Add created_by and last_updated_by columns to entity table.
|
||||
|
||||
Revision ID: k4e5f6g7h8i9
|
||||
Revises: j3d4e5f6g7h8
|
||||
Create Date: 2026-02-23 00:00:00.000000
|
||||
|
||||
These columns track which cloud user created and last modified each entity.
|
||||
Both are nullable — NULL for local/CLI usage and existing entities.
|
||||
"""
|
||||
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy import text
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = "k4e5f6g7h8i9"
|
||||
down_revision: Union[str, None] = "j3d4e5f6g7h8"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def column_exists(connection, table: str, column: str) -> bool:
|
||||
"""Check if a column exists in a table (idempotent migration support)."""
|
||||
if connection.dialect.name == "postgresql":
|
||||
result = connection.execute(
|
||||
text(
|
||||
"SELECT 1 FROM information_schema.columns "
|
||||
"WHERE table_name = :table AND column_name = :column"
|
||||
),
|
||||
{"table": table, "column": column},
|
||||
)
|
||||
return result.fetchone() is not None
|
||||
else:
|
||||
# SQLite
|
||||
result = connection.execute(text(f"PRAGMA table_info({table})"))
|
||||
columns = [row[1] for row in result]
|
||||
return column in columns
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Add created_by and last_updated_by columns to entity table.
|
||||
|
||||
Both columns are nullable strings that store cloud user_profile_id UUIDs.
|
||||
No data backfill — existing rows get NULL.
|
||||
"""
|
||||
connection = op.get_bind()
|
||||
|
||||
if not column_exists(connection, "entity", "created_by"):
|
||||
op.add_column("entity", sa.Column("created_by", sa.String(), nullable=True))
|
||||
|
||||
if not column_exists(connection, "entity", "last_updated_by"):
|
||||
op.add_column("entity", sa.Column("last_updated_by", sa.String(), nullable=True))
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Remove created_by and last_updated_by columns from entity table."""
|
||||
connection = op.get_bind()
|
||||
dialect = connection.dialect.name
|
||||
|
||||
if column_exists(connection, "entity", "last_updated_by"):
|
||||
if dialect == "postgresql":
|
||||
op.drop_column("entity", "last_updated_by")
|
||||
else:
|
||||
with op.batch_alter_table("entity") as batch_op:
|
||||
batch_op.drop_column("last_updated_by")
|
||||
|
||||
if column_exists(connection, "entity", "created_by"):
|
||||
if dialect == "postgresql":
|
||||
op.drop_column("entity", "created_by")
|
||||
else:
|
||||
with op.batch_alter_table("entity") as batch_op:
|
||||
batch_op.drop_column("created_by")
|
||||
@@ -20,7 +20,6 @@ from basic_memory.deps import (
|
||||
ProjectConfigV2ExternalDep,
|
||||
AppConfigDep,
|
||||
EntityRepositoryV2ExternalDep,
|
||||
RelationRepositoryV2ExternalDep,
|
||||
ProjectExternalIdPathDep,
|
||||
TaskSchedulerDep,
|
||||
FileServiceV2ExternalDep,
|
||||
@@ -32,9 +31,6 @@ from basic_memory.schemas.v2 import (
|
||||
EntityResolveRequest,
|
||||
EntityResolveResponse,
|
||||
EntityResponseV2,
|
||||
GraphEdge,
|
||||
GraphNode,
|
||||
GraphResponse,
|
||||
MoveEntityRequestV2,
|
||||
MoveDirectoryRequestV2,
|
||||
DeleteDirectoryRequestV2,
|
||||
@@ -60,50 +56,6 @@ def _schedule_vector_sync_if_enabled(
|
||||
)
|
||||
|
||||
|
||||
## Graph endpoint
|
||||
|
||||
|
||||
@router.get("/graph", response_model=GraphResponse)
|
||||
async def get_graph(
|
||||
project_id: ProjectExternalIdPathDep,
|
||||
entity_repository: EntityRepositoryV2ExternalDep,
|
||||
relation_repository: RelationRepositoryV2ExternalDep,
|
||||
) -> GraphResponse:
|
||||
"""Return all entities and resolved relations for knowledge graph visualization.
|
||||
|
||||
Returns a flat node/edge structure optimized for rendering with graph libraries.
|
||||
Only includes resolved relations (where to_id is not null).
|
||||
"""
|
||||
logger.info("API v2 request: get_graph")
|
||||
|
||||
# Fetch all entities for this project
|
||||
entities = await entity_repository.find_all(use_load_options=False)
|
||||
nodes = [
|
||||
GraphNode(
|
||||
external_id=entity.external_id,
|
||||
title=entity.title,
|
||||
note_type=entity.note_type,
|
||||
file_path=entity.file_path,
|
||||
)
|
||||
for entity in entities
|
||||
]
|
||||
|
||||
# Fetch all resolved relations (to_id is not null) with eager-loaded entities
|
||||
relations = await relation_repository.find_all()
|
||||
edges = [
|
||||
GraphEdge(
|
||||
from_id=relation.from_entity.external_id,
|
||||
to_id=relation.to_entity.external_id,
|
||||
relation_type=relation.relation_type,
|
||||
)
|
||||
for relation in relations
|
||||
if relation.to_entity is not None
|
||||
]
|
||||
|
||||
logger.info(f"API v2 response: graph with {len(nodes)} nodes and {len(edges)} edges")
|
||||
return GraphResponse(nodes=nodes, edges=edges)
|
||||
|
||||
|
||||
## Resolution endpoint
|
||||
|
||||
|
||||
@@ -178,7 +130,7 @@ async def resolve_identifier(
|
||||
resolution_method=resolution_method,
|
||||
)
|
||||
|
||||
logger.debug(
|
||||
logger.info(
|
||||
f"API v2 response: resolved '{data.identifier}' to external_id={result.external_id} via {resolution_method}"
|
||||
)
|
||||
|
||||
@@ -249,7 +201,7 @@ async def create_entity(
|
||||
Created entity with generated external_id (UUID) and file content
|
||||
"""
|
||||
logger.info(
|
||||
"API v2 request", endpoint="create_entity", note_type=data.note_type, title=data.title
|
||||
"API v2 request", endpoint="create_entity", entity_type=data.entity_type, title=data.title
|
||||
)
|
||||
|
||||
if fast:
|
||||
|
||||
@@ -48,7 +48,7 @@ async def list_projects(
|
||||
A list of all projects with metadata
|
||||
"""
|
||||
projects = await project_service.list_projects()
|
||||
default_project = await project_service.get_default_project_name()
|
||||
default_project = project_service.default_project
|
||||
|
||||
project_items = [
|
||||
ProjectItem(
|
||||
|
||||
@@ -145,14 +145,14 @@ async def create_resource(
|
||||
# Determine file details
|
||||
file_name = PathLib(data.file_path).name
|
||||
content_type = file_service.content_type(data.file_path)
|
||||
note_type = "canvas" if data.file_path.endswith(".canvas") else "file"
|
||||
entity_type = "canvas" if data.file_path.endswith(".canvas") else "file"
|
||||
|
||||
# Create a new entity model
|
||||
# Explicitly set external_id to ensure NOT NULL constraint is satisfied (fixes #512)
|
||||
entity = EntityModel(
|
||||
external_id=str(uuid.uuid4()),
|
||||
title=file_name,
|
||||
note_type=note_type,
|
||||
entity_type=entity_type,
|
||||
content_type=content_type,
|
||||
file_path=data.file_path,
|
||||
checksum=checksum,
|
||||
@@ -253,14 +253,14 @@ async def update_resource(
|
||||
# Determine file details
|
||||
file_name = PathLib(target_file_path).name
|
||||
content_type = file_service.content_type(target_file_path)
|
||||
note_type = "canvas" if target_file_path.endswith(".canvas") else "file"
|
||||
entity_type = "canvas" if target_file_path.endswith(".canvas") else "file"
|
||||
|
||||
# Update entity using internal ID
|
||||
updated_entity = await entity_repository.update(
|
||||
entity.id,
|
||||
{
|
||||
"title": file_name,
|
||||
"note_type": note_type,
|
||||
"entity_type": entity_type,
|
||||
"content_type": content_type,
|
||||
"file_path": target_file_path,
|
||||
"checksum": checksum,
|
||||
|
||||
@@ -10,15 +10,9 @@ Flow: Entity loaded with eager observations/relations -> convert to tuples -> co
|
||||
|
||||
from pathlib import Path as FilePath
|
||||
|
||||
import frontmatter
|
||||
from fastapi import APIRouter, Path, Query
|
||||
from loguru import logger
|
||||
|
||||
from basic_memory.deps import (
|
||||
EntityRepositoryV2ExternalDep,
|
||||
FileServiceV2ExternalDep,
|
||||
LinkResolverV2ExternalDep,
|
||||
)
|
||||
from basic_memory.deps import EntityRepositoryV2ExternalDep
|
||||
from basic_memory.models.knowledge import Entity
|
||||
from basic_memory.schemas.schema import (
|
||||
ValidationReport,
|
||||
@@ -57,7 +51,7 @@ def _entity_relations(entity: Entity) -> list[RelationData]:
|
||||
RelationData(
|
||||
relation_type=rel.relation_type,
|
||||
target_name=rel.to_name,
|
||||
target_note_type=rel.to_entity.note_type if rel.to_entity else None,
|
||||
target_entity_type=rel.to_entity.entity_type if rel.to_entity else None,
|
||||
)
|
||||
for rel in entity.outgoing_relations
|
||||
]
|
||||
@@ -73,54 +67,11 @@ def _entity_to_note_data(entity: Entity) -> NoteData:
|
||||
|
||||
|
||||
def _entity_frontmatter(entity: Entity) -> dict:
|
||||
"""Build a frontmatter dict from an entity's database metadata.
|
||||
|
||||
Used for the notes being validated — their type and schema ref are
|
||||
unlikely to change between syncs.
|
||||
"""
|
||||
fm = dict(entity.entity_metadata) if entity.entity_metadata else {}
|
||||
if entity.note_type:
|
||||
fm.setdefault("type", entity.note_type)
|
||||
return fm
|
||||
|
||||
|
||||
async def _schema_frontmatter_from_file(
|
||||
file_service: FileServiceV2ExternalDep,
|
||||
entity: Entity,
|
||||
) -> dict:
|
||||
"""Read a schema entity's frontmatter directly from its file.
|
||||
|
||||
Schema definitions (field declarations, validation mode) are the source
|
||||
of truth for validation. Reading from the file ensures schema-validate
|
||||
always uses the latest settings, even when the file watcher hasn't
|
||||
synced changes to entity_metadata in the database.
|
||||
"""
|
||||
try:
|
||||
content = await file_service.read_file_content(entity.file_path)
|
||||
post = frontmatter.loads(content)
|
||||
metadata = dict(post.metadata)
|
||||
|
||||
# Trigger: file is mid-edit and missing required schema fields
|
||||
# Why: parse_schema_note() raises ValueError for missing entity/schema,
|
||||
# which would turn validation into a 500 response
|
||||
# Outcome: fall back to last-known-good database metadata
|
||||
if not metadata.get("entity") or not isinstance(metadata.get("schema"), dict):
|
||||
logger.warning(
|
||||
"Schema file has incomplete frontmatter, falling back to database metadata",
|
||||
file_path=entity.file_path,
|
||||
)
|
||||
return _entity_frontmatter(entity)
|
||||
|
||||
return metadata
|
||||
except Exception:
|
||||
# Trigger: file is missing, unreadable, or has malformed frontmatter
|
||||
# Why: fall back to database metadata rather than failing validation entirely
|
||||
# Outcome: behaves like before this change — uses potentially stale data
|
||||
logger.warning(
|
||||
"Failed to read schema file, falling back to database metadata",
|
||||
file_path=entity.file_path,
|
||||
)
|
||||
return _entity_frontmatter(entity)
|
||||
"""Build a frontmatter dict from an entity for schema resolution."""
|
||||
frontmatter = dict(entity.entity_metadata) if entity.entity_metadata else {}
|
||||
if entity.entity_type:
|
||||
frontmatter.setdefault("type", entity.entity_type)
|
||||
return frontmatter
|
||||
|
||||
|
||||
# --- Validation ---
|
||||
@@ -129,30 +80,22 @@ async def _schema_frontmatter_from_file(
|
||||
@router.post("/schema/validate", response_model=ValidationReport)
|
||||
async def validate_schema(
|
||||
entity_repository: EntityRepositoryV2ExternalDep,
|
||||
file_service: FileServiceV2ExternalDep,
|
||||
link_resolver: LinkResolverV2ExternalDep,
|
||||
project_id: str = Path(..., description="Project external UUID"),
|
||||
note_type: str | None = Query(None, description="Note type to validate"),
|
||||
entity_type: str | None = Query(None, description="Entity type to validate"),
|
||||
identifier: str | None = Query(None, description="Specific note identifier"),
|
||||
):
|
||||
"""Validate notes against their resolved schemas.
|
||||
|
||||
Validates a specific note (by identifier) or all notes of a given type.
|
||||
Returns warnings/errors based on the schema's validation mode.
|
||||
|
||||
Schema definitions are read directly from their files to ensure the
|
||||
latest settings (validation mode, field declarations) are always used,
|
||||
even when file changes haven't been synced to the database yet.
|
||||
"""
|
||||
results: list[NoteValidationResponse] = []
|
||||
|
||||
# --- Single note validation ---
|
||||
if identifier:
|
||||
# Resolve identifier flexibly (permalink, title, path, fuzzy)
|
||||
# to match how read_note and other tools resolve identifiers
|
||||
entity = await link_resolver.resolve_link(identifier)
|
||||
entity = await entity_repository.get_by_permalink(identifier)
|
||||
if not entity:
|
||||
return ValidationReport(note_type=note_type, total_notes=0, total_entities=0)
|
||||
return ValidationReport(entity_type=entity_type, total_notes=0, results=[])
|
||||
|
||||
frontmatter = _entity_frontmatter(entity)
|
||||
schema_ref = frontmatter.get("schema")
|
||||
@@ -163,31 +106,29 @@ async def validate_schema(
|
||||
query,
|
||||
allow_reference_match=isinstance(schema_ref, str) and query == schema_ref,
|
||||
)
|
||||
return [await _schema_frontmatter_from_file(file_service, e) for e in entities]
|
||||
return [_entity_frontmatter(e) for e in entities]
|
||||
|
||||
schema_def = await resolve_schema(frontmatter, search_fn)
|
||||
if schema_def:
|
||||
result = validate_note(
|
||||
entity.title or entity.permalink or identifier,
|
||||
entity.permalink or identifier,
|
||||
schema_def,
|
||||
_entity_observations(entity),
|
||||
_entity_relations(entity),
|
||||
frontmatter=frontmatter,
|
||||
)
|
||||
results.append(_to_note_validation_response(result))
|
||||
|
||||
return ValidationReport(
|
||||
note_type=note_type or entity.note_type,
|
||||
total_notes=len(results),
|
||||
total_entities=1,
|
||||
entity_type=entity_type or entity.entity_type,
|
||||
total_notes=1,
|
||||
valid_count=1 if (results and results[0].passed) else 0,
|
||||
warning_count=sum(len(r.warnings) for r in results),
|
||||
error_count=sum(len(r.errors) for r in results),
|
||||
results=results,
|
||||
)
|
||||
|
||||
# --- Batch validation by note type ---
|
||||
entities = await _find_by_note_type(entity_repository, note_type) if note_type else []
|
||||
# --- Batch validation by entity type ---
|
||||
entities = await _find_by_entity_type(entity_repository, entity_type) if entity_type else []
|
||||
|
||||
for entity in entities:
|
||||
frontmatter = _entity_frontmatter(entity)
|
||||
@@ -199,22 +140,21 @@ async def validate_schema(
|
||||
query,
|
||||
allow_reference_match=isinstance(schema_ref, str) and query == schema_ref,
|
||||
)
|
||||
return [await _schema_frontmatter_from_file(file_service, e) for e in entities]
|
||||
return [_entity_frontmatter(e) for e in entities]
|
||||
|
||||
schema_def = await resolve_schema(frontmatter, search_fn)
|
||||
if schema_def:
|
||||
result = validate_note(
|
||||
entity.title or entity.permalink or entity.file_path,
|
||||
entity.permalink or entity.file_path,
|
||||
schema_def,
|
||||
_entity_observations(entity),
|
||||
_entity_relations(entity),
|
||||
frontmatter=frontmatter,
|
||||
)
|
||||
results.append(_to_note_validation_response(result))
|
||||
|
||||
valid = sum(1 for r in results if r.passed)
|
||||
return ValidationReport(
|
||||
note_type=note_type,
|
||||
entity_type=entity_type,
|
||||
total_notes=len(results),
|
||||
total_entities=len(entities),
|
||||
valid_count=valid,
|
||||
@@ -231,7 +171,7 @@ async def validate_schema(
|
||||
async def infer_schema_endpoint(
|
||||
entity_repository: EntityRepositoryV2ExternalDep,
|
||||
project_id: str = Path(..., description="Project external UUID"),
|
||||
note_type: str = Query(..., description="Note type to analyze"),
|
||||
entity_type: str = Query(..., description="Entity type to analyze"),
|
||||
threshold: float = Query(0.25, description="Minimum frequency for optional fields"),
|
||||
):
|
||||
"""Infer a schema from existing notes of a given type.
|
||||
@@ -239,13 +179,13 @@ async def infer_schema_endpoint(
|
||||
Examines observation categories and relation types across all notes
|
||||
of the given type. Returns frequency analysis and suggested Picoschema.
|
||||
"""
|
||||
entities = await _find_by_note_type(entity_repository, note_type)
|
||||
entities = await _find_by_entity_type(entity_repository, entity_type)
|
||||
notes_data = [_entity_to_note_data(entity) for entity in entities]
|
||||
|
||||
result = infer_schema(note_type, notes_data, optional_threshold=threshold)
|
||||
result = infer_schema(entity_type, notes_data, optional_threshold=threshold)
|
||||
|
||||
return InferenceReport(
|
||||
note_type=result.note_type,
|
||||
entity_type=result.entity_type,
|
||||
notes_analyzed=result.notes_analyzed,
|
||||
field_frequencies=[
|
||||
FieldFrequencyResponse(
|
||||
@@ -270,11 +210,10 @@ async def infer_schema_endpoint(
|
||||
# --- Drift Detection ---
|
||||
|
||||
|
||||
@router.get("/schema/diff/{note_type}", response_model=DriftReport)
|
||||
@router.get("/schema/diff/{entity_type}", response_model=DriftReport)
|
||||
async def diff_schema_endpoint(
|
||||
entity_repository: EntityRepositoryV2ExternalDep,
|
||||
file_service: FileServiceV2ExternalDep,
|
||||
note_type: str = Path(..., description="Note type to check for drift"),
|
||||
entity_type: str = Path(..., description="Entity type to check for drift"),
|
||||
project_id: str = Path(..., description="Project external UUID"),
|
||||
):
|
||||
"""Show drift between a schema definition and actual note usage.
|
||||
@@ -286,23 +225,23 @@ async def diff_schema_endpoint(
|
||||
|
||||
async def search_fn(query: str) -> list[dict]:
|
||||
entities = await _find_schema_entities(entity_repository, query)
|
||||
return [await _schema_frontmatter_from_file(file_service, e) for e in entities]
|
||||
return [_entity_frontmatter(e) for e in entities]
|
||||
|
||||
# Resolve schema by note type
|
||||
schema_frontmatter = {"type": note_type}
|
||||
# Resolve schema by entity type
|
||||
schema_frontmatter = {"type": entity_type}
|
||||
schema_def = await resolve_schema(schema_frontmatter, search_fn)
|
||||
|
||||
if not schema_def:
|
||||
return DriftReport(note_type=note_type, schema_found=False)
|
||||
return DriftReport(entity_type=entity_type, schema_found=False)
|
||||
|
||||
# Collect all notes of this type
|
||||
entities = await _find_by_note_type(entity_repository, note_type)
|
||||
entities = await _find_by_entity_type(entity_repository, entity_type)
|
||||
notes_data = [_entity_to_note_data(entity) for entity in entities]
|
||||
|
||||
result = diff_schema(schema_def, notes_data)
|
||||
|
||||
return DriftReport(
|
||||
note_type=note_type,
|
||||
entity_type=entity_type,
|
||||
new_fields=[
|
||||
DriftFieldResponse(
|
||||
name=f.name,
|
||||
@@ -330,19 +269,19 @@ async def diff_schema_endpoint(
|
||||
# --- Helpers ---
|
||||
|
||||
|
||||
async def _find_by_note_type(
|
||||
async def _find_by_entity_type(
|
||||
entity_repository: EntityRepositoryV2ExternalDep,
|
||||
note_type: str,
|
||||
entity_type: str,
|
||||
) -> list[Entity]:
|
||||
"""Find all entities of a given type using the repository's select pattern."""
|
||||
query = entity_repository.select().where(Entity.note_type == note_type)
|
||||
query = entity_repository.select().where(Entity.entity_type == entity_type)
|
||||
result = await entity_repository.execute_query(query)
|
||||
return list(result.scalars().all())
|
||||
|
||||
|
||||
async def _find_schema_entities(
|
||||
entity_repository: EntityRepositoryV2ExternalDep,
|
||||
target_note_type: str,
|
||||
target_entity_type: str,
|
||||
*,
|
||||
allow_reference_match: bool = False,
|
||||
) -> list[Entity]:
|
||||
@@ -354,11 +293,11 @@ async def _find_schema_entities(
|
||||
2) Only when allow_reference_match=True and no entity match was found, try
|
||||
exact reference matching by title/permalink (explicit schema references)
|
||||
"""
|
||||
query = entity_repository.select().where(Entity.note_type == "schema")
|
||||
query = entity_repository.select().where(Entity.entity_type == "schema")
|
||||
result = await entity_repository.execute_query(query)
|
||||
entities = list(result.scalars().all())
|
||||
|
||||
normalized_target = generate_permalink(target_note_type)
|
||||
normalized_target = generate_permalink(target_entity_type)
|
||||
|
||||
entity_matches = [
|
||||
e
|
||||
|
||||
@@ -47,28 +47,21 @@ async def search(
|
||||
Returns:
|
||||
SearchResponse with paginated search results
|
||||
"""
|
||||
limit = page_size
|
||||
offset = (page - 1) * page_size
|
||||
# Fetch one extra item to detect whether more pages exist (N+1 trick)
|
||||
fetch_limit = page_size + 1
|
||||
try:
|
||||
results = await search_service.search(query, limit=fetch_limit, offset=offset)
|
||||
results = await search_service.search(query, limit=limit, offset=offset)
|
||||
except SemanticSearchDisabledError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
except SemanticDependenciesMissingError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
|
||||
has_more = len(results) > page_size
|
||||
if has_more:
|
||||
results = results[:page_size]
|
||||
|
||||
search_results = await to_search_results(entity_service, results)
|
||||
return SearchResponse(
|
||||
results=search_results,
|
||||
current_page=page,
|
||||
page_size=page_size,
|
||||
has_more=has_more,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -146,7 +146,6 @@ async def to_graph_context(
|
||||
metadata=metadata,
|
||||
page=page,
|
||||
page_size=page_size,
|
||||
has_more=context_result.metadata.has_more,
|
||||
)
|
||||
|
||||
|
||||
@@ -177,7 +176,6 @@ async def to_search_results(entity_service: EntityService, results: List[SearchI
|
||||
score=r.score, # pyright: ignore
|
||||
entity=entities[0].permalink if entities else None,
|
||||
content=r.content,
|
||||
matched_chunk=r.matched_chunk_text,
|
||||
file_path=r.file_path,
|
||||
metadata=r.metadata,
|
||||
entity_id=entity_id,
|
||||
|
||||
@@ -1,114 +0,0 @@
|
||||
"""Lightweight CLI analytics via Umami event collector.
|
||||
|
||||
Sends anonymous, non-blocking usage events to help understand how the
|
||||
CLI-to-cloud conversion funnel performs. No PII, no fingerprinting,
|
||||
no cookies. Respects the same opt-out mechanisms as promo messaging.
|
||||
|
||||
Events are fire-and-forget — analytics never blocks or breaks the CLI.
|
||||
|
||||
Defaults point to the Basic Memory Umami Cloud instance. Override via:
|
||||
BASIC_MEMORY_UMAMI_HOST — Custom Umami instance URL
|
||||
BASIC_MEMORY_UMAMI_SITE_ID — Custom Website ID
|
||||
Opt out entirely with BASIC_MEMORY_NO_PROMOS=1.
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import threading
|
||||
import urllib.request
|
||||
from typing import Optional
|
||||
|
||||
import basic_memory
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Configuration — defaults baked in, overridable via environment
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_DEFAULT_UMAMI_HOST = "https://api-gateway.umami.dev"
|
||||
_DEFAULT_UMAMI_SITE_ID = "f6479898-ebaf-4e60-bce2-6dc60a3f6c5c"
|
||||
|
||||
|
||||
def _umami_host() -> Optional[str]:
|
||||
return os.getenv("BASIC_MEMORY_UMAMI_HOST", "").strip() or _DEFAULT_UMAMI_HOST
|
||||
|
||||
|
||||
def _umami_site_id() -> Optional[str]:
|
||||
return os.getenv("BASIC_MEMORY_UMAMI_SITE_ID", "").strip() or _DEFAULT_UMAMI_SITE_ID
|
||||
|
||||
|
||||
def _analytics_disabled() -> bool:
|
||||
"""True when analytics should not fire."""
|
||||
value = os.getenv("BASIC_MEMORY_NO_PROMOS", "").strip().lower()
|
||||
return value in {"1", "true", "yes"}
|
||||
|
||||
|
||||
def _is_configured() -> bool:
|
||||
"""True when both host and site ID are available."""
|
||||
return _umami_host() is not None and _umami_site_id() is not None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Public API
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Well-known event names for the promo/cloud funnel
|
||||
EVENT_PROMO_SHOWN = "cli-promo-shown"
|
||||
EVENT_PROMO_OPTED_OUT = "cli-promo-opted-out"
|
||||
EVENT_CLOUD_LOGIN_STARTED = "cli-cloud-login-started"
|
||||
EVENT_CLOUD_LOGIN_SUCCESS = "cli-cloud-login-success"
|
||||
EVENT_CLOUD_LOGIN_SUB_REQUIRED = "cli-cloud-login-sub-required"
|
||||
|
||||
|
||||
def track(event_name: str, data: Optional[dict] = None) -> None:
|
||||
"""Send an analytics event to Umami. Non-blocking, silent on failure.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
event_name:
|
||||
Short kebab-case name (e.g. "cli-promo-shown").
|
||||
data:
|
||||
Optional dict of event properties (all values should be strings/numbers).
|
||||
"""
|
||||
if _analytics_disabled() or not _is_configured():
|
||||
return
|
||||
|
||||
host = _umami_host()
|
||||
site_id = _umami_site_id()
|
||||
|
||||
# Umami v2 /api/send requires "type" at top level alongside "payload"
|
||||
payload = {
|
||||
"type": "event",
|
||||
"payload": {
|
||||
"hostname": "cli.basicmemory.com",
|
||||
"language": "en",
|
||||
"url": f"/cli/{event_name}",
|
||||
"website": site_id,
|
||||
"name": event_name,
|
||||
"data": {
|
||||
"version": basic_memory.__version__,
|
||||
**(data or {}),
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
def _send():
|
||||
try:
|
||||
req = urllib.request.Request(
|
||||
f"{host}/api/send",
|
||||
data=json.dumps(payload).encode("utf-8"),
|
||||
headers={
|
||||
"Content-Type": "application/json",
|
||||
# Umami's bot detection rejects non-browser User-Agents
|
||||
"User-Agent": "Mozilla/5.0 (compatible; BasicMemoryCLI/"
|
||||
f"{basic_memory.__version__})",
|
||||
},
|
||||
)
|
||||
urllib.request.urlopen(req, timeout=3)
|
||||
except Exception:
|
||||
pass # Never break the CLI for analytics
|
||||
|
||||
# Non-daemon so the process waits for the request to complete.
|
||||
# The 3s urllib timeout caps the worst-case exit delay.
|
||||
t = threading.Thread(target=_send)
|
||||
t.start()
|
||||
@@ -8,6 +8,8 @@ from . import (
|
||||
project,
|
||||
format,
|
||||
schema,
|
||||
watch,
|
||||
workspace,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
@@ -23,4 +25,6 @@ __all__ = [
|
||||
"project",
|
||||
"format",
|
||||
"schema",
|
||||
"watch",
|
||||
"workspace",
|
||||
]
|
||||
|
||||
@@ -6,14 +6,11 @@ from basic_memory.cli.app import cloud_app
|
||||
from basic_memory.cli.commands.cloud.core_commands import * # noqa: F401,F403
|
||||
from basic_memory.cli.commands.cloud.api_client import get_authenticated_headers, get_cloud_config # noqa: F401
|
||||
from basic_memory.cli.commands.cloud.upload_command import * # noqa: F401,F403
|
||||
from basic_memory.cli.commands.cloud.project_sync import * # noqa: F401,F403
|
||||
|
||||
# Register snapshot sub-command group
|
||||
from basic_memory.cli.commands.cloud.snapshot import snapshot_app
|
||||
from basic_memory.cli.commands.cloud.workspace import workspace_app
|
||||
|
||||
cloud_app.add_typer(snapshot_app, name="snapshot")
|
||||
cloud_app.add_typer(workspace_app, name="workspace")
|
||||
|
||||
# Register restore command (directly on cloud_app via decorator)
|
||||
from basic_memory.cli.commands.cloud.restore import restore # noqa: F401, E402
|
||||
|
||||
@@ -52,7 +52,7 @@ async def get_authenticated_headers(auth: CLIAuth | None = None) -> dict[str, st
|
||||
auth_obj = auth or CLIAuth(client_id=client_id, authkit_domain=domain)
|
||||
token = await auth_obj.get_valid_token()
|
||||
if not token:
|
||||
console.print("[red]Not authenticated. Please run 'bm cloud login' first.[/red]")
|
||||
console.print("[red]Not authenticated. Please run 'basic-memory cloud login' first.[/red]")
|
||||
raise typer.Exit(1)
|
||||
|
||||
return {"Authorization": f"Bearer {token}"}
|
||||
|
||||
@@ -6,13 +6,6 @@ from rich.console import Console
|
||||
from basic_memory.cli.app import cloud_app
|
||||
from basic_memory.cli.commands.command_utils import run_with_cleanup
|
||||
from basic_memory.cli.auth import CLIAuth
|
||||
from basic_memory.cli.analytics import (
|
||||
track,
|
||||
EVENT_CLOUD_LOGIN_STARTED,
|
||||
EVENT_CLOUD_LOGIN_SUCCESS,
|
||||
EVENT_CLOUD_LOGIN_SUB_REQUIRED,
|
||||
EVENT_PROMO_OPTED_OUT,
|
||||
)
|
||||
from basic_memory.cli.promo import OSS_DISCOUNT_CODE
|
||||
from basic_memory.config import ConfigManager
|
||||
from basic_memory.cli.commands.cloud.api_client import (
|
||||
@@ -40,7 +33,6 @@ def login():
|
||||
"""Authenticate with WorkOS using OAuth Device Authorization flow."""
|
||||
|
||||
async def _login():
|
||||
track(EVENT_CLOUD_LOGIN_STARTED)
|
||||
client_id, domain, host_url = get_cloud_config()
|
||||
auth = CLIAuth(client_id=client_id, authkit_domain=domain)
|
||||
|
||||
@@ -54,12 +46,10 @@ def login():
|
||||
console.print("[dim]Verifying subscription access...[/dim]")
|
||||
await make_api_request("GET", f"{host_url.rstrip('/')}/proxy/health")
|
||||
|
||||
track(EVENT_CLOUD_LOGIN_SUCCESS)
|
||||
console.print("[green]Cloud authentication successful[/green]")
|
||||
console.print(f"[dim]Cloud host ready: {host_url}[/dim]")
|
||||
|
||||
except SubscriptionRequiredError as e:
|
||||
track(EVENT_CLOUD_LOGIN_SUB_REQUIRED)
|
||||
console.print("\n[red]Subscription Required[/red]\n")
|
||||
console.print(f"[yellow]{e.args[0]}[/yellow]\n")
|
||||
console.print(
|
||||
@@ -85,13 +75,13 @@ def logout():
|
||||
|
||||
@cloud_app.command("status")
|
||||
def status() -> None:
|
||||
"""Check cloud authentication and connection status."""
|
||||
"""Check cloud authentication state and cloud instance health."""
|
||||
config_manager = ConfigManager()
|
||||
config = config_manager.load_config()
|
||||
auth = CLIAuth(client_id=config.cloud_client_id, authkit_domain=config.cloud_domain)
|
||||
tokens = auth.load_tokens()
|
||||
|
||||
console.print("[bold blue]Cloud Status[/bold blue]")
|
||||
console.print("[bold blue]Cloud Authentication Status[/bold blue]")
|
||||
console.print(f" Host: {config.cloud_host}")
|
||||
console.print(
|
||||
f" API Key: {'[green]configured[/green]' if config.cloud_api_key else '[yellow]not set[/yellow]'}"
|
||||
@@ -99,33 +89,51 @@ def status() -> None:
|
||||
|
||||
oauth_status = "[yellow]not logged in[/yellow]"
|
||||
if tokens:
|
||||
if auth.is_token_valid(tokens):
|
||||
oauth_status = "[green]token valid[/green]"
|
||||
else:
|
||||
oauth_status = "[yellow]token expired[/yellow]"
|
||||
oauth_status = (
|
||||
"[green]token valid[/green]"
|
||||
if auth.is_token_valid(tokens)
|
||||
else "[yellow]token expired[/yellow]"
|
||||
)
|
||||
console.print(f" OAuth: {oauth_status}")
|
||||
|
||||
# Get cloud configuration
|
||||
_, _, host_url = get_cloud_config()
|
||||
host_url = host_url.rstrip("/")
|
||||
|
||||
has_credentials = bool(config.cloud_api_key) or tokens is not None
|
||||
if not has_credentials:
|
||||
console.print(
|
||||
"\n[dim]No cloud credentials found. Run: bm cloud login or bm cloud api-key save <key>[/dim]"
|
||||
"\n[dim]No cloud credentials found. Run: bm cloud login or bm cloud set-key <key>[/dim]"
|
||||
)
|
||||
return
|
||||
|
||||
# Quick connection check — just verify we can reach the cloud
|
||||
_, _, host_url = get_cloud_config()
|
||||
host_url = host_url.rstrip("/")
|
||||
|
||||
try:
|
||||
run_with_cleanup(make_api_request(method="GET", url=f"{host_url}/proxy/health"))
|
||||
console.print("\n[green]Cloud connected[/green]")
|
||||
except CloudAPIError:
|
||||
console.print("\n[yellow]Cloud not connected[/yellow]")
|
||||
console.print("\n[blue]Checking cloud instance health...[/blue]")
|
||||
|
||||
# Make API request to check health
|
||||
response = run_with_cleanup(make_api_request(method="GET", url=f"{host_url}/proxy/health"))
|
||||
|
||||
health_data = response.json()
|
||||
|
||||
console.print("[green]Cloud instance is healthy[/green]")
|
||||
|
||||
# Display status details
|
||||
if "status" in health_data:
|
||||
console.print(f" Status: {health_data['status']}")
|
||||
if "version" in health_data:
|
||||
console.print(f" Version: {health_data['version']}")
|
||||
if "timestamp" in health_data:
|
||||
console.print(f" Timestamp: {health_data['timestamp']}")
|
||||
|
||||
console.print("\n[dim]To sync projects, use: bm project bisync --name <project>[/dim]")
|
||||
|
||||
except CloudAPIError as e:
|
||||
console.print(f"[yellow]Cloud health check failed: {e}[/yellow]")
|
||||
console.print(
|
||||
"[dim]Try re-authenticating with 'bm cloud login' or 'bm cloud api-key save'.[/dim]"
|
||||
"[dim]Try re-authenticating with 'bm cloud login' or setting API key with 'bm cloud set-key'.[/dim]"
|
||||
)
|
||||
except Exception:
|
||||
console.print("\n[yellow]Cloud not connected[/yellow]")
|
||||
except Exception as e:
|
||||
console.print(f"[yellow]Unexpected health check error: {e}[/yellow]")
|
||||
|
||||
|
||||
@cloud_app.command("setup")
|
||||
@@ -133,7 +141,7 @@ def setup() -> None:
|
||||
"""Set up cloud sync by installing rclone and configuring credentials.
|
||||
|
||||
After setup, use project commands for syncing:
|
||||
bm project add <name> --cloud --local-path ~/projects/<name>
|
||||
bm project add <name> <path> --local-path ~/projects/<name>
|
||||
bm project bisync --name <name> --resync # First time
|
||||
bm project bisync --name <name> # Subsequent syncs
|
||||
"""
|
||||
@@ -165,7 +173,7 @@ def setup() -> None:
|
||||
console.print("\n[bold green]Cloud setup completed successfully![/bold green]")
|
||||
console.print("\n[bold]Next steps:[/bold]")
|
||||
console.print("1. Add a project with local sync path:")
|
||||
console.print(" bm project add research --cloud --local-path ~/Documents/research")
|
||||
console.print(" bm project add research --local-path ~/Documents/research")
|
||||
console.print("\n Or configure sync for an existing project:")
|
||||
console.print(" bm project sync-setup research ~/Documents/research")
|
||||
console.print("\n2. Preview the initial sync (recommended):")
|
||||
@@ -197,26 +205,20 @@ def promo(enabled: bool = typer.Option(True, "--on/--off", help="Enable or disab
|
||||
if enabled:
|
||||
console.print("[green]Cloud promo messages enabled[/green]")
|
||||
else:
|
||||
track(EVENT_PROMO_OPTED_OUT)
|
||||
console.print("[yellow]Cloud promo messages disabled[/yellow]")
|
||||
|
||||
|
||||
# --- API key management subcommand group ---
|
||||
|
||||
api_key_app = typer.Typer(help="Manage cloud API keys")
|
||||
cloud_app.add_typer(api_key_app, name="api-key")
|
||||
|
||||
|
||||
@api_key_app.command("save")
|
||||
def api_key_save(
|
||||
@cloud_app.command("set-key")
|
||||
def set_key(
|
||||
api_key: str = typer.Argument(..., help="API key (bmc_ prefixed) for cloud access"),
|
||||
) -> None:
|
||||
"""Save an existing API key to local config.
|
||||
"""Save a cloud API key for per-project cloud routing.
|
||||
|
||||
Use when you already have an API key (e.g., from the web app).
|
||||
The API key is account-level and used by projects set to cloud mode.
|
||||
Create a key in the web app or use 'bm cloud create-key'.
|
||||
|
||||
Example:
|
||||
bm cloud api-key save bmc_abc123...
|
||||
bm cloud set-key bmc_abc123...
|
||||
"""
|
||||
if not api_key.startswith("bmc_"):
|
||||
console.print("[red]Error: API key must start with 'bmc_'[/red]")
|
||||
@@ -232,16 +234,17 @@ def api_key_save(
|
||||
console.print("[dim]Set a project to cloud mode: bm project set-cloud <name>[/dim]")
|
||||
|
||||
|
||||
@api_key_app.command("create")
|
||||
def api_key_create(
|
||||
@cloud_app.command("create-key")
|
||||
def create_key(
|
||||
name: str = typer.Argument(..., help="Human-readable name for the API key"),
|
||||
) -> None:
|
||||
"""Create a new API key via the cloud API and save it locally.
|
||||
"""Create a new cloud API key and save it locally.
|
||||
|
||||
Requires active OAuth session (run 'bm cloud login' first).
|
||||
The key is created via the cloud API and saved to local config.
|
||||
|
||||
Example:
|
||||
bm cloud api-key create "my-laptop"
|
||||
bm cloud create-key "my-laptop"
|
||||
"""
|
||||
|
||||
async def _create_key():
|
||||
|
||||
@@ -1,372 +0,0 @@
|
||||
"""Cloud sync commands for Basic Memory projects.
|
||||
|
||||
Commands for syncing, bisyncing, and checking integrity between local and cloud
|
||||
project instances. These were previously in project.py but belong here since
|
||||
they are cloud-specific operations.
|
||||
"""
|
||||
|
||||
import os
|
||||
from datetime import datetime
|
||||
|
||||
import typer
|
||||
from rich.console import Console
|
||||
|
||||
from basic_memory.cli.app import cloud_app
|
||||
from basic_memory.cli.commands.cloud.bisync_commands import get_mount_info
|
||||
from basic_memory.cli.commands.cloud.rclone_commands import (
|
||||
RcloneError,
|
||||
SyncProject,
|
||||
get_project_bisync_state,
|
||||
project_bisync,
|
||||
project_check,
|
||||
project_sync,
|
||||
)
|
||||
from basic_memory.cli.commands.command_utils import run_with_cleanup
|
||||
from basic_memory.cli.commands.routing import force_routing
|
||||
from basic_memory.config import ConfigManager, ProjectEntry
|
||||
from basic_memory.mcp.async_client import get_client
|
||||
from basic_memory.mcp.clients import ProjectClient
|
||||
from basic_memory.schemas.project_info import ProjectItem
|
||||
from basic_memory.utils import generate_permalink, normalize_project_path
|
||||
|
||||
console = Console()
|
||||
|
||||
|
||||
# --- Shared helpers ---
|
||||
|
||||
|
||||
def _has_cloud_credentials(config) -> bool:
|
||||
"""Return whether cloud credentials are available (API key or OAuth token)."""
|
||||
from basic_memory.config import has_cloud_credentials
|
||||
|
||||
return has_cloud_credentials(config)
|
||||
|
||||
|
||||
def _require_cloud_credentials(config) -> None:
|
||||
"""Exit with actionable guidance when cloud credentials are missing."""
|
||||
if _has_cloud_credentials(config):
|
||||
return
|
||||
|
||||
console.print("[red]Error: cloud credentials are required for this command[/red]")
|
||||
console.print("[dim]Run 'bm cloud login' or 'bm cloud api-key save <key>' first[/dim]")
|
||||
raise typer.Exit(1)
|
||||
|
||||
|
||||
async def _get_cloud_project(name: str) -> ProjectItem | None:
|
||||
"""Fetch a project by name from the cloud API."""
|
||||
async with get_client() as client:
|
||||
projects_list = await ProjectClient(client).list_projects()
|
||||
for proj in projects_list.projects:
|
||||
if generate_permalink(proj.name) == generate_permalink(name):
|
||||
return proj
|
||||
return None
|
||||
|
||||
|
||||
def _get_sync_project(
|
||||
name: str, config, project_data: ProjectItem
|
||||
) -> tuple[SyncProject, str | None]:
|
||||
"""Build a SyncProject and resolve local_sync_path from config.
|
||||
|
||||
Returns (sync_project, local_sync_path). Exits if no local_sync_path configured.
|
||||
"""
|
||||
sync_entry = config.projects.get(name)
|
||||
# Support both new (path) and legacy (local_sync_path) configs
|
||||
local_sync_path = (sync_entry.local_sync_path or sync_entry.path) if sync_entry else None
|
||||
|
||||
if not local_sync_path or not os.path.isabs(local_sync_path):
|
||||
console.print(f"[red]Error: Project '{name}' has no local sync path configured[/red]")
|
||||
console.print(f"\nConfigure sync with: bm cloud sync-setup {name} ~/path/to/local")
|
||||
raise typer.Exit(1)
|
||||
|
||||
sync_project = SyncProject(
|
||||
name=project_data.name,
|
||||
path=normalize_project_path(project_data.path),
|
||||
local_sync_path=local_sync_path,
|
||||
)
|
||||
return sync_project, local_sync_path
|
||||
|
||||
|
||||
# --- Commands ---
|
||||
|
||||
|
||||
@cloud_app.command("sync")
|
||||
def sync_project_command(
|
||||
name: str = typer.Option(..., "--name", help="Project name to sync"),
|
||||
dry_run: bool = typer.Option(False, "--dry-run", help="Preview changes without syncing"),
|
||||
verbose: bool = typer.Option(False, "--verbose", "-v", help="Show detailed output"),
|
||||
) -> None:
|
||||
"""One-way sync: local -> cloud (make cloud identical to local).
|
||||
|
||||
Example:
|
||||
bm cloud sync --name research
|
||||
bm cloud sync --name research --dry-run
|
||||
"""
|
||||
config = ConfigManager().config
|
||||
_require_cloud_credentials(config)
|
||||
|
||||
try:
|
||||
# Get tenant info for bucket name
|
||||
tenant_info = run_with_cleanup(get_mount_info())
|
||||
bucket_name = tenant_info.bucket_name
|
||||
|
||||
# Get project info
|
||||
with force_routing(cloud=True):
|
||||
project_data = run_with_cleanup(_get_cloud_project(name))
|
||||
if not project_data:
|
||||
console.print(f"[red]Error: Project '{name}' not found[/red]")
|
||||
raise typer.Exit(1)
|
||||
|
||||
sync_project, local_sync_path = _get_sync_project(name, config, project_data)
|
||||
|
||||
# Run sync
|
||||
console.print(f"[blue]Syncing {name} (local -> cloud)...[/blue]")
|
||||
success = project_sync(sync_project, bucket_name, dry_run=dry_run, verbose=verbose)
|
||||
|
||||
if success:
|
||||
console.print(f"[green]{name} synced successfully[/green]")
|
||||
|
||||
# Trigger database sync if not a dry run
|
||||
if not dry_run:
|
||||
|
||||
async def _trigger_db_sync():
|
||||
async with get_client() as client:
|
||||
return await ProjectClient(client).sync(
|
||||
project_data.external_id, force_full=True
|
||||
)
|
||||
|
||||
try:
|
||||
with force_routing(cloud=True):
|
||||
result = run_with_cleanup(_trigger_db_sync())
|
||||
console.print(f"[dim]Database sync initiated: {result.get('message')}[/dim]")
|
||||
except Exception as e:
|
||||
console.print(f"[yellow]Warning: Could not trigger database sync: {e}[/yellow]")
|
||||
else:
|
||||
console.print(f"[red]{name} sync failed[/red]")
|
||||
raise typer.Exit(1)
|
||||
|
||||
except RcloneError as e:
|
||||
console.print(f"[red]Sync error: {e}[/red]")
|
||||
raise typer.Exit(1)
|
||||
except Exception as e:
|
||||
console.print(f"[red]Error: {e}[/red]")
|
||||
raise typer.Exit(1)
|
||||
|
||||
|
||||
@cloud_app.command("bisync")
|
||||
def bisync_project_command(
|
||||
name: str = typer.Option(..., "--name", help="Project name to bisync"),
|
||||
dry_run: bool = typer.Option(False, "--dry-run", help="Preview changes without syncing"),
|
||||
resync: bool = typer.Option(False, "--resync", help="Force new baseline"),
|
||||
verbose: bool = typer.Option(False, "--verbose", "-v", help="Show detailed output"),
|
||||
) -> None:
|
||||
"""Two-way sync: local <-> cloud (bidirectional sync).
|
||||
|
||||
Examples:
|
||||
bm cloud bisync --name research --resync # First time
|
||||
bm cloud bisync --name research # Subsequent syncs
|
||||
bm cloud bisync --name research --dry-run # Preview changes
|
||||
"""
|
||||
config = ConfigManager().config
|
||||
_require_cloud_credentials(config)
|
||||
|
||||
try:
|
||||
# Get tenant info for bucket name
|
||||
tenant_info = run_with_cleanup(get_mount_info())
|
||||
bucket_name = tenant_info.bucket_name
|
||||
|
||||
# Get project info
|
||||
with force_routing(cloud=True):
|
||||
project_data = run_with_cleanup(_get_cloud_project(name))
|
||||
if not project_data:
|
||||
console.print(f"[red]Error: Project '{name}' not found[/red]")
|
||||
raise typer.Exit(1)
|
||||
|
||||
sync_project, local_sync_path = _get_sync_project(name, config, project_data)
|
||||
|
||||
# Run bisync
|
||||
console.print(f"[blue]Bisync {name} (local <-> cloud)...[/blue]")
|
||||
success = project_bisync(
|
||||
sync_project, bucket_name, dry_run=dry_run, resync=resync, verbose=verbose
|
||||
)
|
||||
|
||||
if success:
|
||||
console.print(f"[green]{name} bisync completed successfully[/green]")
|
||||
|
||||
# Update config — sync_entry is guaranteed non-None because
|
||||
# _get_sync_project validated local_sync_path (which comes from sync_entry)
|
||||
sync_entry = config.projects.get(name)
|
||||
assert sync_entry is not None
|
||||
sync_entry.last_sync = datetime.now()
|
||||
sync_entry.bisync_initialized = True
|
||||
ConfigManager().save_config(config)
|
||||
|
||||
# Trigger database sync if not a dry run
|
||||
if not dry_run:
|
||||
|
||||
async def _trigger_db_sync():
|
||||
async with get_client() as client:
|
||||
return await ProjectClient(client).sync(
|
||||
project_data.external_id, force_full=True
|
||||
)
|
||||
|
||||
try:
|
||||
with force_routing(cloud=True):
|
||||
result = run_with_cleanup(_trigger_db_sync())
|
||||
console.print(f"[dim]Database sync initiated: {result.get('message')}[/dim]")
|
||||
except Exception as e:
|
||||
console.print(f"[yellow]Warning: Could not trigger database sync: {e}[/yellow]")
|
||||
else:
|
||||
console.print(f"[red]{name} bisync failed[/red]")
|
||||
raise typer.Exit(1)
|
||||
|
||||
except RcloneError as e:
|
||||
console.print(f"[red]Bisync error: {e}[/red]")
|
||||
raise typer.Exit(1)
|
||||
except Exception as e:
|
||||
console.print(f"[red]Error: {e}[/red]")
|
||||
raise typer.Exit(1)
|
||||
|
||||
|
||||
@cloud_app.command("check")
|
||||
def check_project_command(
|
||||
name: str = typer.Option(..., "--name", help="Project name to check"),
|
||||
one_way: bool = typer.Option(False, "--one-way", help="Check one direction only (faster)"),
|
||||
) -> None:
|
||||
"""Verify file integrity between local and cloud.
|
||||
|
||||
Example:
|
||||
bm cloud check --name research
|
||||
"""
|
||||
config = ConfigManager().config
|
||||
_require_cloud_credentials(config)
|
||||
|
||||
try:
|
||||
# Get tenant info for bucket name
|
||||
tenant_info = run_with_cleanup(get_mount_info())
|
||||
bucket_name = tenant_info.bucket_name
|
||||
|
||||
# Get project info
|
||||
with force_routing(cloud=True):
|
||||
project_data = run_with_cleanup(_get_cloud_project(name))
|
||||
if not project_data:
|
||||
console.print(f"[red]Error: Project '{name}' not found[/red]")
|
||||
raise typer.Exit(1)
|
||||
|
||||
sync_project, local_sync_path = _get_sync_project(name, config, project_data)
|
||||
|
||||
# Run check
|
||||
console.print(f"[blue]Checking {name} integrity...[/blue]")
|
||||
match = project_check(sync_project, bucket_name, one_way=one_way)
|
||||
|
||||
if match:
|
||||
console.print(f"[green]{name} files match[/green]")
|
||||
else:
|
||||
console.print(f"[yellow]!{name} has differences[/yellow]")
|
||||
|
||||
except RcloneError as e:
|
||||
console.print(f"[red]Check error: {e}[/red]")
|
||||
raise typer.Exit(1)
|
||||
except Exception as e:
|
||||
console.print(f"[red]Error: {e}[/red]")
|
||||
raise typer.Exit(1)
|
||||
|
||||
|
||||
@cloud_app.command("bisync-reset")
|
||||
def bisync_reset(
|
||||
name: str = typer.Argument(..., help="Project name to reset bisync state for"),
|
||||
) -> None:
|
||||
"""Clear bisync state for a project.
|
||||
|
||||
This removes the bisync metadata files, forcing a fresh --resync on next bisync.
|
||||
Useful when bisync gets into an inconsistent state or when remote path changes.
|
||||
"""
|
||||
import shutil
|
||||
|
||||
try:
|
||||
state_path = get_project_bisync_state(name)
|
||||
|
||||
if not state_path.exists():
|
||||
console.print(f"[yellow]No bisync state found for project '{name}'[/yellow]")
|
||||
return
|
||||
|
||||
# Remove the entire state directory
|
||||
shutil.rmtree(state_path)
|
||||
console.print(f"[green]Cleared bisync state for project '{name}'[/green]")
|
||||
console.print("\nNext steps:")
|
||||
console.print(f" 1. Preview: bm cloud bisync --name {name} --resync --dry-run")
|
||||
console.print(f" 2. Sync: bm cloud bisync --name {name} --resync")
|
||||
|
||||
except Exception as e:
|
||||
console.print(f"[red]Error clearing bisync state: {str(e)}[/red]")
|
||||
raise typer.Exit(1)
|
||||
|
||||
|
||||
@cloud_app.command("sync-setup")
|
||||
def setup_project_sync(
|
||||
name: str = typer.Argument(..., help="Project name"),
|
||||
local_path: str = typer.Argument(..., help="Local sync directory"),
|
||||
) -> None:
|
||||
"""Configure local sync for an existing cloud project.
|
||||
|
||||
Example:
|
||||
bm cloud sync-setup research ~/Documents/research
|
||||
"""
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
config_manager = ConfigManager()
|
||||
config = config_manager.config
|
||||
_require_cloud_credentials(config)
|
||||
|
||||
async def _verify_project_exists():
|
||||
"""Verify the project exists on cloud by listing all projects."""
|
||||
async with get_client() as client:
|
||||
projects_list = await ProjectClient(client).list_projects()
|
||||
project_names = [p.name for p in projects_list.projects]
|
||||
if name not in project_names:
|
||||
raise ValueError(f"Project '{name}' not found on cloud")
|
||||
return True
|
||||
|
||||
try:
|
||||
# Verify project exists on cloud
|
||||
with force_routing(cloud=True):
|
||||
run_with_cleanup(_verify_project_exists())
|
||||
|
||||
# Resolve and create local path
|
||||
resolved_path = Path(os.path.abspath(os.path.expanduser(local_path)))
|
||||
resolved_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Update project entry with sync path — path is always the local directory
|
||||
entry = config.projects.get(name)
|
||||
if entry:
|
||||
entry.path = resolved_path.as_posix()
|
||||
entry.local_sync_path = resolved_path.as_posix()
|
||||
entry.bisync_initialized = False
|
||||
entry.last_sync = None
|
||||
else:
|
||||
config.projects[name] = ProjectEntry(
|
||||
path=resolved_path.as_posix(),
|
||||
local_sync_path=resolved_path.as_posix(),
|
||||
)
|
||||
config_manager.save_config(config)
|
||||
|
||||
# Create the project in the local DB so the MCP server can immediately use it
|
||||
async def _create_local_project():
|
||||
async with get_client() as client:
|
||||
data = {"name": name, "path": resolved_path.as_posix(), "set_default": False}
|
||||
return await ProjectClient(client).create_project(data)
|
||||
|
||||
with force_routing(local=True):
|
||||
try:
|
||||
run_with_cleanup(_create_local_project())
|
||||
except Exception:
|
||||
pass # Project may already exist locally; reconcile on next startup
|
||||
|
||||
console.print(f"[green]Sync configured for project '{name}'[/green]")
|
||||
console.print(f"\nLocal sync path: {resolved_path}")
|
||||
console.print("\nNext steps:")
|
||||
console.print(f" 1. Preview: bm cloud bisync --name {name} --resync --dry-run")
|
||||
console.print(f" 2. Sync: bm cloud bisync --name {name} --resync")
|
||||
except Exception as e:
|
||||
console.print(f"[red]Error configuring sync: {str(e)}[/red]")
|
||||
raise typer.Exit(1)
|
||||
@@ -223,9 +223,6 @@ def project_sync(
|
||||
*TIGRIS_CONSISTENCY_HEADERS,
|
||||
"--filter-from",
|
||||
str(filter_path),
|
||||
# Prevent NUL byte padding on virtual filesystems (e.g. Google Drive File Stream)
|
||||
# See: rclone/rclone#6801
|
||||
"--local-no-preallocate",
|
||||
]
|
||||
|
||||
if verbose:
|
||||
@@ -302,9 +299,6 @@ def project_bisync(
|
||||
str(filter_path),
|
||||
"--workdir",
|
||||
str(state_path),
|
||||
# Prevent NUL byte padding on virtual filesystems (e.g. Google Drive File Stream)
|
||||
# See: rclone/rclone#6801
|
||||
"--local-no-preallocate",
|
||||
]
|
||||
|
||||
# Add --create-empty-src-dirs if rclone version supports it (v1.64+)
|
||||
|
||||
@@ -10,6 +10,7 @@ import httpx
|
||||
|
||||
from basic_memory.ignore_utils import load_gitignore_patterns, should_ignore_path
|
||||
from basic_memory.mcp.async_client import get_client
|
||||
from basic_memory.mcp.tools.utils import call_put
|
||||
|
||||
# Archive file extensions that should be skipped during upload
|
||||
ARCHIVE_EXTENSIONS = {".zip", ".tar", ".gz", ".bz2", ".xz", ".7z", ".rar", ".tgz", ".tbz2"}
|
||||
@@ -23,7 +24,7 @@ async def upload_path(
|
||||
dry_run: bool = False,
|
||||
*,
|
||||
client_cm_factory: Callable[[], AbstractAsyncContextManager[httpx.AsyncClient]] | None = None,
|
||||
put_func: Callable | None = None,
|
||||
put_func=call_put,
|
||||
) -> bool:
|
||||
"""
|
||||
Upload a file or directory to cloud project via WebDAV.
|
||||
@@ -116,20 +117,9 @@ async def upload_path(
|
||||
|
||||
# Upload via HTTP PUT to WebDAV endpoint with mtime header
|
||||
# Using X-OC-Mtime (ownCloud/Nextcloud standard)
|
||||
if put_func is not None:
|
||||
# Test injection path
|
||||
response = await put_func(
|
||||
client,
|
||||
remote_path,
|
||||
content=content,
|
||||
headers={"X-OC-Mtime": str(mtime)},
|
||||
)
|
||||
else:
|
||||
response = await client.put(
|
||||
remote_path,
|
||||
content=content,
|
||||
headers={"X-OC-Mtime": str(mtime)},
|
||||
)
|
||||
response = await put_func(
|
||||
client, remote_path, content=content, headers={"X-OC-Mtime": str(mtime)}
|
||||
)
|
||||
response.raise_for_status()
|
||||
|
||||
# Format total size based on magnitude
|
||||
|
||||
@@ -13,7 +13,6 @@ from basic_memory.cli.commands.cloud.cloud_utils import (
|
||||
sync_project,
|
||||
)
|
||||
from basic_memory.cli.commands.cloud.upload import upload_path
|
||||
from basic_memory.mcp.async_client import get_cloud_control_plane_client
|
||||
|
||||
console = Console()
|
||||
|
||||
@@ -87,7 +86,7 @@ def upload(
|
||||
console.print(
|
||||
f"[red]Project '{project}' does not exist.[/red]\n"
|
||||
f"[yellow]Options:[/yellow]\n"
|
||||
f" 1. Create it first: bm project add {project} --cloud\n"
|
||||
f" 1. Create it first: bm project add {project}\n"
|
||||
f" 2. Use --create-project flag to create automatically"
|
||||
)
|
||||
raise typer.Exit(1)
|
||||
@@ -101,12 +100,7 @@ def upload(
|
||||
console.print(f"[blue]Uploading {path} to project '{project}'...[/blue]")
|
||||
|
||||
success = await upload_path(
|
||||
path,
|
||||
project,
|
||||
verbose=verbose,
|
||||
use_gitignore=not no_gitignore,
|
||||
dry_run=dry_run,
|
||||
client_cm_factory=get_cloud_control_plane_client,
|
||||
path, project, verbose=verbose, use_gitignore=not no_gitignore, dry_run=dry_run
|
||||
)
|
||||
if not success:
|
||||
console.print("[red]Upload failed[/red]")
|
||||
|
||||
@@ -1,113 +0,0 @@
|
||||
"""Workspace commands for Basic Memory cloud workspaces."""
|
||||
|
||||
import typer
|
||||
from rich.console import Console
|
||||
from rich.table import Table
|
||||
|
||||
from basic_memory.cli.commands.command_utils import run_with_cleanup
|
||||
from basic_memory.config import ConfigManager
|
||||
from basic_memory.mcp.project_context import (
|
||||
_workspace_choices,
|
||||
_workspace_matches_identifier,
|
||||
get_available_workspaces,
|
||||
)
|
||||
|
||||
console = Console()
|
||||
|
||||
workspace_app = typer.Typer(help="Manage cloud workspaces")
|
||||
|
||||
|
||||
@workspace_app.command("list")
|
||||
def list_workspaces() -> None:
|
||||
"""List cloud workspaces available to the current OAuth session."""
|
||||
|
||||
async def _list():
|
||||
return await get_available_workspaces()
|
||||
|
||||
try:
|
||||
workspaces = run_with_cleanup(_list())
|
||||
except RuntimeError as exc:
|
||||
console.print(f"[red]Error: {exc}[/red]")
|
||||
raise typer.Exit(1)
|
||||
except Exception as exc: # pragma: no cover
|
||||
console.print(f"[red]Error listing workspaces: {exc}[/red]")
|
||||
raise typer.Exit(1)
|
||||
|
||||
if not workspaces:
|
||||
console.print("[yellow]No accessible workspaces found.[/yellow]")
|
||||
return
|
||||
|
||||
config = ConfigManager().config
|
||||
default_ws = config.default_workspace
|
||||
|
||||
table = Table(title="Available Workspaces")
|
||||
table.add_column("Name", style="cyan")
|
||||
table.add_column("Type", style="blue")
|
||||
table.add_column("Role", style="green")
|
||||
table.add_column("Tenant ID", style="yellow")
|
||||
table.add_column("Default", style="magenta")
|
||||
|
||||
for workspace in workspaces:
|
||||
is_default = "[X]" if workspace.tenant_id == default_ws else ""
|
||||
table.add_row(
|
||||
workspace.name,
|
||||
workspace.workspace_type,
|
||||
workspace.role,
|
||||
workspace.tenant_id,
|
||||
is_default,
|
||||
)
|
||||
|
||||
console.print(table)
|
||||
|
||||
|
||||
@workspace_app.command("set-default")
|
||||
def set_default_workspace(
|
||||
identifier: str = typer.Argument(..., help="Workspace name or tenant_id to set as default"),
|
||||
) -> None:
|
||||
"""Set the default cloud workspace.
|
||||
|
||||
The default workspace is used as fallback when no per-project workspace
|
||||
is configured. Resolves the identifier against available workspaces.
|
||||
|
||||
Examples:
|
||||
bm cloud workspace set-default Personal
|
||||
bm cloud workspace set-default 11111111-1111-1111-1111-111111111111
|
||||
"""
|
||||
|
||||
async def _list():
|
||||
return await get_available_workspaces()
|
||||
|
||||
try:
|
||||
workspaces = run_with_cleanup(_list())
|
||||
except RuntimeError as exc:
|
||||
console.print(f"[red]Error: {exc}[/red]")
|
||||
raise typer.Exit(1)
|
||||
|
||||
if not workspaces:
|
||||
console.print("[yellow]No accessible workspaces found.[/yellow]")
|
||||
raise typer.Exit(1)
|
||||
|
||||
matches = [ws for ws in workspaces if _workspace_matches_identifier(ws, identifier)]
|
||||
|
||||
if not matches:
|
||||
console.print(f"[red]Error: Workspace '{identifier}' not found[/red]")
|
||||
console.print(f"[dim]Available:\n{_workspace_choices(workspaces)}[/dim]")
|
||||
raise typer.Exit(1)
|
||||
|
||||
if len(matches) > 1:
|
||||
console.print(
|
||||
f"[red]Error: Workspace name '{identifier}' matches multiple workspaces. "
|
||||
f"Use tenant_id instead.[/red]"
|
||||
)
|
||||
console.print(f"[dim]Available:\n{_workspace_choices(workspaces)}[/dim]")
|
||||
raise typer.Exit(1)
|
||||
|
||||
selected = matches[0]
|
||||
config_manager = ConfigManager()
|
||||
config = config_manager.config
|
||||
config.default_workspace = selected.tenant_id
|
||||
config_manager.save_config(config)
|
||||
|
||||
console.print(
|
||||
f"[green]Default workspace set to '{selected.name}' ({selected.tenant_id})[/green]"
|
||||
)
|
||||
@@ -11,8 +11,9 @@ from rich.console import Console
|
||||
from basic_memory import db
|
||||
from basic_memory.config import ConfigManager
|
||||
from basic_memory.mcp.async_client import get_client
|
||||
from basic_memory.mcp.clients import ProjectClient
|
||||
from basic_memory.mcp.tools.utils import call_post, call_get
|
||||
from basic_memory.mcp.project_context import get_active_project
|
||||
from basic_memory.schemas import ProjectInfoResponse
|
||||
|
||||
console = Console()
|
||||
|
||||
@@ -60,12 +61,16 @@ async def run_sync(
|
||||
try:
|
||||
async with get_client(project_name=project) as client:
|
||||
project_item = await get_active_project(client, project, None)
|
||||
project_client = ProjectClient(client)
|
||||
data = await project_client.sync(
|
||||
project_item.external_id,
|
||||
force_full=force_full,
|
||||
run_in_background=run_in_background,
|
||||
)
|
||||
url = f"/v2/projects/{project_item.external_id}/sync"
|
||||
params = []
|
||||
if force_full:
|
||||
params.append("force_full=true")
|
||||
if not run_in_background:
|
||||
params.append("run_in_background=false")
|
||||
if params:
|
||||
url += "?" + "&".join(params)
|
||||
response = await call_post(client, url)
|
||||
data = response.json()
|
||||
# Background mode returns {"message": "..."}, foreground returns SyncReportResponse
|
||||
if "message" in data:
|
||||
console.print(f"[green]{data['message']}[/green]")
|
||||
@@ -89,21 +94,8 @@ async def get_project_info(project: str):
|
||||
try:
|
||||
async with get_client(project_name=project) as client:
|
||||
project_item = await get_active_project(client, project, None)
|
||||
return await ProjectClient(client).get_info(project_item.external_id)
|
||||
response = await call_get(client, f"/v2/projects/{project_item.external_id}/info")
|
||||
return ProjectInfoResponse.model_validate(response.json())
|
||||
except (ToolError, ValueError) as e:
|
||||
error_text = str(e)
|
||||
if "internal proxy error" in error_text.lower() and "not found in configuration" in (
|
||||
error_text.lower()
|
||||
):
|
||||
console.print(
|
||||
"[red]Project info failed: cloud returned an internal configuration error for "
|
||||
"this project.[/red]"
|
||||
)
|
||||
console.print(
|
||||
"[yellow]This is a cloud backend issue for detailed info lookups. "
|
||||
"Use `bm project list --cloud` for project metadata until the service is updated."
|
||||
"[/yellow]"
|
||||
)
|
||||
else:
|
||||
console.print(f"[red]Project info failed: {e}[/red]")
|
||||
console.print(f"[red]Sync failed: {e}[/red]")
|
||||
raise typer.Exit(1)
|
||||
|
||||
@@ -11,7 +11,7 @@ from sqlalchemy.exc import OperationalError
|
||||
from basic_memory import db
|
||||
from basic_memory.cli.app import app
|
||||
from basic_memory.cli.commands.command_utils import run_with_cleanup
|
||||
from basic_memory.config import ConfigManager, ProjectMode
|
||||
from basic_memory.config import ConfigManager
|
||||
from basic_memory.repository import ProjectRepository
|
||||
from basic_memory.services.initialization import reconcile_projects_with_config
|
||||
from basic_memory.sync.sync_service import get_sync_service
|
||||
@@ -169,16 +169,7 @@ async def _reindex(app_config, search: bool, embeddings: bool, project: str | No
|
||||
if project:
|
||||
projects = [p for p in projects if p.name == project]
|
||||
if not projects:
|
||||
# Check if it's a cloud-only project — those can't be reindexed locally
|
||||
project_mode = app_config.get_project_mode(project)
|
||||
if project_mode == ProjectMode.CLOUD:
|
||||
console.print(
|
||||
f"[yellow]Project '{project}' is a cloud project.[/yellow]\n"
|
||||
"Reindexing is a local operation — cloud projects are "
|
||||
"indexed on the server."
|
||||
)
|
||||
else:
|
||||
console.print(f"[red]Project '{project}' not found.[/red]")
|
||||
console.print(f"[red]Project '{project}' not found.[/red]")
|
||||
raise typer.Exit(1)
|
||||
|
||||
for proj in projects:
|
||||
|
||||
@@ -19,6 +19,7 @@ from basic_memory.markdown.markdown_processor import MarkdownProcessor
|
||||
from basic_memory.markdown.schemas import EntityFrontmatter, EntityMarkdown
|
||||
from basic_memory.mcp.async_client import get_client
|
||||
from basic_memory.mcp.clients import KnowledgeClient, ProjectClient, SearchClient
|
||||
from basic_memory.mcp.tools.utils import call_post
|
||||
from basic_memory.schemas.base import Entity
|
||||
from basic_memory.schemas.project_info import ProjectInfoRequest
|
||||
from basic_memory.schemas.search import SearchQuery
|
||||
@@ -61,7 +62,7 @@ async def run_doctor() -> None:
|
||||
api_note = Entity(
|
||||
title=api_note_title,
|
||||
directory="doctor",
|
||||
note_type="note",
|
||||
entity_type="note",
|
||||
content_type="text/markdown",
|
||||
content=f"# {api_note_title}\n\n- [note] API to file check",
|
||||
entity_metadata={"tags": ["doctor"]},
|
||||
@@ -97,10 +98,11 @@ async def run_doctor() -> None:
|
||||
await processor.write_file(manual_path, manual_markdown)
|
||||
console.print("[green]OK[/green] Manual file written")
|
||||
|
||||
sync_data = await project_client.sync(
|
||||
project_id, force_full=True, run_in_background=False
|
||||
sync_response = await call_post(
|
||||
client,
|
||||
f"/v2/projects/{project_id}/sync?force_full=true&run_in_background=false",
|
||||
)
|
||||
sync_report = SyncReportResponse.model_validate(sync_data)
|
||||
sync_report = SyncReportResponse.model_validate(sync_response.json())
|
||||
if sync_report.total == 0:
|
||||
raise ValueError("Sync did not detect any changes")
|
||||
|
||||
@@ -116,7 +118,8 @@ async def run_doctor() -> None:
|
||||
|
||||
console.print("[green]OK[/green] Search confirmed manual file")
|
||||
|
||||
status_report = await project_client.get_status(project_id)
|
||||
status_response = await call_post(client, f"/v2/projects/{project_id}/status")
|
||||
status_report = SyncReportResponse.model_validate(status_response.json())
|
||||
if status_report.total != 0:
|
||||
raise ValueError("Project status not clean after sync")
|
||||
|
||||
@@ -139,9 +142,6 @@ def doctor(
|
||||
"""Run local consistency checks to verify file/database sync."""
|
||||
try:
|
||||
validate_routing_flags(local, cloud)
|
||||
# Doctor runs local filesystem checks — always default to local routing
|
||||
if not local and not cloud:
|
||||
local = True
|
||||
with force_routing(local=local, cloud=cloud):
|
||||
run_with_cleanup(run_doctor())
|
||||
except (ToolError, ValueError) as e:
|
||||
|
||||
@@ -183,10 +183,10 @@ def format(
|
||||
By default, formats all .md, .json, and .canvas files in the current project.
|
||||
|
||||
Examples:
|
||||
bm format # Format all files in current project
|
||||
bm format --project research # Format files in specific project
|
||||
bm format notes/meeting.md # Format a specific file
|
||||
bm format notes/ # Format all files in directory
|
||||
basic-memory format # Format all files in current project
|
||||
basic-memory format --project research # Format files in specific project
|
||||
basic-memory format notes/meeting.md # Format a specific file
|
||||
basic-memory format notes/ # Format all files in directory
|
||||
"""
|
||||
try:
|
||||
run_with_cleanup(run_format(path, project))
|
||||
|
||||
@@ -44,7 +44,7 @@ def import_chatgpt(
|
||||
2. Convert them to linear markdown conversations
|
||||
3. Save as clean, readable markdown files
|
||||
|
||||
After importing, run 'bm reindex --search' to index the new files.
|
||||
After importing, run 'basic-memory sync' to index the new files.
|
||||
"""
|
||||
|
||||
try:
|
||||
@@ -81,7 +81,7 @@ def import_chatgpt(
|
||||
)
|
||||
)
|
||||
|
||||
console.print("\nRun 'bm reindex --search' to index the new files.")
|
||||
console.print("\nRun 'basic-memory sync' to index the new files.")
|
||||
|
||||
except Exception as e:
|
||||
logger.error("Import failed")
|
||||
|
||||
@@ -44,7 +44,7 @@ def import_claude(
|
||||
2. Create markdown files for each conversation
|
||||
3. Format content in clean, readable markdown
|
||||
|
||||
After importing, run 'bm reindex --search' to index the new files.
|
||||
After importing, run 'basic-memory sync' to index the new files.
|
||||
"""
|
||||
|
||||
config = get_project_config()
|
||||
@@ -84,7 +84,7 @@ def import_claude(
|
||||
)
|
||||
)
|
||||
|
||||
console.print("\nRun 'bm reindex --search' to index the new files.")
|
||||
console.print("\nRun 'basic-memory sync' to index the new files.")
|
||||
|
||||
except Exception as e:
|
||||
logger.error("Import failed")
|
||||
|
||||
@@ -44,7 +44,7 @@ def import_projects(
|
||||
2. Store docs in a docs/ subdirectory
|
||||
3. Place prompt template in project root
|
||||
|
||||
After importing, run 'bm reindex --search' to index the new files.
|
||||
After importing, run 'basic-memory sync' to index the new files.
|
||||
"""
|
||||
config = get_project_config()
|
||||
try:
|
||||
@@ -83,7 +83,7 @@ def import_projects(
|
||||
)
|
||||
)
|
||||
|
||||
console.print("\nRun 'bm reindex --search' to index the new files.")
|
||||
console.print("\nRun 'basic-memory sync' to index the new files.")
|
||||
|
||||
except Exception as e:
|
||||
logger.error("Import failed")
|
||||
|
||||
@@ -36,7 +36,7 @@ def mcp(
|
||||
This command starts an MCP server using one of three transport options:
|
||||
|
||||
- stdio: Standard I/O (good for local usage)
|
||||
- streamable-http: Recommended for web deployments
|
||||
- streamable-http: Recommended for web deployments (default)
|
||||
- sse: Server-Sent Events (for compatibility with existing clients)
|
||||
|
||||
Initialization, file sync, and cleanup are handled by the MCP server's lifespan.
|
||||
@@ -45,20 +45,14 @@ def mcp(
|
||||
Users who have cloud mode enabled can still use local MCP for Claude Code
|
||||
and Claude Desktop while using cloud MCP for web and mobile access.
|
||||
"""
|
||||
# --- Routing setup ---
|
||||
# Trigger: MCP server command invocation.
|
||||
# Why: HTTP/SSE transports serve as local API endpoints and must never
|
||||
# route through cloud. Stdio is a client-facing protocol that
|
||||
# should honor per-project routing (local or cloud).
|
||||
# Outcome: HTTP/SSE get explicit local override; stdio passes through
|
||||
# whatever env vars are already set (honoring external overrides)
|
||||
# and defaults to per-project routing resolution.
|
||||
if transport in ("streamable-http", "sse"):
|
||||
os.environ["BASIC_MEMORY_FORCE_LOCAL"] = "true"
|
||||
os.environ.pop("BASIC_MEMORY_FORCE_CLOUD", None)
|
||||
os.environ["BASIC_MEMORY_EXPLICIT_ROUTING"] = "true"
|
||||
# stdio: no env var manipulation — per-project routing applies by default,
|
||||
# and externally-set env vars (e.g. BASIC_MEMORY_FORCE_CLOUD) are honored.
|
||||
# Force local routing for local MCP server.
|
||||
# Trigger: MCP server command invocation (all transports).
|
||||
# Why: local MCP must never route through cloud; stdio in particular must
|
||||
# remain local-only to avoid cross-environment ambiguity.
|
||||
# Outcome: explicit local override disables per-project cloud routing.
|
||||
os.environ["BASIC_MEMORY_FORCE_LOCAL"] = "true"
|
||||
os.environ.pop("BASIC_MEMORY_FORCE_CLOUD", None)
|
||||
os.environ["BASIC_MEMORY_EXPLICIT_ROUTING"] = "true"
|
||||
|
||||
# Import mcp tools/prompts to register them with the server
|
||||
import basic_memory.mcp.tools # noqa: F401 # pragma: no cover
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -2,10 +2,6 @@
|
||||
|
||||
Provides CLI access to schema validation, inference, and drift detection.
|
||||
Registered as a subcommand group: `bm schema validate`, `bm schema infer`, `bm schema diff`.
|
||||
|
||||
Each command calls the corresponding MCP tool with output_format="json" and
|
||||
renders the result as Rich tables — same code path as `bm tool schema-*` but
|
||||
with human-friendly formatting.
|
||||
"""
|
||||
|
||||
import json
|
||||
@@ -20,9 +16,8 @@ from basic_memory.cli.app import app
|
||||
from basic_memory.cli.commands.command_utils import run_with_cleanup
|
||||
from basic_memory.cli.commands.routing import force_routing, validate_routing_flags
|
||||
from basic_memory.config import ConfigManager
|
||||
from basic_memory.mcp.tools import schema_diff as mcp_schema_diff
|
||||
from basic_memory.mcp.tools import schema_infer as mcp_schema_infer
|
||||
from basic_memory.mcp.tools import schema_validate as mcp_schema_validate
|
||||
from basic_memory.mcp.async_client import get_client
|
||||
from basic_memory.mcp.project_context import get_active_project
|
||||
|
||||
console = Console()
|
||||
|
||||
@@ -42,124 +37,77 @@ def _resolve_project_name(project: Optional[str]) -> Optional[str]:
|
||||
return config_manager.default_project
|
||||
|
||||
|
||||
# --- Rendering helpers ---
|
||||
# --- Validate ---
|
||||
|
||||
|
||||
def _render_validate_table(data: dict) -> None:
|
||||
"""Render a validation report dict as a Rich table."""
|
||||
note_type = data.get("note_type")
|
||||
title_label = note_type or "all"
|
||||
async def _run_validate(
|
||||
target: Optional[str] = None,
|
||||
project: Optional[str] = None,
|
||||
strict: bool = False,
|
||||
):
|
||||
"""Run schema validation via the API."""
|
||||
from basic_memory.mcp.clients.schema import SchemaClient
|
||||
|
||||
table = Table(title=f"Schema Validation: {title_label}")
|
||||
table.add_column("Note", style="cyan")
|
||||
table.add_column("Status", justify="center")
|
||||
table.add_column("Warnings", justify="right")
|
||||
table.add_column("Errors", justify="right")
|
||||
async with get_client(project_name=project) as client:
|
||||
active_project = await get_active_project(client, project, None)
|
||||
schema_client = SchemaClient(client, active_project.external_id)
|
||||
|
||||
for result in data.get("results", []):
|
||||
warnings = result.get("warnings", [])
|
||||
errors = result.get("errors", [])
|
||||
passed = result.get("passed", True)
|
||||
# Determine if target is a note identifier or note type
|
||||
# Heuristic: if target contains / or ., treat as identifier
|
||||
entity_type = None
|
||||
identifier = None
|
||||
if target:
|
||||
if "/" in target or "." in target:
|
||||
identifier = target
|
||||
else:
|
||||
entity_type = target
|
||||
|
||||
if passed and not warnings:
|
||||
status = "[green]pass[/green]"
|
||||
elif passed:
|
||||
status = "[yellow]warn[/yellow]"
|
||||
else:
|
||||
status = "[red]fail[/red]"
|
||||
|
||||
table.add_row(
|
||||
result.get("note_identifier", ""),
|
||||
status,
|
||||
str(len(warnings)),
|
||||
str(len(errors)),
|
||||
report = await schema_client.validate(
|
||||
entity_type=entity_type,
|
||||
identifier=identifier,
|
||||
)
|
||||
|
||||
console.print(table)
|
||||
console.print(
|
||||
f"\nSummary: {data.get('valid_count', 0)}/{data.get('total_notes', 0)} valid, "
|
||||
f"{data.get('warning_count', 0)} warnings, {data.get('error_count', 0)} errors"
|
||||
)
|
||||
# --- Display results ---
|
||||
if report.total_notes == 0:
|
||||
if report.total_entities == 0:
|
||||
console.print(f"[yellow]No notes of type '{entity_type}' found.[/yellow]")
|
||||
else:
|
||||
console.print(
|
||||
f"[yellow]Found {report.total_entities} notes but no schema "
|
||||
f"defined for '{entity_type}'.[/yellow]"
|
||||
)
|
||||
return
|
||||
|
||||
table = Table(title=f"Schema Validation: {entity_type or identifier or 'all'}")
|
||||
table.add_column("Note", style="cyan")
|
||||
table.add_column("Status", justify="center")
|
||||
table.add_column("Warnings", justify="right")
|
||||
table.add_column("Errors", justify="right")
|
||||
|
||||
def _render_infer_table(data: dict) -> None:
|
||||
"""Render an inference report dict as a Rich table."""
|
||||
note_type = data.get("note_type", "")
|
||||
notes_analyzed = data.get("notes_analyzed", 0)
|
||||
suggested_required = data.get("suggested_required", [])
|
||||
suggested_optional = data.get("suggested_optional", [])
|
||||
for result in report.results:
|
||||
if result.passed and not result.warnings:
|
||||
status = "[green]pass[/green]"
|
||||
elif result.passed:
|
||||
status = "[yellow]warn[/yellow]"
|
||||
else:
|
||||
status = "[red]fail[/red]"
|
||||
|
||||
console.print(f"\n[bold]Analyzing {notes_analyzed} notes with type: {note_type}...[/bold]\n")
|
||||
table.add_row(
|
||||
result.note_identifier,
|
||||
status,
|
||||
str(len(result.warnings)),
|
||||
str(len(result.errors)),
|
||||
)
|
||||
|
||||
table = Table(title="Field Frequencies")
|
||||
table.add_column("Field", style="cyan")
|
||||
table.add_column("Source")
|
||||
table.add_column("Count", justify="right")
|
||||
table.add_column("Percentage", justify="right")
|
||||
table.add_column("Suggested")
|
||||
|
||||
for freq in data.get("field_frequencies", []):
|
||||
pct = f"{freq.get('percentage', 0):.0%}"
|
||||
name = freq.get("name", "")
|
||||
if name in suggested_required:
|
||||
suggested = "[green]required[/green]"
|
||||
elif name in suggested_optional:
|
||||
suggested = "[yellow]optional[/yellow]"
|
||||
else:
|
||||
suggested = "[dim]excluded[/dim]"
|
||||
|
||||
table.add_row(
|
||||
name,
|
||||
freq.get("source", ""),
|
||||
str(freq.get("count", 0)),
|
||||
pct,
|
||||
suggested,
|
||||
console.print(table)
|
||||
console.print(
|
||||
f"\nSummary: {report.valid_count}/{report.total_notes} valid, "
|
||||
f"{report.warning_count} warnings, {report.error_count} errors"
|
||||
)
|
||||
|
||||
console.print(table)
|
||||
|
||||
suggested_schema = data.get("suggested_schema", {})
|
||||
if suggested_schema:
|
||||
console.print("\n[bold]Suggested schema:[/bold]")
|
||||
console.print(json.dumps(suggested_schema, indent=2))
|
||||
|
||||
|
||||
def _render_diff_output(data: dict) -> None:
|
||||
"""Render a drift report dict as Rich output."""
|
||||
note_type = data.get("note_type", "")
|
||||
new_fields = data.get("new_fields", [])
|
||||
dropped_fields = data.get("dropped_fields", [])
|
||||
cardinality_changes = data.get("cardinality_changes", [])
|
||||
|
||||
has_drift = new_fields or dropped_fields or cardinality_changes
|
||||
|
||||
if not has_drift:
|
||||
console.print(f"[green]No drift detected for {note_type} schema.[/green]")
|
||||
return
|
||||
|
||||
console.print(f"\n[bold]Schema drift detected for {note_type}:[/bold]\n")
|
||||
|
||||
if new_fields:
|
||||
console.print("[green]+ New fields (common in notes, not in schema):[/green]")
|
||||
for f in new_fields:
|
||||
console.print(
|
||||
f" + {f['name']}: {f.get('percentage', 0):.0%} of notes ({f.get('source', '')})"
|
||||
)
|
||||
|
||||
if dropped_fields:
|
||||
console.print("[red]- Dropped fields (in schema, rare in notes):[/red]")
|
||||
for f in dropped_fields:
|
||||
console.print(
|
||||
f" - {f['name']}: {f.get('percentage', 0):.0%} of notes ({f.get('source', '')})"
|
||||
)
|
||||
|
||||
if cardinality_changes:
|
||||
console.print("[yellow]~ Cardinality changes:[/yellow]")
|
||||
for change in cardinality_changes:
|
||||
console.print(f" ~ {change}")
|
||||
|
||||
|
||||
# --- Commands ---
|
||||
# Exit with error code in strict mode if there are failures
|
||||
if strict and report.error_count > 0:
|
||||
raise typer.Exit(1)
|
||||
|
||||
|
||||
@schema_app.command()
|
||||
@@ -173,7 +121,6 @@ def validate(
|
||||
typer.Option(help="The project name."),
|
||||
] = None,
|
||||
strict: bool = typer.Option(False, "--strict", help="Exit with error on validation failures"),
|
||||
json_output: bool = typer.Option(False, "--json", help="Output in JSON format"),
|
||||
local: bool = typer.Option(
|
||||
False, "--local", help="Force local API routing (ignore cloud mode)"
|
||||
),
|
||||
@@ -184,7 +131,6 @@ def validate(
|
||||
TARGET can be a note path (e.g., people/ada-lovelace.md) or a note type
|
||||
(e.g., person). If omitted, validates all notes that have schemas.
|
||||
|
||||
Use --json for machine-readable output.
|
||||
Use --strict to exit with error code 1 if any validation errors are found.
|
||||
Use --local to force local routing when cloud mode is enabled.
|
||||
Use --cloud to force cloud routing when cloud mode is disabled.
|
||||
@@ -192,43 +138,8 @@ def validate(
|
||||
try:
|
||||
validate_routing_flags(local, cloud)
|
||||
project_name = _resolve_project_name(project)
|
||||
|
||||
# Heuristic: if target contains / or ., treat as identifier; otherwise as note type
|
||||
note_type, identifier = None, None
|
||||
if target:
|
||||
if "/" in target or "." in target:
|
||||
identifier = target
|
||||
else:
|
||||
note_type = target
|
||||
|
||||
with force_routing(local=local, cloud=cloud):
|
||||
result = run_with_cleanup(
|
||||
mcp_schema_validate(
|
||||
note_type=note_type,
|
||||
identifier=identifier,
|
||||
project=project_name,
|
||||
output_format="json",
|
||||
)
|
||||
)
|
||||
|
||||
# Handle error responses
|
||||
if isinstance(result, dict) and "error" in result:
|
||||
if json_output:
|
||||
print(json.dumps(result, indent=2, default=str))
|
||||
else:
|
||||
console.print(f"[yellow]{result['error']}[/yellow]")
|
||||
return
|
||||
|
||||
# output_format="json" guarantees a dict return
|
||||
assert isinstance(result, dict)
|
||||
|
||||
if json_output:
|
||||
print(json.dumps(result, indent=2, default=str))
|
||||
else:
|
||||
_render_validate_table(result)
|
||||
|
||||
if strict and result.get("error_count", 0) > 0:
|
||||
raise typer.Exit(1)
|
||||
run_with_cleanup(_run_validate(target, project_name, strict))
|
||||
except ValueError as e:
|
||||
console.print(f"[red]Error: {e}[/red]")
|
||||
raise typer.Exit(1)
|
||||
@@ -240,9 +151,94 @@ def validate(
|
||||
raise
|
||||
|
||||
|
||||
# --- Infer ---
|
||||
|
||||
|
||||
async def _run_infer(
|
||||
entity_type: str,
|
||||
project: Optional[str] = None,
|
||||
threshold: float = 0.25,
|
||||
save: bool = False,
|
||||
):
|
||||
"""Run schema inference via the API."""
|
||||
from basic_memory.mcp.clients.schema import SchemaClient
|
||||
|
||||
async with get_client(project_name=project) as client:
|
||||
active_project = await get_active_project(client, project, None)
|
||||
schema_client = SchemaClient(client, active_project.external_id)
|
||||
|
||||
report = await schema_client.infer(entity_type, threshold=threshold)
|
||||
|
||||
if report.notes_analyzed == 0:
|
||||
console.print(f"[yellow]No notes found with type: {entity_type}[/yellow]")
|
||||
return
|
||||
|
||||
# --- Empty schema guard ---
|
||||
# Trigger: notes were analyzed but no fields met the threshold
|
||||
# Why: dumping hundreds of excluded fields is not useful output
|
||||
# Outcome: show count and suggest a more specific type
|
||||
if not report.suggested_schema:
|
||||
console.print(
|
||||
f"\n[yellow]Analyzed {report.notes_analyzed} notes of type '{entity_type}', "
|
||||
f"but no fields met the {threshold:.0%} threshold.[/yellow]\n"
|
||||
)
|
||||
console.print(
|
||||
f"This usually means '{entity_type}' is too broad — "
|
||||
f"the notes don't share a consistent structure.\n"
|
||||
)
|
||||
console.print("[bold]Suggestions:[/bold]")
|
||||
console.print(" 1. Use a more specific type")
|
||||
console.print(
|
||||
f" 2. Lower the threshold: bm schema infer {entity_type} --threshold 0.1"
|
||||
)
|
||||
console.print(" 3. Create typed notes with write_note using a specific note_type")
|
||||
return
|
||||
|
||||
# --- Display frequency analysis ---
|
||||
console.print(
|
||||
f"\n[bold]Analyzing {report.notes_analyzed} notes with type: {entity_type}...[/bold]\n"
|
||||
)
|
||||
|
||||
table = Table(title="Field Frequencies")
|
||||
table.add_column("Field", style="cyan")
|
||||
table.add_column("Source")
|
||||
table.add_column("Count", justify="right")
|
||||
table.add_column("Percentage", justify="right")
|
||||
table.add_column("Suggested")
|
||||
|
||||
for freq in report.field_frequencies:
|
||||
pct = f"{freq.percentage:.0%}"
|
||||
if freq.name in report.suggested_required:
|
||||
suggested = "[green]required[/green]"
|
||||
elif freq.name in report.suggested_optional:
|
||||
suggested = "[yellow]optional[/yellow]"
|
||||
else:
|
||||
suggested = "[dim]excluded[/dim]"
|
||||
|
||||
table.add_row(
|
||||
freq.name,
|
||||
freq.source,
|
||||
str(freq.count),
|
||||
pct,
|
||||
suggested,
|
||||
)
|
||||
|
||||
console.print(table)
|
||||
|
||||
# --- Display suggested schema ---
|
||||
console.print("\n[bold]Suggested schema:[/bold]")
|
||||
console.print(json.dumps(report.suggested_schema, indent=2))
|
||||
|
||||
if save:
|
||||
console.print(
|
||||
f"\n[yellow]--save not yet implemented. "
|
||||
f"Copy the schema above into schema/{entity_type}.md[/yellow]"
|
||||
)
|
||||
|
||||
|
||||
@schema_app.command()
|
||||
def infer(
|
||||
note_type: Annotated[
|
||||
entity_type: Annotated[
|
||||
str,
|
||||
typer.Argument(help="Note type to analyze (e.g., person, meeting)"),
|
||||
],
|
||||
@@ -254,7 +250,6 @@ def infer(
|
||||
0.25, "--threshold", help="Minimum frequency for optional fields (0-1)"
|
||||
),
|
||||
save: bool = typer.Option(False, "--save", help="Save inferred schema to schema/ directory"),
|
||||
json_output: bool = typer.Option(False, "--json", help="Output in JSON format"),
|
||||
local: bool = typer.Option(
|
||||
False, "--local", help="Force local API routing (ignore cloud mode)"
|
||||
),
|
||||
@@ -268,53 +263,14 @@ def infer(
|
||||
Fields present in 95%+ of notes become required. Fields above the
|
||||
threshold (default 25%) become optional. Fields below threshold are excluded.
|
||||
|
||||
Use --json for machine-readable output.
|
||||
Use --local to force local routing when cloud mode is enabled.
|
||||
Use --cloud to force cloud routing when cloud mode is disabled.
|
||||
"""
|
||||
try:
|
||||
validate_routing_flags(local, cloud)
|
||||
project_name = _resolve_project_name(project)
|
||||
|
||||
with force_routing(local=local, cloud=cloud):
|
||||
result = run_with_cleanup(
|
||||
mcp_schema_infer(
|
||||
note_type=note_type,
|
||||
threshold=threshold,
|
||||
project=project_name,
|
||||
output_format="json",
|
||||
)
|
||||
)
|
||||
|
||||
# Handle error responses
|
||||
if isinstance(result, dict) and "error" in result:
|
||||
if json_output:
|
||||
print(json.dumps(result, indent=2, default=str))
|
||||
else:
|
||||
console.print(f"[yellow]{result['error']}[/yellow]")
|
||||
return
|
||||
|
||||
# output_format="json" guarantees a dict return
|
||||
assert isinstance(result, dict)
|
||||
|
||||
# Handle zero notes
|
||||
if result.get("notes_analyzed", 0) == 0:
|
||||
if json_output:
|
||||
print(json.dumps(result, indent=2, default=str))
|
||||
else:
|
||||
console.print(f"[yellow]No notes found with type: {note_type}[/yellow]")
|
||||
return
|
||||
|
||||
if json_output:
|
||||
print(json.dumps(result, indent=2, default=str))
|
||||
else:
|
||||
_render_infer_table(result)
|
||||
|
||||
if save:
|
||||
console.print(
|
||||
f"\n[yellow]--save not yet implemented. "
|
||||
f"Copy the schema above into schema/{note_type}.md[/yellow]"
|
||||
)
|
||||
run_with_cleanup(_run_infer(entity_type, project_name, threshold, save))
|
||||
except ValueError as e:
|
||||
console.print(f"[red]Error: {e}[/red]")
|
||||
raise typer.Exit(1)
|
||||
@@ -326,9 +282,49 @@ def infer(
|
||||
raise
|
||||
|
||||
|
||||
# --- Diff ---
|
||||
|
||||
|
||||
async def _run_diff(
|
||||
entity_type: str,
|
||||
project: Optional[str] = None,
|
||||
):
|
||||
"""Run schema drift detection via the API."""
|
||||
from basic_memory.mcp.clients.schema import SchemaClient
|
||||
|
||||
async with get_client(project_name=project) as client:
|
||||
active_project = await get_active_project(client, project, None)
|
||||
schema_client = SchemaClient(client, active_project.external_id)
|
||||
|
||||
report = await schema_client.diff(entity_type)
|
||||
|
||||
has_drift = report.new_fields or report.dropped_fields or report.cardinality_changes
|
||||
|
||||
if not has_drift:
|
||||
console.print(f"[green]No drift detected for {entity_type} schema.[/green]")
|
||||
return
|
||||
|
||||
console.print(f"\n[bold]Schema drift detected for {entity_type}:[/bold]\n")
|
||||
|
||||
if report.new_fields:
|
||||
console.print("[green]+ New fields (common in notes, not in schema):[/green]")
|
||||
for f in report.new_fields:
|
||||
console.print(f" + {f.name}: {f.percentage:.0%} of notes ({f.source})")
|
||||
|
||||
if report.dropped_fields:
|
||||
console.print("[red]- Dropped fields (in schema, rare in notes):[/red]")
|
||||
for f in report.dropped_fields:
|
||||
console.print(f" - {f.name}: {f.percentage:.0%} of notes ({f.source})")
|
||||
|
||||
if report.cardinality_changes:
|
||||
console.print("[yellow]~ Cardinality changes:[/yellow]")
|
||||
for change in report.cardinality_changes:
|
||||
console.print(f" ~ {change}")
|
||||
|
||||
|
||||
@schema_app.command()
|
||||
def diff(
|
||||
note_type: Annotated[
|
||||
entity_type: Annotated[
|
||||
str,
|
||||
typer.Argument(help="Note type to check for drift"),
|
||||
],
|
||||
@@ -336,7 +332,6 @@ def diff(
|
||||
Optional[str],
|
||||
typer.Option(help="The project name."),
|
||||
] = None,
|
||||
json_output: bool = typer.Option(False, "--json", help="Output in JSON format"),
|
||||
local: bool = typer.Option(
|
||||
False, "--local", help="Force local API routing (ignore cloud mode)"
|
||||
),
|
||||
@@ -348,38 +343,14 @@ def diff(
|
||||
are actually structured. Identifies new fields,
|
||||
dropped fields, and cardinality changes.
|
||||
|
||||
Use --json for machine-readable output.
|
||||
Use --local to force local routing when cloud mode is enabled.
|
||||
Use --cloud to force cloud routing when cloud mode is disabled.
|
||||
"""
|
||||
try:
|
||||
validate_routing_flags(local, cloud)
|
||||
project_name = _resolve_project_name(project)
|
||||
|
||||
with force_routing(local=local, cloud=cloud):
|
||||
result = run_with_cleanup(
|
||||
mcp_schema_diff(
|
||||
note_type=note_type,
|
||||
project=project_name,
|
||||
output_format="json",
|
||||
)
|
||||
)
|
||||
|
||||
# Handle error responses
|
||||
if isinstance(result, dict) and "error" in result:
|
||||
if json_output:
|
||||
print(json.dumps(result, indent=2, default=str))
|
||||
else:
|
||||
console.print(f"[yellow]{result['error']}[/yellow]")
|
||||
return
|
||||
|
||||
# output_format="json" guarantees a dict return
|
||||
assert isinstance(result, dict)
|
||||
|
||||
if json_output:
|
||||
print(json.dumps(result, indent=2, default=str))
|
||||
else:
|
||||
_render_diff_output(result)
|
||||
run_with_cleanup(_run_diff(entity_type, project_name))
|
||||
except ValueError as e:
|
||||
console.print(f"[red]Error: {e}[/red]")
|
||||
raise typer.Exit(1)
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
"""Status command for basic-memory CLI."""
|
||||
|
||||
import json
|
||||
from typing import Set, Dict
|
||||
from typing import Annotated, Optional
|
||||
|
||||
@@ -15,7 +14,7 @@ from basic_memory.cli.app import app
|
||||
from basic_memory.cli.commands.routing import force_routing, validate_routing_flags
|
||||
from basic_memory.config import ConfigManager
|
||||
from basic_memory.mcp.async_client import get_client
|
||||
from basic_memory.mcp.clients import ProjectClient
|
||||
from basic_memory.mcp.tools.utils import call_post
|
||||
from basic_memory.schemas import SyncReportResponse
|
||||
from basic_memory.mcp.project_context import get_active_project
|
||||
|
||||
@@ -142,20 +141,22 @@ def display_changes(
|
||||
console.print(Panel(tree, expand=False))
|
||||
|
||||
|
||||
async def run_status(
|
||||
project: Optional[str] = None,
|
||||
) -> tuple[str, SyncReportResponse]:
|
||||
"""Fetch sync status of files vs database.
|
||||
|
||||
Returns (project_name, sync_report) for the caller to render.
|
||||
"""
|
||||
async def run_status(project: Optional[str] = None, verbose: bool = False): # pragma: no cover
|
||||
"""Check sync status of files vs database."""
|
||||
# Resolve default project so get_client() can route per-project
|
||||
project = project or ConfigManager().default_project
|
||||
|
||||
async with get_client(project_name=project) as client:
|
||||
project_item = await get_active_project(client, project, None)
|
||||
sync_report = await ProjectClient(client).get_status(project_item.external_id)
|
||||
return project_item.name, sync_report
|
||||
try:
|
||||
async with get_client(project_name=project) as client:
|
||||
project_item = await get_active_project(client, project, None)
|
||||
response = await call_post(client, f"/v2/projects/{project_item.external_id}/status")
|
||||
sync_report = SyncReportResponse.model_validate(response.json())
|
||||
|
||||
display_changes(project_item.name, "Status", sync_report, verbose)
|
||||
|
||||
except (ValueError, ToolError) as e:
|
||||
console.print(f"[red]Error: {e}[/red]")
|
||||
raise typer.Exit(1)
|
||||
|
||||
|
||||
@app.command()
|
||||
@@ -165,7 +166,6 @@ def status(
|
||||
typer.Option(help="The project name."),
|
||||
] = None,
|
||||
verbose: bool = typer.Option(False, "--verbose", "-v", help="Show detailed file information"),
|
||||
json_output: bool = typer.Option(False, "--json", help="Output in JSON format"),
|
||||
local: bool = typer.Option(
|
||||
False, "--local", help="Force local API routing (ignore cloud mode)"
|
||||
),
|
||||
@@ -173,7 +173,6 @@ def status(
|
||||
):
|
||||
"""Show sync status between files and database.
|
||||
|
||||
Use --json for machine-readable output.
|
||||
Use --local to force local routing when cloud mode is enabled.
|
||||
Use --cloud to force cloud routing when cloud mode is disabled.
|
||||
"""
|
||||
@@ -181,32 +180,12 @@ def status(
|
||||
|
||||
try:
|
||||
validate_routing_flags(local, cloud)
|
||||
# Trigger: no explicit routing flag provided
|
||||
# Why: status scans the local filesystem — cloud routing would use the
|
||||
# Docker-internal path stored in the cloud database, which doesn't
|
||||
# exist locally.
|
||||
# Outcome: default to local routing unless --cloud was explicitly requested.
|
||||
if not local and not cloud:
|
||||
local = True
|
||||
with force_routing(local=local, cloud=cloud):
|
||||
project_name, sync_report = run_with_cleanup(run_status(project))
|
||||
|
||||
if json_output:
|
||||
print(json.dumps(sync_report.model_dump(mode="json"), indent=2, default=str))
|
||||
else:
|
||||
display_changes(project_name, "Status", sync_report, verbose)
|
||||
except (ValueError, ToolError) as e:
|
||||
if json_output:
|
||||
print(json.dumps({"error": str(e)}, indent=2))
|
||||
else:
|
||||
console.print(f"[red]Error: {e}[/red]")
|
||||
run_with_cleanup(run_status(project, verbose)) # pragma: no cover
|
||||
except ValueError as e:
|
||||
console.print(f"[red]Error: {e}[/red]")
|
||||
raise typer.Exit(code=1)
|
||||
except typer.Exit:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error(f"Error checking status: {e}")
|
||||
if json_output:
|
||||
print(json.dumps({"error": str(e)}, indent=2))
|
||||
else:
|
||||
typer.echo(f"Error checking status: {e}", err=True)
|
||||
typer.echo(f"Error checking status: {e}", err=True)
|
||||
raise typer.Exit(code=1) # pragma: no cover
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,97 @@
|
||||
"""Watch command - run file watcher as a standalone long-running process."""
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import signal
|
||||
import sys
|
||||
from typing import Optional
|
||||
|
||||
import typer
|
||||
from loguru import logger
|
||||
|
||||
from basic_memory import db
|
||||
from basic_memory.cli.app import app
|
||||
from basic_memory.cli.container import get_container
|
||||
from basic_memory.config import ConfigManager
|
||||
from basic_memory.services.initialization import initialize_app
|
||||
from basic_memory.sync.coordinator import SyncCoordinator
|
||||
|
||||
|
||||
async def run_watch(project: Optional[str] = None) -> None:
|
||||
"""Run the file watcher as a long-running process.
|
||||
|
||||
This is the async core of the watch command. It:
|
||||
1. Initializes the app (DB migrations + project reconciliation)
|
||||
2. Validates and sets project constraint if --project given
|
||||
3. Creates a SyncCoordinator with quiet=False for Rich console output
|
||||
4. Blocks until SIGINT/SIGTERM, then shuts down cleanly
|
||||
"""
|
||||
container = get_container()
|
||||
config = container.config
|
||||
|
||||
# --- Initialization ---
|
||||
# Wrapped in try/finally so DB resources are cleaned up on all exit paths,
|
||||
# including early exits from invalid --project names.
|
||||
await initialize_app(config)
|
||||
sync_coordinator = None
|
||||
|
||||
try:
|
||||
# --- Project constraint ---
|
||||
if project:
|
||||
config_manager = ConfigManager()
|
||||
project_name, _ = config_manager.get_project(project)
|
||||
if not project_name:
|
||||
typer.echo(f"No project found named: {project}", err=True)
|
||||
raise typer.Exit(1)
|
||||
|
||||
os.environ["BASIC_MEMORY_MCP_PROJECT"] = project_name
|
||||
logger.info(f"Watch constrained to project: {project_name}")
|
||||
|
||||
# --- Sync coordinator ---
|
||||
# quiet=False so file change events are printed to the terminal
|
||||
sync_coordinator = SyncCoordinator(config=config, should_sync=True, quiet=False)
|
||||
|
||||
# --- Signal handling ---
|
||||
shutdown_event = asyncio.Event()
|
||||
|
||||
def _signal_handler() -> None:
|
||||
logger.info("Shutdown signal received")
|
||||
shutdown_event.set()
|
||||
|
||||
loop = asyncio.get_running_loop()
|
||||
|
||||
# Windows ProactorEventLoop does not support add_signal_handler;
|
||||
# fall back to the stdlib signal module which works cross-platform.
|
||||
try:
|
||||
for sig in (signal.SIGINT, signal.SIGTERM):
|
||||
loop.add_signal_handler(sig, _signal_handler)
|
||||
except NotImplementedError:
|
||||
for sig in (signal.SIGINT, signal.SIGTERM):
|
||||
signal.signal(sig, lambda _signum, _frame: _signal_handler())
|
||||
|
||||
# --- Run ---
|
||||
await sync_coordinator.start()
|
||||
logger.info("Watch service running, press Ctrl+C to stop")
|
||||
await shutdown_event.wait()
|
||||
finally:
|
||||
if sync_coordinator is not None:
|
||||
await sync_coordinator.stop()
|
||||
await db.shutdown_db()
|
||||
logger.info("Watch service stopped")
|
||||
|
||||
|
||||
@app.command()
|
||||
def watch(
|
||||
project: Optional[str] = typer.Option(None, help="Restrict watcher to a single project"),
|
||||
) -> None:
|
||||
"""Run file watcher as a long-running process (no MCP server).
|
||||
|
||||
Watches for file changes in project directories and syncs them to the
|
||||
database. Useful for running Basic Memory sync alongside external tools
|
||||
that don't use the MCP server.
|
||||
"""
|
||||
# On Windows, use SelectorEventLoop to avoid ProactorEventLoop cleanup issues
|
||||
if sys.platform == "win32": # pragma: no cover
|
||||
asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
|
||||
|
||||
asyncio.run(run_watch(project=project))
|
||||
@@ -0,0 +1,57 @@
|
||||
"""Workspace commands for Basic Memory cloud workspaces."""
|
||||
|
||||
import typer
|
||||
from rich.console import Console
|
||||
from rich.table import Table
|
||||
|
||||
from basic_memory.cli.app import app
|
||||
from basic_memory.cli.commands.command_utils import run_with_cleanup
|
||||
from basic_memory.mcp.project_context import get_available_workspaces
|
||||
|
||||
console = Console()
|
||||
|
||||
workspace_app = typer.Typer(help="Manage cloud workspaces")
|
||||
app.add_typer(workspace_app, name="workspace")
|
||||
|
||||
|
||||
@workspace_app.command("list")
|
||||
def list_workspaces() -> None:
|
||||
"""List cloud workspaces available to the current OAuth session."""
|
||||
|
||||
async def _list():
|
||||
return await get_available_workspaces()
|
||||
|
||||
try:
|
||||
workspaces = run_with_cleanup(_list())
|
||||
except RuntimeError as exc:
|
||||
console.print(f"[red]Error: {exc}[/red]")
|
||||
raise typer.Exit(1)
|
||||
except Exception as exc: # pragma: no cover
|
||||
console.print(f"[red]Error listing workspaces: {exc}[/red]")
|
||||
raise typer.Exit(1)
|
||||
|
||||
if not workspaces:
|
||||
console.print("[yellow]No accessible workspaces found.[/yellow]")
|
||||
return
|
||||
|
||||
table = Table(title="Available Workspaces")
|
||||
table.add_column("Name", style="cyan")
|
||||
table.add_column("Type", style="blue")
|
||||
table.add_column("Role", style="green")
|
||||
table.add_column("Tenant ID", style="yellow")
|
||||
|
||||
for workspace in workspaces:
|
||||
table.add_row(
|
||||
workspace.name,
|
||||
workspace.workspace_type,
|
||||
workspace.role,
|
||||
workspace.tenant_id,
|
||||
)
|
||||
|
||||
console.print(table)
|
||||
|
||||
|
||||
@app.command("workspaces")
|
||||
def workspaces_alias() -> None:
|
||||
"""Alias for `bm workspace list`."""
|
||||
list_workspaces()
|
||||
@@ -28,6 +28,7 @@ if not _version_only_invocation(sys.argv[1:]):
|
||||
schema,
|
||||
status,
|
||||
tool,
|
||||
workspace,
|
||||
)
|
||||
|
||||
warnings.filterwarnings("ignore") # pragma: no cover
|
||||
|
||||
@@ -7,13 +7,10 @@ from rich.console import Console
|
||||
from rich.panel import Panel
|
||||
|
||||
import basic_memory
|
||||
from basic_memory.cli.analytics import track, EVENT_PROMO_SHOWN
|
||||
from basic_memory.config import ConfigManager
|
||||
|
||||
OSS_DISCOUNT_CODE = "BMFOSS"
|
||||
CLOUD_LEARN_MORE_URL = (
|
||||
"https://basicmemory.com?utm_source=bm-foss&utm_medium=promo&utm_campaign=cloud-upsell"
|
||||
)
|
||||
CLOUD_LEARN_MORE_URL = "https://basicmemory.com"
|
||||
|
||||
|
||||
def _promos_disabled_by_env() -> bool:
|
||||
@@ -24,13 +21,7 @@ def _promos_disabled_by_env() -> bool:
|
||||
|
||||
def _is_interactive_session() -> bool:
|
||||
"""Return whether stdin/stdout are interactive terminals."""
|
||||
try:
|
||||
return sys.stdin.isatty() and sys.stdout.isatty()
|
||||
except ValueError:
|
||||
# Trigger: stdin/stdout already closed (e.g., MCP stdio transport shutdown)
|
||||
# Why: isatty() raises ValueError on closed file descriptors
|
||||
# Outcome: treat as non-interactive, suppressing promo output
|
||||
return False
|
||||
return sys.stdin.isatty() and sys.stdout.isatty()
|
||||
|
||||
|
||||
def _build_cloud_promo_message() -> str:
|
||||
@@ -122,9 +113,6 @@ def maybe_show_cloud_promo(
|
||||
out.print(f"Learn more at [link={CLOUD_LEARN_MORE_URL}]{CLOUD_LEARN_MORE_URL}[/link]")
|
||||
out.print("[dim]Disable with: bm cloud promo --off[/dim]")
|
||||
|
||||
trigger = "first_run" if show_first_run else "version_bump"
|
||||
track(EVENT_PROMO_SHOWN, {"trigger": trigger})
|
||||
|
||||
config.cloud_promo_first_run_shown = True
|
||||
config.cloud_promo_last_version_shown = basic_memory.__version__
|
||||
manager.save_config(config)
|
||||
|
||||
+31
-135
@@ -3,7 +3,6 @@
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
@@ -11,7 +10,7 @@ from typing import Any, Dict, Literal, Optional, List, Tuple
|
||||
from enum import Enum
|
||||
|
||||
from loguru import logger
|
||||
from pydantic import AliasChoices, BaseModel, Field, model_validator
|
||||
from pydantic import BaseModel, Field, model_validator
|
||||
from pydantic_settings import BaseSettings, SettingsConfigDict
|
||||
|
||||
from basic_memory.utils import setup_logging, generate_permalink
|
||||
@@ -41,11 +40,8 @@ class DatabaseBackend(str, Enum):
|
||||
|
||||
|
||||
def _default_semantic_search_enabled() -> bool:
|
||||
"""Enable semantic search by default when required local semantic dependencies exist."""
|
||||
required_modules = ("fastembed", "sqlite_vec")
|
||||
return all(
|
||||
importlib.util.find_spec(module_name) is not None for module_name in required_modules
|
||||
)
|
||||
"""Enable semantic search by default when semantic extras are installed."""
|
||||
return importlib.util.find_spec("fastembed") is not None
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -97,15 +93,10 @@ class ProjectEntry(BaseModel):
|
||||
default=ProjectMode.LOCAL,
|
||||
description="Routing mode: local (in-process ASGI) or cloud (remote API)",
|
||||
)
|
||||
workspace_id: Optional[str] = Field(
|
||||
default=None,
|
||||
description="Cloud workspace tenant_id. Set by 'bm project set-cloud --workspace'.",
|
||||
)
|
||||
# Cloud sync state (replaces CloudProjectConfig)
|
||||
local_sync_path: Optional[str] = Field(
|
||||
cloud_sync_path: Optional[str] = Field(
|
||||
default=None,
|
||||
description="Local working directory for bisync",
|
||||
validation_alias=AliasChoices("local_sync_path", "cloud_sync_path"),
|
||||
description="Local working directory for bisync (formerly CloudProjectConfig.local_path)",
|
||||
)
|
||||
bisync_initialized: bool = Field(
|
||||
default=False,
|
||||
@@ -133,7 +124,7 @@ class BasicMemoryConfig(BaseSettings):
|
||||
description="Mapping of project names to their ProjectEntry configuration",
|
||||
)
|
||||
default_project: Optional[str] = Field(
|
||||
default=None,
|
||||
default="main",
|
||||
description="Name of the default project to use. When set, acts as fallback when no project parameter is specified. Set to null to disable automatic project resolution.",
|
||||
)
|
||||
|
||||
@@ -154,7 +145,7 @@ class BasicMemoryConfig(BaseSettings):
|
||||
# Semantic search configuration
|
||||
semantic_search_enabled: bool = Field(
|
||||
default_factory=_default_semantic_search_enabled,
|
||||
description="Enable semantic search (vector/hybrid retrieval). Works on both SQLite and Postgres backends. Requires semantic dependencies (included by default).",
|
||||
description="Enable semantic search (vector/hybrid retrieval). Works on both SQLite and Postgres backends. Requires semantic extras.",
|
||||
)
|
||||
semantic_embedding_provider: str = Field(
|
||||
default="fastembed",
|
||||
@@ -173,25 +164,6 @@ class BasicMemoryConfig(BaseSettings):
|
||||
description="Batch size for embedding generation.",
|
||||
gt=0,
|
||||
)
|
||||
semantic_embedding_sync_batch_size: int = Field(
|
||||
default=64,
|
||||
description="Batch size for vector sync orchestration flushes.",
|
||||
gt=0,
|
||||
)
|
||||
semantic_embedding_cache_dir: str | None = Field(
|
||||
default=None,
|
||||
description="Optional cache directory for FastEmbed model artifacts.",
|
||||
)
|
||||
semantic_embedding_threads: int | None = Field(
|
||||
default=None,
|
||||
description="Optional FastEmbed runtime thread count override.",
|
||||
gt=0,
|
||||
)
|
||||
semantic_embedding_parallel: int | None = Field(
|
||||
default=None,
|
||||
description="Optional FastEmbed embed() parallelism override.",
|
||||
gt=0,
|
||||
)
|
||||
semantic_vector_k: int = Field(
|
||||
default=100,
|
||||
description="Vector candidate count for vector and hybrid retrieval.",
|
||||
@@ -265,18 +237,8 @@ class BasicMemoryConfig(BaseSettings):
|
||||
description="Disable automatic permalink generation in frontmatter. When enabled, new notes won't have permalinks added and sync won't update permalinks. Existing permalinks will still work for reading.",
|
||||
)
|
||||
|
||||
write_note_overwrite_default: bool = Field(
|
||||
default=False,
|
||||
description=(
|
||||
"Default value for write_note's overwrite parameter. "
|
||||
"When False (default), write_note errors if note already exists. "
|
||||
"Set to True to restore pre-v0.20 upsert behavior. "
|
||||
"Env: BASIC_MEMORY_WRITE_NOTE_OVERWRITE_DEFAULT"
|
||||
),
|
||||
)
|
||||
|
||||
ensure_frontmatter_on_sync: bool = Field(
|
||||
default=True,
|
||||
default=False,
|
||||
description="Ensure markdown files have frontmatter during sync by adding derived title/type/permalink when missing. When combined with disable_permalinks=True, this setting takes precedence for missing-frontmatter files and still writes permalinks.",
|
||||
)
|
||||
|
||||
@@ -356,11 +318,6 @@ class BasicMemoryConfig(BaseSettings):
|
||||
description="API key for cloud access (bmc_ prefixed). Account-level, not per-project.",
|
||||
)
|
||||
|
||||
default_workspace: Optional[str] = Field(
|
||||
default=None,
|
||||
description="Default cloud workspace tenant_id. Set by 'bm cloud workspace set-default'.",
|
||||
)
|
||||
|
||||
@model_validator(mode="before")
|
||||
@classmethod
|
||||
def migrate_legacy_projects(cls, data: Any) -> Any:
|
||||
@@ -402,27 +359,26 @@ class BasicMemoryConfig(BaseSettings):
|
||||
if name in cloud_projects:
|
||||
cp = cloud_projects[name]
|
||||
if isinstance(cp, dict):
|
||||
entry["local_sync_path"] = cp.get("local_path")
|
||||
entry["cloud_sync_path"] = cp.get("local_path")
|
||||
entry["bisync_initialized"] = cp.get("bisync_initialized", False)
|
||||
entry["last_sync"] = cp.get("last_sync")
|
||||
else:
|
||||
# Already a CloudProjectConfig-like object
|
||||
entry["local_sync_path"] = getattr(cp, "local_path", None)
|
||||
entry["cloud_sync_path"] = getattr(cp, "local_path", None)
|
||||
entry["bisync_initialized"] = getattr(cp, "bisync_initialized", False)
|
||||
entry["last_sync"] = getattr(cp, "last_sync", None)
|
||||
new_projects[name] = entry
|
||||
|
||||
# Pick up cloud_projects entries not already in projects
|
||||
# These are cloud-only projects — path should be the local working
|
||||
# directory (if one exists), local_path goes into local_sync_path for bisync
|
||||
# These are cloud-only projects — path is the cloud permalink,
|
||||
# local_path goes into cloud_sync_path for bisync
|
||||
for name, cp in cloud_projects.items():
|
||||
if name not in new_projects:
|
||||
if isinstance(cp, dict):
|
||||
local_path = cp.get("local_path", "")
|
||||
new_projects[name] = {
|
||||
"path": local_path or "",
|
||||
"path": generate_permalink(name),
|
||||
"mode": project_modes.get(name, "cloud"),
|
||||
"local_sync_path": local_path,
|
||||
"cloud_sync_path": cp.get("local_path"),
|
||||
"bisync_initialized": cp.get("bisync_initialized", False),
|
||||
"last_sync": cp.get("last_sync"),
|
||||
}
|
||||
@@ -433,18 +389,6 @@ class BasicMemoryConfig(BaseSettings):
|
||||
data.pop("project_modes", None)
|
||||
data.pop("cloud_projects", None)
|
||||
|
||||
# --- Promote local_sync_path into path for cloud projects with slug paths ---
|
||||
# Trigger: project entry has local_sync_path set but path is a cloud slug (not absolute)
|
||||
# Why: path must always be the local filesystem path; the cloud remote is derivable
|
||||
# Outcome: path becomes the local directory, local_sync_path kept for backwards compat
|
||||
projects = data.get("projects", {})
|
||||
for name, entry in projects.items():
|
||||
if isinstance(entry, dict):
|
||||
lsp = entry.get("local_sync_path")
|
||||
path = entry.get("path", "")
|
||||
if lsp and not os.path.isabs(path):
|
||||
entry["path"] = lsp
|
||||
|
||||
return data
|
||||
|
||||
@property
|
||||
@@ -467,12 +411,10 @@ class BasicMemoryConfig(BaseSettings):
|
||||
def get_project_mode(self, project_name: str) -> ProjectMode:
|
||||
"""Get the routing mode for a project.
|
||||
|
||||
Returns the per-project mode if set.
|
||||
Unknown projects (not in local config) default to CLOUD —
|
||||
local projects are always registered in config.
|
||||
Returns the per-project mode if set, otherwise LOCAL.
|
||||
"""
|
||||
entry = self.projects.get(project_name)
|
||||
return entry.mode if entry else ProjectMode.CLOUD
|
||||
return entry.mode if entry else ProjectMode.LOCAL
|
||||
|
||||
def set_project_mode(self, project_name: str, mode: ProjectMode) -> None:
|
||||
"""Set the routing mode for a project.
|
||||
@@ -535,25 +477,18 @@ class BasicMemoryConfig(BaseSettings):
|
||||
if self.database_backend == DatabaseBackend.POSTGRES: # pragma: no cover
|
||||
return # pragma: no cover
|
||||
|
||||
# Trigger: no projects configured (fresh install or empty config)
|
||||
# Why: every config needs at least one project to be functional
|
||||
# Outcome: creates "main" project using BASIC_MEMORY_HOME or ~/basic-memory
|
||||
if not self.projects:
|
||||
# Ensure at least one project exists; if none exist then create main
|
||||
if not self.projects: # pragma: no cover
|
||||
self.projects["main"] = ProjectEntry(
|
||||
path=str(Path(os.getenv("BASIC_MEMORY_HOME", Path.home() / "basic-memory")))
|
||||
)
|
||||
|
||||
# Trigger: default_project was not explicitly provided in the input data
|
||||
# (config file omitted the key, or BasicMemoryConfig() called with no args)
|
||||
# Why: callers like get_project_config() expect a valid project name;
|
||||
# but explicit None (discovery mode) must be preserved
|
||||
# Outcome: sets default_project to the first available project
|
||||
if "default_project" not in self.model_fields_set:
|
||||
self.default_project = next(iter(self.projects.keys()))
|
||||
# Trigger: default_project was explicitly set but references a non-existent project
|
||||
# Why: project may have been removed or renamed since config was saved
|
||||
# Outcome: corrects to the first available project
|
||||
elif self.default_project is not None and self.default_project not in self.projects:
|
||||
# Ensure default project is valid (i.e. points to an existing project)
|
||||
# None means "no default" — intentionally left unset
|
||||
if (
|
||||
self.default_project is not None and self.default_project not in self.projects
|
||||
): # pragma: no cover
|
||||
# Set default to first available project
|
||||
self.default_project = next(iter(self.projects.keys()))
|
||||
|
||||
@property
|
||||
@@ -606,9 +541,6 @@ class BasicMemoryConfig(BaseSettings):
|
||||
|
||||
for name, entry in self.projects.items():
|
||||
path = Path(entry.path)
|
||||
# Skip cloud-only projects whose path is a slug, not a local directory
|
||||
if not path.is_absolute():
|
||||
continue
|
||||
if not path.exists():
|
||||
try:
|
||||
path.mkdir(parents=True)
|
||||
@@ -690,17 +622,6 @@ class ConfigManager:
|
||||
if isinstance(first_val, str):
|
||||
needs_resave = True
|
||||
|
||||
# Check if any project has local_sync_path set but path is a cloud slug
|
||||
# (will be migrated by migrate_legacy_projects validator)
|
||||
if not needs_resave:
|
||||
for entry_data in projects_raw.values():
|
||||
if isinstance(entry_data, dict):
|
||||
lsp = entry_data.get("local_sync_path")
|
||||
p = entry_data.get("path", "")
|
||||
if lsp and not os.path.isabs(p):
|
||||
needs_resave = True
|
||||
break
|
||||
|
||||
# First, create config from environment variables (Pydantic will read them)
|
||||
# Then overlay with file data for fields that aren't set via env vars
|
||||
# This ensures env vars take precedence
|
||||
@@ -725,27 +646,13 @@ class ConfigManager:
|
||||
|
||||
# Re-save to normalize legacy config into current format
|
||||
if needs_resave:
|
||||
# Create backup before overwriting so users can revert if needed
|
||||
backup_path = self.config_file.with_suffix(".json.bak")
|
||||
shutil.copy2(self.config_file, backup_path)
|
||||
logger.info(f"Migrating config to current format (backup: {backup_path})")
|
||||
logger.info("Migrating config to current format")
|
||||
save_basic_memory_config(self.config_file, _CONFIG_CACHE)
|
||||
|
||||
return _CONFIG_CACHE
|
||||
except json.JSONDecodeError as e: # pragma: no cover
|
||||
logger.error(f"Invalid JSON in config file {self.config_file}: {e}")
|
||||
raise SystemExit(
|
||||
f"Error: config file is not valid JSON: {self.config_file}\n"
|
||||
f" {e}\n"
|
||||
f"Fix or delete the file and re-run."
|
||||
)
|
||||
except Exception as e: # pragma: no cover
|
||||
logger.error(f"Failed to load config from {self.config_file}: {e}")
|
||||
raise SystemExit(
|
||||
f"Error: failed to load config from {self.config_file}\n"
|
||||
f" {e}\n"
|
||||
f"Fix or delete the file and re-run."
|
||||
)
|
||||
logger.exception(f"Failed to load config: {e}")
|
||||
raise e
|
||||
else:
|
||||
config = BasicMemoryConfig()
|
||||
self.save_config(config)
|
||||
@@ -778,8 +685,11 @@ class ConfigManager:
|
||||
if project_name: # pragma: no cover
|
||||
raise ValueError(f"Project '{name}' already exists")
|
||||
|
||||
# Load config, modify it, and save it
|
||||
# Ensure the path exists
|
||||
project_path = Path(path)
|
||||
project_path.mkdir(parents=True, exist_ok=True) # pragma: no cover
|
||||
|
||||
# Load config, modify it, and save it
|
||||
config = self.load_config()
|
||||
config.projects[name] = ProjectEntry(path=str(project_path))
|
||||
self.save_config(config)
|
||||
@@ -864,20 +774,6 @@ def get_project_config(project_name: Optional[str] = None) -> ProjectConfig:
|
||||
raise ValueError(f"Project '{actual_project_name}' not found") # pragma: no cover
|
||||
|
||||
|
||||
def has_cloud_credentials(config: BasicMemoryConfig) -> bool:
|
||||
"""Check if cloud credentials are available (API key or OAuth token).
|
||||
|
||||
Shared utility used by both MCP tools and CLI commands to determine
|
||||
whether cloud project discovery is possible.
|
||||
"""
|
||||
if config.cloud_api_key:
|
||||
return True
|
||||
from basic_memory.cli.auth import CLIAuth
|
||||
|
||||
auth = CLIAuth(client_id=config.cloud_client_id, authkit_domain=config.cloud_domain)
|
||||
return auth.load_tokens() is not None
|
||||
|
||||
|
||||
def save_basic_memory_config(file_path: Path, config: BasicMemoryConfig) -> None:
|
||||
"""Save configuration to file."""
|
||||
try:
|
||||
|
||||
+1
-116
@@ -44,101 +44,6 @@ _engine: Optional[AsyncEngine] = None
|
||||
_session_maker: Optional[async_sessionmaker[AsyncSession]] = None
|
||||
|
||||
|
||||
async def _needs_semantic_embedding_backfill(
|
||||
app_config: BasicMemoryConfig,
|
||||
session_maker: async_sessionmaker[AsyncSession],
|
||||
) -> bool:
|
||||
"""Check if entities exist but vector embeddings are empty.
|
||||
|
||||
This is the reliable way to detect that embeddings need to be generated,
|
||||
regardless of how migrations were applied (fresh DB, upgrade, reset, etc.).
|
||||
"""
|
||||
if not app_config.semantic_search_enabled:
|
||||
return False
|
||||
|
||||
try:
|
||||
async with scoped_session(session_maker) as session:
|
||||
entity_count = (
|
||||
await session.execute(text("SELECT COUNT(*) FROM entity"))
|
||||
).scalar() or 0
|
||||
if entity_count == 0:
|
||||
return False
|
||||
|
||||
# Check if vector chunks table exists and is empty
|
||||
embedding_count = (
|
||||
await session.execute(text("SELECT COUNT(*) FROM search_vector_chunks"))
|
||||
).scalar() or 0
|
||||
|
||||
return embedding_count == 0
|
||||
except Exception as exc:
|
||||
# Table might not exist yet (pre-migration)
|
||||
logger.debug(f"Could not check embedding status: {exc}")
|
||||
return False
|
||||
|
||||
|
||||
async def _run_semantic_embedding_backfill(
|
||||
app_config: BasicMemoryConfig,
|
||||
session_maker: async_sessionmaker[AsyncSession],
|
||||
) -> None:
|
||||
"""Backfill semantic embeddings for all active projects/entities."""
|
||||
if not app_config.semantic_search_enabled:
|
||||
logger.info("Skipping automatic semantic embedding backfill: semantic search is disabled.")
|
||||
return
|
||||
|
||||
async with scoped_session(session_maker) as session:
|
||||
project_result = await session.execute(
|
||||
text("SELECT id, name FROM project WHERE is_active = :is_active ORDER BY id"),
|
||||
{"is_active": True},
|
||||
)
|
||||
projects = [(int(row[0]), str(row[1])) for row in project_result.fetchall()]
|
||||
|
||||
if not projects:
|
||||
logger.info("Skipping automatic semantic embedding backfill: no active projects found.")
|
||||
return
|
||||
|
||||
repository_class = (
|
||||
PostgresSearchRepository
|
||||
if app_config.database_backend == DatabaseBackend.POSTGRES
|
||||
else SQLiteSearchRepository
|
||||
)
|
||||
|
||||
total_entities = 0
|
||||
for project_id, project_name in projects:
|
||||
async with scoped_session(session_maker) as session:
|
||||
entity_result = await session.execute(
|
||||
text("SELECT id FROM entity WHERE project_id = :project_id ORDER BY id"),
|
||||
{"project_id": project_id},
|
||||
)
|
||||
entity_ids = [int(row[0]) for row in entity_result.fetchall()]
|
||||
|
||||
if not entity_ids:
|
||||
continue
|
||||
|
||||
total_entities += len(entity_ids)
|
||||
logger.info(
|
||||
"Automatic semantic embedding backfill: "
|
||||
f"project={project_name}, entities={len(entity_ids)}"
|
||||
)
|
||||
|
||||
search_repository = repository_class(
|
||||
session_maker,
|
||||
project_id=project_id,
|
||||
app_config=app_config,
|
||||
)
|
||||
batch_result = await search_repository.sync_entity_vectors_batch(entity_ids)
|
||||
if batch_result.entities_failed > 0:
|
||||
logger.warning(
|
||||
"Automatic semantic embedding backfill encountered entity failures: "
|
||||
f"project={project_name}, failed={batch_result.entities_failed}, "
|
||||
f"failed_entity_ids={batch_result.failed_entity_ids}"
|
||||
)
|
||||
|
||||
logger.info(
|
||||
"Automatic semantic embedding backfill complete: "
|
||||
f"projects={len(projects)}, entities={total_entities}"
|
||||
)
|
||||
|
||||
|
||||
class DatabaseType(Enum):
|
||||
"""Types of supported databases."""
|
||||
|
||||
@@ -478,7 +383,6 @@ async def run_migrations(
|
||||
so it's safe to call this multiple times - it will only run pending migrations.
|
||||
"""
|
||||
logger.info("Running database migrations...")
|
||||
temp_engine: AsyncEngine | None = None
|
||||
try:
|
||||
# Get the absolute path to the alembic directory relative to this file
|
||||
alembic_dir = Path(__file__).parent / "alembic"
|
||||
@@ -503,9 +407,7 @@ async def run_migrations(
|
||||
|
||||
# Get session maker - ensure we don't trigger recursive migration calls
|
||||
if _session_maker is None:
|
||||
temp_engine, session_maker = _create_engine_and_session(
|
||||
app_config.database_path, database_type, app_config
|
||||
)
|
||||
_, session_maker = _create_engine_and_session(app_config.database_path, database_type)
|
||||
else:
|
||||
session_maker = _session_maker
|
||||
|
||||
@@ -520,23 +422,6 @@ async def run_migrations(
|
||||
await PostgresSearchRepository(session_maker, 1).init_search_index()
|
||||
else:
|
||||
await SQLiteSearchRepository(session_maker, 1).init_search_index()
|
||||
|
||||
# Check if backfill is needed — actual backfill runs in background
|
||||
# from the MCP server lifespan to avoid blocking startup.
|
||||
if await _needs_semantic_embedding_backfill(app_config, session_maker):
|
||||
logger.info(
|
||||
"Semantic embeddings missing — backfill will run in background after startup"
|
||||
)
|
||||
else:
|
||||
logger.info("Semantic embeddings: up to date")
|
||||
except Exception as e: # pragma: no cover
|
||||
logger.error(f"Error running migrations: {e}")
|
||||
raise
|
||||
finally:
|
||||
# Trigger: run_migrations() created a temporary engine while module-level
|
||||
# session maker was not initialized.
|
||||
# Why: temporary aiosqlite worker threads can outlive CLI command execution
|
||||
# and block process shutdown if the engine is not disposed.
|
||||
# Outcome: always dispose temporary engines after migration work completes.
|
||||
if temp_engine is not None:
|
||||
await temp_engine.dispose()
|
||||
|
||||
@@ -9,7 +9,6 @@ This module provides service-layer dependencies:
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Annotated, Any, Callable, Coroutine, Mapping, Protocol
|
||||
|
||||
from fastapi import Depends
|
||||
@@ -309,13 +308,11 @@ async def get_context_service(
|
||||
search_repository: SearchRepositoryDep,
|
||||
entity_repository: EntityRepositoryDep,
|
||||
observation_repository: ObservationRepositoryDep,
|
||||
link_resolver: LinkResolverDep,
|
||||
) -> ContextService:
|
||||
return ContextService(
|
||||
search_repository=search_repository,
|
||||
entity_repository=entity_repository,
|
||||
observation_repository=observation_repository,
|
||||
link_resolver=link_resolver,
|
||||
)
|
||||
|
||||
|
||||
@@ -326,14 +323,12 @@ async def get_context_service_v2( # pragma: no cover
|
||||
search_repository: SearchRepositoryV2Dep,
|
||||
entity_repository: EntityRepositoryV2Dep,
|
||||
observation_repository: ObservationRepositoryV2Dep,
|
||||
link_resolver: LinkResolverV2Dep,
|
||||
) -> ContextService:
|
||||
"""Create ContextService for v2 API."""
|
||||
return ContextService(
|
||||
search_repository=search_repository,
|
||||
entity_repository=entity_repository,
|
||||
observation_repository=observation_repository,
|
||||
link_resolver=link_resolver,
|
||||
)
|
||||
|
||||
|
||||
@@ -344,14 +339,12 @@ async def get_context_service_v2_external(
|
||||
search_repository: SearchRepositoryV2ExternalDep,
|
||||
entity_repository: EntityRepositoryV2ExternalDep,
|
||||
observation_repository: ObservationRepositoryV2ExternalDep,
|
||||
link_resolver: LinkResolverV2ExternalDep,
|
||||
) -> ContextService:
|
||||
"""Create ContextService for v2 API (uses external_id)."""
|
||||
return ContextService(
|
||||
search_repository=search_repository,
|
||||
entity_repository=entity_repository,
|
||||
observation_repository=observation_repository,
|
||||
link_resolver=link_resolver,
|
||||
)
|
||||
|
||||
|
||||
@@ -556,15 +549,9 @@ TaskSchedulerDep = Annotated[TaskScheduler, Depends(get_task_scheduler)]
|
||||
|
||||
async def get_project_service(
|
||||
project_repository: ProjectRepositoryDep,
|
||||
app_config: AppConfigDep,
|
||||
) -> ProjectService:
|
||||
"""Create ProjectService with repository and a system-level FileService for directory operations."""
|
||||
# A system-level FileService for project directory creation (no project-specific base_path needed).
|
||||
# ensure_directory() accepts absolute paths and ignores base_path for those, so Path.home() is safe.
|
||||
entity_parser = EntityParser(Path.home())
|
||||
markdown_processor = MarkdownProcessor(entity_parser, app_config=app_config)
|
||||
file_service = FileService(Path.home(), markdown_processor, app_config=app_config)
|
||||
return ProjectService(repository=project_repository, file_service=file_service)
|
||||
"""Create ProjectService with repository."""
|
||||
return ProjectService(repository=project_repository)
|
||||
|
||||
|
||||
ProjectServiceDep = Annotated[ProjectService, Depends(get_project_service)]
|
||||
|
||||
@@ -88,22 +88,6 @@ def normalize_frontmatter_value(value: Any) -> Any:
|
||||
return value
|
||||
|
||||
|
||||
def _coerce_to_string(value: Any) -> str:
|
||||
"""Coerce a frontmatter value to a string.
|
||||
|
||||
YAML can parse scalar-looking fields as lists when the author uses block
|
||||
sequence syntax. For fields like ``title`` and ``type`` that *must* be
|
||||
strings, this helper converts lists to a comma-separated string and any
|
||||
other non-string type via ``str()``.
|
||||
"""
|
||||
if isinstance(value, str):
|
||||
return value
|
||||
if isinstance(value, list):
|
||||
# Join list items, converting each to string first
|
||||
return ", ".join(str(item) for item in value)
|
||||
return str(value)
|
||||
|
||||
|
||||
def normalize_frontmatter_metadata(metadata: dict) -> dict:
|
||||
"""Normalize all values in frontmatter metadata dict.
|
||||
|
||||
@@ -249,15 +233,9 @@ class EntityParser:
|
||||
|
||||
content = strip_bom(content)
|
||||
|
||||
# Parse frontmatter with proper error handling for malformed YAML.
|
||||
# We use frontmatter.parse() instead of frontmatter.loads() because
|
||||
# loads() does Post(content, handler, **metadata), which crashes when
|
||||
# the YAML contains reserved keys like 'content' or 'handler'.
|
||||
# See basic-memory-cloud#375.
|
||||
# Parse frontmatter with proper error handling for malformed YAML
|
||||
try:
|
||||
fm_metadata, fm_content = frontmatter.parse(content)
|
||||
post = frontmatter.Post(fm_content)
|
||||
post.metadata.update(fm_metadata)
|
||||
post = frontmatter.loads(content)
|
||||
except yaml.YAMLError as e:
|
||||
logger.warning(
|
||||
f"Failed to parse YAML frontmatter in {file_path}: {e}. "
|
||||
@@ -270,22 +248,15 @@ class EntityParser:
|
||||
# Normalize frontmatter values
|
||||
metadata = normalize_frontmatter_metadata(post.metadata)
|
||||
|
||||
# Ensure required string fields are always strings.
|
||||
# YAML can parse these as lists when authors use block sequence syntax
|
||||
# (e.g. "title:\n - My Title"), causing 'list' has no attribute 'strip'
|
||||
# downstream. See basic-memory-cloud#376.
|
||||
# Ensure required fields have defaults
|
||||
title = metadata.get("title")
|
||||
if title is not None:
|
||||
title = _coerce_to_string(title)
|
||||
if not title or title == "None":
|
||||
metadata["title"] = file_path.stem
|
||||
else:
|
||||
metadata["title"] = title
|
||||
|
||||
note_type = metadata.get("type")
|
||||
if note_type is not None:
|
||||
note_type = _coerce_to_string(note_type)
|
||||
metadata["type"] = note_type if note_type is not None else "note"
|
||||
entity_type = metadata.get("type")
|
||||
metadata["type"] = entity_type if entity_type is not None else "note"
|
||||
|
||||
tags = parse_tags(metadata.get("tags", [])) # pyright: ignore
|
||||
if tags:
|
||||
|
||||
@@ -50,7 +50,7 @@ def entity_model_from_markdown(
|
||||
|
||||
# Update basic fields
|
||||
model.title = markdown.frontmatter.title
|
||||
model.note_type = markdown.frontmatter.type
|
||||
model.entity_type = markdown.frontmatter.type
|
||||
# Only update permalink if it exists in frontmatter, otherwise preserve existing
|
||||
if markdown.frontmatter.permalink is not None:
|
||||
model.permalink = markdown.frontmatter.permalink
|
||||
@@ -86,7 +86,7 @@ async def schema_to_markdown(schema: Any) -> Post:
|
||||
Convert schema to markdown Post object.
|
||||
|
||||
Args:
|
||||
schema: Schema to convert (must have title, note_type, and permalink attributes)
|
||||
schema: Schema to convert (must have title, entity_type, and permalink attributes)
|
||||
|
||||
Returns:
|
||||
Post object with frontmatter metadata
|
||||
@@ -113,7 +113,7 @@ async def schema_to_markdown(schema: Any) -> Post:
|
||||
post = Post(
|
||||
content,
|
||||
title=schema.title,
|
||||
type=schema.note_type,
|
||||
type=schema.entity_type,
|
||||
)
|
||||
# set the permalink if passed in
|
||||
if schema.permalink:
|
||||
|
||||
@@ -56,7 +56,7 @@ async def _resolve_cloud_token(config) -> str:
|
||||
|
||||
raise RuntimeError(
|
||||
"Cloud routing requested but no credentials found. "
|
||||
"Run 'bm cloud api-key save <key>' or 'bm cloud login' first."
|
||||
"Run 'bm cloud set-key <key>' or 'bm cloud login' first."
|
||||
)
|
||||
|
||||
|
||||
@@ -106,27 +106,6 @@ def set_client_factory(factory: Callable[[], AbstractAsyncContextManager[AsyncCl
|
||||
_client_factory = factory
|
||||
|
||||
|
||||
def is_factory_mode() -> bool:
|
||||
"""Return True when a client factory override is active (e.g., cloud app)."""
|
||||
return _client_factory is not None
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def get_cloud_proxy_client(
|
||||
workspace: Optional[str] = None,
|
||||
) -> AsyncIterator[AsyncClient]:
|
||||
"""Create a cloud proxy client for project-level operations.
|
||||
|
||||
Used by MCP tools to fetch cloud project lists independently of the
|
||||
default get_client() routing, which always goes through the local ASGI
|
||||
transport in stdio mode.
|
||||
"""
|
||||
config = ConfigManager().config
|
||||
timeout = _build_timeout()
|
||||
async with _cloud_client(config, timeout, workspace=workspace) as client:
|
||||
yield client
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def get_client(
|
||||
project_name: Optional[str] = None,
|
||||
@@ -154,13 +133,13 @@ async def get_client(
|
||||
# Outcome: route strictly based on explicit flag.
|
||||
if _explicit_routing():
|
||||
if _force_local_mode():
|
||||
logger.debug("Explicit local routing enabled - using ASGI client")
|
||||
logger.info("Explicit local routing enabled - using ASGI client")
|
||||
async with _asgi_client(timeout) as client:
|
||||
yield client
|
||||
return
|
||||
|
||||
if _force_cloud_mode():
|
||||
logger.debug("Explicit cloud routing enabled - using cloud proxy client")
|
||||
logger.info("Explicit cloud routing enabled - using cloud proxy client")
|
||||
async with _cloud_client(config, timeout, workspace=workspace) as client:
|
||||
yield client
|
||||
return
|
||||
@@ -172,24 +151,24 @@ async def get_client(
|
||||
if project_name is not None and not _explicit_routing():
|
||||
project_mode = config.get_project_mode(project_name)
|
||||
if project_mode == ProjectMode.CLOUD:
|
||||
logger.debug(f"Project '{project_name}' is cloud mode - using cloud proxy client")
|
||||
logger.info(f"Project '{project_name}' is cloud mode - using cloud proxy client")
|
||||
try:
|
||||
async with _cloud_client(config, timeout, workspace=workspace) as client:
|
||||
yield client
|
||||
except RuntimeError as exc:
|
||||
raise RuntimeError(
|
||||
f"Project '{project_name}' is set to cloud mode but no credentials found. "
|
||||
"Run 'bm cloud api-key save <key>' or 'bm cloud login' first."
|
||||
"Run 'bm cloud set-key <key>' or 'bm cloud login' first."
|
||||
) from exc
|
||||
return
|
||||
|
||||
logger.debug(f"Project '{project_name}' is local mode - using ASGI client")
|
||||
logger.info(f"Project '{project_name}' is local mode - using ASGI client")
|
||||
async with _asgi_client(timeout) as client:
|
||||
yield client
|
||||
return
|
||||
|
||||
# --- Default fallback ---
|
||||
logger.debug("Default routing - using ASGI client for local Basic Memory API")
|
||||
logger.info("Default routing - using ASGI client for local Basic Memory API")
|
||||
async with _asgi_client(timeout) as client:
|
||||
yield client
|
||||
|
||||
|
||||
@@ -7,16 +7,8 @@ from typing import Any
|
||||
|
||||
from httpx import AsyncClient
|
||||
|
||||
from basic_memory.mcp.tools.utils import (
|
||||
call_delete,
|
||||
call_get,
|
||||
call_patch,
|
||||
call_post,
|
||||
call_put,
|
||||
)
|
||||
from basic_memory.schemas import ProjectInfoResponse, SyncReportResponse
|
||||
from basic_memory.mcp.tools.utils import call_get, call_post, call_delete
|
||||
from basic_memory.schemas.project_info import ProjectList, ProjectStatusResponse
|
||||
from basic_memory.schemas.v2 import ProjectResolveResponse
|
||||
|
||||
|
||||
class ProjectClient:
|
||||
@@ -78,14 +70,11 @@ class ProjectClient:
|
||||
)
|
||||
return ProjectStatusResponse.model_validate(response.json())
|
||||
|
||||
async def delete_project(
|
||||
self, project_external_id: str, delete_notes: bool = False
|
||||
) -> ProjectStatusResponse:
|
||||
async def delete_project(self, project_external_id: str) -> ProjectStatusResponse:
|
||||
"""Delete a project by its external ID.
|
||||
|
||||
Args:
|
||||
project_external_id: Project external ID (UUID)
|
||||
delete_notes: If True, also delete project files from disk
|
||||
|
||||
Returns:
|
||||
ProjectStatusResponse with deletion result
|
||||
@@ -93,137 +82,8 @@ class ProjectClient:
|
||||
Raises:
|
||||
ToolError: If the request fails
|
||||
"""
|
||||
url = f"/v2/projects/{project_external_id}"
|
||||
if delete_notes:
|
||||
url += "?delete_notes=true"
|
||||
response = await call_delete(
|
||||
self.http_client,
|
||||
url,
|
||||
)
|
||||
return ProjectStatusResponse.model_validate(response.json())
|
||||
|
||||
async def resolve_project(self, identifier: str) -> ProjectResolveResponse:
|
||||
"""Resolve a project name/permalink to its full project record.
|
||||
|
||||
Args:
|
||||
identifier: Project name or permalink
|
||||
|
||||
Returns:
|
||||
ProjectResolveResponse with project metadata
|
||||
|
||||
Raises:
|
||||
ToolError: If the request fails
|
||||
"""
|
||||
response = await call_post(
|
||||
self.http_client,
|
||||
"/v2/projects/resolve",
|
||||
json={"identifier": identifier},
|
||||
)
|
||||
return ProjectResolveResponse.model_validate(response.json())
|
||||
|
||||
async def set_default(self, project_external_id: str) -> ProjectStatusResponse:
|
||||
"""Set a project as the default.
|
||||
|
||||
Args:
|
||||
project_external_id: Project external ID (UUID)
|
||||
|
||||
Returns:
|
||||
ProjectStatusResponse with result
|
||||
|
||||
Raises:
|
||||
ToolError: If the request fails
|
||||
"""
|
||||
response = await call_put(
|
||||
self.http_client,
|
||||
f"/v2/projects/{project_external_id}/default",
|
||||
)
|
||||
return ProjectStatusResponse.model_validate(response.json())
|
||||
|
||||
async def update_project(
|
||||
self, project_external_id: str, data: dict[str, Any]
|
||||
) -> ProjectStatusResponse:
|
||||
"""Update a project's configuration (e.g. path).
|
||||
|
||||
Args:
|
||||
project_external_id: Project external ID (UUID)
|
||||
data: Fields to update
|
||||
|
||||
Returns:
|
||||
ProjectStatusResponse with update result
|
||||
|
||||
Raises:
|
||||
ToolError: If the request fails
|
||||
"""
|
||||
response = await call_patch(
|
||||
self.http_client,
|
||||
f"/v2/projects/{project_external_id}",
|
||||
json=data,
|
||||
)
|
||||
return ProjectStatusResponse.model_validate(response.json())
|
||||
|
||||
async def sync(
|
||||
self,
|
||||
project_external_id: str,
|
||||
force_full: bool = False,
|
||||
run_in_background: bool = True,
|
||||
) -> dict[str, Any]:
|
||||
"""Trigger a sync operation for a project.
|
||||
|
||||
Args:
|
||||
project_external_id: Project external ID (UUID)
|
||||
force_full: If True, force a full scan bypassing watermark optimization
|
||||
run_in_background: If True, return immediately; if False, wait for completion
|
||||
|
||||
Returns:
|
||||
Raw response dict — background mode returns {"message": ...},
|
||||
foreground mode returns a SyncReportResponse-shaped dict.
|
||||
|
||||
Raises:
|
||||
ToolError: If the request fails
|
||||
"""
|
||||
url = f"/v2/projects/{project_external_id}/sync"
|
||||
params = []
|
||||
if force_full:
|
||||
params.append("force_full=true")
|
||||
if not run_in_background:
|
||||
params.append("run_in_background=false")
|
||||
if params:
|
||||
url += "?" + "&".join(params)
|
||||
response = await call_post(self.http_client, url)
|
||||
return response.json()
|
||||
|
||||
async def get_status(self, project_external_id: str) -> SyncReportResponse:
|
||||
"""Get the sync status for a project.
|
||||
|
||||
Args:
|
||||
project_external_id: Project external ID (UUID)
|
||||
|
||||
Returns:
|
||||
SyncReportResponse describing pending changes
|
||||
|
||||
Raises:
|
||||
ToolError: If the request fails
|
||||
"""
|
||||
response = await call_post(
|
||||
self.http_client,
|
||||
f"/v2/projects/{project_external_id}/status",
|
||||
)
|
||||
return SyncReportResponse.model_validate(response.json())
|
||||
|
||||
async def get_info(self, project_external_id: str) -> ProjectInfoResponse:
|
||||
"""Get detailed project information and statistics.
|
||||
|
||||
Args:
|
||||
project_external_id: Project external ID (UUID)
|
||||
|
||||
Returns:
|
||||
ProjectInfoResponse with project details
|
||||
|
||||
Raises:
|
||||
ToolError: If the request fails
|
||||
"""
|
||||
response = await call_get(
|
||||
self.http_client,
|
||||
f"/v2/projects/{project_external_id}/info",
|
||||
)
|
||||
return ProjectInfoResponse.model_validate(response.json())
|
||||
|
||||
@@ -24,7 +24,7 @@ class SchemaClient:
|
||||
Usage:
|
||||
async with get_client() as http_client:
|
||||
client = SchemaClient(http_client, project_id)
|
||||
report = await client.validate(note_type="person")
|
||||
report = await client.validate(entity_type="Person")
|
||||
"""
|
||||
|
||||
def __init__(self, http_client: AsyncClient, project_id: str):
|
||||
@@ -41,13 +41,13 @@ class SchemaClient:
|
||||
async def validate(
|
||||
self,
|
||||
*,
|
||||
note_type: str | None = None,
|
||||
entity_type: str | None = None,
|
||||
identifier: str | None = None,
|
||||
) -> ValidationReport:
|
||||
"""Validate notes against their resolved schemas.
|
||||
|
||||
Args:
|
||||
note_type: Optional note type to batch-validate
|
||||
entity_type: Optional entity type to batch-validate
|
||||
identifier: Optional specific note to validate
|
||||
|
||||
Returns:
|
||||
@@ -57,8 +57,8 @@ class SchemaClient:
|
||||
ToolError: If the request fails
|
||||
"""
|
||||
params: dict[str, str] = {}
|
||||
if note_type:
|
||||
params["note_type"] = note_type
|
||||
if entity_type:
|
||||
params["entity_type"] = entity_type
|
||||
if identifier:
|
||||
params["identifier"] = identifier
|
||||
|
||||
@@ -71,14 +71,14 @@ class SchemaClient:
|
||||
|
||||
async def infer(
|
||||
self,
|
||||
note_type: str,
|
||||
entity_type: str,
|
||||
*,
|
||||
threshold: float = 0.25,
|
||||
) -> InferenceReport:
|
||||
"""Infer a schema from existing notes of a given type.
|
||||
|
||||
Args:
|
||||
note_type: The note type to analyze
|
||||
entity_type: The entity type to analyze
|
||||
threshold: Minimum frequency for optional fields (0-1)
|
||||
|
||||
Returns:
|
||||
@@ -90,15 +90,15 @@ class SchemaClient:
|
||||
response = await call_post(
|
||||
self.http_client,
|
||||
f"{self._base_path}/infer",
|
||||
params={"note_type": note_type, "threshold": threshold},
|
||||
params={"entity_type": entity_type, "threshold": threshold},
|
||||
)
|
||||
return InferenceReport.model_validate(response.json())
|
||||
|
||||
async def diff(self, note_type: str) -> DriftReport:
|
||||
async def diff(self, entity_type: str) -> DriftReport:
|
||||
"""Show drift between schema definition and actual usage.
|
||||
|
||||
Args:
|
||||
note_type: The note type to check for drift
|
||||
entity_type: The entity type to check for drift
|
||||
|
||||
Returns:
|
||||
DriftReport with detected differences
|
||||
@@ -108,6 +108,6 @@ class SchemaClient:
|
||||
"""
|
||||
response = await call_get(
|
||||
self.http_client,
|
||||
f"{self._base_path}/diff/{note_type}",
|
||||
f"{self._base_path}/diff/{entity_type}",
|
||||
)
|
||||
return DriftReport.model_validate(response.json())
|
||||
|
||||
@@ -9,7 +9,7 @@ compatibility with existing MCP tools.
|
||||
"""
|
||||
|
||||
from contextlib import asynccontextmanager
|
||||
from typing import AsyncIterator, Awaitable, Callable, Optional, List, Tuple
|
||||
from typing import AsyncIterator, Optional, List, Tuple
|
||||
|
||||
from httpx import AsyncClient
|
||||
from httpx._types import (
|
||||
@@ -19,7 +19,7 @@ from loguru import logger
|
||||
from fastmcp import Context
|
||||
from mcp.server.fastmcp.exceptions import ToolError
|
||||
|
||||
from basic_memory.config import BasicMemoryConfig, ConfigManager, ProjectMode
|
||||
from basic_memory.config import ConfigManager, ProjectMode
|
||||
from basic_memory.project_resolver import ProjectResolver
|
||||
from basic_memory.schemas.cloud import WorkspaceInfo, WorkspaceListResponse
|
||||
from basic_memory.schemas.project_info import ProjectItem, ProjectList
|
||||
@@ -27,41 +27,6 @@ from basic_memory.schemas.v2 import ProjectResolveResponse
|
||||
from basic_memory.schemas.memory import memory_url_path
|
||||
from basic_memory.utils import generate_permalink, normalize_project_reference
|
||||
|
||||
# --- Workspace provider injection ---
|
||||
# Mirrors the set_client_factory() pattern in async_client.py.
|
||||
# The cloud MCP server sets a provider that queries its own database directly,
|
||||
# avoiding the control-plane HTTP round-trip that requires local credentials.
|
||||
_workspace_provider: Optional[Callable[[], Awaitable[list[WorkspaceInfo]]]] = None
|
||||
|
||||
|
||||
def set_workspace_provider(provider: Callable[[], Awaitable[list[WorkspaceInfo]]]) -> None:
|
||||
"""Override workspace discovery (for cloud app, testing, etc)."""
|
||||
global _workspace_provider
|
||||
_workspace_provider = provider
|
||||
|
||||
|
||||
async def _resolve_default_project_from_api() -> Optional[str]:
|
||||
"""Query the projects API for the default project.
|
||||
|
||||
Used as a fallback when ConfigManager has no local config (cloud mode).
|
||||
"""
|
||||
from basic_memory.mcp.async_client import get_client
|
||||
|
||||
try:
|
||||
async with get_client() as client:
|
||||
response = await client.get("/v2/projects/")
|
||||
if response.status_code == 200:
|
||||
project_list = ProjectList.model_validate(response.json())
|
||||
if project_list.default_project:
|
||||
return project_list.default_project
|
||||
# Fallback: find project with is_default=True
|
||||
for p in project_list.projects:
|
||||
if p.is_default:
|
||||
return p.name
|
||||
except Exception:
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
async def resolve_project_parameter(
|
||||
project: Optional[str] = None,
|
||||
@@ -89,16 +54,11 @@ async def resolve_project_parameter(
|
||||
Returns:
|
||||
Resolved project name or None if no resolution possible
|
||||
"""
|
||||
# Load config for any values not explicitly provided.
|
||||
# ConfigManager reads from the local config file, which doesn't exist in cloud mode.
|
||||
# When it returns None, fall back to querying the projects API for the is_default flag.
|
||||
# Load config for any values not explicitly provided
|
||||
if default_project is None:
|
||||
config = ConfigManager().config
|
||||
default_project = config.default_project
|
||||
|
||||
if default_project is None:
|
||||
default_project = await _resolve_default_project_from_api()
|
||||
|
||||
# Create resolver with configuration and resolve
|
||||
resolver = ProjectResolver.from_env(
|
||||
default_project=default_project,
|
||||
@@ -139,22 +99,11 @@ def _workspace_choices(workspaces: list[WorkspaceInfo]) -> str:
|
||||
async def get_available_workspaces(context: Optional[Context] = None) -> list[WorkspaceInfo]:
|
||||
"""Load available cloud workspaces for the current authenticated user."""
|
||||
if context:
|
||||
cached_raw = await context.get_state("available_workspaces")
|
||||
if isinstance(cached_raw, list):
|
||||
return [WorkspaceInfo.model_validate(item) for item in cached_raw]
|
||||
|
||||
# Trigger: workspace provider was injected (e.g., by cloud MCP server)
|
||||
# Why: the cloud server IS the cloud — it can query its own database
|
||||
# directly instead of making an HTTP round-trip that requires local credentials
|
||||
# Outcome: use provider result, cache in context, skip control-plane client
|
||||
if _workspace_provider is not None:
|
||||
workspaces = await _workspace_provider()
|
||||
if context:
|
||||
await context.set_state(
|
||||
"available_workspaces",
|
||||
[ws.model_dump() for ws in workspaces],
|
||||
)
|
||||
return workspaces
|
||||
cached_workspaces = context.get_state("available_workspaces")
|
||||
if isinstance(cached_workspaces, list) and all(
|
||||
isinstance(item, WorkspaceInfo) for item in cached_workspaces
|
||||
):
|
||||
return cached_workspaces
|
||||
|
||||
from basic_memory.mcp.async_client import get_cloud_control_plane_client
|
||||
from basic_memory.mcp.tools.utils import call_get
|
||||
@@ -164,10 +113,7 @@ async def get_available_workspaces(context: Optional[Context] = None) -> list[Wo
|
||||
workspace_list = WorkspaceListResponse.model_validate(response.json())
|
||||
|
||||
if context:
|
||||
await context.set_state(
|
||||
"available_workspaces",
|
||||
[ws.model_dump() for ws in workspace_list.workspaces],
|
||||
)
|
||||
context.set_state("available_workspaces", workspace_list.workspaces)
|
||||
|
||||
return workspace_list.workspaces
|
||||
|
||||
@@ -178,12 +124,12 @@ async def resolve_workspace_parameter(
|
||||
) -> WorkspaceInfo:
|
||||
"""Resolve workspace using explicit input, session cache, and cloud discovery."""
|
||||
if context:
|
||||
cached_raw = await context.get_state("active_workspace")
|
||||
if isinstance(cached_raw, dict):
|
||||
cached_workspace = WorkspaceInfo.model_validate(cached_raw)
|
||||
if workspace is None or _workspace_matches_identifier(cached_workspace, workspace):
|
||||
logger.debug(f"Using cached workspace from context: {cached_workspace.tenant_id}")
|
||||
return cached_workspace
|
||||
cached_workspace = context.get_state("active_workspace")
|
||||
if isinstance(cached_workspace, WorkspaceInfo) and (
|
||||
workspace is None or _workspace_matches_identifier(cached_workspace, workspace)
|
||||
):
|
||||
logger.debug(f"Using cached workspace from context: {cached_workspace.tenant_id}")
|
||||
return cached_workspace
|
||||
|
||||
workspaces = await get_available_workspaces(context=context)
|
||||
if not workspaces:
|
||||
@@ -218,7 +164,7 @@ async def resolve_workspace_parameter(
|
||||
)
|
||||
|
||||
if context:
|
||||
await context.set_state("active_workspace", selected_workspace.model_dump())
|
||||
context.set_state("active_workspace", selected_workspace)
|
||||
logger.debug(f"Cached workspace in context: {selected_workspace.tenant_id}")
|
||||
|
||||
return selected_workspace
|
||||
@@ -260,12 +206,10 @@ async def get_active_project(
|
||||
|
||||
# Check if already cached in context
|
||||
if context:
|
||||
cached_raw = await context.get_state("active_project")
|
||||
if isinstance(cached_raw, dict):
|
||||
cached_project = ProjectItem.model_validate(cached_raw)
|
||||
if cached_project.name == project:
|
||||
logger.debug(f"Using cached project from context: {project}")
|
||||
return cached_project
|
||||
cached_project = context.get_state("active_project")
|
||||
if cached_project and cached_project.name == project:
|
||||
logger.debug(f"Using cached project from context: {project}")
|
||||
return cached_project
|
||||
|
||||
# Validate project exists by calling API
|
||||
logger.debug(f"Validating project: {project}")
|
||||
@@ -286,7 +230,7 @@ async def get_active_project(
|
||||
|
||||
# Cache in context if available
|
||||
if context:
|
||||
await context.set_state("active_project", active_project.model_dump())
|
||||
context.set_state("active_project", active_project)
|
||||
logger.debug(f"Cached project in context: {project}")
|
||||
|
||||
logger.debug(f"Validated project: {active_project.name}")
|
||||
@@ -363,7 +307,7 @@ async def resolve_project_and_path(
|
||||
is_default=resolved.is_default,
|
||||
)
|
||||
if context:
|
||||
await context.set_state("active_project", active_project.model_dump())
|
||||
context.set_state("active_project", active_project)
|
||||
|
||||
resolved_path = f"{resolved.permalink}/{remainder}" if include_project else remainder
|
||||
return active_project, resolved_path, True
|
||||
@@ -399,35 +343,6 @@ def add_project_metadata(result: str, project_name: str) -> str:
|
||||
return f"{result}\n\n[Session: Using project '{project_name}']"
|
||||
|
||||
|
||||
def detect_project_from_url_prefix(identifier: str, config: BasicMemoryConfig) -> Optional[str]:
|
||||
"""Check if a memory URL's first path segment matches a known project in config.
|
||||
|
||||
This enables automatic project routing from memory URLs like
|
||||
``memory://specs/in-progress`` without requiring the caller to pass
|
||||
an explicit ``project`` parameter.
|
||||
|
||||
Uses local config only — no network calls.
|
||||
|
||||
Args:
|
||||
identifier: Raw identifier string (may or may not start with ``memory://``).
|
||||
config: Current BasicMemoryConfig with project entries.
|
||||
|
||||
Returns:
|
||||
Matching project name from config, or None if no match.
|
||||
"""
|
||||
path = memory_url_path(identifier) if identifier.strip().startswith("memory://") else identifier
|
||||
normalized = normalize_project_reference(path)
|
||||
prefix, _ = _split_project_prefix(normalized)
|
||||
if prefix is None:
|
||||
return None
|
||||
|
||||
prefix_permalink = generate_permalink(prefix)
|
||||
for project_name in config.projects:
|
||||
if generate_permalink(project_name) == prefix_permalink:
|
||||
return project_name
|
||||
return None
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def get_project_client(
|
||||
project: Optional[str] = None,
|
||||
@@ -441,20 +356,6 @@ async def get_project_client(
|
||||
the project. This helper resolves the project from config first (no
|
||||
network), creates the correctly-routed client, then validates via API.
|
||||
|
||||
Routing decision order:
|
||||
1. Explicit --local/--cloud flags → skip workspace, use flag routing
|
||||
2. Cloud routing (explicit --cloud OR project mode CLOUD) →
|
||||
resolve workspace via priority chain, create cloud client
|
||||
3. Otherwise → local ASGI client
|
||||
|
||||
Workspace resolution priority (when cloud routing):
|
||||
1. Explicit ``workspace`` parameter
|
||||
2. Per-project ``workspace_id`` from config
|
||||
3. Global ``default_workspace`` from config
|
||||
4. MCP session cache (context)
|
||||
5. Auto-select if single workspace
|
||||
6. Error listing choices
|
||||
|
||||
Args:
|
||||
project: Optional explicit project parameter
|
||||
workspace: Optional cloud workspace selector (tenant_id or unique name)
|
||||
@@ -467,13 +368,8 @@ async def get_project_client(
|
||||
ValueError: If no project can be resolved
|
||||
RuntimeError: If cloud project but no API key configured
|
||||
"""
|
||||
# Deferred imports to avoid circular dependency
|
||||
from basic_memory.mcp.async_client import (
|
||||
_explicit_routing,
|
||||
_force_local_mode,
|
||||
get_client,
|
||||
is_factory_mode,
|
||||
)
|
||||
# Deferred import to avoid circular dependency
|
||||
from basic_memory.mcp.async_client import get_client
|
||||
|
||||
# Step 1: Resolve project name from config (no network call)
|
||||
resolved_project = await resolve_project_parameter(project)
|
||||
@@ -487,76 +383,28 @@ async def get_project_client(
|
||||
f"Available projects: {project_names}"
|
||||
)
|
||||
|
||||
# Step 1b: Factory injection (in-process cloud server)
|
||||
# Trigger: set_client_factory() was called (e.g., by cloud MCP server)
|
||||
# Why: the transport layer already resolved workspace and tenant context;
|
||||
# attempting cloud workspace resolution here would call the production
|
||||
# control-plane API with no valid credentials and fail with 401
|
||||
# Outcome: use the factory client directly, skip workspace resolution
|
||||
if is_factory_mode():
|
||||
async with get_client() as client:
|
||||
active_project = await get_active_project(client, resolved_project, context)
|
||||
yield client, active_project
|
||||
return
|
||||
|
||||
# Step 2: Check explicit routing BEFORE workspace resolution
|
||||
# Trigger: CLI passed --local or --cloud
|
||||
# Why: explicit flags must be deterministic — skip workspace entirely for --local
|
||||
# Outcome: route strictly based on explicit flag, no workspace network calls
|
||||
if _explicit_routing() and _force_local_mode():
|
||||
async with get_client(project_name=resolved_project) as client:
|
||||
active_project = await get_active_project(client, resolved_project, context)
|
||||
yield client, active_project
|
||||
return
|
||||
|
||||
# Step 3: Determine if cloud routing is needed
|
||||
# Step 2: Resolve project mode and optional workspace selection
|
||||
config = ConfigManager().config
|
||||
project_entry = config.projects.get(resolved_project)
|
||||
project_mode = config.get_project_mode(resolved_project)
|
||||
active_workspace: WorkspaceInfo | None = None
|
||||
|
||||
# Trigger: workspace provided for a local project (without explicit --cloud)
|
||||
# Trigger: workspace provided for a local project
|
||||
# Why: workspace selection is a cloud routing concern only
|
||||
# Outcome: fail fast with a deterministic guidance message
|
||||
if project_mode != ProjectMode.CLOUD and workspace is not None and not _explicit_routing():
|
||||
if project_mode != ProjectMode.CLOUD and workspace is not None:
|
||||
raise ValueError(
|
||||
f"Workspace '{workspace}' cannot be used with local project '{resolved_project}'. "
|
||||
"Workspace selection is only supported for cloud-mode projects."
|
||||
)
|
||||
|
||||
if project_mode == ProjectMode.CLOUD or (_explicit_routing() and not _force_local_mode()):
|
||||
# --- Cloud routing: resolve workspace with priority chain ---
|
||||
effective_workspace = workspace
|
||||
if project_mode == ProjectMode.CLOUD:
|
||||
active_workspace = await resolve_workspace_parameter(workspace=workspace, context=context)
|
||||
|
||||
# Priority 2: per-project workspace_id from config
|
||||
if effective_workspace is None and project_entry and project_entry.workspace_id:
|
||||
effective_workspace = project_entry.workspace_id
|
||||
|
||||
# Priority 3: global default_workspace from config
|
||||
if effective_workspace is None and config.default_workspace:
|
||||
effective_workspace = config.default_workspace
|
||||
|
||||
# Priorities 4-6: if still unresolved, fall back to resolve_workspace_parameter
|
||||
# which checks context cache, auto-selects single workspace, or errors
|
||||
if effective_workspace is not None:
|
||||
# Config-resolved workspace — pass directly to get_client, skip network lookup
|
||||
async with get_client(
|
||||
project_name=resolved_project,
|
||||
workspace=effective_workspace,
|
||||
) as client:
|
||||
active_project = await get_active_project(client, resolved_project, context)
|
||||
yield client, active_project
|
||||
else:
|
||||
# No config-based workspace — use resolve_workspace_parameter for discovery
|
||||
active_ws = await resolve_workspace_parameter(workspace=None, context=context)
|
||||
async with get_client(
|
||||
project_name=resolved_project,
|
||||
workspace=active_ws.tenant_id,
|
||||
) as client:
|
||||
active_project = await get_active_project(client, resolved_project, context)
|
||||
yield client, active_project
|
||||
return
|
||||
|
||||
# Step 4: Local routing (default)
|
||||
async with get_client(project_name=resolved_project) as client:
|
||||
# Step 2: Create client routed based on project's mode
|
||||
async with get_client(
|
||||
project_name=resolved_project,
|
||||
workspace=active_workspace.tenant_id if active_workspace else None,
|
||||
) as client:
|
||||
# Step 3: Validate project exists via API
|
||||
active_project = await get_active_project(client, resolved_project, context)
|
||||
yield client, active_project
|
||||
|
||||
@@ -4,15 +4,17 @@ These prompts help users continue conversations and work across sessions,
|
||||
providing context from previous interactions to maintain continuity.
|
||||
"""
|
||||
|
||||
from textwrap import dedent
|
||||
from typing import Annotated, Optional
|
||||
|
||||
from loguru import logger
|
||||
from pydantic import Field
|
||||
|
||||
from basic_memory.config import ConfigManager
|
||||
from basic_memory.mcp.async_client import get_client
|
||||
from basic_memory.mcp.project_context import get_active_project
|
||||
from basic_memory.mcp.server import mcp
|
||||
from basic_memory.mcp.tools.recent_activity import recent_activity
|
||||
from basic_memory.mcp.tools.search import search_notes
|
||||
from basic_memory.mcp.tools.utils import call_post
|
||||
from basic_memory.schemas.prompt import ContinueConversationRequest
|
||||
|
||||
|
||||
@mcp.prompt(
|
||||
@@ -40,92 +42,22 @@ async def continue_conversation(
|
||||
"""
|
||||
logger.info(f"Continuing session, topic: {topic}, timeframe: {timeframe}")
|
||||
|
||||
if topic:
|
||||
# Use json format to get structured data for result counting and branching
|
||||
result = await search_notes(query=topic, after_date=timeframe, output_format="json")
|
||||
async with get_client() as client:
|
||||
config = ConfigManager().config
|
||||
active_project = await get_active_project(client, project=config.default_project)
|
||||
|
||||
if isinstance(result, dict):
|
||||
results = result.get("results", [])
|
||||
context_text = _format_continuation_results(results, topic)
|
||||
result_count = len(results)
|
||||
else:
|
||||
# Error string
|
||||
context_text = str(result)
|
||||
result_count = 0
|
||||
else:
|
||||
# No topic — show recent activity
|
||||
effective_timeframe = timeframe or "7d"
|
||||
activity_text = await recent_activity(timeframe=effective_timeframe)
|
||||
context_text = str(activity_text)
|
||||
result_count = -1 # Signals we used recent_activity
|
||||
# Create request model
|
||||
request = ContinueConversationRequest( # pyright: ignore [reportCallIssue]
|
||||
topic=topic, timeframe=timeframe
|
||||
)
|
||||
|
||||
target = f"'{topic}'" if topic else "recent activity"
|
||||
# Call the prompt API endpoint
|
||||
response = await call_post(
|
||||
client,
|
||||
f"/v2/projects/{active_project.external_id}/prompt/continue-conversation",
|
||||
json=request.model_dump(exclude_none=True),
|
||||
)
|
||||
|
||||
prompt = dedent(f"""
|
||||
# Continuing conversation on: {target}
|
||||
|
||||
This is a memory retrieval session.
|
||||
|
||||
Please use the available basic-memory tools to gather relevant context before responding.
|
||||
Start by executing one of the suggested commands below to retrieve content.
|
||||
|
||||
{context_text}
|
||||
|
||||
---
|
||||
|
||||
## Next Steps
|
||||
""")
|
||||
|
||||
if topic and result_count > 0:
|
||||
prompt += dedent(f"""
|
||||
Found {result_count} results related to '{topic}'.
|
||||
|
||||
1. **Read full content** - Use `read_note("permalink")` to dive into specific notes
|
||||
2. **Build context** - Use `build_context("memory://path")` to see relationships
|
||||
3. **Search deeper** - Use `search_notes("{topic}")` with different filters
|
||||
|
||||
> **Knowledge Capture:** As you continue this conversation, actively look for
|
||||
> opportunities to record new information, decisions, or insights using `write_note()`.
|
||||
""")
|
||||
elif topic:
|
||||
prompt += dedent(f"""
|
||||
No previous context found for '{topic}'.
|
||||
|
||||
This is an opportunity to start documenting this topic:
|
||||
|
||||
1. **Create a new note** - Use `write_note(title="{topic}", content="...")` to start
|
||||
2. **Search with variations** - Try `search_notes("{topic}")` with different terms
|
||||
3. **Check recent activity** - Use `recent_activity(timeframe="7d")` to see what's new
|
||||
""")
|
||||
else:
|
||||
prompt += dedent("""
|
||||
1. **Explore specific items** - Use `read_note("permalink")` to dive deeper
|
||||
2. **Search for topics** - Use `search_notes("topic")` to find specific content
|
||||
3. **Build context** - Use `build_context("memory://path")` to see relationships
|
||||
""")
|
||||
|
||||
return prompt
|
||||
|
||||
|
||||
def _format_continuation_results(results: list[dict], topic: str) -> str:
|
||||
"""Format search result dicts for conversation continuation context."""
|
||||
if not results:
|
||||
return f"No previous context found for '{topic}'."
|
||||
|
||||
lines = [f"## Previous Context for '{topic}'\n"]
|
||||
|
||||
for item in results:
|
||||
title = item.get("title", "Untitled")
|
||||
permalink = item.get("permalink", "")
|
||||
|
||||
lines.append(f"### {title}")
|
||||
if permalink:
|
||||
lines.append(f"permalink: {permalink}")
|
||||
lines.append(f'Read with: `read_note("{permalink}")`')
|
||||
content = item.get("content")
|
||||
if content:
|
||||
content = content[:300] + "..." if len(content) > 300 else content
|
||||
lines.append(f"\n{content}")
|
||||
lines.append("")
|
||||
|
||||
return "\n".join(lines)
|
||||
# Extract the rendered prompt from the response
|
||||
result = response.json()
|
||||
return result["prompt"]
|
||||
|
||||
@@ -46,7 +46,7 @@ async def recent_activity_prompt(
|
||||
logger.info(f"Getting recent activity, timeframe: {timeframe}, project: {project}")
|
||||
|
||||
# Call the tool function - it returns a well-formatted string
|
||||
activity_summary = await recent_activity(project=project, timeframe=timeframe)
|
||||
activity_summary = await recent_activity.fn(project=project, timeframe=timeframe)
|
||||
|
||||
# Build the prompt response
|
||||
# The tool already returns formatted markdown, so we use it directly
|
||||
|
||||
@@ -3,14 +3,17 @@
|
||||
These prompts help users search and explore their knowledge base.
|
||||
"""
|
||||
|
||||
from textwrap import dedent
|
||||
from typing import Annotated, Optional
|
||||
|
||||
from loguru import logger
|
||||
from pydantic import Field
|
||||
|
||||
from basic_memory.config import ConfigManager
|
||||
from basic_memory.mcp.async_client import get_client
|
||||
from basic_memory.mcp.project_context import get_active_project
|
||||
from basic_memory.mcp.server import mcp
|
||||
from basic_memory.mcp.tools.search import search_notes
|
||||
from basic_memory.mcp.tools.utils import call_post
|
||||
from basic_memory.schemas.prompt import SearchPromptRequest
|
||||
|
||||
|
||||
@mcp.prompt(
|
||||
@@ -38,60 +41,20 @@ async def search_prompt(
|
||||
"""
|
||||
logger.info(f"Searching knowledge base, query: {query}, timeframe: {timeframe}")
|
||||
|
||||
# Use json format to get structured data for result counting and formatting
|
||||
result = await search_notes(query=query, after_date=timeframe, output_format="json")
|
||||
async with get_client() as client:
|
||||
config = ConfigManager().config
|
||||
active_project = await get_active_project(client, project=config.default_project)
|
||||
|
||||
# Format the tool output into a prompt with guidance
|
||||
if isinstance(result, dict):
|
||||
results = result.get("results", [])
|
||||
result_count = len(results)
|
||||
result_text = _format_search_results(results, query)
|
||||
else:
|
||||
# Error string from search tool
|
||||
result_count = 0
|
||||
result_text = str(result)
|
||||
# Create request model
|
||||
request = SearchPromptRequest(query=query, timeframe=timeframe)
|
||||
|
||||
return dedent(f"""
|
||||
# Search Results: "{query}"
|
||||
# Call the prompt API endpoint
|
||||
response = await call_post(
|
||||
client,
|
||||
f"/v2/projects/{active_project.external_id}/prompt/search",
|
||||
json=request.model_dump(exclude_none=True),
|
||||
)
|
||||
|
||||
This is a memory retrieval session showing search results.
|
||||
|
||||
{result_text}
|
||||
|
||||
---
|
||||
|
||||
## Next Steps
|
||||
|
||||
Based on these {result_count} results, you can:
|
||||
|
||||
1. **Read a specific note** - Use `read_note("permalink")` to see full content
|
||||
2. **Build context** - Use `build_context("memory://path")` to see relationships
|
||||
3. **Refine search** - Use `search_notes("refined query")` to narrow results
|
||||
4. **Check recent activity** - Use `recent_activity(timeframe="7d")` for recent changes
|
||||
""")
|
||||
|
||||
|
||||
def _format_search_results(results: list[dict], query: str) -> str:
|
||||
"""Format search result dicts into readable markdown."""
|
||||
if not results:
|
||||
return f"No results found for '{query}'."
|
||||
|
||||
lines = [f"Found {len(results)} results:\n"]
|
||||
|
||||
for item in results:
|
||||
title = item.get("title", "Untitled")
|
||||
permalink = item.get("permalink", "")
|
||||
score = item.get("score")
|
||||
score_text = f" (score: {score:.2f})" if score else ""
|
||||
|
||||
lines.append(f"- **{title}**{score_text}")
|
||||
if permalink:
|
||||
lines.append(f" permalink: {permalink}")
|
||||
content = item.get("content")
|
||||
if content:
|
||||
# Truncate content snippet
|
||||
content = content[:200] + "..." if len(content) > 200 else content
|
||||
lines.append(f" {content}")
|
||||
lines.append("")
|
||||
|
||||
return "\n".join(lines)
|
||||
# Extract the rendered prompt from the response
|
||||
result = response.json()
|
||||
return result["prompt"]
|
||||
|
||||
@@ -57,16 +57,6 @@ await write_note(
|
||||
)
|
||||
```
|
||||
|
||||
> **Important**: `write_note` errors if the note already exists. Use `edit_note` for incremental changes, or pass `overwrite=True` to replace.
|
||||
|
||||
```python
|
||||
# Preferred: update an existing note incrementally
|
||||
await edit_note(identifier="Topic", operation="append", content="\n- [category] new fact")
|
||||
|
||||
# Alternative: replace the entire note
|
||||
await write_note(title="Topic", content="...", folder="notes", overwrite=True)
|
||||
```
|
||||
|
||||
### Reading Knowledge
|
||||
|
||||
```python
|
||||
@@ -80,27 +70,11 @@ content = await read_note("memory://folder/topic", project="main")
|
||||
### Searching
|
||||
|
||||
```python
|
||||
# Basic text search
|
||||
results = await search_notes(query="authentication", project="main")
|
||||
|
||||
# Search types: "text" (default), "title", "permalink", "vector"/"semantic", "hybrid"
|
||||
# Default is "hybrid" when semantic search is enabled, "text" otherwise
|
||||
results = await search_notes(query="auth flow", search_type="hybrid")
|
||||
|
||||
# Tag shorthand in query (multiple tags: "tag:x AND tag:y" or "tag:x tag:y")
|
||||
results = await search_notes(query="tag:security")
|
||||
results = await search_notes(query="tag:coffee AND tag:brewing")
|
||||
|
||||
# Filter-only search (no query needed)
|
||||
results = await search_notes(tags=["security", "auth"], status="active")
|
||||
|
||||
# Metadata filters with operators: $in, $gt, $gte, $lt, $lte, $between
|
||||
results = await search_notes(
|
||||
metadata_filters={"priority": {"$in": ["high", "critical"]}}
|
||||
query="authentication",
|
||||
project="main",
|
||||
page_size=10
|
||||
)
|
||||
|
||||
# Override similarity threshold for vector/hybrid search
|
||||
results = await search_notes(query="auth", search_type="hybrid", min_similarity=0.5)
|
||||
```
|
||||
|
||||
### Building Context
|
||||
@@ -188,8 +162,6 @@ activity = await recent_activity(project="main")
|
||||
- 2-3 relations per note
|
||||
- Meaningful categories and relation types
|
||||
|
||||
**Prefer `edit_note` for updates** — use `write_note` only for new notes.
|
||||
|
||||
**Search before creating:**
|
||||
```python
|
||||
# Find existing entities to reference
|
||||
@@ -229,14 +201,6 @@ except:
|
||||
results = await search_notes(query="test", project=projects[0].name)
|
||||
```
|
||||
|
||||
**Note already exists:**
|
||||
```python
|
||||
# write_note returns an error if the note exists — use edit_note or overwrite
|
||||
await edit_note(identifier="Existing Topic", operation="append", content="\n- [update] new info")
|
||||
# Or replace entirely:
|
||||
await write_note(title="Existing Topic", content="...", folder="notes", overwrite=True)
|
||||
```
|
||||
|
||||
**Forward references:**
|
||||
```python
|
||||
# Check response for unresolved relations
|
||||
@@ -292,14 +256,13 @@ context = await build_context(url=f"memory://{results[0].permalink}", project="m
|
||||
|
||||
| Tool | Purpose | Key Params |
|
||||
|------|---------|------------|
|
||||
| `write_note` | Create new | title, content, folder, project, overwrite |
|
||||
| `write_note` | Create/update | title, content, folder, project |
|
||||
| `read_note` | Read content | identifier, project |
|
||||
| `edit_note` | Modify existing | identifier, operation, content, project |
|
||||
| `search_notes` | Find notes | query, search_type, tags, metadata_filters, project |
|
||||
| `search_notes` | Find notes | query, project |
|
||||
| `build_context` | Graph traversal | url, depth, project |
|
||||
| `recent_activity` | Recent changes | timeframe, project |
|
||||
| `list_memory_projects` | Show projects | (none) |
|
||||
| `list_workspaces` | Show workspaces | (none) |
|
||||
|
||||
## memory:// URL Format
|
||||
|
||||
@@ -307,7 +270,6 @@ context = await build_context(url=f"memory://{results[0].permalink}", project="m
|
||||
- `memory://folder/title` - By folder + title
|
||||
- `memory://permalink` - By permalink
|
||||
- `memory://folder/*` - All in folder
|
||||
- `memory://project-name/folder/title` - Cross-project (auto-routes to the correct project)
|
||||
|
||||
For full documentation: https://docs.basicmemory.com
|
||||
|
||||
|
||||
@@ -2,71 +2,18 @@
|
||||
Basic Memory FastMCP server.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import time
|
||||
from contextlib import asynccontextmanager
|
||||
|
||||
from fastmcp import FastMCP
|
||||
from loguru import logger
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.ext.asyncio import async_sessionmaker, AsyncSession
|
||||
|
||||
from basic_memory import db
|
||||
from basic_memory.cli.auth import CLIAuth
|
||||
from basic_memory.config import BasicMemoryConfig
|
||||
from basic_memory.db import (
|
||||
scoped_session,
|
||||
_needs_semantic_embedding_backfill,
|
||||
_run_semantic_embedding_backfill,
|
||||
)
|
||||
from basic_memory.mcp.container import McpContainer, set_container
|
||||
from basic_memory.services.initialization import initialize_app
|
||||
|
||||
|
||||
async def _log_embedding_status(session_maker: async_sessionmaker[AsyncSession]) -> None:
|
||||
"""Log a clear summary of semantic embedding status at startup."""
|
||||
try:
|
||||
async with scoped_session(session_maker) as session:
|
||||
entity_count = (
|
||||
await session.execute(text("SELECT COUNT(*) FROM entity"))
|
||||
).scalar() or 0
|
||||
chunk_count = (
|
||||
await session.execute(text("SELECT COUNT(*) FROM search_vector_chunks"))
|
||||
).scalar() or 0
|
||||
embedding_count = (
|
||||
await session.execute(text("SELECT COUNT(*) FROM search_vector_embeddings_rowids"))
|
||||
).scalar() or 0
|
||||
|
||||
if entity_count == 0:
|
||||
logger.info("Semantic embeddings: no entities yet")
|
||||
elif embedding_count == 0:
|
||||
logger.warning(
|
||||
f"Semantic embeddings: EMPTY — {entity_count} entities have no embeddings. "
|
||||
"Backfill running in background..."
|
||||
)
|
||||
else:
|
||||
logger.info(
|
||||
f"Semantic embeddings: {embedding_count} embeddings "
|
||||
f"across {chunk_count} chunks for {entity_count} entities"
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.debug(f"Could not check embedding status at startup: {exc}")
|
||||
|
||||
|
||||
async def _background_embedding_backfill(
|
||||
config: BasicMemoryConfig,
|
||||
session_maker: async_sessionmaker[AsyncSession],
|
||||
) -> None:
|
||||
"""Run semantic embedding backfill in the background without blocking startup."""
|
||||
try:
|
||||
if await _needs_semantic_embedding_backfill(config, session_maker):
|
||||
logger.info("Background embedding backfill starting...")
|
||||
await _run_semantic_embedding_backfill(config, session_maker)
|
||||
await _log_embedding_status(session_maker)
|
||||
except Exception as exc:
|
||||
logger.error(f"Background embedding backfill failed: {exc}")
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastMCP):
|
||||
"""Lifecycle manager for the MCP server.
|
||||
@@ -123,16 +70,6 @@ async def lifespan(app: FastMCP):
|
||||
# Initialize app (runs migrations, reconciles projects)
|
||||
await initialize_app(container.config)
|
||||
|
||||
# Log embedding status so it's easy to spot in the logs
|
||||
backfill_task: asyncio.Task | None = None # type: ignore[type-arg]
|
||||
if config.semantic_search_enabled and db._session_maker is not None:
|
||||
await _log_embedding_status(db._session_maker)
|
||||
# Launch backfill in background so MCP server is ready immediately
|
||||
backfill_task = asyncio.create_task(
|
||||
_background_embedding_backfill(config, db._session_maker),
|
||||
name="embedding-backfill",
|
||||
)
|
||||
|
||||
# Create and start sync coordinator (lifecycle centralized in coordinator)
|
||||
sync_coordinator = container.create_sync_coordinator()
|
||||
await sync_coordinator.start()
|
||||
@@ -142,15 +79,6 @@ async def lifespan(app: FastMCP):
|
||||
finally:
|
||||
# Shutdown - coordinator handles clean task cancellation
|
||||
logger.debug("Shutting down Basic Memory MCP server")
|
||||
|
||||
# Cancel embedding backfill if still running
|
||||
if backfill_task is not None and not backfill_task.done():
|
||||
backfill_task.cancel()
|
||||
try:
|
||||
await backfill_task
|
||||
except asyncio.CancelledError:
|
||||
logger.info("Background embedding backfill cancelled during shutdown")
|
||||
|
||||
await sync_coordinator.stop()
|
||||
|
||||
# Only shutdown DB if we created it (not if test fixture provided it)
|
||||
|
||||
@@ -18,7 +18,7 @@ from basic_memory.mcp.tools.view_note import view_note
|
||||
from basic_memory.mcp.tools.write_note import write_note
|
||||
from basic_memory.mcp.tools.cloud_info import cloud_info
|
||||
from basic_memory.mcp.tools.release_notes import release_notes
|
||||
from basic_memory.mcp.tools.search import search_notes
|
||||
from basic_memory.mcp.tools.search import search_notes, search_by_metadata
|
||||
from basic_memory.mcp.tools.canvas import canvas
|
||||
from basic_memory.mcp.tools.list_directory import list_directory
|
||||
from basic_memory.mcp.tools.edit_note import edit_note
|
||||
@@ -58,6 +58,7 @@ __all__ = [
|
||||
"schema_infer",
|
||||
"schema_validate",
|
||||
"search",
|
||||
"search_by_metadata",
|
||||
"search_notes",
|
||||
# "search_notes_ui",
|
||||
"view_note",
|
||||
|
||||
@@ -5,12 +5,7 @@ from typing import Optional, Literal
|
||||
from loguru import logger
|
||||
from fastmcp import Context
|
||||
|
||||
from basic_memory.config import ConfigManager
|
||||
from basic_memory.mcp.project_context import (
|
||||
detect_project_from_url_prefix,
|
||||
get_project_client,
|
||||
resolve_project_and_path,
|
||||
)
|
||||
from basic_memory.mcp.project_context import get_project_client, resolve_project_and_path
|
||||
from basic_memory.mcp.server import mcp
|
||||
from basic_memory.schemas.base import TimeFrame
|
||||
from basic_memory.schemas.memory import (
|
||||
@@ -22,6 +17,74 @@ from basic_memory.schemas.memory import (
|
||||
RelationSummary,
|
||||
)
|
||||
|
||||
# --- Fields to strip from each model (redundant with parent entity) ---
|
||||
|
||||
_OBSERVATION_STRIP = {
|
||||
"observation_id",
|
||||
"entity_id",
|
||||
"entity_external_id",
|
||||
"title",
|
||||
"file_path",
|
||||
"created_at",
|
||||
}
|
||||
_RELATION_STRIP = {
|
||||
"relation_id",
|
||||
"entity_id",
|
||||
"from_entity_id",
|
||||
"from_entity_external_id",
|
||||
"to_entity_id",
|
||||
"to_entity_external_id",
|
||||
"title",
|
||||
"file_path",
|
||||
"created_at",
|
||||
}
|
||||
_ENTITY_STRIP = {"entity_id", "created_at"}
|
||||
_METADATA_STRIP = {"total_results", "generated_at"}
|
||||
|
||||
|
||||
def _slim_summary(summary: EntitySummary | RelationSummary | ObservationSummary) -> dict:
|
||||
"""Strip redundant fields from a summary model based on its type."""
|
||||
if isinstance(summary, ObservationSummary):
|
||||
strip = _OBSERVATION_STRIP
|
||||
elif isinstance(summary, RelationSummary):
|
||||
strip = _RELATION_STRIP
|
||||
else:
|
||||
strip = _ENTITY_STRIP
|
||||
|
||||
data = summary.model_dump()
|
||||
for key in strip:
|
||||
data.pop(key, None)
|
||||
return data
|
||||
|
||||
|
||||
def _slim_context(graph: GraphContext) -> dict:
|
||||
"""Transform GraphContext into a slimmed dict, stripping redundant fields.
|
||||
|
||||
Reduces payload size ~40% by removing fields on nested objects that
|
||||
duplicate information already present on the parent entity (IDs,
|
||||
timestamps, file paths).
|
||||
"""
|
||||
slimmed_results = []
|
||||
for result in graph.results:
|
||||
slimmed_results.append(
|
||||
{
|
||||
"primary_result": _slim_summary(result.primary_result),
|
||||
"observations": [_slim_summary(obs) for obs in result.observations],
|
||||
"related_results": [_slim_summary(rel) for rel in result.related_results],
|
||||
}
|
||||
)
|
||||
|
||||
metadata = graph.metadata.model_dump()
|
||||
for key in _METADATA_STRIP:
|
||||
metadata.pop(key, None)
|
||||
|
||||
return {
|
||||
"results": slimmed_results,
|
||||
"metadata": metadata,
|
||||
"page": graph.page,
|
||||
"page_size": graph.page_size,
|
||||
}
|
||||
|
||||
|
||||
def _format_entity_block(result: ContextResult) -> str:
|
||||
"""Format a single context result as a markdown block."""
|
||||
@@ -126,10 +189,9 @@ def _format_context_markdown(graph: GraphContext, project: str) -> str:
|
||||
- Or standard formats like "7d", "24h"
|
||||
|
||||
Format options:
|
||||
- "json" (default): Structured JSON with internal fields excluded
|
||||
- "json" (default): Slimmed JSON with redundant fields removed
|
||||
- "text": Compact markdown text for LLM consumption
|
||||
""",
|
||||
annotations={"readOnlyHint": True, "openWorldHint": False},
|
||||
)
|
||||
async def build_context(
|
||||
url: MemoryUrl,
|
||||
@@ -163,12 +225,12 @@ async def build_context(
|
||||
page: Page number of results to return (default: 1)
|
||||
page_size: Number of results to return per page (default: 10)
|
||||
max_related: Maximum number of related results to return (default: 10)
|
||||
output_format: Response format - "json" for structured JSON dict,
|
||||
output_format: Response format - "json" for slimmed JSON dict,
|
||||
"text" for compact markdown text
|
||||
context: Optional FastMCP context for performance caching.
|
||||
|
||||
Returns:
|
||||
dict (output_format="json"): Structured JSON with internal fields excluded
|
||||
dict (output_format="json"): Slimmed JSON with redundant fields removed
|
||||
str (output_format="text"): Compact markdown representation
|
||||
|
||||
Examples:
|
||||
@@ -184,12 +246,6 @@ async def build_context(
|
||||
Raises:
|
||||
ToolError: If project doesn't exist or depth parameter is invalid
|
||||
"""
|
||||
# Detect project from memory URL prefix before routing
|
||||
if project is None:
|
||||
detected = detect_project_from_url_prefix(url, ConfigManager().config)
|
||||
if detected:
|
||||
project = detected
|
||||
|
||||
logger.info(f"Building context from {url} in project {project}")
|
||||
|
||||
# Convert string depth to integer if needed
|
||||
@@ -224,4 +280,4 @@ async def build_context(
|
||||
if output_format == "text":
|
||||
return _format_context_markdown(graph, active_project.name)
|
||||
|
||||
return graph.model_dump()
|
||||
return _slim_context(graph)
|
||||
|
||||
@@ -4,25 +4,22 @@ This tool creates Obsidian canvas files (.canvas) using the JSON Canvas 1.0 spec
|
||||
"""
|
||||
|
||||
import json
|
||||
from typing import Annotated, Dict, List, Any, Optional
|
||||
from typing import Dict, List, Any, Optional
|
||||
|
||||
from loguru import logger
|
||||
from fastmcp import Context
|
||||
from pydantic import BeforeValidator
|
||||
|
||||
from basic_memory.mcp.project_context import get_project_client
|
||||
from basic_memory.utils import coerce_list
|
||||
from basic_memory.mcp.server import mcp
|
||||
from basic_memory.mcp.tools.utils import call_put, call_post, resolve_entity_id
|
||||
|
||||
|
||||
@mcp.tool(
|
||||
description="Create an Obsidian canvas file to visualize concepts and connections.",
|
||||
annotations={"destructiveHint": False, "idempotentHint": True, "openWorldHint": False},
|
||||
)
|
||||
async def canvas(
|
||||
nodes: Annotated[List[Dict[str, Any]], BeforeValidator(coerce_list)],
|
||||
edges: Annotated[List[Dict[str, Any]], BeforeValidator(coerce_list)],
|
||||
nodes: List[Dict[str, Any]],
|
||||
edges: List[Dict[str, Any]],
|
||||
title: str,
|
||||
directory: str,
|
||||
project: Optional[str] = None,
|
||||
|
||||
@@ -7,13 +7,13 @@ a list containing a single `{"type": "text", "text": "{...json...}"}` item.
|
||||
|
||||
import json
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from fastmcp import Context
|
||||
from loguru import logger
|
||||
from fastmcp import Context
|
||||
|
||||
from basic_memory.mcp.server import mcp
|
||||
from basic_memory.mcp.tools.read_note import read_note
|
||||
from basic_memory.mcp.tools.search import search_notes
|
||||
from basic_memory.mcp.tools.read_note import read_note
|
||||
from basic_memory.config import ConfigManager
|
||||
from basic_memory.schemas.search import SearchResponse, SearchResult
|
||||
|
||||
|
||||
@@ -92,10 +92,7 @@ def _format_document_for_chatgpt(
|
||||
}
|
||||
|
||||
|
||||
@mcp.tool(
|
||||
description="Search for content across the knowledge base",
|
||||
annotations={"readOnlyHint": True, "openWorldHint": False},
|
||||
)
|
||||
@mcp.tool(description="Search for content across the knowledge base")
|
||||
async def search(
|
||||
query: str,
|
||||
context: Context | None = None,
|
||||
@@ -113,12 +110,17 @@ async def search(
|
||||
logger.info(f"ChatGPT search request: query='{query}'")
|
||||
|
||||
try:
|
||||
# Let search_notes resolve the default project via get_project_client(),
|
||||
# which works in both local mode (ConfigManager) and cloud mode (database).
|
||||
results = await search_notes(
|
||||
# ChatGPT tools don't expose project parameter, so use default project
|
||||
config = ConfigManager().config
|
||||
default_project = config.default_project
|
||||
|
||||
# Call underlying search_notes with sensible defaults for ChatGPT
|
||||
results = await search_notes.fn(
|
||||
query=query,
|
||||
project=default_project, # Use default project for ChatGPT
|
||||
page=1,
|
||||
page_size=10,
|
||||
page_size=10, # Reasonable default for ChatGPT consumption
|
||||
search_type="text", # Default to full-text search
|
||||
output_format="json",
|
||||
context=context,
|
||||
)
|
||||
@@ -155,10 +157,7 @@ async def search(
|
||||
return [{"type": "text", "text": json.dumps(error_results, ensure_ascii=False)}]
|
||||
|
||||
|
||||
@mcp.tool(
|
||||
description="Fetch the full contents of a search result document",
|
||||
annotations={"readOnlyHint": True, "openWorldHint": False},
|
||||
)
|
||||
@mcp.tool(description="Fetch the full contents of a search result document")
|
||||
async def fetch(
|
||||
id: str,
|
||||
context: Context | None = None,
|
||||
@@ -176,15 +175,17 @@ async def fetch(
|
||||
logger.info(f"ChatGPT fetch request: id='{id}'")
|
||||
|
||||
try:
|
||||
# Let read_note resolve the default project via get_project_client(),
|
||||
# which works in both local mode (ConfigManager) and cloud mode (database).
|
||||
content = str(
|
||||
await read_note(
|
||||
identifier=id,
|
||||
page=1,
|
||||
page_size=10,
|
||||
context=context,
|
||||
)
|
||||
# ChatGPT tools don't expose project parameter, so use default project
|
||||
config = ConfigManager().config
|
||||
default_project = config.default_project
|
||||
|
||||
# Call underlying read_note function
|
||||
content = await read_note.fn(
|
||||
identifier=id,
|
||||
project=default_project, # Use default project for ChatGPT
|
||||
page=1,
|
||||
page_size=10, # Default pagination
|
||||
context=context,
|
||||
)
|
||||
|
||||
# Format the document for ChatGPT
|
||||
|
||||
@@ -5,10 +5,7 @@ from pathlib import Path
|
||||
from basic_memory.mcp.server import mcp
|
||||
|
||||
|
||||
@mcp.tool(
|
||||
"cloud_info",
|
||||
annotations={"readOnlyHint": True, "openWorldHint": False},
|
||||
)
|
||||
@mcp.tool("cloud_info")
|
||||
def cloud_info() -> str:
|
||||
"""Return optional Basic Memory Cloud information and setup guidance."""
|
||||
content_path = Path(__file__).parent.parent / "resources" / "cloud_info.md"
|
||||
|
||||
@@ -146,10 +146,7 @@ delete_note("{project}", "correct-identifier-from-search")
|
||||
If the note should be deleted but the operation keeps failing, send a message to support@basicmemory.com."""
|
||||
|
||||
|
||||
@mcp.tool(
|
||||
description="Delete a note or directory by title, permalink, or path",
|
||||
annotations={"destructiveHint": True, "openWorldHint": False},
|
||||
)
|
||||
@mcp.tool(description="Delete a note or directory by title, permalink, or path")
|
||||
async def delete_note(
|
||||
identifier: str,
|
||||
is_directory: bool = False,
|
||||
@@ -284,16 +281,6 @@ async def delete_note(
|
||||
|
||||
except Exception as e: # pragma: no cover
|
||||
logger.error(f"Directory delete failed for '{identifier}': {e}")
|
||||
if output_format == "json":
|
||||
return {
|
||||
"deleted": False,
|
||||
"is_directory": True,
|
||||
"identifier": identifier,
|
||||
"total_files": 0,
|
||||
"successful_deletes": 0,
|
||||
"failed_deletes": 0,
|
||||
"error": str(e),
|
||||
}
|
||||
return f"""# Directory Delete Failed
|
||||
|
||||
Error deleting directory '{identifier}': {str(e)}
|
||||
@@ -318,7 +305,7 @@ delete_note("path/to/file.md")
|
||||
note_file_path = None
|
||||
try:
|
||||
# Resolve identifier to entity ID
|
||||
entity_id = await knowledge_client.resolve_entity(identifier, strict=True)
|
||||
entity_id = await knowledge_client.resolve_entity(identifier)
|
||||
if output_format == "json":
|
||||
entity = await knowledge_client.get_entity(entity_id)
|
||||
note_title = entity.title
|
||||
|
||||
@@ -7,36 +7,6 @@ from fastmcp import Context
|
||||
|
||||
from basic_memory.mcp.project_context import get_project_client, add_project_metadata
|
||||
from basic_memory.mcp.server import mcp
|
||||
from basic_memory.schemas.base import Entity
|
||||
from basic_memory.schemas.response import EntityResponse
|
||||
from basic_memory.utils import validate_project_path
|
||||
|
||||
|
||||
def _parse_identifier_to_title_and_directory(identifier: str) -> tuple[str, str]:
|
||||
"""Parse an identifier into (title, directory) for creating a new note.
|
||||
|
||||
Strips memory:// prefix if present, then splits on the last '/' to
|
||||
separate the directory path from the note title.
|
||||
|
||||
Examples:
|
||||
"conversations/my-note" → ("my-note", "conversations")
|
||||
"my-note" → ("my-note", "")
|
||||
"a/b/c/my-note" → ("my-note", "a/b/c")
|
||||
"memory://a/b/note" → ("note", "a/b")
|
||||
"""
|
||||
cleaned = identifier
|
||||
if cleaned.startswith("memory://"):
|
||||
cleaned = cleaned[len("memory://") :]
|
||||
|
||||
if "/" in cleaned:
|
||||
last_slash = cleaned.rfind("/")
|
||||
directory = cleaned[:last_slash]
|
||||
title = cleaned[last_slash + 1 :]
|
||||
else:
|
||||
directory = ""
|
||||
title = cleaned
|
||||
|
||||
return title, directory
|
||||
|
||||
|
||||
def _format_error_response(
|
||||
@@ -49,19 +19,15 @@ def _format_error_response(
|
||||
) -> str:
|
||||
"""Format helpful error responses for edit_note failures that guide the AI to retry successfully."""
|
||||
|
||||
# Entity not found errors — only reachable for find_replace/replace_section
|
||||
# because append/prepend auto-create the note when it doesn't exist
|
||||
# Entity not found errors
|
||||
if "Entity not found" in error_message or "entity not found" in error_message.lower():
|
||||
return f"""# Edit Failed - Note Not Found
|
||||
|
||||
The note with identifier '{identifier}' could not be found. The `find_replace` and `replace_section` operations require an existing note with content to modify.
|
||||
|
||||
**Tip:** `append` and `prepend` operations automatically create the note if it doesn't exist.
|
||||
The note with identifier '{identifier}' could not be found. Edit operations require an exact match (no fuzzy matching).
|
||||
|
||||
## Suggestions to try:
|
||||
1. **Use append/prepend instead**: These operations will create the note automatically if it doesn't exist
|
||||
2. **Search for the note first**: Use `search_notes("{project or "project-name"}", "{identifier.split("/")[-1]}")` to find similar notes with exact identifiers
|
||||
3. **Try different exact identifier formats**:
|
||||
1. **Search for the note first**: Use `search_notes("{project or "project-name"}", "{identifier.split("/")[-1]}")` to find similar notes with exact identifiers
|
||||
2. **Try different exact identifier formats**:
|
||||
- If you used a permalink like "folder/note-title", try the exact title: "{identifier.split("/")[-1].replace("-", " ").title()}"
|
||||
- If you used a title, try the exact permalink format: "{identifier.lower().replace(" ", "-")}"
|
||||
- Use `read_note("{project or "project-name"}", "{identifier}")` first to verify the note exists and get the exact identifier
|
||||
@@ -158,8 +124,7 @@ Error editing note '{identifier}': {error_message}
|
||||
|
||||
|
||||
@mcp.tool(
|
||||
description="Edit an existing markdown note using various operations like append, prepend, find_replace, replace_section, insert_before_section, or insert_after_section.",
|
||||
annotations={"destructiveHint": False, "openWorldHint": False},
|
||||
description="Edit an existing markdown note using various operations like append, prepend, find_replace, or replace_section.",
|
||||
)
|
||||
async def edit_note(
|
||||
identifier: str,
|
||||
@@ -169,7 +134,7 @@ async def edit_note(
|
||||
workspace: Optional[str] = None,
|
||||
section: Optional[str] = None,
|
||||
find_text: Optional[str] = None,
|
||||
expected_replacements: Optional[int] = None,
|
||||
expected_replacements: int = 1,
|
||||
output_format: Literal["text", "json"] = "text",
|
||||
context: Context | None = None,
|
||||
) -> str | dict:
|
||||
@@ -186,12 +151,10 @@ async def edit_note(
|
||||
Must be an exact match - fuzzy matching is not supported for edit operations.
|
||||
Use search_notes() or read_note() first to find the correct identifier if uncertain.
|
||||
operation: The editing operation to perform:
|
||||
- "append": Add content to the end of the note (creates the note if it doesn't exist)
|
||||
- "prepend": Add content to the beginning of the note (creates the note if it doesn't exist)
|
||||
- "find_replace": Replace occurrences of find_text with content (note must exist)
|
||||
- "replace_section": Replace content under a specific markdown header (note must exist)
|
||||
- "insert_before_section": Insert content before a section heading without consuming it (note must exist)
|
||||
- "insert_after_section": Insert content after a section heading without consuming it (note must exist)
|
||||
- "append": Add content to the end of the note
|
||||
- "prepend": Add content to the beginning of the note
|
||||
- "find_replace": Replace occurrences of find_text with content
|
||||
- "replace_section": Replace content under a specific markdown header
|
||||
content: The content to add or use for replacement
|
||||
project: Project name to edit in. Optional - server will resolve using hierarchy.
|
||||
If unknown, use list_memory_projects() to discover available projects.
|
||||
@@ -252,21 +215,11 @@ async def edit_note(
|
||||
search_notes() first to find the correct identifier. The tool provides detailed
|
||||
error messages with suggestions if operations fail.
|
||||
"""
|
||||
# Resolve effective default: allow MCP clients to send null for optional int field
|
||||
effective_replacements = expected_replacements if expected_replacements is not None else 1
|
||||
|
||||
async with get_project_client(project, workspace, context) as (client, active_project):
|
||||
logger.info("MCP tool call", tool="edit_note", identifier=identifier, operation=operation)
|
||||
|
||||
# Validate operation
|
||||
valid_operations = [
|
||||
"append",
|
||||
"prepend",
|
||||
"find_replace",
|
||||
"replace_section",
|
||||
"insert_before_section",
|
||||
"insert_after_section",
|
||||
]
|
||||
valid_operations = ["append", "prepend", "find_replace", "replace_section"]
|
||||
if operation not in valid_operations:
|
||||
raise ValueError(
|
||||
f"Invalid operation '{operation}'. Must be one of: {', '.join(valid_operations)}"
|
||||
@@ -275,9 +228,8 @@ async def edit_note(
|
||||
# Validate required parameters for specific operations
|
||||
if operation == "find_replace" and not find_text:
|
||||
raise ValueError("find_text parameter is required for find_replace operation")
|
||||
section_ops = ("replace_section", "insert_before_section", "insert_after_section")
|
||||
if operation in section_ops and not section:
|
||||
raise ValueError("section parameter is required for section-based operations")
|
||||
if operation == "replace_section" and not section:
|
||||
raise ValueError("section parameter is required for replace_section operation")
|
||||
|
||||
# Use the PATCH endpoint to edit the entity
|
||||
try:
|
||||
@@ -287,122 +239,48 @@ async def edit_note(
|
||||
# Use typed KnowledgeClient for API calls
|
||||
knowledge_client = KnowledgeClient(client, active_project.external_id)
|
||||
|
||||
file_created = False
|
||||
entity_id = ""
|
||||
result: EntityResponse | None = None
|
||||
# Resolve identifier to entity ID
|
||||
entity_id = await knowledge_client.resolve_entity(identifier)
|
||||
|
||||
# Try to resolve the entity; for append/prepend, create it if not found
|
||||
try:
|
||||
entity_id = await knowledge_client.resolve_entity(identifier, strict=True)
|
||||
except Exception as resolve_error:
|
||||
# Trigger: entity does not exist yet
|
||||
# Why: append/prepend can meaningfully create a new note from the content,
|
||||
# while find_replace/replace_section require existing content to modify
|
||||
# Outcome: note is created via the same path as write_note
|
||||
error_msg = str(resolve_error).lower()
|
||||
is_not_found = "entity not found" in error_msg or "not found" in error_msg
|
||||
# Prepare the edit request data
|
||||
edit_data = {
|
||||
"operation": operation,
|
||||
"content": content,
|
||||
}
|
||||
|
||||
if is_not_found and operation in ("append", "prepend"):
|
||||
title, directory = _parse_identifier_to_title_and_directory(identifier)
|
||||
# Add optional parameters
|
||||
if section:
|
||||
edit_data["section"] = section
|
||||
if find_text:
|
||||
edit_data["find_text"] = find_text
|
||||
if expected_replacements != 1: # Only send if different from default
|
||||
edit_data["expected_replacements"] = str(expected_replacements)
|
||||
|
||||
# Validate directory path (same security check as write_note)
|
||||
project_path = active_project.home
|
||||
if directory and not validate_project_path(directory, project_path):
|
||||
logger.warning(
|
||||
"Attempted path traversal attack blocked",
|
||||
directory=directory,
|
||||
project=active_project.name,
|
||||
)
|
||||
if output_format == "json":
|
||||
return {
|
||||
"title": title,
|
||||
"permalink": None,
|
||||
"file_path": None,
|
||||
"checksum": None,
|
||||
"operation": operation,
|
||||
"fileCreated": False,
|
||||
"error": "SECURITY_VALIDATION_ERROR",
|
||||
}
|
||||
return f"# Error\n\nDirectory path '{directory}' is not allowed - paths must stay within project boundaries"
|
||||
# Call the PATCH endpoint
|
||||
result = await knowledge_client.patch_entity(entity_id, edit_data, fast=False)
|
||||
|
||||
entity = Entity(
|
||||
title=title,
|
||||
directory=directory,
|
||||
content_type="text/markdown",
|
||||
content=content,
|
||||
)
|
||||
# Format summary
|
||||
summary = [
|
||||
f"# Edited note ({operation})",
|
||||
f"project: {active_project.name}",
|
||||
f"file_path: {result.file_path}",
|
||||
f"permalink: {result.permalink}",
|
||||
f"checksum: {result.checksum[:8] if result.checksum else 'unknown'}",
|
||||
]
|
||||
|
||||
logger.info(
|
||||
"Creating note via edit_note auto-create",
|
||||
title=title,
|
||||
directory=directory,
|
||||
operation=operation,
|
||||
)
|
||||
result = await knowledge_client.create_entity(entity.model_dump(), fast=False)
|
||||
file_created = True
|
||||
else:
|
||||
# find_replace/replace_section require existing content — re-raise
|
||||
raise resolve_error
|
||||
|
||||
# --- Standard edit path (entity already existed) ---
|
||||
if not file_created:
|
||||
# Prepare the edit request data
|
||||
edit_data = {
|
||||
"operation": operation,
|
||||
"content": content,
|
||||
}
|
||||
|
||||
# Add optional parameters
|
||||
if section:
|
||||
edit_data["section"] = section
|
||||
if find_text:
|
||||
edit_data["find_text"] = find_text
|
||||
if effective_replacements != 1: # Only send if different from default
|
||||
edit_data["expected_replacements"] = str(effective_replacements)
|
||||
|
||||
# Call the PATCH endpoint
|
||||
result = await knowledge_client.patch_entity(entity_id, edit_data, fast=False)
|
||||
|
||||
# --- Format response ---
|
||||
# result is always set: either by create_entity (auto-create) or patch_entity (edit)
|
||||
assert result is not None
|
||||
if file_created:
|
||||
summary = [
|
||||
f"# Created note ({operation})",
|
||||
f"project: {active_project.name}",
|
||||
f"file_path: {result.file_path}",
|
||||
f"permalink: {result.permalink}",
|
||||
f"checksum: {result.checksum[:8] if result.checksum else 'unknown'}",
|
||||
"fileCreated: true",
|
||||
]
|
||||
# Add operation-specific details
|
||||
if operation == "append":
|
||||
lines_added = len(content.split("\n"))
|
||||
summary.append(f"operation: Created note with {lines_added} lines")
|
||||
else:
|
||||
summary = [
|
||||
f"# Edited note ({operation})",
|
||||
f"project: {active_project.name}",
|
||||
f"file_path: {result.file_path}",
|
||||
f"permalink: {result.permalink}",
|
||||
f"checksum: {result.checksum[:8] if result.checksum else 'unknown'}",
|
||||
]
|
||||
|
||||
# Add operation-specific details
|
||||
if operation == "append":
|
||||
lines_added = len(content.split("\n"))
|
||||
summary.append(f"operation: Added {lines_added} lines to end of note")
|
||||
elif operation == "prepend":
|
||||
lines_added = len(content.split("\n"))
|
||||
summary.append(f"operation: Added {lines_added} lines to beginning of note")
|
||||
elif operation == "find_replace":
|
||||
# For find_replace, we can't easily count replacements from here
|
||||
# since we don't have the original content, but the server handled it
|
||||
summary.append("operation: Find and replace operation completed")
|
||||
elif operation == "replace_section":
|
||||
summary.append(f"operation: Replaced content under section '{section}'")
|
||||
elif operation == "insert_before_section":
|
||||
summary.append(f"operation: Inserted content before section '{section}'")
|
||||
elif operation == "insert_after_section":
|
||||
summary.append(f"operation: Inserted content after section '{section}'")
|
||||
summary.append(f"operation: Added {lines_added} lines to end of note")
|
||||
elif operation == "prepend":
|
||||
lines_added = len(content.split("\n"))
|
||||
summary.append(f"operation: Added {lines_added} lines to beginning of note")
|
||||
elif operation == "find_replace":
|
||||
# For find_replace, we can't easily count replacements from here
|
||||
# since we don't have the original content, but the server handled it
|
||||
summary.append("operation: Find and replace operation completed")
|
||||
elif operation == "replace_section":
|
||||
summary.append(f"operation: Replaced content under section '{section}'")
|
||||
|
||||
# Count observations by category (reuse logic from write_note)
|
||||
categories = {}
|
||||
@@ -410,7 +288,7 @@ async def edit_note(
|
||||
for obs in result.observations:
|
||||
categories[obs.category] = categories.get(obs.category, 0) + 1
|
||||
|
||||
summary.append("\n## Observations")
|
||||
summary.append("\\n## Observations")
|
||||
for category, count in sorted(categories.items()):
|
||||
summary.append(f"- {category}: {count}")
|
||||
|
||||
@@ -421,7 +299,7 @@ async def edit_note(
|
||||
unresolved = sum(1 for r in result.relations if not r.to_id)
|
||||
resolved = len(result.relations) - unresolved
|
||||
|
||||
summary.append("\n## Relations")
|
||||
summary.append("\\n## Relations")
|
||||
summary.append(f"- Resolved: {resolved}")
|
||||
if unresolved:
|
||||
summary.append(f"- Unresolved: {unresolved}")
|
||||
@@ -434,7 +312,6 @@ async def edit_note(
|
||||
permalink=result.permalink,
|
||||
observations_count=len(result.observations),
|
||||
relations_count=len(result.relations),
|
||||
file_created=file_created,
|
||||
)
|
||||
|
||||
if output_format == "json":
|
||||
@@ -444,7 +321,6 @@ async def edit_note(
|
||||
"file_path": result.file_path,
|
||||
"checksum": result.checksum,
|
||||
"operation": operation,
|
||||
"fileCreated": file_created,
|
||||
}
|
||||
|
||||
summary_result = "\n".join(summary)
|
||||
@@ -459,14 +335,8 @@ async def edit_note(
|
||||
"file_path": None,
|
||||
"checksum": None,
|
||||
"operation": operation,
|
||||
"fileCreated": False,
|
||||
"error": str(e),
|
||||
}
|
||||
return _format_error_response(
|
||||
str(e),
|
||||
operation,
|
||||
identifier,
|
||||
find_text,
|
||||
effective_replacements,
|
||||
active_project.name,
|
||||
str(e), operation, identifier, find_text, expected_replacements, active_project.name
|
||||
)
|
||||
|
||||
@@ -11,7 +11,6 @@ from basic_memory.mcp.server import mcp
|
||||
|
||||
@mcp.tool(
|
||||
description="List directory contents with filtering and depth control.",
|
||||
annotations={"readOnlyHint": True, "openWorldHint": False},
|
||||
)
|
||||
async def list_directory(
|
||||
dir_name: str = "/",
|
||||
|
||||
@@ -1,12 +1,10 @@
|
||||
"""Move note tool for Basic Memory MCP server."""
|
||||
|
||||
from pathlib import Path, PureWindowsPath
|
||||
from textwrap import dedent
|
||||
from typing import Optional, Literal
|
||||
|
||||
from loguru import logger
|
||||
from fastmcp import Context
|
||||
from mcp.server.fastmcp.exceptions import ToolError
|
||||
|
||||
from basic_memory.mcp.server import mcp
|
||||
from basic_memory.mcp.project_context import get_project_client
|
||||
@@ -344,12 +342,10 @@ delete_note("{identifier}")
|
||||
|
||||
@mcp.tool(
|
||||
description="Move a note or directory to a new location, updating database and maintaining links.",
|
||||
annotations={"destructiveHint": False, "openWorldHint": False},
|
||||
)
|
||||
async def move_note(
|
||||
identifier: str,
|
||||
destination_path: str = "",
|
||||
destination_folder: Optional[str] = None,
|
||||
destination_path: str,
|
||||
is_directory: bool = False,
|
||||
project: Optional[str] = None,
|
||||
workspace: Optional[str] = None,
|
||||
@@ -369,9 +365,6 @@ async def move_note(
|
||||
Use search_notes() or list_directory() first to find the correct path if uncertain.
|
||||
destination_path: For files: new path relative to project root (e.g., "work/meetings/note.md")
|
||||
For directories: new directory path (e.g., "archive/docs")
|
||||
Mutually exclusive with destination_folder.
|
||||
destination_folder: Move the note into this folder, preserving the original filename.
|
||||
Mutually exclusive with destination_path. Only for single-file moves.
|
||||
is_directory: If True, moves an entire directory and all its contents.
|
||||
When True, identifier and destination_path should be directory paths
|
||||
(without file extensions). Defaults to False.
|
||||
@@ -392,9 +385,6 @@ async def move_note(
|
||||
# Move by exact permalink
|
||||
move_note("my-note-permalink", "archive/old-notes/my-note.md")
|
||||
|
||||
# Move note to archive folder (filename preserved automatically)
|
||||
move_note("my-note", destination_folder="archive")
|
||||
|
||||
# Move with complex path structure
|
||||
move_note("experiments/ml-results", "archive/2025/ml-experiments.md")
|
||||
|
||||
@@ -425,57 +415,6 @@ async def move_note(
|
||||
- Re-indexes the entity for search
|
||||
- Maintains all observations and relations
|
||||
"""
|
||||
# --- Parameter Validation ---
|
||||
# Trigger: both destination_path and destination_folder provided
|
||||
# Why: they are mutually exclusive — one specifies full path, the other just the folder
|
||||
# Outcome: early error before any entity resolution or API calls
|
||||
if destination_folder and destination_path:
|
||||
error_msg = (
|
||||
"Cannot specify both destination_path and destination_folder. Use one or the other."
|
||||
)
|
||||
if output_format == "json":
|
||||
return {
|
||||
"moved": False,
|
||||
"title": None,
|
||||
"permalink": None,
|
||||
"file_path": None,
|
||||
"source": identifier,
|
||||
"destination": None,
|
||||
"error": "MUTUALLY_EXCLUSIVE_PARAMS",
|
||||
}
|
||||
return f"# Move Failed - Invalid Parameters\n\n{error_msg}"
|
||||
|
||||
if not destination_folder and not destination_path:
|
||||
error_msg = "Either destination_path or destination_folder must be provided."
|
||||
if output_format == "json":
|
||||
return {
|
||||
"moved": False,
|
||||
"title": None,
|
||||
"permalink": None,
|
||||
"file_path": None,
|
||||
"source": identifier,
|
||||
"destination": None,
|
||||
"error": "MISSING_DESTINATION",
|
||||
}
|
||||
return f"# Move Failed - Missing Destination\n\n{error_msg}"
|
||||
|
||||
# Trigger: destination_folder used with is_directory=True
|
||||
# Why: destination_folder preserves a single file's name — meaningless for directory moves
|
||||
if destination_folder and is_directory:
|
||||
error_msg = (
|
||||
"destination_folder is only supported for single-file moves, not directory moves."
|
||||
)
|
||||
if output_format == "json":
|
||||
return {
|
||||
"moved": False,
|
||||
"title": None,
|
||||
"permalink": None,
|
||||
"file_path": None,
|
||||
"source": identifier,
|
||||
"destination": None,
|
||||
"error": "DESTINATION_FOLDER_NOT_FOR_DIRECTORIES",
|
||||
}
|
||||
return f"# Move Failed - Invalid Parameters\n\n{error_msg}"
|
||||
async with get_project_client(project, workspace, context) as (client, active_project):
|
||||
logger.debug(
|
||||
f"Moving {'directory' if is_directory else 'note'}: {identifier} to {destination_path} in project: {active_project.name}"
|
||||
@@ -629,105 +568,19 @@ move_note("path/to/file.md", "{destination_path}/file.md")
|
||||
# Use typed KnowledgeClient for API calls
|
||||
knowledge_client = KnowledgeClient(client, active_project.external_id)
|
||||
|
||||
# Resolve once and reuse the entity ID across extension validation and move.
|
||||
# Get the source entity information for extension validation
|
||||
source_ext = "md" # Default to .md if we can't determine source extension
|
||||
resolved_entity_id: str | None = None
|
||||
source_entity = None
|
||||
|
||||
async def _ensure_resolved_entity_id() -> str:
|
||||
"""Resolve and cache the source entity ID for the duration of this move."""
|
||||
nonlocal resolved_entity_id
|
||||
if resolved_entity_id is None:
|
||||
resolved_entity_id = await knowledge_client.resolve_entity(identifier, strict=True)
|
||||
return resolved_entity_id
|
||||
|
||||
try:
|
||||
resolved_entity_id = await _ensure_resolved_entity_id()
|
||||
source_entity = await knowledge_client.get_entity(resolved_entity_id)
|
||||
# Resolve identifier to entity ID
|
||||
entity_id = await knowledge_client.resolve_entity(identifier)
|
||||
# Fetch source entity information to get the current file extension
|
||||
source_entity = await knowledge_client.get_entity(entity_id)
|
||||
if "." in source_entity.file_path:
|
||||
source_ext = source_entity.file_path.split(".")[-1]
|
||||
except ToolError as e:
|
||||
# Trigger: strict=True resolve_entity raised because the entity was not found.
|
||||
# Why: fail fast with a formatted error instead of silently falling through
|
||||
# to extension defaults and failing later with a confusing message.
|
||||
# Outcome: move_note returns a user-facing not-found error immediately.
|
||||
logger.error(f"Move failed for '{identifier}' to '{destination_path}': {e}")
|
||||
if output_format == "json":
|
||||
return {
|
||||
"moved": False,
|
||||
"title": None,
|
||||
"permalink": None,
|
||||
"file_path": None,
|
||||
"source": identifier,
|
||||
"destination": destination_path,
|
||||
"error": str(e),
|
||||
}
|
||||
return _format_move_error_response(str(e), identifier, destination_path)
|
||||
except Exception as e:
|
||||
# If we can't fetch source metadata (e.g. get_entity or file_path parsing fails),
|
||||
# continue with extension defaults — the entity was at least resolved.
|
||||
# If we can't fetch the source entity, default to .md extension
|
||||
logger.debug(f"Could not fetch source entity for extension check: {e}")
|
||||
|
||||
# --- Resolve destination_folder into destination_path ---
|
||||
# Trigger: caller passed destination_folder instead of destination_path
|
||||
# Why: extract the original filename from the resolved entity so callers
|
||||
# don't need a separate read_note round-trip
|
||||
# Outcome: destination_path is set to folder/original-filename.ext
|
||||
if destination_folder is not None:
|
||||
if source_entity is None:
|
||||
error_msg = (
|
||||
f"Could not resolve source entity '{identifier}' to extract filename "
|
||||
f"for destination_folder. Use destination_path with an explicit filename instead."
|
||||
)
|
||||
if output_format == "json":
|
||||
return {
|
||||
"moved": False,
|
||||
"title": None,
|
||||
"permalink": None,
|
||||
"file_path": None,
|
||||
"source": identifier,
|
||||
"destination": None,
|
||||
"error": "ENTITY_RESOLUTION_FAILED",
|
||||
}
|
||||
return f"# Move Failed - Entity Resolution Failed\n\n{error_msg}"
|
||||
|
||||
source_filename = Path(source_entity.file_path).name
|
||||
# Normalize backslashes to forward slashes for Windows compatibility,
|
||||
# then strip leading/trailing separators
|
||||
folder = PureWindowsPath(destination_folder).as_posix().strip("/")
|
||||
destination_path = f"{folder}/{source_filename}" if folder else source_filename
|
||||
|
||||
# Validate resolved path to prevent path traversal via destination_folder
|
||||
if not validate_project_path(destination_path, project_path):
|
||||
logger.warning(
|
||||
"Attempted path traversal attack blocked via destination_folder",
|
||||
destination_folder=destination_folder,
|
||||
project=active_project.name,
|
||||
)
|
||||
if output_format == "json":
|
||||
return {
|
||||
"moved": False,
|
||||
"title": None,
|
||||
"permalink": None,
|
||||
"file_path": None,
|
||||
"source": identifier,
|
||||
"destination": destination_path,
|
||||
"error": "SECURITY_VALIDATION_ERROR",
|
||||
}
|
||||
return f"""# Move Failed - Security Validation Error
|
||||
|
||||
The destination folder '{destination_folder}' is not allowed - paths must stay within project boundaries.
|
||||
|
||||
## Valid folder examples:
|
||||
- `notes`
|
||||
- `projects/2025`
|
||||
- `archive/old-notes`
|
||||
|
||||
## Try again with a safe folder:
|
||||
```
|
||||
move_note("{identifier}", destination_folder="notes")
|
||||
```"""
|
||||
|
||||
# Validate that destination path includes a file extension
|
||||
if "." not in destination_path or not destination_path.split(".")[-1]:
|
||||
logger.warning(f"Move failed - no file extension provided: {destination_path}")
|
||||
@@ -759,15 +612,14 @@ move_note("{identifier}", destination_folder="notes")
|
||||
All examples in Basic Memory expect file extensions to be explicitly provided.
|
||||
""").strip()
|
||||
|
||||
# Validate extension consistency when source metadata is available.
|
||||
if source_entity is None:
|
||||
try:
|
||||
resolved_entity_id = await _ensure_resolved_entity_id()
|
||||
source_entity = await knowledge_client.get_entity(resolved_entity_id)
|
||||
except Exception as e:
|
||||
logger.debug(f"Could not fetch source entity for extension check: {e}")
|
||||
# Get the source entity to check its file extension
|
||||
try:
|
||||
# Resolve identifier to entity ID (might already be cached from above)
|
||||
entity_id = await knowledge_client.resolve_entity(identifier)
|
||||
# Fetch source entity information
|
||||
source_entity = await knowledge_client.get_entity(entity_id)
|
||||
|
||||
if source_entity is not None:
|
||||
# Extract file extensions
|
||||
source_ext = (
|
||||
source_entity.file_path.split(".")[-1] if "." in source_entity.file_path else ""
|
||||
)
|
||||
@@ -804,13 +656,17 @@ move_note("{identifier}", destination_folder="notes")
|
||||
move_note("{identifier}", "{destination_path.rsplit(".", 1)[0]}.{source_ext}")
|
||||
```
|
||||
""").strip()
|
||||
except Exception as e:
|
||||
# If we can't fetch the source entity, log it but continue
|
||||
# This might happen if the identifier is not yet resolved
|
||||
logger.debug(f"Could not fetch source entity for extension check: {e}")
|
||||
|
||||
try:
|
||||
# Resolve identifier only if earlier checks could not.
|
||||
resolved_entity_id = await _ensure_resolved_entity_id()
|
||||
# Resolve identifier to entity ID for the move operation
|
||||
entity_id = await knowledge_client.resolve_entity(identifier)
|
||||
|
||||
# Call the move API using KnowledgeClient
|
||||
result = await knowledge_client.move_entity(resolved_entity_id, destination_path)
|
||||
result = await knowledge_client.move_entity(entity_id, destination_path)
|
||||
if output_format == "json":
|
||||
return {
|
||||
"moved": True,
|
||||
|
||||
@@ -6,276 +6,73 @@ and manage project context during conversations.
|
||||
|
||||
import os
|
||||
from typing import Literal
|
||||
|
||||
from fastmcp import Context
|
||||
from loguru import logger
|
||||
|
||||
from basic_memory.config import ConfigManager, has_cloud_credentials
|
||||
from basic_memory.mcp.async_client import get_client, get_cloud_proxy_client, is_factory_mode
|
||||
from basic_memory.mcp.async_client import get_client
|
||||
from basic_memory.mcp.server import mcp
|
||||
from basic_memory.schemas.project_info import ProjectInfoRequest, ProjectItem, ProjectList
|
||||
from basic_memory.schemas.project_info import ProjectInfoRequest
|
||||
from basic_memory.utils import generate_permalink
|
||||
|
||||
|
||||
# --- Helpers for dual-fetch + merge ---
|
||||
|
||||
|
||||
async def _fetch_cloud_projects(
|
||||
workspace: str | None = None,
|
||||
context: Context | None = None,
|
||||
) -> ProjectList | None:
|
||||
"""Fetch projects from the cloud API, returning None on failure.
|
||||
|
||||
Logs warnings on failure so the caller can fall back to local-only results.
|
||||
"""
|
||||
try:
|
||||
from basic_memory.mcp.clients import ProjectClient
|
||||
|
||||
async with get_cloud_proxy_client(workspace=workspace) as cloud_client:
|
||||
cloud_project_client = ProjectClient(cloud_client)
|
||||
cloud_list = await cloud_project_client.list_projects()
|
||||
if context: # pragma: no cover
|
||||
await context.info(f"Discovered {len(cloud_list.projects)} cloud projects")
|
||||
return cloud_list
|
||||
except Exception as exc:
|
||||
logger.warning(f"Cloud project discovery failed: {exc}")
|
||||
if context: # pragma: no cover
|
||||
await context.info("Cloud project discovery failed, showing local projects only")
|
||||
return None
|
||||
|
||||
|
||||
def _merge_projects(
|
||||
local_list: ProjectList | None,
|
||||
cloud_list: ProjectList | None,
|
||||
*,
|
||||
cloud_workspace_name: str | None = None,
|
||||
cloud_workspace_type: str | None = None,
|
||||
cloud_workspace_tenant_id: str | None = None,
|
||||
) -> list[dict]:
|
||||
"""Merge local and cloud project lists by permalink.
|
||||
|
||||
Returns a sorted list of dicts with unified project metadata.
|
||||
Same merge-by-permalink algorithm used by the CLI `bm project list`.
|
||||
"""
|
||||
names_by_permalink: dict[str, str] = {}
|
||||
local_by_permalink: dict[str, ProjectItem] = {}
|
||||
cloud_by_permalink: dict[str, ProjectItem] = {}
|
||||
|
||||
if local_list:
|
||||
for project in local_list.projects:
|
||||
permalink = generate_permalink(project.name)
|
||||
names_by_permalink[permalink] = project.name
|
||||
local_by_permalink[permalink] = project
|
||||
|
||||
if cloud_list:
|
||||
for project in cloud_list.projects:
|
||||
permalink = generate_permalink(project.name)
|
||||
names_by_permalink[permalink] = project.name
|
||||
cloud_by_permalink[permalink] = project
|
||||
|
||||
merged: list[dict] = []
|
||||
for permalink in sorted(names_by_permalink):
|
||||
name = names_by_permalink[permalink]
|
||||
local_proj = local_by_permalink.get(permalink)
|
||||
cloud_proj = cloud_by_permalink.get(permalink)
|
||||
|
||||
# Determine source label
|
||||
if local_proj and cloud_proj:
|
||||
source = "local+cloud"
|
||||
elif cloud_proj:
|
||||
source = "cloud"
|
||||
else:
|
||||
source = "local"
|
||||
|
||||
# Prefer local path for backward compat; fall back to cloud path
|
||||
local_path = local_proj.path if local_proj else None
|
||||
cloud_path = cloud_proj.path if cloud_proj else None
|
||||
path = local_path or cloud_path or ""
|
||||
|
||||
is_default = False
|
||||
if local_proj and local_proj.is_default:
|
||||
is_default = True
|
||||
if cloud_proj and cloud_proj.is_default:
|
||||
is_default = True
|
||||
|
||||
# Prefer cloud display_name / is_private (cloud injects these)
|
||||
display_name = None
|
||||
is_private = False
|
||||
if cloud_proj:
|
||||
display_name = cloud_proj.display_name
|
||||
is_private = cloud_proj.is_private
|
||||
elif local_proj:
|
||||
display_name = local_proj.display_name
|
||||
is_private = local_proj.is_private
|
||||
|
||||
# Attach workspace info for cloud-sourced projects
|
||||
ws_name = cloud_workspace_name if cloud_proj else None
|
||||
ws_type = cloud_workspace_type if cloud_proj else None
|
||||
ws_tenant_id = cloud_workspace_tenant_id if cloud_proj else None
|
||||
|
||||
merged.append(
|
||||
{
|
||||
"name": name,
|
||||
"path": path,
|
||||
"local_path": local_path,
|
||||
"cloud_path": cloud_path,
|
||||
"source": source,
|
||||
"is_default": is_default,
|
||||
"is_private": is_private,
|
||||
"display_name": display_name,
|
||||
"workspace_name": ws_name,
|
||||
"workspace_type": ws_type,
|
||||
"workspace_tenant_id": ws_tenant_id,
|
||||
}
|
||||
)
|
||||
|
||||
return merged
|
||||
|
||||
|
||||
def _format_project_list_text(merged: list[dict]) -> str:
|
||||
"""Format merged project list as human-readable text."""
|
||||
result = "Available projects:\n"
|
||||
for project in merged:
|
||||
display_name = project["display_name"]
|
||||
name = project["name"]
|
||||
label = f"{display_name} ({name})" if display_name else name
|
||||
source = project["source"]
|
||||
result += f"• {label} ({source})\n"
|
||||
|
||||
result += "\n" + "─" * 40 + "\n"
|
||||
result += "Next: Ask which project to use for this session.\n"
|
||||
result += "Example: 'Which project should I use for this task?'\n\n"
|
||||
result += (
|
||||
"Session reminder: Track the selected project for all subsequent "
|
||||
"operations in this conversation.\n"
|
||||
)
|
||||
result += "The user can say 'switch to [project]' to change projects."
|
||||
return result
|
||||
|
||||
|
||||
def _format_project_list_json(
|
||||
merged: list[dict],
|
||||
default_project: str | None,
|
||||
constrained_project: str | None,
|
||||
) -> dict:
|
||||
"""Format merged project list as structured JSON."""
|
||||
return {
|
||||
"projects": merged,
|
||||
"default_project": default_project,
|
||||
"constrained_project": constrained_project,
|
||||
}
|
||||
|
||||
|
||||
@mcp.tool(
|
||||
"list_memory_projects",
|
||||
annotations={"readOnlyHint": True, "openWorldHint": False},
|
||||
)
|
||||
@mcp.tool("list_memory_projects")
|
||||
async def list_memory_projects(
|
||||
output_format: Literal["text", "json"] = "text",
|
||||
workspace: str | None = None,
|
||||
context: Context | None = None,
|
||||
) -> str | dict:
|
||||
"""List all available projects with their status.
|
||||
|
||||
Shows projects from both local and cloud sources when cloud credentials
|
||||
are available, merging by permalink to give a unified view.
|
||||
|
||||
Args:
|
||||
output_format: "text" returns the existing human-readable project list.
|
||||
"json" returns structured project metadata.
|
||||
workspace: Cloud workspace name or tenant_id. Falls back to
|
||||
config.default_workspace when not specified.
|
||||
context: Optional FastMCP context for progress/status logging.
|
||||
"""
|
||||
if context: # pragma: no cover
|
||||
await context.info("Listing all available projects")
|
||||
|
||||
constrained_project = os.environ.get("BASIC_MEMORY_MCP_PROJECT")
|
||||
|
||||
from basic_memory.mcp.clients import ProjectClient
|
||||
|
||||
# --- Factory mode (cloud app) ---
|
||||
# Trigger: set_client_factory() was called (e.g., basic-memory-cloud)
|
||||
# Why: there is no local ASGI server; the factory IS the only source
|
||||
# Outcome: single fetch, no merge needed
|
||||
if is_factory_mode():
|
||||
async with get_client() as client:
|
||||
project_client = ProjectClient(client)
|
||||
project_list = await project_client.list_projects()
|
||||
|
||||
merged = _merge_projects(project_list, None)
|
||||
if output_format == "json":
|
||||
return _format_project_list_json(
|
||||
merged, project_list.default_project, constrained_project
|
||||
)
|
||||
if constrained_project:
|
||||
return _format_constrained_text(constrained_project)
|
||||
return _format_project_list_text(merged)
|
||||
|
||||
# --- Normal MCP stdio mode ---
|
||||
# Always fetch local projects via the ASGI transport
|
||||
async with get_client() as client:
|
||||
if context: # pragma: no cover
|
||||
await context.info("Listing all available projects")
|
||||
|
||||
constrained_project = os.environ.get("BASIC_MEMORY_MCP_PROJECT")
|
||||
|
||||
from basic_memory.mcp.clients import ProjectClient
|
||||
|
||||
project_client = ProjectClient(client)
|
||||
local_list = await project_client.list_projects()
|
||||
project_list = await project_client.list_projects()
|
||||
|
||||
# Fetch cloud projects when credentials are available
|
||||
cloud_list: ProjectList | None = None
|
||||
cloud_ws_name: str | None = None
|
||||
cloud_ws_type: str | None = None
|
||||
cloud_ws_tenant_id: str | None = None
|
||||
config = ConfigManager().config
|
||||
if has_cloud_credentials(config):
|
||||
# Use explicit workspace, fall back to config default
|
||||
effective_workspace = workspace or config.default_workspace
|
||||
cloud_list = await _fetch_cloud_projects(effective_workspace, context)
|
||||
if output_format == "json":
|
||||
projects = [
|
||||
{
|
||||
"name": project.name,
|
||||
"path": project.path,
|
||||
"is_default": project.is_default,
|
||||
"is_private": False,
|
||||
"display_name": None,
|
||||
}
|
||||
for project in project_list.projects
|
||||
]
|
||||
return {
|
||||
"projects": projects,
|
||||
"default_project": project_list.default_project,
|
||||
"constrained_project": constrained_project,
|
||||
}
|
||||
|
||||
# Resolve workspace metadata so each cloud project carries its workspace info
|
||||
if cloud_list:
|
||||
cloud_ws_tenant_id = effective_workspace
|
||||
try:
|
||||
from basic_memory.mcp.project_context import get_available_workspaces
|
||||
if constrained_project:
|
||||
result = f"Project: {constrained_project}\n\n"
|
||||
result += "Note: This MCP server is constrained to a single project.\n"
|
||||
result += "All operations will automatically use this project."
|
||||
return result
|
||||
|
||||
workspaces = await get_available_workspaces(context)
|
||||
matched = next(
|
||||
(ws for ws in workspaces if ws.tenant_id == effective_workspace),
|
||||
None,
|
||||
)
|
||||
if matched:
|
||||
cloud_ws_name = matched.name
|
||||
cloud_ws_type = matched.workspace_type
|
||||
except Exception:
|
||||
pass # workspace lookup is best-effort
|
||||
result = "Available projects:\n"
|
||||
for project in project_list.projects:
|
||||
result += f"• {project.name}\n"
|
||||
|
||||
merged = _merge_projects(
|
||||
local_list,
|
||||
cloud_list,
|
||||
cloud_workspace_name=cloud_ws_name,
|
||||
cloud_workspace_type=cloud_ws_type,
|
||||
cloud_workspace_tenant_id=cloud_ws_tenant_id,
|
||||
)
|
||||
default_project = local_list.default_project
|
||||
|
||||
if output_format == "json":
|
||||
return _format_project_list_json(merged, default_project, constrained_project)
|
||||
|
||||
if constrained_project:
|
||||
return _format_constrained_text(constrained_project)
|
||||
|
||||
return _format_project_list_text(merged)
|
||||
result += "\n" + "─" * 40 + "\n"
|
||||
result += "Next: Ask which project to use for this session.\n"
|
||||
result += "Example: 'Which project should I use for this task?'\n\n"
|
||||
result += "Session reminder: Track the selected project for all subsequent operations in this conversation.\n"
|
||||
result += "The user can say 'switch to [project]' to change projects."
|
||||
return result
|
||||
|
||||
|
||||
def _format_constrained_text(constrained_project: str) -> str:
|
||||
"""Format text output when the MCP server is constrained to a single project."""
|
||||
result = f"Project: {constrained_project}\n\n"
|
||||
result += "Note: This MCP server is constrained to a single project.\n"
|
||||
result += "All operations will automatically use this project."
|
||||
return result
|
||||
|
||||
|
||||
@mcp.tool(
|
||||
"create_memory_project",
|
||||
annotations={"destructiveHint": False, "openWorldHint": False},
|
||||
)
|
||||
@mcp.tool("create_memory_project")
|
||||
async def create_memory_project(
|
||||
project_name: str,
|
||||
project_path: str,
|
||||
@@ -357,7 +154,7 @@ async def create_memory_project(
|
||||
f"Project Details:\n"
|
||||
f"• Name: {existing_match.name}\n"
|
||||
f"• Path: {existing_match.path}\n"
|
||||
f"{'• Set as default project\n' if is_default else ''}"
|
||||
f"{'• Set as default project\\n' if is_default else ''}"
|
||||
"\nProject is already available for use in tool calls.\n"
|
||||
)
|
||||
|
||||
@@ -391,9 +188,7 @@ async def create_memory_project(
|
||||
return result
|
||||
|
||||
|
||||
@mcp.tool(
|
||||
annotations={"destructiveHint": True, "openWorldHint": False},
|
||||
)
|
||||
@mcp.tool()
|
||||
async def delete_project(project_name: str, context: Context | None = None) -> str:
|
||||
"""Delete a Basic Memory project.
|
||||
|
||||
|
||||
@@ -15,12 +15,7 @@ from PIL import Image as PILImage
|
||||
from fastmcp import Context
|
||||
from mcp.server.fastmcp.exceptions import ToolError
|
||||
|
||||
from basic_memory.config import ConfigManager
|
||||
from basic_memory.mcp.project_context import (
|
||||
detect_project_from_url_prefix,
|
||||
get_project_client,
|
||||
resolve_project_and_path,
|
||||
)
|
||||
from basic_memory.mcp.project_context import get_project_client, resolve_project_and_path
|
||||
from basic_memory.mcp.server import mcp
|
||||
from basic_memory.mcp.tools.utils import call_get, resolve_entity_id
|
||||
from basic_memory.schemas.memory import memory_url_path
|
||||
@@ -153,10 +148,7 @@ def optimize_image(img, content_length, max_output_bytes=350000):
|
||||
return buf.getvalue()
|
||||
|
||||
|
||||
@mcp.tool(
|
||||
description="Read a file's raw content by path or permalink",
|
||||
annotations={"readOnlyHint": True, "openWorldHint": False},
|
||||
)
|
||||
@mcp.tool(description="Read a file's raw content by path or permalink")
|
||||
async def read_content(
|
||||
path: str,
|
||||
project: Optional[str] = None,
|
||||
@@ -210,12 +202,6 @@ async def read_content(
|
||||
HTTPError: If project doesn't exist or is inaccessible
|
||||
SecurityError: If path attempts path traversal
|
||||
"""
|
||||
# Detect project from memory URL prefix before routing
|
||||
if project is None:
|
||||
detected = detect_project_from_url_prefix(path, ConfigManager().config)
|
||||
if detected:
|
||||
project = detected
|
||||
|
||||
logger.info("Reading file", path=path, project=project)
|
||||
|
||||
async with get_project_client(project, workspace, context) as (client, active_project):
|
||||
|
||||
@@ -8,12 +8,7 @@ import yaml
|
||||
from loguru import logger
|
||||
from fastmcp import Context
|
||||
|
||||
from basic_memory.config import ConfigManager
|
||||
from basic_memory.mcp.project_context import (
|
||||
detect_project_from_url_prefix,
|
||||
get_project_client,
|
||||
resolve_project_and_path,
|
||||
)
|
||||
from basic_memory.mcp.project_context import get_project_client, resolve_project_and_path
|
||||
from basic_memory.mcp.server import mcp
|
||||
from basic_memory.mcp.tools.search import search_notes
|
||||
from basic_memory.schemas.memory import memory_url_path
|
||||
@@ -64,7 +59,6 @@ def _parse_opening_frontmatter(content: str) -> tuple[str, dict | None]:
|
||||
description="Read a markdown note by title or permalink.",
|
||||
# TODO: re-enable once MCP client rendering is working
|
||||
# meta={"ui/resourceUri": "ui://basic-memory/note-preview"},
|
||||
annotations={"readOnlyHint": True, "openWorldHint": False},
|
||||
)
|
||||
async def read_note(
|
||||
identifier: str,
|
||||
@@ -133,12 +127,6 @@ async def read_note(
|
||||
If the exact note isn't found, this tool provides helpful suggestions
|
||||
including related notes, search commands, and note creation templates.
|
||||
"""
|
||||
# Detect project from memory URL prefix before routing
|
||||
if project is None:
|
||||
detected = detect_project_from_url_prefix(identifier, ConfigManager().config)
|
||||
if detected:
|
||||
project = detected
|
||||
|
||||
async with get_project_client(project, workspace, context) as (client, active_project):
|
||||
# Resolve identifier with project-prefix awareness for memory:// URLs
|
||||
_, entity_path, _ = await resolve_project_and_path(client, identifier, project, context)
|
||||
@@ -204,21 +192,32 @@ async def read_note(
|
||||
"frontmatter": None,
|
||||
}
|
||||
|
||||
def _search_results(payload: object) -> list[dict]:
|
||||
if not isinstance(payload, dict):
|
||||
return []
|
||||
results = payload.get("results")
|
||||
return results if isinstance(results, list) else []
|
||||
def _search_results(payload: object) -> list:
|
||||
if isinstance(payload, dict):
|
||||
results = payload.get("results")
|
||||
return results if isinstance(results, list) else []
|
||||
if hasattr(payload, "results"):
|
||||
results = getattr(payload, "results")
|
||||
return results if isinstance(results, list) else []
|
||||
return []
|
||||
|
||||
def _result_title(item: dict) -> str:
|
||||
return str(item.get("title") or "")
|
||||
def _result_title(item: object) -> str:
|
||||
if isinstance(item, dict):
|
||||
return str(item.get("title") or "")
|
||||
return str(getattr(item, "title", "") or "")
|
||||
|
||||
def _result_permalink(item: dict) -> Optional[str]:
|
||||
value = item.get("permalink")
|
||||
def _result_permalink(item: object) -> Optional[str]:
|
||||
if isinstance(item, dict):
|
||||
value = item.get("permalink")
|
||||
return str(value) if value else None
|
||||
value = getattr(item, "permalink", None)
|
||||
return str(value) if value else None
|
||||
|
||||
def _result_file_path(item: dict) -> Optional[str]:
|
||||
value = item.get("file_path")
|
||||
def _result_file_path(item: object) -> Optional[str]:
|
||||
if isinstance(item, dict):
|
||||
value = item.get("file_path")
|
||||
return str(value) if value else None
|
||||
value = getattr(item, "file_path", None)
|
||||
return str(value) if value else None
|
||||
|
||||
try:
|
||||
@@ -240,7 +239,7 @@ async def read_note(
|
||||
|
||||
# Fallback 1: Try title search via API
|
||||
logger.info(f"Search title for: {identifier}")
|
||||
title_results = await search_notes(
|
||||
title_results = await search_notes.fn(
|
||||
query=identifier,
|
||||
search_type="title",
|
||||
project=active_project.name,
|
||||
@@ -292,7 +291,7 @@ async def read_note(
|
||||
|
||||
# Fallback 2: Text search as a last resort
|
||||
logger.info(f"Title search failed, trying text search for: {identifier}")
|
||||
text_results = await search_notes(
|
||||
text_results = await search_notes.fn(
|
||||
query=identifier,
|
||||
search_type="text",
|
||||
project=active_project.name,
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
"""Recent activity tool for Basic Memory MCP server."""
|
||||
|
||||
from datetime import timezone
|
||||
from pathlib import PurePosixPath
|
||||
from typing import List, Union, Optional, Literal
|
||||
|
||||
from loguru import logger
|
||||
@@ -35,14 +34,11 @@ from basic_memory.schemas.search import SearchItemType
|
||||
- "3 weeks ago"
|
||||
Or standard formats like "7d"
|
||||
""",
|
||||
annotations={"readOnlyHint": True, "openWorldHint": False},
|
||||
)
|
||||
async def recent_activity(
|
||||
type: Union[str, List[str]] = "",
|
||||
depth: int = 1,
|
||||
timeframe: TimeFrame = "7d",
|
||||
page: int = 1,
|
||||
page_size: int = 10,
|
||||
project: Optional[str] = None,
|
||||
workspace: Optional[str] = None,
|
||||
output_format: Literal["text", "json"] = "text",
|
||||
@@ -74,11 +70,8 @@ async def recent_activity(
|
||||
- "observation" or ["observation"] for notes and observations
|
||||
Multiple types can be combined: ["entity", "relation"]
|
||||
Case-insensitive: "ENTITY" and "entity" are treated the same.
|
||||
Default is entity-only. Specify other types explicitly to include
|
||||
observations and relations.
|
||||
Default is an empty string, which returns all types.
|
||||
depth: How many relation hops to traverse (1-3 recommended)
|
||||
page: Page number for pagination (default 1)
|
||||
page_size: Number of items per page (default 10)
|
||||
timeframe: Time window to search. Supports natural language:
|
||||
- Relative: "2 days ago", "last week", "yesterday"
|
||||
- Points in time: "2024-01-01", "January 1st"
|
||||
@@ -87,7 +80,7 @@ async def recent_activity(
|
||||
hierarchy above. If unknown, use list_memory_projects() to discover
|
||||
available projects.
|
||||
output_format: "text" returns human-readable summary text. "json" returns
|
||||
a flat list of recent items.
|
||||
a flat list of recent entity items.
|
||||
context: Optional FastMCP context for performance caching.
|
||||
|
||||
Returns:
|
||||
@@ -113,19 +106,10 @@ async def recent_activity(
|
||||
- For focused queries, consider using build_context with a specific URI
|
||||
- Max timeframe is 1 year in the past
|
||||
"""
|
||||
# Validate pagination arguments before they reach the API layer,
|
||||
# where negative offset would cause a database error.
|
||||
if page < 1:
|
||||
raise ValueError(f"page must be >= 1, got {page}")
|
||||
if page_size < 1:
|
||||
raise ValueError(f"page_size must be >= 1, got {page_size}")
|
||||
if page_size > 100:
|
||||
raise ValueError(f"page_size must be <= 100, got {page_size}")
|
||||
|
||||
# Build common parameters for API calls
|
||||
params: dict = {
|
||||
"page": page,
|
||||
"page_size": page_size,
|
||||
"page": 1,
|
||||
"page_size": 10,
|
||||
"max_related": 10,
|
||||
}
|
||||
if depth:
|
||||
@@ -155,12 +139,6 @@ async def recent_activity(
|
||||
# Add validated types to params
|
||||
params["type"] = [t.value for t in validated_types] # pyright: ignore
|
||||
|
||||
# Default to entity-only when no explicit type was provided.
|
||||
# This prevents a single well-connected entity from filling the page
|
||||
# with its observations and relations.
|
||||
if "type" not in params:
|
||||
params["type"] = [SearchItemType.ENTITY.value]
|
||||
|
||||
# Resolve project parameter using the three-tier hierarchy
|
||||
# allow_discovery=True enables Discovery Mode, so a project is not required
|
||||
resolved_project = await resolve_project_parameter(project, allow_discovery=True)
|
||||
@@ -214,7 +192,7 @@ async def recent_activity(
|
||||
if output_format == "json":
|
||||
rows: list[dict] = []
|
||||
for project_name, project_activity in projects_activity.items():
|
||||
rows.extend(_extract_recent_rows(project_activity.activity, project_name))
|
||||
rows.extend(_extract_recent_entity_rows(project_activity.activity, project_name))
|
||||
return rows
|
||||
|
||||
# Build summary stats
|
||||
@@ -290,10 +268,10 @@ async def recent_activity(
|
||||
activity_data = GraphContext.model_validate(response.json())
|
||||
|
||||
if output_format == "json":
|
||||
return _extract_recent_rows(activity_data)
|
||||
return _extract_recent_entity_rows(activity_data)
|
||||
|
||||
# Format project-specific mode output
|
||||
return _format_project_output(resolved_project, activity_data, timeframe, type, page)
|
||||
return _format_project_output(resolved_project, activity_data, timeframe, type)
|
||||
|
||||
|
||||
async def _get_project_activity(
|
||||
@@ -334,9 +312,9 @@ async def _get_project_activity(
|
||||
last_activity = current_time
|
||||
|
||||
# Extract folder from file_path
|
||||
if result.primary_result.file_path:
|
||||
folder = str(PurePosixPath(result.primary_result.file_path).parent)
|
||||
if folder and folder != ".":
|
||||
if hasattr(result.primary_result, "file_path") and result.primary_result.file_path:
|
||||
folder = "/".join(result.primary_result.file_path.split("/")[:-1])
|
||||
if folder:
|
||||
active_folders.add(folder)
|
||||
|
||||
return ProjectActivity(
|
||||
@@ -349,19 +327,22 @@ async def _get_project_activity(
|
||||
)
|
||||
|
||||
|
||||
def _extract_recent_rows(
|
||||
def _extract_recent_entity_rows(
|
||||
activity_data: GraphContext, project_name: Optional[str] = None
|
||||
) -> list[dict]:
|
||||
"""Flatten GraphContext into a list of recent rows."""
|
||||
"""Flatten GraphContext into a list of recent entity rows."""
|
||||
rows: list[dict] = []
|
||||
for result in activity_data.results:
|
||||
primary = result.primary_result
|
||||
if primary.type != "entity":
|
||||
continue
|
||||
row = {
|
||||
"type": primary.type,
|
||||
"title": primary.title,
|
||||
"permalink": primary.permalink,
|
||||
"file_path": primary.file_path,
|
||||
"created_at": primary.created_at.isoformat() if primary.created_at else None,
|
||||
"created_at": (
|
||||
primary.created_at.isoformat() if getattr(primary, "created_at", None) else None
|
||||
),
|
||||
}
|
||||
if project_name is not None:
|
||||
row["project"] = project_name
|
||||
@@ -385,7 +366,7 @@ def _format_discovery_output(
|
||||
# Get latest activity from most active project
|
||||
if most_active.activity.results:
|
||||
latest = most_active.activity.results[0].primary_result
|
||||
title = latest.title or "Recent activity"
|
||||
title = latest.title if hasattr(latest, "title") and latest.title else "Recent activity"
|
||||
# Format relative time
|
||||
time_str = (
|
||||
_format_relative_time(latest.created_at) if latest.created_at else "unknown time"
|
||||
@@ -414,7 +395,9 @@ def _format_discovery_output(
|
||||
for name, activity in projects_activity.items():
|
||||
if activity.item_count > 0:
|
||||
for result in activity.activity.results[:3]: # Top 3 from each active project
|
||||
if result.primary_result.type == "entity":
|
||||
if result.primary_result.type == "entity" and hasattr(
|
||||
result.primary_result, "title"
|
||||
):
|
||||
title = result.primary_result.title
|
||||
# Look for status indicators in titles
|
||||
if any(word in title.lower() for word in ["complete", "fix", "test", "spec"]):
|
||||
@@ -442,7 +425,6 @@ def _format_project_output(
|
||||
activity_data: GraphContext,
|
||||
timeframe: str,
|
||||
type_filter: Union[str, List[str]],
|
||||
page: int = 1,
|
||||
) -> str:
|
||||
"""Format project-specific mode output as human-readable text."""
|
||||
lines = [f"## Recent Activity: {project_name} ({timeframe})"]
|
||||
@@ -468,12 +450,12 @@ def _format_project_output(
|
||||
if entities:
|
||||
lines.append(f"\n**📄 Recent Notes & Documents ({len(entities)}):**")
|
||||
for entity in entities[:5]: # Show top 5
|
||||
title = entity.title or "Untitled"
|
||||
# Get folder from file_path
|
||||
title = entity.title if hasattr(entity, "title") and entity.title else "Untitled"
|
||||
# Get folder from file_path if available
|
||||
folder = ""
|
||||
if entity.file_path:
|
||||
folder_path = str(PurePosixPath(entity.file_path).parent)
|
||||
if folder_path and folder_path != ".":
|
||||
if hasattr(entity, "file_path") and entity.file_path:
|
||||
folder_path = "/".join(entity.file_path.split("/")[:-1])
|
||||
if folder_path:
|
||||
folder = f" ({folder_path})"
|
||||
lines.append(f" • {title}{folder}")
|
||||
|
||||
@@ -483,7 +465,9 @@ def _format_project_output(
|
||||
# Group by category
|
||||
by_category = {}
|
||||
for obs in observations[:10]: # Limit to recent ones
|
||||
category = obs.category
|
||||
category = (
|
||||
getattr(obs, "category", "general") if hasattr(obs, "category") else "general"
|
||||
)
|
||||
if category not in by_category:
|
||||
by_category[category] = []
|
||||
by_category[category].append(obs)
|
||||
@@ -491,7 +475,11 @@ def _format_project_output(
|
||||
for category, obs_list in list(by_category.items())[:5]: # Show top 5 categories
|
||||
lines.append(f" **{category}:** {len(obs_list)} items")
|
||||
for obs in obs_list[:2]: # Show 2 examples per category
|
||||
content = obs.content
|
||||
content = (
|
||||
getattr(obs, "content", "No content")
|
||||
if hasattr(obs, "content")
|
||||
else "No content"
|
||||
)
|
||||
# Truncate at word boundary
|
||||
if len(content) > 80:
|
||||
content = _truncate_at_word(content, 80)
|
||||
@@ -501,9 +489,15 @@ def _format_project_output(
|
||||
if relations:
|
||||
lines.append(f"\n**🔗 Recent Connections ({len(relations)}):**")
|
||||
for rel in relations[:5]: # Show top 5
|
||||
rel_type = rel.relation_type
|
||||
from_entity = rel.from_entity or "Unknown"
|
||||
to_entity = rel.to_entity
|
||||
rel_type = (
|
||||
getattr(rel, "relation_type", "relates_to")
|
||||
if hasattr(rel, "relation_type")
|
||||
else "relates_to"
|
||||
)
|
||||
from_entity = (
|
||||
getattr(rel, "from_entity", "Unknown") if hasattr(rel, "from_entity") else "Unknown"
|
||||
)
|
||||
to_entity = getattr(rel, "to_entity", None) if hasattr(rel, "to_entity") else None
|
||||
|
||||
# Format as WikiLinks to show they're readable notes
|
||||
from_link = f"[[{from_entity}]]" if from_entity != "Unknown" else from_entity
|
||||
@@ -511,15 +505,12 @@ def _format_project_output(
|
||||
|
||||
lines.append(f" • {from_link} → {rel_type} → {to_link}")
|
||||
|
||||
# Activity summary with pagination guidance
|
||||
# Activity summary
|
||||
total = len(activity_data.results)
|
||||
if activity_data.has_more:
|
||||
lines.append(
|
||||
f"\n**Activity Summary:** Showing {total} items (page {page}). "
|
||||
f"Use page={page + 1} to see more."
|
||||
)
|
||||
else:
|
||||
lines.append(f"\n**Activity Summary:** {total} items found.")
|
||||
lines.append(f"\n**Activity Summary:** {total} items found")
|
||||
if hasattr(activity_data, "metadata") and activity_data.metadata:
|
||||
if hasattr(activity_data.metadata, "total_results"):
|
||||
lines.append(f"Total available: {activity_data.metadata.total_results}")
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
@@ -5,10 +5,7 @@ from pathlib import Path
|
||||
from basic_memory.mcp.server import mcp
|
||||
|
||||
|
||||
@mcp.tool(
|
||||
"release_notes",
|
||||
annotations={"readOnlyHint": True, "openWorldHint": False},
|
||||
)
|
||||
@mcp.tool("release_notes")
|
||||
def release_notes() -> str:
|
||||
"""Return the latest product release notes for optional user review."""
|
||||
content_path = Path(__file__).parent.parent / "resources" / "release_notes.md"
|
||||
|
||||
@@ -4,148 +4,14 @@ Provides tools for schema validation, inference, and drift detection through the
|
||||
These tools call the schema API endpoints via the typed SchemaClient.
|
||||
"""
|
||||
|
||||
from typing import Literal, Optional
|
||||
from typing import Optional
|
||||
|
||||
from loguru import logger
|
||||
from fastmcp import Context
|
||||
|
||||
from basic_memory.mcp.project_context import get_project_client
|
||||
from basic_memory.mcp.server import mcp
|
||||
from basic_memory.schemas.schema import DriftReport, InferenceReport, ValidationReport
|
||||
|
||||
|
||||
def _format_validation_report(report: ValidationReport) -> str:
|
||||
"""Render a ValidationReport as readable markdown.
|
||||
|
||||
Produces output the LLM can display directly instead of trying to
|
||||
interpret raw JSON, which leads to "undefined — invalid" rendering.
|
||||
"""
|
||||
lines: list[str] = []
|
||||
|
||||
# --- Header ---
|
||||
type_label = report.note_type or "all"
|
||||
lines.append(f"# Schema Validation: {type_label}")
|
||||
lines.append("")
|
||||
lines.append(
|
||||
f"Notes: {report.total_notes} | Valid: {report.valid_count} "
|
||||
f"| Warnings: {report.warning_count} | Errors: {report.error_count}"
|
||||
)
|
||||
lines.append("")
|
||||
|
||||
# --- Per-note results ---
|
||||
for r in report.results:
|
||||
status = "valid" if r.passed else "INVALID"
|
||||
lines.append(f"- **{r.note_identifier}** — {status}")
|
||||
for w in r.warnings:
|
||||
lines.append(f" - warning: {w}")
|
||||
for e in r.errors:
|
||||
lines.append(f" - error: {e}")
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def _format_inference_report(report: InferenceReport) -> str:
|
||||
"""Render an InferenceReport as readable markdown.
|
||||
|
||||
Without this formatter the LLM receives raw JSON and renders
|
||||
field names as "undefined".
|
||||
"""
|
||||
lines: list[str] = []
|
||||
|
||||
# --- Header ---
|
||||
lines.append(f"# Schema Inference: {report.note_type}")
|
||||
lines.append("")
|
||||
lines.append(f"Notes analyzed: {report.notes_analyzed}")
|
||||
lines.append("")
|
||||
|
||||
# --- Suggested schema YAML ---
|
||||
if report.suggested_schema:
|
||||
lines.append("## Suggested Schema")
|
||||
lines.append("")
|
||||
lines.append("```yaml")
|
||||
lines.append("---")
|
||||
lines.append(f"title: {report.note_type.title()}")
|
||||
lines.append("type: schema")
|
||||
lines.append(f"entity: {report.note_type}")
|
||||
lines.append("version: 1")
|
||||
lines.append("schema:")
|
||||
for field_name, field_def in report.suggested_schema.items():
|
||||
lines.append(f" {field_name}: {field_def}")
|
||||
lines.append("---")
|
||||
lines.append("```")
|
||||
lines.append("")
|
||||
|
||||
# --- Field frequency table ---
|
||||
if report.field_frequencies:
|
||||
lines.append("## Field Frequencies")
|
||||
lines.append("")
|
||||
for f in report.field_frequencies:
|
||||
pct = f"{f.percentage:.0%}"
|
||||
req_marker = "required" if f.name in report.suggested_required else "optional"
|
||||
samples = ", ".join(f.sample_values[:3]) if f.sample_values else ""
|
||||
sample_str = f" (e.g. {samples})" if samples else ""
|
||||
lines.append(
|
||||
f"- **{f.name}** ({f.source}) — {pct} ({f.count}/{f.total}) "
|
||||
f"[{req_marker}]{sample_str}"
|
||||
)
|
||||
lines.append("")
|
||||
|
||||
# --- Excluded fields ---
|
||||
if report.excluded:
|
||||
lines.append("## Excluded (below threshold)")
|
||||
lines.append("")
|
||||
for name in report.excluded:
|
||||
lines.append(f"- {name}")
|
||||
lines.append("")
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def _format_drift_report(report: DriftReport) -> str:
|
||||
"""Render a DriftReport as readable markdown.
|
||||
|
||||
Without this formatter the LLM receives raw JSON and renders
|
||||
field names as "undefined".
|
||||
"""
|
||||
lines: list[str] = []
|
||||
|
||||
# --- Header ---
|
||||
lines.append(f"# Schema Drift: {report.note_type}")
|
||||
lines.append("")
|
||||
|
||||
has_drift = report.new_fields or report.dropped_fields or report.cardinality_changes
|
||||
|
||||
if not has_drift:
|
||||
lines.append("No drift detected — schema matches actual usage.")
|
||||
return "\n".join(lines)
|
||||
|
||||
# --- New fields ---
|
||||
if report.new_fields:
|
||||
lines.append("## New Fields (in notes but not in schema)")
|
||||
lines.append("")
|
||||
for f in report.new_fields:
|
||||
pct = f"{f.percentage:.0%}"
|
||||
lines.append(f"- **{f.name}** ({f.source}) — {pct} ({f.count}/{f.total})")
|
||||
lines.append("")
|
||||
|
||||
# --- Dropped fields ---
|
||||
if report.dropped_fields:
|
||||
lines.append("## Dropped Fields (in schema but rare in notes)")
|
||||
lines.append("")
|
||||
for f in report.dropped_fields:
|
||||
pct = f"{f.percentage:.0%}"
|
||||
lines.append(f"- **{f.name}** ({f.source}) — {pct} ({f.count}/{f.total})")
|
||||
lines.append("")
|
||||
|
||||
# --- Cardinality changes ---
|
||||
if report.cardinality_changes:
|
||||
lines.append("## Cardinality Changes")
|
||||
lines.append("")
|
||||
for change in report.cardinality_changes:
|
||||
lines.append(f"- {change}")
|
||||
lines.append("")
|
||||
|
||||
return "\n".join(lines)
|
||||
from basic_memory.schemas.schema import ValidationReport, InferenceReport, DriftReport
|
||||
|
||||
|
||||
def _no_notes_guidance(note_type: str, tool_name: str) -> str:
|
||||
@@ -160,7 +26,7 @@ def _no_notes_guidance(note_type: str, tool_name: str) -> str:
|
||||
f"## Next Steps\n\n"
|
||||
f"1. **Create notes of this type** — use `write_note` with "
|
||||
f'`note_type="{note_type}"` to create notes\n'
|
||||
f"2. **Check existing types** — use `search_notes` with `note_types` "
|
||||
f"2. **Check existing types** — use `search_notes` with `entity_types` "
|
||||
f"filter to see what types exist\n"
|
||||
f"3. **Browse content** — use `list_directory` or `recent_activity` to "
|
||||
f"see what's in the project\n"
|
||||
@@ -205,16 +71,14 @@ def _no_schema_guidance(note_type: str, tool_name: str) -> str:
|
||||
|
||||
@mcp.tool(
|
||||
description="Validate notes against their Picoschema definitions.",
|
||||
annotations={"readOnlyHint": True, "openWorldHint": False},
|
||||
)
|
||||
async def schema_validate(
|
||||
note_type: Optional[str] = None,
|
||||
identifier: Optional[str] = None,
|
||||
project: Optional[str] = None,
|
||||
workspace: Optional[str] = None,
|
||||
output_format: Literal["text", "json"] = "text",
|
||||
context: Context | None = None,
|
||||
) -> ValidationReport | str | dict:
|
||||
) -> ValidationReport | str:
|
||||
"""Validate notes against their resolved schema.
|
||||
|
||||
Validates a specific note (by identifier) or all notes of a given type.
|
||||
@@ -262,7 +126,7 @@ async def schema_validate(
|
||||
|
||||
schema_client = SchemaClient(client, active_project.external_id)
|
||||
result = await schema_client.validate(
|
||||
note_type=note_type,
|
||||
entity_type=note_type,
|
||||
identifier=identifier,
|
||||
)
|
||||
|
||||
@@ -276,30 +140,20 @@ async def schema_validate(
|
||||
# Trigger: no entities of this type exist in the project
|
||||
# Why: can't validate notes that don't exist yet
|
||||
# Outcome: return guidance on creating notes of this type
|
||||
effective_type = note_type or result.note_type or "unknown"
|
||||
if result.total_entities == 0:
|
||||
if output_format == "json":
|
||||
return {"error": f"No notes found of type '{effective_type}'"}
|
||||
return _no_notes_guidance(effective_type, "schema_validate")
|
||||
if note_type and result.total_entities == 0:
|
||||
return _no_notes_guidance(note_type, "schema_validate")
|
||||
|
||||
# --- No schema guard ---
|
||||
# Trigger: entities exist but none were validated (no schema found)
|
||||
# Why: notes of this type exist but no schema was found, so none were validated
|
||||
# Outcome: return guidance on how to create a schema
|
||||
if result.total_notes == 0:
|
||||
if output_format == "json":
|
||||
return {"error": f"No schema found for type '{effective_type}'"}
|
||||
return _no_schema_guidance(effective_type, "schema_validate")
|
||||
if note_type and result.total_notes == 0:
|
||||
return _no_schema_guidance(note_type, "schema_validate")
|
||||
|
||||
if output_format == "json":
|
||||
return result.model_dump(mode="json", exclude_none=True)
|
||||
|
||||
return _format_validation_report(result)
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Schema validation failed: {e}, project: {active_project.name}")
|
||||
if output_format == "json":
|
||||
return {"error": f"Schema validation failed: {e}"}
|
||||
return (
|
||||
f"# Schema Validation Failed\n\n"
|
||||
f"Error validating schemas: {e}\n\n"
|
||||
@@ -312,16 +166,14 @@ async def schema_validate(
|
||||
|
||||
@mcp.tool(
|
||||
description="Analyze existing notes and suggest a Picoschema definition.",
|
||||
annotations={"readOnlyHint": True, "openWorldHint": False},
|
||||
)
|
||||
async def schema_infer(
|
||||
note_type: str,
|
||||
threshold: float = 0.25,
|
||||
project: Optional[str] = None,
|
||||
workspace: Optional[str] = None,
|
||||
output_format: Literal["text", "json"] = "text",
|
||||
context: Context | None = None,
|
||||
) -> str | dict:
|
||||
) -> InferenceReport | str:
|
||||
"""Analyze existing notes and suggest a schema definition.
|
||||
|
||||
Examines observation categories and relation types across all notes
|
||||
@@ -381,13 +233,6 @@ async def schema_infer(
|
||||
# Why: returning hundreds of excluded fields overwhelms the LLM context
|
||||
# Outcome: return actionable guidance instead of a massive empty result
|
||||
if result.notes_analyzed > 0 and not result.suggested_schema:
|
||||
if output_format == "json":
|
||||
return {
|
||||
"error": (
|
||||
f"No schema pattern found for '{note_type}' "
|
||||
f"(threshold: {threshold:.0%})"
|
||||
)
|
||||
}
|
||||
return (
|
||||
f"# No Schema Pattern Found\n\n"
|
||||
f"Analyzed {result.notes_analyzed} notes of type '{note_type}', "
|
||||
@@ -397,7 +242,7 @@ async def schema_infer(
|
||||
f"share a consistent structure.\n\n"
|
||||
f"## Suggestions\n"
|
||||
f"1. **Use a more specific type** — try `search_notes` with "
|
||||
f"`note_types` filter to see what types exist\n"
|
||||
f"`entity_types` filter to see what types exist\n"
|
||||
f"2. **Lower the threshold** — "
|
||||
f'`schema_infer("{note_type}", threshold=0.1)` to include '
|
||||
f"rarer fields\n"
|
||||
@@ -406,36 +251,29 @@ async def schema_infer(
|
||||
f"structure\n"
|
||||
)
|
||||
|
||||
if output_format == "json":
|
||||
return result.model_dump(mode="json", exclude_none=True)
|
||||
|
||||
return _format_inference_report(result)
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Schema inference failed: {e}, project: {active_project.name}")
|
||||
if output_format == "json":
|
||||
return {"error": f"Schema inference failed: {e}"}
|
||||
return (
|
||||
f"# Schema Inference Failed\n\n"
|
||||
f"Error inferring schema for type '{note_type}': {e}\n\n"
|
||||
f"## Troubleshooting\n"
|
||||
f"1. Ensure notes of type '{note_type}' exist in the project\n"
|
||||
f'2. Try searching: `search_notes("{note_type}", note_types=["{note_type}"])`\n'
|
||||
f'2. Try searching: `search_notes("{note_type}", types=["{note_type}"])`\n'
|
||||
f"3. Verify the project has been synced: `basic-memory status`\n"
|
||||
)
|
||||
|
||||
|
||||
@mcp.tool(
|
||||
description="Detect drift between a schema definition and actual note usage.",
|
||||
annotations={"readOnlyHint": True, "openWorldHint": False},
|
||||
)
|
||||
async def schema_diff(
|
||||
note_type: str,
|
||||
project: Optional[str] = None,
|
||||
workspace: Optional[str] = None,
|
||||
output_format: Literal["text", "json"] = "text",
|
||||
context: Context | None = None,
|
||||
) -> str | dict:
|
||||
) -> DriftReport | str:
|
||||
"""Detect drift between a schema definition and actual note usage.
|
||||
|
||||
Compares the existing schema for a note type against how notes of
|
||||
@@ -490,19 +328,12 @@ async def schema_diff(
|
||||
# Why: diff requires a schema to compare against
|
||||
# Outcome: return guidance on how to create a schema
|
||||
if not result.schema_found:
|
||||
if output_format == "json":
|
||||
return {"error": f"No schema found for type '{note_type}'"}
|
||||
return _no_schema_guidance(note_type, "schema_diff")
|
||||
|
||||
if output_format == "json":
|
||||
return result.model_dump(mode="json", exclude_none=True)
|
||||
|
||||
return _format_drift_report(result)
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Schema diff failed: {e}, project: {active_project.name}")
|
||||
if output_format == "json":
|
||||
return {"error": f"Schema diff failed: {e}"}
|
||||
return (
|
||||
f"# Schema Diff Failed\n\n"
|
||||
f"Error detecting drift for type '{note_type}': {e}\n\n"
|
||||
|
||||
@@ -1,21 +1,14 @@
|
||||
"""Search tools for Basic Memory MCP server."""
|
||||
|
||||
import re
|
||||
from textwrap import dedent
|
||||
from typing import Annotated, List, Optional, Dict, Any, Literal
|
||||
from typing import List, Optional, Dict, Any, Literal
|
||||
|
||||
from loguru import logger
|
||||
from fastmcp import Context
|
||||
from pydantic import BeforeValidator
|
||||
|
||||
from basic_memory.config import ConfigManager
|
||||
from basic_memory.utils import coerce_dict, coerce_list
|
||||
from basic_memory.mcp.container import get_container
|
||||
from basic_memory.mcp.project_context import (
|
||||
detect_project_from_url_prefix,
|
||||
get_project_client,
|
||||
resolve_project_and_path,
|
||||
)
|
||||
from basic_memory.mcp.project_context import get_project_client, resolve_project_and_path
|
||||
from basic_memory.mcp.server import mcp
|
||||
from basic_memory.schemas.search import (
|
||||
SearchItemType,
|
||||
@@ -36,11 +29,6 @@ def _semantic_search_enabled_for_text_search() -> bool:
|
||||
return ConfigManager().config.semantic_search_enabled
|
||||
|
||||
|
||||
def _default_search_type() -> str:
|
||||
"""Pick default search mode from semantic-search config."""
|
||||
return "hybrid" if _semantic_search_enabled_for_text_search() else "text"
|
||||
|
||||
|
||||
def _format_search_error_response(
|
||||
project: str, error_message: str, query: str, search_type: str = "text"
|
||||
) -> str:
|
||||
@@ -69,7 +57,7 @@ def _format_search_error_response(
|
||||
Semantic retrieval is enabled but required packages are not installed.
|
||||
|
||||
## Fix
|
||||
1. Install/update Basic Memory: `pip install -U basic-memory`
|
||||
1. Install semantic extras: `pip install 'basic-memory[semantic]'`
|
||||
2. Restart Basic Memory
|
||||
3. Retry your query:
|
||||
`search_notes("{project}", "{query}", search_type="{search_type}")`
|
||||
@@ -116,7 +104,7 @@ def _format_search_error_response(
|
||||
## Alternative search strategies:
|
||||
- Break into simpler terms: `search_notes("{project}", "{" ".join(clean_query.split()[:2])}")`
|
||||
- Try different search types: `search_notes("{project}","{clean_query}", search_type="title")`
|
||||
- Use filtering: `search_notes("{project}","{clean_query}", note_types=["note"])`
|
||||
- Use filtering: `search_notes("{project}","{clean_query}", types=["entity"])`
|
||||
""").strip()
|
||||
|
||||
# Project not found errors (check before general "not found")
|
||||
@@ -167,7 +155,7 @@ def _format_search_error_response(
|
||||
- Remove restrictive terms: Focus on the most important keywords
|
||||
|
||||
5. **Use filtering to narrow scope**:
|
||||
- By note type in frontmatter: `search_notes("{project}","{query}", note_types=["note"])`
|
||||
- By content type: `search_notes("{project}","{query}", types=["entity"])`
|
||||
- By recent content: `search_notes("{project}","{query}", after_date="1 week")`
|
||||
- By entity type: `search_notes("{project}","{query}", entity_types=["observation"])`
|
||||
|
||||
@@ -236,7 +224,7 @@ Error searching for '{query}': {error_message}
|
||||
- **Different search types**:
|
||||
- Title only: `search_notes("{project}","{query}", search_type="title")`
|
||||
- Permalink patterns: `search_notes("{project}","{query}*", search_type="permalink")`
|
||||
- **With filters**: `search_notes("{project}","{query}", note_types=["note"])`
|
||||
- **With filters**: `search_notes("{project}","{query}", types=["entity"])`
|
||||
- **Recent content**: `search_notes("{project}","{query}", after_date="1 week")`
|
||||
- **Boolean variations**: `search_notes("{project}","{" OR ".join(query.split()[:2])}")`
|
||||
|
||||
@@ -253,86 +241,28 @@ Error searching for '{query}': {error_message}
|
||||
- **Patterns**: `tag:example`, `category:observation`"""
|
||||
|
||||
|
||||
def _format_search_markdown(result: SearchResponse, project: str, query: str | None) -> str:
|
||||
"""Format SearchResponse as compact markdown text.
|
||||
|
||||
Produces a human-readable markdown representation suitable for LLM
|
||||
consumption when structured data isn't needed.
|
||||
"""
|
||||
if not result.results:
|
||||
return f"No results found for '{query or ''}' in project '{project}'."
|
||||
|
||||
parts = []
|
||||
|
||||
# --- Header ---
|
||||
if query:
|
||||
parts.append(f"# Search Results: {query}")
|
||||
else:
|
||||
parts.append("# Search Results")
|
||||
parts.append(f"*project: {project}*")
|
||||
parts.append("")
|
||||
|
||||
# --- Result blocks ---
|
||||
for r in result.results:
|
||||
parts.append(f"### {r.title}")
|
||||
parts.append(f"- permalink: {r.permalink}")
|
||||
parts.append(f"- score: {r.score:.4f}")
|
||||
if r.matched_chunk:
|
||||
parts.append(f"- match: {r.matched_chunk[:200]}")
|
||||
parts.append("")
|
||||
|
||||
# --- Footer with pagination ---
|
||||
parts.append("---")
|
||||
count = len(result.results)
|
||||
parts.append(
|
||||
f"*{count} result{'s' if count != 1 else ''}"
|
||||
f" | page {result.current_page}, page_size {result.page_size}"
|
||||
f"{' | more available' if result.has_more else ''}*"
|
||||
)
|
||||
|
||||
return "\n".join(parts)
|
||||
|
||||
|
||||
@mcp.tool(
|
||||
description="Search across all content in the knowledge base with advanced syntax support.",
|
||||
# TODO: re-enable once MCP client rendering is working
|
||||
# meta={"ui/resourceUri": "ui://basic-memory/search-results"},
|
||||
annotations={"readOnlyHint": True, "openWorldHint": False},
|
||||
)
|
||||
async def search_notes(
|
||||
query: Optional[str] = None,
|
||||
query: str,
|
||||
project: Optional[str] = None,
|
||||
workspace: Optional[str] = None,
|
||||
page: int = 1,
|
||||
page_size: int = 10,
|
||||
search_type: str | None = None,
|
||||
search_type: str = "text",
|
||||
output_format: Literal["text", "json"] = "text",
|
||||
note_types: Annotated[
|
||||
List[str] | None,
|
||||
BeforeValidator(coerce_list),
|
||||
"Filter by the 'type' field in note frontmatter (e.g. 'note', 'chapter', 'person'). "
|
||||
"Case-insensitive.",
|
||||
] = None,
|
||||
entity_types: Annotated[
|
||||
List[str] | None,
|
||||
BeforeValidator(coerce_list),
|
||||
"Filter by knowledge graph item type: 'entity' (whole notes), 'observation', or "
|
||||
"'relation'. Defaults to 'entity'. Do NOT pass schema/frontmatter types like "
|
||||
"'Chapter' here — use note_types instead.",
|
||||
] = None,
|
||||
types: List[str] | None = None,
|
||||
entity_types: List[str] | None = None,
|
||||
after_date: Optional[str] = None,
|
||||
metadata_filters: Annotated[
|
||||
Dict[str, Any] | None,
|
||||
BeforeValidator(coerce_dict),
|
||||
] = None,
|
||||
tags: Annotated[
|
||||
List[str] | None,
|
||||
BeforeValidator(coerce_list),
|
||||
] = None,
|
||||
metadata_filters: Optional[Dict[str, Any]] = None,
|
||||
tags: Optional[List[str]] = None,
|
||||
status: Optional[str] = None,
|
||||
min_similarity: Optional[float] = None,
|
||||
context: Context | None = None,
|
||||
) -> dict | str:
|
||||
) -> SearchResponse | dict | str:
|
||||
"""Search across all content in the knowledge base with comprehensive syntax support.
|
||||
|
||||
This tool searches the knowledge base using full-text search, pattern matching,
|
||||
@@ -362,20 +292,15 @@ async def search_notes(
|
||||
- `search_notes("work-project", "category:observation")` - Filter by observation categories
|
||||
- `search_notes("team-docs", "author:username")` - Find content by author (if metadata available)
|
||||
|
||||
**Note:** `tag:` shorthand is automatically converted to a `tags` filter, so it works
|
||||
with any search type (text, hybrid, vector). You can also use the `tags` parameter
|
||||
directly: `search_notes("project", "query", tags=["my-tag"])`
|
||||
|
||||
### Search Type Examples
|
||||
- `search_notes("my-project", "Meeting", search_type="title")` - Search only in titles
|
||||
- `search_notes("work-docs", "docs/meeting-*", search_type="permalink")` - Pattern match permalinks
|
||||
Note: Permalink patterns match the full path (e.g., "project/folder/chapter-13*", not just "chapter-13*").
|
||||
- `search_notes("research", "keyword")` - Default search (hybrid when semantic is enabled,
|
||||
text when disabled)
|
||||
- `search_notes("research", "keyword", search_type="text")` - Text search (default; auto-upgrades
|
||||
to hybrid when semantic search is enabled)
|
||||
|
||||
### Filtering Options
|
||||
- `search_notes("my-project", "query", note_types=["note"])` - Search only notes
|
||||
- `search_notes("work-docs", "query", note_types=["note", "person"])` - Multiple note types
|
||||
- `search_notes("my-project", "query", types=["entity"])` - Search only entities
|
||||
- `search_notes("work-docs", "query", types=["note", "person"])` - Multiple content types
|
||||
- `search_notes("research", "query", entity_types=["observation"])` - Filter by entity type
|
||||
- `search_notes("team-docs", "query", after_date="2024-01-01")` - Recent content only
|
||||
- `search_notes("my-project", "query", after_date="1 week")` - Relative date filtering
|
||||
@@ -394,10 +319,8 @@ async def search_notes(
|
||||
- Nested keys use dot notation (e.g., `"schema.confidence"`).
|
||||
|
||||
### Filter-only Searches
|
||||
Omit `query` (or pass None) when only using structured filters:
|
||||
- `search_notes(metadata_filters={"type": "spec"}, project="my-project")`
|
||||
- `search_notes(tags=["security"], project="my-project")`
|
||||
- `search_notes(status="draft", project="my-project")`
|
||||
You can pass an empty query string when only using structured filters:
|
||||
- `search_notes("my-project", "", metadata_filters={"type": "spec"})`
|
||||
|
||||
### Convenience Filters
|
||||
`tags` and `status` are shorthand for metadata_filters. If the same key exists in
|
||||
@@ -410,18 +333,17 @@ async def search_notes(
|
||||
- `search_notes("archive", "docs/2024-*", search_type="permalink")` - Year-based permalink search
|
||||
|
||||
Args:
|
||||
query: Optional search query string (supports boolean operators, phrases, patterns).
|
||||
Omit or pass None for filter-only searches using metadata_filters, tags, or status.
|
||||
query: The search query string (supports boolean operators, phrases, patterns)
|
||||
project: Project name to search in. Optional - server will resolve using hierarchy.
|
||||
If unknown, use list_memory_projects() to discover available projects.
|
||||
page: The page number of results to return (default 1)
|
||||
page_size: The number of results to return per page (default 10)
|
||||
search_type: Type of search to perform, one of:
|
||||
"text", "title", "permalink", "vector", "semantic", "hybrid".
|
||||
Default is dynamic: "hybrid" when semantic search is enabled, otherwise "text".
|
||||
"text", "title", "permalink", "vector", "semantic", "hybrid" (default: "text";
|
||||
text mode auto-upgrades to hybrid when semantic search is enabled)
|
||||
output_format: "text" preserves existing structured search response behavior.
|
||||
"json" returns a machine-readable dictionary payload.
|
||||
note_types: Optional list of note types to search (e.g., ["note", "person"])
|
||||
types: Optional list of note types to search (e.g., ["note", "person"])
|
||||
entity_types: Optional list of entity types to filter by (e.g., ["entity", "observation"])
|
||||
after_date: Optional date filter for recent content (e.g., "1 week", "2d", "2024-01-01")
|
||||
metadata_filters: Optional structured frontmatter filters (e.g., {"status": "in-progress"})
|
||||
@@ -433,8 +355,7 @@ async def search_notes(
|
||||
context: Optional FastMCP context for performance caching.
|
||||
|
||||
Returns:
|
||||
Formatted markdown text (output_format="text"), dict (output_format="json"),
|
||||
or helpful error guidance string if search fails
|
||||
SearchResponse with results and pagination info, or helpful error guidance if search fails
|
||||
|
||||
Examples:
|
||||
# Basic text search
|
||||
@@ -456,10 +377,10 @@ async def search_notes(
|
||||
# Exact phrase search
|
||||
results = await search_notes("\"weekly standup meeting\"")
|
||||
|
||||
# Search with note type filter - type property in frontmatter
|
||||
# Search with type filter
|
||||
results = await search_notes(
|
||||
"meeting notes",
|
||||
note_types=["note"],
|
||||
types=["entity"],
|
||||
)
|
||||
|
||||
# Search with entity type filter
|
||||
@@ -489,7 +410,7 @@ async def search_notes(
|
||||
# Complex search with multiple filters
|
||||
results = await search_notes(
|
||||
"(bug OR issue) AND NOT resolved",
|
||||
note_types=["note"],
|
||||
types=["entity"],
|
||||
after_date="2024-01-01"
|
||||
)
|
||||
|
||||
@@ -497,97 +418,56 @@ async def search_notes(
|
||||
results = await search_notes("project planning", project="my-project")
|
||||
"""
|
||||
# Avoid mutable-default-argument footguns. Treat None as "no filter".
|
||||
# Lowercase note_types so "Chapter" matches the stored "chapter".
|
||||
note_types = [t.lower() for t in note_types] if note_types else []
|
||||
types = types or []
|
||||
entity_types = entity_types or []
|
||||
|
||||
# Parse tag:<value> shorthand at tool level so it works with all search modes.
|
||||
# Handles "tag:security", "tag:coffee tag:brewing", "tag:coffee AND tag:brewing".
|
||||
# Without this, hybrid/vector modes fail because they require non-empty text,
|
||||
# but the service-layer tag: parser clears the text after the mode is set.
|
||||
if query and "tag:" in query.lower():
|
||||
# Extract tag values, splitting comma-separated lists (e.g. "tag:coffee,brewing")
|
||||
raw_values = re.findall(r"tag:(\S+)", query, flags=re.IGNORECASE)
|
||||
tag_values = [v for raw in raw_values for v in raw.split(",") if v]
|
||||
if tag_values:
|
||||
# Merge with any explicitly provided tags
|
||||
tags = list(set((tags or []) + tag_values))
|
||||
# Remove tag: tokens and boolean connectors, keep remaining text as query
|
||||
remainder = re.sub(r"tag:\S+", "", query, flags=re.IGNORECASE)
|
||||
remainder = re.sub(r"\b(AND|OR|NOT)\b", "", remainder).strip()
|
||||
query = remainder or None
|
||||
|
||||
# Detect project from memory URL prefix before routing
|
||||
if project is None and query is not None:
|
||||
detected = detect_project_from_url_prefix(query, ConfigManager().config)
|
||||
if detected:
|
||||
project = detected
|
||||
|
||||
async with get_project_client(project, workspace, context) as (client, active_project):
|
||||
# Handle memory:// URLs by resolving to permalink search
|
||||
is_memory_url = False
|
||||
if query is not None:
|
||||
_, resolved_query, is_memory_url = await resolve_project_and_path(
|
||||
client, query, project, context
|
||||
)
|
||||
if is_memory_url:
|
||||
query = resolved_query
|
||||
effective_search_type = search_type or _default_search_type()
|
||||
_, resolved_query, is_memory_url = await resolve_project_and_path(
|
||||
client, query, project, context
|
||||
)
|
||||
if is_memory_url:
|
||||
effective_search_type = "permalink"
|
||||
query = resolved_query
|
||||
search_type = "permalink"
|
||||
|
||||
try:
|
||||
# Create a SearchQuery object based on the parameters
|
||||
search_query = SearchQuery()
|
||||
|
||||
# Only map search_type to query fields when there is an actual query string.
|
||||
# When query is None/empty, skip the search mode block — filters-only path.
|
||||
effective_query = (query or "").strip()
|
||||
if effective_query:
|
||||
valid_search_types = {
|
||||
"text",
|
||||
"title",
|
||||
"permalink",
|
||||
"vector",
|
||||
"semantic",
|
||||
"hybrid",
|
||||
}
|
||||
if effective_search_type == "text":
|
||||
search_query.text = effective_query
|
||||
search_query.retrieval_mode = SearchRetrievalMode.FTS
|
||||
elif effective_search_type in ("vector", "semantic"):
|
||||
search_query.text = effective_query
|
||||
search_query.retrieval_mode = SearchRetrievalMode.VECTOR
|
||||
elif effective_search_type == "hybrid":
|
||||
search_query.text = effective_query
|
||||
# Map search_type to the appropriate query field and retrieval mode
|
||||
valid_search_types = {"text", "title", "permalink", "vector", "semantic", "hybrid"}
|
||||
if search_type == "text":
|
||||
search_query.text = query
|
||||
# Upgrade to hybrid when semantic search is available —
|
||||
# combines FTS keyword matching with vector similarity for better results
|
||||
if _semantic_search_enabled_for_text_search():
|
||||
search_query.retrieval_mode = SearchRetrievalMode.HYBRID
|
||||
elif effective_search_type == "title":
|
||||
search_query.title = effective_query
|
||||
elif effective_search_type == "permalink" and "*" in effective_query:
|
||||
search_query.permalink_match = effective_query
|
||||
elif effective_search_type == "permalink":
|
||||
search_query.permalink = effective_query
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Invalid search_type '{effective_search_type}'. "
|
||||
f"Valid options: {', '.join(sorted(valid_search_types))}"
|
||||
)
|
||||
elif search_type in ("vector", "semantic"):
|
||||
search_query.text = query
|
||||
search_query.retrieval_mode = SearchRetrievalMode.VECTOR
|
||||
elif search_type == "hybrid":
|
||||
search_query.text = query
|
||||
search_query.retrieval_mode = SearchRetrievalMode.HYBRID
|
||||
elif search_type == "title":
|
||||
search_query.title = query
|
||||
elif search_type == "permalink" and "*" in query:
|
||||
search_query.permalink_match = query
|
||||
elif search_type == "permalink":
|
||||
search_query.permalink = query
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Invalid search_type '{search_type}'. "
|
||||
f"Valid options: {', '.join(sorted(valid_search_types))}"
|
||||
)
|
||||
|
||||
# Add optional filters if provided (empty lists are treated as no filter)
|
||||
if entity_types:
|
||||
search_query.entity_types = [SearchItemType(t) for t in entity_types]
|
||||
if note_types:
|
||||
search_query.note_types = note_types
|
||||
if types:
|
||||
search_query.types = types
|
||||
if after_date:
|
||||
search_query.after_date = after_date
|
||||
if metadata_filters:
|
||||
# Alias common column/model names to their frontmatter key equivalents.
|
||||
# Users often pass "note_type" (the entity model column) when the
|
||||
# frontmatter field is actually "type".
|
||||
_METADATA_KEY_ALIASES = {"note_type": "type"}
|
||||
metadata_filters = {
|
||||
_METADATA_KEY_ALIASES.get(k, k): v for k, v in metadata_filters.items()
|
||||
}
|
||||
search_query.metadata_filters = metadata_filters
|
||||
if tags:
|
||||
search_query.tags = tags
|
||||
@@ -596,22 +476,7 @@ async def search_notes(
|
||||
if min_similarity is not None:
|
||||
search_query.min_similarity = min_similarity
|
||||
|
||||
# Reject searches with no criteria at all
|
||||
if search_query.no_criteria():
|
||||
return (
|
||||
"# No Search Criteria\n\n"
|
||||
"Please provide at least one of: `query`, `metadata_filters`, "
|
||||
"`tags`, `status`, `note_types`, `entity_types`, or `after_date`."
|
||||
)
|
||||
|
||||
# Default to entity-level results to avoid returning individual
|
||||
# observations/relations as separate search results (see issue #31).
|
||||
# Applied after no_criteria() so that the implicit default doesn't
|
||||
# mask a truly empty search request.
|
||||
if not search_query.entity_types:
|
||||
search_query.entity_types = [SearchItemType("entity")]
|
||||
|
||||
logger.debug(f"Searching for {search_query} in project {active_project.name}")
|
||||
logger.info(f"Searching for {search_query} in project {active_project.name}")
|
||||
# Import here to avoid circular import (tools → clients → utils → tools)
|
||||
from basic_memory.mcp.clients import SearchClient
|
||||
|
||||
@@ -625,7 +490,7 @@ async def search_notes(
|
||||
|
||||
# Check if we got no results and provide helpful guidance
|
||||
if not result.results:
|
||||
logger.debug(
|
||||
logger.info(
|
||||
f"Search returned no results for query: {query} in project {active_project.name}"
|
||||
)
|
||||
# Don't treat this as an error, but the user might want guidance
|
||||
@@ -634,13 +499,88 @@ async def search_notes(
|
||||
if output_format == "json":
|
||||
return result.model_dump(mode="json", exclude_none=True)
|
||||
|
||||
return _format_search_markdown(result, active_project.name, query)
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Search failed for query '{query}': {e}, project: {active_project.name}")
|
||||
# Return formatted error message as string for better user experience
|
||||
return _format_search_error_response(active_project.name, str(e), query, search_type)
|
||||
|
||||
|
||||
@mcp.tool(
|
||||
description="Search entities by structured frontmatter metadata.",
|
||||
)
|
||||
async def search_by_metadata(
|
||||
filters: Dict[str, Any],
|
||||
project: Optional[str] = None,
|
||||
workspace: Optional[str] = None,
|
||||
limit: int = 20,
|
||||
offset: int = 0,
|
||||
context: Context | None = None,
|
||||
) -> SearchResponse | str:
|
||||
"""Search entities by structured frontmatter metadata.
|
||||
|
||||
Args:
|
||||
filters: Dictionary of metadata filters (e.g., {"status": "in-progress"})
|
||||
project: Project name to search in. Optional - server will resolve using hierarchy.
|
||||
limit: Maximum number of results to return
|
||||
offset: Number of results to skip (for pagination)
|
||||
context: Optional FastMCP context for performance caching.
|
||||
|
||||
Returns:
|
||||
SearchResponse with results, or helpful error guidance if search fails
|
||||
"""
|
||||
if limit <= 0:
|
||||
return "# Error\n\n`limit` must be greater than 0."
|
||||
|
||||
# Build a structured-only search query
|
||||
search_query = SearchQuery()
|
||||
search_query.metadata_filters = filters
|
||||
search_query.entity_types = [SearchItemType.ENTITY]
|
||||
|
||||
# Convert offset/limit to page/page_size (API uses paging)
|
||||
page_size = limit
|
||||
page = (offset // limit) + 1
|
||||
offset_within_page = offset % limit
|
||||
|
||||
async with get_project_client(project, workspace, context) as (client, active_project):
|
||||
logger.info(
|
||||
f"Structured search in project {active_project.name} filters={filters} limit={limit} offset={offset}"
|
||||
)
|
||||
|
||||
try:
|
||||
from basic_memory.mcp.clients import SearchClient
|
||||
|
||||
search_client = SearchClient(client, active_project.external_id)
|
||||
result = await search_client.search(
|
||||
search_query.model_dump(),
|
||||
page=page,
|
||||
page_size=page_size,
|
||||
)
|
||||
|
||||
# Apply offset within page, fetch next page if needed
|
||||
if offset_within_page:
|
||||
remaining = result.results[offset_within_page:]
|
||||
if len(remaining) < limit:
|
||||
next_page = page + 1
|
||||
extra = await search_client.search(
|
||||
search_query.model_dump(),
|
||||
page=next_page,
|
||||
page_size=page_size,
|
||||
)
|
||||
remaining.extend(extra.results[: max(0, limit - len(remaining))])
|
||||
result = SearchResponse(
|
||||
results=remaining[:limit],
|
||||
current_page=page,
|
||||
page_size=page_size,
|
||||
)
|
||||
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
f"Search failed for query '{query or ''}': {e}, project: {active_project.name}"
|
||||
f"Metadata search failed for filters '{filters}': {e}, project: {active_project.name}"
|
||||
)
|
||||
# Return formatted error message as string for better user experience
|
||||
return _format_search_error_response(
|
||||
active_project.name, str(e), query or "", effective_search_type
|
||||
active_project.name, str(e), str(filters), "metadata"
|
||||
)
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Annotated, Any, Dict, List, Optional
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from fastmcp import Context
|
||||
from mcp.types import ContentBlock, TextContent
|
||||
@@ -20,25 +20,15 @@ def _text_block(message: str) -> List[ContentBlock]:
|
||||
@mcp.tool(
|
||||
description="Search notes and return an embedded MCP-UI resource (raw HTML).",
|
||||
output_schema=None,
|
||||
annotations={"readOnlyHint": True, "openWorldHint": False},
|
||||
)
|
||||
async def search_notes_ui(
|
||||
query: str,
|
||||
project: Optional[str] = None,
|
||||
page: int = 1,
|
||||
page_size: int = 10,
|
||||
search_type: Optional[str] = None,
|
||||
note_types: Annotated[
|
||||
List[str] | None,
|
||||
"Filter by the 'type' field in note frontmatter (e.g. 'note', 'chapter', 'person'). "
|
||||
"Case-insensitive.",
|
||||
] = None,
|
||||
entity_types: Annotated[
|
||||
List[str] | None,
|
||||
"Filter by knowledge graph item type: 'entity' (whole notes), 'observation', or "
|
||||
"'relation'. Defaults to 'entity'. Do NOT pass schema/frontmatter types like "
|
||||
"'Chapter' here — use note_types instead.",
|
||||
] = None,
|
||||
search_type: str = "text",
|
||||
types: List[str] | None = None,
|
||||
entity_types: List[str] | None = None,
|
||||
after_date: Optional[str] = None,
|
||||
metadata_filters: Optional[Dict[str, Any]] = None,
|
||||
tags: Optional[List[str]] = None,
|
||||
@@ -46,14 +36,14 @@ async def search_notes_ui(
|
||||
context: Context | None = None,
|
||||
) -> List[ContentBlock]:
|
||||
"""Return a search results UI as an embedded MCP-UI resource."""
|
||||
result = await search_notes(
|
||||
result = await search_notes.fn(
|
||||
query=query,
|
||||
project=project,
|
||||
page=page,
|
||||
page_size=page_size,
|
||||
search_type=search_type,
|
||||
output_format="json",
|
||||
note_types=note_types,
|
||||
types=types,
|
||||
entity_types=entity_types,
|
||||
after_date=after_date,
|
||||
metadata_filters=metadata_filters,
|
||||
@@ -92,7 +82,6 @@ async def search_notes_ui(
|
||||
@mcp.tool(
|
||||
description="Read a note and return an embedded MCP-UI resource (raw HTML).",
|
||||
output_schema=None,
|
||||
annotations={"readOnlyHint": True, "openWorldHint": False},
|
||||
)
|
||||
async def read_note_ui(
|
||||
identifier: str,
|
||||
@@ -102,7 +91,7 @@ async def read_note_ui(
|
||||
context: Context | None = None,
|
||||
) -> List[ContentBlock]:
|
||||
"""Return a note preview UI as an embedded MCP-UI resource."""
|
||||
content = await read_note(
|
||||
content = await read_note.fn(
|
||||
identifier=identifier,
|
||||
project=project,
|
||||
page=page,
|
||||
|
||||
@@ -23,8 +23,6 @@ from httpx._types import (
|
||||
from loguru import logger
|
||||
from mcp.server.fastmcp.exceptions import ToolError
|
||||
|
||||
from basic_memory.config import ConfigManager
|
||||
|
||||
|
||||
def get_error_message(
|
||||
status_code: int, url: URL | str, method: str, msg: Optional[str] = None
|
||||
@@ -76,65 +74,6 @@ def get_error_message(
|
||||
return f"HTTP error {status_code}: {method} request to '{path}' failed"
|
||||
|
||||
|
||||
def _extract_response_data(response: Response) -> typing.Any:
|
||||
"""Safely decode response payload for error reporting."""
|
||||
try:
|
||||
return response.json()
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _response_detail_text(response_data: typing.Any) -> str | None:
|
||||
"""Extract textual error detail from API payloads."""
|
||||
if isinstance(response_data, dict):
|
||||
detail = response_data.get("detail")
|
||||
if isinstance(detail, str):
|
||||
return detail
|
||||
if isinstance(detail, dict):
|
||||
nested_message = detail.get("message")
|
||||
if isinstance(nested_message, str):
|
||||
return nested_message
|
||||
return str(detail)
|
||||
if detail is not None:
|
||||
return str(detail)
|
||||
return None
|
||||
|
||||
|
||||
def _has_configured_cloud_api_key() -> bool:
|
||||
"""Check whether a cloud API key is currently configured."""
|
||||
try:
|
||||
return bool(ConfigManager().config.cloud_api_key)
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def _resolve_error_message(
|
||||
status_code: int, url: URL | str, method: str, response_data: typing.Any
|
||||
) -> str:
|
||||
"""Resolve a user-facing error message with cloud auth remediation when relevant."""
|
||||
detail_text = _response_detail_text(response_data)
|
||||
|
||||
if status_code == 401 and _has_configured_cloud_api_key():
|
||||
detail_lower = detail_text.lower() if detail_text else ""
|
||||
if (
|
||||
"invalid jwt" in detail_lower
|
||||
or "invalid token" in detail_lower
|
||||
or "authentication required" in detail_lower
|
||||
or not detail_lower
|
||||
):
|
||||
return (
|
||||
"Authentication failed: the configured cloud API key was rejected by the server. "
|
||||
"Basic Memory prioritizes cloud_api_key over OAuth for cloud routing. "
|
||||
"Fix by running `bm cloud api-key save <valid-key>` "
|
||||
"or remove `cloud_api_key` and use `bm cloud login`."
|
||||
)
|
||||
|
||||
if detail_text:
|
||||
return detail_text
|
||||
|
||||
return get_error_message(status_code, url, method)
|
||||
|
||||
|
||||
async def call_get(
|
||||
client: AsyncClient,
|
||||
url: URL | str,
|
||||
@@ -186,8 +125,12 @@ async def call_get(
|
||||
|
||||
# Handle different status codes differently
|
||||
status_code = response.status_code
|
||||
response_data = _extract_response_data(response)
|
||||
error_message = _resolve_error_message(status_code, url, "GET", response_data)
|
||||
# get the message if available
|
||||
response_data = response.json()
|
||||
if isinstance(response_data, dict) and "detail" in response_data:
|
||||
error_message = response_data["detail"]
|
||||
else:
|
||||
error_message = get_error_message(status_code, url, "PUT")
|
||||
|
||||
# Log at appropriate level based on status code
|
||||
if 400 <= status_code < 500:
|
||||
@@ -272,8 +215,12 @@ async def call_put(
|
||||
# Handle different status codes differently
|
||||
status_code = response.status_code
|
||||
|
||||
response_data = _extract_response_data(response)
|
||||
error_message = _resolve_error_message(status_code, url, "PUT", response_data)
|
||||
# get the message if available
|
||||
response_data = response.json()
|
||||
if isinstance(response_data, dict) and "detail" in response_data:
|
||||
error_message = response_data["detail"] # pragma: no cover
|
||||
else:
|
||||
error_message = get_error_message(status_code, url, "PUT")
|
||||
|
||||
# Log at appropriate level based on status code
|
||||
if 400 <= status_code < 500:
|
||||
@@ -357,8 +304,15 @@ async def call_patch(
|
||||
# Handle different status codes differently
|
||||
status_code = response.status_code
|
||||
|
||||
response_data = _extract_response_data(response)
|
||||
error_message = _resolve_error_message(status_code, url, "PATCH", response_data)
|
||||
# Try to extract specific error message from response body
|
||||
try:
|
||||
response_data = response.json()
|
||||
if isinstance(response_data, dict) and "detail" in response_data:
|
||||
error_message = response_data["detail"]
|
||||
else:
|
||||
error_message = get_error_message(status_code, url, "PATCH") # pragma: no cover
|
||||
except Exception: # pragma: no cover
|
||||
error_message = get_error_message(status_code, url, "PATCH") # pragma: no cover
|
||||
|
||||
# Log at appropriate level based on status code
|
||||
if 400 <= status_code < 500:
|
||||
@@ -378,8 +332,15 @@ async def call_patch(
|
||||
except HTTPStatusError as e:
|
||||
status_code = e.response.status_code
|
||||
|
||||
response_data = _extract_response_data(e.response)
|
||||
error_message = _resolve_error_message(status_code, url, "PATCH", response_data)
|
||||
# Try to extract specific error message from response body
|
||||
try:
|
||||
response_data = e.response.json()
|
||||
if isinstance(response_data, dict) and "detail" in response_data:
|
||||
error_message = response_data["detail"]
|
||||
else:
|
||||
error_message = get_error_message(status_code, url, "PATCH") # pragma: no cover
|
||||
except Exception: # pragma: no cover
|
||||
error_message = get_error_message(status_code, url, "PATCH") # pragma: no cover
|
||||
|
||||
raise ToolError(error_message) from e
|
||||
|
||||
@@ -448,8 +409,12 @@ async def call_post(
|
||||
|
||||
# Handle different status codes differently
|
||||
status_code = response.status_code
|
||||
response_data = _extract_response_data(response)
|
||||
error_message = _resolve_error_message(status_code, url, "POST", response_data)
|
||||
# get the message if available
|
||||
response_data = response.json()
|
||||
if isinstance(response_data, dict) and "detail" in response_data:
|
||||
error_message = response_data["detail"]
|
||||
else:
|
||||
error_message = get_error_message(status_code, url, "POST")
|
||||
|
||||
# Log at appropriate level based on status code
|
||||
if 400 <= status_code < 500:
|
||||
@@ -553,8 +518,12 @@ async def call_delete(
|
||||
|
||||
# Handle different status codes differently
|
||||
status_code = response.status_code
|
||||
response_data = _extract_response_data(response)
|
||||
error_message = _resolve_error_message(status_code, url, "DELETE", response_data)
|
||||
# get the message if available
|
||||
response_data = response.json()
|
||||
if isinstance(response_data, dict) and "detail" in response_data:
|
||||
error_message = response_data["detail"] # pragma: no cover
|
||||
else:
|
||||
error_message = get_error_message(status_code, url, "DELETE")
|
||||
|
||||
# Log at appropriate level based on status code
|
||||
if 400 <= status_code < 500:
|
||||
|
||||
@@ -12,7 +12,6 @@ from basic_memory.mcp.tools.read_note import read_note
|
||||
|
||||
@mcp.tool(
|
||||
description="View a note as a formatted artifact for better readability.",
|
||||
annotations={"readOnlyHint": True, "openWorldHint": False},
|
||||
)
|
||||
async def view_note(
|
||||
identifier: str,
|
||||
@@ -58,16 +57,14 @@ async def view_note(
|
||||
"""
|
||||
logger.info(f"Viewing note: {identifier} in project: {project}")
|
||||
|
||||
# Call the existing read_note logic (default output_format="text" returns str)
|
||||
content = str(
|
||||
await read_note(
|
||||
identifier=identifier,
|
||||
project=project,
|
||||
workspace=workspace,
|
||||
page=page,
|
||||
page_size=page_size,
|
||||
context=context,
|
||||
)
|
||||
# Call the existing read_note logic
|
||||
content = await read_note.fn(
|
||||
identifier=identifier,
|
||||
project=project,
|
||||
workspace=workspace,
|
||||
page=page,
|
||||
page_size=page_size,
|
||||
context=context,
|
||||
)
|
||||
|
||||
# Check if this is an error message (note not found)
|
||||
|
||||
@@ -1,46 +1,16 @@
|
||||
"""Workspace discovery MCP tool."""
|
||||
|
||||
from typing import Literal
|
||||
|
||||
from fastmcp import Context
|
||||
|
||||
from basic_memory.mcp.project_context import get_available_workspaces
|
||||
from basic_memory.mcp.server import mcp
|
||||
|
||||
|
||||
@mcp.tool(
|
||||
description="List available cloud workspaces (tenant_id, type, role, and name).",
|
||||
annotations={"readOnlyHint": True, "openWorldHint": False},
|
||||
)
|
||||
async def list_workspaces(
|
||||
output_format: Literal["text", "json"] = "text",
|
||||
context: Context | None = None,
|
||||
) -> str | dict:
|
||||
"""List workspaces available to the current cloud user.
|
||||
|
||||
Args:
|
||||
output_format: "text" returns human-readable workspace list.
|
||||
"json" returns structured workspace metadata.
|
||||
context: Optional FastMCP context for progress/status logging.
|
||||
"""
|
||||
@mcp.tool(description="List available cloud workspaces (tenant_id, type, role, and name).")
|
||||
async def list_workspaces(context: Context | None = None) -> str:
|
||||
"""List workspaces available to the current cloud user."""
|
||||
workspaces = await get_available_workspaces(context=context)
|
||||
|
||||
if output_format == "json":
|
||||
return {
|
||||
"workspaces": [
|
||||
{
|
||||
"tenant_id": ws.tenant_id,
|
||||
"name": ws.name,
|
||||
"workspace_type": ws.workspace_type,
|
||||
"role": ws.role,
|
||||
"organization_id": ws.organization_id,
|
||||
"has_active_subscription": ws.has_active_subscription,
|
||||
}
|
||||
for ws in workspaces
|
||||
],
|
||||
"count": len(workspaces),
|
||||
}
|
||||
|
||||
if not workspaces:
|
||||
return (
|
||||
"# No Workspaces Available\n\n"
|
||||
|
||||
@@ -1,25 +1,21 @@
|
||||
"""Write note tool for Basic Memory MCP server."""
|
||||
|
||||
import textwrap
|
||||
from typing import Annotated, List, Union, Optional, Literal
|
||||
from typing import List, Union, Optional, Literal
|
||||
|
||||
from loguru import logger
|
||||
from pydantic import BeforeValidator
|
||||
|
||||
from basic_memory.config import ConfigManager
|
||||
from basic_memory.mcp.project_context import get_project_client, add_project_metadata
|
||||
from basic_memory.mcp.server import mcp
|
||||
from fastmcp import Context
|
||||
from basic_memory.schemas.base import Entity
|
||||
from basic_memory.utils import coerce_dict, parse_tags, validate_project_path
|
||||
from basic_memory.utils import parse_tags, validate_project_path
|
||||
|
||||
# Define TagType as a Union that can accept either a string or a list of strings or None
|
||||
TagType = Union[List[str], str, None]
|
||||
|
||||
|
||||
@mcp.tool(
|
||||
description="Create a markdown note. If the note already exists, returns an error by default — pass overwrite=True to replace.",
|
||||
annotations={"destructiveHint": True, "idempotentHint": False, "openWorldHint": False},
|
||||
description="Create or update a markdown note. Returns a markdown formatted summary of the semantic content.",
|
||||
)
|
||||
async def write_note(
|
||||
title: str,
|
||||
@@ -29,16 +25,13 @@ async def write_note(
|
||||
workspace: Optional[str] = None,
|
||||
tags: list[str] | str | None = None,
|
||||
note_type: str = "note",
|
||||
metadata: Annotated[dict | None, BeforeValidator(coerce_dict)] = None,
|
||||
overwrite: bool | None = None,
|
||||
metadata: dict | None = None,
|
||||
output_format: Literal["text", "json"] = "text",
|
||||
context: Context | None = None,
|
||||
) -> str | dict:
|
||||
"""Write a markdown note to the knowledge base.
|
||||
|
||||
Creates a markdown note with semantic observations and relations.
|
||||
If the note already exists, returns an error by default. Pass overwrite=True
|
||||
to replace the existing note. For incremental updates, use edit_note instead.
|
||||
Creates or updates a markdown note with semantic observations and relations.
|
||||
|
||||
Project Resolution:
|
||||
Server resolves projects using a unified priority chain (same in local and cloud modes):
|
||||
@@ -80,8 +73,6 @@ async def write_note(
|
||||
metadata: Optional dict of extra frontmatter fields merged into entity_metadata.
|
||||
Useful for schema notes or any note that needs custom YAML frontmatter
|
||||
beyond title/type/tags. Nested dicts are supported.
|
||||
overwrite: If True, replace existing note on conflict. If False, error on conflict.
|
||||
If None (default), consult write_note_overwrite_default config setting.
|
||||
output_format: "text" returns the existing markdown summary. "json" returns
|
||||
machine-readable metadata.
|
||||
context: Optional FastMCP context for performance caching.
|
||||
@@ -114,13 +105,12 @@ async def write_note(
|
||||
note_type="guide"
|
||||
)
|
||||
|
||||
# Overwrite an existing note explicitly
|
||||
# Update existing note (same title/directory)
|
||||
write_note(
|
||||
project="my-research",
|
||||
title="Meeting Notes",
|
||||
directory="meetings",
|
||||
content="# Weekly Standup\\n\\n- [decision] Use PostgreSQL instead #tech",
|
||||
overwrite=True
|
||||
content="# Weekly Standup\\n\\n- [decision] Use PostgreSQL instead #tech"
|
||||
)
|
||||
|
||||
# Create a schema note with custom frontmatter via metadata
|
||||
@@ -141,13 +131,6 @@ async def write_note(
|
||||
HTTPError: If project doesn't exist or is inaccessible
|
||||
SecurityError: If directory path attempts path traversal
|
||||
"""
|
||||
# Resolve overwrite flag: explicit parameter > config default
|
||||
# Trigger: caller omitted the parameter (None)
|
||||
# Why: lets users set a global default without breaking per-call overrides
|
||||
effective_overwrite = (
|
||||
overwrite if overwrite is not None else ConfigManager().config.write_note_overwrite_default
|
||||
)
|
||||
|
||||
async with get_project_client(project, workspace, context) as (client, active_project):
|
||||
logger.info(
|
||||
f"MCP tool call tool=write_note project={active_project.name} directory={directory}, title={title}, tags={tags}"
|
||||
@@ -190,7 +173,7 @@ async def write_note(
|
||||
entity = Entity(
|
||||
title=title,
|
||||
directory=directory,
|
||||
note_type=note_type,
|
||||
entity_type=note_type,
|
||||
content_type="text/markdown",
|
||||
content=content,
|
||||
entity_metadata=entity_metadata or None,
|
||||
@@ -215,23 +198,6 @@ async def write_note(
|
||||
or "conflict" in str(e).lower()
|
||||
or "already exists" in str(e).lower()
|
||||
):
|
||||
# Guard: block overwrite unless explicitly enabled
|
||||
if not effective_overwrite:
|
||||
logger.warning(
|
||||
f"write_note blocked: note already exists (overwrite not enabled) "
|
||||
f"permalink={entity.permalink}"
|
||||
)
|
||||
if output_format == "json":
|
||||
return {
|
||||
"title": title,
|
||||
"permalink": entity.permalink,
|
||||
"file_path": None,
|
||||
"checksum": None,
|
||||
"action": "conflict",
|
||||
"error": "NOTE_ALREADY_EXISTS",
|
||||
}
|
||||
return _format_overwrite_error(title, entity.permalink, active_project.name)
|
||||
|
||||
logger.debug(f"Entity exists, updating instead permalink={entity.permalink}")
|
||||
try:
|
||||
if not entity.permalink:
|
||||
@@ -303,23 +269,3 @@ async def write_note(
|
||||
|
||||
summary_result = "\n".join(summary)
|
||||
return add_project_metadata(summary_result, active_project.name)
|
||||
|
||||
|
||||
def _format_overwrite_error(title: str, permalink: str | None, project_name: str) -> str:
|
||||
"""Format a helpful error when write_note is blocked by the overwrite guard."""
|
||||
return textwrap.dedent(f"""\
|
||||
# Error: Note already exists
|
||||
|
||||
**"{title}"** already exists (permalink: `{permalink}`).
|
||||
|
||||
`write_note` does not overwrite by default. Choose an option:
|
||||
|
||||
| Goal | Action |
|
||||
|------|--------|
|
||||
| Append content | `edit_note("{permalink}", operation="append", content="...")` |
|
||||
| Prepend content | `edit_note("{permalink}", operation="prepend", content="...")` |
|
||||
| Replace a section | `edit_note("{permalink}", operation="replace_section", section="...", content="...")` |
|
||||
| Full replace | `write_note("{title}", ..., overwrite=True)` |
|
||||
| Inspect first | `read_note("{permalink}")` |
|
||||
|
||||
Project: {project_name}""")
|
||||
|
||||
@@ -37,7 +37,7 @@ class Entity(Base):
|
||||
__tablename__ = "entity"
|
||||
__table_args__ = (
|
||||
# Regular indexes
|
||||
Index("ix_note_type", "note_type"),
|
||||
Index("ix_entity_type", "entity_type"),
|
||||
Index("ix_entity_title", "title"),
|
||||
Index("ix_entity_external_id", "external_id", unique=True),
|
||||
Index("ix_entity_created_at", "created_at"), # For timeline queries
|
||||
@@ -64,7 +64,7 @@ class Entity(Base):
|
||||
# External UUID for API references - stable identifier that won't change
|
||||
external_id: Mapped[str] = mapped_column(String, unique=True, default=lambda: str(uuid.uuid4()))
|
||||
title: Mapped[str] = mapped_column(String)
|
||||
note_type: Mapped[str] = mapped_column(String)
|
||||
entity_type: Mapped[str] = mapped_column(String)
|
||||
entity_metadata: Mapped[Optional[dict]] = mapped_column(JSON, nullable=True)
|
||||
content_type: Mapped[str] = mapped_column(String)
|
||||
|
||||
@@ -94,11 +94,6 @@ class Entity(Base):
|
||||
onupdate=lambda: datetime.now().astimezone(),
|
||||
)
|
||||
|
||||
# Who created this entity (cloud user_profile_id UUID, null for local/CLI usage)
|
||||
created_by: Mapped[Optional[str]] = mapped_column(String, nullable=True, default=None)
|
||||
# Who last modified this entity (cloud user_profile_id UUID, null for local/CLI usage)
|
||||
last_updated_by: Mapped[Optional[str]] = mapped_column(String, nullable=True, default=None)
|
||||
|
||||
# Relationships
|
||||
project = relationship("Project", back_populates="entities")
|
||||
observations = relationship(
|
||||
@@ -138,7 +133,7 @@ class Entity(Base):
|
||||
return value
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return f"Entity(id={self.id}, external_id='{self.external_id}', name='{self.title}', type='{self.note_type}', checksum='{self.checksum}')"
|
||||
return f"Entity(id={self.id}, external_id='{self.external_id}', name='{self.title}', type='{self.entity_type}', checksum='{self.checksum}')"
|
||||
|
||||
|
||||
class Observation(Base):
|
||||
|
||||
@@ -93,27 +93,6 @@ CREATE VIRTUAL TABLE IF NOT EXISTS search_index USING fts5(
|
||||
);
|
||||
""")
|
||||
|
||||
# Postgres semantic chunk metadata table.
|
||||
# Matches the Alembic migration (h1b2c3d4e5f6) schema.
|
||||
# Used by tests to create the table without running full migrations.
|
||||
CREATE_POSTGRES_SEARCH_VECTOR_CHUNKS_TABLE = DDL("""
|
||||
CREATE TABLE IF NOT EXISTS search_vector_chunks (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
entity_id INTEGER NOT NULL,
|
||||
project_id INTEGER NOT NULL,
|
||||
chunk_key TEXT NOT NULL,
|
||||
chunk_text TEXT NOT NULL,
|
||||
source_hash TEXT NOT NULL,
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
UNIQUE (project_id, entity_id, chunk_key)
|
||||
)
|
||||
""")
|
||||
|
||||
CREATE_POSTGRES_SEARCH_VECTOR_CHUNKS_INDEX = DDL("""
|
||||
CREATE INDEX IF NOT EXISTS idx_search_vector_chunks_project_entity
|
||||
ON search_vector_chunks (project_id, entity_id)
|
||||
""")
|
||||
|
||||
# Local semantic chunk metadata table for SQLite.
|
||||
# Embedding vectors live in sqlite-vec virtual table keyed by this table rowid.
|
||||
CREATE_SQLITE_SEARCH_VECTOR_CHUNKS = DDL("""
|
||||
|
||||
@@ -1,34 +1,8 @@
|
||||
"""Factory for creating configured semantic embedding providers."""
|
||||
|
||||
from threading import Lock
|
||||
|
||||
from basic_memory.config import BasicMemoryConfig
|
||||
from basic_memory.repository.embedding_provider import EmbeddingProvider
|
||||
|
||||
type ProviderCacheKey = tuple[str, str, int | None, int, str | None, int | None, int | None]
|
||||
|
||||
_EMBEDDING_PROVIDER_CACHE: dict[ProviderCacheKey, EmbeddingProvider] = {}
|
||||
_EMBEDDING_PROVIDER_CACHE_LOCK = Lock()
|
||||
|
||||
|
||||
def _provider_cache_key(app_config: BasicMemoryConfig) -> ProviderCacheKey:
|
||||
"""Build a stable cache key from provider-relevant semantic embedding config."""
|
||||
return (
|
||||
app_config.semantic_embedding_provider.strip().lower(),
|
||||
app_config.semantic_embedding_model,
|
||||
app_config.semantic_embedding_dimensions,
|
||||
app_config.semantic_embedding_batch_size,
|
||||
app_config.semantic_embedding_cache_dir,
|
||||
app_config.semantic_embedding_threads,
|
||||
app_config.semantic_embedding_parallel,
|
||||
)
|
||||
|
||||
|
||||
def reset_embedding_provider_cache() -> None:
|
||||
"""Clear process-level embedding provider cache (used by tests)."""
|
||||
with _EMBEDDING_PROVIDER_CACHE_LOCK:
|
||||
_EMBEDDING_PROVIDER_CACHE.clear()
|
||||
|
||||
|
||||
def create_embedding_provider(app_config: BasicMemoryConfig) -> EmbeddingProvider:
|
||||
"""Create an embedding provider based on semantic config.
|
||||
@@ -36,50 +10,32 @@ def create_embedding_provider(app_config: BasicMemoryConfig) -> EmbeddingProvide
|
||||
When semantic_embedding_dimensions is set in config, it overrides
|
||||
the provider's default dimensions (384 for FastEmbed, 1536 for OpenAI).
|
||||
"""
|
||||
cache_key = _provider_cache_key(app_config)
|
||||
with _EMBEDDING_PROVIDER_CACHE_LOCK:
|
||||
if cached_provider := _EMBEDDING_PROVIDER_CACHE.get(cache_key):
|
||||
return cached_provider
|
||||
|
||||
provider_name = app_config.semantic_embedding_provider.strip().lower()
|
||||
extra_kwargs: dict = {}
|
||||
if app_config.semantic_embedding_dimensions is not None:
|
||||
extra_kwargs["dimensions"] = app_config.semantic_embedding_dimensions
|
||||
|
||||
provider: EmbeddingProvider
|
||||
if provider_name == "fastembed":
|
||||
# Deferred import: fastembed (and its onnxruntime dep) may not be installed
|
||||
from basic_memory.repository.fastembed_provider import FastEmbedEmbeddingProvider
|
||||
|
||||
if app_config.semantic_embedding_cache_dir is not None:
|
||||
extra_kwargs["cache_dir"] = app_config.semantic_embedding_cache_dir
|
||||
if app_config.semantic_embedding_threads is not None:
|
||||
extra_kwargs["threads"] = app_config.semantic_embedding_threads
|
||||
if app_config.semantic_embedding_parallel is not None:
|
||||
extra_kwargs["parallel"] = app_config.semantic_embedding_parallel
|
||||
|
||||
provider = FastEmbedEmbeddingProvider(
|
||||
return FastEmbedEmbeddingProvider(
|
||||
model_name=app_config.semantic_embedding_model,
|
||||
batch_size=app_config.semantic_embedding_batch_size,
|
||||
**extra_kwargs,
|
||||
)
|
||||
elif provider_name == "openai":
|
||||
|
||||
if provider_name == "openai":
|
||||
# Deferred import: openai may not be installed
|
||||
from basic_memory.repository.openai_provider import OpenAIEmbeddingProvider
|
||||
|
||||
model_name = app_config.semantic_embedding_model or "text-embedding-3-small"
|
||||
if model_name == "bge-small-en-v1.5":
|
||||
model_name = "text-embedding-3-small"
|
||||
provider = OpenAIEmbeddingProvider(
|
||||
return OpenAIEmbeddingProvider(
|
||||
model_name=model_name,
|
||||
batch_size=app_config.semantic_embedding_batch_size,
|
||||
**extra_kwargs,
|
||||
)
|
||||
else:
|
||||
raise ValueError(f"Unsupported semantic embedding provider: {provider_name}")
|
||||
|
||||
with _EMBEDDING_PROVIDER_CACHE_LOCK:
|
||||
if cached_provider := _EMBEDDING_PROVIDER_CACHE.get(cache_key):
|
||||
return cached_provider
|
||||
_EMBEDDING_PROVIDER_CACHE[cache_key] = provider
|
||||
return provider
|
||||
raise ValueError(f"Unsupported semantic embedding provider: {provider_name}")
|
||||
|
||||
@@ -5,13 +5,11 @@ from __future__ import annotations
|
||||
import asyncio
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from loguru import logger
|
||||
|
||||
from basic_memory.repository.embedding_provider import EmbeddingProvider
|
||||
from basic_memory.repository.semantic_errors import SemanticDependenciesMissingError
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from fastembed import TextEmbedding # type: ignore[import-not-found] # pragma: no cover
|
||||
from fastembed import TextEmbedding # pragma: no cover
|
||||
|
||||
|
||||
class FastEmbedEmbeddingProvider(EmbeddingProvider):
|
||||
@@ -21,25 +19,16 @@ class FastEmbedEmbeddingProvider(EmbeddingProvider):
|
||||
"bge-small-en-v1.5": "BAAI/bge-small-en-v1.5",
|
||||
}
|
||||
|
||||
def _effective_parallel(self) -> int | None:
|
||||
return self.parallel if self.parallel is not None and self.parallel > 1 else None
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
model_name: str = "bge-small-en-v1.5",
|
||||
*,
|
||||
batch_size: int = 64,
|
||||
dimensions: int = 384,
|
||||
cache_dir: str | None = None,
|
||||
threads: int | None = None,
|
||||
parallel: int | None = None,
|
||||
) -> None:
|
||||
self.model_name = model_name
|
||||
self.dimensions = dimensions
|
||||
self.batch_size = batch_size
|
||||
self.cache_dir = cache_dir
|
||||
self.threads = threads
|
||||
self.parallel = parallel
|
||||
self._model: TextEmbedding | None = None
|
||||
self._model_lock = asyncio.Lock()
|
||||
|
||||
@@ -53,39 +42,18 @@ class FastEmbedEmbeddingProvider(EmbeddingProvider):
|
||||
|
||||
def _create_model() -> "TextEmbedding":
|
||||
try:
|
||||
from fastembed import TextEmbedding # type: ignore[import-not-found]
|
||||
from fastembed import TextEmbedding
|
||||
except (
|
||||
ImportError
|
||||
) as exc: # pragma: no cover - exercised via tests with monkeypatch
|
||||
raise SemanticDependenciesMissingError(
|
||||
"fastembed package is missing. "
|
||||
"Install/update basic-memory to include semantic dependencies: "
|
||||
"pip install -U basic-memory"
|
||||
"Install semantic extras: pip install 'basic-memory[semantic]'"
|
||||
) from exc
|
||||
resolved_model_name = self._MODEL_ALIASES.get(self.model_name, self.model_name)
|
||||
if self.cache_dir is not None and self.threads is not None:
|
||||
return TextEmbedding(
|
||||
model_name=resolved_model_name,
|
||||
cache_dir=self.cache_dir,
|
||||
threads=self.threads,
|
||||
)
|
||||
if self.cache_dir is not None:
|
||||
return TextEmbedding(model_name=resolved_model_name, cache_dir=self.cache_dir)
|
||||
if self.threads is not None:
|
||||
return TextEmbedding(model_name=resolved_model_name, threads=self.threads)
|
||||
return TextEmbedding(model_name=resolved_model_name)
|
||||
|
||||
self._model = await asyncio.to_thread(_create_model)
|
||||
logger.info(
|
||||
"FastEmbed model loaded: model_name={model_name} batch_size={batch_size} "
|
||||
"threads={threads} configured_parallel={configured_parallel} "
|
||||
"effective_parallel={effective_parallel}",
|
||||
model_name=self._MODEL_ALIASES.get(self.model_name, self.model_name),
|
||||
batch_size=self.batch_size,
|
||||
threads=self.threads,
|
||||
configured_parallel=self.parallel,
|
||||
effective_parallel=self._effective_parallel(),
|
||||
)
|
||||
return self._model
|
||||
|
||||
async def embed_documents(self, texts: list[str]) -> list[list[float]]:
|
||||
@@ -93,23 +61,9 @@ class FastEmbedEmbeddingProvider(EmbeddingProvider):
|
||||
return []
|
||||
|
||||
model = await self._load_model()
|
||||
effective_parallel = self._effective_parallel()
|
||||
logger.debug(
|
||||
"FastEmbed embed_documents call: text_count={text_count} batch_size={batch_size} "
|
||||
"threads={threads} configured_parallel={configured_parallel} "
|
||||
"effective_parallel={effective_parallel}",
|
||||
text_count=len(texts),
|
||||
batch_size=self.batch_size,
|
||||
threads=self.threads,
|
||||
configured_parallel=self.parallel,
|
||||
effective_parallel=effective_parallel,
|
||||
)
|
||||
|
||||
def _embed_batch() -> list[list[float]]:
|
||||
embed_kwargs: dict[str, int] = {"batch_size": self.batch_size}
|
||||
if effective_parallel is not None:
|
||||
embed_kwargs["parallel"] = effective_parallel
|
||||
vectors = list(model.embed(texts, **embed_kwargs))
|
||||
vectors = list(model.embed(texts, batch_size=self.batch_size))
|
||||
normalized: list[list[float]] = []
|
||||
for vector in vectors:
|
||||
values = vector.tolist() if hasattr(vector, "tolist") else vector
|
||||
|
||||
@@ -2,10 +2,9 @@
|
||||
|
||||
from typing import Dict, List, Sequence
|
||||
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import async_sessionmaker
|
||||
from sqlalchemy.orm import selectinload
|
||||
from sqlalchemy.orm.interfaces import LoaderOption
|
||||
|
||||
from basic_memory.models import Observation
|
||||
from basic_memory.repository.repository import Repository
|
||||
@@ -23,10 +22,6 @@ class ObservationRepository(Repository[Observation]):
|
||||
"""
|
||||
super().__init__(session_maker, Observation, project_id=project_id)
|
||||
|
||||
def get_load_options(self) -> List[LoaderOption]:
|
||||
"""Eager-load parent entity to prevent N+1 if obs.entity is accessed."""
|
||||
return [selectinload(Observation.entity)]
|
||||
|
||||
async def find_by_entity(self, entity_id: int) -> Sequence[Observation]:
|
||||
"""Find all observations for a specific entity."""
|
||||
query = select(Observation).filter(Observation.entity_id == entity_id)
|
||||
|
||||
@@ -41,12 +41,11 @@ class OpenAIEmbeddingProvider(EmbeddingProvider):
|
||||
return self._client
|
||||
|
||||
try:
|
||||
from openai import AsyncOpenAI # type: ignore[import-not-found]
|
||||
from openai import AsyncOpenAI
|
||||
except ImportError as exc: # pragma: no cover - covered via monkeypatch tests
|
||||
raise SemanticDependenciesMissingError(
|
||||
"OpenAI dependency is missing. "
|
||||
"Install/update basic-memory to include semantic dependencies: "
|
||||
"pip install -U basic-memory"
|
||||
"Install semantic extras: pip install 'basic-memory[semantic]'"
|
||||
) from exc
|
||||
|
||||
api_key = self._api_key or os.getenv("OPENAI_API_KEY")
|
||||
|
||||
@@ -16,20 +16,14 @@ from basic_memory.repository.embedding_provider import EmbeddingProvider
|
||||
from basic_memory.repository.embedding_provider_factory import create_embedding_provider
|
||||
from basic_memory.repository.search_index_row import SearchIndexRow
|
||||
from basic_memory.repository.search_repository_base import SearchRepositoryBase
|
||||
from basic_memory.repository.metadata_filters import parse_metadata_filters
|
||||
from basic_memory.repository.metadata_filters import (
|
||||
parse_metadata_filters,
|
||||
build_postgres_json_path,
|
||||
)
|
||||
from basic_memory.repository.semantic_errors import SemanticDependenciesMissingError
|
||||
from basic_memory.schemas.search import SearchItemType, SearchRetrievalMode
|
||||
|
||||
|
||||
def _strip_nul_from_row(row_data: dict) -> dict:
|
||||
"""Strip NUL bytes from all string values in a row dict.
|
||||
|
||||
Secondary defense: PostgreSQL text columns cannot store \\x00.
|
||||
Primary sanitization happens in SearchService.index_entity_markdown().
|
||||
"""
|
||||
return {k: v.replace("\x00", "") if isinstance(v, str) else v for k, v in row_data.items()}
|
||||
|
||||
|
||||
class PostgresSearchRepository(SearchRepositoryBase):
|
||||
"""PostgreSQL tsvector implementation of search repository.
|
||||
|
||||
@@ -58,9 +52,6 @@ class PostgresSearchRepository(SearchRepositoryBase):
|
||||
self._semantic_enabled = self._app_config.semantic_search_enabled
|
||||
self._semantic_vector_k = self._app_config.semantic_vector_k
|
||||
self._semantic_min_similarity = self._app_config.semantic_min_similarity
|
||||
self._semantic_embedding_sync_batch_size = (
|
||||
self._app_config.semantic_embedding_sync_batch_size
|
||||
)
|
||||
self._embedding_provider = embedding_provider
|
||||
self._vector_dimensions = 384
|
||||
self._vector_tables_initialized = False
|
||||
@@ -101,7 +92,6 @@ class PostgresSearchRepository(SearchRepositoryBase):
|
||||
# Serialize JSON for raw SQL
|
||||
insert_data = search_index_row.to_insert(serialize_json=True)
|
||||
insert_data["project_id"] = self.project_id
|
||||
insert_data = _strip_nul_from_row(insert_data)
|
||||
|
||||
# Use upsert to handle race conditions during parallel indexing
|
||||
# ON CONFLICT (permalink, project_id) matches the partial unique index
|
||||
@@ -270,7 +260,7 @@ class PostgresSearchRepository(SearchRepositoryBase):
|
||||
if self._vector_tables_initialized:
|
||||
return
|
||||
|
||||
logger.debug("Ensuring Postgres vector tables exist for semantic search")
|
||||
logger.info("Ensuring Postgres vector tables exist for semantic search")
|
||||
|
||||
async with self._vector_tables_lock:
|
||||
if self._vector_tables_initialized:
|
||||
@@ -361,7 +351,7 @@ class PostgresSearchRepository(SearchRepositoryBase):
|
||||
)
|
||||
await session.commit()
|
||||
|
||||
logger.debug(f"Postgres vector tables ready (dimensions={self._vector_dimensions})")
|
||||
logger.info(f"Postgres vector tables ready (dimensions={self._vector_dimensions})")
|
||||
self._vector_tables_initialized = True
|
||||
|
||||
async def _get_existing_embedding_dims(self, session: AsyncSession) -> int | None:
|
||||
@@ -424,7 +414,7 @@ class PostgresSearchRepository(SearchRepositoryBase):
|
||||
ORDER BY e.embedding <=> CAST(:query_embedding AS vector)
|
||||
LIMIT :vector_k
|
||||
)
|
||||
SELECT c.entity_id, c.chunk_key, c.chunk_text, vector_matches.distance AS best_distance
|
||||
SELECT c.entity_id, c.chunk_key, vector_matches.distance AS best_distance
|
||||
FROM vector_matches
|
||||
JOIN search_vector_chunks c ON c.id = vector_matches.chunk_id
|
||||
WHERE c.project_id = :project_id
|
||||
@@ -509,14 +499,6 @@ class PostgresSearchRepository(SearchRepositoryBase):
|
||||
async def _update_timestamp_sql(self) -> str:
|
||||
return "NOW()" # pragma: no cover
|
||||
|
||||
def _distance_to_similarity(self, distance: float) -> float:
|
||||
"""Convert pgvector cosine distance to cosine similarity.
|
||||
|
||||
pgvector's <=> operator returns cosine distance in [0, 2],
|
||||
where cos_distance = 1 - cos_similarity.
|
||||
"""
|
||||
return max(0.0, 1.0 - distance)
|
||||
|
||||
def _timestamp_now_expr(self) -> str:
|
||||
return "NOW()"
|
||||
|
||||
@@ -551,7 +533,7 @@ class PostgresSearchRepository(SearchRepositoryBase):
|
||||
for row in search_index_rows:
|
||||
insert_data = row.to_insert(serialize_json=True)
|
||||
insert_data["project_id"] = self.project_id
|
||||
insert_data_list.append(_strip_nul_from_row(insert_data))
|
||||
insert_data_list.append(insert_data)
|
||||
|
||||
# Use upsert to handle race conditions during parallel indexing
|
||||
# ON CONFLICT (permalink, project_id) matches the partial unique index
|
||||
@@ -603,7 +585,7 @@ class PostgresSearchRepository(SearchRepositoryBase):
|
||||
permalink: Optional[str] = None,
|
||||
permalink_match: Optional[str] = None,
|
||||
title: Optional[str] = None,
|
||||
note_types: Optional[List[str]] = None,
|
||||
types: Optional[List[str]] = None,
|
||||
after_date: Optional[datetime] = None,
|
||||
search_item_types: Optional[List[SearchItemType]] = None,
|
||||
metadata_filters: Optional[dict] = None,
|
||||
@@ -619,7 +601,7 @@ class PostgresSearchRepository(SearchRepositoryBase):
|
||||
permalink=permalink,
|
||||
permalink_match=permalink_match,
|
||||
title=title,
|
||||
note_types=note_types,
|
||||
types=types,
|
||||
after_date=after_date,
|
||||
search_item_types=search_item_types,
|
||||
metadata_filters=metadata_filters,
|
||||
@@ -677,22 +659,20 @@ class PostgresSearchRepository(SearchRepositoryBase):
|
||||
else:
|
||||
conditions.append("search_index.permalink = :permalink")
|
||||
|
||||
# Handle search item type filter (parameterized for defense-in-depth)
|
||||
# Handle search item type filter
|
||||
if search_item_types:
|
||||
type_placeholders = []
|
||||
for idx, t in enumerate(search_item_types):
|
||||
param_name = f"search_type_{idx}"
|
||||
params[param_name] = t.value
|
||||
type_placeholders.append(f":{param_name}")
|
||||
conditions.append(f"search_index.type IN ({', '.join(type_placeholders)})")
|
||||
type_list = ", ".join(f"'{t.value}'" for t in search_item_types)
|
||||
conditions.append(f"search_index.type IN ({type_list})")
|
||||
|
||||
# Handle note type filter using JSONB containment (parameterized)
|
||||
if note_types:
|
||||
# Handle entity type filter using JSONB containment
|
||||
if types:
|
||||
# Use JSONB @> operator for efficient containment queries
|
||||
type_conditions = []
|
||||
for idx, note_type in enumerate(note_types):
|
||||
param_name = f"note_type_{idx}"
|
||||
params[param_name] = json.dumps({"note_type": note_type})
|
||||
type_conditions.append(f"search_index.metadata @> CAST(:{param_name} AS jsonb)")
|
||||
for entity_type in types:
|
||||
# Create JSONB containment condition for each type
|
||||
type_conditions.append(
|
||||
f'search_index.metadata @> \'{{"entity_type": "{entity_type}"}}\''
|
||||
)
|
||||
conditions.append(f"({' OR '.join(type_conditions)})")
|
||||
|
||||
# Handle date filter
|
||||
@@ -703,23 +683,15 @@ class PostgresSearchRepository(SearchRepositoryBase):
|
||||
order_by_clause = ", search_index.updated_at DESC"
|
||||
|
||||
# Handle structured metadata filters (frontmatter)
|
||||
# Uses jsonb_extract_path_text() / jsonb_extract_path() with parameterized
|
||||
# path parts instead of #>> / #> with interpolated paths.
|
||||
if metadata_filters:
|
||||
parsed_filters = parse_metadata_filters(metadata_filters)
|
||||
from_clause = "search_index JOIN entity ON search_index.entity_id = entity.id"
|
||||
metadata_expr = "entity.entity_metadata::jsonb"
|
||||
|
||||
for idx, filt in enumerate(parsed_filters):
|
||||
# Parameterize each JSON path part individually
|
||||
path_param_names = []
|
||||
for j, part in enumerate(filt.path_parts):
|
||||
path_param = f"meta_path_{idx}_{j}"
|
||||
params[path_param] = part
|
||||
path_param_names.append(f":{path_param}")
|
||||
path_args = ", ".join(path_param_names)
|
||||
text_expr = f"jsonb_extract_path_text({metadata_expr}, {path_args})"
|
||||
json_expr = f"jsonb_extract_path({metadata_expr}, {path_args})"
|
||||
path = build_postgres_json_path(filt.path_parts)
|
||||
text_expr = f"({metadata_expr} #>> '{path}')"
|
||||
json_expr = f"({metadata_expr} #> '{path}')"
|
||||
|
||||
if filt.op == "eq":
|
||||
value_param = f"meta_val_{idx}"
|
||||
@@ -737,12 +709,14 @@ class PostgresSearchRepository(SearchRepositoryBase):
|
||||
continue
|
||||
|
||||
if filt.op == "contains":
|
||||
import json as _json
|
||||
|
||||
base_param = f"meta_val_{idx}"
|
||||
tag_conditions = []
|
||||
# Require all values to be present
|
||||
for j, val in enumerate(filt.value):
|
||||
tag_param = f"{base_param}_{j}"
|
||||
params[tag_param] = json.dumps([val])
|
||||
params[tag_param] = _json.dumps([val])
|
||||
like_param = f"{base_param}_{j}_like"
|
||||
params[like_param] = f'%"{val}"%'
|
||||
like_param_single = f"{base_param}_{j}_like_single"
|
||||
@@ -757,7 +731,7 @@ class PostgresSearchRepository(SearchRepositoryBase):
|
||||
|
||||
if filt.op in {"gt", "gte", "lt", "lte", "between"}:
|
||||
compare_expr = (
|
||||
f"{text_expr}::double precision"
|
||||
f"({metadata_expr} #>> '{path}')::double precision"
|
||||
if filt.comparison == "numeric"
|
||||
else text_expr
|
||||
)
|
||||
|
||||
@@ -38,10 +38,7 @@ class SearchIndexRow:
|
||||
to_id: Optional[int] = None # relations
|
||||
relation_type: Optional[str] = None # relations
|
||||
|
||||
# Matched chunk text from vector search (the actual content that matched the query)
|
||||
matched_chunk_text: Optional[str] = None
|
||||
|
||||
CONTENT_DISPLAY_LIMIT = 4000
|
||||
CONTENT_DISPLAY_LIMIT = 250
|
||||
|
||||
@property
|
||||
def content(self):
|
||||
|
||||
@@ -7,7 +7,7 @@ The actual repository implementations are backend-specific:
|
||||
"""
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Any, Callable, List, Optional, Protocol
|
||||
from typing import List, Optional, Protocol
|
||||
|
||||
from sqlalchemy import Result
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
@@ -15,7 +15,6 @@ from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
from basic_memory.config import BasicMemoryConfig, ConfigManager, DatabaseBackend
|
||||
from basic_memory.repository.postgres_search_repository import PostgresSearchRepository
|
||||
from basic_memory.repository.search_index_row import SearchIndexRow
|
||||
from basic_memory.repository.search_repository_base import VectorSyncBatchResult
|
||||
from basic_memory.repository.sqlite_search_repository import SQLiteSearchRepository
|
||||
from basic_memory.schemas.search import SearchItemType, SearchRetrievalMode
|
||||
|
||||
@@ -38,7 +37,7 @@ class SearchRepository(Protocol):
|
||||
permalink: Optional[str] = None,
|
||||
permalink_match: Optional[str] = None,
|
||||
title: Optional[str] = None,
|
||||
note_types: Optional[List[str]] = None,
|
||||
types: Optional[List[str]] = None,
|
||||
after_date: Optional[datetime] = None,
|
||||
search_item_types: Optional[List[SearchItemType]] = None,
|
||||
metadata_filters: Optional[dict] = None,
|
||||
@@ -70,14 +69,6 @@ class SearchRepository(Protocol):
|
||||
"""Sync semantic vector chunks for an entity."""
|
||||
...
|
||||
|
||||
async def sync_entity_vectors_batch(
|
||||
self,
|
||||
entity_ids: list[int],
|
||||
progress_callback: Optional[Callable[[int, int, int], Any]] = None,
|
||||
) -> VectorSyncBatchResult:
|
||||
"""Sync semantic vector chunks for a batch of entities."""
|
||||
...
|
||||
|
||||
async def execute_query(self, query, params: dict) -> Result:
|
||||
"""Execute a raw SQL query."""
|
||||
...
|
||||
|
||||
@@ -5,9 +5,9 @@ import json
|
||||
import re
|
||||
import time
|
||||
from abc import ABC, abstractmethod
|
||||
from dataclasses import dataclass, field, replace
|
||||
from dataclasses import replace
|
||||
from datetime import datetime
|
||||
from typing import Any, Callable, Dict, List, Optional
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from loguru import logger
|
||||
from sqlalchemy import Executable, Result, text
|
||||
@@ -25,67 +25,20 @@ from basic_memory.schemas.search import SearchItemType, SearchRetrievalMode
|
||||
# --- Semantic search constants ---
|
||||
|
||||
VECTOR_FILTER_SCAN_LIMIT = 50000
|
||||
FUSION_BONUS = 0.3
|
||||
FTS_GATE_THRESHOLD = 0.0
|
||||
RRF_K = 60
|
||||
MAX_VECTOR_CHUNK_CHARS = 900
|
||||
VECTOR_CHUNK_OVERLAP_CHARS = 120
|
||||
TOP_CHUNKS_PER_RESULT = 5
|
||||
SMALL_NOTE_CONTENT_LIMIT = 2000
|
||||
HEADER_LINE_PATTERN = re.compile(r"^\s*#{1,6}\s+")
|
||||
BULLET_PATTERN = re.compile(r"^[\-\*]\s+")
|
||||
|
||||
|
||||
@dataclass
|
||||
class VectorSyncBatchResult:
|
||||
"""Aggregate result for batched semantic vector sync runs."""
|
||||
|
||||
entities_total: int
|
||||
entities_synced: int
|
||||
entities_failed: int
|
||||
failed_entity_ids: list[int] = field(default_factory=list)
|
||||
embedding_jobs_total: int = 0
|
||||
embed_seconds_total: float = 0.0
|
||||
write_seconds_total: float = 0.0
|
||||
|
||||
|
||||
@dataclass
|
||||
class _PreparedEntityVectorSync:
|
||||
"""Prepared chunk mutations + embedding jobs for one entity."""
|
||||
|
||||
entity_id: int
|
||||
sync_start: float
|
||||
source_rows_count: int
|
||||
embedding_jobs: list[tuple[int, str]]
|
||||
|
||||
|
||||
@dataclass
|
||||
class _PendingEmbeddingJob:
|
||||
"""Pending embedding write entry with entity ownership metadata."""
|
||||
|
||||
entity_id: int
|
||||
chunk_row_id: int
|
||||
chunk_text: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class _EntitySyncRuntime:
|
||||
"""Per-entity runtime counters used while flushes are in flight."""
|
||||
|
||||
sync_start: float
|
||||
source_rows_count: int
|
||||
embedding_jobs_count: int
|
||||
remaining_jobs: int
|
||||
embed_seconds: float = 0.0
|
||||
write_seconds: float = 0.0
|
||||
|
||||
|
||||
class SearchRepositoryBase(ABC):
|
||||
"""Abstract base class for backend-specific search repository implementations.
|
||||
|
||||
This class defines the common interface that all search repositories must implement,
|
||||
regardless of whether they use SQLite FTS5 or Postgres tsvector for full-text search.
|
||||
|
||||
Shared semantic search logic (chunking, embedding orchestration, hybrid score-based fusion)
|
||||
Shared semantic search logic (chunking, embedding orchestration, hybrid RRF fusion)
|
||||
lives here. Backend-specific operations are delegated to abstract hooks.
|
||||
|
||||
Concrete implementations:
|
||||
@@ -98,7 +51,6 @@ class SearchRepositoryBase(ABC):
|
||||
_semantic_vector_k: int
|
||||
_semantic_min_similarity: float
|
||||
_embedding_provider: Optional[EmbeddingProvider]
|
||||
_semantic_embedding_sync_batch_size: int
|
||||
_vector_dimensions: int
|
||||
_vector_tables_initialized: bool
|
||||
|
||||
@@ -156,7 +108,7 @@ class SearchRepositoryBase(ABC):
|
||||
permalink: Optional[str] = None,
|
||||
permalink_match: Optional[str] = None,
|
||||
title: Optional[str] = None,
|
||||
note_types: Optional[List[str]] = None,
|
||||
types: Optional[List[str]] = None,
|
||||
after_date: Optional[datetime] = None,
|
||||
search_item_types: Optional[List[SearchItemType]] = None,
|
||||
metadata_filters: Optional[Dict[str, Any]] = None,
|
||||
@@ -172,7 +124,7 @@ class SearchRepositoryBase(ABC):
|
||||
permalink: Exact permalink match
|
||||
permalink_match: Permalink pattern match (supports *)
|
||||
title: Title search
|
||||
note_types: Filter by note types (from metadata.note_type)
|
||||
types: Filter by entity types (from metadata.entity_type)
|
||||
after_date: Filter by created_at > after_date
|
||||
search_item_types: Filter by SearchItemType (ENTITY, OBSERVATION, RELATION)
|
||||
metadata_filters: Structured frontmatter metadata filters
|
||||
@@ -252,16 +204,6 @@ class SearchRepositoryBase(ABC):
|
||||
"""Return the SQL expression for current timestamp in the backend."""
|
||||
pass # pragma: no cover
|
||||
|
||||
@abstractmethod
|
||||
def _distance_to_similarity(self, distance: float) -> float:
|
||||
"""Convert a backend-specific vector distance to cosine similarity in [0, 1].
|
||||
|
||||
Backend-specific implementations:
|
||||
- SQLite (vec0): L2/Euclidean distance → cosine similarity via 1 - d²/2
|
||||
- Postgres (pgvector <=>): Cosine distance → cosine similarity via 1 - d
|
||||
"""
|
||||
pass # pragma: no cover
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Shared index / delete operations
|
||||
# ------------------------------------------------------------------
|
||||
@@ -413,8 +355,7 @@ class SearchRepositoryBase(ABC):
|
||||
if self._embedding_provider is None:
|
||||
raise SemanticDependenciesMissingError(
|
||||
"No embedding provider configured. "
|
||||
"Install/update basic-memory to include semantic dependencies "
|
||||
"(pip install -U basic-memory) "
|
||||
"Install semantic extras: pip install 'basic-memory[semantic]' "
|
||||
"and set semantic_search_enabled=true."
|
||||
)
|
||||
|
||||
@@ -451,36 +392,21 @@ class SearchRepositoryBase(ABC):
|
||||
return "\n\n".join(part for part in row_parts if part)
|
||||
|
||||
def _build_chunk_records(self, rows) -> list[dict[str, str]]:
|
||||
records_by_key: dict[str, dict[str, str]] = {}
|
||||
duplicate_chunk_keys = 0
|
||||
records: list[dict[str, str]] = []
|
||||
for row in rows:
|
||||
source_text = self._compose_row_source_text(row)
|
||||
chunks = self._split_text_into_chunks(source_text)
|
||||
for chunk_index, chunk_text in enumerate(chunks):
|
||||
chunk_key = f"{row.type}:{row.id}:{chunk_index}"
|
||||
source_hash = hashlib.sha256(chunk_text.encode("utf-8")).hexdigest()
|
||||
# Trigger: SQLite FTS5 can accumulate duplicate logical rows for the
|
||||
# same search_index id because it does not enforce relational uniqueness.
|
||||
# Why: duplicate chunk keys would schedule duplicate writes for the same
|
||||
# chunk row and eventually trip UNIQUE(rowid) in search_vector_embeddings.
|
||||
# Outcome: collapse chunk work to one deterministic record per chunk key.
|
||||
if chunk_key in records_by_key:
|
||||
duplicate_chunk_keys += 1
|
||||
records_by_key[chunk_key] = {
|
||||
"chunk_key": chunk_key,
|
||||
"chunk_text": chunk_text,
|
||||
"source_hash": source_hash,
|
||||
}
|
||||
|
||||
if duplicate_chunk_keys:
|
||||
logger.warning(
|
||||
"Collapsed duplicate vector chunk keys before embedding sync: "
|
||||
"project_id={project_id} duplicate_chunk_keys={duplicate_chunk_keys}",
|
||||
project_id=self.project_id,
|
||||
duplicate_chunk_keys=duplicate_chunk_keys,
|
||||
)
|
||||
|
||||
return list(records_by_key.values())
|
||||
records.append(
|
||||
{
|
||||
"chunk_key": chunk_key,
|
||||
"chunk_text": chunk_text,
|
||||
"source_hash": source_hash,
|
||||
}
|
||||
)
|
||||
return records
|
||||
|
||||
# --- Text splitting ---
|
||||
|
||||
@@ -623,205 +549,15 @@ class SearchRepositoryBase(ABC):
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
async def sync_entity_vectors(self, entity_id: int) -> None:
|
||||
"""Sync semantic chunk rows + embeddings for a single entity."""
|
||||
await self._sync_entity_vectors_internal(
|
||||
[entity_id],
|
||||
progress_callback=None,
|
||||
continue_on_error=False,
|
||||
)
|
||||
"""Sync semantic chunk rows + embeddings for a single entity.
|
||||
|
||||
async def sync_entity_vectors_batch(
|
||||
self,
|
||||
entity_ids: list[int],
|
||||
progress_callback: Optional[Callable[[int, int, int], Any]] = None,
|
||||
) -> VectorSyncBatchResult:
|
||||
"""Sync semantic chunk rows + embeddings for a batch of entities."""
|
||||
return await self._sync_entity_vectors_internal(
|
||||
entity_ids,
|
||||
progress_callback=progress_callback,
|
||||
continue_on_error=True,
|
||||
)
|
||||
|
||||
async def _sync_entity_vectors_internal(
|
||||
self,
|
||||
entity_ids: list[int],
|
||||
progress_callback: Optional[Callable[[int, int, int], Any]],
|
||||
continue_on_error: bool,
|
||||
) -> VectorSyncBatchResult:
|
||||
"""Run shared vector sync orchestration for one or many entities."""
|
||||
This is the shared orchestration logic. Backend-specific SQL operations
|
||||
are delegated to abstract hooks (_delete_entity_chunks, _write_embeddings, etc.).
|
||||
"""
|
||||
self._assert_semantic_available()
|
||||
await self._ensure_vector_tables()
|
||||
assert self._embedding_provider is not None
|
||||
|
||||
total_entities = len(entity_ids)
|
||||
result = VectorSyncBatchResult(
|
||||
entities_total=total_entities,
|
||||
entities_synced=0,
|
||||
entities_failed=0,
|
||||
)
|
||||
if total_entities == 0:
|
||||
return result
|
||||
|
||||
logger.info(
|
||||
"Vector batch sync start: project_id={project_id} entities_total={entities_total} "
|
||||
"sync_batch_size={sync_batch_size}",
|
||||
project_id=self.project_id,
|
||||
entities_total=total_entities,
|
||||
sync_batch_size=self._semantic_embedding_sync_batch_size,
|
||||
)
|
||||
|
||||
pending_jobs: list[_PendingEmbeddingJob] = []
|
||||
entity_runtime: dict[int, _EntitySyncRuntime] = {}
|
||||
failed_entity_ids: set[int] = set()
|
||||
synced_entity_ids: set[int] = set()
|
||||
|
||||
for index, entity_id in enumerate(entity_ids):
|
||||
if progress_callback is not None:
|
||||
progress_callback(entity_id, index, total_entities)
|
||||
|
||||
try:
|
||||
prepared = await self._prepare_entity_vector_jobs(entity_id)
|
||||
except Exception as exc:
|
||||
if not continue_on_error:
|
||||
raise
|
||||
failed_entity_ids.add(entity_id)
|
||||
logger.warning(
|
||||
"Vector batch sync entity prepare failed: project_id={project_id} "
|
||||
"entity_id={entity_id} error={error}",
|
||||
project_id=self.project_id,
|
||||
entity_id=entity_id,
|
||||
error=str(exc),
|
||||
)
|
||||
continue
|
||||
|
||||
embedding_jobs_count = len(prepared.embedding_jobs)
|
||||
result.embedding_jobs_total += embedding_jobs_count
|
||||
|
||||
if embedding_jobs_count == 0:
|
||||
synced_entity_ids.add(entity_id)
|
||||
total_seconds = time.perf_counter() - prepared.sync_start
|
||||
self._log_vector_sync_complete(
|
||||
entity_id=entity_id,
|
||||
total_seconds=total_seconds,
|
||||
embed_seconds=0.0,
|
||||
write_seconds=0.0,
|
||||
source_rows_count=prepared.source_rows_count,
|
||||
embedding_jobs_count=0,
|
||||
)
|
||||
continue
|
||||
|
||||
entity_runtime[entity_id] = _EntitySyncRuntime(
|
||||
sync_start=prepared.sync_start,
|
||||
source_rows_count=prepared.source_rows_count,
|
||||
embedding_jobs_count=embedding_jobs_count,
|
||||
remaining_jobs=embedding_jobs_count,
|
||||
)
|
||||
pending_jobs.extend(
|
||||
_PendingEmbeddingJob(
|
||||
entity_id=entity_id, chunk_row_id=row_id, chunk_text=chunk_text
|
||||
)
|
||||
for row_id, chunk_text in prepared.embedding_jobs
|
||||
)
|
||||
|
||||
while len(pending_jobs) >= self._semantic_embedding_sync_batch_size:
|
||||
flush_jobs = pending_jobs[: self._semantic_embedding_sync_batch_size]
|
||||
pending_jobs = pending_jobs[self._semantic_embedding_sync_batch_size :]
|
||||
try:
|
||||
embed_seconds, write_seconds = await self._flush_embedding_jobs(
|
||||
flush_jobs=flush_jobs,
|
||||
entity_runtime=entity_runtime,
|
||||
synced_entity_ids=synced_entity_ids,
|
||||
)
|
||||
result.embed_seconds_total += embed_seconds
|
||||
result.write_seconds_total += write_seconds
|
||||
except Exception as exc:
|
||||
if not continue_on_error:
|
||||
raise
|
||||
affected_entity_ids = sorted({job.entity_id for job in flush_jobs})
|
||||
failed_entity_ids.update(affected_entity_ids)
|
||||
for failed_entity_id in affected_entity_ids:
|
||||
entity_runtime.pop(failed_entity_id, None)
|
||||
logger.warning(
|
||||
"Vector batch sync flush failed: project_id={project_id} "
|
||||
"affected_entities={affected_entities} chunk_count={chunk_count} error={error}",
|
||||
project_id=self.project_id,
|
||||
affected_entities=affected_entity_ids,
|
||||
chunk_count=len(flush_jobs),
|
||||
error=str(exc),
|
||||
)
|
||||
|
||||
if pending_jobs:
|
||||
flush_jobs = list(pending_jobs)
|
||||
pending_jobs = []
|
||||
try:
|
||||
embed_seconds, write_seconds = await self._flush_embedding_jobs(
|
||||
flush_jobs=flush_jobs,
|
||||
entity_runtime=entity_runtime,
|
||||
synced_entity_ids=synced_entity_ids,
|
||||
)
|
||||
result.embed_seconds_total += embed_seconds
|
||||
result.write_seconds_total += write_seconds
|
||||
except Exception as exc:
|
||||
if not continue_on_error:
|
||||
raise
|
||||
affected_entity_ids = sorted({job.entity_id for job in flush_jobs})
|
||||
failed_entity_ids.update(affected_entity_ids)
|
||||
for failed_entity_id in affected_entity_ids:
|
||||
entity_runtime.pop(failed_entity_id, None)
|
||||
logger.warning(
|
||||
"Vector batch sync final flush failed: project_id={project_id} "
|
||||
"affected_entities={affected_entities} chunk_count={chunk_count} error={error}",
|
||||
project_id=self.project_id,
|
||||
affected_entities=affected_entity_ids,
|
||||
chunk_count=len(flush_jobs),
|
||||
error=str(exc),
|
||||
)
|
||||
|
||||
# Trigger: this should never happen after all flushes succeed.
|
||||
# Why: remaining jobs mean runtime tracking drifted from queued jobs.
|
||||
# Outcome: fail-safe marks these entities as failed to avoid false positives.
|
||||
if entity_runtime:
|
||||
orphan_runtime_entities = sorted(entity_runtime.keys())
|
||||
failed_entity_ids.update(orphan_runtime_entities)
|
||||
logger.warning(
|
||||
"Vector batch sync left unfinished entities after flushes: "
|
||||
"project_id={project_id} unfinished_entities={unfinished_entities}",
|
||||
project_id=self.project_id,
|
||||
unfinished_entities=orphan_runtime_entities,
|
||||
)
|
||||
|
||||
# Keep result counters aligned with successful/failed terminal states.
|
||||
synced_entity_ids.difference_update(failed_entity_ids)
|
||||
result.failed_entity_ids = sorted(failed_entity_ids)
|
||||
result.entities_failed = len(result.failed_entity_ids)
|
||||
result.entities_synced = len(synced_entity_ids)
|
||||
|
||||
logger.info(
|
||||
"Vector batch sync complete: project_id={project_id} entities_total={entities_total} "
|
||||
"entities_synced={entities_synced} entities_failed={entities_failed} "
|
||||
"embedding_jobs_total={embedding_jobs_total} embed_seconds_total={embed_seconds_total:.3f} "
|
||||
"write_seconds_total={write_seconds_total:.3f}",
|
||||
project_id=self.project_id,
|
||||
entities_total=result.entities_total,
|
||||
entities_synced=result.entities_synced,
|
||||
entities_failed=result.entities_failed,
|
||||
embedding_jobs_total=result.embedding_jobs_total,
|
||||
embed_seconds_total=result.embed_seconds_total,
|
||||
write_seconds_total=result.write_seconds_total,
|
||||
)
|
||||
|
||||
return result
|
||||
|
||||
async def _prepare_entity_vector_jobs(self, entity_id: int) -> _PreparedEntityVectorSync:
|
||||
"""Prepare chunk mutations and embedding jobs for one entity."""
|
||||
sync_start = time.perf_counter()
|
||||
|
||||
logger.info(
|
||||
"Vector sync start: project_id={project_id} entity_id={entity_id}",
|
||||
project_id=self.project_id,
|
||||
entity_id=entity_id,
|
||||
)
|
||||
|
||||
async with db.scoped_session(self.session_maker) as session:
|
||||
await self._prepare_vector_session(session)
|
||||
|
||||
@@ -847,49 +583,18 @@ class SearchRepositoryBase(ABC):
|
||||
},
|
||||
)
|
||||
rows = row_result.fetchall()
|
||||
source_rows_count = len(rows)
|
||||
built_chunk_records_count = 0
|
||||
|
||||
# No search_index rows → delete all chunk/embedding data for this entity.
|
||||
if not rows:
|
||||
logger.info(
|
||||
"Vector sync source prepared: project_id={project_id} entity_id={entity_id} "
|
||||
"source_rows_count={source_rows_count} "
|
||||
"built_chunk_records_count={built_chunk_records_count}",
|
||||
project_id=self.project_id,
|
||||
entity_id=entity_id,
|
||||
source_rows_count=source_rows_count,
|
||||
built_chunk_records_count=built_chunk_records_count,
|
||||
)
|
||||
await self._delete_entity_chunks(session, entity_id)
|
||||
await session.commit()
|
||||
return _PreparedEntityVectorSync(
|
||||
entity_id=entity_id,
|
||||
sync_start=sync_start,
|
||||
source_rows_count=source_rows_count,
|
||||
embedding_jobs=[],
|
||||
)
|
||||
return
|
||||
|
||||
chunk_records = self._build_chunk_records(rows)
|
||||
built_chunk_records_count = len(chunk_records)
|
||||
logger.info(
|
||||
"Vector sync source prepared: project_id={project_id} entity_id={entity_id} "
|
||||
"source_rows_count={source_rows_count} "
|
||||
"built_chunk_records_count={built_chunk_records_count}",
|
||||
project_id=self.project_id,
|
||||
entity_id=entity_id,
|
||||
source_rows_count=source_rows_count,
|
||||
built_chunk_records_count=built_chunk_records_count,
|
||||
)
|
||||
if not chunk_records:
|
||||
await self._delete_entity_chunks(session, entity_id)
|
||||
await session.commit()
|
||||
return _PreparedEntityVectorSync(
|
||||
entity_id=entity_id,
|
||||
sync_start=sync_start,
|
||||
source_rows_count=source_rows_count,
|
||||
embedding_jobs=[],
|
||||
)
|
||||
return
|
||||
|
||||
# --- Diff existing chunks against incoming ---
|
||||
existing_rows_result = await session.execute(
|
||||
@@ -901,7 +606,6 @@ class SearchRepositoryBase(ABC):
|
||||
{"project_id": self.project_id, "entity_id": entity_id},
|
||||
)
|
||||
existing_by_key = {row.chunk_key: row for row in existing_rows_result.fetchall()}
|
||||
existing_chunks_count = len(existing_by_key)
|
||||
incoming_hashes = {
|
||||
record["chunk_key"]: record["source_hash"] for record in chunk_records
|
||||
}
|
||||
@@ -910,7 +614,6 @@ class SearchRepositoryBase(ABC):
|
||||
for chunk_key, row in existing_by_key.items()
|
||||
if chunk_key not in incoming_hashes
|
||||
]
|
||||
stale_chunks_count = len(stale_ids)
|
||||
|
||||
if stale_ids:
|
||||
await self._delete_stale_chunks(session, stale_ids, entity_id)
|
||||
@@ -924,8 +627,6 @@ class SearchRepositoryBase(ABC):
|
||||
{"project_id": self.project_id, "entity_id": entity_id},
|
||||
)
|
||||
orphan_rows = orphan_result.fetchall()
|
||||
orphan_ids = {int(row.id) for row in orphan_rows}
|
||||
orphan_chunks_count = len(orphan_ids)
|
||||
|
||||
# --- Upsert changed / new chunks, collect embedding jobs ---
|
||||
timestamp_expr = self._timestamp_now_expr()
|
||||
@@ -936,7 +637,7 @@ class SearchRepositoryBase(ABC):
|
||||
# Trigger: chunk exists and hash matches (no content change)
|
||||
# but chunk has no embedding (orphan from crash).
|
||||
# Outcome: schedule re-embedding without touching chunk metadata.
|
||||
is_orphan = current and int(current.id) in orphan_ids
|
||||
is_orphan = current and any(o.id == current.id for o in orphan_rows)
|
||||
if current and current.source_hash == record["source_hash"] and not is_orphan:
|
||||
continue
|
||||
|
||||
@@ -979,141 +680,20 @@ class SearchRepositoryBase(ABC):
|
||||
row_id = int(inserted.scalar_one())
|
||||
embedding_jobs.append((row_id, record["chunk_text"]))
|
||||
|
||||
logger.info(
|
||||
"Vector sync diff complete: project_id={project_id} entity_id={entity_id} "
|
||||
"existing_chunks_count={existing_chunks_count} "
|
||||
"stale_chunks_count={stale_chunks_count} "
|
||||
"orphan_chunks_count={orphan_chunks_count} "
|
||||
"embedding_jobs_count={embedding_jobs_count}",
|
||||
project_id=self.project_id,
|
||||
entity_id=entity_id,
|
||||
existing_chunks_count=existing_chunks_count,
|
||||
stale_chunks_count=stale_chunks_count,
|
||||
orphan_chunks_count=orphan_chunks_count,
|
||||
embedding_jobs_count=len(embedding_jobs),
|
||||
)
|
||||
await session.commit()
|
||||
|
||||
return _PreparedEntityVectorSync(
|
||||
entity_id=entity_id,
|
||||
sync_start=sync_start,
|
||||
source_rows_count=source_rows_count,
|
||||
embedding_jobs=embedding_jobs,
|
||||
)
|
||||
if not embedding_jobs:
|
||||
return
|
||||
|
||||
async def _flush_embedding_jobs(
|
||||
self,
|
||||
flush_jobs: list[_PendingEmbeddingJob],
|
||||
entity_runtime: dict[int, _EntitySyncRuntime],
|
||||
synced_entity_ids: set[int],
|
||||
) -> tuple[float, float]:
|
||||
"""Embed and persist one queued flush chunk."""
|
||||
if not flush_jobs:
|
||||
return 0.0, 0.0
|
||||
assert self._embedding_provider is not None
|
||||
|
||||
embed_start = time.perf_counter()
|
||||
texts = [job.chunk_text for job in flush_jobs]
|
||||
texts = [t for _, t in embedding_jobs]
|
||||
embeddings = await self._embedding_provider.embed_documents(texts)
|
||||
embed_seconds = time.perf_counter() - embed_start
|
||||
embed_rate = (len(flush_jobs) / embed_seconds) if embed_seconds > 0 else 0.0
|
||||
logger.info(
|
||||
"Vector batch embed flush: project_id={project_id} chunk_count={chunk_count} "
|
||||
"embed_seconds={embed_seconds:.3f} embed_rate_chunks_per_second={embed_rate:.2f}",
|
||||
project_id=self.project_id,
|
||||
chunk_count=len(flush_jobs),
|
||||
embed_seconds=embed_seconds,
|
||||
embed_rate=embed_rate,
|
||||
)
|
||||
if len(embeddings) != len(flush_jobs):
|
||||
if len(embeddings) != len(embedding_jobs):
|
||||
raise RuntimeError("Embedding provider returned an unexpected number of vectors.")
|
||||
|
||||
write_start = time.perf_counter()
|
||||
async with db.scoped_session(self.session_maker) as session:
|
||||
await self._prepare_vector_session(session)
|
||||
write_jobs = [(job.chunk_row_id, job.chunk_text) for job in flush_jobs]
|
||||
await self._write_embeddings(session, write_jobs, embeddings)
|
||||
await self._write_embeddings(session, embedding_jobs, embeddings)
|
||||
await session.commit()
|
||||
write_seconds = time.perf_counter() - write_start
|
||||
write_rate = (len(flush_jobs) / write_seconds) if write_seconds > 0 else 0.0
|
||||
logger.info(
|
||||
"Vector batch write flush: project_id={project_id} row_count={row_count} "
|
||||
"write_seconds={write_seconds:.3f} write_rate_rows_per_second={write_rate:.2f}",
|
||||
project_id=self.project_id,
|
||||
row_count=len(flush_jobs),
|
||||
write_seconds=write_seconds,
|
||||
write_rate=write_rate,
|
||||
)
|
||||
|
||||
flush_size = len(flush_jobs)
|
||||
entity_job_counts: dict[int, int] = {}
|
||||
for job in flush_jobs:
|
||||
entity_job_counts[job.entity_id] = entity_job_counts.get(job.entity_id, 0) + 1
|
||||
|
||||
for entity_id, entity_job_count in entity_job_counts.items():
|
||||
runtime = entity_runtime.get(entity_id)
|
||||
if runtime is None:
|
||||
continue
|
||||
runtime.remaining_jobs -= entity_job_count
|
||||
|
||||
# Attribute flush wall-clock to entities in proportion to rows written.
|
||||
flush_share = entity_job_count / flush_size
|
||||
runtime.embed_seconds += embed_seconds * flush_share
|
||||
runtime.write_seconds += write_seconds * flush_share
|
||||
|
||||
if runtime.remaining_jobs <= 0:
|
||||
synced_entity_ids.add(entity_id)
|
||||
total_seconds = time.perf_counter() - runtime.sync_start
|
||||
self._log_vector_sync_complete(
|
||||
entity_id=entity_id,
|
||||
total_seconds=total_seconds,
|
||||
embed_seconds=runtime.embed_seconds,
|
||||
write_seconds=runtime.write_seconds,
|
||||
source_rows_count=runtime.source_rows_count,
|
||||
embedding_jobs_count=runtime.embedding_jobs_count,
|
||||
)
|
||||
entity_runtime.pop(entity_id, None)
|
||||
|
||||
return embed_seconds, write_seconds
|
||||
|
||||
def _log_vector_sync_complete(
|
||||
self,
|
||||
*,
|
||||
entity_id: int,
|
||||
total_seconds: float,
|
||||
embed_seconds: float,
|
||||
write_seconds: float,
|
||||
source_rows_count: int,
|
||||
embedding_jobs_count: int,
|
||||
) -> None:
|
||||
"""Log completion and slow-entity warnings with a consistent format."""
|
||||
logger.info(
|
||||
"Vector sync complete: project_id={project_id} entity_id={entity_id} "
|
||||
"total_seconds={total_seconds:.3f} embed_seconds={embed_seconds:.3f} "
|
||||
"write_seconds={write_seconds:.3f} source_rows_count={source_rows_count} "
|
||||
"embedding_jobs_count={embedding_jobs_count}",
|
||||
project_id=self.project_id,
|
||||
entity_id=entity_id,
|
||||
total_seconds=total_seconds,
|
||||
embed_seconds=embed_seconds,
|
||||
write_seconds=write_seconds,
|
||||
source_rows_count=source_rows_count,
|
||||
embedding_jobs_count=embedding_jobs_count,
|
||||
)
|
||||
if total_seconds > 10:
|
||||
logger.warning(
|
||||
"Vector sync slow entity: project_id={project_id} entity_id={entity_id} "
|
||||
"total_seconds={total_seconds:.3f} embed_seconds={embed_seconds:.3f} "
|
||||
"write_seconds={write_seconds:.3f} source_rows_count={source_rows_count} "
|
||||
"embedding_jobs_count={embedding_jobs_count}",
|
||||
project_id=self.project_id,
|
||||
entity_id=entity_id,
|
||||
total_seconds=total_seconds,
|
||||
embed_seconds=embed_seconds,
|
||||
write_seconds=write_seconds,
|
||||
source_rows_count=source_rows_count,
|
||||
embedding_jobs_count=embedding_jobs_count,
|
||||
)
|
||||
|
||||
async def _prepare_vector_session(self, session: AsyncSession) -> None:
|
||||
"""Hook for per-session setup (e.g. loading sqlite-vec extension).
|
||||
@@ -1170,7 +750,7 @@ class SearchRepositoryBase(ABC):
|
||||
permalink: Optional[str],
|
||||
permalink_match: Optional[str],
|
||||
title: Optional[str],
|
||||
note_types: Optional[List[str]],
|
||||
types: Optional[List[str]],
|
||||
after_date: Optional[datetime],
|
||||
search_item_types: Optional[List[SearchItemType]],
|
||||
metadata_filters: Optional[dict],
|
||||
@@ -1203,7 +783,7 @@ class SearchRepositoryBase(ABC):
|
||||
permalink=permalink,
|
||||
permalink_match=permalink_match,
|
||||
title=title,
|
||||
note_types=note_types,
|
||||
types=types,
|
||||
after_date=after_date,
|
||||
search_item_types=search_item_types,
|
||||
metadata_filters=metadata_filters,
|
||||
@@ -1222,7 +802,7 @@ class SearchRepositoryBase(ABC):
|
||||
permalink=permalink,
|
||||
permalink_match=permalink_match,
|
||||
title=title,
|
||||
note_types=note_types,
|
||||
types=types,
|
||||
after_date=after_date,
|
||||
search_item_types=search_item_types,
|
||||
metadata_filters=metadata_filters,
|
||||
@@ -1251,14 +831,13 @@ class SearchRepositoryBase(ABC):
|
||||
permalink: Optional[str],
|
||||
permalink_match: Optional[str],
|
||||
title: Optional[str],
|
||||
note_types: Optional[List[str]],
|
||||
types: Optional[List[str]],
|
||||
after_date: Optional[datetime],
|
||||
search_item_types: Optional[List[SearchItemType]],
|
||||
metadata_filters: Optional[dict],
|
||||
min_similarity: Optional[float] = None,
|
||||
limit: int,
|
||||
offset: int,
|
||||
_emit_observability_log: bool = True,
|
||||
) -> List[SearchIndexRow]:
|
||||
"""Run vector-only search returning chunk-level results.
|
||||
|
||||
@@ -1269,75 +848,24 @@ class SearchRepositoryBase(ABC):
|
||||
self._assert_semantic_available()
|
||||
await self._ensure_vector_tables()
|
||||
assert self._embedding_provider is not None
|
||||
query_text = search_text.strip()
|
||||
query_embedding = await self._embedding_provider.embed_query(search_text.strip())
|
||||
candidate_limit = max(self._semantic_vector_k, (limit + offset) * 10)
|
||||
query_start = time.perf_counter()
|
||||
embed_start = time.perf_counter()
|
||||
query_embedding = await self._embedding_provider.embed_query(query_text)
|
||||
embed_ms = (time.perf_counter() - embed_start) * 1000
|
||||
vector_query_start = time.perf_counter()
|
||||
|
||||
async with db.scoped_session(self.session_maker) as session:
|
||||
await self._prepare_vector_session(session)
|
||||
vector_rows = await self._run_vector_query(session, query_embedding, candidate_limit)
|
||||
vector_query_ms = (time.perf_counter() - vector_query_start) * 1000
|
||||
vector_row_count = len(vector_rows)
|
||||
hydrate_ms = 0.0
|
||||
|
||||
def _log_vector_summary() -> None:
|
||||
if not _emit_observability_log:
|
||||
return
|
||||
|
||||
total_ms = (time.perf_counter() - query_start) * 1000
|
||||
logger.info(
|
||||
"Semantic query timing: project_id={project_id} retrieval_mode={retrieval_mode} "
|
||||
"query_length={query_length} candidate_limit={candidate_limit} "
|
||||
"vector_row_count={vector_row_count} embed_ms={embed_ms:.2f} "
|
||||
"vector_query_ms={vector_query_ms:.2f} hydrate_ms={hydrate_ms:.2f} "
|
||||
"total_ms={total_ms:.2f}",
|
||||
project_id=self.project_id,
|
||||
retrieval_mode="vector",
|
||||
query_length=len(query_text),
|
||||
candidate_limit=candidate_limit,
|
||||
vector_row_count=vector_row_count,
|
||||
embed_ms=embed_ms,
|
||||
vector_query_ms=vector_query_ms,
|
||||
hydrate_ms=hydrate_ms,
|
||||
total_ms=total_ms,
|
||||
)
|
||||
if total_ms > 2000:
|
||||
logger.warning(
|
||||
"[SEMANTIC_SLOW_QUERY] Semantic query timing: project_id={project_id} "
|
||||
"retrieval_mode={retrieval_mode} query_length={query_length} "
|
||||
"candidate_limit={candidate_limit} vector_row_count={vector_row_count} "
|
||||
"embed_ms={embed_ms:.2f} vector_query_ms={vector_query_ms:.2f} "
|
||||
"hydrate_ms={hydrate_ms:.2f} total_ms={total_ms:.2f}",
|
||||
project_id=self.project_id,
|
||||
retrieval_mode="vector",
|
||||
query_length=len(query_text),
|
||||
candidate_limit=candidate_limit,
|
||||
vector_row_count=vector_row_count,
|
||||
embed_ms=embed_ms,
|
||||
vector_query_ms=vector_query_ms,
|
||||
hydrate_ms=hydrate_ms,
|
||||
total_ms=total_ms,
|
||||
)
|
||||
|
||||
if not vector_rows:
|
||||
_log_vector_summary()
|
||||
return []
|
||||
|
||||
hydrate_start = time.perf_counter()
|
||||
# Build per-search_index_row similarity scores from chunk-level results.
|
||||
# Each chunk_key encodes the search_index row type and id.
|
||||
# Track the best similarity per row (for ranking) and all chunks (for context).
|
||||
# Keep the best similarity per search_index row id.
|
||||
similarity_by_si_id: dict[int, float] = {}
|
||||
chunks_by_si_id: dict[int, list[tuple[float, str]]] = {}
|
||||
for row in vector_rows:
|
||||
chunk_key = row.get("chunk_key", "")
|
||||
distance = float(row["best_distance"])
|
||||
similarity = self._distance_to_similarity(distance)
|
||||
chunk_text = row.get("chunk_text", "")
|
||||
similarity = 1.0 / (1.0 + max(distance, 0.0))
|
||||
try:
|
||||
_, si_id = self._parse_chunk_key(chunk_key)
|
||||
except (ValueError, IndexError):
|
||||
@@ -1346,11 +874,8 @@ class SearchRepositoryBase(ABC):
|
||||
current = similarity_by_si_id.get(si_id)
|
||||
if current is None or similarity > current:
|
||||
similarity_by_si_id[si_id] = similarity
|
||||
chunks_by_si_id.setdefault(si_id, []).append((similarity, chunk_text))
|
||||
|
||||
if not similarity_by_si_id:
|
||||
hydrate_ms = (time.perf_counter() - hydrate_start) * 1000
|
||||
_log_vector_summary()
|
||||
return []
|
||||
|
||||
# Filter out results below the minimum similarity threshold.
|
||||
@@ -1363,8 +888,6 @@ class SearchRepositoryBase(ABC):
|
||||
k: v for k, v in similarity_by_si_id.items() if v >= effective_min_similarity
|
||||
}
|
||||
if not similarity_by_si_id:
|
||||
hydrate_ms = (time.perf_counter() - hydrate_start) * 1000
|
||||
_log_vector_summary()
|
||||
return []
|
||||
|
||||
# Fetch the actual search_index rows
|
||||
@@ -1377,7 +900,7 @@ class SearchRepositoryBase(ABC):
|
||||
permalink,
|
||||
permalink_match,
|
||||
title,
|
||||
note_types,
|
||||
types,
|
||||
after_date,
|
||||
search_item_types,
|
||||
metadata_filters,
|
||||
@@ -1390,7 +913,7 @@ class SearchRepositoryBase(ABC):
|
||||
permalink=permalink,
|
||||
permalink_match=permalink_match,
|
||||
title=title,
|
||||
note_types=note_types,
|
||||
types=types,
|
||||
after_date=after_date,
|
||||
search_item_types=search_item_types,
|
||||
metadata_filters=metadata_filters,
|
||||
@@ -1410,29 +933,9 @@ class SearchRepositoryBase(ABC):
|
||||
row = search_index_rows.get(si_id)
|
||||
if row is None:
|
||||
continue
|
||||
|
||||
# Small notes: return full content so the answer is always present.
|
||||
# Large notes: return top-N most relevant chunks for richer context.
|
||||
content_snippet = row.content_snippet or ""
|
||||
if content_snippet and len(content_snippet) <= SMALL_NOTE_CONTENT_LIMIT:
|
||||
matched_chunk_text = content_snippet
|
||||
else:
|
||||
si_chunks = chunks_by_si_id.get(si_id, [])
|
||||
si_chunks.sort(key=lambda c: c[0], reverse=True)
|
||||
top_texts = [text for _, text in si_chunks[:TOP_CHUNKS_PER_RESULT]]
|
||||
matched_chunk_text = "\n---\n".join(top_texts) if top_texts else None
|
||||
|
||||
ranked_rows.append(
|
||||
replace(
|
||||
row,
|
||||
score=similarity,
|
||||
matched_chunk_text=matched_chunk_text,
|
||||
)
|
||||
)
|
||||
ranked_rows.append(replace(row, score=similarity))
|
||||
|
||||
ranked_rows.sort(key=lambda item: item.score or 0.0, reverse=True)
|
||||
hydrate_ms = (time.perf_counter() - hydrate_start) * 1000
|
||||
_log_vector_summary()
|
||||
return ranked_rows[offset : offset + limit]
|
||||
|
||||
async def _fetch_entity_rows_by_ids(self, entity_ids: list[int]) -> dict[int, SearchIndexRow]:
|
||||
@@ -1530,7 +1033,7 @@ class SearchRepositoryBase(ABC):
|
||||
return result
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Shared semantic search: hybrid score-based fusion
|
||||
# Shared semantic search: hybrid RRF fusion
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
async def _search_hybrid(
|
||||
@@ -1540,7 +1043,7 @@ class SearchRepositoryBase(ABC):
|
||||
permalink: Optional[str],
|
||||
permalink_match: Optional[str],
|
||||
title: Optional[str],
|
||||
note_types: Optional[List[str]],
|
||||
types: Optional[List[str]],
|
||||
after_date: Optional[datetime],
|
||||
search_item_types: Optional[List[SearchItemType]],
|
||||
metadata_filters: Optional[dict],
|
||||
@@ -1548,23 +1051,19 @@ class SearchRepositoryBase(ABC):
|
||||
limit: int,
|
||||
offset: int,
|
||||
) -> List[SearchIndexRow]:
|
||||
"""Fuse FTS and vector results using score-based fusion.
|
||||
"""Fuse FTS and vector rankings using reciprocal rank fusion (RRF).
|
||||
|
||||
Uses search_index row id as the fusion key. The formula
|
||||
``max(vec, fts) + FUSION_BONUS * min(vec, fts)`` preserves
|
||||
the dominant signal and rewards dual-source agreement.
|
||||
Uses entity_id as the fusion key (not permalink) to correctly handle
|
||||
entities with NULL permalinks.
|
||||
"""
|
||||
self._assert_semantic_available()
|
||||
query_text = search_text.strip()
|
||||
query_start = time.perf_counter()
|
||||
candidate_limit = max(self._semantic_vector_k, (limit + offset) * 10)
|
||||
fts_start = time.perf_counter()
|
||||
fts_results = await self.search(
|
||||
search_text=search_text,
|
||||
permalink=permalink,
|
||||
permalink_match=permalink_match,
|
||||
title=title,
|
||||
note_types=note_types,
|
||||
types=types,
|
||||
after_date=after_date,
|
||||
search_item_types=search_item_types,
|
||||
metadata_filters=metadata_filters,
|
||||
@@ -1572,28 +1071,24 @@ class SearchRepositoryBase(ABC):
|
||||
limit=candidate_limit,
|
||||
offset=0,
|
||||
)
|
||||
fts_ms = (time.perf_counter() - fts_start) * 1000
|
||||
vector_start = time.perf_counter()
|
||||
vector_results = await self._search_vector_only(
|
||||
search_text=search_text,
|
||||
permalink=permalink,
|
||||
permalink_match=permalink_match,
|
||||
title=title,
|
||||
note_types=note_types,
|
||||
types=types,
|
||||
after_date=after_date,
|
||||
search_item_types=search_item_types,
|
||||
metadata_filters=metadata_filters,
|
||||
min_similarity=min_similarity,
|
||||
limit=candidate_limit,
|
||||
offset=0,
|
||||
_emit_observability_log=False,
|
||||
)
|
||||
vector_ms = (time.perf_counter() - vector_start) * 1000
|
||||
fusion_start = time.perf_counter()
|
||||
|
||||
# --- Score-based fusion keyed on search_index row id ---
|
||||
# FTS scores are normalized to [0, 1] (BM25 is unbounded).
|
||||
# Vector scores are used raw — already calibrated [0, 1] by _distance_to_similarity().
|
||||
# Score-weighted RRF fusion keyed on search_index row id.
|
||||
# Multiplies the standard 1/(k+rank) score by the normalized original score
|
||||
# so that high-confidence matches contribute more than weak ones at the same rank.
|
||||
fused_scores: dict[int, float] = {}
|
||||
rows_by_id: dict[int, SearchIndexRow] = {}
|
||||
|
||||
# Normalize FTS scores to [0, 1] — handles both SQLite (negative bm25)
|
||||
@@ -1601,80 +1096,27 @@ class SearchRepositoryBase(ABC):
|
||||
fts_abs = [abs(row.score or 0.0) for row in fts_results]
|
||||
fts_max = max(fts_abs) if fts_abs else 1.0
|
||||
|
||||
fts_scores: dict[int, float] = {}
|
||||
for row in fts_results:
|
||||
for rank, row in enumerate(fts_results, start=1):
|
||||
if row.id is None:
|
||||
continue
|
||||
norm = abs(row.score or 0.0) / fts_max if fts_max > 0 else 0.0
|
||||
# Gate: FTS scores below threshold contribute zero
|
||||
if norm < FTS_GATE_THRESHOLD:
|
||||
norm = 0.0
|
||||
fts_scores[row.id] = norm
|
||||
weight = max(norm, 0.1) # floor preserves RRF stability
|
||||
fused_scores[row.id] = fused_scores.get(row.id, 0.0) + weight * (1.0 / (RRF_K + rank))
|
||||
rows_by_id[row.id] = row
|
||||
|
||||
vec_scores: dict[int, float] = {}
|
||||
for row in vector_results:
|
||||
# Vector scores already in [0, 1] from the similarity formula
|
||||
vec_max = max((row.score or 0.0) for row in vector_results) if vector_results else 1.0
|
||||
|
||||
for rank, row in enumerate(vector_results, start=1):
|
||||
if row.id is None:
|
||||
continue
|
||||
# Trigger: no re-normalization by vec_max
|
||||
# Why: vector similarity is already calibrated [0, 1]; re-normalizing
|
||||
# inflates weak matches when the entire result set is mediocre
|
||||
vec_scores[row.id] = row.score or 0.0
|
||||
norm = (row.score or 0.0) / vec_max if vec_max > 0 else 0.0
|
||||
weight = max(norm, 0.1) # floor preserves RRF stability
|
||||
fused_scores[row.id] = fused_scores.get(row.id, 0.0) + weight * (1.0 / (RRF_K + rank))
|
||||
rows_by_id[row.id] = row
|
||||
|
||||
# Fuse: max(v, f) + FUSION_BONUS * min(v, f)
|
||||
# Preserves the dominant signal; bonus rewards dual-source agreement.
|
||||
# Output range: [0, 1.3] for dual-source, [0, 1.0] for single-source.
|
||||
fused_scores: dict[int, float] = {}
|
||||
for row_id in fts_scores.keys() | vec_scores.keys():
|
||||
v = vec_scores.get(row_id, 0.0)
|
||||
f = fts_scores.get(row_id, 0.0)
|
||||
fused_scores[row_id] = max(v, f) + FUSION_BONUS * min(v, f)
|
||||
|
||||
ranked = sorted(fused_scores.items(), key=lambda item: item[1], reverse=True)
|
||||
output: list[SearchIndexRow] = []
|
||||
for row_id, fused_score in ranked[offset : offset + limit]:
|
||||
row = rows_by_id[row_id]
|
||||
# Trigger: FTS-only results have no matched_chunk_text from vector search.
|
||||
# Why: without chunk text, API falls back to truncated content, losing answer text.
|
||||
# Outcome: FTS-only results get full content_snippet as matched_chunk.
|
||||
if row.matched_chunk_text is None and row.content_snippet:
|
||||
row = replace(row, matched_chunk_text=row.content_snippet)
|
||||
output.append(replace(row, score=fused_score))
|
||||
fusion_ms = (time.perf_counter() - fusion_start) * 1000
|
||||
total_ms = (time.perf_counter() - query_start) * 1000
|
||||
logger.info(
|
||||
"Semantic query timing: project_id={project_id} retrieval_mode={retrieval_mode} "
|
||||
"query_length={query_length} candidate_limit={candidate_limit} "
|
||||
"fts_count={fts_count} vector_count={vector_count} fts_ms={fts_ms:.2f} "
|
||||
"vector_ms={vector_ms:.2f} fusion_ms={fusion_ms:.2f} total_ms={total_ms:.2f}",
|
||||
project_id=self.project_id,
|
||||
retrieval_mode="hybrid",
|
||||
query_length=len(query_text),
|
||||
candidate_limit=candidate_limit,
|
||||
fts_count=len(fts_results),
|
||||
vector_count=len(vector_results),
|
||||
fts_ms=fts_ms,
|
||||
vector_ms=vector_ms,
|
||||
fusion_ms=fusion_ms,
|
||||
total_ms=total_ms,
|
||||
)
|
||||
if total_ms > 2500:
|
||||
logger.warning(
|
||||
"[SEMANTIC_SLOW_QUERY] Semantic query timing: project_id={project_id} "
|
||||
"retrieval_mode={retrieval_mode} query_length={query_length} "
|
||||
"candidate_limit={candidate_limit} fts_count={fts_count} "
|
||||
"vector_count={vector_count} fts_ms={fts_ms:.2f} vector_ms={vector_ms:.2f} "
|
||||
"fusion_ms={fusion_ms:.2f} total_ms={total_ms:.2f}",
|
||||
project_id=self.project_id,
|
||||
retrieval_mode="hybrid",
|
||||
query_length=len(query_text),
|
||||
candidate_limit=candidate_limit,
|
||||
fts_count=len(fts_results),
|
||||
vector_count=len(vector_results),
|
||||
fts_ms=fts_ms,
|
||||
vector_ms=vector_ms,
|
||||
fusion_ms=fusion_ms,
|
||||
total_ms=total_ms,
|
||||
)
|
||||
output.append(replace(rows_by_id[row_id], score=fused_score))
|
||||
return output
|
||||
|
||||
@@ -52,18 +52,12 @@ class SQLiteSearchRepository(SearchRepositoryBase):
|
||||
self._semantic_enabled = self._app_config.semantic_search_enabled
|
||||
self._semantic_vector_k = self._app_config.semantic_vector_k
|
||||
self._semantic_min_similarity = self._app_config.semantic_min_similarity
|
||||
self._semantic_embedding_sync_batch_size = (
|
||||
self._app_config.semantic_embedding_sync_batch_size
|
||||
)
|
||||
self._embedding_provider = embedding_provider
|
||||
self._sqlite_vec_lock = asyncio.Lock()
|
||||
self._vector_tables_initialized = False
|
||||
self._vector_dimensions = 384
|
||||
|
||||
if self._semantic_enabled and self._embedding_provider is None:
|
||||
# Constraint: SQLite maps L2 distance to cosine similarity via 1 - L2²/2.
|
||||
# This conversion is correct only for unit-normalized embeddings.
|
||||
# Provider implementations must return normalized vectors.
|
||||
self._embedding_provider = create_embedding_provider(self._app_config)
|
||||
if self._embedding_provider is not None:
|
||||
self._vector_dimensions = self._embedding_provider.dimensions
|
||||
@@ -82,7 +76,7 @@ class SQLiteSearchRepository(SearchRepositoryBase):
|
||||
across server restarts. Also creates vector tables when semantic search
|
||||
is enabled so missing dependencies are caught at startup, not first query.
|
||||
"""
|
||||
logger.debug("Initializing SQLite FTS5 search index")
|
||||
logger.info("Initializing SQLite FTS5 search index")
|
||||
try:
|
||||
async with db.scoped_session(self.session_maker) as session:
|
||||
# Create FTS5 virtual table if it doesn't exist
|
||||
@@ -349,12 +343,11 @@ class SQLiteSearchRepository(SearchRepositoryBase):
|
||||
pass
|
||||
|
||||
try:
|
||||
import sqlite_vec # type: ignore[import-not-found]
|
||||
import sqlite_vec
|
||||
except ImportError as exc:
|
||||
raise SemanticDependenciesMissingError(
|
||||
"sqlite-vec package is missing. "
|
||||
"Install/update basic-memory to include semantic dependencies: "
|
||||
"pip install -U basic-memory"
|
||||
"Install semantic extras: pip install 'basic-memory[semantic]'"
|
||||
) from exc
|
||||
|
||||
async with self._sqlite_vec_lock:
|
||||
@@ -381,7 +374,7 @@ class SQLiteSearchRepository(SearchRepositoryBase):
|
||||
if self._vector_tables_initialized:
|
||||
return
|
||||
|
||||
logger.debug("Ensuring SQLite vector tables exist for semantic search")
|
||||
logger.info("Ensuring SQLite vector tables exist for semantic search")
|
||||
|
||||
async with db.scoped_session(self.session_maker) as session:
|
||||
await self._ensure_sqlite_vec_loaded(session)
|
||||
@@ -434,24 +427,19 @@ class SQLiteSearchRepository(SearchRepositoryBase):
|
||||
await session.execute(create_sqlite_search_vector_embeddings(self._vector_dimensions))
|
||||
await session.commit()
|
||||
|
||||
logger.debug(f"SQLite vector tables ready (dimensions={self._vector_dimensions})")
|
||||
logger.info(f"SQLite vector tables ready (dimensions={self._vector_dimensions})")
|
||||
self._vector_tables_initialized = True
|
||||
|
||||
async def _prepare_vector_session(self, session: AsyncSession) -> None:
|
||||
"""Load sqlite-vec extension for the session."""
|
||||
await self._ensure_sqlite_vec_loaded(session)
|
||||
|
||||
# sqlite-vec hard limit for knn k parameter
|
||||
SQLITE_VEC_MAX_K = 4096
|
||||
|
||||
async def _run_vector_query(
|
||||
self,
|
||||
session: AsyncSession,
|
||||
query_embedding: list[float],
|
||||
candidate_limit: int,
|
||||
) -> list[dict]:
|
||||
# Constraint: sqlite-vec enforces k <= 4096 for knn queries
|
||||
vector_k = min(candidate_limit, self.SQLITE_VEC_MAX_K)
|
||||
query_embedding_json = json.dumps(query_embedding)
|
||||
vector_result = await session.execute(
|
||||
text(
|
||||
@@ -461,18 +449,17 @@ class SQLiteSearchRepository(SearchRepositoryBase):
|
||||
" WHERE embedding MATCH :query_embedding "
|
||||
" AND k = :vector_k"
|
||||
") "
|
||||
"SELECT c.entity_id, c.chunk_key, c.chunk_text, vector_matches.distance AS best_distance "
|
||||
"SELECT c.entity_id, c.chunk_key, vector_matches.distance AS best_distance "
|
||||
"FROM vector_matches "
|
||||
"JOIN search_vector_chunks c ON c.id = vector_matches.rowid "
|
||||
"WHERE c.project_id = :project_id "
|
||||
"ORDER BY best_distance ASC "
|
||||
"LIMIT :candidate_limit"
|
||||
"LIMIT :vector_k"
|
||||
),
|
||||
{
|
||||
"query_embedding": query_embedding_json,
|
||||
"project_id": self.project_id,
|
||||
"vector_k": vector_k,
|
||||
"candidate_limit": candidate_limit,
|
||||
"vector_k": candidate_limit,
|
||||
},
|
||||
)
|
||||
return [dict(row) for row in vector_result.mappings().all()]
|
||||
@@ -555,14 +542,6 @@ class SQLiteSearchRepository(SearchRepositoryBase):
|
||||
async def _update_timestamp_sql(self) -> str:
|
||||
return "CURRENT_TIMESTAMP" # pragma: no cover
|
||||
|
||||
def _distance_to_similarity(self, distance: float) -> float:
|
||||
"""Convert L2 distance to cosine similarity for normalized embeddings.
|
||||
|
||||
sqlite-vec vec0 returns Euclidean (L2) distance by default.
|
||||
For unit-normalized vectors: L2² = 2·(1 - cos_sim), so cos_sim = 1 - L2²/2.
|
||||
"""
|
||||
return max(0.0, 1.0 - (distance * distance) / 2.0)
|
||||
|
||||
def _orphan_detection_sql(self) -> str:
|
||||
"""SQLite sqlite-vec uses rowid-based embedding table."""
|
||||
return (
|
||||
@@ -597,7 +576,7 @@ class SQLiteSearchRepository(SearchRepositoryBase):
|
||||
permalink: Optional[str] = None,
|
||||
permalink_match: Optional[str] = None,
|
||||
title: Optional[str] = None,
|
||||
note_types: Optional[List[str]] = None,
|
||||
types: Optional[List[str]] = None,
|
||||
after_date: Optional[datetime] = None,
|
||||
search_item_types: Optional[List[SearchItemType]] = None,
|
||||
metadata_filters: Optional[dict] = None,
|
||||
@@ -613,7 +592,7 @@ class SQLiteSearchRepository(SearchRepositoryBase):
|
||||
permalink=permalink,
|
||||
permalink_match=permalink_match,
|
||||
title=title,
|
||||
note_types=note_types,
|
||||
types=types,
|
||||
after_date=after_date,
|
||||
search_item_types=search_item_types,
|
||||
metadata_filters=metadata_filters,
|
||||
@@ -627,7 +606,6 @@ class SQLiteSearchRepository(SearchRepositoryBase):
|
||||
|
||||
# --- FTS mode (SQLite-specific) ---
|
||||
conditions = []
|
||||
match_conditions = []
|
||||
params = {}
|
||||
order_by_clause = ""
|
||||
from_clause = "search_index"
|
||||
@@ -642,7 +620,7 @@ class SQLiteSearchRepository(SearchRepositoryBase):
|
||||
# Use _prepare_search_term to handle both Boolean and non-Boolean queries
|
||||
processed_text = self._prepare_search_term(search_text.strip())
|
||||
params["text"] = processed_text
|
||||
match_conditions.append(
|
||||
conditions.append(
|
||||
"(search_index.title MATCH :text OR search_index.content_stems MATCH :text)"
|
||||
)
|
||||
|
||||
@@ -650,7 +628,7 @@ class SQLiteSearchRepository(SearchRepositoryBase):
|
||||
if title:
|
||||
title_text = self._prepare_search_term(title.strip(), is_prefix=False)
|
||||
params["title_text"] = title_text
|
||||
match_conditions.append("search_index.title MATCH :title_text")
|
||||
conditions.append("search_index.title MATCH :title_text")
|
||||
|
||||
# Handle permalink exact search
|
||||
if permalink:
|
||||
@@ -673,26 +651,18 @@ class SQLiteSearchRepository(SearchRepositoryBase):
|
||||
else:
|
||||
permalink_text = self._prepare_search_term(permalink_text, is_prefix=False)
|
||||
params["permalink"] = permalink_text
|
||||
match_conditions.append("search_index.permalink MATCH :permalink")
|
||||
conditions.append("search_index.permalink MATCH :permalink")
|
||||
|
||||
# Handle entity type filter (parameterized for defense-in-depth)
|
||||
# Handle entity type filter
|
||||
if search_item_types:
|
||||
type_placeholders = []
|
||||
for idx, t in enumerate(search_item_types):
|
||||
param_name = f"search_type_{idx}"
|
||||
params[param_name] = t.value
|
||||
type_placeholders.append(f":{param_name}")
|
||||
conditions.append(f"search_index.type IN ({', '.join(type_placeholders)})")
|
||||
type_list = ", ".join(f"'{t.value}'" for t in search_item_types)
|
||||
conditions.append(f"search_index.type IN ({type_list})")
|
||||
|
||||
# Handle note type filter (frontmatter type field, parameterized)
|
||||
if note_types:
|
||||
type_placeholders = []
|
||||
for idx, t in enumerate(note_types):
|
||||
param_name = f"note_type_{idx}"
|
||||
params[param_name] = t
|
||||
type_placeholders.append(f":{param_name}")
|
||||
# Handle type filter
|
||||
if types:
|
||||
type_list = ", ".join(f"'{t}'" for t in types)
|
||||
conditions.append(
|
||||
f"json_extract(search_index.metadata, '$.note_type') IN ({', '.join(type_placeholders)})"
|
||||
f"json_extract(search_index.metadata, '$.entity_type') IN ({type_list})"
|
||||
)
|
||||
|
||||
# Handle date filter using datetime() for proper comparison
|
||||
@@ -785,18 +755,6 @@ class SQLiteSearchRepository(SearchRepositoryBase):
|
||||
conditions.append(f"{compare_expr} {operator} :{value_param}")
|
||||
continue
|
||||
|
||||
# Trigger: SQLite FTS MATCH predicates combined with JOINs can fail with
|
||||
# "unable to use function MATCH in the requested context".
|
||||
# Why: MATCH needs to run in an FTS-valid context.
|
||||
# Outcome: evaluate MATCH clauses in an FTS subquery and filter outer rows by rowid.
|
||||
if metadata_filters and match_conditions:
|
||||
match_where = " AND ".join(match_conditions)
|
||||
conditions.append(
|
||||
f"search_index.rowid IN (SELECT rowid FROM search_index WHERE {match_where})"
|
||||
)
|
||||
else:
|
||||
conditions.extend(match_conditions)
|
||||
|
||||
# Always filter by project_id
|
||||
params["project_id"] = self.project_id
|
||||
conditions.append("search_index.project_id = :project_id")
|
||||
|
||||
@@ -30,14 +30,14 @@ class FieldFrequency:
|
||||
percentage: float
|
||||
sample_values: list[str] = field(default_factory=list)
|
||||
is_array: bool = False # True if typically appears multiple times per note
|
||||
target_type: str | None = None # For relations, the most common target note type
|
||||
target_type: str | None = None # For relations, the most common target entity type
|
||||
|
||||
|
||||
@dataclass
|
||||
class InferenceResult:
|
||||
"""Complete inference result with frequency analysis and suggested schema."""
|
||||
|
||||
note_type: str
|
||||
entity_type: str
|
||||
notes_analyzed: int
|
||||
field_frequencies: list[FieldFrequency]
|
||||
suggested_schema: dict # Ready-to-use Picoschema YAML dict
|
||||
@@ -65,7 +65,7 @@ class RelationData:
|
||||
|
||||
relation_type: str
|
||||
target_name: str
|
||||
target_note_type: str | None = None
|
||||
target_entity_type: str | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -85,7 +85,7 @@ class NoteData:
|
||||
|
||||
|
||||
def infer_schema(
|
||||
note_type: str,
|
||||
entity_type: str,
|
||||
notes: list[NoteData],
|
||||
required_threshold: float = 0.95,
|
||||
optional_threshold: float = 0.25,
|
||||
@@ -98,7 +98,7 @@ def infer_schema(
|
||||
appear less frequently become optional.
|
||||
|
||||
Args:
|
||||
note_type: The note type being analyzed (e.g., "person").
|
||||
entity_type: The entity type being analyzed (e.g., "Person").
|
||||
notes: List of NoteData objects to analyze.
|
||||
required_threshold: Frequency at or above which a field is required (default 0.95).
|
||||
optional_threshold: Frequency at or above which a field is optional (default 0.25).
|
||||
@@ -110,7 +110,7 @@ def infer_schema(
|
||||
total = len(notes)
|
||||
if total == 0:
|
||||
return InferenceResult(
|
||||
note_type=note_type,
|
||||
entity_type=entity_type,
|
||||
notes_analyzed=0,
|
||||
field_frequencies=[],
|
||||
suggested_schema={},
|
||||
@@ -145,7 +145,7 @@ def infer_schema(
|
||||
)
|
||||
|
||||
return InferenceResult(
|
||||
note_type=note_type,
|
||||
entity_type=entity_type,
|
||||
notes_analyzed=total,
|
||||
field_frequencies=all_frequencies,
|
||||
suggested_schema=suggested_schema,
|
||||
@@ -255,8 +255,8 @@ def analyze_relations(
|
||||
# Track target entity types from individual relations (not the source note)
|
||||
target_counter = rel_target_types.setdefault(rel_type, Counter())
|
||||
for rel in note_rel_objects[rel_type]:
|
||||
if rel.target_note_type:
|
||||
target_counter[rel.target_note_type] += 1
|
||||
if rel.target_entity_type:
|
||||
target_counter[rel.target_entity_type] += 1
|
||||
|
||||
frequencies: list[FieldFrequency] = []
|
||||
for rel_type, count in rel_note_count.most_common():
|
||||
|
||||
@@ -14,7 +14,6 @@ Syntax reference:
|
||||
EntityName as type (capitalized) # entity reference
|
||||
"""
|
||||
|
||||
import re
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
|
||||
@@ -50,7 +49,6 @@ class SchemaDefinition:
|
||||
version: int # Schema version
|
||||
fields: list[SchemaField] # Parsed fields
|
||||
validation_mode: str # "warn" | "strict" | "off"
|
||||
frontmatter_fields: list[SchemaField] = field(default_factory=list) # From settings.frontmatter
|
||||
|
||||
|
||||
# --- Built-in scalar types ---
|
||||
@@ -125,31 +123,6 @@ def _is_entity_ref_type(type_str: str) -> bool:
|
||||
return len(type_str) > 0 and type_str[0].isupper()
|
||||
|
||||
|
||||
# --- Enum String Parsing ---
|
||||
|
||||
|
||||
def _parse_enum_string(value: str) -> tuple[list[str], str | None]:
|
||||
"""Parse a string-typed enum value into enum values and optional description.
|
||||
|
||||
When picoschema enum values are quoted in YAML frontmatter (required when a
|
||||
description follows the list), YAML parses the whole thing as a string. This
|
||||
function extracts the enum values and description from that string.
|
||||
|
||||
Examples:
|
||||
"[active, blocked, done], current state" -> (['active', 'blocked', 'done'], 'current state')
|
||||
"[active, blocked]" -> (['active', 'blocked'], None)
|
||||
"active" -> (['active'], None)
|
||||
"""
|
||||
# Match bracketed list with optional trailing description
|
||||
m = re.match(r"\[([^\]]+)\](?:\s*,\s*(.+))?", value)
|
||||
if m:
|
||||
items = [item.strip() for item in m.group(1).split(",")]
|
||||
description = m.group(2).strip() if m.group(2) else None
|
||||
return items, description
|
||||
# Plain string — single enum value
|
||||
return [value.strip()], None
|
||||
|
||||
|
||||
# --- Main Parser ---
|
||||
|
||||
|
||||
@@ -173,25 +146,18 @@ def parse_picoschema(yaml_dict: dict) -> list[SchemaField]:
|
||||
name, required, is_array, is_enum, is_object = _parse_field_key(key)
|
||||
|
||||
# --- Enum fields ---
|
||||
# Trigger: value is a list or a string containing bracketed enum values
|
||||
# Why: enums declare allowed values directly as a YAML list, or as a quoted
|
||||
# string when a description follows (e.g., "[a, b], desc" must be quoted
|
||||
# in YAML to avoid parse errors)
|
||||
# Trigger: value is a list (e.g., [active, inactive])
|
||||
# Why: enums declare allowed values directly as a YAML list
|
||||
# Outcome: SchemaField with is_enum=True and enum_values populated
|
||||
if is_enum:
|
||||
description = None
|
||||
if isinstance(value, list):
|
||||
enum_values = [str(v) for v in value]
|
||||
else:
|
||||
enum_values, description = _parse_enum_string(str(value))
|
||||
enum_values = value if isinstance(value, list) else [str(value)]
|
||||
fields.append(
|
||||
SchemaField(
|
||||
name=name,
|
||||
type="enum",
|
||||
required=required,
|
||||
is_enum=True,
|
||||
enum_values=enum_values,
|
||||
description=description,
|
||||
enum_values=[str(v) for v in enum_values],
|
||||
)
|
||||
)
|
||||
continue
|
||||
@@ -262,19 +228,9 @@ def parse_schema_note(frontmatter: dict) -> SchemaDefinition:
|
||||
|
||||
fields = parse_picoschema(schema_dict)
|
||||
|
||||
# --- Frontmatter validation rules ---
|
||||
# Trigger: settings.frontmatter is a dict of Picoschema field declarations
|
||||
# Why: allows schema notes to validate frontmatter keys (tags, status, etc.)
|
||||
# Outcome: frontmatter_fields populated using same parser as schema fields
|
||||
frontmatter_dict = settings.get("frontmatter") if isinstance(settings, dict) else None
|
||||
frontmatter_fields = (
|
||||
parse_picoschema(frontmatter_dict) if isinstance(frontmatter_dict, dict) else []
|
||||
)
|
||||
|
||||
return SchemaDefinition(
|
||||
entity=entity,
|
||||
version=version,
|
||||
fields=fields,
|
||||
validation_mode=validation_mode,
|
||||
frontmatter_fields=frontmatter_fields,
|
||||
)
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user