mirror of
https://github.com/basicmachines-co/basic-memory
synced 2026-06-21 13:47:35 +00:00
Compare commits
32 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 4e10e21a6c | |||
| 69808b23ca | |||
| 6f207c20c0 | |||
| cff31c5797 | |||
| 2d1ccfa36c | |||
| a2e0f935d6 | |||
| e6b98a15c7 | |||
| 733c4f7514 | |||
| cfa70004be | |||
| 7696fca826 | |||
| 98a2a3cbaf | |||
| 01cbad1dbe | |||
| a4e0422926 | |||
| 552a835669 | |||
| 888e3c2909 | |||
| d1320f671e | |||
| 94bdfe77e4 | |||
| 4791e19685 | |||
| 36848410a1 | |||
| a77b51a28e | |||
| 1a6a65571e | |||
| c8b00449d2 | |||
| 013864ebf0 | |||
| dd91b49054 | |||
| 7c96a0777d | |||
| 148e07c580 | |||
| 21334cc29b | |||
| db60942267 | |||
| 7bfac158df | |||
| 87616924ff | |||
| 5cb0502ed2 | |||
| a94a717b1b |
+32
-10
@@ -6,7 +6,6 @@ concurrency:
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ "main" ]
|
||||
pull_request:
|
||||
branches: [ "main" ]
|
||||
|
||||
@@ -52,7 +51,6 @@ jobs:
|
||||
test-sqlite-unit:
|
||||
name: Test SQLite Unit (${{ matrix.os }}, Python ${{ matrix.python-version }})
|
||||
timeout-minutes: 30
|
||||
needs: [static-checks]
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
@@ -99,7 +97,6 @@ jobs:
|
||||
test-sqlite-integration:
|
||||
name: Test SQLite Integration (${{ matrix.os }}, Python ${{ matrix.python-version }})
|
||||
timeout-minutes: 45
|
||||
needs: [static-checks]
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
@@ -146,7 +143,6 @@ jobs:
|
||||
test-postgres-unit:
|
||||
name: Test Postgres Unit (Python ${{ matrix.python-version }})
|
||||
timeout-minutes: 30
|
||||
needs: [static-checks]
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
@@ -155,8 +151,22 @@ jobs:
|
||||
- python-version: "3.13"
|
||||
- python-version: "3.14"
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
# Note: No services section needed - testcontainers handles Postgres in Docker
|
||||
services:
|
||||
postgres:
|
||||
image: pgvector/pgvector:pg16
|
||||
env:
|
||||
POSTGRES_USER: basic_memory_user
|
||||
POSTGRES_PASSWORD: dev_password
|
||||
POSTGRES_DB: basic_memory_test
|
||||
ports:
|
||||
- 5432:5432
|
||||
options: >-
|
||||
--health-cmd "pg_isready -U basic_memory_user -d basic_memory_test"
|
||||
--health-interval 10s
|
||||
--health-timeout 5s
|
||||
--health-retries 5
|
||||
env:
|
||||
BASIC_MEMORY_TEST_POSTGRES_URL: postgresql://basic_memory_user:dev_password@127.0.0.1:5432/basic_memory_test
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
@@ -190,7 +200,6 @@ jobs:
|
||||
test-postgres-integration:
|
||||
name: Test Postgres Integration (Python ${{ matrix.python-version }})
|
||||
timeout-minutes: 45
|
||||
needs: [static-checks]
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
@@ -199,8 +208,22 @@ jobs:
|
||||
- python-version: "3.13"
|
||||
- python-version: "3.14"
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
# Note: No services section needed - testcontainers handles Postgres in Docker
|
||||
services:
|
||||
postgres:
|
||||
image: pgvector/pgvector:pg16
|
||||
env:
|
||||
POSTGRES_USER: basic_memory_user
|
||||
POSTGRES_PASSWORD: dev_password
|
||||
POSTGRES_DB: basic_memory_test
|
||||
ports:
|
||||
- 5432:5432
|
||||
options: >-
|
||||
--health-cmd "pg_isready -U basic_memory_user -d basic_memory_test"
|
||||
--health-interval 10s
|
||||
--health-timeout 5s
|
||||
--health-retries 5
|
||||
env:
|
||||
BASIC_MEMORY_TEST_POSTGRES_URL: postgresql://basic_memory_user:dev_password@127.0.0.1:5432/basic_memory_test
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
@@ -234,7 +257,6 @@ jobs:
|
||||
test-semantic:
|
||||
name: Test Semantic (Python 3.12)
|
||||
timeout-minutes: 45
|
||||
needs: [static-checks]
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
|
||||
@@ -442,5 +442,9 @@ With GitHub integration, the development workflow includes:
|
||||
3. **Branch management** - Claude can create feature branches for implementations
|
||||
4. **Documentation maintenance** - Claude can keep documentation updated as the code evolves
|
||||
5. **Code Commits**: ALWAYS sign off commits with `git commit -s`
|
||||
6. **Pull Request Titles**: PR titles must follow the semantic format enforced by `.github/workflows/pr-title.yml`: `type(scope): summary`
|
||||
- Allowed types: `feat`, `fix`, `chore`, `docs`, `style`, `refactor`, `perf`, `test`, `build`, `ci`
|
||||
- Allowed scopes: `core`, `cli`, `api`, `mcp`, `sync`, `ui`, `deps`, `installer`
|
||||
- Example: `fix(cli): propagate cloud workspace routing`
|
||||
|
||||
This level of integration represents a new paradigm in AI-human collaboration, where the AI assistant becomes a full-fledged team member rather than just a tool for generating code snippets.
|
||||
|
||||
@@ -2,6 +2,49 @@
|
||||
|
||||
## Unreleased
|
||||
|
||||
## v0.20.3 (2026-03-26)
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- **#698**: CLI cloud commands now use API key when configured
|
||||
- `get_authenticated_headers()` only checked OAuth tokens, ignoring `config.cloud_api_key`
|
||||
- All CLI cloud commands (`upload`, `status`, `snapshot`, `restore`, etc.) failed for API-key-only users while MCP tools worked fine
|
||||
- Now mirrors the same credential priority as MCP: API key first, OAuth fallback
|
||||
- Fixes `bm cloud upload --project` returning "project does not exist" when authenticated with `bmc_*` API key
|
||||
|
||||
## v0.20.2 (2026-03-10)
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- Fix auto-update Homebrew detection: `brew outdated` exits 1 when a formula is outdated, not on error
|
||||
- Previously treated exit code 1 as a failure, causing "Automatic update check failed" instead of detecting the available update
|
||||
|
||||
## v0.20.1 (2026-03-10)
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- **#661**: Fix `bm project list` MCP column to show transport type (stdio/https) instead of DB presence
|
||||
- Renamed "MCP (stdio)" column to "MCP"
|
||||
- Shows actual routing mode: `stdio` for local, `https` for cloud projects
|
||||
- Clears local path display for cloud-mode projects
|
||||
- **#662**: Invalidate config cache when file is modified by another process
|
||||
- Adds mtime-based cache validation to `ConfigManager.load_config()`
|
||||
- Long-lived processes (MCP stdio server) now detect external config changes
|
||||
- Fixes `bm project set-cloud` having no effect on running MCP server
|
||||
|
||||
## v0.20.0 (2026-03-10)
|
||||
|
||||
### Features
|
||||
|
||||
- **#643**: Default-on auto-update system and `bm update` command
|
||||
- Automatic background update checks for CLI installs (uv tool, Homebrew)
|
||||
- Install-source detection (homebrew, uv_tool, uvx, unknown) with uvx skip behavior
|
||||
- Periodic check gating via `auto_update_last_checked_at` + `update_check_interval` config
|
||||
- Manager-specific update flows: Homebrew (`brew upgrade`) and uv tool (`uv tool upgrade`)
|
||||
- Silent, non-blocking MCP behavior via daemon thread before server run
|
||||
- Manual commands: `bm update` (force check + apply) and `bm update --check` (check only)
|
||||
- New config fields: `auto_update`, `update_check_interval`, `auto_update_last_checked_at`
|
||||
|
||||
## v0.19.2 (2026-03-09)
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
@@ -75,6 +75,36 @@ uv tool install basic-memory
|
||||
|
||||
You can view shared context via files in `~/basic-memory` (default directory location).
|
||||
|
||||
## Automatic Updates
|
||||
|
||||
Basic Memory includes a default-on auto-update flow for CLI installs.
|
||||
|
||||
- **Auto-install supported:** `uv tool` and Homebrew installs
|
||||
- **Default check interval:** every 24 hours (`86400` seconds)
|
||||
- **MCP-safe behavior:** update checks run silently in `basic-memory mcp` mode
|
||||
- **`uvx` behavior:** skipped (runtime is ephemeral and managed by `uvx`)
|
||||
|
||||
Manual update commands:
|
||||
|
||||
```bash
|
||||
# Check now and install if supported
|
||||
bm update
|
||||
|
||||
# Check only, do not install
|
||||
bm update --check
|
||||
```
|
||||
|
||||
Config options in `~/.basic-memory/config.json`:
|
||||
|
||||
```json
|
||||
{
|
||||
"auto_update": true,
|
||||
"update_check_interval": 86400
|
||||
}
|
||||
```
|
||||
|
||||
To disable automatic updates, set `"auto_update": false`.
|
||||
|
||||
## Why Basic Memory?
|
||||
|
||||
Most LLM interactions are ephemeral - you ask a question, get an answer, and everything is forgotten. Each conversation
|
||||
|
||||
@@ -0,0 +1,499 @@
|
||||
# Logfire Instrumentation Strategy
|
||||
|
||||
## Why
|
||||
|
||||
We want Logfire in Basic Memory for two specific use cases:
|
||||
|
||||
1. Local development and performance investigation
|
||||
2. Cloud deployments where Basic Memory runs inside Basic Memory Cloud
|
||||
|
||||
This instrumentation must be:
|
||||
|
||||
- Disabled by default
|
||||
- Useful when enabled
|
||||
- Safe for local-first users
|
||||
- Searchable in Logfire over time
|
||||
|
||||
The previous integration added telemetry, but it leaned too much on generic framework instrumentation. That created noisy spans with weak names and made the trace view harder to navigate. This strategy favors manual instrumentation around Basic Memory's real units of work.
|
||||
|
||||
## Core Principles
|
||||
|
||||
### 1. Default-off
|
||||
|
||||
Basic Memory should ship with Logfire disabled unless the operator explicitly enables it.
|
||||
|
||||
That means:
|
||||
|
||||
- no required token for normal local usage
|
||||
- no surprise outbound telemetry
|
||||
- no behavior change for existing users
|
||||
|
||||
### 2. Manual spans over automatic framework spans
|
||||
|
||||
We should not rely on broad auto-instrumentation for FastAPI, MCP, SQLAlchemy, or HTTP as the primary experience.
|
||||
|
||||
Why:
|
||||
|
||||
- auto-generated span names are often generic
|
||||
- routes and middleware produce too many low-signal spans
|
||||
- it becomes harder to answer product questions like "why was `write_note` slow?" or "where did sync time go?"
|
||||
|
||||
The preferred model is:
|
||||
|
||||
- one meaningful root span per high-level operation
|
||||
- a small number of child spans for important phases
|
||||
- optional targeted instrumentation only where it adds clear value
|
||||
|
||||
### 3. Logs must live inside traces
|
||||
|
||||
Basic Memory already uses `loguru` pervasively. The Logfire integration should preserve that and make those logs visible inside the active trace/span context.
|
||||
|
||||
If traces exist but the logs are detached from them, the integration is not doing its job.
|
||||
|
||||
### 4. Stable names, selective attributes
|
||||
|
||||
Span names should describe the operation class, not the specific input.
|
||||
|
||||
Good:
|
||||
|
||||
- `mcp.tool.write_note`
|
||||
- `sync.project.scan`
|
||||
- `search.execute`
|
||||
- `routing.resolve_project`
|
||||
|
||||
Bad:
|
||||
|
||||
- `Searching for "foo bar baz"`
|
||||
- `POST /v2/projects/123/search/`
|
||||
- `write note to /specs/api.md`
|
||||
|
||||
Dynamic values belong in attributes, not in the span name.
|
||||
|
||||
## What We Should Not Do
|
||||
|
||||
### Avoid broad FastAPI auto-instrumentation
|
||||
|
||||
We should not turn on `instrument_fastapi()` and treat that as the main telemetry story.
|
||||
|
||||
It may still be useful in narrowly scoped debugging, but it should not define the production trace shape. The meaningful root spans should come from Basic Memory's own entrypoints and service boundaries.
|
||||
|
||||
### Avoid per-file spans by default
|
||||
|
||||
`sync` can process many files. A span per file will explode trace cardinality and make performance views noisy.
|
||||
|
||||
Default behavior should be:
|
||||
|
||||
- one span for the project sync
|
||||
- child spans for scan, move handling, delete handling, markdown sync batch, relation resolution, embedding sync, watermark update
|
||||
- per-file spans only for failures or very slow outliers
|
||||
|
||||
### Avoid high-cardinality attributes on every span
|
||||
|
||||
Do not attach large or highly variable values everywhere:
|
||||
|
||||
- raw note content
|
||||
- file bodies
|
||||
- long search text
|
||||
- arbitrary metadata blobs
|
||||
- unique IDs that make every span shape distinct
|
||||
|
||||
Prefer compact, queryable attributes:
|
||||
|
||||
- `project_name`
|
||||
- `workspace_id`
|
||||
- `route_mode`
|
||||
- `scan_type`
|
||||
- `file_count`
|
||||
- `result_count`
|
||||
- `search_type`
|
||||
- `retrieval_mode`
|
||||
- `duration_ms`
|
||||
|
||||
## Proposed Architecture
|
||||
|
||||
Add a dedicated telemetry module in core Basic Memory, separate from logging setup.
|
||||
|
||||
Suggested shape:
|
||||
|
||||
```python
|
||||
# basic_memory/telemetry.py
|
||||
|
||||
def configure_telemetry(service_name: str, *, enable_logfire: bool) -> None: ...
|
||||
def telemetry_enabled() -> bool: ...
|
||||
def span(name: str, **attrs): ...
|
||||
def bind_telemetry_context(**attrs): ...
|
||||
```
|
||||
|
||||
This module should:
|
||||
|
||||
- configure Logfire only when explicitly enabled
|
||||
- set up the Logfire `loguru` handler
|
||||
- expose lightweight helpers so application code does not import `logfire` directly everywhere
|
||||
- degrade cleanly to no-op behavior when disabled
|
||||
|
||||
This keeps the rest of the codebase readable and makes it easy to reason about what telemetry is doing.
|
||||
|
||||
## Logging Integration Strategy
|
||||
|
||||
### Goal
|
||||
|
||||
When a span is active, logs emitted through `loguru` during that operation should show up in the same trace.
|
||||
|
||||
### Preferred design
|
||||
|
||||
1. Configure Logfire once in the telemetry bootstrap
|
||||
2. Add the Logfire `loguru` handler to the existing `loguru` configuration
|
||||
3. At operation boundaries, bind stable contextual fields with `loguru`
|
||||
4. Let logs emitted inside the span inherit the active trace context
|
||||
|
||||
### Context to bind
|
||||
|
||||
Bind only the fields that help correlate work across the system:
|
||||
|
||||
- `service_name`
|
||||
- `entrypoint`
|
||||
- `project_name`
|
||||
- `workspace_id`
|
||||
- `route_mode`
|
||||
- `tool_name`
|
||||
- `command_name`
|
||||
|
||||
This binding should happen at the root of an operation, not deep in leaf functions.
|
||||
|
||||
### Important nuance
|
||||
|
||||
We should not try to encode the entire trace model into logger extras. The logger context should be a human-meaningful slice of the active operation. Trace linkage comes from the active Logfire/OpenTelemetry context; logger extras are there to improve searchability and readability.
|
||||
|
||||
## Span Model
|
||||
|
||||
### Root spans
|
||||
|
||||
Each user-visible or system-visible operation should get one root span.
|
||||
|
||||
Examples:
|
||||
|
||||
- `cli.command.status`
|
||||
- `cli.command.project_sync`
|
||||
- `api.request.search`
|
||||
- `mcp.tool.write_note`
|
||||
- `mcp.tool.read_note`
|
||||
- `mcp.tool.search_notes`
|
||||
- `sync.project.run`
|
||||
- `db.semantic_backfill`
|
||||
|
||||
### Child spans
|
||||
|
||||
Child spans should represent real phases whose duration we care about.
|
||||
|
||||
Examples:
|
||||
|
||||
- `routing.client_session`
|
||||
- `routing.resolve_project`
|
||||
- `routing.resolve_workspace`
|
||||
- `api.search.execute`
|
||||
- `sync.project.scan`
|
||||
- `sync.project.detect_moves`
|
||||
- `sync.project.apply_changes`
|
||||
- `sync.project.resolve_relations`
|
||||
- `sync.project.sync_embeddings`
|
||||
- `sync.file.markdown`
|
||||
- `sync.file.regular`
|
||||
- `search.execute`
|
||||
- `search.relaxed_fts_retry`
|
||||
- `db.init`
|
||||
- `db.migrate`
|
||||
|
||||
### Span naming rules
|
||||
|
||||
- Use dot-separated names
|
||||
- Start with subsystem
|
||||
- Keep the verb at the end
|
||||
- Keep names stable across runs
|
||||
- Never include request-specific text in the span name
|
||||
|
||||
## Attribute Taxonomy
|
||||
|
||||
### Required attributes on root spans
|
||||
|
||||
Every root span should have a small common set:
|
||||
|
||||
- `service_name`
|
||||
- `entrypoint`
|
||||
- `project_name` when applicable
|
||||
- `workspace_id` when applicable
|
||||
- `route_mode` with values like `local_asgi`, `cloud_proxy`, `factory`
|
||||
|
||||
### Operation-specific attributes
|
||||
|
||||
Examples:
|
||||
|
||||
For search:
|
||||
|
||||
- `search_type`
|
||||
- `retrieval_mode`
|
||||
- `page`
|
||||
- `page_size`
|
||||
- `result_count`
|
||||
- `fallback_used`
|
||||
|
||||
For sync:
|
||||
|
||||
- `scan_type`
|
||||
- `force_full`
|
||||
- `new_count`
|
||||
- `modified_count`
|
||||
- `deleted_count`
|
||||
- `move_count`
|
||||
- `skipped_count`
|
||||
- `embeddings_enabled`
|
||||
|
||||
For note operations:
|
||||
|
||||
- `tool_name`
|
||||
- `note_type`
|
||||
- `directory`
|
||||
- `overwrite`
|
||||
- `output_format`
|
||||
|
||||
### Attributes to avoid by default
|
||||
|
||||
- full `query.text`
|
||||
- full note titles if they create privacy or cardinality issues
|
||||
- file content
|
||||
- raw frontmatter
|
||||
- raw HTTP bodies
|
||||
|
||||
If we need richer payloads for a local debugging session, that should be an explicit temporary mode, not the default telemetry shape.
|
||||
|
||||
## Instrumentation Plan By Layer
|
||||
|
||||
### 1. Entrypoints
|
||||
|
||||
Instrument these first:
|
||||
|
||||
- `cli.app` callback and major commands
|
||||
- API lifespan and selected routers
|
||||
- MCP server lifespan
|
||||
- MCP tool entrypoints
|
||||
|
||||
Why:
|
||||
|
||||
- this establishes clean root spans
|
||||
- it gives us trace boundaries that match how users think about the product
|
||||
|
||||
### 2. Routing and context resolution
|
||||
|
||||
Instrument:
|
||||
|
||||
- client routing decisions
|
||||
- workspace resolution
|
||||
- project resolution
|
||||
- default-project fallback
|
||||
|
||||
Why:
|
||||
|
||||
- Basic Memory has local/cloud/per-project routing logic
|
||||
- when something is slow or surprising, we need to know which path was taken
|
||||
|
||||
### 3. Sync and indexing
|
||||
|
||||
This is the highest-value area to instrument deeply.
|
||||
|
||||
Instrument:
|
||||
|
||||
- sync root
|
||||
- scan strategy decision
|
||||
- filesystem scan
|
||||
- move detection
|
||||
- delete handling
|
||||
- markdown sync phase
|
||||
- relation resolution
|
||||
- vector embedding sync
|
||||
- scan watermark update
|
||||
|
||||
Why:
|
||||
|
||||
- this is where performance work will happen
|
||||
- cloud and local both benefit from this visibility
|
||||
|
||||
### 4. Search
|
||||
|
||||
Instrument:
|
||||
|
||||
- search execution
|
||||
- retrieval mode
|
||||
- relaxed FTS fallback
|
||||
- result shaping
|
||||
|
||||
Why:
|
||||
|
||||
- search is user-facing and latency-sensitive
|
||||
- hybrid/vector/FTS paths need to be distinguishable
|
||||
|
||||
### 5. Database and initialization
|
||||
|
||||
Instrument selectively:
|
||||
|
||||
- DB init
|
||||
- migrations
|
||||
- semantic backfill
|
||||
- connection mode selection
|
||||
|
||||
Avoid full automatic SQL span firehose by default.
|
||||
|
||||
## Recommended Rollout Phases
|
||||
|
||||
## Task List
|
||||
|
||||
- [x] Phase 1: Bootstrap and config gating
|
||||
- [x] Phase 2: Root spans for entrypoints and primary operations
|
||||
- [x] Phase 3: Child spans for sync, search, and routing
|
||||
- [x] Phase 4: Failure-focused detail and final verification
|
||||
- [x] Phase 5: Loguru context binding and scoped context inheritance
|
||||
|
||||
## Recommended Rollout Phases
|
||||
|
||||
### Phase 1: Bootstrap and config gating
|
||||
|
||||
Add:
|
||||
|
||||
- telemetry bootstrap module
|
||||
- config/env gating
|
||||
- `loguru` + Logfire handler integration
|
||||
|
||||
This gives immediate value with low noise.
|
||||
|
||||
### Phase 2: Root spans for entrypoints and primary operations
|
||||
|
||||
Add:
|
||||
|
||||
- root spans for CLI, API, MCP, and main MCP tools
|
||||
- stable root attributes for project, workspace, route mode, and operation type
|
||||
|
||||
This gives us clean top-level traces that match how users think about the product.
|
||||
|
||||
### Phase 3: Child spans for sync, search, and routing
|
||||
|
||||
Add child spans to:
|
||||
|
||||
- sync
|
||||
- search
|
||||
- routing
|
||||
|
||||
This is the main performance-investigation layer.
|
||||
|
||||
### Phase 4: Failure-focused detail
|
||||
|
||||
Add selective deeper spans/log enrichment for:
|
||||
|
||||
- sync failures
|
||||
- relation resolution failures
|
||||
- slow file operations
|
||||
- cloud routing/auth failures
|
||||
|
||||
This keeps normal traces clean while improving debuggability.
|
||||
|
||||
### Phase 5: Loguru context binding and scoped context inheritance
|
||||
|
||||
Add:
|
||||
|
||||
- context-local telemetry state in `basic_memory.telemetry`
|
||||
- a shared `scope(...)` helper that opens a span and binds stable logger context together
|
||||
- context inheritance for routing, sync, and search so downstream `loguru` logs carry the active operation fields
|
||||
|
||||
This makes the trace view and the log stream tell the same story without forcing logger rewrites across the codebase.
|
||||
|
||||
## Local Dev Playbook
|
||||
|
||||
The fastest way to sanity-check the current trace shape is:
|
||||
|
||||
```bash
|
||||
LOGFIRE_TOKEN=lf_... just telemetry-smoke
|
||||
```
|
||||
|
||||
What this does:
|
||||
|
||||
- creates an isolated temp home, config dir, and project path
|
||||
- enables Logfire for the run
|
||||
- automatically exports to Logfire when `LOGFIRE_TOKEN` is present
|
||||
- defaults `BASIC_MEMORY_SEMANTIC_SEARCH_ENABLED=false` so the smoke run stays fast and trace-friendly
|
||||
- disables promo telemetry so the trace is about Basic Memory work, not analytics noise
|
||||
- runs a small CLI workflow:
|
||||
- `project add`
|
||||
- `tool write-note`
|
||||
- `tool read-note`
|
||||
- `tool edit-note`
|
||||
- `tool build-context`
|
||||
- `tool search-notes`
|
||||
- `doctor`
|
||||
|
||||
If you want to exercise the instrumentation without exporting anything upstream:
|
||||
|
||||
```bash
|
||||
BASIC_MEMORY_LOGFIRE_SEND_TO_LOGFIRE=false just telemetry-smoke
|
||||
```
|
||||
|
||||
If you want the smoke run to include vector or hybrid retrieval spans too:
|
||||
|
||||
```bash
|
||||
LOGFIRE_TOKEN=lf_... BASIC_MEMORY_SEMANTIC_SEARCH_ENABLED=true just telemetry-smoke
|
||||
```
|
||||
|
||||
The recipe sets `BASIC_MEMORY_LOGFIRE_ENVIRONMENT=telemetry-smoke` by default so these traces are easy to isolate in Logfire. Override it if you want the smoke traces grouped under a different environment name.
|
||||
|
||||
### What to look for
|
||||
|
||||
You should see a small set of comparable root spans rather than a framework-generated span forest:
|
||||
|
||||
- `cli.command.project`
|
||||
- `cli.command.tool`
|
||||
- `mcp.tool.write_note`
|
||||
- `mcp.tool.read_note`
|
||||
- `mcp.tool.edit_note`
|
||||
- `mcp.tool.build_context`
|
||||
- `mcp.tool.search_notes`
|
||||
- `sync.project.run`
|
||||
|
||||
You should also see correlated logs under those traces with stable fields like:
|
||||
|
||||
- `project_name`
|
||||
- `route_mode`
|
||||
- `tool_name`
|
||||
- `entrypoint`
|
||||
|
||||
### Expected nuance
|
||||
|
||||
`doctor` creates its own temporary project on purpose. That means the sync trace will usually show a different project name than the `telemetry-smoke` write/search traces. That is fine for smoke testing because the goal is to confirm:
|
||||
|
||||
- root span names are meaningful
|
||||
- scoped logs stay attached to the active trace
|
||||
- routing, tool, search, and sync phases are easy to distinguish
|
||||
|
||||
## Validation Checklist
|
||||
|
||||
We should consider the integration successful when the following are true:
|
||||
|
||||
1. With telemetry disabled, Basic Memory behaves exactly as it does today.
|
||||
2. With telemetry enabled, one user action produces one obvious root span.
|
||||
3. Logs emitted during that action are visible inside the same trace.
|
||||
4. A search in Logfire for `mcp.tool.write_note` or `sync.project.run` returns comparable spans across runs.
|
||||
5. Trace views show phase timing clearly without drowning in framework noise.
|
||||
6. Sensitive payloads are not captured by default.
|
||||
|
||||
## Immediate Implementation Direction
|
||||
|
||||
When we start coding, the first pass should be:
|
||||
|
||||
1. Add `basic_memory.telemetry`
|
||||
2. Add config/env switches for `enabled`, `send_to_logfire`, and service name
|
||||
3. Wire telemetry bootstrap into CLI, API, and MCP entrypoints
|
||||
4. Configure `loguru` to emit to both existing sinks and the Logfire handler when enabled
|
||||
5. Add manual root spans around:
|
||||
- CLI commands
|
||||
- API request handlers we care about
|
||||
- MCP tool entrypoints
|
||||
- sync root
|
||||
- search root
|
||||
6. Add child spans to the sync and routing phases only after the root span model feels clean
|
||||
|
||||
That gives us a strong foundation without repeating the earlier "turn on instrumentation everywhere" approach.
|
||||
@@ -205,6 +205,51 @@ doctor:
|
||||
BASIC_MEMORY_CONFIG_DIR="$TMP_CONFIG" \
|
||||
./.venv/bin/python -m basic_memory.cli.main doctor --local
|
||||
|
||||
# Run an isolated Logfire smoke workflow for local trace inspection
|
||||
telemetry-smoke:
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
TMP_HOME=$(mktemp -d)
|
||||
TMP_CONFIG=$(mktemp -d)
|
||||
TMP_PROJECT=$(mktemp -d)
|
||||
export HOME="$TMP_HOME"
|
||||
export BASIC_MEMORY_ENV="${BASIC_MEMORY_ENV:-dev}"
|
||||
export BASIC_MEMORY_HOME="$TMP_PROJECT/home-root"
|
||||
export BASIC_MEMORY_CONFIG_DIR="$TMP_CONFIG"
|
||||
export BASIC_MEMORY_NO_PROMOS=1
|
||||
export BASIC_MEMORY_LOG_LEVEL="${BASIC_MEMORY_LOG_LEVEL:-INFO}"
|
||||
export BASIC_MEMORY_SEMANTIC_SEARCH_ENABLED="${BASIC_MEMORY_SEMANTIC_SEARCH_ENABLED:-false}"
|
||||
export BASIC_MEMORY_LOGFIRE_ENABLED="${BASIC_MEMORY_LOGFIRE_ENABLED:-true}"
|
||||
export BASIC_MEMORY_LOGFIRE_ENVIRONMENT="${BASIC_MEMORY_LOGFIRE_ENVIRONMENT:-telemetry-smoke}"
|
||||
if [[ -z "${BASIC_MEMORY_LOGFIRE_SEND_TO_LOGFIRE:-}" ]]; then
|
||||
if [[ -n "${LOGFIRE_TOKEN:-}" ]]; then
|
||||
export BASIC_MEMORY_LOGFIRE_SEND_TO_LOGFIRE=true
|
||||
else
|
||||
export BASIC_MEMORY_LOGFIRE_SEND_TO_LOGFIRE=false
|
||||
fi
|
||||
fi
|
||||
mkdir -p "$BASIC_MEMORY_HOME"
|
||||
echo "Telemetry smoke setup:"
|
||||
echo " logfire_enabled=$BASIC_MEMORY_LOGFIRE_ENABLED"
|
||||
echo " send_to_logfire=$BASIC_MEMORY_LOGFIRE_SEND_TO_LOGFIRE"
|
||||
echo " log_level=$BASIC_MEMORY_LOG_LEVEL"
|
||||
echo " semantic_search_enabled=$BASIC_MEMORY_SEMANTIC_SEARCH_ENABLED"
|
||||
echo " logfire_environment=$BASIC_MEMORY_LOGFIRE_ENVIRONMENT"
|
||||
echo " project_path=$TMP_PROJECT"
|
||||
./.venv/bin/python -m basic_memory.cli.main project add telemetry-smoke "$TMP_PROJECT" --default --local
|
||||
./.venv/bin/python -m basic_memory.cli.main tool write-note --title "Telemetry Smoke" --folder notes --content "hello from smoke" --project telemetry-smoke --local
|
||||
./.venv/bin/python -m basic_memory.cli.main tool read-note notes/telemetry-smoke --project telemetry-smoke --local
|
||||
./.venv/bin/python -m basic_memory.cli.main tool edit-note notes/telemetry-smoke --operation append --content $'\n\nsmoke edit line' --project telemetry-smoke --local
|
||||
./.venv/bin/python -m basic_memory.cli.main tool build-context notes/telemetry-smoke --project telemetry-smoke --local --page-size 5 --max-related 5
|
||||
./.venv/bin/python -m basic_memory.cli.main tool search-notes telemetry --project telemetry-smoke --local
|
||||
./.venv/bin/python -m basic_memory.cli.main doctor --local
|
||||
echo ""
|
||||
echo "Telemetry smoke complete."
|
||||
echo "Search Logfire for:"
|
||||
echo " service_name: basic-memory-cli"
|
||||
echo " environment: $BASIC_MEMORY_LOGFIRE_ENVIRONMENT"
|
||||
echo " span names: mcp.tool.write_note, mcp.tool.read_note, mcp.tool.edit_note, mcp.tool.build_context, mcp.tool.search_notes, sync.project.run"
|
||||
|
||||
|
||||
# Update all dependencies to latest versions
|
||||
update-deps:
|
||||
|
||||
+17
-1
@@ -54,6 +54,22 @@ Or for a one-time sync:
|
||||
basic-memory sync
|
||||
```
|
||||
|
||||
### 4. Updating Basic Memory
|
||||
|
||||
Basic Memory supports automatic updates by default for `uv tool` and Homebrew installs.
|
||||
|
||||
For manual checks and upgrades:
|
||||
|
||||
```bash
|
||||
# Check now and install if supported
|
||||
bm update
|
||||
|
||||
# Check only, do not install
|
||||
bm update --check
|
||||
```
|
||||
|
||||
To disable automatic updates, set `"auto_update": false` in `~/.basic-memory/config.json`.
|
||||
|
||||
## Configuration Options
|
||||
|
||||
### Custom Directory
|
||||
@@ -125,4 +141,4 @@ If you encounter issues:
|
||||
cat ~/.basic-memory/basic-memory.log
|
||||
```
|
||||
|
||||
For more detailed information, refer to the [full documentation](https://memory.basicmachines.co/).
|
||||
For more detailed information, refer to the [full documentation](https://docs.basicmemory.com/).
|
||||
|
||||
@@ -58,6 +58,9 @@ Documentation = "https://github.com/basicmachines-co/basic-memory#readme"
|
||||
basic-memory = "basic_memory.cli.main:app"
|
||||
bm = "basic_memory.cli.main:app"
|
||||
|
||||
[project.optional-dependencies]
|
||||
telemetry = ["logfire>=4.19.0"]
|
||||
|
||||
[build-system]
|
||||
requires = ["hatchling", "uv-dynamic-versioning>=0.7.0"]
|
||||
build-backend = "hatchling.build"
|
||||
@@ -83,6 +86,7 @@ target-version = "py312"
|
||||
|
||||
[dependency-groups]
|
||||
dev = [
|
||||
"logfire>=4.19.0",
|
||||
"gevent>=24.11.1",
|
||||
"icecream>=2.1.3",
|
||||
"pytest>=8.3.4",
|
||||
|
||||
+2
-2
@@ -6,12 +6,12 @@
|
||||
"url": "https://github.com/basicmachines-co/basic-memory.git",
|
||||
"source": "github"
|
||||
},
|
||||
"version": "0.19.2",
|
||||
"version": "0.20.3",
|
||||
"packages": [
|
||||
{
|
||||
"registryType": "pypi",
|
||||
"identifier": "basic-memory",
|
||||
"version": "0.19.2",
|
||||
"version": "0.20.3",
|
||||
"runtimeHint": "uvx",
|
||||
"runtimeArguments": [
|
||||
{"type": "positional", "value": "basic-memory"},
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
"""basic-memory - Local-first knowledge management combining Zettelkasten with knowledge graphs"""
|
||||
|
||||
# Package version - updated by release automation
|
||||
__version__ = "0.19.2"
|
||||
__version__ = "0.20.3"
|
||||
|
||||
# API version for FastAPI - independent of package version
|
||||
__api_version__ = "v0"
|
||||
|
||||
+26
-16
@@ -25,6 +25,7 @@ from basic_memory.api.v2.routers.project_router import (
|
||||
list_projects,
|
||||
synchronize_projects,
|
||||
)
|
||||
from basic_memory import telemetry
|
||||
from basic_memory.config import init_api_logging
|
||||
from basic_memory.services.exceptions import EntityAlreadyExistsError
|
||||
from basic_memory.services.initialization import initialize_app
|
||||
@@ -43,30 +44,39 @@ async def lifespan(app: FastAPI): # pragma: no cover
|
||||
set_container(container)
|
||||
app.state.container = container
|
||||
|
||||
logger.info(f"Starting Basic Memory API (mode={container.mode.name})")
|
||||
with telemetry.operation(
|
||||
"api.lifecycle.startup",
|
||||
entrypoint="api",
|
||||
mode=container.mode.name.lower(),
|
||||
):
|
||||
logger.info(f"Starting Basic Memory API (mode={container.mode.name})")
|
||||
|
||||
await initialize_app(container.config)
|
||||
await initialize_app(container.config)
|
||||
|
||||
# Cache database connections in app state for performance
|
||||
logger.info("Initializing database and caching connections...")
|
||||
engine, session_maker = await container.init_database()
|
||||
app.state.engine = engine
|
||||
app.state.session_maker = session_maker
|
||||
logger.info("Database connections cached in app state")
|
||||
# Cache database connections in app state for performance
|
||||
logger.info("Initializing database and caching connections...")
|
||||
engine, session_maker = await container.init_database()
|
||||
app.state.engine = engine
|
||||
app.state.session_maker = session_maker
|
||||
logger.info("Database connections cached in app state")
|
||||
|
||||
# Create and start sync coordinator (lifecycle centralized in coordinator)
|
||||
sync_coordinator = container.create_sync_coordinator()
|
||||
await sync_coordinator.start()
|
||||
app.state.sync_coordinator = sync_coordinator
|
||||
# Create and start sync coordinator (lifecycle centralized in coordinator)
|
||||
sync_coordinator = container.create_sync_coordinator()
|
||||
await sync_coordinator.start()
|
||||
app.state.sync_coordinator = sync_coordinator
|
||||
|
||||
# Proceed with startup
|
||||
yield
|
||||
|
||||
# Shutdown - coordinator handles clean task cancellation
|
||||
logger.info("Shutting down Basic Memory API")
|
||||
await sync_coordinator.stop()
|
||||
|
||||
await container.shutdown_database()
|
||||
with telemetry.operation(
|
||||
"api.lifecycle.shutdown",
|
||||
entrypoint="api",
|
||||
mode=container.mode.name.lower(),
|
||||
):
|
||||
logger.info("Shutting down Basic Memory API")
|
||||
await sync_coordinator.stop()
|
||||
await container.shutdown_database()
|
||||
|
||||
|
||||
# Initialize FastAPI app
|
||||
|
||||
@@ -13,6 +13,7 @@ Key improvements:
|
||||
from fastapi import APIRouter, HTTPException, BackgroundTasks, Depends, Response, Path, Query
|
||||
from loguru import logger
|
||||
|
||||
from basic_memory import telemetry
|
||||
from basic_memory.deps import (
|
||||
EntityServiceV2ExternalDep,
|
||||
SearchServiceV2ExternalDep,
|
||||
@@ -142,47 +143,66 @@ async def resolve_identifier(
|
||||
"resolution_method": "permalink"
|
||||
}
|
||||
"""
|
||||
logger.info(f"API v2 request: resolve_identifier for '{data.identifier}'")
|
||||
with telemetry.operation(
|
||||
"api.request.knowledge.resolve_entity",
|
||||
entrypoint="api",
|
||||
domain="knowledge",
|
||||
action="resolve_entity",
|
||||
):
|
||||
logger.info(f"API v2 request: resolve_identifier for '{data.identifier}'")
|
||||
|
||||
# Try to resolve by external_id first
|
||||
entity = await entity_repository.get_by_external_id(data.identifier)
|
||||
resolution_method = "external_id" if entity else "search"
|
||||
with telemetry.scope(
|
||||
"api.knowledge.resolve_entity.lookup_entity",
|
||||
domain="knowledge",
|
||||
action="resolve_entity",
|
||||
phase="lookup_entity",
|
||||
):
|
||||
entity = await entity_repository.get_by_external_id(data.identifier)
|
||||
resolution_method = "external_id" if entity else "search"
|
||||
|
||||
# If not found by external_id, try other resolution methods
|
||||
# Pass source_path for context-aware resolution (prefers notes closer to source)
|
||||
# Pass strict to control fuzzy search fallback (default False allows fuzzy matching)
|
||||
if not entity:
|
||||
entity = await link_resolver.resolve_link(
|
||||
data.identifier, source_path=data.source_path, strict=data.strict
|
||||
if not entity:
|
||||
with telemetry.scope(
|
||||
"api.knowledge.resolve_entity.resolve_link",
|
||||
domain="knowledge",
|
||||
action="resolve_entity",
|
||||
phase="resolve_link",
|
||||
):
|
||||
entity = await link_resolver.resolve_link(
|
||||
data.identifier, source_path=data.source_path, strict=data.strict
|
||||
)
|
||||
if entity:
|
||||
if entity.permalink == data.identifier:
|
||||
resolution_method = "permalink"
|
||||
elif entity.title == data.identifier:
|
||||
resolution_method = "title"
|
||||
elif entity.file_path == data.identifier:
|
||||
resolution_method = "path"
|
||||
else:
|
||||
resolution_method = "search"
|
||||
|
||||
if not entity:
|
||||
raise HTTPException(status_code=404, detail=f"Entity not found: '{data.identifier}'")
|
||||
|
||||
with telemetry.scope(
|
||||
"api.knowledge.resolve_entity.shape_response",
|
||||
domain="knowledge",
|
||||
action="resolve_entity",
|
||||
phase="shape_response",
|
||||
):
|
||||
result = EntityResolveResponse(
|
||||
external_id=entity.external_id,
|
||||
entity_id=entity.id,
|
||||
permalink=entity.permalink,
|
||||
file_path=entity.file_path,
|
||||
title=entity.title,
|
||||
resolution_method=resolution_method,
|
||||
)
|
||||
|
||||
logger.debug(
|
||||
f"API v2 response: resolved '{data.identifier}' to external_id={result.external_id} via {resolution_method}"
|
||||
)
|
||||
if entity:
|
||||
# Determine resolution method
|
||||
if entity.permalink == data.identifier:
|
||||
resolution_method = "permalink"
|
||||
elif entity.title == data.identifier:
|
||||
resolution_method = "title"
|
||||
elif entity.file_path == data.identifier:
|
||||
resolution_method = "path"
|
||||
else:
|
||||
resolution_method = "search"
|
||||
|
||||
if not entity:
|
||||
raise HTTPException(status_code=404, detail=f"Entity not found: '{data.identifier}'")
|
||||
|
||||
result = EntityResolveResponse(
|
||||
external_id=entity.external_id,
|
||||
entity_id=entity.id,
|
||||
permalink=entity.permalink,
|
||||
file_path=entity.file_path,
|
||||
title=entity.title,
|
||||
resolution_method=resolution_method,
|
||||
)
|
||||
|
||||
logger.debug(
|
||||
f"API v2 response: resolved '{data.identifier}' to external_id={result.external_id} via {resolution_method}"
|
||||
)
|
||||
|
||||
return result
|
||||
return result
|
||||
|
||||
|
||||
## Read endpoints
|
||||
@@ -208,18 +228,36 @@ async def get_entity_by_id(
|
||||
Raises:
|
||||
HTTPException: 404 if entity not found
|
||||
"""
|
||||
logger.info(f"API v2 request: get_entity_by_id entity_id={entity_id}")
|
||||
with telemetry.operation(
|
||||
"api.request.knowledge.get_entity",
|
||||
entrypoint="api",
|
||||
domain="knowledge",
|
||||
action="get_entity",
|
||||
):
|
||||
logger.info(f"API v2 request: get_entity_by_id entity_id={entity_id}")
|
||||
|
||||
entity = await entity_repository.get_by_external_id(entity_id)
|
||||
if not entity:
|
||||
raise HTTPException(
|
||||
status_code=404, detail=f"Entity with external_id '{entity_id}' not found"
|
||||
)
|
||||
with telemetry.scope(
|
||||
"api.knowledge.get_entity.load_entity",
|
||||
domain="knowledge",
|
||||
action="get_entity",
|
||||
phase="load_entity",
|
||||
):
|
||||
entity = await entity_repository.get_by_external_id(entity_id)
|
||||
if not entity:
|
||||
raise HTTPException(
|
||||
status_code=404, detail=f"Entity with external_id '{entity_id}' not found"
|
||||
)
|
||||
|
||||
result = EntityResponseV2.model_validate(entity)
|
||||
logger.info(f"API v2 response: external_id={entity_id}, title='{result.title}'")
|
||||
with telemetry.scope(
|
||||
"api.knowledge.get_entity.shape_response",
|
||||
domain="knowledge",
|
||||
action="get_entity",
|
||||
phase="shape_response",
|
||||
):
|
||||
result = EntityResponseV2.model_validate(entity)
|
||||
logger.info(f"API v2 response: external_id={entity_id}, title='{result.title}'")
|
||||
|
||||
return result
|
||||
return result
|
||||
|
||||
|
||||
## Create endpoints
|
||||
@@ -248,39 +286,92 @@ async def create_entity(
|
||||
Returns:
|
||||
Created entity with generated external_id (UUID) and file content
|
||||
"""
|
||||
logger.info(
|
||||
"API v2 request", endpoint="create_entity", note_type=data.note_type, title=data.title
|
||||
)
|
||||
|
||||
if fast:
|
||||
entity = await entity_service.fast_write_entity(data)
|
||||
task_scheduler.schedule(
|
||||
"reindex_entity",
|
||||
entity_id=entity.id,
|
||||
project_id=project_id,
|
||||
)
|
||||
else:
|
||||
entity = await entity_service.create_entity(data)
|
||||
await search_service.index_entity(entity)
|
||||
_schedule_vector_sync_if_enabled(
|
||||
task_scheduler=task_scheduler,
|
||||
app_config=app_config,
|
||||
entity_id=entity.id,
|
||||
project_id=project_id,
|
||||
with telemetry.operation(
|
||||
"api.request.knowledge.create_entity",
|
||||
entrypoint="api",
|
||||
domain="knowledge",
|
||||
action="create_entity",
|
||||
fast=fast,
|
||||
):
|
||||
logger.info(
|
||||
"API v2 request", endpoint="create_entity", note_type=data.note_type, title=data.title
|
||||
)
|
||||
|
||||
result = EntityResponseV2.model_validate(entity)
|
||||
if fast:
|
||||
result = result.model_copy(update={"observations": [], "relations": []})
|
||||
with telemetry.scope(
|
||||
"api.knowledge.create_entity.write_entity",
|
||||
domain="knowledge",
|
||||
action="create_entity",
|
||||
phase="write_entity",
|
||||
fast=fast,
|
||||
):
|
||||
if fast:
|
||||
entity = await entity_service.fast_write_entity(data)
|
||||
written_content = None
|
||||
search_content = None
|
||||
else:
|
||||
write_result = await entity_service.create_entity_with_content(data)
|
||||
entity = write_result.entity
|
||||
written_content = write_result.content
|
||||
search_content = write_result.search_content
|
||||
|
||||
# Always read and return file content
|
||||
content = await file_service.read_file_content(entity.file_path)
|
||||
result = result.model_copy(update={"content": content})
|
||||
if fast:
|
||||
with telemetry.scope(
|
||||
"api.knowledge.create_entity.enqueue_reindex",
|
||||
domain="knowledge",
|
||||
action="create_entity",
|
||||
phase="enqueue_reindex",
|
||||
fast=fast,
|
||||
):
|
||||
task_scheduler.schedule(
|
||||
"reindex_entity",
|
||||
entity_id=entity.id,
|
||||
project_id=project_id,
|
||||
)
|
||||
else:
|
||||
with telemetry.scope(
|
||||
"api.knowledge.create_entity.search_index",
|
||||
domain="knowledge",
|
||||
action="create_entity",
|
||||
phase="search_index",
|
||||
):
|
||||
await search_service.index_entity(entity, content=search_content)
|
||||
with telemetry.scope(
|
||||
"api.knowledge.create_entity.vector_sync",
|
||||
domain="knowledge",
|
||||
action="create_entity",
|
||||
phase="vector_sync",
|
||||
):
|
||||
_schedule_vector_sync_if_enabled(
|
||||
task_scheduler=task_scheduler,
|
||||
app_config=app_config,
|
||||
entity_id=entity.id,
|
||||
project_id=project_id,
|
||||
)
|
||||
|
||||
logger.info(
|
||||
f"API v2 response: endpoint='create_entity' external_id={entity.external_id}, title={result.title}, permalink={result.permalink}, status_code=201"
|
||||
)
|
||||
return result
|
||||
result = EntityResponseV2.model_validate(entity)
|
||||
if fast:
|
||||
result = result.model_copy(update={"observations": [], "relations": []})
|
||||
|
||||
with telemetry.scope(
|
||||
"api.knowledge.create_entity.read_content",
|
||||
domain="knowledge",
|
||||
action="create_entity",
|
||||
phase="read_content",
|
||||
source="file" if fast else "memory",
|
||||
):
|
||||
if fast:
|
||||
content = await file_service.read_file_content(entity.file_path)
|
||||
else:
|
||||
# Non-fast writes already captured the markdown in memory. Reuse it here
|
||||
# instead of re-reading the file; format_on_save is the one config that can
|
||||
# still make the persisted file diverge because write_file only returns a checksum.
|
||||
content = written_content
|
||||
result = result.model_copy(update={"content": content})
|
||||
|
||||
logger.info(
|
||||
f"API v2 response: endpoint='create_entity' external_id={entity.external_id}, title={result.title}, permalink={result.permalink}, status_code=201"
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
## Update endpoints
|
||||
@@ -315,61 +406,121 @@ async def update_entity_by_id(
|
||||
Returns:
|
||||
Updated entity with file content
|
||||
"""
|
||||
logger.info(f"API v2 request: update_entity_by_id entity_id={entity_id}")
|
||||
with telemetry.operation(
|
||||
"api.request.knowledge.update_entity",
|
||||
entrypoint="api",
|
||||
domain="knowledge",
|
||||
action="update_entity",
|
||||
fast=fast,
|
||||
):
|
||||
logger.info(f"API v2 request: update_entity_by_id entity_id={entity_id}")
|
||||
|
||||
# Check if entity exists (external_id is the source of truth for v2)
|
||||
existing = await entity_repository.get_by_external_id(entity_id)
|
||||
created = existing is None
|
||||
with telemetry.scope(
|
||||
"api.knowledge.update_entity.load_entity",
|
||||
domain="knowledge",
|
||||
action="update_entity",
|
||||
phase="load_entity",
|
||||
):
|
||||
existing = await entity_repository.get_by_external_id(entity_id)
|
||||
created = existing is None
|
||||
|
||||
if fast:
|
||||
entity = await entity_service.fast_write_entity(data, external_id=entity_id)
|
||||
response.status_code = 200 if existing else 201
|
||||
task_scheduler.schedule(
|
||||
"reindex_entity",
|
||||
entity_id=entity.id,
|
||||
project_id=project_id,
|
||||
resolve_relations=created,
|
||||
)
|
||||
else:
|
||||
if existing:
|
||||
# Update the existing entity in-place to avoid path-based duplication
|
||||
entity = await entity_service.update_entity(existing, data)
|
||||
response.status_code = 200
|
||||
else:
|
||||
# Create new entity, then bind external_id to the requested UUID
|
||||
entity = await entity_service.create_entity(data)
|
||||
if entity.external_id != entity_id:
|
||||
entity = await entity_repository.update(
|
||||
entity.id,
|
||||
{"external_id": entity_id},
|
||||
with telemetry.scope(
|
||||
"api.knowledge.update_entity.write_entity",
|
||||
domain="knowledge",
|
||||
action="update_entity",
|
||||
phase="write_entity",
|
||||
fast=fast,
|
||||
):
|
||||
if fast:
|
||||
entity = await entity_service.fast_write_entity(data, external_id=entity_id)
|
||||
written_content = None
|
||||
search_content = None
|
||||
response.status_code = 200 if existing else 201
|
||||
else:
|
||||
if existing:
|
||||
write_result = await entity_service.update_entity_with_content(existing, data)
|
||||
entity = write_result.entity
|
||||
written_content = write_result.content
|
||||
search_content = write_result.search_content
|
||||
response.status_code = 200
|
||||
else:
|
||||
write_result = await entity_service.create_entity_with_content(data)
|
||||
entity = write_result.entity
|
||||
written_content = write_result.content
|
||||
search_content = write_result.search_content
|
||||
if entity.external_id != entity_id:
|
||||
entity = await entity_repository.update(
|
||||
entity.id,
|
||||
{"external_id": entity_id},
|
||||
)
|
||||
# external_id fixup only changes the DB row. The file content is unchanged,
|
||||
# so the markdown captured during the write remains valid downstream.
|
||||
if not entity:
|
||||
raise HTTPException(
|
||||
status_code=404,
|
||||
detail=f"Entity with external_id '{entity_id}' not found",
|
||||
)
|
||||
response.status_code = 201
|
||||
|
||||
if fast:
|
||||
with telemetry.scope(
|
||||
"api.knowledge.update_entity.enqueue_reindex",
|
||||
domain="knowledge",
|
||||
action="update_entity",
|
||||
phase="enqueue_reindex",
|
||||
fast=fast,
|
||||
):
|
||||
task_scheduler.schedule(
|
||||
"reindex_entity",
|
||||
entity_id=entity.id,
|
||||
project_id=project_id,
|
||||
resolve_relations=created,
|
||||
)
|
||||
else:
|
||||
with telemetry.scope(
|
||||
"api.knowledge.update_entity.search_index",
|
||||
domain="knowledge",
|
||||
action="update_entity",
|
||||
phase="search_index",
|
||||
):
|
||||
await search_service.index_entity(entity, content=search_content)
|
||||
with telemetry.scope(
|
||||
"api.knowledge.update_entity.vector_sync",
|
||||
domain="knowledge",
|
||||
action="update_entity",
|
||||
phase="vector_sync",
|
||||
):
|
||||
_schedule_vector_sync_if_enabled(
|
||||
task_scheduler=task_scheduler,
|
||||
app_config=app_config,
|
||||
entity_id=entity.id,
|
||||
project_id=project_id,
|
||||
)
|
||||
if not entity:
|
||||
raise HTTPException(
|
||||
status_code=404,
|
||||
detail=f"Entity with external_id '{entity_id}' not found",
|
||||
)
|
||||
response.status_code = 201
|
||||
|
||||
await search_service.index_entity(entity)
|
||||
_schedule_vector_sync_if_enabled(
|
||||
task_scheduler=task_scheduler,
|
||||
app_config=app_config,
|
||||
entity_id=entity.id,
|
||||
project_id=project_id,
|
||||
result = EntityResponseV2.model_validate(entity)
|
||||
if fast:
|
||||
result = result.model_copy(update={"observations": [], "relations": []})
|
||||
|
||||
with telemetry.scope(
|
||||
"api.knowledge.update_entity.read_content",
|
||||
domain="knowledge",
|
||||
action="update_entity",
|
||||
phase="read_content",
|
||||
source="file" if fast else "memory",
|
||||
):
|
||||
if fast:
|
||||
content = await file_service.read_file_content(entity.file_path)
|
||||
else:
|
||||
# Non-fast writes already captured the markdown in memory. Reuse it here
|
||||
# instead of re-reading the file; format_on_save is the one config that can
|
||||
# still make the persisted file diverge because write_file only returns a checksum.
|
||||
content = written_content
|
||||
result = result.model_copy(update={"content": content})
|
||||
|
||||
logger.info(
|
||||
f"API v2 response: external_id={entity_id}, created={created}, status_code={response.status_code}"
|
||||
)
|
||||
|
||||
result = EntityResponseV2.model_validate(entity)
|
||||
if fast:
|
||||
result = result.model_copy(update={"observations": [], "relations": []})
|
||||
|
||||
# Always read and return file content
|
||||
content = await file_service.read_file_content(entity.file_path)
|
||||
result = result.model_copy(update={"content": content})
|
||||
|
||||
logger.info(
|
||||
f"API v2 response: external_id={entity_id}, created={created}, status_code={response.status_code}"
|
||||
)
|
||||
return result
|
||||
return result
|
||||
|
||||
|
||||
@router.patch("/entities/{entity_id}", response_model=EntityResponseV2)
|
||||
@@ -401,69 +552,125 @@ async def edit_entity_by_id(
|
||||
Raises:
|
||||
HTTPException: 404 if entity not found, 400 if edit fails
|
||||
"""
|
||||
logger.info(
|
||||
f"API v2 request: edit_entity_by_id entity_id={entity_id}, operation='{data.operation}'"
|
||||
)
|
||||
|
||||
# Verify entity exists
|
||||
entity = await entity_repository.get_by_external_id(entity_id)
|
||||
if not entity: # pragma: no cover
|
||||
raise HTTPException(
|
||||
status_code=404, detail=f"Entity with external_id '{entity_id}' not found"
|
||||
)
|
||||
|
||||
try:
|
||||
if fast:
|
||||
updated_entity = await entity_service.fast_edit_entity(
|
||||
entity=entity,
|
||||
operation=data.operation,
|
||||
content=data.content,
|
||||
section=data.section,
|
||||
find_text=data.find_text,
|
||||
expected_replacements=data.expected_replacements,
|
||||
)
|
||||
task_scheduler.schedule(
|
||||
"reindex_entity",
|
||||
entity_id=updated_entity.id,
|
||||
project_id=project_id,
|
||||
)
|
||||
else:
|
||||
# Edit using the entity's permalink or path
|
||||
identifier = entity.permalink or entity.file_path
|
||||
updated_entity = await entity_service.edit_entity(
|
||||
identifier=identifier,
|
||||
operation=data.operation,
|
||||
content=data.content,
|
||||
section=data.section,
|
||||
find_text=data.find_text,
|
||||
expected_replacements=data.expected_replacements,
|
||||
)
|
||||
|
||||
await search_service.index_entity(updated_entity)
|
||||
_schedule_vector_sync_if_enabled(
|
||||
task_scheduler=task_scheduler,
|
||||
app_config=app_config,
|
||||
entity_id=updated_entity.id,
|
||||
project_id=project_id,
|
||||
)
|
||||
|
||||
result = EntityResponseV2.model_validate(updated_entity)
|
||||
if fast:
|
||||
result = result.model_copy(update={"observations": [], "relations": []})
|
||||
|
||||
# Always read and return file content
|
||||
content = await file_service.read_file_content(updated_entity.file_path)
|
||||
result = result.model_copy(update={"content": content})
|
||||
|
||||
with telemetry.operation(
|
||||
"api.request.knowledge.edit_entity",
|
||||
entrypoint="api",
|
||||
domain="knowledge",
|
||||
action="edit_entity",
|
||||
fast=fast,
|
||||
):
|
||||
logger.info(
|
||||
f"API v2 response: external_id={entity_id}, operation='{data.operation}', status_code=200"
|
||||
f"API v2 request: edit_entity_by_id entity_id={entity_id}, operation='{data.operation}'"
|
||||
)
|
||||
|
||||
return result
|
||||
with telemetry.scope(
|
||||
"api.knowledge.edit_entity.load_entity",
|
||||
domain="knowledge",
|
||||
action="edit_entity",
|
||||
phase="load_entity",
|
||||
):
|
||||
entity = await entity_repository.get_by_external_id(entity_id)
|
||||
if not entity: # pragma: no cover
|
||||
raise HTTPException(
|
||||
status_code=404, detail=f"Entity with external_id '{entity_id}' not found"
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error editing entity {entity_id}: {e}")
|
||||
raise HTTPException(status_code=400, detail=str(e))
|
||||
try:
|
||||
with telemetry.scope(
|
||||
"api.knowledge.edit_entity.write_entity",
|
||||
domain="knowledge",
|
||||
action="edit_entity",
|
||||
phase="write_entity",
|
||||
fast=fast,
|
||||
):
|
||||
if fast:
|
||||
updated_entity = await entity_service.fast_edit_entity(
|
||||
entity=entity,
|
||||
operation=data.operation,
|
||||
content=data.content,
|
||||
section=data.section,
|
||||
find_text=data.find_text,
|
||||
expected_replacements=data.expected_replacements,
|
||||
)
|
||||
written_content = None
|
||||
search_content = None
|
||||
else:
|
||||
identifier = entity.permalink or entity.file_path
|
||||
write_result = await entity_service.edit_entity_with_content(
|
||||
identifier=identifier,
|
||||
operation=data.operation,
|
||||
content=data.content,
|
||||
section=data.section,
|
||||
find_text=data.find_text,
|
||||
expected_replacements=data.expected_replacements,
|
||||
)
|
||||
updated_entity = write_result.entity
|
||||
written_content = write_result.content
|
||||
search_content = write_result.search_content
|
||||
|
||||
if fast:
|
||||
with telemetry.scope(
|
||||
"api.knowledge.edit_entity.enqueue_reindex",
|
||||
domain="knowledge",
|
||||
action="edit_entity",
|
||||
phase="enqueue_reindex",
|
||||
fast=fast,
|
||||
):
|
||||
task_scheduler.schedule(
|
||||
"reindex_entity",
|
||||
entity_id=updated_entity.id,
|
||||
project_id=project_id,
|
||||
)
|
||||
else:
|
||||
with telemetry.scope(
|
||||
"api.knowledge.edit_entity.search_index",
|
||||
domain="knowledge",
|
||||
action="edit_entity",
|
||||
phase="search_index",
|
||||
):
|
||||
await search_service.index_entity(updated_entity, content=search_content)
|
||||
with telemetry.scope(
|
||||
"api.knowledge.edit_entity.vector_sync",
|
||||
domain="knowledge",
|
||||
action="edit_entity",
|
||||
phase="vector_sync",
|
||||
):
|
||||
_schedule_vector_sync_if_enabled(
|
||||
task_scheduler=task_scheduler,
|
||||
app_config=app_config,
|
||||
entity_id=updated_entity.id,
|
||||
project_id=project_id,
|
||||
)
|
||||
|
||||
result = EntityResponseV2.model_validate(updated_entity)
|
||||
if fast:
|
||||
result = result.model_copy(update={"observations": [], "relations": []})
|
||||
|
||||
with telemetry.scope(
|
||||
"api.knowledge.edit_entity.read_content",
|
||||
domain="knowledge",
|
||||
action="edit_entity",
|
||||
phase="read_content",
|
||||
source="file" if fast else "memory",
|
||||
):
|
||||
if fast:
|
||||
content = await file_service.read_file_content(updated_entity.file_path)
|
||||
else:
|
||||
# Non-fast writes already captured the markdown in memory. Reuse it here
|
||||
# instead of re-reading the file; format_on_save is the one config that can
|
||||
# still make the persisted file diverge because write_file only returns a checksum.
|
||||
content = written_content
|
||||
result = result.model_copy(update={"content": content})
|
||||
|
||||
logger.info(
|
||||
f"API v2 response: external_id={entity_id}, operation='{data.operation}', status_code=200"
|
||||
)
|
||||
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error editing entity {entity_id}: {e}")
|
||||
raise HTTPException(status_code=400, detail=str(e))
|
||||
|
||||
|
||||
## Delete endpoints
|
||||
|
||||
@@ -9,6 +9,7 @@ from typing import Annotated, Optional
|
||||
from fastapi import APIRouter, Query, Path
|
||||
from loguru import logger
|
||||
|
||||
from basic_memory import telemetry
|
||||
from basic_memory.deps import ContextServiceV2ExternalDep, EntityRepositoryV2ExternalDep
|
||||
from basic_memory.schemas.base import TimeFrame, parse_timeframe
|
||||
from basic_memory.schemas.memory import (
|
||||
@@ -50,30 +51,55 @@ async def recent(
|
||||
Returns:
|
||||
GraphContext with recent activity and related entities
|
||||
"""
|
||||
# return all types by default
|
||||
types = (
|
||||
[SearchItemType.ENTITY, SearchItemType.RELATION, SearchItemType.OBSERVATION]
|
||||
if not type
|
||||
else type
|
||||
)
|
||||
with telemetry.operation(
|
||||
"api.request.memory.recent_activity",
|
||||
entrypoint="api",
|
||||
domain="memory",
|
||||
action="recent_activity",
|
||||
page=page,
|
||||
page_size=page_size,
|
||||
):
|
||||
types = (
|
||||
[SearchItemType.ENTITY, SearchItemType.RELATION, SearchItemType.OBSERVATION]
|
||||
if not type
|
||||
else type
|
||||
)
|
||||
|
||||
logger.debug(
|
||||
f"V2 Getting recent context for project {project_id}: `{types}` depth: `{depth}` timeframe: `{timeframe}` page: `{page}` page_size: `{page_size}` max_related: `{max_related}`"
|
||||
)
|
||||
# Parse timeframe
|
||||
since = parse_timeframe(timeframe)
|
||||
limit = page_size
|
||||
offset = (page - 1) * page_size
|
||||
logger.debug(
|
||||
f"V2 Getting recent context for project {project_id}: `{types}` depth: `{depth}` timeframe: `{timeframe}` page: `{page}` page_size: `{page_size}` max_related: `{max_related}`"
|
||||
)
|
||||
since = parse_timeframe(timeframe)
|
||||
limit = page_size
|
||||
offset = (page - 1) * page_size
|
||||
|
||||
# Build context
|
||||
context = await context_service.build_context(
|
||||
types=types, depth=depth, since=since, limit=limit, offset=offset, max_related=max_related
|
||||
)
|
||||
recent_context = await to_graph_context(
|
||||
context, entity_repository=entity_repository, page=page, page_size=page_size
|
||||
)
|
||||
logger.debug(f"V2 Recent context: {recent_context.model_dump_json()}")
|
||||
return recent_context
|
||||
with telemetry.scope(
|
||||
"api.memory.recent_activity.build_context",
|
||||
domain="memory",
|
||||
action="recent_activity",
|
||||
phase="build_context",
|
||||
page=page,
|
||||
page_size=page_size,
|
||||
):
|
||||
context = await context_service.build_context(
|
||||
types=types,
|
||||
depth=depth,
|
||||
since=since,
|
||||
limit=limit,
|
||||
offset=offset,
|
||||
max_related=max_related,
|
||||
)
|
||||
with telemetry.scope(
|
||||
"api.memory.recent_activity.shape_response",
|
||||
domain="memory",
|
||||
action="recent_activity",
|
||||
phase="shape_response",
|
||||
result_count=len(context.results),
|
||||
):
|
||||
recent_context = await to_graph_context(
|
||||
context, entity_repository=entity_repository, page=page, page_size=page_size
|
||||
)
|
||||
logger.debug(f"V2 Recent context: {recent_context.model_dump_json()}")
|
||||
return recent_context
|
||||
|
||||
|
||||
# get_memory_context needs to be declared last so other paths can match
|
||||
@@ -111,20 +137,46 @@ async def get_memory_context(
|
||||
Returns:
|
||||
GraphContext with the entity and its related context
|
||||
"""
|
||||
logger.debug(
|
||||
f"V2 Getting context for project {project_id}, URI: `{uri}` depth: `{depth}` timeframe: `{timeframe}` page: `{page}` page_size: `{page_size}` max_related: `{max_related}`"
|
||||
)
|
||||
memory_url = normalize_memory_url(uri)
|
||||
with telemetry.operation(
|
||||
"api.request.memory.build_context",
|
||||
entrypoint="api",
|
||||
domain="memory",
|
||||
action="build_context",
|
||||
page=page,
|
||||
page_size=page_size,
|
||||
):
|
||||
logger.debug(
|
||||
f"V2 Getting context for project {project_id}, URI: `{uri}` depth: `{depth}` timeframe: `{timeframe}` page: `{page}` page_size: `{page_size}` max_related: `{max_related}`"
|
||||
)
|
||||
memory_url = normalize_memory_url(uri)
|
||||
|
||||
# Parse timeframe
|
||||
since = parse_timeframe(timeframe) if timeframe else None
|
||||
limit = page_size
|
||||
offset = (page - 1) * page_size
|
||||
since = parse_timeframe(timeframe) if timeframe else None
|
||||
limit = page_size
|
||||
offset = (page - 1) * page_size
|
||||
|
||||
# Build context
|
||||
context = await context_service.build_context(
|
||||
memory_url, depth=depth, since=since, limit=limit, offset=offset, max_related=max_related
|
||||
)
|
||||
return await to_graph_context(
|
||||
context, entity_repository=entity_repository, page=page, page_size=page_size
|
||||
)
|
||||
with telemetry.scope(
|
||||
"api.memory.build_context.build_context",
|
||||
domain="memory",
|
||||
action="build_context",
|
||||
phase="build_context",
|
||||
page=page,
|
||||
page_size=page_size,
|
||||
):
|
||||
context = await context_service.build_context(
|
||||
memory_url,
|
||||
depth=depth,
|
||||
since=since,
|
||||
limit=limit,
|
||||
offset=offset,
|
||||
max_related=max_related,
|
||||
)
|
||||
with telemetry.scope(
|
||||
"api.memory.build_context.shape_response",
|
||||
domain="memory",
|
||||
action="build_context",
|
||||
phase="shape_response",
|
||||
result_count=len(context.results),
|
||||
):
|
||||
return await to_graph_context(
|
||||
context, entity_repository=entity_repository, page=page, page_size=page_size
|
||||
)
|
||||
|
||||
@@ -15,6 +15,7 @@ from pathlib import Path as PathLib
|
||||
from fastapi import APIRouter, HTTPException, Response, Path
|
||||
from loguru import logger
|
||||
|
||||
from basic_memory import telemetry
|
||||
from basic_memory.deps import (
|
||||
ProjectConfigV2ExternalDep,
|
||||
FileServiceV2ExternalDep,
|
||||
@@ -55,36 +56,62 @@ async def get_resource_content(
|
||||
Raises:
|
||||
HTTPException: 404 if entity or file not found
|
||||
"""
|
||||
logger.debug(f"V2 Getting content for project {project_id}, entity_id: {entity_id}")
|
||||
with telemetry.operation(
|
||||
"api.request.resource.get_content",
|
||||
entrypoint="api",
|
||||
domain="resource",
|
||||
action="get_content",
|
||||
):
|
||||
logger.debug(f"V2 Getting content for project {project_id}, entity_id: {entity_id}")
|
||||
|
||||
# Get entity by external_id
|
||||
entity = await entity_repository.get_by_external_id(entity_id)
|
||||
if not entity:
|
||||
raise HTTPException(status_code=404, detail=f"Entity {entity_id} not found")
|
||||
with telemetry.scope(
|
||||
"api.resource.get_content.load_entity",
|
||||
domain="resource",
|
||||
action="get_content",
|
||||
phase="load_entity",
|
||||
):
|
||||
entity = await entity_repository.get_by_external_id(entity_id)
|
||||
if not entity:
|
||||
raise HTTPException(status_code=404, detail=f"Entity {entity_id} not found")
|
||||
|
||||
# Validate entity file path to prevent path traversal
|
||||
project_path = PathLib(config.home)
|
||||
if not validate_project_path(entity.file_path, project_path):
|
||||
logger.error( # pragma: no cover
|
||||
f"Invalid file path in entity {entity.id}: {entity.file_path}"
|
||||
)
|
||||
raise HTTPException( # pragma: no cover
|
||||
status_code=500,
|
||||
detail="Entity contains invalid file path",
|
||||
)
|
||||
with telemetry.scope(
|
||||
"api.resource.get_content.validate_path",
|
||||
domain="resource",
|
||||
action="get_content",
|
||||
phase="validate_path",
|
||||
):
|
||||
project_path = PathLib(config.home)
|
||||
if not validate_project_path(entity.file_path, project_path):
|
||||
logger.error( # pragma: no cover
|
||||
f"Invalid file path in entity {entity.id}: {entity.file_path}"
|
||||
)
|
||||
raise HTTPException( # pragma: no cover
|
||||
status_code=500,
|
||||
detail="Entity contains invalid file path",
|
||||
)
|
||||
|
||||
# Check file exists via file_service (for cloud compatibility)
|
||||
if not await file_service.exists(entity.file_path):
|
||||
raise HTTPException( # pragma: no cover
|
||||
status_code=404,
|
||||
detail=f"File not found: {entity.file_path}",
|
||||
)
|
||||
with telemetry.scope(
|
||||
"api.resource.get_content.ensure_exists",
|
||||
domain="resource",
|
||||
action="get_content",
|
||||
phase="ensure_exists",
|
||||
):
|
||||
if not await file_service.exists(entity.file_path):
|
||||
raise HTTPException( # pragma: no cover
|
||||
status_code=404,
|
||||
detail=f"File not found: {entity.file_path}",
|
||||
)
|
||||
|
||||
# Read content via file_service as bytes (works with both local and S3)
|
||||
content = await file_service.read_file_bytes(entity.file_path)
|
||||
content_type = file_service.content_type(entity.file_path)
|
||||
with telemetry.scope(
|
||||
"api.resource.get_content.read_content",
|
||||
domain="resource",
|
||||
action="get_content",
|
||||
phase="read_content",
|
||||
):
|
||||
content = await file_service.read_file_bytes(entity.file_path)
|
||||
content_type = file_service.content_type(entity.file_path)
|
||||
|
||||
return Response(content=content, media_type=content_type)
|
||||
return Response(content=content, media_type=content_type)
|
||||
|
||||
|
||||
@router.post("", response_model=ResourceResponse)
|
||||
@@ -112,74 +139,94 @@ async def create_resource(
|
||||
Raises:
|
||||
HTTPException: 400 for invalid file paths, 409 if file already exists
|
||||
"""
|
||||
try:
|
||||
# Validate path to prevent path traversal attacks
|
||||
project_path = PathLib(config.home)
|
||||
if not validate_project_path(data.file_path, project_path):
|
||||
logger.warning(
|
||||
f"Invalid file path attempted: {data.file_path} in project {config.name}"
|
||||
with telemetry.operation(
|
||||
"api.request.resource.create",
|
||||
entrypoint="api",
|
||||
domain="resource",
|
||||
action="create",
|
||||
):
|
||||
try:
|
||||
# Validate path to prevent path traversal attacks
|
||||
project_path = PathLib(config.home)
|
||||
if not validate_project_path(data.file_path, project_path):
|
||||
logger.warning(
|
||||
f"Invalid file path attempted: {data.file_path} in project {config.name}"
|
||||
)
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"Invalid file path: {data.file_path}. "
|
||||
"Path must be relative and stay within project boundaries.",
|
||||
)
|
||||
|
||||
existing_entity = await entity_repository.get_by_file_path(data.file_path)
|
||||
if existing_entity:
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail=f"Resource already exists at {data.file_path} with entity_id {existing_entity.external_id}. "
|
||||
f"Use PUT /resource/{existing_entity.external_id} to update it.",
|
||||
)
|
||||
|
||||
with telemetry.scope(
|
||||
"api.resource.create.write_file",
|
||||
domain="resource",
|
||||
action="create",
|
||||
phase="write_file",
|
||||
):
|
||||
await file_service.ensure_directory(PathLib(data.file_path).parent)
|
||||
checksum = await file_service.write_file(data.file_path, data.content)
|
||||
|
||||
with telemetry.scope(
|
||||
"api.resource.create.read_metadata",
|
||||
domain="resource",
|
||||
action="create",
|
||||
phase="read_metadata",
|
||||
):
|
||||
file_metadata = await file_service.get_file_metadata(data.file_path)
|
||||
|
||||
file_name = PathLib(data.file_path).name
|
||||
content_type = file_service.content_type(data.file_path)
|
||||
note_type = "canvas" if data.file_path.endswith(".canvas") else "file"
|
||||
|
||||
entity = EntityModel(
|
||||
external_id=str(uuid.uuid4()),
|
||||
title=file_name,
|
||||
note_type=note_type,
|
||||
content_type=content_type,
|
||||
file_path=data.file_path,
|
||||
checksum=checksum,
|
||||
created_at=file_metadata.created_at,
|
||||
updated_at=file_metadata.modified_at,
|
||||
)
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"Invalid file path: {data.file_path}. "
|
||||
"Path must be relative and stay within project boundaries.",
|
||||
with telemetry.scope(
|
||||
"api.resource.create.upsert_entity",
|
||||
domain="resource",
|
||||
action="create",
|
||||
phase="upsert_entity",
|
||||
):
|
||||
entity = await entity_repository.add(entity)
|
||||
|
||||
with telemetry.scope(
|
||||
"api.resource.create.search_index",
|
||||
domain="resource",
|
||||
action="create",
|
||||
phase="search_index",
|
||||
):
|
||||
await search_service.index_entity(entity) # pyright: ignore
|
||||
|
||||
return ResourceResponse(
|
||||
entity_id=entity.id,
|
||||
external_id=entity.external_id,
|
||||
file_path=data.file_path,
|
||||
checksum=checksum,
|
||||
size=file_metadata.size,
|
||||
created_at=file_metadata.created_at.timestamp(),
|
||||
modified_at=file_metadata.modified_at.timestamp(),
|
||||
)
|
||||
|
||||
# Check if entity already exists
|
||||
existing_entity = await entity_repository.get_by_file_path(data.file_path)
|
||||
if existing_entity:
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail=f"Resource already exists at {data.file_path} with entity_id {existing_entity.external_id}. "
|
||||
f"Use PUT /resource/{existing_entity.external_id} to update it.",
|
||||
)
|
||||
|
||||
# Cloud compatibility: avoid assuming a local filesystem path.
|
||||
# Delegate directory creation + writes to FileService (local or S3).
|
||||
await file_service.ensure_directory(PathLib(data.file_path).parent)
|
||||
checksum = await file_service.write_file(data.file_path, data.content)
|
||||
|
||||
# Get file info
|
||||
file_metadata = await file_service.get_file_metadata(data.file_path)
|
||||
|
||||
# Determine file details
|
||||
file_name = PathLib(data.file_path).name
|
||||
content_type = file_service.content_type(data.file_path)
|
||||
note_type = "canvas" if data.file_path.endswith(".canvas") else "file"
|
||||
|
||||
# Create a new entity model
|
||||
# Explicitly set external_id to ensure NOT NULL constraint is satisfied (fixes #512)
|
||||
entity = EntityModel(
|
||||
external_id=str(uuid.uuid4()),
|
||||
title=file_name,
|
||||
note_type=note_type,
|
||||
content_type=content_type,
|
||||
file_path=data.file_path,
|
||||
checksum=checksum,
|
||||
created_at=file_metadata.created_at,
|
||||
updated_at=file_metadata.modified_at,
|
||||
)
|
||||
entity = await entity_repository.add(entity)
|
||||
|
||||
# Index the file for search
|
||||
await search_service.index_entity(entity) # pyright: ignore
|
||||
|
||||
# Return success response
|
||||
return ResourceResponse(
|
||||
entity_id=entity.id,
|
||||
external_id=entity.external_id,
|
||||
file_path=data.file_path,
|
||||
checksum=checksum,
|
||||
size=file_metadata.size,
|
||||
created_at=file_metadata.created_at.timestamp(),
|
||||
modified_at=file_metadata.modified_at.timestamp(),
|
||||
)
|
||||
except HTTPException:
|
||||
# Re-raise HTTP exceptions without wrapping
|
||||
raise
|
||||
except Exception as e: # pragma: no cover
|
||||
logger.error(f"Error creating resource {data.file_path}: {e}")
|
||||
raise HTTPException(status_code=500, detail=f"Failed to create resource: {str(e)}")
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e: # pragma: no cover
|
||||
logger.error(f"Error creating resource {data.file_path}: {e}")
|
||||
raise HTTPException(status_code=500, detail=f"Failed to create resource: {str(e)}")
|
||||
|
||||
|
||||
@router.put("/{entity_id}", response_model=ResourceResponse)
|
||||
@@ -211,79 +258,94 @@ async def update_resource(
|
||||
Raises:
|
||||
HTTPException: 404 if entity not found, 400 for invalid paths
|
||||
"""
|
||||
try:
|
||||
# Get existing entity by external_id
|
||||
entity = await entity_repository.get_by_external_id(entity_id)
|
||||
if not entity:
|
||||
raise HTTPException(status_code=404, detail=f"Entity {entity_id} not found")
|
||||
with telemetry.operation(
|
||||
"api.request.resource.update",
|
||||
entrypoint="api",
|
||||
domain="resource",
|
||||
action="update",
|
||||
):
|
||||
try:
|
||||
entity = await entity_repository.get_by_external_id(entity_id)
|
||||
if not entity:
|
||||
raise HTTPException(status_code=404, detail=f"Entity {entity_id} not found")
|
||||
|
||||
# Determine target file path
|
||||
target_file_path = data.file_path if data.file_path else entity.file_path
|
||||
target_file_path = data.file_path if data.file_path else entity.file_path
|
||||
|
||||
# Validate path to prevent path traversal attacks
|
||||
project_path = PathLib(config.home)
|
||||
if not validate_project_path(target_file_path, project_path):
|
||||
logger.warning(
|
||||
f"Invalid file path attempted: {target_file_path} in project {config.name}"
|
||||
project_path = PathLib(config.home)
|
||||
if not validate_project_path(target_file_path, project_path):
|
||||
logger.warning(
|
||||
f"Invalid file path attempted: {target_file_path} in project {config.name}"
|
||||
)
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"Invalid file path: {target_file_path}. "
|
||||
"Path must be relative and stay within project boundaries.",
|
||||
)
|
||||
|
||||
with telemetry.scope(
|
||||
"api.resource.update.write_file",
|
||||
domain="resource",
|
||||
action="update",
|
||||
phase="write_file",
|
||||
):
|
||||
if data.file_path and data.file_path != entity.file_path:
|
||||
await file_service.ensure_directory(PathLib(target_file_path).parent)
|
||||
if await file_service.exists(entity.file_path):
|
||||
await file_service.delete_file(entity.file_path)
|
||||
else:
|
||||
await file_service.ensure_directory(PathLib(target_file_path).parent)
|
||||
|
||||
checksum = await file_service.write_file(target_file_path, data.content)
|
||||
|
||||
with telemetry.scope(
|
||||
"api.resource.update.read_metadata",
|
||||
domain="resource",
|
||||
action="update",
|
||||
phase="read_metadata",
|
||||
):
|
||||
file_metadata = await file_service.get_file_metadata(target_file_path)
|
||||
|
||||
file_name = PathLib(target_file_path).name
|
||||
content_type = file_service.content_type(target_file_path)
|
||||
note_type = "canvas" if target_file_path.endswith(".canvas") else "file"
|
||||
|
||||
with telemetry.scope(
|
||||
"api.resource.update.update_entity",
|
||||
domain="resource",
|
||||
action="update",
|
||||
phase="update_entity",
|
||||
):
|
||||
updated_entity = await entity_repository.update(
|
||||
entity.id,
|
||||
{
|
||||
"title": file_name,
|
||||
"note_type": note_type,
|
||||
"content_type": content_type,
|
||||
"file_path": target_file_path,
|
||||
"checksum": checksum,
|
||||
"updated_at": file_metadata.modified_at,
|
||||
},
|
||||
)
|
||||
|
||||
with telemetry.scope(
|
||||
"api.resource.update.search_index",
|
||||
domain="resource",
|
||||
action="update",
|
||||
phase="search_index",
|
||||
):
|
||||
await search_service.index_entity(updated_entity) # pyright: ignore
|
||||
|
||||
return ResourceResponse(
|
||||
entity_id=entity.id,
|
||||
external_id=entity.external_id,
|
||||
file_path=target_file_path,
|
||||
checksum=checksum,
|
||||
size=file_metadata.size,
|
||||
created_at=file_metadata.created_at.timestamp(),
|
||||
modified_at=file_metadata.modified_at.timestamp(),
|
||||
)
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"Invalid file path: {target_file_path}. "
|
||||
"Path must be relative and stay within project boundaries.",
|
||||
)
|
||||
|
||||
# If moving file, handle the move
|
||||
if data.file_path and data.file_path != entity.file_path:
|
||||
# Ensure new parent directory exists (no-op for S3)
|
||||
await file_service.ensure_directory(PathLib(target_file_path).parent)
|
||||
|
||||
# If old file exists, remove it via file_service (for cloud compatibility)
|
||||
if await file_service.exists(entity.file_path):
|
||||
await file_service.delete_file(entity.file_path)
|
||||
else:
|
||||
# Ensure directory exists for in-place update
|
||||
await file_service.ensure_directory(PathLib(target_file_path).parent)
|
||||
|
||||
# Write content to target file
|
||||
checksum = await file_service.write_file(target_file_path, data.content)
|
||||
|
||||
# Get file info
|
||||
file_metadata = await file_service.get_file_metadata(target_file_path)
|
||||
|
||||
# Determine file details
|
||||
file_name = PathLib(target_file_path).name
|
||||
content_type = file_service.content_type(target_file_path)
|
||||
note_type = "canvas" if target_file_path.endswith(".canvas") else "file"
|
||||
|
||||
# Update entity using internal ID
|
||||
updated_entity = await entity_repository.update(
|
||||
entity.id,
|
||||
{
|
||||
"title": file_name,
|
||||
"note_type": note_type,
|
||||
"content_type": content_type,
|
||||
"file_path": target_file_path,
|
||||
"checksum": checksum,
|
||||
"updated_at": file_metadata.modified_at,
|
||||
},
|
||||
)
|
||||
|
||||
# Index the updated file for search
|
||||
await search_service.index_entity(updated_entity) # pyright: ignore
|
||||
|
||||
# Return success response
|
||||
return ResourceResponse(
|
||||
entity_id=entity.id,
|
||||
external_id=entity.external_id,
|
||||
file_path=target_file_path,
|
||||
checksum=checksum,
|
||||
size=file_metadata.size,
|
||||
created_at=file_metadata.created_at.timestamp(),
|
||||
modified_at=file_metadata.modified_at.timestamp(),
|
||||
)
|
||||
except HTTPException:
|
||||
# Re-raise HTTP exceptions without wrapping
|
||||
raise
|
||||
except Exception as e: # pragma: no cover
|
||||
logger.error(f"Error updating resource {entity_id}: {e}")
|
||||
raise HTTPException(status_code=500, detail=f"Failed to update resource: {str(e)}")
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e: # pragma: no cover
|
||||
logger.error(f"Error updating resource {entity_id}: {e}")
|
||||
raise HTTPException(status_code=500, detail=f"Failed to update resource: {str(e)}")
|
||||
|
||||
@@ -6,6 +6,7 @@ V1 uses string-based project names which are less efficient and less stable.
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Path
|
||||
|
||||
from basic_memory import telemetry
|
||||
from basic_memory.api.v2.utils import to_search_results
|
||||
from basic_memory.repository.semantic_errors import (
|
||||
SemanticDependenciesMissingError,
|
||||
@@ -47,29 +48,73 @@ async def search(
|
||||
Returns:
|
||||
SearchResponse with paginated search results
|
||||
"""
|
||||
offset = (page - 1) * page_size
|
||||
# Fetch one extra item to detect whether more pages exist (N+1 trick)
|
||||
fetch_limit = page_size + 1
|
||||
try:
|
||||
results = await search_service.search(query, limit=fetch_limit, offset=offset)
|
||||
except SemanticSearchDisabledError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
except SemanticDependenciesMissingError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
|
||||
has_more = len(results) > page_size
|
||||
if has_more:
|
||||
results = results[:page_size]
|
||||
|
||||
search_results = await to_search_results(entity_service, results)
|
||||
return SearchResponse(
|
||||
results=search_results,
|
||||
current_page=page,
|
||||
with telemetry.operation(
|
||||
"api.request.search",
|
||||
entrypoint="api",
|
||||
domain="search",
|
||||
action="search",
|
||||
page=page,
|
||||
page_size=page_size,
|
||||
has_more=has_more,
|
||||
)
|
||||
retrieval_mode=query.retrieval_mode.value,
|
||||
has_query=bool(
|
||||
(query.text and query.text.strip())
|
||||
or query.title
|
||||
or query.permalink
|
||||
or query.permalink_match
|
||||
),
|
||||
has_filters=bool(query.note_types or query.entity_types or query.metadata_filters),
|
||||
):
|
||||
offset = (page - 1) * page_size
|
||||
fetch_limit = page_size + 1
|
||||
try:
|
||||
with telemetry.scope(
|
||||
"api.search.search.execute_query",
|
||||
domain="search",
|
||||
action="search",
|
||||
phase="execute_query",
|
||||
page=page,
|
||||
page_size=page_size,
|
||||
):
|
||||
results = await search_service.search(query, limit=fetch_limit, offset=offset)
|
||||
except SemanticSearchDisabledError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
except SemanticDependenciesMissingError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
|
||||
with telemetry.scope(
|
||||
"api.search.search.paginate_results",
|
||||
domain="search",
|
||||
action="search",
|
||||
phase="paginate_results",
|
||||
result_count=len(results),
|
||||
):
|
||||
has_more = len(results) > page_size
|
||||
if has_more:
|
||||
results = results[:page_size]
|
||||
|
||||
with telemetry.scope(
|
||||
"api.search.search.hydrate_results",
|
||||
domain="search",
|
||||
action="search",
|
||||
phase="hydrate_results",
|
||||
result_count=len(results),
|
||||
):
|
||||
search_results = await to_search_results(entity_service, results)
|
||||
with telemetry.scope(
|
||||
"api.search.search.build_response",
|
||||
domain="search",
|
||||
action="search",
|
||||
phase="build_response",
|
||||
result_count=len(search_results),
|
||||
):
|
||||
return SearchResponse(
|
||||
results=search_results,
|
||||
current_page=page,
|
||||
page_size=page_size,
|
||||
has_more=has_more,
|
||||
)
|
||||
|
||||
|
||||
@router.post("/search/reindex")
|
||||
|
||||
+215
-158
@@ -1,5 +1,7 @@
|
||||
from typing import Optional, List
|
||||
|
||||
from basic_memory import telemetry
|
||||
from basic_memory.models import Entity as EntityModel
|
||||
from basic_memory.repository import EntityRepository
|
||||
from basic_memory.repository.search_repository import SearchIndexRow
|
||||
from basic_memory.schemas.memory import (
|
||||
@@ -24,169 +26,224 @@ async def to_graph_context(
|
||||
page: Optional[int] = None,
|
||||
page_size: Optional[int] = None,
|
||||
):
|
||||
# First pass: collect all entity IDs needed for external_id lookup
|
||||
# This includes: entity primary results, observation parent entities, relation from/to entities
|
||||
entity_ids_needed: set[int] = set()
|
||||
for context_item in context_result.results:
|
||||
for item in (
|
||||
[context_item.primary_result] + context_item.observations + context_item.related_results
|
||||
):
|
||||
if item.type == SearchItemType.ENTITY:
|
||||
# Entity's own ID for its external_id
|
||||
entity_ids_needed.add(item.id)
|
||||
elif item.type == SearchItemType.OBSERVATION:
|
||||
# Parent entity ID for entity_external_id
|
||||
if item.entity_id: # pyright: ignore
|
||||
entity_ids_needed.add(item.entity_id) # pyright: ignore
|
||||
elif item.type == SearchItemType.RELATION:
|
||||
# Source and target entity IDs for external_ids
|
||||
if item.from_id: # pyright: ignore
|
||||
entity_ids_needed.add(item.from_id) # pyright: ignore
|
||||
if item.to_id:
|
||||
entity_ids_needed.add(item.to_id)
|
||||
|
||||
# Batch fetch all entities at once - get both title and external_id
|
||||
entity_title_lookup: dict[int, str] = {}
|
||||
entity_external_id_lookup: dict[int, str] = {}
|
||||
if entity_ids_needed:
|
||||
entities = await entity_repository.find_by_ids(list(entity_ids_needed))
|
||||
for e in entities:
|
||||
entity_title_lookup[e.id] = e.title
|
||||
entity_external_id_lookup[e.id] = e.external_id
|
||||
|
||||
# Helper function to convert items to summaries
|
||||
def to_summary(item: SearchIndexRow | ContextResultRow):
|
||||
match item.type:
|
||||
case SearchItemType.ENTITY:
|
||||
return EntitySummary(
|
||||
external_id=entity_external_id_lookup.get(item.id, ""),
|
||||
entity_id=item.id,
|
||||
title=item.title, # pyright: ignore
|
||||
permalink=item.permalink,
|
||||
content=item.content,
|
||||
file_path=item.file_path,
|
||||
created_at=item.created_at,
|
||||
)
|
||||
case SearchItemType.OBSERVATION:
|
||||
entity_ext_id = None
|
||||
if item.entity_id: # pyright: ignore
|
||||
entity_ext_id = entity_external_id_lookup.get(item.entity_id) # pyright: ignore
|
||||
return ObservationSummary(
|
||||
observation_id=item.id,
|
||||
entity_id=item.entity_id, # pyright: ignore
|
||||
entity_external_id=entity_ext_id,
|
||||
title=entity_title_lookup.get(item.entity_id), # pyright: ignore
|
||||
file_path=item.file_path,
|
||||
category=item.category, # pyright: ignore
|
||||
content=item.content, # pyright: ignore
|
||||
permalink=item.permalink, # pyright: ignore
|
||||
created_at=item.created_at,
|
||||
)
|
||||
case SearchItemType.RELATION:
|
||||
from_title = entity_title_lookup.get(item.from_id) if item.from_id else None # pyright: ignore
|
||||
to_title = entity_title_lookup.get(item.to_id) if item.to_id else None
|
||||
from_ext_id = entity_external_id_lookup.get(item.from_id) if item.from_id else None # pyright: ignore
|
||||
to_ext_id = entity_external_id_lookup.get(item.to_id) if item.to_id else None
|
||||
return RelationSummary(
|
||||
relation_id=item.id,
|
||||
entity_id=item.entity_id, # pyright: ignore
|
||||
title=item.title, # pyright: ignore
|
||||
file_path=item.file_path,
|
||||
permalink=item.permalink, # pyright: ignore
|
||||
relation_type=item.relation_type, # pyright: ignore
|
||||
from_entity=from_title,
|
||||
from_entity_id=item.from_id, # pyright: ignore
|
||||
from_entity_external_id=from_ext_id,
|
||||
to_entity=to_title,
|
||||
to_entity_id=item.to_id,
|
||||
to_entity_external_id=to_ext_id,
|
||||
created_at=item.created_at,
|
||||
)
|
||||
case _: # pragma: no cover
|
||||
raise ValueError(f"Unexpected type: {item.type}")
|
||||
|
||||
# Process the hierarchical results
|
||||
hierarchical_results = []
|
||||
for context_item in context_result.results:
|
||||
# Process primary result
|
||||
primary_result = to_summary(context_item.primary_result)
|
||||
|
||||
# Process observations (always ObservationSummary, validated by context_service)
|
||||
observations = [to_summary(obs) for obs in context_item.observations]
|
||||
|
||||
# Process related results
|
||||
related = [to_summary(rel) for rel in context_item.related_results]
|
||||
|
||||
# Add to hierarchical results
|
||||
hierarchical_results.append(
|
||||
ContextResult(
|
||||
primary_result=primary_result,
|
||||
observations=observations, # pyright: ignore[reportArgumentType]
|
||||
related_results=related,
|
||||
)
|
||||
)
|
||||
|
||||
# Create schema metadata from service metadata
|
||||
metadata = MemoryMetadata(
|
||||
uri=context_result.metadata.uri,
|
||||
types=context_result.metadata.types,
|
||||
depth=context_result.metadata.depth,
|
||||
timeframe=context_result.metadata.timeframe,
|
||||
generated_at=context_result.metadata.generated_at,
|
||||
primary_count=context_result.metadata.primary_count,
|
||||
related_count=context_result.metadata.related_count,
|
||||
total_results=context_result.metadata.primary_count + context_result.metadata.related_count,
|
||||
total_relations=context_result.metadata.total_relations,
|
||||
total_observations=context_result.metadata.total_observations,
|
||||
)
|
||||
|
||||
# Return new GraphContext with just hierarchical results
|
||||
return GraphContext(
|
||||
results=hierarchical_results,
|
||||
metadata=metadata,
|
||||
with telemetry.scope(
|
||||
"memory.hydrate_context",
|
||||
domain="memory",
|
||||
action="build_context",
|
||||
phase="hydrate_context",
|
||||
page=page,
|
||||
page_size=page_size,
|
||||
has_more=context_result.metadata.has_more,
|
||||
)
|
||||
result_count=len(context_result.results),
|
||||
):
|
||||
# First pass: collect all entity IDs needed for external_id lookup
|
||||
# This includes: entity primary results, observation parent entities, relation from/to entities
|
||||
entity_ids_needed: set[int] = set()
|
||||
for context_item in context_result.results:
|
||||
for item in (
|
||||
[context_item.primary_result]
|
||||
+ context_item.observations
|
||||
+ context_item.related_results
|
||||
):
|
||||
if item.type == SearchItemType.ENTITY:
|
||||
# Entity's own ID for its external_id
|
||||
entity_ids_needed.add(item.id)
|
||||
elif item.type == SearchItemType.OBSERVATION:
|
||||
# Parent entity ID for entity_external_id
|
||||
if item.entity_id: # pyright: ignore
|
||||
entity_ids_needed.add(item.entity_id) # pyright: ignore
|
||||
elif item.type == SearchItemType.RELATION:
|
||||
# Source and target entity IDs for external_ids
|
||||
if item.from_id: # pyright: ignore
|
||||
entity_ids_needed.add(item.from_id) # pyright: ignore
|
||||
if item.to_id:
|
||||
entity_ids_needed.add(item.to_id)
|
||||
|
||||
# Batch fetch all entities at once - get both title and external_id
|
||||
entity_title_lookup: dict[int, str] = {}
|
||||
entity_external_id_lookup: dict[int, str] = {}
|
||||
if entity_ids_needed:
|
||||
with telemetry.scope(
|
||||
"memory.hydrate_context.lookup_entities",
|
||||
domain="memory",
|
||||
action="build_context",
|
||||
phase="lookup_entities",
|
||||
result_count=len(entity_ids_needed),
|
||||
):
|
||||
entities = await entity_repository.find_by_ids(list(entity_ids_needed))
|
||||
for e in entities:
|
||||
entity_title_lookup[e.id] = e.title
|
||||
entity_external_id_lookup[e.id] = e.external_id
|
||||
|
||||
# Helper function to convert items to summaries
|
||||
def to_summary(item: SearchIndexRow | ContextResultRow):
|
||||
match item.type:
|
||||
case SearchItemType.ENTITY:
|
||||
return EntitySummary(
|
||||
external_id=entity_external_id_lookup.get(item.id, ""),
|
||||
entity_id=item.id,
|
||||
title=item.title, # pyright: ignore
|
||||
permalink=item.permalink,
|
||||
content=item.content,
|
||||
file_path=item.file_path,
|
||||
created_at=item.created_at,
|
||||
)
|
||||
case SearchItemType.OBSERVATION:
|
||||
entity_ext_id = None
|
||||
if item.entity_id: # pyright: ignore
|
||||
entity_ext_id = entity_external_id_lookup.get(item.entity_id) # pyright: ignore
|
||||
return ObservationSummary(
|
||||
observation_id=item.id,
|
||||
entity_id=item.entity_id, # pyright: ignore
|
||||
entity_external_id=entity_ext_id,
|
||||
title=entity_title_lookup.get(item.entity_id), # pyright: ignore
|
||||
file_path=item.file_path,
|
||||
category=item.category, # pyright: ignore
|
||||
content=item.content, # pyright: ignore
|
||||
permalink=item.permalink, # pyright: ignore
|
||||
created_at=item.created_at,
|
||||
)
|
||||
case SearchItemType.RELATION:
|
||||
from_title = entity_title_lookup.get(item.from_id) if item.from_id else None # pyright: ignore
|
||||
to_title = entity_title_lookup.get(item.to_id) if item.to_id else None
|
||||
from_ext_id = (
|
||||
entity_external_id_lookup.get(item.from_id) if item.from_id else None
|
||||
) # pyright: ignore
|
||||
to_ext_id = entity_external_id_lookup.get(item.to_id) if item.to_id else None
|
||||
return RelationSummary(
|
||||
relation_id=item.id,
|
||||
entity_id=item.entity_id, # pyright: ignore
|
||||
title=item.title, # pyright: ignore
|
||||
file_path=item.file_path,
|
||||
permalink=item.permalink, # pyright: ignore
|
||||
relation_type=item.relation_type, # pyright: ignore
|
||||
from_entity=from_title,
|
||||
from_entity_id=item.from_id, # pyright: ignore
|
||||
from_entity_external_id=from_ext_id,
|
||||
to_entity=to_title,
|
||||
to_entity_id=item.to_id,
|
||||
to_entity_external_id=to_ext_id,
|
||||
created_at=item.created_at,
|
||||
)
|
||||
case _: # pragma: no cover
|
||||
raise ValueError(f"Unexpected type: {item.type}")
|
||||
|
||||
with telemetry.scope(
|
||||
"memory.hydrate_context.shape_results",
|
||||
domain="memory",
|
||||
action="build_context",
|
||||
phase="shape_results",
|
||||
result_count=len(context_result.results),
|
||||
):
|
||||
hierarchical_results = []
|
||||
for context_item in context_result.results:
|
||||
primary_result = to_summary(context_item.primary_result)
|
||||
observations = [to_summary(obs) for obs in context_item.observations]
|
||||
related = [to_summary(rel) for rel in context_item.related_results]
|
||||
hierarchical_results.append(
|
||||
ContextResult(
|
||||
primary_result=primary_result,
|
||||
observations=observations, # pyright: ignore[reportArgumentType]
|
||||
related_results=related,
|
||||
)
|
||||
)
|
||||
|
||||
metadata = MemoryMetadata(
|
||||
uri=context_result.metadata.uri,
|
||||
types=context_result.metadata.types,
|
||||
depth=context_result.metadata.depth,
|
||||
timeframe=context_result.metadata.timeframe,
|
||||
generated_at=context_result.metadata.generated_at,
|
||||
primary_count=context_result.metadata.primary_count,
|
||||
related_count=context_result.metadata.related_count,
|
||||
total_results=context_result.metadata.primary_count
|
||||
+ context_result.metadata.related_count,
|
||||
total_relations=context_result.metadata.total_relations,
|
||||
total_observations=context_result.metadata.total_observations,
|
||||
)
|
||||
|
||||
return GraphContext(
|
||||
results=hierarchical_results,
|
||||
metadata=metadata,
|
||||
page=page,
|
||||
page_size=page_size,
|
||||
has_more=context_result.metadata.has_more,
|
||||
)
|
||||
|
||||
|
||||
async def to_search_results(entity_service: EntityService, results: List[SearchIndexRow]):
|
||||
search_results = []
|
||||
for r in results:
|
||||
entities = await entity_service.get_entities_by_id([r.entity_id, r.from_id, r.to_id]) # pyright: ignore
|
||||
with telemetry.scope(
|
||||
"search.hydrate_results",
|
||||
domain="search",
|
||||
action="search",
|
||||
phase="hydrate_results",
|
||||
result_count=len(results),
|
||||
):
|
||||
# Collect all unique entity IDs across all results in a single pass
|
||||
# This avoids N+1 queries — one batch fetch instead of one per result
|
||||
all_entity_ids: set[int] = set()
|
||||
for result in results:
|
||||
for eid in (result.entity_id, result.from_id, result.to_id):
|
||||
if eid is not None:
|
||||
all_entity_ids.add(eid)
|
||||
|
||||
# Determine which IDs to set based on type
|
||||
entity_id = None
|
||||
observation_id = None
|
||||
relation_id = None
|
||||
# Single batch fetch for all entities
|
||||
entities_by_id: dict[int, EntityModel] = {}
|
||||
with telemetry.scope(
|
||||
"search.hydrate_results.fetch_entities",
|
||||
domain="search",
|
||||
action="search",
|
||||
phase="fetch_entities",
|
||||
result_count=len(all_entity_ids),
|
||||
):
|
||||
if all_entity_ids:
|
||||
entities = await entity_service.get_entities_by_id(list(all_entity_ids))
|
||||
entities_by_id = {e.id: e for e in entities}
|
||||
|
||||
if r.type == SearchItemType.ENTITY:
|
||||
entity_id = r.id
|
||||
elif r.type == SearchItemType.OBSERVATION:
|
||||
observation_id = r.id
|
||||
entity_id = r.entity_id # Parent entity
|
||||
elif r.type == SearchItemType.RELATION:
|
||||
relation_id = r.id
|
||||
entity_id = r.entity_id # Parent entity
|
||||
search_results = []
|
||||
with telemetry.scope(
|
||||
"search.hydrate_results.shape_results",
|
||||
domain="search",
|
||||
action="search",
|
||||
phase="shape_results",
|
||||
result_count=len(results),
|
||||
):
|
||||
for result in results:
|
||||
entity_id = None
|
||||
observation_id = None
|
||||
relation_id = None
|
||||
|
||||
search_results.append(
|
||||
SearchResult(
|
||||
title=r.title, # pyright: ignore
|
||||
type=r.type, # pyright: ignore
|
||||
permalink=r.permalink,
|
||||
score=r.score, # pyright: ignore
|
||||
entity=entities[0].permalink if entities else None,
|
||||
content=r.content,
|
||||
matched_chunk=r.matched_chunk_text,
|
||||
file_path=r.file_path,
|
||||
metadata=r.metadata,
|
||||
entity_id=entity_id,
|
||||
observation_id=observation_id,
|
||||
relation_id=relation_id,
|
||||
category=r.category,
|
||||
from_entity=entities[0].permalink if entities else None,
|
||||
to_entity=entities[1].permalink if len(entities) > 1 else None,
|
||||
relation_type=r.relation_type,
|
||||
)
|
||||
)
|
||||
return search_results
|
||||
if result.type == SearchItemType.ENTITY:
|
||||
entity_id = result.id
|
||||
elif result.type == SearchItemType.OBSERVATION:
|
||||
observation_id = result.id
|
||||
entity_id = result.entity_id
|
||||
elif result.type == SearchItemType.RELATION:
|
||||
relation_id = result.id
|
||||
entity_id = result.entity_id
|
||||
|
||||
# Look up entities by their specific IDs
|
||||
parent_entity = entities_by_id.get(result.entity_id) if result.entity_id else None # pyright: ignore
|
||||
from_entity = entities_by_id.get(result.from_id) if result.from_id else None # pyright: ignore
|
||||
to_entity = entities_by_id.get(result.to_id) if result.to_id else None
|
||||
|
||||
search_results.append(
|
||||
SearchResult(
|
||||
title=result.title, # pyright: ignore
|
||||
type=result.type, # pyright: ignore
|
||||
permalink=result.permalink,
|
||||
score=result.score, # pyright: ignore
|
||||
entity=parent_entity.permalink if parent_entity else None,
|
||||
content=result.content,
|
||||
matched_chunk=result.matched_chunk_text,
|
||||
file_path=result.file_path,
|
||||
metadata=result.metadata,
|
||||
entity_id=entity_id,
|
||||
observation_id=observation_id,
|
||||
relation_id=relation_id,
|
||||
category=result.category,
|
||||
from_entity=from_entity.permalink if from_entity else None,
|
||||
to_entity=to_entity.permalink if to_entity else None,
|
||||
relation_type=result.relation_type,
|
||||
)
|
||||
)
|
||||
return search_results
|
||||
|
||||
@@ -8,9 +8,11 @@ from typing import Optional # noqa: E402
|
||||
|
||||
import typer # noqa: E402
|
||||
|
||||
from basic_memory.cli.auto_update import maybe_run_periodic_auto_update # noqa: E402
|
||||
from basic_memory.cli.container import CliContainer, set_container # noqa: E402
|
||||
from basic_memory.cli.promo import maybe_show_cloud_promo, maybe_show_init_line # noqa: E402
|
||||
from basic_memory.config import init_cli_logging # noqa: E402
|
||||
from basic_memory import telemetry # noqa: E402
|
||||
|
||||
|
||||
def version_callback(value: bool) -> None:
|
||||
@@ -41,6 +43,14 @@ def app_callback(
|
||||
|
||||
# Initialize logging for CLI (file only, no stdout)
|
||||
init_cli_logging()
|
||||
command_name = ctx.invoked_subcommand or "root"
|
||||
ctx.with_resource(
|
||||
telemetry.operation(
|
||||
f"cli.command.{command_name}",
|
||||
entrypoint="cli",
|
||||
command_name=command_name,
|
||||
)
|
||||
)
|
||||
|
||||
# --- Composition Root ---
|
||||
# Create container and read config (single point of config access)
|
||||
@@ -52,10 +62,14 @@ def app_callback(
|
||||
# Outcome: one-time plain line printed before the subcommand runs.
|
||||
maybe_show_init_line(ctx.invoked_subcommand)
|
||||
|
||||
# Trigger: register promo as a post-command callback.
|
||||
# Why: promo output should appear after the command's own output, not before.
|
||||
# Outcome: promo panel renders below the command results (status tree, table, etc.).
|
||||
ctx.call_on_close(lambda: maybe_show_cloud_promo(ctx.invoked_subcommand))
|
||||
# Trigger: register post-command messaging callbacks.
|
||||
# Why: informational/promo/update output belongs below command results.
|
||||
# Outcome: command output remains primary, with optional follow-up notices afterwards.
|
||||
def _post_command_messages() -> None:
|
||||
maybe_show_cloud_promo(ctx.invoked_subcommand)
|
||||
maybe_run_periodic_auto_update(ctx.invoked_subcommand)
|
||||
|
||||
ctx.call_on_close(_post_command_messages)
|
||||
|
||||
# Run initialization for commands that don't use the API
|
||||
# Skip for 'mcp' command - it has its own lifespan that handles initialization
|
||||
@@ -70,6 +84,7 @@ def app_callback(
|
||||
"tool",
|
||||
"reset",
|
||||
"reindex",
|
||||
"update",
|
||||
"watch",
|
||||
}
|
||||
if (
|
||||
|
||||
@@ -0,0 +1,389 @@
|
||||
"""Automatic update checks and upgrades for the Basic Memory CLI."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timedelta
|
||||
from enum import Enum
|
||||
|
||||
from loguru import logger
|
||||
from packaging.version import InvalidVersion, Version
|
||||
from rich.console import Console
|
||||
|
||||
import basic_memory
|
||||
from basic_memory.config import ConfigManager
|
||||
|
||||
PACKAGE_NAME = "basic-memory"
|
||||
PYPI_JSON_URL = "https://pypi.org/pypi/basic-memory/json"
|
||||
|
||||
PYPI_TIMEOUT_SECONDS = 5
|
||||
BREW_OUTDATED_TIMEOUT_SECONDS = 15
|
||||
UV_UPGRADE_TIMEOUT_SECONDS = 180
|
||||
BREW_UPGRADE_TIMEOUT_SECONDS = 600
|
||||
|
||||
|
||||
class InstallSource(str, Enum):
|
||||
"""How the running CLI appears to have been installed."""
|
||||
|
||||
HOMEBREW = "homebrew"
|
||||
UV_TOOL = "uv_tool"
|
||||
UVX = "uvx"
|
||||
UNKNOWN = "unknown"
|
||||
|
||||
|
||||
class AutoUpdateStatus(str, Enum):
|
||||
"""Result classification for update checks and installs."""
|
||||
|
||||
SKIPPED = "skipped"
|
||||
UP_TO_DATE = "up_to_date"
|
||||
UPDATE_AVAILABLE = "update_available"
|
||||
UPDATED = "updated"
|
||||
FAILED = "failed"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class AutoUpdateResult:
|
||||
"""Structured result for update checks/install attempts."""
|
||||
|
||||
status: AutoUpdateStatus
|
||||
source: InstallSource
|
||||
checked: bool
|
||||
update_available: bool
|
||||
updated: bool
|
||||
latest_version: str | None = None
|
||||
message: str | None = None
|
||||
error: str | None = None
|
||||
restart_recommended: bool = False
|
||||
|
||||
|
||||
def detect_install_source(executable: str | None = None) -> InstallSource:
|
||||
"""Infer installation source from the active interpreter path."""
|
||||
active_executable = executable or sys.executable
|
||||
normalized = active_executable.lower().replace("\\", "/")
|
||||
|
||||
if "cellar/basic-memory" in normalized:
|
||||
return InstallSource.HOMEBREW
|
||||
if "uv/tools/basic-memory" in normalized:
|
||||
return InstallSource.UV_TOOL
|
||||
if "/uv/archive-" in normalized:
|
||||
return InstallSource.UVX
|
||||
return InstallSource.UNKNOWN
|
||||
|
||||
|
||||
def _is_interactive_session() -> bool:
|
||||
"""Return whether stdin/stdout are interactive terminals."""
|
||||
try:
|
||||
return sys.stdin.isatty() and sys.stdout.isatty()
|
||||
except ValueError:
|
||||
# Trigger: stdin/stdout may be closed during transport teardown.
|
||||
# Why: isatty() raises ValueError on closed descriptors.
|
||||
# Outcome: treat as non-interactive and suppress periodic output.
|
||||
return False
|
||||
|
||||
|
||||
def _run_subprocess(
|
||||
command: list[str],
|
||||
*,
|
||||
timeout_seconds: int,
|
||||
silent: bool,
|
||||
capture_output: bool,
|
||||
) -> subprocess.CompletedProcess[str]:
|
||||
"""Run a subprocess with explicit stdio behavior for protocol safety."""
|
||||
# Trigger: silent operation (MCP/background) with no need for subprocess output.
|
||||
# Why: prevent protocol/terminal pollution from child process output.
|
||||
# Outcome: stdout/stderr are discarded unless explicit capture is requested.
|
||||
use_devnull = silent and not capture_output
|
||||
stdout_target = subprocess.DEVNULL if use_devnull else subprocess.PIPE
|
||||
stderr_target = subprocess.DEVNULL if use_devnull else subprocess.PIPE
|
||||
|
||||
return subprocess.run(
|
||||
command,
|
||||
stdin=subprocess.DEVNULL,
|
||||
stdout=stdout_target,
|
||||
stderr=stderr_target,
|
||||
text=True,
|
||||
timeout=timeout_seconds,
|
||||
check=False,
|
||||
)
|
||||
|
||||
|
||||
def _version_from_pypi() -> str:
|
||||
"""Fetch the latest published package version from PyPI."""
|
||||
request = urllib.request.Request(
|
||||
PYPI_JSON_URL,
|
||||
headers={"User-Agent": f"basic-memory-cli/{basic_memory.__version__}"},
|
||||
)
|
||||
with urllib.request.urlopen(request, timeout=PYPI_TIMEOUT_SECONDS) as response:
|
||||
payload = json.loads(response.read().decode("utf-8"))
|
||||
latest = payload.get("info", {}).get("version")
|
||||
if not latest:
|
||||
raise RuntimeError("PyPI JSON response did not include info.version")
|
||||
return str(latest)
|
||||
|
||||
|
||||
def _check_homebrew_update_available(silent: bool) -> tuple[bool, str | None]:
|
||||
"""Check whether Homebrew reports an outdated basic-memory formula."""
|
||||
result = _run_subprocess(
|
||||
["brew", "outdated", "--quiet", PACKAGE_NAME],
|
||||
timeout_seconds=BREW_OUTDATED_TIMEOUT_SECONDS,
|
||||
silent=silent,
|
||||
capture_output=True,
|
||||
)
|
||||
# Trigger: brew outdated exits 1 when the formula IS outdated (with name on stdout).
|
||||
# Why: non-zero exit here means "outdated", not "error".
|
||||
# Outcome: check stdout for the package name to determine outdated status.
|
||||
stdout = (result.stdout or "").strip()
|
||||
is_outdated = PACKAGE_NAME in stdout
|
||||
return is_outdated, None
|
||||
|
||||
|
||||
def _check_pypi_update_available() -> tuple[bool, str]:
|
||||
"""Compare installed package version with PyPI latest version."""
|
||||
latest = _version_from_pypi()
|
||||
try:
|
||||
current_version = Version(basic_memory.__version__)
|
||||
latest_version = Version(latest)
|
||||
except InvalidVersion as exc:
|
||||
raise RuntimeError(
|
||||
f"Could not compare versions (current={basic_memory.__version__}, latest={latest})"
|
||||
) from exc
|
||||
|
||||
return latest_version > current_version, latest
|
||||
|
||||
|
||||
def _manual_update_hint(source: InstallSource) -> str:
|
||||
"""Return manager-appropriate manual update instructions."""
|
||||
if source == InstallSource.UV_TOOL:
|
||||
return "Run `uv tool upgrade basic-memory`."
|
||||
if source == InstallSource.HOMEBREW:
|
||||
return "Run `brew upgrade basic-memory`."
|
||||
return (
|
||||
"Automatic install is not supported for this environment. "
|
||||
"Update with your package manager (for pip: `python3 -m pip install -U basic-memory`)."
|
||||
)
|
||||
|
||||
|
||||
def _save_last_checked_timestamp(config_manager: ConfigManager, checked_at: datetime) -> None:
|
||||
"""Persist the timestamp for the most recent attempted update check."""
|
||||
config = config_manager.load_config()
|
||||
config.auto_update_last_checked_at = checked_at
|
||||
config_manager.save_config(config)
|
||||
|
||||
|
||||
def run_auto_update(
|
||||
*,
|
||||
force: bool = False,
|
||||
check_only: bool = False,
|
||||
silent: bool = False,
|
||||
config_manager: ConfigManager | None = None,
|
||||
now: datetime | None = None,
|
||||
executable: str | None = None,
|
||||
) -> AutoUpdateResult:
|
||||
"""Run update check/install flow and return a structured result."""
|
||||
manager = config_manager or ConfigManager()
|
||||
config = manager.load_config()
|
||||
source = detect_install_source(executable)
|
||||
checked_at = now or datetime.now()
|
||||
|
||||
if source == InstallSource.UVX:
|
||||
return AutoUpdateResult(
|
||||
status=AutoUpdateStatus.SKIPPED,
|
||||
source=source,
|
||||
checked=False,
|
||||
update_available=False,
|
||||
updated=False,
|
||||
message="uvx runtime detected; updates are managed by uvx cache resolution.",
|
||||
)
|
||||
|
||||
if not force and not config.auto_update:
|
||||
return AutoUpdateResult(
|
||||
status=AutoUpdateStatus.SKIPPED,
|
||||
source=source,
|
||||
checked=False,
|
||||
update_available=False,
|
||||
updated=False,
|
||||
message="Auto-update is disabled in config.",
|
||||
)
|
||||
|
||||
if not force and config.auto_update_last_checked_at is not None:
|
||||
try:
|
||||
elapsed = checked_at - config.auto_update_last_checked_at
|
||||
except TypeError:
|
||||
# Trigger: mixed naive/aware datetimes from manual config edits.
|
||||
# Why: datetime subtraction fails for mixed tz-awareness.
|
||||
# Outcome: ignore the gate once and continue with a forced check path.
|
||||
logger.warning("Auto-update interval gate skipped due to incompatible timestamp format")
|
||||
else:
|
||||
if elapsed < timedelta(seconds=config.update_check_interval):
|
||||
return AutoUpdateResult(
|
||||
status=AutoUpdateStatus.SKIPPED,
|
||||
source=source,
|
||||
checked=False,
|
||||
update_available=False,
|
||||
updated=False,
|
||||
message="Update check interval has not elapsed.",
|
||||
)
|
||||
|
||||
try:
|
||||
# --- Availability check ---
|
||||
latest_version: str | None = None
|
||||
if source == InstallSource.HOMEBREW:
|
||||
update_available, latest_version = _check_homebrew_update_available(silent=silent)
|
||||
else:
|
||||
update_available, latest_version = _check_pypi_update_available()
|
||||
|
||||
if not update_available:
|
||||
return AutoUpdateResult(
|
||||
status=AutoUpdateStatus.UP_TO_DATE,
|
||||
source=source,
|
||||
checked=True,
|
||||
update_available=False,
|
||||
updated=False,
|
||||
latest_version=latest_version,
|
||||
message=f"Basic Memory is up to date ({basic_memory.__version__}).",
|
||||
)
|
||||
|
||||
if check_only:
|
||||
return AutoUpdateResult(
|
||||
status=AutoUpdateStatus.UPDATE_AVAILABLE,
|
||||
source=source,
|
||||
checked=True,
|
||||
update_available=True,
|
||||
updated=False,
|
||||
latest_version=latest_version,
|
||||
message=(
|
||||
f"Update available (latest: {latest_version or 'unknown'}). "
|
||||
f"{_manual_update_hint(source)}"
|
||||
),
|
||||
)
|
||||
|
||||
if source == InstallSource.UNKNOWN:
|
||||
return AutoUpdateResult(
|
||||
status=AutoUpdateStatus.UPDATE_AVAILABLE,
|
||||
source=source,
|
||||
checked=True,
|
||||
update_available=True,
|
||||
updated=False,
|
||||
latest_version=latest_version,
|
||||
message=(
|
||||
f"Update available (latest: {latest_version or 'unknown'}). "
|
||||
f"{_manual_update_hint(source)}"
|
||||
),
|
||||
)
|
||||
|
||||
# --- Automatic install ---
|
||||
command = (
|
||||
["uv", "tool", "upgrade", PACKAGE_NAME]
|
||||
if source == InstallSource.UV_TOOL
|
||||
else ["brew", "upgrade", PACKAGE_NAME]
|
||||
)
|
||||
timeout = (
|
||||
UV_UPGRADE_TIMEOUT_SECONDS
|
||||
if source == InstallSource.UV_TOOL
|
||||
else BREW_UPGRADE_TIMEOUT_SECONDS
|
||||
)
|
||||
|
||||
install_result = _run_subprocess(
|
||||
command,
|
||||
timeout_seconds=timeout,
|
||||
silent=silent,
|
||||
capture_output=not silent,
|
||||
)
|
||||
if install_result.returncode != 0:
|
||||
stderr = (install_result.stderr or "").strip() if install_result.stderr else ""
|
||||
stdout = (install_result.stdout or "").strip() if install_result.stdout else ""
|
||||
detail = stderr or stdout or "update command failed"
|
||||
return AutoUpdateResult(
|
||||
status=AutoUpdateStatus.FAILED,
|
||||
source=source,
|
||||
checked=True,
|
||||
update_available=True,
|
||||
updated=False,
|
||||
latest_version=latest_version,
|
||||
message="Automatic update failed.",
|
||||
error=detail,
|
||||
)
|
||||
|
||||
return AutoUpdateResult(
|
||||
status=AutoUpdateStatus.UPDATED,
|
||||
source=source,
|
||||
checked=True,
|
||||
update_available=True,
|
||||
updated=True,
|
||||
latest_version=latest_version,
|
||||
message=(
|
||||
"Basic Memory was updated successfully. "
|
||||
"Restart running sessions to use the new version."
|
||||
),
|
||||
restart_recommended=True,
|
||||
)
|
||||
|
||||
except (
|
||||
RuntimeError,
|
||||
urllib.error.URLError,
|
||||
ValueError,
|
||||
TimeoutError,
|
||||
subprocess.SubprocessError,
|
||||
OSError,
|
||||
) as exc:
|
||||
logger.warning(f"Auto-update check failed: {exc}")
|
||||
return AutoUpdateResult(
|
||||
status=AutoUpdateStatus.FAILED,
|
||||
source=source,
|
||||
checked=True,
|
||||
update_available=False,
|
||||
updated=False,
|
||||
message="Automatic update check failed.",
|
||||
error=str(exc),
|
||||
)
|
||||
finally:
|
||||
# Trigger: we attempted a check path (including failures).
|
||||
# Why: repeated failing checks on every command create noise and unnecessary network load.
|
||||
# Outcome: next periodic check is gated by update_check_interval.
|
||||
try:
|
||||
_save_last_checked_timestamp(manager, checked_at)
|
||||
except Exception as exc: # pragma: no cover
|
||||
logger.warning(f"Failed to persist auto-update timestamp: {exc}")
|
||||
|
||||
|
||||
def maybe_run_periodic_auto_update(
|
||||
invoked_subcommand: str | None,
|
||||
*,
|
||||
config_manager: ConfigManager | None = None,
|
||||
is_interactive: bool | None = None,
|
||||
console: Console | None = None,
|
||||
) -> AutoUpdateResult | None:
|
||||
"""Run a periodic auto-update check for interactive CLI sessions."""
|
||||
interactive = _is_interactive_session() if is_interactive is None else is_interactive
|
||||
if not interactive:
|
||||
return None
|
||||
if invoked_subcommand in {None, "mcp", "update"}:
|
||||
return None
|
||||
|
||||
result = run_auto_update(
|
||||
force=False,
|
||||
check_only=False,
|
||||
silent=False,
|
||||
config_manager=config_manager,
|
||||
)
|
||||
|
||||
if result.status in {
|
||||
AutoUpdateStatus.UPDATE_AVAILABLE,
|
||||
AutoUpdateStatus.UPDATED,
|
||||
AutoUpdateStatus.FAILED,
|
||||
}:
|
||||
out = console or Console()
|
||||
if result.status == AutoUpdateStatus.UPDATED:
|
||||
out.print(f"[green]{result.message}[/green]")
|
||||
elif result.status == AutoUpdateStatus.FAILED:
|
||||
error_detail = f" {result.error}" if result.error else ""
|
||||
out.print(f"[yellow]{result.message}{error_detail}[/yellow]")
|
||||
elif result.message:
|
||||
out.print(f"[cyan]{result.message}[/cyan]")
|
||||
|
||||
return result
|
||||
@@ -8,6 +8,7 @@ from . import (
|
||||
project,
|
||||
format,
|
||||
schema,
|
||||
update,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
@@ -23,4 +24,5 @@ __all__ = [
|
||||
"project",
|
||||
"format",
|
||||
"schema",
|
||||
"update",
|
||||
]
|
||||
|
||||
@@ -45,14 +45,26 @@ def get_cloud_config() -> tuple[str, str, str]:
|
||||
|
||||
async def get_authenticated_headers(auth: CLIAuth | None = None) -> dict[str, str]:
|
||||
"""
|
||||
Get authentication headers with JWT token.
|
||||
handles jwt refresh if needed.
|
||||
Get authentication headers for cloud API requests.
|
||||
|
||||
Credential priority mirrors async_client._resolve_cloud_token():
|
||||
1. API key (config.cloud_api_key) — fast, no refresh needed
|
||||
2. OAuth token via CLIAuth — handles JWT refresh automatically
|
||||
"""
|
||||
# --- API key (preferred) ---
|
||||
config_manager = ConfigManager()
|
||||
api_key = config_manager.config.cloud_api_key
|
||||
if api_key:
|
||||
return {"Authorization": f"Bearer {api_key}"}
|
||||
|
||||
# --- OAuth fallback ---
|
||||
client_id, domain, _ = get_cloud_config()
|
||||
auth_obj = auth or CLIAuth(client_id=client_id, authkit_domain=domain)
|
||||
token = await auth_obj.get_valid_token()
|
||||
if not token:
|
||||
console.print("[red]Not authenticated. Please run 'bm cloud login' first.[/red]")
|
||||
console.print(
|
||||
"[red]Not authenticated. Run 'bm cloud set-key <key>' or 'bm cloud login' first.[/red]"
|
||||
)
|
||||
raise typer.Exit(1)
|
||||
|
||||
return {"Authorization": f"Bearer {token}"}
|
||||
|
||||
@@ -2,10 +2,12 @@
|
||||
|
||||
from basic_memory.cli.commands.cloud.api_client import make_api_request
|
||||
from basic_memory.config import ConfigManager
|
||||
from basic_memory.mcp.async_client import resolve_configured_workspace
|
||||
from basic_memory.schemas.cloud import (
|
||||
CloudProjectList,
|
||||
CloudProjectCreateRequest,
|
||||
CloudProjectCreateResponse,
|
||||
ProjectVisibility,
|
||||
)
|
||||
from basic_memory.utils import generate_permalink
|
||||
|
||||
@@ -16,8 +18,25 @@ class CloudUtilsError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def _workspace_headers(
|
||||
*,
|
||||
project_name: str | None = None,
|
||||
workspace: str | None = None,
|
||||
) -> dict[str, str]:
|
||||
"""Build optional workspace headers using the CLI config resolution chain."""
|
||||
resolved_workspace = resolve_configured_workspace(
|
||||
project_name=project_name,
|
||||
workspace=workspace,
|
||||
)
|
||||
if resolved_workspace is None:
|
||||
return {}
|
||||
return {"X-Workspace-ID": resolved_workspace}
|
||||
|
||||
|
||||
async def fetch_cloud_projects(
|
||||
*,
|
||||
project_name: str | None = None,
|
||||
workspace: str | None = None,
|
||||
api_request=make_api_request,
|
||||
) -> CloudProjectList:
|
||||
"""Fetch list of projects from cloud API.
|
||||
@@ -30,7 +49,11 @@ async def fetch_cloud_projects(
|
||||
config = config_manager.config
|
||||
host_url = config.cloud_host.rstrip("/")
|
||||
|
||||
response = await api_request(method="GET", url=f"{host_url}/proxy/v2/projects/")
|
||||
response = await api_request(
|
||||
method="GET",
|
||||
url=f"{host_url}/proxy/v2/projects/",
|
||||
headers=_workspace_headers(project_name=project_name, workspace=workspace),
|
||||
)
|
||||
|
||||
return CloudProjectList.model_validate(response.json())
|
||||
except Exception as e:
|
||||
@@ -40,12 +63,16 @@ async def fetch_cloud_projects(
|
||||
async def create_cloud_project(
|
||||
project_name: str,
|
||||
*,
|
||||
workspace: str | None = None,
|
||||
visibility: ProjectVisibility = "workspace",
|
||||
api_request=make_api_request,
|
||||
) -> CloudProjectCreateResponse:
|
||||
"""Create a new project on cloud.
|
||||
|
||||
Args:
|
||||
project_name: Name of project to create
|
||||
workspace: Optional workspace override for tenant-scoped project creation
|
||||
visibility: Visibility for the created cloud project
|
||||
|
||||
Returns:
|
||||
CloudProjectCreateResponse with project details from API
|
||||
@@ -62,12 +89,16 @@ async def create_cloud_project(
|
||||
name=project_name,
|
||||
path=project_path,
|
||||
set_default=False,
|
||||
visibility=visibility,
|
||||
)
|
||||
|
||||
response = await api_request(
|
||||
method="POST",
|
||||
url=f"{host_url}/proxy/v2/projects/",
|
||||
headers={"Content-Type": "application/json"},
|
||||
headers={
|
||||
"Content-Type": "application/json",
|
||||
**_workspace_headers(project_name=project_name, workspace=workspace),
|
||||
},
|
||||
json_data=project_data.model_dump(),
|
||||
)
|
||||
|
||||
@@ -91,18 +122,28 @@ async def sync_project(project_name: str, force_full: bool = False) -> None:
|
||||
raise CloudUtilsError(f"Failed to sync project '{project_name}': {e}") from e
|
||||
|
||||
|
||||
async def project_exists(project_name: str, *, api_request=make_api_request) -> bool:
|
||||
async def project_exists(
|
||||
project_name: str,
|
||||
*,
|
||||
workspace: str | None = None,
|
||||
api_request=make_api_request,
|
||||
) -> bool:
|
||||
"""Check if a project exists on cloud.
|
||||
|
||||
Args:
|
||||
project_name: Name of project to check
|
||||
workspace: Optional workspace override for tenant-scoped project lookup
|
||||
|
||||
Returns:
|
||||
True if project exists, False otherwise
|
||||
|
||||
Raises:
|
||||
CloudUtilsError: If the project list cannot be fetched from cloud
|
||||
"""
|
||||
try:
|
||||
projects = await fetch_cloud_projects(api_request=api_request)
|
||||
project_names = {p.name for p in projects.projects}
|
||||
return project_name in project_names
|
||||
except Exception:
|
||||
return False
|
||||
projects = await fetch_cloud_projects(
|
||||
project_name=project_name,
|
||||
workspace=workspace,
|
||||
api_request=api_request,
|
||||
)
|
||||
project_names = {p.name for p in projects.projects}
|
||||
return project_name in project_names
|
||||
|
||||
@@ -54,7 +54,7 @@ def _require_cloud_credentials(config) -> None:
|
||||
|
||||
async def _get_cloud_project(name: str) -> ProjectItem | None:
|
||||
"""Fetch a project by name from the cloud API."""
|
||||
async with get_client() as client:
|
||||
async with get_client(project_name=name) as client:
|
||||
projects_list = await ProjectClient(client).list_projects()
|
||||
for proj in projects_list.projects:
|
||||
if generate_permalink(proj.name) == generate_permalink(name):
|
||||
@@ -129,9 +129,9 @@ def sync_project_command(
|
||||
if not dry_run:
|
||||
|
||||
async def _trigger_db_sync():
|
||||
async with get_client() as client:
|
||||
async with get_client(project_name=name) as client:
|
||||
return await ProjectClient(client).sync(
|
||||
project_data.external_id, force_full=True
|
||||
project_data.external_id, force_full=False
|
||||
)
|
||||
|
||||
try:
|
||||
@@ -195,7 +195,10 @@ def bisync_project_command(
|
||||
# Update config — sync_entry is guaranteed non-None because
|
||||
# _get_sync_project validated local_sync_path (which comes from sync_entry)
|
||||
sync_entry = config.projects.get(name)
|
||||
assert sync_entry is not None
|
||||
if sync_entry is None:
|
||||
raise RuntimeError(
|
||||
f"Sync entry for project '{name}' unexpectedly missing after validation"
|
||||
)
|
||||
sync_entry.last_sync = datetime.now()
|
||||
sync_entry.bisync_initialized = True
|
||||
ConfigManager().save_config(config)
|
||||
@@ -204,9 +207,9 @@ def bisync_project_command(
|
||||
if not dry_run:
|
||||
|
||||
async def _trigger_db_sync():
|
||||
async with get_client() as client:
|
||||
async with get_client(project_name=name) as client:
|
||||
return await ProjectClient(client).sync(
|
||||
project_data.external_id, force_full=True
|
||||
project_data.external_id, force_full=False
|
||||
)
|
||||
|
||||
try:
|
||||
@@ -320,7 +323,7 @@ def setup_project_sync(
|
||||
|
||||
async def _verify_project_exists():
|
||||
"""Verify the project exists on cloud by listing all projects."""
|
||||
async with get_client() as client:
|
||||
async with get_client(project_name=name) as client:
|
||||
projects_list = await ProjectClient(client).list_projects()
|
||||
project_names = [p.name for p in projects_list.projects]
|
||||
if name not in project_names:
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
"""Upload CLI commands for basic-memory projects."""
|
||||
|
||||
from functools import partial
|
||||
from pathlib import Path
|
||||
|
||||
import typer
|
||||
@@ -8,12 +9,16 @@ from rich.console import Console
|
||||
from basic_memory.cli.app import cloud_app
|
||||
from basic_memory.cli.commands.command_utils import run_with_cleanup
|
||||
from basic_memory.cli.commands.cloud.cloud_utils import (
|
||||
CloudUtilsError,
|
||||
create_cloud_project,
|
||||
project_exists,
|
||||
sync_project,
|
||||
)
|
||||
from basic_memory.cli.commands.cloud.upload import upload_path
|
||||
from basic_memory.mcp.async_client import get_cloud_control_plane_client
|
||||
from basic_memory.mcp.async_client import (
|
||||
get_cloud_control_plane_client,
|
||||
resolve_configured_workspace,
|
||||
)
|
||||
|
||||
console = Console()
|
||||
|
||||
@@ -73,12 +78,20 @@ def upload(
|
||||
"""
|
||||
|
||||
async def _upload():
|
||||
resolved_workspace = resolve_configured_workspace(project_name=project)
|
||||
|
||||
try:
|
||||
project_already_exists = await project_exists(project, workspace=resolved_workspace)
|
||||
except CloudUtilsError as e:
|
||||
console.print(f"[red]Failed to check cloud project '{project}': {e}[/red]")
|
||||
raise typer.Exit(1)
|
||||
|
||||
# Check if project exists
|
||||
if not await project_exists(project):
|
||||
if not project_already_exists:
|
||||
if create_project:
|
||||
console.print(f"[blue]Creating cloud project '{project}'...[/blue]")
|
||||
try:
|
||||
await create_cloud_project(project)
|
||||
await create_cloud_project(project, workspace=resolved_workspace)
|
||||
console.print(f"[green]Created project '{project}'[/green]")
|
||||
except Exception as e:
|
||||
console.print(f"[red]Failed to create project: {e}[/red]")
|
||||
@@ -106,7 +119,10 @@ def upload(
|
||||
verbose=verbose,
|
||||
use_gitignore=not no_gitignore,
|
||||
dry_run=dry_run,
|
||||
client_cm_factory=get_cloud_control_plane_client,
|
||||
client_cm_factory=partial(
|
||||
get_cloud_control_plane_client,
|
||||
workspace=resolved_workspace,
|
||||
),
|
||||
)
|
||||
if not success:
|
||||
console.print("[red]Upload failed[/red]")
|
||||
@@ -117,8 +133,10 @@ def upload(
|
||||
else:
|
||||
console.print(f"[green]Successfully uploaded to '{project}'[/green]")
|
||||
|
||||
# Sync project if requested (skip on dry run)
|
||||
# Force full scan after bisync to ensure database is up-to-date with synced files
|
||||
# Sync project if requested (skip on dry run).
|
||||
# Trigger: upload adds new files the watcher has not observed locally.
|
||||
# Why: force_full ensures those freshly uploaded files are indexed immediately.
|
||||
# Outcome: upload keeps its eager reindex while sync/bisync stay incremental.
|
||||
if sync and not dry_run:
|
||||
console.print(f"[blue]Syncing project '{project}'...[/blue]")
|
||||
try:
|
||||
|
||||
@@ -54,6 +54,9 @@ async def run_doctor() -> None:
|
||||
if not status.new_project:
|
||||
raise ValueError("Failed to create doctor project")
|
||||
project_id = status.new_project.external_id
|
||||
# Use the resolved path from the server — when project_root is configured,
|
||||
# the actual project directory differs from the requested temp_path
|
||||
project_path = Path(status.new_project.path)
|
||||
console.print(f"[green]OK[/green] Created doctor project: {project_name}")
|
||||
|
||||
# --- DB -> File: create an entity via API ---
|
||||
@@ -68,7 +71,7 @@ async def run_doctor() -> None:
|
||||
)
|
||||
api_result = await knowledge_client.create_entity(api_note.model_dump(), fast=False)
|
||||
|
||||
api_file = temp_path / api_result.file_path
|
||||
api_file = project_path / api_result.file_path
|
||||
if not api_file.exists():
|
||||
raise ValueError(f"API note file missing: {api_result.file_path}")
|
||||
|
||||
@@ -79,7 +82,7 @@ async def run_doctor() -> None:
|
||||
console.print("[green]OK[/green] API write created file")
|
||||
|
||||
# --- File -> DB: write markdown file directly, then sync ---
|
||||
parser = EntityParser(temp_path)
|
||||
parser = EntityParser(project_path)
|
||||
processor = MarkdownProcessor(parser)
|
||||
manual_markdown = EntityMarkdown(
|
||||
frontmatter=EntityFrontmatter(
|
||||
@@ -93,7 +96,7 @@ async def run_doctor() -> None:
|
||||
content=f"# {manual_note_title}\n\n- [note] File to DB check",
|
||||
)
|
||||
|
||||
manual_path = temp_path / "doctor" / "manual-note.md"
|
||||
manual_path = project_path / "doctor" / "manual-note.md"
|
||||
await processor.write_file(manual_path, manual_markdown)
|
||||
console.print("[green]OK[/green] Manual file written")
|
||||
|
||||
|
||||
@@ -1,12 +1,14 @@
|
||||
"""MCP server command with streamable HTTP transport."""
|
||||
|
||||
import os
|
||||
import threading
|
||||
from typing import Any, Optional
|
||||
|
||||
import typer
|
||||
from loguru import logger
|
||||
|
||||
from basic_memory.cli.app import app
|
||||
from basic_memory.cli.auto_update import AutoUpdateStatus, run_auto_update
|
||||
from basic_memory.config import ConfigManager, init_mcp_logging
|
||||
|
||||
|
||||
@@ -80,6 +82,22 @@ def mcp(
|
||||
os.environ["BASIC_MEMORY_MCP_PROJECT"] = project_name
|
||||
logger.info(f"MCP server constrained to project: {project_name}")
|
||||
|
||||
def _run_background_auto_update() -> None:
|
||||
result = run_auto_update(force=False, check_only=False, silent=True)
|
||||
if result.restart_recommended:
|
||||
logger.info(
|
||||
"A newer Basic Memory version was installed and will apply on next restart."
|
||||
)
|
||||
elif result.status == AutoUpdateStatus.FAILED and result.error:
|
||||
logger.warning(f"MCP background auto-update failed: {result.error}")
|
||||
|
||||
# Trigger: stdio transport corresponds to local user installs.
|
||||
# Why: server transports (HTTP/SSE) run in managed environments where
|
||||
# package-manager self-upgrades are inappropriate.
|
||||
# Outcome: background auto-update runs only for local stdio MCP sessions.
|
||||
if transport == "stdio":
|
||||
threading.Thread(target=_run_background_auto_update, daemon=True).start()
|
||||
|
||||
# Run the MCP server (blocks)
|
||||
# Lifespan handles: initialization, migrations, file sync, cleanup
|
||||
logger.info(f"Starting MCP server with {transport.upper()} transport")
|
||||
|
||||
@@ -4,6 +4,7 @@ import json
|
||||
import os
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import cast
|
||||
|
||||
import typer
|
||||
from rich.console import Console, Group
|
||||
@@ -27,6 +28,7 @@ from basic_memory.cli.commands.routing import force_routing, validate_routing_fl
|
||||
from basic_memory.config import ConfigManager, ProjectEntry, ProjectMode
|
||||
from basic_memory.mcp.async_client import get_client
|
||||
from basic_memory.mcp.clients import ProjectClient
|
||||
from basic_memory.schemas.cloud import ProjectVisibility
|
||||
from basic_memory.schemas.project_info import ProjectItem, ProjectList
|
||||
from basic_memory.utils import generate_permalink, normalize_project_path
|
||||
|
||||
@@ -56,6 +58,57 @@ def make_bar(value: int, max_value: int, width: int = 40) -> Text:
|
||||
return bar
|
||||
|
||||
|
||||
def _normalize_project_visibility(visibility: str | None) -> ProjectVisibility:
|
||||
"""Normalize CLI visibility input to the cloud API contract."""
|
||||
if visibility is None:
|
||||
return "workspace"
|
||||
|
||||
normalized = visibility.strip().lower()
|
||||
if normalized in {"workspace", "shared", "private"}:
|
||||
return cast(ProjectVisibility, normalized)
|
||||
|
||||
raise ValueError("Invalid visibility. Expected one of: workspace, shared, private.")
|
||||
|
||||
|
||||
def _resolve_workspace_id(config, workspace: str | None) -> str | None:
|
||||
"""Resolve a workspace name or tenant_id to a tenant_id."""
|
||||
from basic_memory.mcp.project_context import (
|
||||
_workspace_choices,
|
||||
_workspace_matches_identifier,
|
||||
get_available_workspaces,
|
||||
)
|
||||
|
||||
if workspace is not None:
|
||||
workspaces = run_with_cleanup(get_available_workspaces())
|
||||
matches = [ws for ws in workspaces if _workspace_matches_identifier(ws, workspace)]
|
||||
if not matches:
|
||||
console.print(f"[red]Error: Workspace '{workspace}' not found[/red]")
|
||||
if workspaces:
|
||||
console.print(f"[dim]Available:\n{_workspace_choices(workspaces)}[/dim]")
|
||||
raise typer.Exit(1)
|
||||
if len(matches) > 1:
|
||||
console.print(
|
||||
f"[red]Error: Workspace name '{workspace}' matches multiple workspaces. "
|
||||
f"Use tenant_id instead.[/red]"
|
||||
)
|
||||
console.print(f"[dim]Available:\n{_workspace_choices(workspaces)}[/dim]")
|
||||
raise typer.Exit(1)
|
||||
return matches[0].tenant_id
|
||||
|
||||
if config.default_workspace:
|
||||
return config.default_workspace
|
||||
|
||||
try:
|
||||
workspaces = run_with_cleanup(get_available_workspaces())
|
||||
if len(workspaces) == 1:
|
||||
return workspaces[0].tenant_id
|
||||
except Exception:
|
||||
# Workspace resolution is optional until a command needs a specific tenant.
|
||||
pass
|
||||
|
||||
return None
|
||||
|
||||
|
||||
@project_app.command("list")
|
||||
def list_projects(
|
||||
local: bool = typer.Option(False, "--local", help="Force local routing for this command"),
|
||||
@@ -128,7 +181,7 @@ def list_projects(
|
||||
table.add_column("Cloud Path", style="green")
|
||||
table.add_column("Workspace", style="green")
|
||||
table.add_column("CLI Route", style="blue")
|
||||
table.add_column("MCP (stdio)", style="blue")
|
||||
table.add_column("MCP", style="blue")
|
||||
table.add_column("Sync", style="green")
|
||||
table.add_column("Default", style="magenta")
|
||||
|
||||
@@ -164,6 +217,11 @@ def list_projects(
|
||||
elif entry and entry.mode == ProjectMode.LOCAL and entry.path:
|
||||
local_path = format_path(normalize_project_path(entry.path))
|
||||
|
||||
# Clear local path for cloud-mode projects — only local projects
|
||||
# should display a local path
|
||||
if entry and entry.mode == ProjectMode.CLOUD:
|
||||
local_path = ""
|
||||
|
||||
cloud_path = ""
|
||||
if cloud_project is not None:
|
||||
cloud_path = normalize_project_path(cloud_project.path)
|
||||
@@ -182,7 +240,13 @@ def list_projects(
|
||||
is_default = config.default_project == project_name
|
||||
|
||||
has_sync = bool(entry and entry.local_sync_path)
|
||||
mcp_stdio_target = "local" if local_project is not None else "n/a"
|
||||
# Determine MCP transport based on project routing mode
|
||||
if entry and entry.mode == ProjectMode.CLOUD:
|
||||
mcp_transport = "https"
|
||||
elif entry is None and cloud_project is not None:
|
||||
mcp_transport = "https"
|
||||
else:
|
||||
mcp_transport = "stdio"
|
||||
|
||||
# Show workspace name (type) for cloud-sourced projects
|
||||
ws_label = ""
|
||||
@@ -195,7 +259,7 @@ def list_projects(
|
||||
"local_path": local_path,
|
||||
"cloud_path": cloud_path,
|
||||
"cli_route": cli_route,
|
||||
"mcp_stdio": mcp_stdio_target,
|
||||
"mcp_stdio": mcp_transport,
|
||||
"sync": has_sync,
|
||||
"is_default": is_default,
|
||||
}
|
||||
@@ -246,6 +310,16 @@ def add_project(
|
||||
local_path: str = typer.Option(
|
||||
None, "--local-path", help="Local sync path for cloud mode (optional)"
|
||||
),
|
||||
workspace: str = typer.Option(
|
||||
None,
|
||||
"--workspace",
|
||||
help="Cloud workspace name or tenant_id (cloud mode only)",
|
||||
),
|
||||
visibility: str = typer.Option(
|
||||
None,
|
||||
"--visibility",
|
||||
help="Cloud project visibility: workspace, shared, or private",
|
||||
),
|
||||
set_default: bool = typer.Option(False, "--default", help="Set as default project"),
|
||||
local: bool = typer.Option(
|
||||
False, "--local", help="Force local API routing (ignore cloud mode)"
|
||||
@@ -260,6 +334,8 @@ def add_project(
|
||||
Cloud mode examples:\n
|
||||
bm project add research # No local sync\n
|
||||
bm project add research --local-path ~/docs # With local sync\n
|
||||
bm project add research --cloud --visibility shared\n
|
||||
bm project add research --cloud --workspace Personal --visibility shared\n
|
||||
|
||||
Local mode example:\n
|
||||
bm project add research ~/Documents/research
|
||||
@@ -274,6 +350,7 @@ def add_project(
|
||||
|
||||
# Determine effective mode: default local, cloud only when explicitly requested.
|
||||
effective_cloud_mode = cloud and not local
|
||||
resolved_workspace_id: str | None = None
|
||||
|
||||
# Resolve local sync path early (needed for both cloud and local mode)
|
||||
local_sync_path: str | None = None
|
||||
@@ -282,18 +359,31 @@ def add_project(
|
||||
|
||||
if effective_cloud_mode:
|
||||
_require_cloud_credentials(config)
|
||||
try:
|
||||
resolved_visibility = _normalize_project_visibility(visibility)
|
||||
except ValueError as e:
|
||||
console.print(f"[red]Error: {e}[/red]")
|
||||
raise typer.Exit(1)
|
||||
resolved_workspace_id = _resolve_workspace_id(config, workspace)
|
||||
# Cloud mode: path auto-generated from name, local sync is optional
|
||||
|
||||
async def _add_project():
|
||||
async with get_client() as client:
|
||||
async with get_client(workspace=resolved_workspace_id) as client:
|
||||
data = {
|
||||
"name": name,
|
||||
"path": generate_permalink(name),
|
||||
"local_sync_path": local_sync_path,
|
||||
"set_default": set_default,
|
||||
"visibility": resolved_visibility,
|
||||
}
|
||||
return await ProjectClient(client).create_project(data)
|
||||
else:
|
||||
if workspace is not None:
|
||||
console.print("[red]Error: --workspace is only supported in cloud mode[/red]")
|
||||
raise typer.Exit(1)
|
||||
if visibility is not None:
|
||||
console.print("[red]Error: --visibility is only supported in cloud mode[/red]")
|
||||
raise typer.Exit(1)
|
||||
# Local mode: path is required
|
||||
if path is None:
|
||||
console.print("[red]Error: path argument is required in local mode[/red]")
|
||||
@@ -312,25 +402,34 @@ def add_project(
|
||||
result = run_with_cleanup(_add_project())
|
||||
console.print(f"[green]{result.message}[/green]")
|
||||
|
||||
# Trigger: local config needs enough metadata to route future commands back to cloud.
|
||||
# Why: explicit workspace selection and local sync state should persist across CLI sessions.
|
||||
# Outcome: cloud-backed projects keep cloud mode, workspace_id, and optional local sync path.
|
||||
if effective_cloud_mode and (local_sync_path or resolved_workspace_id):
|
||||
entry = config.projects.get(name)
|
||||
if entry:
|
||||
entry.mode = ProjectMode.CLOUD
|
||||
if local_sync_path:
|
||||
entry.path = local_sync_path
|
||||
entry.local_sync_path = local_sync_path
|
||||
if resolved_workspace_id:
|
||||
entry.workspace_id = resolved_workspace_id
|
||||
else:
|
||||
# Project may not be in local config yet (cloud-only add)
|
||||
config.projects[name] = ProjectEntry(
|
||||
path=local_sync_path or "",
|
||||
mode=ProjectMode.CLOUD,
|
||||
local_sync_path=local_sync_path,
|
||||
workspace_id=resolved_workspace_id,
|
||||
)
|
||||
ConfigManager().save_config(config)
|
||||
|
||||
# Save local sync path to config if in cloud mode
|
||||
if effective_cloud_mode and local_sync_path:
|
||||
# Create local directory if it doesn't exist
|
||||
local_dir = Path(local_sync_path)
|
||||
local_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Update project entry — path is always the local directory
|
||||
entry = config.projects.get(name)
|
||||
if entry:
|
||||
entry.path = local_sync_path
|
||||
entry.local_sync_path = local_sync_path
|
||||
else:
|
||||
# Project may not be in local config yet (cloud-only add)
|
||||
config.projects[name] = ProjectEntry(
|
||||
path=local_sync_path,
|
||||
local_sync_path=local_sync_path,
|
||||
)
|
||||
ConfigManager().save_config(config)
|
||||
|
||||
console.print(f"\n[green]Local sync path configured: {local_sync_path}[/green]")
|
||||
console.print("\nNext steps:")
|
||||
console.print(f" 1. Preview: bm cloud bisync --name {name} --resync --dry-run")
|
||||
@@ -564,45 +663,7 @@ def set_cloud(
|
||||
console.print("[dim]Run 'bm cloud api-key save <key>' or 'bm cloud login' first[/dim]")
|
||||
raise typer.Exit(1)
|
||||
|
||||
# --- Resolve workspace to tenant_id ---
|
||||
resolved_workspace_id: str | None = None
|
||||
|
||||
if workspace is not None:
|
||||
# Explicit --workspace: resolve to tenant_id via cloud lookup
|
||||
from basic_memory.mcp.project_context import (
|
||||
get_available_workspaces,
|
||||
_workspace_matches_identifier,
|
||||
_workspace_choices,
|
||||
)
|
||||
|
||||
workspaces = run_with_cleanup(get_available_workspaces())
|
||||
matches = [ws for ws in workspaces if _workspace_matches_identifier(ws, workspace)]
|
||||
if not matches:
|
||||
console.print(f"[red]Error: Workspace '{workspace}' not found[/red]")
|
||||
if workspaces:
|
||||
console.print(f"[dim]Available:\n{_workspace_choices(workspaces)}[/dim]")
|
||||
raise typer.Exit(1)
|
||||
if len(matches) > 1:
|
||||
console.print(
|
||||
f"[red]Error: Workspace name '{workspace}' matches multiple workspaces. "
|
||||
f"Use tenant_id instead.[/red]"
|
||||
)
|
||||
console.print(f"[dim]Available:\n{_workspace_choices(workspaces)}[/dim]")
|
||||
raise typer.Exit(1)
|
||||
resolved_workspace_id = matches[0].tenant_id
|
||||
elif config.default_workspace:
|
||||
# Fall back to global default
|
||||
resolved_workspace_id = config.default_workspace
|
||||
else:
|
||||
# Try auto-select if single workspace
|
||||
try:
|
||||
from basic_memory.mcp.project_context import get_available_workspaces
|
||||
|
||||
workspaces = run_with_cleanup(get_available_workspaces())
|
||||
if len(workspaces) == 1:
|
||||
resolved_workspace_id = workspaces[0].tenant_id
|
||||
except Exception:
|
||||
pass # Workspace resolution is optional at set-cloud time
|
||||
resolved_workspace_id = _resolve_workspace_id(config, workspace)
|
||||
|
||||
config.set_project_mode(name, ProjectMode.CLOUD)
|
||||
if resolved_workspace_id:
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
"""Manual update command for Basic Memory CLI."""
|
||||
|
||||
import typer
|
||||
from rich.console import Console
|
||||
|
||||
from basic_memory.cli.app import app
|
||||
from basic_memory.cli.auto_update import AutoUpdateStatus, run_auto_update
|
||||
|
||||
console = Console()
|
||||
|
||||
|
||||
@app.command("update")
|
||||
def update(
|
||||
check: bool = typer.Option(
|
||||
False,
|
||||
"--check",
|
||||
help="Check for updates only (do not install).",
|
||||
),
|
||||
) -> None:
|
||||
"""Check for updates and install when supported."""
|
||||
result = run_auto_update(force=True, check_only=check, silent=False)
|
||||
|
||||
if result.status == AutoUpdateStatus.FAILED:
|
||||
detail = f" {result.error}" if result.error else ""
|
||||
console.print(f"[red]{result.message or 'Update failed.'}{detail}[/red]")
|
||||
raise typer.Exit(1)
|
||||
|
||||
if result.status == AutoUpdateStatus.UPDATED:
|
||||
console.print(f"[green]{result.message or 'Basic Memory updated successfully.'}[/green]")
|
||||
return
|
||||
|
||||
if result.status == AutoUpdateStatus.UP_TO_DATE:
|
||||
console.print(f"[green]{result.message or 'Basic Memory is up to date.'}[/green]")
|
||||
return
|
||||
|
||||
if result.status == AutoUpdateStatus.UPDATE_AVAILABLE:
|
||||
console.print(f"[cyan]{result.message or 'Update available.'}[/cyan]")
|
||||
return
|
||||
|
||||
console.print(f"[dim]{result.message or 'No update action was performed.'}[/dim]")
|
||||
@@ -28,6 +28,7 @@ if not _version_only_invocation(sys.argv[1:]):
|
||||
schema,
|
||||
status,
|
||||
tool,
|
||||
update,
|
||||
)
|
||||
|
||||
warnings.filterwarnings("ignore") # pragma: no cover
|
||||
|
||||
+111
-38
@@ -6,14 +6,16 @@ import os
|
||||
import shutil
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from enum import Enum
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, Literal, Optional, List, Tuple
|
||||
from enum import Enum
|
||||
|
||||
from loguru import logger
|
||||
from pydantic import AliasChoices, BaseModel, Field, model_validator
|
||||
from pydantic_settings import BaseSettings, SettingsConfigDict
|
||||
|
||||
from basic_memory import __version__
|
||||
from basic_memory.telemetry import configure_telemetry
|
||||
from basic_memory.utils import setup_logging, generate_permalink
|
||||
|
||||
|
||||
@@ -140,6 +142,24 @@ class BasicMemoryConfig(BaseSettings):
|
||||
# overridden by ~/.basic-memory/config.json
|
||||
log_level: str = "INFO"
|
||||
|
||||
# Optional Logfire telemetry (disabled by default)
|
||||
logfire_enabled: bool = Field(
|
||||
default=False,
|
||||
description="Enable Logfire instrumentation for local development or managed deployments.",
|
||||
)
|
||||
logfire_send_to_logfire: bool = Field(
|
||||
default=False,
|
||||
description="When true, allow Logfire to export telemetry to the configured backend.",
|
||||
)
|
||||
logfire_service_name: str = Field(
|
||||
default="basic-memory",
|
||||
description="Base service name used when constructing entrypoint-specific Logfire service names.",
|
||||
)
|
||||
logfire_environment: str | None = Field(
|
||||
default=None,
|
||||
description="Optional override for Logfire environment. Defaults to env when unset.",
|
||||
)
|
||||
|
||||
# Database configuration
|
||||
database_backend: DatabaseBackend = Field(
|
||||
default=DatabaseBackend.SQLITE,
|
||||
@@ -203,6 +223,12 @@ class BasicMemoryConfig(BaseSettings):
|
||||
ge=0.0,
|
||||
le=1.0,
|
||||
)
|
||||
default_search_type: Literal["text", "vector", "hybrid"] | None = Field(
|
||||
default=None,
|
||||
description="Default search type for search_notes when not specified per-query. "
|
||||
"Valid values: text, vector, hybrid. "
|
||||
"When unset, defaults to 'hybrid' if semantic search is enabled, otherwise 'text'.",
|
||||
)
|
||||
|
||||
# Database connection pool configuration (Postgres only)
|
||||
db_pool_size: int = Field(
|
||||
@@ -351,6 +377,22 @@ class BasicMemoryConfig(BaseSettings):
|
||||
description="Most recent cloud promo version shown in CLI.",
|
||||
)
|
||||
|
||||
auto_update: bool = Field(
|
||||
default=True,
|
||||
description="Enable automatic CLI update checks and installs when supported.",
|
||||
)
|
||||
|
||||
update_check_interval: int = Field(
|
||||
default=86400,
|
||||
description="Seconds between automatic update checks.",
|
||||
gt=0,
|
||||
)
|
||||
|
||||
auto_update_last_checked_at: Optional[datetime] = Field(
|
||||
default=None,
|
||||
description="Timestamp of the last attempted automatic update check.",
|
||||
)
|
||||
|
||||
cloud_api_key: Optional[str] = Field(
|
||||
default=None,
|
||||
description="API key for cloud access (bmc_ prefixed). Account-level, not per-project.",
|
||||
@@ -629,10 +671,12 @@ class BasicMemoryConfig(BaseSettings):
|
||||
|
||||
# Module-level cache for configuration
|
||||
_CONFIG_CACHE: Optional[BasicMemoryConfig] = None
|
||||
# mtime of the config file when the cache was last populated.
|
||||
# Used to detect out-of-process changes (e.g. CLI set-cloud/set-local while
|
||||
# the MCP stdio server is running) so the long-lived process picks up updates.
|
||||
_CONFIG_CACHE_MTIME: Optional[float] = None
|
||||
# Track config file mtime+size so cross-process changes (e.g. `bm project set-cloud`
|
||||
# in a separate terminal) invalidate the cache in long-lived processes like the
|
||||
# MCP stdio server. Using both mtime and size guards against coarse-granularity
|
||||
# filesystems where two writes within the same second share the same mtime.
|
||||
_CONFIG_MTIME: Optional[float] = None
|
||||
_CONFIG_SIZE: Optional[int] = None
|
||||
|
||||
|
||||
class ConfigManager:
|
||||
@@ -666,32 +710,38 @@ class ConfigManager:
|
||||
Environment variables take precedence over file config values,
|
||||
following Pydantic Settings best practices.
|
||||
|
||||
Uses module-level cache for performance across ConfigManager instances.
|
||||
The cache is validated against the config file's mtime on every call so
|
||||
that long-lived processes (MCP stdio server) automatically pick up
|
||||
changes made by external commands like `bm project set-cloud`.
|
||||
Uses module-level cache with file mtime validation so that
|
||||
cross-process config changes (e.g. `bm project set-cloud` in a
|
||||
separate terminal) are picked up by long-lived processes like
|
||||
the MCP stdio server.
|
||||
"""
|
||||
global _CONFIG_CACHE, _CONFIG_CACHE_MTIME
|
||||
global _CONFIG_CACHE, _CONFIG_MTIME, _CONFIG_SIZE
|
||||
|
||||
# --- Detect out-of-process config changes via mtime ---
|
||||
# Trigger: another process (CLI set-cloud/set-local) wrote a new config
|
||||
# while this process was running and has a cached copy.
|
||||
# Why: _CONFIG_CACHE is never invalidated across process boundaries;
|
||||
# a cheap stat() call lets us detect the change without polling.
|
||||
# Outcome: stale cache is dropped and the file is re-read below.
|
||||
if _CONFIG_CACHE is not None and self.config_file.exists():
|
||||
current_mtime = self.config_file.stat().st_mtime
|
||||
if current_mtime != _CONFIG_CACHE_MTIME:
|
||||
logger.debug(
|
||||
f"Config file modified since last load (mtime changed), "
|
||||
f"invalidating cache: {self.config_file}"
|
||||
)
|
||||
_CONFIG_CACHE = None
|
||||
_CONFIG_CACHE_MTIME = None
|
||||
|
||||
# Return cached config if still valid
|
||||
# Trigger: cached config exists but the on-disk file may have been
|
||||
# modified by another process (CLI command in a different terminal).
|
||||
# Why: the MCP server is long-lived; without this check it would
|
||||
# serve stale project routing forever.
|
||||
# Outcome: cheap os.stat() per access; re-read only when mtime or size differs.
|
||||
if _CONFIG_CACHE is not None:
|
||||
return _CONFIG_CACHE
|
||||
try:
|
||||
st = self.config_file.stat()
|
||||
current_mtime = st.st_mtime
|
||||
current_size = st.st_size
|
||||
except OSError:
|
||||
current_mtime = None
|
||||
current_size = None
|
||||
|
||||
if (
|
||||
current_mtime is not None
|
||||
and current_mtime == _CONFIG_MTIME
|
||||
and current_size == _CONFIG_SIZE
|
||||
):
|
||||
return _CONFIG_CACHE
|
||||
|
||||
# mtime/size changed or file gone — invalidate and fall through to re-read
|
||||
_CONFIG_CACHE = None
|
||||
_CONFIG_MTIME = None
|
||||
_CONFIG_SIZE = None
|
||||
|
||||
if self.config_file.exists():
|
||||
try:
|
||||
@@ -745,8 +795,15 @@ class ConfigManager:
|
||||
merged_data[field_name] = env_dict[field_name]
|
||||
|
||||
_CONFIG_CACHE = BasicMemoryConfig(**merged_data)
|
||||
# Record mtime so we can detect future out-of-process changes.
|
||||
_CONFIG_CACHE_MTIME = self.config_file.stat().st_mtime
|
||||
|
||||
# Record mtime+size so subsequent calls detect cross-process changes
|
||||
try:
|
||||
st = self.config_file.stat()
|
||||
_CONFIG_MTIME = st.st_mtime
|
||||
_CONFIG_SIZE = st.st_size
|
||||
except OSError:
|
||||
_CONFIG_MTIME = None
|
||||
_CONFIG_SIZE = None
|
||||
|
||||
# Re-save to normalize legacy config into current format
|
||||
if needs_resave:
|
||||
@@ -778,13 +835,12 @@ class ConfigManager:
|
||||
|
||||
def save_config(self, config: BasicMemoryConfig) -> None:
|
||||
"""Save configuration to file and invalidate cache."""
|
||||
global _CONFIG_CACHE, _CONFIG_CACHE_MTIME
|
||||
global _CONFIG_CACHE, _CONFIG_MTIME, _CONFIG_SIZE
|
||||
save_basic_memory_config(self.config_file, config)
|
||||
# Invalidate cache so next load_config() reads fresh data.
|
||||
# Also reset mtime so the in-process re-read after save picks up the
|
||||
# new file rather than seeing the newly-written mtime as "unchanged".
|
||||
# Invalidate cache so next load_config() reads fresh data
|
||||
_CONFIG_CACHE = None
|
||||
_CONFIG_CACHE_MTIME = None
|
||||
_CONFIG_MTIME = None
|
||||
_CONFIG_SIZE = None
|
||||
|
||||
@property
|
||||
def projects(self) -> Dict[str, str]:
|
||||
@@ -919,33 +975,50 @@ def save_basic_memory_config(file_path: Path, config: BasicMemoryConfig) -> None
|
||||
# Logging initialization functions for different entry points
|
||||
|
||||
|
||||
def init_cli_logging() -> None: # pragma: no cover
|
||||
def _configure_logfire_for_entrypoint(entrypoint: str) -> None:
|
||||
"""Configure optional Logfire telemetry for a specific entrypoint."""
|
||||
config = ConfigManager().config
|
||||
service_name = f"{config.logfire_service_name}-{entrypoint}"
|
||||
environment = config.logfire_environment or config.env
|
||||
configure_telemetry(
|
||||
service_name=service_name,
|
||||
environment=environment,
|
||||
service_version=__version__,
|
||||
enable_logfire=config.logfire_enabled,
|
||||
send_to_logfire=config.logfire_send_to_logfire,
|
||||
)
|
||||
|
||||
|
||||
def init_cli_logging() -> None:
|
||||
"""Initialize logging for CLI commands - file only.
|
||||
|
||||
CLI commands should not log to stdout to avoid interfering with
|
||||
command output and shell integration.
|
||||
"""
|
||||
log_level = os.getenv("BASIC_MEMORY_LOG_LEVEL", "INFO")
|
||||
_configure_logfire_for_entrypoint("cli")
|
||||
setup_logging(log_level=log_level, log_to_file=True)
|
||||
|
||||
|
||||
def init_mcp_logging() -> None: # pragma: no cover
|
||||
def init_mcp_logging() -> None:
|
||||
"""Initialize logging for MCP server - file only.
|
||||
|
||||
MCP server must not log to stdout as it would corrupt the
|
||||
JSON-RPC protocol communication.
|
||||
"""
|
||||
log_level = os.getenv("BASIC_MEMORY_LOG_LEVEL", "INFO")
|
||||
_configure_logfire_for_entrypoint("mcp")
|
||||
setup_logging(log_level=log_level, log_to_file=True)
|
||||
|
||||
|
||||
def init_api_logging() -> None: # pragma: no cover
|
||||
def init_api_logging() -> None:
|
||||
"""Initialize logging for API server.
|
||||
|
||||
Cloud mode (BASIC_MEMORY_CLOUD_MODE=1): stdout with structured context
|
||||
Local mode: file only
|
||||
"""
|
||||
log_level = os.getenv("BASIC_MEMORY_LOG_LEVEL", "INFO")
|
||||
_configure_logfire_for_entrypoint("api")
|
||||
cloud_mode = os.getenv("BASIC_MEMORY_CLOUD_MODE", "").lower() in ("1", "true")
|
||||
if cloud_mode:
|
||||
setup_logging(log_level=log_level, log_to_stdout=True, structured_context=True)
|
||||
|
||||
@@ -447,6 +447,11 @@ def sanitize_for_filename(text: str, replacement: str = "-") -> str:
|
||||
# compress multiple, repeated replacements
|
||||
text = re.sub(f"{re.escape(replacement)}+", replacement, text)
|
||||
|
||||
# Strip trailing periods — they cause "hi-everyone..md" double-dot filenames
|
||||
# when ".md" is appended, which triggers path traversal false positives.
|
||||
# Trailing periods are also invalid on Windows filesystems.
|
||||
text = text.strip(".")
|
||||
|
||||
return text.strip(replacement)
|
||||
|
||||
|
||||
|
||||
@@ -5,6 +5,7 @@ from typing import AsyncIterator, Callable, Optional
|
||||
from httpx import ASGITransport, AsyncClient, Timeout
|
||||
from loguru import logger
|
||||
|
||||
from basic_memory import telemetry
|
||||
from basic_memory.api.app import app as fastapi_app
|
||||
from basic_memory.config import ConfigManager, ProjectMode
|
||||
|
||||
@@ -43,21 +44,47 @@ def _asgi_client(timeout: Timeout) -> AsyncClient:
|
||||
|
||||
async def _resolve_cloud_token(config) -> str:
|
||||
"""Resolve cloud token with API key preferred, OAuth fallback."""
|
||||
token = config.cloud_api_key
|
||||
if token:
|
||||
return token
|
||||
with telemetry.span(
|
||||
"routing.resolve_cloud_credentials",
|
||||
has_api_key=bool(config.cloud_api_key),
|
||||
):
|
||||
token = config.cloud_api_key
|
||||
if token:
|
||||
return token
|
||||
|
||||
from basic_memory.cli.auth import CLIAuth
|
||||
from basic_memory.cli.auth import CLIAuth
|
||||
|
||||
auth = CLIAuth(client_id=config.cloud_client_id, authkit_domain=config.cloud_domain)
|
||||
token = await auth.get_valid_token()
|
||||
if token:
|
||||
return token
|
||||
auth = CLIAuth(client_id=config.cloud_client_id, authkit_domain=config.cloud_domain)
|
||||
token = await auth.get_valid_token()
|
||||
if token:
|
||||
return token
|
||||
|
||||
raise RuntimeError(
|
||||
"Cloud routing requested but no credentials found. "
|
||||
"Run 'bm cloud api-key save <key>' or 'bm cloud login' first."
|
||||
)
|
||||
logger.error("Cloud routing requested but no credentials were available")
|
||||
raise RuntimeError(
|
||||
"Cloud routing requested but no credentials found. "
|
||||
"Run 'bm cloud api-key save <key>' or 'bm cloud login' first."
|
||||
)
|
||||
|
||||
|
||||
def resolve_configured_workspace(
|
||||
*,
|
||||
config=None,
|
||||
project_name: Optional[str] = None,
|
||||
workspace: Optional[str] = None,
|
||||
) -> Optional[str]:
|
||||
"""Resolve workspace from explicit input, per-project config, then global default."""
|
||||
if workspace is not None:
|
||||
return workspace
|
||||
|
||||
if config is None:
|
||||
config = ConfigManager().config
|
||||
|
||||
if project_name is not None:
|
||||
project_entry = config.projects.get(project_name)
|
||||
if project_entry and project_entry.workspace_id:
|
||||
return project_entry.workspace_id
|
||||
|
||||
return config.default_workspace
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
@@ -82,15 +109,20 @@ async def _cloud_client(
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def get_cloud_control_plane_client() -> AsyncIterator[AsyncClient]:
|
||||
async def get_cloud_control_plane_client(
|
||||
workspace: Optional[str] = None,
|
||||
) -> AsyncIterator[AsyncClient]:
|
||||
"""Create a control-plane cloud client for endpoints outside /proxy."""
|
||||
config = ConfigManager().config
|
||||
timeout = _build_timeout()
|
||||
token = await _resolve_cloud_token(config)
|
||||
headers = {"Authorization": f"Bearer {token}"}
|
||||
if workspace:
|
||||
headers["X-Workspace-ID"] = workspace
|
||||
logger.info(f"Creating HTTP client for cloud control plane at: {config.cloud_host}")
|
||||
async with AsyncClient(
|
||||
base_url=config.cloud_host,
|
||||
headers={"Authorization": f"Bearer {token}"},
|
||||
headers=headers,
|
||||
timeout=timeout,
|
||||
) as client:
|
||||
yield client
|
||||
@@ -161,7 +193,12 @@ async def get_client(
|
||||
|
||||
if _force_cloud_mode():
|
||||
logger.debug("Explicit cloud routing enabled - using cloud proxy client")
|
||||
async with _cloud_client(config, timeout, workspace=workspace) as client:
|
||||
effective_workspace = resolve_configured_workspace(
|
||||
config=config,
|
||||
project_name=project_name,
|
||||
workspace=workspace,
|
||||
)
|
||||
async with _cloud_client(config, timeout, workspace=effective_workspace) as client:
|
||||
yield client
|
||||
return
|
||||
|
||||
@@ -173,8 +210,13 @@ async def get_client(
|
||||
project_mode = config.get_project_mode(project_name)
|
||||
if project_mode == ProjectMode.CLOUD:
|
||||
logger.debug(f"Project '{project_name}' is cloud mode - using cloud proxy client")
|
||||
effective_workspace = resolve_configured_workspace(
|
||||
config=config,
|
||||
project_name=project_name,
|
||||
workspace=workspace,
|
||||
)
|
||||
try:
|
||||
async with _cloud_client(config, timeout, workspace=workspace) as client:
|
||||
async with _cloud_client(config, timeout, workspace=effective_workspace) as client:
|
||||
yield client
|
||||
except RuntimeError as exc:
|
||||
raise RuntimeError(
|
||||
|
||||
@@ -7,6 +7,7 @@ from typing import Any
|
||||
|
||||
from httpx import AsyncClient
|
||||
|
||||
from basic_memory import telemetry
|
||||
from basic_memory.mcp.tools.utils import call_get, call_post, call_put, call_patch, call_delete
|
||||
from basic_memory.schemas.response import (
|
||||
EntityResponse,
|
||||
@@ -58,12 +59,21 @@ class KnowledgeClient:
|
||||
ToolError: If the request fails
|
||||
"""
|
||||
params = {"fast": fast} if fast is not None else None
|
||||
response = await call_post(
|
||||
self.http_client,
|
||||
f"{self._base_path}/entities",
|
||||
json=entity_data,
|
||||
params=params,
|
||||
)
|
||||
with telemetry.scope(
|
||||
"mcp.client.knowledge.create_entity",
|
||||
client_name="knowledge",
|
||||
operation="create_entity",
|
||||
fast=fast,
|
||||
):
|
||||
response = await call_post(
|
||||
self.http_client,
|
||||
f"{self._base_path}/entities",
|
||||
json=entity_data,
|
||||
params=params,
|
||||
client_name="knowledge",
|
||||
operation="create_entity",
|
||||
path_template="/v2/projects/{project_id}/knowledge/entities",
|
||||
)
|
||||
return EntityResponse.model_validate(response.json())
|
||||
|
||||
async def update_entity(
|
||||
@@ -86,12 +96,21 @@ class KnowledgeClient:
|
||||
ToolError: If the request fails
|
||||
"""
|
||||
params = {"fast": fast} if fast is not None else None
|
||||
response = await call_put(
|
||||
self.http_client,
|
||||
f"{self._base_path}/entities/{entity_id}",
|
||||
json=entity_data,
|
||||
params=params,
|
||||
)
|
||||
with telemetry.scope(
|
||||
"mcp.client.knowledge.update_entity",
|
||||
client_name="knowledge",
|
||||
operation="update_entity",
|
||||
fast=fast,
|
||||
):
|
||||
response = await call_put(
|
||||
self.http_client,
|
||||
f"{self._base_path}/entities/{entity_id}",
|
||||
json=entity_data,
|
||||
params=params,
|
||||
client_name="knowledge",
|
||||
operation="update_entity",
|
||||
path_template="/v2/projects/{project_id}/knowledge/entities/{entity_id}",
|
||||
)
|
||||
return EntityResponse.model_validate(response.json())
|
||||
|
||||
async def get_entity(self, entity_id: str) -> EntityResponse:
|
||||
@@ -106,10 +125,18 @@ class KnowledgeClient:
|
||||
Raises:
|
||||
ToolError: If the entity is not found or request fails
|
||||
"""
|
||||
response = await call_get(
|
||||
self.http_client,
|
||||
f"{self._base_path}/entities/{entity_id}",
|
||||
)
|
||||
with telemetry.scope(
|
||||
"mcp.client.knowledge.get_entity",
|
||||
client_name="knowledge",
|
||||
operation="get_entity",
|
||||
):
|
||||
response = await call_get(
|
||||
self.http_client,
|
||||
f"{self._base_path}/entities/{entity_id}",
|
||||
client_name="knowledge",
|
||||
operation="get_entity",
|
||||
path_template="/v2/projects/{project_id}/knowledge/entities/{entity_id}",
|
||||
)
|
||||
return EntityResponse.model_validate(response.json())
|
||||
|
||||
async def patch_entity(
|
||||
@@ -132,12 +159,21 @@ class KnowledgeClient:
|
||||
ToolError: If the request fails
|
||||
"""
|
||||
params = {"fast": fast} if fast is not None else None
|
||||
response = await call_patch(
|
||||
self.http_client,
|
||||
f"{self._base_path}/entities/{entity_id}",
|
||||
json=patch_data,
|
||||
params=params,
|
||||
)
|
||||
with telemetry.scope(
|
||||
"mcp.client.knowledge.patch_entity",
|
||||
client_name="knowledge",
|
||||
operation="patch_entity",
|
||||
fast=fast,
|
||||
):
|
||||
response = await call_patch(
|
||||
self.http_client,
|
||||
f"{self._base_path}/entities/{entity_id}",
|
||||
json=patch_data,
|
||||
params=params,
|
||||
client_name="knowledge",
|
||||
operation="patch_entity",
|
||||
path_template="/v2/projects/{project_id}/knowledge/entities/{entity_id}",
|
||||
)
|
||||
return EntityResponse.model_validate(response.json())
|
||||
|
||||
async def delete_entity(self, entity_id: str) -> DeleteEntitiesResponse:
|
||||
@@ -152,10 +188,18 @@ class KnowledgeClient:
|
||||
Raises:
|
||||
ToolError: If the entity is not found or request fails
|
||||
"""
|
||||
response = await call_delete(
|
||||
self.http_client,
|
||||
f"{self._base_path}/entities/{entity_id}",
|
||||
)
|
||||
with telemetry.scope(
|
||||
"mcp.client.knowledge.delete_entity",
|
||||
client_name="knowledge",
|
||||
operation="delete_entity",
|
||||
):
|
||||
response = await call_delete(
|
||||
self.http_client,
|
||||
f"{self._base_path}/entities/{entity_id}",
|
||||
client_name="knowledge",
|
||||
operation="delete_entity",
|
||||
path_template="/v2/projects/{project_id}/knowledge/entities/{entity_id}",
|
||||
)
|
||||
return DeleteEntitiesResponse.model_validate(response.json())
|
||||
|
||||
async def move_entity(self, entity_id: str, destination_path: str) -> EntityResponse:
|
||||
@@ -171,11 +215,19 @@ class KnowledgeClient:
|
||||
Raises:
|
||||
ToolError: If the request fails
|
||||
"""
|
||||
response = await call_put(
|
||||
self.http_client,
|
||||
f"{self._base_path}/entities/{entity_id}/move",
|
||||
json={"destination_path": destination_path},
|
||||
)
|
||||
with telemetry.scope(
|
||||
"mcp.client.knowledge.move_entity",
|
||||
client_name="knowledge",
|
||||
operation="move_entity",
|
||||
):
|
||||
response = await call_put(
|
||||
self.http_client,
|
||||
f"{self._base_path}/entities/{entity_id}/move",
|
||||
json={"destination_path": destination_path},
|
||||
client_name="knowledge",
|
||||
operation="move_entity",
|
||||
path_template="/v2/projects/{project_id}/knowledge/entities/{entity_id}/move",
|
||||
)
|
||||
return EntityResponse.model_validate(response.json())
|
||||
|
||||
async def move_directory(
|
||||
@@ -193,14 +245,22 @@ class KnowledgeClient:
|
||||
Raises:
|
||||
ToolError: If the request fails
|
||||
"""
|
||||
response = await call_post(
|
||||
self.http_client,
|
||||
f"{self._base_path}/move-directory",
|
||||
json={
|
||||
"source_directory": source_directory,
|
||||
"destination_directory": destination_directory,
|
||||
},
|
||||
)
|
||||
with telemetry.scope(
|
||||
"mcp.client.knowledge.move_directory",
|
||||
client_name="knowledge",
|
||||
operation="move_directory",
|
||||
):
|
||||
response = await call_post(
|
||||
self.http_client,
|
||||
f"{self._base_path}/move-directory",
|
||||
json={
|
||||
"source_directory": source_directory,
|
||||
"destination_directory": destination_directory,
|
||||
},
|
||||
client_name="knowledge",
|
||||
operation="move_directory",
|
||||
path_template="/v2/projects/{project_id}/knowledge/move-directory",
|
||||
)
|
||||
return DirectoryMoveResult.model_validate(response.json())
|
||||
|
||||
async def delete_directory(self, directory: str) -> DirectoryDeleteResult:
|
||||
@@ -215,11 +275,19 @@ class KnowledgeClient:
|
||||
Raises:
|
||||
ToolError: If the request fails
|
||||
"""
|
||||
response = await call_post(
|
||||
self.http_client,
|
||||
f"{self._base_path}/delete-directory",
|
||||
json={"directory": directory},
|
||||
)
|
||||
with telemetry.scope(
|
||||
"mcp.client.knowledge.delete_directory",
|
||||
client_name="knowledge",
|
||||
operation="delete_directory",
|
||||
):
|
||||
response = await call_post(
|
||||
self.http_client,
|
||||
f"{self._base_path}/delete-directory",
|
||||
json={"directory": directory},
|
||||
client_name="knowledge",
|
||||
operation="delete_directory",
|
||||
path_template="/v2/projects/{project_id}/knowledge/delete-directory",
|
||||
)
|
||||
return DirectoryDeleteResult.model_validate(response.json())
|
||||
|
||||
# --- Resolution ---
|
||||
@@ -237,10 +305,18 @@ class KnowledgeClient:
|
||||
Raises:
|
||||
ToolError: If the identifier cannot be resolved
|
||||
"""
|
||||
response = await call_post(
|
||||
self.http_client,
|
||||
f"{self._base_path}/resolve",
|
||||
json={"identifier": identifier, "strict": strict},
|
||||
)
|
||||
with telemetry.scope(
|
||||
"mcp.client.knowledge.resolve_entity",
|
||||
client_name="knowledge",
|
||||
operation="resolve_entity",
|
||||
):
|
||||
response = await call_post(
|
||||
self.http_client,
|
||||
f"{self._base_path}/resolve",
|
||||
json={"identifier": identifier, "strict": strict},
|
||||
client_name="knowledge",
|
||||
operation="resolve_entity",
|
||||
path_template="/v2/projects/{project_id}/knowledge/resolve",
|
||||
)
|
||||
data = response.json()
|
||||
return data["external_id"]
|
||||
|
||||
@@ -7,6 +7,7 @@ from typing import Optional
|
||||
|
||||
from httpx import AsyncClient
|
||||
|
||||
from basic_memory import telemetry
|
||||
from basic_memory.mcp.tools.utils import call_get
|
||||
from basic_memory.schemas.memory import GraphContext
|
||||
|
||||
@@ -71,11 +72,21 @@ class MemoryClient:
|
||||
if timeframe:
|
||||
params["timeframe"] = timeframe
|
||||
|
||||
response = await call_get(
|
||||
self.http_client,
|
||||
f"{self._base_path}/{path}",
|
||||
params=params,
|
||||
)
|
||||
with telemetry.scope(
|
||||
"mcp.client.memory.build_context",
|
||||
client_name="memory",
|
||||
operation="build_context",
|
||||
page=page,
|
||||
page_size=page_size,
|
||||
):
|
||||
response = await call_get(
|
||||
self.http_client,
|
||||
f"{self._base_path}/{path}",
|
||||
params=params,
|
||||
client_name="memory",
|
||||
operation="build_context",
|
||||
path_template="/v2/projects/{project_id}/memory/{path}",
|
||||
)
|
||||
return GraphContext.model_validate(response.json())
|
||||
|
||||
async def recent(
|
||||
@@ -112,9 +123,19 @@ class MemoryClient:
|
||||
# Join types as comma-separated string if provided
|
||||
params["type"] = ",".join(types) if isinstance(types, list) else types
|
||||
|
||||
response = await call_get(
|
||||
self.http_client,
|
||||
f"{self._base_path}/recent",
|
||||
params=params,
|
||||
)
|
||||
with telemetry.scope(
|
||||
"mcp.client.memory.recent_activity",
|
||||
client_name="memory",
|
||||
operation="recent_activity",
|
||||
page=page,
|
||||
page_size=page_size,
|
||||
):
|
||||
response = await call_get(
|
||||
self.http_client,
|
||||
f"{self._base_path}/recent",
|
||||
params=params,
|
||||
client_name="memory",
|
||||
operation="recent_activity",
|
||||
path_template="/v2/projects/{project_id}/memory/recent",
|
||||
)
|
||||
return GraphContext.model_validate(response.json())
|
||||
|
||||
@@ -7,6 +7,7 @@ from typing import Optional
|
||||
|
||||
from httpx import AsyncClient, Response
|
||||
|
||||
from basic_memory import telemetry
|
||||
from basic_memory.mcp.tools.utils import call_get
|
||||
|
||||
|
||||
@@ -64,8 +65,18 @@ class ResourceClient:
|
||||
if page_size is not None:
|
||||
params["page_size"] = page_size
|
||||
|
||||
return await call_get(
|
||||
self.http_client,
|
||||
f"{self._base_path}/{entity_id}",
|
||||
params=params if params else None,
|
||||
)
|
||||
with telemetry.scope(
|
||||
"mcp.client.resource.read",
|
||||
client_name="resource",
|
||||
operation="read",
|
||||
page=page,
|
||||
page_size=page_size,
|
||||
):
|
||||
return await call_get(
|
||||
self.http_client,
|
||||
f"{self._base_path}/{entity_id}",
|
||||
params=params if params else None,
|
||||
client_name="resource",
|
||||
operation="read",
|
||||
path_template="/v2/projects/{project_id}/resource/{entity_id}",
|
||||
)
|
||||
|
||||
@@ -7,6 +7,7 @@ from typing import Any
|
||||
|
||||
from httpx import AsyncClient
|
||||
|
||||
from basic_memory import telemetry
|
||||
from basic_memory.mcp.tools.utils import call_post
|
||||
from basic_memory.schemas.search import SearchResponse
|
||||
|
||||
@@ -56,10 +57,20 @@ class SearchClient:
|
||||
Raises:
|
||||
ToolError: If the request fails
|
||||
"""
|
||||
response = await call_post(
|
||||
self.http_client,
|
||||
f"{self._base_path}/",
|
||||
json=query,
|
||||
params={"page": page, "page_size": page_size},
|
||||
)
|
||||
with telemetry.scope(
|
||||
"mcp.client.search.search",
|
||||
client_name="search",
|
||||
operation="search",
|
||||
page=page,
|
||||
page_size=page_size,
|
||||
):
|
||||
response = await call_post(
|
||||
self.http_client,
|
||||
f"{self._base_path}/",
|
||||
json=query,
|
||||
params={"page": page, "page_size": page_size},
|
||||
client_name="search",
|
||||
operation="search",
|
||||
path_template="/v2/projects/{project_id}/search/",
|
||||
)
|
||||
return SearchResponse.model_validate(response.json())
|
||||
|
||||
@@ -19,6 +19,7 @@ from loguru import logger
|
||||
from fastmcp import Context
|
||||
from mcp.server.fastmcp.exceptions import ToolError
|
||||
|
||||
from basic_memory import telemetry
|
||||
from basic_memory.config import BasicMemoryConfig, ConfigManager, ProjectMode
|
||||
from basic_memory.project_resolver import ProjectResolver
|
||||
from basic_memory.schemas.cloud import WorkspaceInfo, WorkspaceListResponse
|
||||
@@ -63,10 +64,79 @@ async def _resolve_default_project_from_api() -> Optional[str]:
|
||||
return None
|
||||
|
||||
|
||||
async def _get_cached_active_project(context: Optional[Context]) -> Optional[ProjectItem]:
|
||||
"""Return the cached active project from context when available."""
|
||||
if not context:
|
||||
return None
|
||||
|
||||
cached_raw = await context.get_state("active_project")
|
||||
if isinstance(cached_raw, dict):
|
||||
return ProjectItem.model_validate(cached_raw)
|
||||
return None
|
||||
|
||||
|
||||
async def _set_cached_active_project(
|
||||
context: Optional[Context],
|
||||
active_project: ProjectItem,
|
||||
) -> None:
|
||||
"""Persist the active project and known default-project metadata in context."""
|
||||
if not context:
|
||||
return
|
||||
|
||||
await context.set_state("active_project", active_project.model_dump())
|
||||
if active_project.is_default:
|
||||
await context.set_state("default_project_name", active_project.name)
|
||||
|
||||
|
||||
async def _get_cached_default_project(context: Optional[Context]) -> Optional[str]:
|
||||
"""Return the cached default project name from context when available."""
|
||||
if not context:
|
||||
return None
|
||||
|
||||
cached_default = await context.get_state("default_project_name")
|
||||
if isinstance(cached_default, str):
|
||||
return cached_default
|
||||
return None
|
||||
|
||||
|
||||
def _canonicalize_project_name(
|
||||
project_name: Optional[str],
|
||||
config: BasicMemoryConfig,
|
||||
) -> Optional[str]:
|
||||
"""Return the configured project name when the identifier matches by permalink.
|
||||
|
||||
Project routing happens before API validation, so we normalize explicit inputs
|
||||
here to keep local/cloud routing aligned with the database's case-insensitive
|
||||
project resolver.
|
||||
"""
|
||||
if project_name is None:
|
||||
return None
|
||||
|
||||
requested_permalink = generate_permalink(project_name)
|
||||
for configured_name in config.projects:
|
||||
if generate_permalink(configured_name) == requested_permalink:
|
||||
return configured_name
|
||||
|
||||
return project_name
|
||||
|
||||
|
||||
def _project_matches_identifier(project_item: ProjectItem, identifier: Optional[str]) -> bool:
|
||||
"""Return True when the identifier refers to the cached project."""
|
||||
if identifier is None:
|
||||
return True
|
||||
|
||||
normalized_identifier = generate_permalink(identifier)
|
||||
return normalized_identifier in {
|
||||
generate_permalink(project_item.name),
|
||||
project_item.permalink,
|
||||
}
|
||||
|
||||
|
||||
async def resolve_project_parameter(
|
||||
project: Optional[str] = None,
|
||||
allow_discovery: bool = False,
|
||||
default_project: Optional[str] = None,
|
||||
context: Optional[Context] = None,
|
||||
) -> Optional[str]:
|
||||
"""Resolve project parameter using unified linear priority chain.
|
||||
|
||||
@@ -89,22 +159,46 @@ async def resolve_project_parameter(
|
||||
Returns:
|
||||
Resolved project name or None if no resolution possible
|
||||
"""
|
||||
# Load config for any values not explicitly provided.
|
||||
# ConfigManager reads from the local config file, which doesn't exist in cloud mode.
|
||||
# When it returns None, fall back to querying the projects API for the is_default flag.
|
||||
if default_project is None:
|
||||
with telemetry.span(
|
||||
"routing.resolve_project",
|
||||
requested_project=project,
|
||||
allow_discovery=allow_discovery,
|
||||
):
|
||||
config = ConfigManager().config
|
||||
default_project = config.default_project
|
||||
|
||||
if default_project is None:
|
||||
default_project = await _resolve_default_project_from_api()
|
||||
# Trigger: project already resolved earlier in the same MCP request
|
||||
# Why: the active project is request-constant, so re-discovering the
|
||||
# default project via /v2/projects/ just repeats work
|
||||
# Outcome: reuse the cached project name as the explicit candidate
|
||||
if project is None:
|
||||
cached_project = await _get_cached_active_project(context)
|
||||
if cached_project is not None:
|
||||
project = cached_project.name
|
||||
|
||||
# Create resolver with configuration and resolve
|
||||
resolver = ProjectResolver.from_env(
|
||||
default_project=default_project,
|
||||
)
|
||||
result = resolver.resolve(project=project, allow_discovery=allow_discovery)
|
||||
return result.project
|
||||
# Trigger: there is no explicit project after env/context normalization
|
||||
# Why: default-project discovery is only needed as a fallback; doing it
|
||||
# for explicit requests adds an avoidable /v2/projects/ round-trip
|
||||
# Outcome: skip default lookup when the active project is already known
|
||||
if default_project is None and project is None:
|
||||
# Load config for any values not explicitly provided.
|
||||
# ConfigManager reads from the local config file, which doesn't exist in cloud mode.
|
||||
# When it returns None, fall back to querying the projects API for the is_default flag.
|
||||
default_project = config.default_project
|
||||
|
||||
if default_project is None:
|
||||
default_project = await _get_cached_default_project(context)
|
||||
|
||||
if default_project is None:
|
||||
default_project = await _resolve_default_project_from_api()
|
||||
if default_project and context:
|
||||
await context.set_state("default_project_name", default_project)
|
||||
|
||||
# Create resolver with configuration and resolve
|
||||
resolver = ProjectResolver.from_env(
|
||||
default_project=default_project,
|
||||
)
|
||||
result = resolver.resolve(project=project, allow_discovery=allow_discovery)
|
||||
return _canonicalize_project_name(result.project, config)
|
||||
|
||||
|
||||
async def get_project_names(client: AsyncClient, headers: HeaderTypes | None = None) -> List[str]:
|
||||
@@ -177,51 +271,60 @@ async def resolve_workspace_parameter(
|
||||
context: Optional[Context] = None,
|
||||
) -> WorkspaceInfo:
|
||||
"""Resolve workspace using explicit input, session cache, and cloud discovery."""
|
||||
if context:
|
||||
cached_raw = await context.get_state("active_workspace")
|
||||
if isinstance(cached_raw, dict):
|
||||
cached_workspace = WorkspaceInfo.model_validate(cached_raw)
|
||||
if workspace is None or _workspace_matches_identifier(cached_workspace, workspace):
|
||||
logger.debug(f"Using cached workspace from context: {cached_workspace.tenant_id}")
|
||||
return cached_workspace
|
||||
with telemetry.scope(
|
||||
"routing.resolve_workspace",
|
||||
workspace_requested=workspace is not None,
|
||||
has_context=context is not None,
|
||||
):
|
||||
if context:
|
||||
cached_raw = await context.get_state("active_workspace")
|
||||
if isinstance(cached_raw, dict):
|
||||
cached_workspace = WorkspaceInfo.model_validate(cached_raw)
|
||||
if workspace is None or _workspace_matches_identifier(cached_workspace, workspace):
|
||||
logger.debug(
|
||||
f"Using cached workspace from context: {cached_workspace.tenant_id}"
|
||||
)
|
||||
return cached_workspace
|
||||
|
||||
workspaces = await get_available_workspaces(context=context)
|
||||
if not workspaces:
|
||||
raise ValueError(
|
||||
"No accessible workspaces found for this account. "
|
||||
"Ensure you have an active subscription and tenant access."
|
||||
)
|
||||
|
||||
selected_workspace: WorkspaceInfo | None = None
|
||||
|
||||
if workspace:
|
||||
matches = [item for item in workspaces if _workspace_matches_identifier(item, workspace)]
|
||||
if not matches:
|
||||
workspaces = await get_available_workspaces(context=context)
|
||||
if not workspaces:
|
||||
raise ValueError(
|
||||
f"Workspace '{workspace}' was not found.\n"
|
||||
"No accessible workspaces found for this account. "
|
||||
"Ensure you have an active subscription and tenant access."
|
||||
)
|
||||
|
||||
selected_workspace: WorkspaceInfo | None = None
|
||||
|
||||
if workspace:
|
||||
matches = [
|
||||
item for item in workspaces if _workspace_matches_identifier(item, workspace)
|
||||
]
|
||||
if not matches:
|
||||
raise ValueError(
|
||||
f"Workspace '{workspace}' was not found.\n"
|
||||
f"Available workspaces:\n{_workspace_choices(workspaces)}"
|
||||
)
|
||||
if len(matches) > 1:
|
||||
raise ValueError(
|
||||
f"Workspace name '{workspace}' matches multiple workspaces. "
|
||||
"Use tenant_id instead.\n"
|
||||
f"Available workspaces:\n{_workspace_choices(workspaces)}"
|
||||
)
|
||||
selected_workspace = matches[0]
|
||||
elif len(workspaces) == 1:
|
||||
selected_workspace = workspaces[0]
|
||||
else:
|
||||
raise ValueError(
|
||||
"Multiple workspaces are available. Ask the user which workspace to use, then retry "
|
||||
"with the 'workspace' argument set to the tenant_id or unique name.\n"
|
||||
f"Available workspaces:\n{_workspace_choices(workspaces)}"
|
||||
)
|
||||
if len(matches) > 1:
|
||||
raise ValueError(
|
||||
f"Workspace name '{workspace}' matches multiple workspaces. "
|
||||
"Use tenant_id instead.\n"
|
||||
f"Available workspaces:\n{_workspace_choices(workspaces)}"
|
||||
)
|
||||
selected_workspace = matches[0]
|
||||
elif len(workspaces) == 1:
|
||||
selected_workspace = workspaces[0]
|
||||
else:
|
||||
raise ValueError(
|
||||
"Multiple workspaces are available. Ask the user which workspace to use, then retry "
|
||||
"with the 'workspace' argument set to the tenant_id or unique name.\n"
|
||||
f"Available workspaces:\n{_workspace_choices(workspaces)}"
|
||||
)
|
||||
|
||||
if context:
|
||||
await context.set_state("active_workspace", selected_workspace.model_dump())
|
||||
logger.debug(f"Cached workspace in context: {selected_workspace.tenant_id}")
|
||||
if context:
|
||||
await context.set_state("active_workspace", selected_workspace.model_dump())
|
||||
logger.debug(f"Cached workspace in context: {selected_workspace.tenant_id}")
|
||||
|
||||
return selected_workspace
|
||||
return selected_workspace
|
||||
|
||||
|
||||
async def get_active_project(
|
||||
@@ -244,53 +347,58 @@ async def get_active_project(
|
||||
ValueError: If no project can be resolved
|
||||
HTTPError: If project doesn't exist or is inaccessible
|
||||
"""
|
||||
# Deferred import to avoid circular dependency with tools
|
||||
from basic_memory.mcp.tools.utils import call_post
|
||||
with telemetry.scope(
|
||||
"routing.validate_project",
|
||||
requested_project=project,
|
||||
has_context=context is not None,
|
||||
):
|
||||
# Deferred import to avoid circular dependency with tools
|
||||
from basic_memory.mcp.tools.utils import call_post
|
||||
|
||||
resolved_project = await resolve_project_parameter(project)
|
||||
if not resolved_project:
|
||||
project_names = await get_project_names(client, headers)
|
||||
raise ValueError(
|
||||
"No project specified. "
|
||||
"Either set 'default_project' in config, or use 'project' argument.\n"
|
||||
f"Available projects: {project_names}"
|
||||
cached_project = await _get_cached_active_project(context)
|
||||
if cached_project and _project_matches_identifier(cached_project, project):
|
||||
logger.debug(f"Using cached project from context: {cached_project.name}")
|
||||
return cached_project
|
||||
|
||||
resolved_project = await resolve_project_parameter(project, context=context)
|
||||
if not resolved_project:
|
||||
project_names = await get_project_names(client, headers)
|
||||
raise ValueError(
|
||||
"No project specified. "
|
||||
"Either set 'default_project' in config, or use 'project' argument.\n"
|
||||
f"Available projects: {project_names}"
|
||||
)
|
||||
|
||||
project = resolved_project
|
||||
|
||||
if cached_project and _project_matches_identifier(cached_project, project):
|
||||
logger.debug(f"Using cached project from context: {cached_project.name}")
|
||||
return cached_project
|
||||
|
||||
# Validate project exists by calling API
|
||||
logger.debug(f"Validating project: {project}")
|
||||
response = await call_post(
|
||||
client,
|
||||
"/v2/projects/resolve",
|
||||
json={"identifier": project},
|
||||
headers=headers,
|
||||
)
|
||||
resolved = ProjectResolveResponse.model_validate(response.json())
|
||||
active_project = ProjectItem(
|
||||
id=resolved.project_id,
|
||||
external_id=resolved.external_id,
|
||||
name=resolved.name,
|
||||
path=resolved.path,
|
||||
is_default=resolved.is_default,
|
||||
)
|
||||
|
||||
project = resolved_project
|
||||
# Cache in context if available
|
||||
await _set_cached_active_project(context, active_project)
|
||||
if context:
|
||||
logger.debug(f"Cached project in context: {project}")
|
||||
|
||||
# Check if already cached in context
|
||||
if context:
|
||||
cached_raw = await context.get_state("active_project")
|
||||
if isinstance(cached_raw, dict):
|
||||
cached_project = ProjectItem.model_validate(cached_raw)
|
||||
if cached_project.name == project:
|
||||
logger.debug(f"Using cached project from context: {project}")
|
||||
return cached_project
|
||||
|
||||
# Validate project exists by calling API
|
||||
logger.debug(f"Validating project: {project}")
|
||||
response = await call_post(
|
||||
client,
|
||||
"/v2/projects/resolve",
|
||||
json={"identifier": project},
|
||||
headers=headers,
|
||||
)
|
||||
resolved = ProjectResolveResponse.model_validate(response.json())
|
||||
active_project = ProjectItem(
|
||||
id=resolved.project_id,
|
||||
external_id=resolved.external_id,
|
||||
name=resolved.name,
|
||||
path=resolved.path,
|
||||
is_default=resolved.is_default,
|
||||
)
|
||||
|
||||
# Cache in context if available
|
||||
if context:
|
||||
await context.set_state("active_project", active_project.model_dump())
|
||||
logger.debug(f"Cached project in context: {project}")
|
||||
|
||||
logger.debug(f"Validated project: {active_project.name}")
|
||||
return active_project
|
||||
logger.debug(f"Validated project: {active_project.name}")
|
||||
return active_project
|
||||
|
||||
|
||||
def _split_project_prefix(path: str) -> tuple[Optional[str], str]:
|
||||
@@ -321,66 +429,91 @@ async def resolve_project_and_path(
|
||||
Tuple of (active_project, normalized_path, is_memory_url)
|
||||
"""
|
||||
is_memory_url = identifier.strip().startswith("memory://")
|
||||
if not is_memory_url:
|
||||
active_project = await get_active_project(client, project, context, headers)
|
||||
return active_project, identifier, False
|
||||
config = ConfigManager().config
|
||||
include_project = config.permalinks_include_project if is_memory_url else None
|
||||
with telemetry.scope(
|
||||
"routing.resolve_memory_url",
|
||||
is_memory_url=is_memory_url,
|
||||
requested_project=project,
|
||||
include_project_prefix=include_project,
|
||||
):
|
||||
if not is_memory_url:
|
||||
active_project = await get_active_project(client, project, context, headers)
|
||||
return active_project, identifier, False
|
||||
|
||||
normalized_path = normalize_project_reference(memory_url_path(identifier))
|
||||
project_prefix, remainder = _split_project_prefix(normalized_path)
|
||||
include_project = ConfigManager().config.permalinks_include_project
|
||||
normalized_path = normalize_project_reference(memory_url_path(identifier))
|
||||
project_prefix, remainder = _split_project_prefix(normalized_path)
|
||||
include_project = config.permalinks_include_project
|
||||
# Trigger: memory URL begins with a potential project segment
|
||||
# Why: allow project-scoped memory URLs without requiring a separate project parameter
|
||||
# Outcome: attempt to resolve the prefix as a project and route to it
|
||||
if project_prefix:
|
||||
cached_project = await _get_cached_active_project(context)
|
||||
if cached_project and _project_matches_identifier(cached_project, project_prefix):
|
||||
resolved_project = await resolve_project_parameter(project_prefix, context=context)
|
||||
if resolved_project and generate_permalink(resolved_project) != generate_permalink(
|
||||
project_prefix
|
||||
):
|
||||
raise ValueError(
|
||||
f"Project is constrained to '{resolved_project}', cannot use '{project_prefix}'."
|
||||
)
|
||||
|
||||
# Trigger: memory URL begins with a potential project segment
|
||||
# Why: allow project-scoped memory URLs without requiring a separate project parameter
|
||||
# Outcome: attempt to resolve the prefix as a project and route to it
|
||||
if project_prefix:
|
||||
try:
|
||||
from basic_memory.mcp.tools.utils import call_post
|
||||
|
||||
response = await call_post(
|
||||
client,
|
||||
"/v2/projects/resolve",
|
||||
json={"identifier": project_prefix},
|
||||
headers=headers,
|
||||
)
|
||||
resolved = ProjectResolveResponse.model_validate(response.json())
|
||||
except ToolError as exc:
|
||||
if "project not found" not in str(exc).lower():
|
||||
raise
|
||||
else:
|
||||
resolved_project = await resolve_project_parameter(project_prefix)
|
||||
if resolved_project and generate_permalink(resolved_project) != generate_permalink(
|
||||
project_prefix
|
||||
):
|
||||
raise ValueError(
|
||||
f"Project is constrained to '{resolved_project}', cannot use '{project_prefix}'."
|
||||
resolved_path = (
|
||||
f"{cached_project.permalink}/{remainder}" if include_project else remainder
|
||||
)
|
||||
return cached_project, resolved_path, True
|
||||
|
||||
active_project = ProjectItem(
|
||||
id=resolved.project_id,
|
||||
external_id=resolved.external_id,
|
||||
name=resolved.name,
|
||||
path=resolved.path,
|
||||
is_default=resolved.is_default,
|
||||
)
|
||||
if context:
|
||||
await context.set_state("active_project", active_project.model_dump())
|
||||
try:
|
||||
from basic_memory.mcp.tools.utils import call_post
|
||||
|
||||
resolved_path = f"{resolved.permalink}/{remainder}" if include_project else remainder
|
||||
return active_project, resolved_path, True
|
||||
response = await call_post(
|
||||
client,
|
||||
"/v2/projects/resolve",
|
||||
json={"identifier": project_prefix},
|
||||
headers=headers,
|
||||
)
|
||||
resolved = ProjectResolveResponse.model_validate(response.json())
|
||||
except ToolError as exc:
|
||||
if "project not found" not in str(exc).lower():
|
||||
raise
|
||||
else:
|
||||
resolved_project = await resolve_project_parameter(project_prefix, context=context)
|
||||
if resolved_project and generate_permalink(resolved_project) != generate_permalink(
|
||||
project_prefix
|
||||
):
|
||||
raise ValueError(
|
||||
f"Project is constrained to '{resolved_project}', cannot use '{project_prefix}'."
|
||||
)
|
||||
|
||||
# Trigger: no resolvable project prefix in the memory URL
|
||||
# Why: preserve existing memory URL behavior within the active project
|
||||
# Outcome: use the active project and normalize the path for lookup
|
||||
active_project = await get_active_project(client, project, context, headers)
|
||||
resolved_path = normalized_path
|
||||
if include_project:
|
||||
# Trigger: project-prefixed permalinks are enabled and the path lacks a prefix
|
||||
# Why: ensure memory URL lookups align with canonical permalinks
|
||||
# Outcome: prefix the path with the active project's permalink
|
||||
project_prefix = active_project.permalink
|
||||
if resolved_path != project_prefix and not resolved_path.startswith(f"{project_prefix}/"):
|
||||
resolved_path = f"{project_prefix}/{resolved_path}"
|
||||
return active_project, resolved_path, True
|
||||
active_project = ProjectItem(
|
||||
id=resolved.project_id,
|
||||
external_id=resolved.external_id,
|
||||
name=resolved.name,
|
||||
path=resolved.path,
|
||||
is_default=resolved.is_default,
|
||||
)
|
||||
await _set_cached_active_project(context, active_project)
|
||||
|
||||
resolved_path = (
|
||||
f"{resolved.permalink}/{remainder}" if include_project else remainder
|
||||
)
|
||||
return active_project, resolved_path, True
|
||||
|
||||
# Trigger: no resolvable project prefix in the memory URL
|
||||
# Why: preserve existing memory URL behavior within the active project
|
||||
# Outcome: use the active project and normalize the path for lookup
|
||||
active_project = await get_active_project(client, project, context, headers)
|
||||
resolved_path = normalized_path
|
||||
if include_project:
|
||||
# Trigger: project-prefixed permalinks are enabled and the path lacks a prefix
|
||||
# Why: ensure memory URL lookups align with canonical permalinks
|
||||
# Outcome: prefix the path with the active project's permalink
|
||||
project_prefix = active_project.permalink
|
||||
if resolved_path != project_prefix and not resolved_path.startswith(
|
||||
f"{project_prefix}/"
|
||||
):
|
||||
resolved_path = f"{project_prefix}/{resolved_path}"
|
||||
return active_project, resolved_path, True
|
||||
|
||||
|
||||
def add_project_metadata(result: str, project_name: str) -> str:
|
||||
@@ -476,7 +609,7 @@ async def get_project_client(
|
||||
)
|
||||
|
||||
# Step 1: Resolve project name from config (no network call)
|
||||
resolved_project = await resolve_project_parameter(project)
|
||||
resolved_project = await resolve_project_parameter(project, context=context)
|
||||
if not resolved_project:
|
||||
# Fall back to local client to discover projects and raise helpful error
|
||||
async with get_client() as client:
|
||||
@@ -494,9 +627,17 @@ async def get_project_client(
|
||||
# control-plane API with no valid credentials and fail with 401
|
||||
# Outcome: use the factory client directly, skip workspace resolution
|
||||
if is_factory_mode():
|
||||
async with get_client() as client:
|
||||
active_project = await get_active_project(client, resolved_project, context)
|
||||
yield client, active_project
|
||||
route_mode = "factory"
|
||||
with telemetry.scope(
|
||||
"routing.client_session",
|
||||
project_name=resolved_project,
|
||||
route_mode=route_mode,
|
||||
workspace_id=workspace,
|
||||
):
|
||||
logger.debug("Using injected client factory for project routing")
|
||||
async with get_client() as client:
|
||||
active_project = await get_active_project(client, resolved_project, context)
|
||||
yield client, active_project
|
||||
return
|
||||
|
||||
# Step 2: Check explicit routing BEFORE workspace resolution
|
||||
@@ -504,9 +645,16 @@ async def get_project_client(
|
||||
# Why: explicit flags must be deterministic — skip workspace entirely for --local
|
||||
# Outcome: route strictly based on explicit flag, no workspace network calls
|
||||
if _explicit_routing() and _force_local_mode():
|
||||
async with get_client(project_name=resolved_project) as client:
|
||||
active_project = await get_active_project(client, resolved_project, context)
|
||||
yield client, active_project
|
||||
route_mode = "explicit_local"
|
||||
with telemetry.scope(
|
||||
"routing.client_session",
|
||||
project_name=resolved_project,
|
||||
route_mode=route_mode,
|
||||
):
|
||||
logger.debug("Explicit local routing selected for project client")
|
||||
async with get_client(project_name=resolved_project) as client:
|
||||
active_project = await get_active_project(client, resolved_project, context)
|
||||
yield client, active_project
|
||||
return
|
||||
|
||||
# Step 3: Determine if cloud routing is needed
|
||||
@@ -535,28 +683,51 @@ async def get_project_client(
|
||||
if effective_workspace is None and config.default_workspace:
|
||||
effective_workspace = config.default_workspace
|
||||
|
||||
route_mode = "cloud_proxy"
|
||||
|
||||
# Priorities 4-6: if still unresolved, fall back to resolve_workspace_parameter
|
||||
# which checks context cache, auto-selects single workspace, or errors
|
||||
if effective_workspace is not None:
|
||||
# Config-resolved workspace — pass directly to get_client, skip network lookup
|
||||
async with get_client(
|
||||
with telemetry.scope(
|
||||
"routing.client_session",
|
||||
project_name=resolved_project,
|
||||
workspace=effective_workspace,
|
||||
) as client:
|
||||
active_project = await get_active_project(client, resolved_project, context)
|
||||
yield client, active_project
|
||||
route_mode=route_mode,
|
||||
workspace_id=effective_workspace,
|
||||
):
|
||||
logger.debug("Using configured workspace for cloud project routing")
|
||||
async with get_client(
|
||||
project_name=resolved_project,
|
||||
workspace=effective_workspace,
|
||||
) as client:
|
||||
active_project = await get_active_project(client, resolved_project, context)
|
||||
yield client, active_project
|
||||
else:
|
||||
# No config-based workspace — use resolve_workspace_parameter for discovery
|
||||
active_ws = await resolve_workspace_parameter(workspace=None, context=context)
|
||||
async with get_client(
|
||||
with telemetry.scope(
|
||||
"routing.client_session",
|
||||
project_name=resolved_project,
|
||||
workspace=active_ws.tenant_id,
|
||||
) as client:
|
||||
active_project = await get_active_project(client, resolved_project, context)
|
||||
yield client, active_project
|
||||
route_mode=route_mode,
|
||||
workspace_id=active_ws.tenant_id,
|
||||
):
|
||||
logger.debug("Resolved workspace dynamically for cloud project routing")
|
||||
async with get_client(
|
||||
project_name=resolved_project,
|
||||
workspace=active_ws.tenant_id,
|
||||
) as client:
|
||||
active_project = await get_active_project(client, resolved_project, context)
|
||||
yield client, active_project
|
||||
return
|
||||
|
||||
# Step 4: Local routing (default)
|
||||
async with get_client(project_name=resolved_project) as client:
|
||||
active_project = await get_active_project(client, resolved_project, context)
|
||||
yield client, active_project
|
||||
route_mode = "local_asgi"
|
||||
with telemetry.scope(
|
||||
"routing.client_session",
|
||||
project_name=resolved_project,
|
||||
route_mode=route_mode,
|
||||
):
|
||||
logger.debug("Using default local ASGI routing for project client")
|
||||
async with get_client(project_name=resolved_project) as client:
|
||||
active_project = await get_active_project(client, resolved_project, context)
|
||||
yield client, active_project
|
||||
|
||||
@@ -21,6 +21,7 @@ from basic_memory.db import (
|
||||
)
|
||||
from basic_memory.mcp.container import McpContainer, set_container
|
||||
from basic_memory.services.initialization import initialize_app
|
||||
from basic_memory import telemetry
|
||||
|
||||
|
||||
async def _log_embedding_status(session_maker: async_sessionmaker[AsyncSession]) -> None:
|
||||
@@ -82,83 +83,96 @@ async def lifespan(app: FastMCP):
|
||||
set_container(container)
|
||||
|
||||
config = container.config
|
||||
logger.info(f"Starting Basic Memory MCP server (mode={container.mode.name})")
|
||||
logger.info(
|
||||
f"Config: database_backend={config.database_backend.value}, "
|
||||
f"semantic_search_enabled={config.semantic_search_enabled}, "
|
||||
f"default_project={config.default_project}"
|
||||
)
|
||||
if config.semantic_search_enabled:
|
||||
with telemetry.operation(
|
||||
"mcp.lifecycle.startup",
|
||||
entrypoint="mcp",
|
||||
mode=container.mode.name.lower(),
|
||||
default_project=config.default_project,
|
||||
):
|
||||
logger.info(f"Starting Basic Memory MCP server (mode={container.mode.name})")
|
||||
logger.info(
|
||||
f"Semantic search: provider={config.semantic_embedding_provider}, "
|
||||
f"model={config.semantic_embedding_model}, "
|
||||
f"dimensions={config.semantic_embedding_dimensions or 'auto'}, "
|
||||
f"batch_size={config.semantic_embedding_batch_size}"
|
||||
f"Config: database_backend={config.database_backend.value}, "
|
||||
f"semantic_search_enabled={config.semantic_search_enabled}, "
|
||||
f"default_project={config.default_project}"
|
||||
)
|
||||
if config.semantic_search_enabled:
|
||||
logger.info(
|
||||
f"Semantic search: provider={config.semantic_embedding_provider}, "
|
||||
f"model={config.semantic_embedding_model}, "
|
||||
f"dimensions={config.semantic_embedding_dimensions or 'auto'}, "
|
||||
f"batch_size={config.semantic_embedding_batch_size}"
|
||||
)
|
||||
|
||||
# Log configured projects with their routing mode
|
||||
for name, entry in config.projects.items():
|
||||
default = " (default)" if name == config.default_project else ""
|
||||
logger.info(f"Project: {name} -> {entry.path} [mode={entry.mode.value}]{default}")
|
||||
# Log configured projects with their routing mode
|
||||
for name, entry in config.projects.items():
|
||||
default = " (default)" if name == config.default_project else ""
|
||||
logger.info(f"Project: {name} -> {entry.path} [mode={entry.mode.value}]{default}")
|
||||
|
||||
# Check cloud auth status (local file check, no network call)
|
||||
auth = CLIAuth(client_id=config.cloud_client_id, authkit_domain=config.cloud_domain)
|
||||
tokens = auth.load_tokens()
|
||||
if tokens is not None:
|
||||
if not auth.is_token_valid(tokens):
|
||||
expires_at = tokens.get("expires_at", 0)
|
||||
expired_ago = int(time.time() - expires_at)
|
||||
logger.warning(f"Cloud token expired {expired_ago}s ago - may need 'bm cloud login'")
|
||||
else:
|
||||
logger.info("Cloud: authenticated (OAuth token valid)")
|
||||
# Check cloud auth status (local file check, no network call)
|
||||
auth = CLIAuth(client_id=config.cloud_client_id, authkit_domain=config.cloud_domain)
|
||||
tokens = auth.load_tokens()
|
||||
if tokens is not None:
|
||||
if not auth.is_token_valid(tokens):
|
||||
expires_at = tokens.get("expires_at", 0)
|
||||
expired_ago = int(time.time() - expires_at)
|
||||
logger.warning(
|
||||
f"Cloud token expired {expired_ago}s ago - may need 'bm cloud login'"
|
||||
)
|
||||
else:
|
||||
logger.info("Cloud: authenticated (OAuth token valid)")
|
||||
|
||||
if config.cloud_api_key:
|
||||
logger.info("Cloud: API key configured")
|
||||
if config.cloud_api_key:
|
||||
logger.info("Cloud: API key configured")
|
||||
|
||||
# Track if we created the engine (vs test fixtures providing it)
|
||||
# This prevents disposing an engine provided by test fixtures when
|
||||
# multiple Client connections are made in the same test
|
||||
engine_was_none = db._engine is None
|
||||
# Track if we created the engine (vs test fixtures providing it)
|
||||
# This prevents disposing an engine provided by test fixtures when
|
||||
# multiple Client connections are made in the same test
|
||||
engine_was_none = db._engine is None
|
||||
|
||||
# Initialize app (runs migrations, reconciles projects)
|
||||
await initialize_app(container.config)
|
||||
# Initialize app (runs migrations, reconciles projects)
|
||||
await initialize_app(container.config)
|
||||
|
||||
# Log embedding status so it's easy to spot in the logs
|
||||
backfill_task: asyncio.Task | None = None # type: ignore[type-arg]
|
||||
if config.semantic_search_enabled and db._session_maker is not None:
|
||||
await _log_embedding_status(db._session_maker)
|
||||
# Launch backfill in background so MCP server is ready immediately
|
||||
backfill_task = asyncio.create_task(
|
||||
_background_embedding_backfill(config, db._session_maker),
|
||||
name="embedding-backfill",
|
||||
)
|
||||
# Log embedding status so it's easy to spot in the logs
|
||||
backfill_task: asyncio.Task | None = None # type: ignore[type-arg]
|
||||
if config.semantic_search_enabled and db._session_maker is not None:
|
||||
await _log_embedding_status(db._session_maker)
|
||||
# Launch backfill in background so MCP server is ready immediately
|
||||
backfill_task = asyncio.create_task(
|
||||
_background_embedding_backfill(config, db._session_maker),
|
||||
name="embedding-backfill",
|
||||
)
|
||||
|
||||
# Create and start sync coordinator (lifecycle centralized in coordinator)
|
||||
sync_coordinator = container.create_sync_coordinator()
|
||||
await sync_coordinator.start()
|
||||
# Create and start sync coordinator (lifecycle centralized in coordinator)
|
||||
sync_coordinator = container.create_sync_coordinator()
|
||||
await sync_coordinator.start()
|
||||
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
# Shutdown - coordinator handles clean task cancellation
|
||||
logger.debug("Shutting down Basic Memory MCP server")
|
||||
with telemetry.operation(
|
||||
"mcp.lifecycle.shutdown",
|
||||
entrypoint="mcp",
|
||||
mode=container.mode.name.lower(),
|
||||
):
|
||||
logger.debug("Shutting down Basic Memory MCP server")
|
||||
|
||||
# Cancel embedding backfill if still running
|
||||
if backfill_task is not None and not backfill_task.done():
|
||||
backfill_task.cancel()
|
||||
try:
|
||||
await backfill_task
|
||||
except asyncio.CancelledError:
|
||||
logger.info("Background embedding backfill cancelled during shutdown")
|
||||
# Cancel embedding backfill if still running
|
||||
if backfill_task is not None and not backfill_task.done():
|
||||
backfill_task.cancel()
|
||||
try:
|
||||
await backfill_task
|
||||
except asyncio.CancelledError:
|
||||
logger.info("Background embedding backfill cancelled during shutdown")
|
||||
|
||||
await sync_coordinator.stop()
|
||||
await sync_coordinator.stop()
|
||||
|
||||
# Only shutdown DB if we created it (not if test fixture provided it)
|
||||
if engine_was_none:
|
||||
await db.shutdown_db()
|
||||
logger.debug("Database connections closed")
|
||||
else: # pragma: no cover
|
||||
logger.debug("Skipping DB shutdown - engine provided externally")
|
||||
# Only shutdown DB if we created it (not if test fixture provided it)
|
||||
if engine_was_none:
|
||||
await db.shutdown_db()
|
||||
logger.debug("Database connections closed")
|
||||
else: # pragma: no cover
|
||||
logger.debug("Skipping DB shutdown - engine provided externally")
|
||||
|
||||
|
||||
mcp = FastMCP(
|
||||
|
||||
@@ -6,6 +6,7 @@ from loguru import logger
|
||||
from fastmcp import Context
|
||||
|
||||
from basic_memory.config import ConfigManager
|
||||
from basic_memory import telemetry
|
||||
from basic_memory.mcp.project_context import (
|
||||
detect_project_from_url_prefix,
|
||||
get_project_client,
|
||||
@@ -190,8 +191,6 @@ async def build_context(
|
||||
if detected:
|
||||
project = detected
|
||||
|
||||
logger.info(f"Building context from {url} in project {project}")
|
||||
|
||||
# Convert string depth to integer if needed
|
||||
if isinstance(depth, str):
|
||||
try:
|
||||
@@ -203,25 +202,62 @@ async def build_context(
|
||||
|
||||
# URL is already validated and normalized by MemoryUrl type annotation
|
||||
|
||||
async with get_project_client(project, workspace, context) as (client, active_project):
|
||||
# Resolve memory:// identifier with project-prefix awareness
|
||||
_, resolved_path, _ = await resolve_project_and_path(client, url, project, context)
|
||||
with telemetry.operation(
|
||||
"mcp.tool.build_context",
|
||||
entrypoint="mcp",
|
||||
tool_name="build_context",
|
||||
requested_project=project,
|
||||
workspace_id=workspace,
|
||||
depth=depth or 1,
|
||||
timeframe=timeframe,
|
||||
page=page,
|
||||
page_size=page_size,
|
||||
max_related=max_related,
|
||||
output_format=output_format,
|
||||
is_memory_url=str(url).startswith("memory://"),
|
||||
):
|
||||
async with get_project_client(project, workspace, context) as (client, active_project):
|
||||
with telemetry.contextualize(
|
||||
project_name=active_project.name,
|
||||
workspace_id=workspace,
|
||||
tool_name="build_context",
|
||||
):
|
||||
logger.info(
|
||||
f"MCP tool call tool=build_context project={active_project.name} "
|
||||
f"url={url} depth={depth} timeframe={timeframe} output_format={output_format}"
|
||||
)
|
||||
|
||||
# Import here to avoid circular import
|
||||
from basic_memory.mcp.clients import MemoryClient
|
||||
# Resolve memory:// identifier with project-prefix awareness
|
||||
_, resolved_path, _ = await resolve_project_and_path(
|
||||
client,
|
||||
url,
|
||||
active_project.name,
|
||||
context,
|
||||
)
|
||||
|
||||
# Use typed MemoryClient for API calls
|
||||
memory_client = MemoryClient(client, active_project.external_id)
|
||||
graph = await memory_client.build_context(
|
||||
resolved_path,
|
||||
depth=depth or 1,
|
||||
timeframe=timeframe,
|
||||
page=page,
|
||||
page_size=page_size,
|
||||
max_related=max_related,
|
||||
)
|
||||
# Import here to avoid circular import
|
||||
from basic_memory.mcp.clients import MemoryClient
|
||||
|
||||
if output_format == "text":
|
||||
return _format_context_markdown(graph, active_project.name)
|
||||
# Use typed MemoryClient for API calls
|
||||
memory_client = MemoryClient(client, active_project.external_id)
|
||||
graph = await memory_client.build_context(
|
||||
resolved_path,
|
||||
depth=depth or 1,
|
||||
timeframe=timeframe,
|
||||
page=page,
|
||||
page_size=page_size,
|
||||
max_related=max_related,
|
||||
)
|
||||
|
||||
return graph.model_dump()
|
||||
logger.info(
|
||||
f"MCP tool response: tool=build_context project={active_project.name} "
|
||||
f"uri={graph.metadata.uri or resolved_path} "
|
||||
f"primary_count={graph.metadata.primary_count or 0} "
|
||||
f"related_count={graph.metadata.related_count or 0} "
|
||||
f"output_format={output_format}"
|
||||
)
|
||||
|
||||
if output_format == "text":
|
||||
return _format_context_markdown(graph, active_project.name)
|
||||
|
||||
return graph.model_dump()
|
||||
|
||||
@@ -5,7 +5,8 @@ from loguru import logger
|
||||
from fastmcp import Context
|
||||
from mcp.server.fastmcp.exceptions import ToolError
|
||||
|
||||
from basic_memory.mcp.project_context import get_project_client
|
||||
from basic_memory.config import ConfigManager
|
||||
from basic_memory.mcp.project_context import detect_project_from_url_prefix, get_project_client
|
||||
from basic_memory.mcp.server import mcp
|
||||
|
||||
|
||||
@@ -222,6 +223,16 @@ async def delete_note(
|
||||
with suggestions for finding the correct identifier, including search
|
||||
commands and alternative formats to try.
|
||||
"""
|
||||
# Detect project from memory URL prefix before routing
|
||||
# Trigger: identifier starts with memory:// and no explicit project was provided
|
||||
# Why: only gate on memory:// to avoid misrouting plain paths like "research/note"
|
||||
# where "research" is a directory, not a project name
|
||||
# Outcome: project is set from the URL prefix, routing goes to the correct project
|
||||
if project is None and identifier.strip().startswith("memory://"):
|
||||
detected = detect_project_from_url_prefix(identifier, ConfigManager().config)
|
||||
if detected:
|
||||
project = detected
|
||||
|
||||
async with get_project_client(project, workspace, context) as (client, active_project):
|
||||
logger.debug(
|
||||
f"Deleting {'directory' if is_directory else 'note'}: {identifier} in project: {active_project.name}"
|
||||
|
||||
@@ -5,7 +5,13 @@ from typing import Optional, Literal
|
||||
from loguru import logger
|
||||
from fastmcp import Context
|
||||
|
||||
from basic_memory.mcp.project_context import get_project_client, add_project_metadata
|
||||
from basic_memory.config import ConfigManager
|
||||
from basic_memory import telemetry
|
||||
from basic_memory.mcp.project_context import (
|
||||
detect_project_from_url_prefix,
|
||||
get_project_client,
|
||||
add_project_metadata,
|
||||
)
|
||||
from basic_memory.mcp.server import mcp
|
||||
from basic_memory.schemas.base import Entity
|
||||
from basic_memory.schemas.response import EntityResponse
|
||||
@@ -255,218 +261,253 @@ async def edit_note(
|
||||
# Resolve effective default: allow MCP clients to send null for optional int field
|
||||
effective_replacements = expected_replacements if expected_replacements is not None else 1
|
||||
|
||||
async with get_project_client(project, workspace, context) as (client, active_project):
|
||||
logger.info("MCP tool call", tool="edit_note", identifier=identifier, operation=operation)
|
||||
# Detect project from memory URL prefix before routing
|
||||
# Trigger: identifier starts with memory:// and no explicit project was provided
|
||||
# Why: only gate on memory:// to avoid misrouting plain paths like "research/note"
|
||||
# where "research" is a directory, not a project name
|
||||
# Outcome: project is set from the URL prefix, routing goes to the correct project
|
||||
if project is None and identifier.strip().startswith("memory://"):
|
||||
detected = detect_project_from_url_prefix(identifier, ConfigManager().config)
|
||||
if detected:
|
||||
project = detected
|
||||
|
||||
# Validate operation
|
||||
valid_operations = [
|
||||
"append",
|
||||
"prepend",
|
||||
"find_replace",
|
||||
"replace_section",
|
||||
"insert_before_section",
|
||||
"insert_after_section",
|
||||
]
|
||||
if operation not in valid_operations:
|
||||
raise ValueError(
|
||||
f"Invalid operation '{operation}'. Must be one of: {', '.join(valid_operations)}"
|
||||
)
|
||||
with telemetry.operation(
|
||||
"mcp.tool.edit_note",
|
||||
entrypoint="mcp",
|
||||
tool_name="edit_note",
|
||||
requested_project=project,
|
||||
workspace_id=workspace,
|
||||
edit_operation=operation,
|
||||
output_format=output_format,
|
||||
has_section=bool(section),
|
||||
has_find_text=bool(find_text),
|
||||
expected_replacements=effective_replacements,
|
||||
):
|
||||
async with get_project_client(project, workspace, context) as (client, active_project):
|
||||
with telemetry.contextualize(
|
||||
project_name=active_project.name,
|
||||
workspace_id=workspace,
|
||||
tool_name="edit_note",
|
||||
):
|
||||
logger.info(
|
||||
f"MCP tool call tool=edit_note project={active_project.name} "
|
||||
f"identifier={identifier} operation={operation} output_format={output_format}"
|
||||
)
|
||||
|
||||
# Validate required parameters for specific operations
|
||||
if operation == "find_replace" and not find_text:
|
||||
raise ValueError("find_text parameter is required for find_replace operation")
|
||||
section_ops = ("replace_section", "insert_before_section", "insert_after_section")
|
||||
if operation in section_ops and not section:
|
||||
raise ValueError("section parameter is required for section-based operations")
|
||||
|
||||
# Use the PATCH endpoint to edit the entity
|
||||
try:
|
||||
# Import here to avoid circular import
|
||||
from basic_memory.mcp.clients import KnowledgeClient
|
||||
|
||||
# Use typed KnowledgeClient for API calls
|
||||
knowledge_client = KnowledgeClient(client, active_project.external_id)
|
||||
|
||||
file_created = False
|
||||
entity_id = ""
|
||||
result: EntityResponse | None = None
|
||||
|
||||
# Try to resolve the entity; for append/prepend, create it if not found
|
||||
try:
|
||||
entity_id = await knowledge_client.resolve_entity(identifier, strict=True)
|
||||
except Exception as resolve_error:
|
||||
# Trigger: entity does not exist yet
|
||||
# Why: append/prepend can meaningfully create a new note from the content,
|
||||
# while find_replace/replace_section require existing content to modify
|
||||
# Outcome: note is created via the same path as write_note
|
||||
error_msg = str(resolve_error).lower()
|
||||
is_not_found = "entity not found" in error_msg or "not found" in error_msg
|
||||
|
||||
if is_not_found and operation in ("append", "prepend"):
|
||||
title, directory = _parse_identifier_to_title_and_directory(identifier)
|
||||
|
||||
# Validate directory path (same security check as write_note)
|
||||
project_path = active_project.home
|
||||
if directory and not validate_project_path(directory, project_path):
|
||||
logger.warning(
|
||||
"Attempted path traversal attack blocked",
|
||||
directory=directory,
|
||||
project=active_project.name,
|
||||
)
|
||||
if output_format == "json":
|
||||
return {
|
||||
"title": title,
|
||||
"permalink": None,
|
||||
"file_path": None,
|
||||
"checksum": None,
|
||||
"operation": operation,
|
||||
"fileCreated": False,
|
||||
"error": "SECURITY_VALIDATION_ERROR",
|
||||
}
|
||||
return f"# Error\n\nDirectory path '{directory}' is not allowed - paths must stay within project boundaries"
|
||||
|
||||
entity = Entity(
|
||||
title=title,
|
||||
directory=directory,
|
||||
content_type="text/markdown",
|
||||
content=content,
|
||||
# Validate operation
|
||||
valid_operations = [
|
||||
"append",
|
||||
"prepend",
|
||||
"find_replace",
|
||||
"replace_section",
|
||||
"insert_before_section",
|
||||
"insert_after_section",
|
||||
]
|
||||
if operation not in valid_operations:
|
||||
raise ValueError(
|
||||
f"Invalid operation '{operation}'. Must be one of: {', '.join(valid_operations)}"
|
||||
)
|
||||
|
||||
# Validate required parameters for specific operations
|
||||
if operation == "find_replace" and not find_text:
|
||||
raise ValueError("find_text parameter is required for find_replace operation")
|
||||
section_ops = ("replace_section", "insert_before_section", "insert_after_section")
|
||||
if operation in section_ops and not section:
|
||||
raise ValueError("section parameter is required for section-based operations")
|
||||
|
||||
# Use the PATCH endpoint to edit the entity
|
||||
try:
|
||||
# Import here to avoid circular import
|
||||
from basic_memory.mcp.clients import KnowledgeClient
|
||||
|
||||
# Use typed KnowledgeClient for API calls
|
||||
knowledge_client = KnowledgeClient(client, active_project.external_id)
|
||||
|
||||
file_created = False
|
||||
entity_id = ""
|
||||
result: EntityResponse | None = None
|
||||
|
||||
# Try to resolve the entity; for append/prepend, create it if not found
|
||||
try:
|
||||
entity_id = await knowledge_client.resolve_entity(identifier, strict=True)
|
||||
except Exception as resolve_error:
|
||||
# Trigger: entity does not exist yet
|
||||
# Why: append/prepend can meaningfully create a new note from the content,
|
||||
# while find_replace/replace_section require existing content to modify
|
||||
# Outcome: note is created via the same path as write_note
|
||||
error_msg = str(resolve_error).lower()
|
||||
is_not_found = "entity not found" in error_msg or "not found" in error_msg
|
||||
|
||||
if is_not_found and operation in ("append", "prepend"):
|
||||
title, directory = _parse_identifier_to_title_and_directory(identifier)
|
||||
|
||||
# Validate directory path (same security check as write_note)
|
||||
project_path = active_project.home
|
||||
if directory and not validate_project_path(directory, project_path):
|
||||
logger.warning(
|
||||
"Attempted path traversal attack blocked",
|
||||
directory=directory,
|
||||
project=active_project.name,
|
||||
)
|
||||
if output_format == "json":
|
||||
return {
|
||||
"title": title,
|
||||
"permalink": None,
|
||||
"file_path": None,
|
||||
"checksum": None,
|
||||
"operation": operation,
|
||||
"fileCreated": False,
|
||||
"error": "SECURITY_VALIDATION_ERROR",
|
||||
}
|
||||
return f"# Error\n\nDirectory path '{directory}' is not allowed - paths must stay within project boundaries"
|
||||
|
||||
entity = Entity(
|
||||
title=title,
|
||||
directory=directory,
|
||||
content_type="text/markdown",
|
||||
content=content,
|
||||
)
|
||||
|
||||
logger.info(
|
||||
"Creating note via edit_note auto-create",
|
||||
title=title,
|
||||
directory=directory,
|
||||
operation=operation,
|
||||
)
|
||||
result = await knowledge_client.create_entity(
|
||||
entity.model_dump(), fast=False
|
||||
)
|
||||
file_created = True
|
||||
else:
|
||||
# find_replace/replace_section require existing content — re-raise
|
||||
raise resolve_error
|
||||
|
||||
# --- Standard edit path (entity already existed) ---
|
||||
if not file_created:
|
||||
# Prepare the edit request data
|
||||
edit_data = {
|
||||
"operation": operation,
|
||||
"content": content,
|
||||
}
|
||||
|
||||
# Add optional parameters
|
||||
if section:
|
||||
edit_data["section"] = section
|
||||
if find_text:
|
||||
edit_data["find_text"] = find_text
|
||||
if effective_replacements != 1: # Only send if different from default
|
||||
edit_data["expected_replacements"] = str(effective_replacements)
|
||||
|
||||
# Call the PATCH endpoint
|
||||
result = await knowledge_client.patch_entity(
|
||||
entity_id, edit_data, fast=False
|
||||
)
|
||||
|
||||
# --- Format response ---
|
||||
# result is always set: either by create_entity (auto-create) or patch_entity (edit)
|
||||
assert result is not None
|
||||
if file_created:
|
||||
summary = [
|
||||
f"# Created note ({operation})",
|
||||
f"project: {active_project.name}",
|
||||
f"file_path: {result.file_path}",
|
||||
f"permalink: {result.permalink}",
|
||||
f"checksum: {result.checksum[:8] if result.checksum else 'unknown'}",
|
||||
"fileCreated: true",
|
||||
]
|
||||
lines_added = len(content.split("\n"))
|
||||
summary.append(f"operation: Created note with {lines_added} lines")
|
||||
else:
|
||||
summary = [
|
||||
f"# Edited note ({operation})",
|
||||
f"project: {active_project.name}",
|
||||
f"file_path: {result.file_path}",
|
||||
f"permalink: {result.permalink}",
|
||||
f"checksum: {result.checksum[:8] if result.checksum else 'unknown'}",
|
||||
]
|
||||
|
||||
# Add operation-specific details
|
||||
if operation == "append":
|
||||
lines_added = len(content.split("\n"))
|
||||
summary.append(f"operation: Added {lines_added} lines to end of note")
|
||||
elif operation == "prepend":
|
||||
lines_added = len(content.split("\n"))
|
||||
summary.append(
|
||||
f"operation: Added {lines_added} lines to beginning of note"
|
||||
)
|
||||
elif operation == "find_replace":
|
||||
# For find_replace, we can't easily count replacements from here
|
||||
# since we don't have the original content, but the server handled it
|
||||
summary.append("operation: Find and replace operation completed")
|
||||
elif operation == "replace_section":
|
||||
summary.append(f"operation: Replaced content under section '{section}'")
|
||||
elif operation == "insert_before_section":
|
||||
summary.append(
|
||||
f"operation: Inserted content before section '{section}'"
|
||||
)
|
||||
elif operation == "insert_after_section":
|
||||
summary.append(f"operation: Inserted content after section '{section}'")
|
||||
|
||||
# Count observations by category (reuse logic from write_note)
|
||||
categories = {}
|
||||
if result.observations:
|
||||
for obs in result.observations:
|
||||
categories[obs.category] = categories.get(obs.category, 0) + 1
|
||||
|
||||
summary.append("\n## Observations")
|
||||
for category, count in sorted(categories.items()):
|
||||
summary.append(f"- {category}: {count}")
|
||||
|
||||
# Count resolved/unresolved relations
|
||||
unresolved = 0
|
||||
resolved = 0
|
||||
if result.relations:
|
||||
unresolved = sum(1 for r in result.relations if not r.to_id)
|
||||
resolved = len(result.relations) - unresolved
|
||||
|
||||
summary.append("\n## Relations")
|
||||
summary.append(f"- Resolved: {resolved}")
|
||||
if unresolved:
|
||||
summary.append(f"- Unresolved: {unresolved}")
|
||||
|
||||
logger.info(
|
||||
"Creating note via edit_note auto-create",
|
||||
title=title,
|
||||
directory=directory,
|
||||
operation=operation,
|
||||
f"MCP tool response: tool=edit_note project={active_project.name} "
|
||||
f"operation={operation} permalink={result.permalink} "
|
||||
f"observations_count={len(result.observations)} "
|
||||
f"relations_count={len(result.relations)} "
|
||||
f"file_created={str(file_created).lower()}"
|
||||
)
|
||||
result = await knowledge_client.create_entity(entity.model_dump(), fast=False)
|
||||
file_created = True
|
||||
else:
|
||||
# find_replace/replace_section require existing content — re-raise
|
||||
raise resolve_error
|
||||
|
||||
# --- Standard edit path (entity already existed) ---
|
||||
if not file_created:
|
||||
# Prepare the edit request data
|
||||
edit_data = {
|
||||
"operation": operation,
|
||||
"content": content,
|
||||
}
|
||||
if output_format == "json":
|
||||
return {
|
||||
"title": result.title,
|
||||
"permalink": result.permalink,
|
||||
"file_path": result.file_path,
|
||||
"checksum": result.checksum,
|
||||
"operation": operation,
|
||||
"fileCreated": file_created,
|
||||
}
|
||||
|
||||
# Add optional parameters
|
||||
if section:
|
||||
edit_data["section"] = section
|
||||
if find_text:
|
||||
edit_data["find_text"] = find_text
|
||||
if effective_replacements != 1: # Only send if different from default
|
||||
edit_data["expected_replacements"] = str(effective_replacements)
|
||||
summary_result = "\n".join(summary)
|
||||
return add_project_metadata(summary_result, active_project.name)
|
||||
|
||||
# Call the PATCH endpoint
|
||||
result = await knowledge_client.patch_entity(entity_id, edit_data, fast=False)
|
||||
|
||||
# --- Format response ---
|
||||
# result is always set: either by create_entity (auto-create) or patch_entity (edit)
|
||||
assert result is not None
|
||||
if file_created:
|
||||
summary = [
|
||||
f"# Created note ({operation})",
|
||||
f"project: {active_project.name}",
|
||||
f"file_path: {result.file_path}",
|
||||
f"permalink: {result.permalink}",
|
||||
f"checksum: {result.checksum[:8] if result.checksum else 'unknown'}",
|
||||
"fileCreated: true",
|
||||
]
|
||||
lines_added = len(content.split("\n"))
|
||||
summary.append(f"operation: Created note with {lines_added} lines")
|
||||
else:
|
||||
summary = [
|
||||
f"# Edited note ({operation})",
|
||||
f"project: {active_project.name}",
|
||||
f"file_path: {result.file_path}",
|
||||
f"permalink: {result.permalink}",
|
||||
f"checksum: {result.checksum[:8] if result.checksum else 'unknown'}",
|
||||
]
|
||||
|
||||
# Add operation-specific details
|
||||
if operation == "append":
|
||||
lines_added = len(content.split("\n"))
|
||||
summary.append(f"operation: Added {lines_added} lines to end of note")
|
||||
elif operation == "prepend":
|
||||
lines_added = len(content.split("\n"))
|
||||
summary.append(f"operation: Added {lines_added} lines to beginning of note")
|
||||
elif operation == "find_replace":
|
||||
# For find_replace, we can't easily count replacements from here
|
||||
# since we don't have the original content, but the server handled it
|
||||
summary.append("operation: Find and replace operation completed")
|
||||
elif operation == "replace_section":
|
||||
summary.append(f"operation: Replaced content under section '{section}'")
|
||||
elif operation == "insert_before_section":
|
||||
summary.append(f"operation: Inserted content before section '{section}'")
|
||||
elif operation == "insert_after_section":
|
||||
summary.append(f"operation: Inserted content after section '{section}'")
|
||||
|
||||
# Count observations by category (reuse logic from write_note)
|
||||
categories = {}
|
||||
if result.observations:
|
||||
for obs in result.observations:
|
||||
categories[obs.category] = categories.get(obs.category, 0) + 1
|
||||
|
||||
summary.append("\n## Observations")
|
||||
for category, count in sorted(categories.items()):
|
||||
summary.append(f"- {category}: {count}")
|
||||
|
||||
# Count resolved/unresolved relations
|
||||
unresolved = 0
|
||||
resolved = 0
|
||||
if result.relations:
|
||||
unresolved = sum(1 for r in result.relations if not r.to_id)
|
||||
resolved = len(result.relations) - unresolved
|
||||
|
||||
summary.append("\n## Relations")
|
||||
summary.append(f"- Resolved: {resolved}")
|
||||
if unresolved:
|
||||
summary.append(f"- Unresolved: {unresolved}")
|
||||
|
||||
logger.info(
|
||||
"MCP tool response",
|
||||
tool="edit_note",
|
||||
operation=operation,
|
||||
project=active_project.name,
|
||||
permalink=result.permalink,
|
||||
observations_count=len(result.observations),
|
||||
relations_count=len(result.relations),
|
||||
file_created=file_created,
|
||||
)
|
||||
|
||||
if output_format == "json":
|
||||
return {
|
||||
"title": result.title,
|
||||
"permalink": result.permalink,
|
||||
"file_path": result.file_path,
|
||||
"checksum": result.checksum,
|
||||
"operation": operation,
|
||||
"fileCreated": file_created,
|
||||
}
|
||||
|
||||
summary_result = "\n".join(summary)
|
||||
return add_project_metadata(summary_result, active_project.name)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error editing note: {e}")
|
||||
if output_format == "json":
|
||||
return {
|
||||
"title": None,
|
||||
"permalink": None,
|
||||
"file_path": None,
|
||||
"checksum": None,
|
||||
"operation": operation,
|
||||
"fileCreated": False,
|
||||
"error": str(e),
|
||||
}
|
||||
return _format_error_response(
|
||||
str(e),
|
||||
operation,
|
||||
identifier,
|
||||
find_text,
|
||||
effective_replacements,
|
||||
active_project.name,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Error editing note: {e}")
|
||||
if output_format == "json":
|
||||
return {
|
||||
"title": None,
|
||||
"permalink": None,
|
||||
"file_path": None,
|
||||
"checksum": None,
|
||||
"operation": operation,
|
||||
"fileCreated": False,
|
||||
"error": str(e),
|
||||
}
|
||||
return _format_error_response(
|
||||
str(e),
|
||||
operation,
|
||||
identifier,
|
||||
find_text,
|
||||
effective_replacements,
|
||||
active_project.name,
|
||||
)
|
||||
|
||||
@@ -477,8 +477,11 @@ async def move_note(
|
||||
}
|
||||
return f"# Move Failed - Invalid Parameters\n\n{error_msg}"
|
||||
async with get_project_client(project, workspace, context) as (client, active_project):
|
||||
logger.debug(
|
||||
f"Moving {'directory' if is_directory else 'note'}: {identifier} to {destination_path} in project: {active_project.name}"
|
||||
destination_target = destination_folder or destination_path
|
||||
logger.info(
|
||||
f"MCP tool call tool=move_note project={active_project.name} "
|
||||
f"identifier={identifier} destination={destination_target} "
|
||||
f"is_directory={str(is_directory).lower()}"
|
||||
)
|
||||
|
||||
# Validate destination path to prevent path traversal attacks
|
||||
@@ -834,10 +837,8 @@ move_note("{identifier}", destination_folder="notes")
|
||||
|
||||
# Log the operation
|
||||
logger.info(
|
||||
"Move note completed",
|
||||
identifier=identifier,
|
||||
destination_path=destination_path,
|
||||
project=active_project.name,
|
||||
f"MCP tool response: tool=move_note project={active_project.name} "
|
||||
f"source={identifier} destination={result.file_path} permalink={result.permalink}"
|
||||
)
|
||||
|
||||
return "\n".join(result_lines)
|
||||
|
||||
@@ -216,7 +216,7 @@ async def read_content(
|
||||
if detected:
|
||||
project = detected
|
||||
|
||||
logger.info("Reading file", path=path, project=project)
|
||||
logger.info(f"MCP tool call tool=read_content project={project} path={path}")
|
||||
|
||||
async with get_project_client(project, workspace, context) as (client, active_project):
|
||||
# Resolve path with project-prefix awareness for memory:// URLs
|
||||
@@ -260,6 +260,10 @@ async def read_content(
|
||||
# Handle text or json
|
||||
if content_type.startswith("text/") or content_type == "application/json":
|
||||
logger.debug("Processing text resource")
|
||||
logger.info(
|
||||
f"MCP tool response: tool=read_content project={active_project.name} "
|
||||
f"path={url} type=text content_type={content_type}"
|
||||
)
|
||||
return {
|
||||
"type": "text",
|
||||
"text": response.text,
|
||||
@@ -272,6 +276,10 @@ async def read_content(
|
||||
logger.debug("Processing image")
|
||||
img = PILImage.open(io.BytesIO(response.content))
|
||||
img_bytes = optimize_image(img, content_length)
|
||||
logger.info(
|
||||
f"MCP tool response: tool=read_content project={active_project.name} "
|
||||
f"path={url} type=image content_type=image/jpeg"
|
||||
)
|
||||
|
||||
return {
|
||||
"type": "image",
|
||||
@@ -291,6 +299,10 @@ async def read_content(
|
||||
"type": "error",
|
||||
"error": f"Document size {content_length} bytes exceeds maximum allowed size",
|
||||
}
|
||||
logger.info(
|
||||
f"MCP tool response: tool=read_content project={active_project.name} "
|
||||
f"path={url} type=document content_type={content_type}"
|
||||
)
|
||||
return {
|
||||
"type": "document",
|
||||
"source": {
|
||||
|
||||
@@ -8,6 +8,7 @@ import yaml
|
||||
from loguru import logger
|
||||
from fastmcp import Context
|
||||
|
||||
from basic_memory import telemetry
|
||||
from basic_memory.config import ConfigManager
|
||||
from basic_memory.mcp.project_context import (
|
||||
detect_project_from_url_prefix,
|
||||
@@ -139,186 +140,224 @@ async def read_note(
|
||||
if detected:
|
||||
project = detected
|
||||
|
||||
async with get_project_client(project, workspace, context) as (client, active_project):
|
||||
# Resolve identifier with project-prefix awareness for memory:// URLs
|
||||
_, entity_path, _ = await resolve_project_and_path(client, identifier, project, context)
|
||||
with telemetry.operation(
|
||||
"mcp.tool.read_note",
|
||||
entrypoint="mcp",
|
||||
tool_name="read_note",
|
||||
requested_project=project,
|
||||
workspace_id=workspace,
|
||||
output_format=output_format,
|
||||
page=page,
|
||||
page_size=page_size,
|
||||
include_frontmatter=include_frontmatter,
|
||||
):
|
||||
async with get_project_client(project, workspace, context) as (client, active_project):
|
||||
with telemetry.contextualize(
|
||||
project_name=active_project.name,
|
||||
workspace_id=workspace,
|
||||
tool_name="read_note",
|
||||
):
|
||||
# Resolve identifier with project-prefix awareness for memory:// URLs
|
||||
_, entity_path, _ = await resolve_project_and_path(
|
||||
client, identifier, project, context
|
||||
)
|
||||
|
||||
# Validate identifier to prevent path traversal attacks
|
||||
# For memory:// URLs, validate the extracted path (not the raw URL which
|
||||
# has a scheme prefix that confuses path validation)
|
||||
raw_path = memory_url_path(identifier) if identifier.startswith("memory://") else identifier
|
||||
processed_path = entity_path
|
||||
project_path = active_project.home
|
||||
# Validate identifier to prevent path traversal attacks
|
||||
# For memory:// URLs, validate the extracted path (not the raw URL which
|
||||
# has a scheme prefix that confuses path validation)
|
||||
raw_path = (
|
||||
memory_url_path(identifier)
|
||||
if identifier.startswith("memory://")
|
||||
else identifier
|
||||
)
|
||||
processed_path = entity_path
|
||||
project_path = active_project.home
|
||||
|
||||
if not validate_project_path(raw_path, project_path) or not validate_project_path(
|
||||
processed_path, project_path
|
||||
):
|
||||
logger.warning(
|
||||
"Attempted path traversal attack blocked",
|
||||
identifier=identifier,
|
||||
processed_path=processed_path,
|
||||
project=active_project.name,
|
||||
)
|
||||
if output_format == "json":
|
||||
return {
|
||||
"title": None,
|
||||
"permalink": None,
|
||||
"file_path": None,
|
||||
"content": None,
|
||||
"frontmatter": None,
|
||||
"error": "SECURITY_VALIDATION_ERROR",
|
||||
}
|
||||
return f"# Error\n\nIdentifier '{identifier}' is not allowed - paths must stay within project boundaries"
|
||||
|
||||
# Get the file via REST API - first try direct identifier resolution
|
||||
logger.info(
|
||||
f"Attempting to read note from Project: {active_project.name} identifier: {entity_path}"
|
||||
)
|
||||
|
||||
# Import here to avoid circular import
|
||||
from basic_memory.mcp.clients import KnowledgeClient, ResourceClient
|
||||
|
||||
# Use typed clients for API calls
|
||||
knowledge_client = KnowledgeClient(client, active_project.external_id)
|
||||
resource_client = ResourceClient(client, active_project.external_id)
|
||||
|
||||
async def _read_json_payload(entity_id: str) -> dict:
|
||||
entity = await knowledge_client.get_entity(entity_id)
|
||||
response = await resource_client.read(entity_id, page=page, page_size=page_size)
|
||||
content_text = response.text
|
||||
body_content, parsed_frontmatter = _parse_opening_frontmatter(content_text)
|
||||
return {
|
||||
"title": entity.title,
|
||||
"permalink": entity.permalink,
|
||||
"file_path": entity.file_path,
|
||||
"content": content_text if include_frontmatter else body_content,
|
||||
"frontmatter": parsed_frontmatter,
|
||||
}
|
||||
|
||||
def _empty_json_payload() -> dict:
|
||||
return {
|
||||
"title": None,
|
||||
"permalink": None,
|
||||
"file_path": None,
|
||||
"content": None,
|
||||
"frontmatter": None,
|
||||
}
|
||||
|
||||
def _search_results(payload: object) -> list[dict]:
|
||||
if not isinstance(payload, dict):
|
||||
return []
|
||||
results = payload.get("results")
|
||||
return results if isinstance(results, list) else []
|
||||
|
||||
def _result_title(item: dict) -> str:
|
||||
return str(item.get("title") or "")
|
||||
|
||||
def _result_permalink(item: dict) -> Optional[str]:
|
||||
value = item.get("permalink")
|
||||
return str(value) if value else None
|
||||
|
||||
def _result_file_path(item: dict) -> Optional[str]:
|
||||
value = item.get("file_path")
|
||||
return str(value) if value else None
|
||||
|
||||
try:
|
||||
# Try to resolve identifier to entity ID
|
||||
entity_id = await knowledge_client.resolve_entity(entity_path, strict=True)
|
||||
|
||||
# Fetch content using entity ID
|
||||
response = await resource_client.read(entity_id, page=page, page_size=page_size)
|
||||
|
||||
# If successful, return the content
|
||||
if response.status_code == 200:
|
||||
logger.info("Returning read_note result from resource: {path}", path=entity_path)
|
||||
if output_format == "json":
|
||||
return await _read_json_payload(entity_id)
|
||||
return response.text
|
||||
except Exception as e: # pragma: no cover
|
||||
logger.info(f"Direct lookup failed for '{entity_path}': {e}")
|
||||
# Continue to fallback methods
|
||||
|
||||
# Fallback 1: Try title search via API
|
||||
logger.info(f"Search title for: {identifier}")
|
||||
title_results = await search_notes(
|
||||
query=identifier,
|
||||
search_type="title",
|
||||
project=active_project.name,
|
||||
workspace=workspace,
|
||||
output_format="json",
|
||||
context=context,
|
||||
)
|
||||
|
||||
title_candidates = _search_results(title_results)
|
||||
if title_candidates:
|
||||
# Trigger: direct resolution failed and title search returned candidates.
|
||||
# Why: avoid returning unrelated notes when search yields only fuzzy matches.
|
||||
# Outcome: fetch content only when a true exact title match exists.
|
||||
result = next(
|
||||
(
|
||||
candidate
|
||||
for candidate in title_candidates
|
||||
if _is_exact_title_match(identifier, _result_title(candidate))
|
||||
),
|
||||
None,
|
||||
)
|
||||
if not result:
|
||||
logger.info(f"No exact title match found for: {identifier}")
|
||||
elif _result_permalink(result):
|
||||
try:
|
||||
# Resolve the permalink to entity ID
|
||||
entity_id = await knowledge_client.resolve_entity(
|
||||
_result_permalink(result) or "", strict=True
|
||||
if not validate_project_path(raw_path, project_path) or not validate_project_path(
|
||||
processed_path, project_path
|
||||
):
|
||||
logger.warning(
|
||||
"Attempted path traversal attack blocked",
|
||||
identifier=identifier,
|
||||
processed_path=processed_path,
|
||||
project=active_project.name,
|
||||
)
|
||||
if output_format == "json":
|
||||
return {
|
||||
"title": None,
|
||||
"permalink": None,
|
||||
"file_path": None,
|
||||
"content": None,
|
||||
"frontmatter": None,
|
||||
"error": "SECURITY_VALIDATION_ERROR",
|
||||
}
|
||||
return f"# Error\n\nIdentifier '{identifier}' is not allowed - paths must stay within project boundaries"
|
||||
|
||||
# Fetch content using the entity ID
|
||||
# Get the file via REST API - first try direct identifier resolution
|
||||
logger.info(
|
||||
f"Attempting to read note from Project: {active_project.name} identifier: {entity_path}"
|
||||
)
|
||||
|
||||
# Import here to avoid circular import
|
||||
from basic_memory.mcp.clients import KnowledgeClient, ResourceClient
|
||||
|
||||
# Use typed clients for API calls
|
||||
knowledge_client = KnowledgeClient(client, active_project.external_id)
|
||||
resource_client = ResourceClient(client, active_project.external_id)
|
||||
|
||||
async def _read_json_payload(entity_id: str) -> dict:
|
||||
with telemetry.scope(
|
||||
"mcp.read_note.shape_response",
|
||||
domain="mcp",
|
||||
action="read_note",
|
||||
phase="shape_response",
|
||||
):
|
||||
entity = await knowledge_client.get_entity(entity_id)
|
||||
response = await resource_client.read(
|
||||
entity_id, page=page, page_size=page_size
|
||||
)
|
||||
content_text = response.text
|
||||
body_content, parsed_frontmatter = _parse_opening_frontmatter(content_text)
|
||||
return {
|
||||
"title": entity.title,
|
||||
"permalink": entity.permalink,
|
||||
"file_path": entity.file_path,
|
||||
"content": content_text if include_frontmatter else body_content,
|
||||
"frontmatter": parsed_frontmatter,
|
||||
}
|
||||
|
||||
def _empty_json_payload() -> dict:
|
||||
return {
|
||||
"title": None,
|
||||
"permalink": None,
|
||||
"file_path": None,
|
||||
"content": None,
|
||||
"frontmatter": None,
|
||||
}
|
||||
|
||||
def _search_results(payload: object) -> list[dict]:
|
||||
if not isinstance(payload, dict):
|
||||
return []
|
||||
results = payload.get("results")
|
||||
return results if isinstance(results, list) else []
|
||||
|
||||
async def _search_candidates(identifier_text: str, *, title_only: bool) -> dict:
|
||||
# Trigger: direct entity resolution failed for the caller's identifier.
|
||||
# Why: search_notes applies the same memory:// normalization and tool-level
|
||||
# query handling as the rest of MCP routing, which raw client calls skip.
|
||||
# Outcome: unresolved memory URLs still fall back through normalized search.
|
||||
search_type = "title" if title_only else "text"
|
||||
response = await search_notes(
|
||||
project=active_project.name,
|
||||
workspace=workspace,
|
||||
query=identifier_text,
|
||||
search_type=search_type,
|
||||
page=page,
|
||||
page_size=page_size,
|
||||
output_format="json",
|
||||
context=context,
|
||||
)
|
||||
return response if isinstance(response, dict) else {}
|
||||
|
||||
def _result_title(item: dict) -> str:
|
||||
return str(item.get("title") or "")
|
||||
|
||||
def _result_permalink(item: dict) -> Optional[str]:
|
||||
value = item.get("permalink")
|
||||
return str(value) if value else None
|
||||
|
||||
def _result_file_path(item: dict) -> Optional[str]:
|
||||
value = item.get("file_path")
|
||||
return str(value) if value else None
|
||||
|
||||
try:
|
||||
# Try to resolve identifier to entity ID
|
||||
entity_id = await knowledge_client.resolve_entity(entity_path, strict=True)
|
||||
|
||||
# Fetch content using entity ID
|
||||
response = await resource_client.read(entity_id, page=page, page_size=page_size)
|
||||
|
||||
# If successful, return the content
|
||||
if response.status_code == 200:
|
||||
logger.info(
|
||||
f"Found note by exact title search: {_result_permalink(result)}"
|
||||
"Returning read_note result from resource: {path}", path=entity_path
|
||||
)
|
||||
if output_format == "json":
|
||||
return await _read_json_payload(entity_id)
|
||||
return response.text
|
||||
except Exception as e: # pragma: no cover
|
||||
logger.info(
|
||||
f"Failed to fetch content for found title match {_result_permalink(result)}: {e}"
|
||||
logger.info(f"Direct lookup failed for '{entity_path}': {e}")
|
||||
# Continue to fallback methods
|
||||
|
||||
# Fallback 1: Try title search via API
|
||||
logger.info(f"Search title for: {identifier}")
|
||||
title_results = await _search_candidates(identifier, title_only=True)
|
||||
|
||||
title_candidates = _search_results(title_results)
|
||||
if title_candidates:
|
||||
# Trigger: direct resolution failed and title search returned candidates.
|
||||
# Why: avoid returning unrelated notes when search yields only fuzzy matches.
|
||||
# Outcome: fetch content only when a true exact title match exists.
|
||||
result = next(
|
||||
(
|
||||
candidate
|
||||
for candidate in title_candidates
|
||||
if _is_exact_title_match(identifier, _result_title(candidate))
|
||||
),
|
||||
None,
|
||||
)
|
||||
else:
|
||||
logger.info(
|
||||
f"No results in title search for: {identifier} in project {active_project.name}"
|
||||
)
|
||||
if not result:
|
||||
logger.info(f"No exact title match found for: {identifier}")
|
||||
elif _result_permalink(result):
|
||||
try:
|
||||
# Resolve the permalink to entity ID
|
||||
entity_id = await knowledge_client.resolve_entity(
|
||||
_result_permalink(result) or "", strict=True
|
||||
)
|
||||
|
||||
# Fallback 2: Text search as a last resort
|
||||
logger.info(f"Title search failed, trying text search for: {identifier}")
|
||||
text_results = await search_notes(
|
||||
query=identifier,
|
||||
search_type="text",
|
||||
project=active_project.name,
|
||||
workspace=workspace,
|
||||
output_format="json",
|
||||
context=context,
|
||||
)
|
||||
# Fetch content using the entity ID
|
||||
response = await resource_client.read(
|
||||
entity_id, page=page, page_size=page_size
|
||||
)
|
||||
|
||||
# We didn't find a direct match, construct a helpful error message
|
||||
text_candidates = _search_results(text_results)
|
||||
if not text_candidates:
|
||||
if output_format == "json":
|
||||
return _empty_json_payload()
|
||||
return format_not_found_message(active_project.name, identifier)
|
||||
if output_format == "json":
|
||||
payload = _empty_json_payload()
|
||||
payload["related_results"] = [
|
||||
{
|
||||
"title": _result_title(result),
|
||||
"permalink": _result_permalink(result),
|
||||
"file_path": _result_file_path(result),
|
||||
}
|
||||
for result in text_candidates[:5]
|
||||
]
|
||||
return payload
|
||||
return format_related_results(active_project.name, identifier, text_candidates[:5])
|
||||
if response.status_code == 200:
|
||||
logger.info(
|
||||
f"Found note by exact title search: {_result_permalink(result)}"
|
||||
)
|
||||
if output_format == "json":
|
||||
return await _read_json_payload(entity_id)
|
||||
return response.text
|
||||
except Exception as e: # pragma: no cover
|
||||
logger.info(
|
||||
f"Failed to fetch content for found title match {_result_permalink(result)}: {e}"
|
||||
)
|
||||
else:
|
||||
logger.info(
|
||||
f"No results in title search for: {identifier} in project {active_project.name}"
|
||||
)
|
||||
|
||||
# Fallback 2: Text search as a last resort
|
||||
logger.info(f"Title search failed, trying text search for: {identifier}")
|
||||
text_results = await _search_candidates(identifier, title_only=False)
|
||||
|
||||
# We didn't find a direct match, construct a helpful error message
|
||||
text_candidates = _search_results(text_results)
|
||||
if not text_candidates:
|
||||
if output_format == "json":
|
||||
return _empty_json_payload()
|
||||
return format_not_found_message(active_project.name, identifier)
|
||||
if output_format == "json":
|
||||
payload = _empty_json_payload()
|
||||
payload["related_results"] = [
|
||||
{
|
||||
"title": _result_title(result),
|
||||
"permalink": _result_permalink(result),
|
||||
"file_path": _result_file_path(result),
|
||||
}
|
||||
for result in text_candidates[:5]
|
||||
]
|
||||
return payload
|
||||
return format_related_results(active_project.name, identifier, text_candidates[:5])
|
||||
|
||||
|
||||
def format_not_found_message(project: str | None, identifier: str) -> str:
|
||||
|
||||
@@ -8,6 +8,7 @@ from loguru import logger
|
||||
from fastmcp import Context
|
||||
from pydantic import BeforeValidator
|
||||
|
||||
from basic_memory import telemetry
|
||||
from basic_memory.config import ConfigManager
|
||||
from basic_memory.utils import coerce_dict, coerce_list
|
||||
from basic_memory.mcp.container import get_container
|
||||
@@ -25,20 +26,20 @@ from basic_memory.schemas.search import (
|
||||
)
|
||||
|
||||
|
||||
def _semantic_search_enabled_for_text_search() -> bool:
|
||||
"""Resolve semantic-search enablement in both MCP and CLI invocation paths."""
|
||||
try:
|
||||
return get_container().config.semantic_search_enabled
|
||||
except RuntimeError:
|
||||
# Trigger: MCP container is not initialized (e.g., `bm tool search-notes` direct call).
|
||||
# Why: CLI path still needs the same semantic-default behavior as MCP server path.
|
||||
# Outcome: load config directly and keep text-mode retrieval behavior consistent.
|
||||
return ConfigManager().config.semantic_search_enabled
|
||||
|
||||
|
||||
def _default_search_type() -> str:
|
||||
"""Pick default search mode from semantic-search config."""
|
||||
return "hybrid" if _semantic_search_enabled_for_text_search() else "text"
|
||||
"""Pick default search mode from config, falling back to auto-detection.
|
||||
|
||||
Priority: config default_search_type > auto-detect (hybrid if semantic enabled, else text).
|
||||
"""
|
||||
try:
|
||||
config = get_container().config
|
||||
except RuntimeError:
|
||||
config = ConfigManager().config
|
||||
|
||||
if config.default_search_type:
|
||||
return config.default_search_type
|
||||
|
||||
return "hybrid" if config.semantic_search_enabled else "text"
|
||||
|
||||
|
||||
def _format_search_error_response(
|
||||
@@ -523,124 +524,159 @@ async def search_notes(
|
||||
if detected:
|
||||
project = detected
|
||||
|
||||
async with get_project_client(project, workspace, context) as (client, active_project):
|
||||
# Handle memory:// URLs by resolving to permalink search
|
||||
is_memory_url = False
|
||||
if query is not None:
|
||||
_, resolved_query, is_memory_url = await resolve_project_and_path(
|
||||
client, query, project, context
|
||||
)
|
||||
if is_memory_url:
|
||||
query = resolved_query
|
||||
effective_search_type = search_type or _default_search_type()
|
||||
if is_memory_url:
|
||||
effective_search_type = "permalink"
|
||||
with telemetry.operation(
|
||||
"mcp.tool.search_notes",
|
||||
entrypoint="mcp",
|
||||
tool_name="search_notes",
|
||||
requested_project=project,
|
||||
workspace_id=workspace,
|
||||
search_type=search_type or "default",
|
||||
output_format=output_format,
|
||||
page=page,
|
||||
page_size=page_size,
|
||||
has_query=bool(query and query.strip()),
|
||||
note_type_filter_count=len(note_types),
|
||||
entity_type_filter_count=len(entity_types),
|
||||
has_filters=bool(
|
||||
metadata_filters or tags or status or note_types or entity_types or after_date
|
||||
),
|
||||
has_tags_filter=bool(tags),
|
||||
has_status_filter=bool(status),
|
||||
):
|
||||
async with get_project_client(project, workspace, context) as (client, active_project):
|
||||
with telemetry.contextualize(
|
||||
project_name=active_project.name,
|
||||
workspace_id=workspace,
|
||||
tool_name="search_notes",
|
||||
):
|
||||
# Handle memory:// URLs by resolving to permalink search
|
||||
is_memory_url = False
|
||||
if query is not None:
|
||||
_, resolved_query, is_memory_url = await resolve_project_and_path(
|
||||
client, query, project, context
|
||||
)
|
||||
if is_memory_url:
|
||||
query = resolved_query
|
||||
effective_search_type = search_type or _default_search_type()
|
||||
if is_memory_url:
|
||||
effective_search_type = "permalink"
|
||||
|
||||
try:
|
||||
# Create a SearchQuery object based on the parameters
|
||||
search_query = SearchQuery()
|
||||
try:
|
||||
# Create a SearchQuery object based on the parameters
|
||||
search_query = SearchQuery()
|
||||
|
||||
# Only map search_type to query fields when there is an actual query string.
|
||||
# When query is None/empty, skip the search mode block — filters-only path.
|
||||
effective_query = (query or "").strip()
|
||||
if effective_query:
|
||||
valid_search_types = {
|
||||
"text",
|
||||
"title",
|
||||
"permalink",
|
||||
"vector",
|
||||
"semantic",
|
||||
"hybrid",
|
||||
}
|
||||
if effective_search_type == "text":
|
||||
search_query.text = effective_query
|
||||
search_query.retrieval_mode = SearchRetrievalMode.FTS
|
||||
elif effective_search_type in ("vector", "semantic"):
|
||||
search_query.text = effective_query
|
||||
search_query.retrieval_mode = SearchRetrievalMode.VECTOR
|
||||
elif effective_search_type == "hybrid":
|
||||
search_query.text = effective_query
|
||||
search_query.retrieval_mode = SearchRetrievalMode.HYBRID
|
||||
elif effective_search_type == "title":
|
||||
search_query.title = effective_query
|
||||
elif effective_search_type == "permalink" and "*" in effective_query:
|
||||
search_query.permalink_match = effective_query
|
||||
elif effective_search_type == "permalink":
|
||||
search_query.permalink = effective_query
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Invalid search_type '{effective_search_type}'. "
|
||||
f"Valid options: {', '.join(sorted(valid_search_types))}"
|
||||
# Only map search_type to query fields when there is an actual query string.
|
||||
# When query is None/empty, skip the search mode block — filters-only path.
|
||||
effective_query = (query or "").strip()
|
||||
if effective_query:
|
||||
valid_search_types = {
|
||||
"text",
|
||||
"title",
|
||||
"permalink",
|
||||
"vector",
|
||||
"semantic",
|
||||
"hybrid",
|
||||
}
|
||||
if effective_search_type == "text":
|
||||
search_query.text = effective_query
|
||||
search_query.retrieval_mode = SearchRetrievalMode.FTS
|
||||
elif effective_search_type in ("vector", "semantic"):
|
||||
search_query.text = effective_query
|
||||
search_query.retrieval_mode = SearchRetrievalMode.VECTOR
|
||||
elif effective_search_type == "hybrid":
|
||||
search_query.text = effective_query
|
||||
search_query.retrieval_mode = SearchRetrievalMode.HYBRID
|
||||
elif effective_search_type == "title":
|
||||
search_query.title = effective_query
|
||||
elif effective_search_type == "permalink" and "*" in effective_query:
|
||||
search_query.permalink_match = effective_query
|
||||
elif effective_search_type == "permalink":
|
||||
search_query.permalink = effective_query
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Invalid search_type '{effective_search_type}'. "
|
||||
f"Valid options: {', '.join(sorted(valid_search_types))}"
|
||||
)
|
||||
|
||||
# Add optional filters if provided (empty lists are treated as no filter)
|
||||
if entity_types:
|
||||
search_query.entity_types = [SearchItemType(t) for t in entity_types]
|
||||
if note_types:
|
||||
search_query.note_types = note_types
|
||||
if after_date:
|
||||
search_query.after_date = after_date
|
||||
if metadata_filters:
|
||||
# Alias common column/model names to their frontmatter key equivalents.
|
||||
# Users often pass "note_type" (the entity model column) when the
|
||||
# frontmatter field is actually "type".
|
||||
_METADATA_KEY_ALIASES = {"note_type": "type"}
|
||||
metadata_filters = {
|
||||
_METADATA_KEY_ALIASES.get(k, k): v for k, v in metadata_filters.items()
|
||||
}
|
||||
search_query.metadata_filters = metadata_filters
|
||||
if tags:
|
||||
search_query.tags = tags
|
||||
if status:
|
||||
search_query.status = status
|
||||
if min_similarity is not None:
|
||||
search_query.min_similarity = min_similarity
|
||||
|
||||
# Reject searches with no criteria at all
|
||||
if search_query.no_criteria():
|
||||
return (
|
||||
"# No Search Criteria\n\n"
|
||||
"Please provide at least one of: `query`, `metadata_filters`, "
|
||||
"`tags`, `status`, `note_types`, `entity_types`, or `after_date`."
|
||||
)
|
||||
|
||||
# Default to entity-level results to avoid returning individual
|
||||
# observations/relations as separate search results (see issue #31).
|
||||
# Applied after no_criteria() so that the implicit default doesn't
|
||||
# mask a truly empty search request.
|
||||
if not search_query.entity_types:
|
||||
search_query.entity_types = [SearchItemType("entity")]
|
||||
|
||||
logger.debug(
|
||||
f"Search request: project={active_project.name} "
|
||||
f"search_type={effective_search_type} "
|
||||
f"query={effective_query or '<filters-only>'} "
|
||||
f"note_types={len(note_types)} entity_types={len(search_query.entity_types or [])} "
|
||||
f"page={page} page_size={page_size}"
|
||||
)
|
||||
# Import here to avoid circular import (tools → clients → utils → tools)
|
||||
from basic_memory.mcp.clients import SearchClient
|
||||
|
||||
# Use typed SearchClient for API calls
|
||||
search_client = SearchClient(client, active_project.external_id)
|
||||
result = await search_client.search(
|
||||
search_query.model_dump(),
|
||||
page=page,
|
||||
page_size=page_size,
|
||||
)
|
||||
logger.debug(
|
||||
f"Search response: project={active_project.name} "
|
||||
f"results={len(result.results)} has_more={str(result.has_more).lower()} "
|
||||
f"page={result.current_page} page_size={result.page_size}"
|
||||
)
|
||||
|
||||
# Add optional filters if provided (empty lists are treated as no filter)
|
||||
if entity_types:
|
||||
search_query.entity_types = [SearchItemType(t) for t in entity_types]
|
||||
if note_types:
|
||||
search_query.note_types = note_types
|
||||
if after_date:
|
||||
search_query.after_date = after_date
|
||||
if metadata_filters:
|
||||
# Alias common column/model names to their frontmatter key equivalents.
|
||||
# Users often pass "note_type" (the entity model column) when the
|
||||
# frontmatter field is actually "type".
|
||||
_METADATA_KEY_ALIASES = {"note_type": "type"}
|
||||
metadata_filters = {
|
||||
_METADATA_KEY_ALIASES.get(k, k): v for k, v in metadata_filters.items()
|
||||
}
|
||||
search_query.metadata_filters = metadata_filters
|
||||
if tags:
|
||||
search_query.tags = tags
|
||||
if status:
|
||||
search_query.status = status
|
||||
if min_similarity is not None:
|
||||
search_query.min_similarity = min_similarity
|
||||
# Check if we got no results and provide helpful guidance
|
||||
if not result.results:
|
||||
logger.debug(
|
||||
f"Search returned no results for query: {query} in project {active_project.name}"
|
||||
)
|
||||
# Don't treat this as an error, but the user might want guidance
|
||||
# We return the empty result as normal - the user can decide if they need help
|
||||
|
||||
# Reject searches with no criteria at all
|
||||
if search_query.no_criteria():
|
||||
return (
|
||||
"# No Search Criteria\n\n"
|
||||
"Please provide at least one of: `query`, `metadata_filters`, "
|
||||
"`tags`, `status`, `note_types`, `entity_types`, or `after_date`."
|
||||
)
|
||||
if output_format == "json":
|
||||
return result.model_dump(mode="json", exclude_none=True)
|
||||
|
||||
# Default to entity-level results to avoid returning individual
|
||||
# observations/relations as separate search results (see issue #31).
|
||||
# Applied after no_criteria() so that the implicit default doesn't
|
||||
# mask a truly empty search request.
|
||||
if not search_query.entity_types:
|
||||
search_query.entity_types = [SearchItemType("entity")]
|
||||
return _format_search_markdown(result, active_project.name, query)
|
||||
|
||||
logger.debug(f"Searching for {search_query} in project {active_project.name}")
|
||||
# Import here to avoid circular import (tools → clients → utils → tools)
|
||||
from basic_memory.mcp.clients import SearchClient
|
||||
|
||||
# Use typed SearchClient for API calls
|
||||
search_client = SearchClient(client, active_project.external_id)
|
||||
result = await search_client.search(
|
||||
search_query.model_dump(),
|
||||
page=page,
|
||||
page_size=page_size,
|
||||
)
|
||||
|
||||
# Check if we got no results and provide helpful guidance
|
||||
if not result.results:
|
||||
logger.debug(
|
||||
f"Search returned no results for query: {query} in project {active_project.name}"
|
||||
)
|
||||
# Don't treat this as an error, but the user might want guidance
|
||||
# We return the empty result as normal - the user can decide if they need help
|
||||
|
||||
if output_format == "json":
|
||||
return result.model_dump(mode="json", exclude_none=True)
|
||||
|
||||
return _format_search_markdown(result, active_project.name, query)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
f"Search failed for query '{query or ''}': {e}, project: {active_project.name}"
|
||||
)
|
||||
# Return formatted error message as string for better user experience
|
||||
return _format_search_error_response(
|
||||
active_project.name, str(e), query or "", effective_search_type
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
f"Search failed for query '{query or ''}': {e}, project: {active_project.name}"
|
||||
)
|
||||
# Return formatted error message as string for better user experience
|
||||
return _format_search_error_response(
|
||||
active_project.name, str(e), query or "", effective_search_type
|
||||
)
|
||||
|
||||
@@ -5,6 +5,7 @@ to the Basic Memory API, with improved error handling and logging.
|
||||
"""
|
||||
|
||||
import typing
|
||||
from contextlib import contextmanager
|
||||
from typing import Optional
|
||||
|
||||
from httpx import Response, URL, AsyncClient, HTTPStatusError
|
||||
@@ -23,9 +24,62 @@ from httpx._types import (
|
||||
from loguru import logger
|
||||
from mcp.server.fastmcp.exceptions import ToolError
|
||||
|
||||
from basic_memory import telemetry
|
||||
from basic_memory.config import ConfigManager
|
||||
|
||||
|
||||
def _classify_http_outcome(status_code: int) -> str:
|
||||
"""Map HTTP status codes to a low-cardinality outcome label."""
|
||||
if 200 <= status_code < 300:
|
||||
return "success"
|
||||
if 300 <= status_code < 400: # pragma: no cover
|
||||
return "redirect"
|
||||
if 400 <= status_code < 500:
|
||||
return "client_error"
|
||||
if 500 <= status_code < 600:
|
||||
return "server_error"
|
||||
return "unknown" # pragma: no cover
|
||||
|
||||
|
||||
class _RequestSpan:
|
||||
"""Small adapter for attaching outcome metadata to a live request span."""
|
||||
|
||||
def __init__(self, active_span: typing.Any | None):
|
||||
self._active_span = active_span
|
||||
|
||||
def record_response(self, response: Response) -> None:
|
||||
self._set_attributes(
|
||||
{
|
||||
"status_code": response.status_code,
|
||||
"is_success": response.is_success,
|
||||
"outcome": _classify_http_outcome(response.status_code),
|
||||
}
|
||||
)
|
||||
|
||||
def record_transport_error(self, exc: Exception) -> None:
|
||||
self._set_attributes(
|
||||
{
|
||||
"is_success": False,
|
||||
"outcome": "transport_error",
|
||||
"error_type": type(exc).__name__,
|
||||
}
|
||||
)
|
||||
|
||||
def _set_attributes(self, attrs: dict[str, typing.Any]) -> None:
|
||||
if self._active_span is None:
|
||||
return
|
||||
|
||||
set_attributes = getattr(self._active_span, "set_attributes", None)
|
||||
if callable(set_attributes):
|
||||
set_attributes(attrs)
|
||||
return
|
||||
|
||||
set_attribute = getattr(self._active_span, "set_attribute", None)
|
||||
if callable(set_attribute):
|
||||
for key, value in attrs.items():
|
||||
set_attribute(key, value)
|
||||
|
||||
|
||||
def get_error_message(
|
||||
status_code: int, url: URL | str, method: str, msg: Optional[str] = None
|
||||
) -> str:
|
||||
@@ -135,10 +189,38 @@ def _resolve_error_message(
|
||||
return get_error_message(status_code, url, method)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _request_scope(
|
||||
method: str,
|
||||
*,
|
||||
client_name: str | None,
|
||||
operation: str | None,
|
||||
path_template: str | None,
|
||||
params: QueryParamTypes | None = None,
|
||||
has_body: bool = False,
|
||||
):
|
||||
"""Create the shared MCP transport span used by all HTTP helpers."""
|
||||
attrs = {
|
||||
"method": method,
|
||||
"client_name": client_name,
|
||||
"operation": operation,
|
||||
"path_template": path_template,
|
||||
"phase": "request",
|
||||
"has_query": bool(params),
|
||||
"has_body": has_body,
|
||||
}
|
||||
with telemetry.contextualize(**attrs):
|
||||
with telemetry.started_span("mcp.http.request", **attrs) as active_span:
|
||||
yield _RequestSpan(active_span)
|
||||
|
||||
|
||||
async def call_get(
|
||||
client: AsyncClient,
|
||||
url: URL | str,
|
||||
*,
|
||||
client_name: str | None = None,
|
||||
operation: str | None = None,
|
||||
path_template: str | None = None,
|
||||
params: QueryParamTypes | None = None,
|
||||
headers: HeaderTypes | None = None,
|
||||
cookies: CookieTypes | None = None,
|
||||
@@ -168,18 +250,27 @@ async def call_get(
|
||||
"""
|
||||
logger.debug(f"Calling GET '{url}' params: '{params}'")
|
||||
error_message = None
|
||||
request_span: _RequestSpan | None = None
|
||||
|
||||
try:
|
||||
response = await client.get(
|
||||
url,
|
||||
with _request_scope(
|
||||
"GET",
|
||||
client_name=client_name,
|
||||
operation=operation,
|
||||
path_template=path_template,
|
||||
params=params,
|
||||
headers=headers,
|
||||
cookies=cookies,
|
||||
auth=auth,
|
||||
follow_redirects=follow_redirects,
|
||||
timeout=timeout,
|
||||
extensions=extensions,
|
||||
)
|
||||
) as request_span:
|
||||
response = await client.get(
|
||||
url,
|
||||
params=params,
|
||||
headers=headers,
|
||||
cookies=cookies,
|
||||
auth=auth,
|
||||
follow_redirects=follow_redirects,
|
||||
timeout=timeout,
|
||||
extensions=extensions,
|
||||
)
|
||||
request_span.record_response(response)
|
||||
|
||||
if response.is_success:
|
||||
return response
|
||||
@@ -206,12 +297,19 @@ async def call_get(
|
||||
|
||||
except HTTPStatusError as e:
|
||||
raise ToolError(error_message) from e
|
||||
except Exception as e:
|
||||
if request_span is not None:
|
||||
request_span.record_transport_error(e)
|
||||
raise
|
||||
|
||||
|
||||
async def call_put(
|
||||
client: AsyncClient,
|
||||
url: URL | str,
|
||||
*,
|
||||
client_name: str | None = None,
|
||||
operation: str | None = None,
|
||||
path_template: str | None = None,
|
||||
content: RequestContent | None = None,
|
||||
data: RequestData | None = None,
|
||||
files: RequestFiles | None = None,
|
||||
@@ -249,22 +347,32 @@ async def call_put(
|
||||
"""
|
||||
logger.debug(f"Calling PUT '{url}'")
|
||||
error_message = None
|
||||
request_span: _RequestSpan | None = None
|
||||
|
||||
try:
|
||||
response = await client.put(
|
||||
url,
|
||||
content=content,
|
||||
data=data,
|
||||
files=files,
|
||||
json=json,
|
||||
with _request_scope(
|
||||
"PUT",
|
||||
client_name=client_name,
|
||||
operation=operation,
|
||||
path_template=path_template,
|
||||
params=params,
|
||||
headers=headers,
|
||||
cookies=cookies,
|
||||
auth=auth,
|
||||
follow_redirects=follow_redirects,
|
||||
timeout=timeout,
|
||||
extensions=extensions,
|
||||
)
|
||||
has_body=any(value is not None for value in (content, data, files, json)),
|
||||
) as request_span:
|
||||
response = await client.put(
|
||||
url,
|
||||
content=content,
|
||||
data=data,
|
||||
files=files,
|
||||
json=json,
|
||||
params=params,
|
||||
headers=headers,
|
||||
cookies=cookies,
|
||||
auth=auth,
|
||||
follow_redirects=follow_redirects,
|
||||
timeout=timeout,
|
||||
extensions=extensions,
|
||||
)
|
||||
request_span.record_response(response)
|
||||
|
||||
if response.is_success:
|
||||
return response
|
||||
@@ -292,12 +400,19 @@ async def call_put(
|
||||
|
||||
except HTTPStatusError as e:
|
||||
raise ToolError(error_message) from e
|
||||
except Exception as e:
|
||||
if request_span is not None:
|
||||
request_span.record_transport_error(e)
|
||||
raise
|
||||
|
||||
|
||||
async def call_patch(
|
||||
client: AsyncClient,
|
||||
url: URL | str,
|
||||
*,
|
||||
client_name: str | None = None,
|
||||
operation: str | None = None,
|
||||
path_template: str | None = None,
|
||||
content: RequestContent | None = None,
|
||||
data: RequestData | None = None,
|
||||
files: RequestFiles | None = None,
|
||||
@@ -334,22 +449,32 @@ async def call_patch(
|
||||
ToolError: If the request fails with an appropriate error message
|
||||
"""
|
||||
logger.debug(f"Calling PATCH '{url}'")
|
||||
request_span: _RequestSpan | None = None
|
||||
|
||||
try:
|
||||
response = await client.patch(
|
||||
url,
|
||||
content=content,
|
||||
data=data,
|
||||
files=files,
|
||||
json=json,
|
||||
with _request_scope(
|
||||
"PATCH",
|
||||
client_name=client_name,
|
||||
operation=operation,
|
||||
path_template=path_template,
|
||||
params=params,
|
||||
headers=headers,
|
||||
cookies=cookies,
|
||||
auth=auth,
|
||||
follow_redirects=follow_redirects,
|
||||
timeout=timeout,
|
||||
extensions=extensions,
|
||||
)
|
||||
has_body=any(value is not None for value in (content, data, files, json)),
|
||||
) as request_span:
|
||||
response = await client.patch(
|
||||
url,
|
||||
content=content,
|
||||
data=data,
|
||||
files=files,
|
||||
json=json,
|
||||
params=params,
|
||||
headers=headers,
|
||||
cookies=cookies,
|
||||
auth=auth,
|
||||
follow_redirects=follow_redirects,
|
||||
timeout=timeout,
|
||||
extensions=extensions,
|
||||
)
|
||||
request_span.record_response(response)
|
||||
|
||||
if response.is_success:
|
||||
return response
|
||||
@@ -382,12 +507,19 @@ async def call_patch(
|
||||
error_message = _resolve_error_message(status_code, url, "PATCH", response_data)
|
||||
|
||||
raise ToolError(error_message) from e
|
||||
except Exception as e:
|
||||
if request_span is not None:
|
||||
request_span.record_transport_error(e)
|
||||
raise
|
||||
|
||||
|
||||
async def call_post(
|
||||
client: AsyncClient,
|
||||
url: URL | str,
|
||||
*,
|
||||
client_name: str | None = None,
|
||||
operation: str | None = None,
|
||||
path_template: str | None = None,
|
||||
content: RequestContent | None = None,
|
||||
data: RequestData | None = None,
|
||||
files: RequestFiles | None = None,
|
||||
@@ -425,23 +557,33 @@ async def call_post(
|
||||
"""
|
||||
logger.debug(f"Calling POST '{url}'")
|
||||
error_message = None
|
||||
request_span: _RequestSpan | None = None
|
||||
|
||||
try:
|
||||
response = await client.post(
|
||||
url=url,
|
||||
content=content,
|
||||
data=data,
|
||||
files=files,
|
||||
json=json,
|
||||
with _request_scope(
|
||||
"POST",
|
||||
client_name=client_name,
|
||||
operation=operation,
|
||||
path_template=path_template,
|
||||
params=params,
|
||||
headers=headers,
|
||||
cookies=cookies,
|
||||
auth=auth,
|
||||
follow_redirects=follow_redirects,
|
||||
timeout=timeout,
|
||||
extensions=extensions,
|
||||
)
|
||||
logger.debug(f"response: {response.json()}")
|
||||
has_body=any(value is not None for value in (content, data, files, json)),
|
||||
) as request_span:
|
||||
response = await client.post(
|
||||
url=url,
|
||||
content=content,
|
||||
data=data,
|
||||
files=files,
|
||||
json=json,
|
||||
params=params,
|
||||
headers=headers,
|
||||
cookies=cookies,
|
||||
auth=auth,
|
||||
follow_redirects=follow_redirects,
|
||||
timeout=timeout,
|
||||
extensions=extensions,
|
||||
)
|
||||
request_span.record_response(response)
|
||||
logger.debug(f"response: {_extract_response_data(response)}")
|
||||
|
||||
if response.is_success:
|
||||
return response
|
||||
@@ -468,6 +610,10 @@ async def call_post(
|
||||
|
||||
except HTTPStatusError as e:
|
||||
raise ToolError(error_message) from e
|
||||
except Exception as e:
|
||||
if request_span is not None:
|
||||
request_span.record_transport_error(e)
|
||||
raise
|
||||
|
||||
|
||||
async def resolve_entity_id(client: AsyncClient, project_external_id: str, identifier: str) -> str:
|
||||
@@ -506,6 +652,9 @@ async def call_delete(
|
||||
client: AsyncClient,
|
||||
url: URL | str,
|
||||
*,
|
||||
client_name: str | None = None,
|
||||
operation: str | None = None,
|
||||
path_template: str | None = None,
|
||||
params: QueryParamTypes | None = None,
|
||||
headers: HeaderTypes | None = None,
|
||||
cookies: CookieTypes | None = None,
|
||||
@@ -535,18 +684,27 @@ async def call_delete(
|
||||
"""
|
||||
logger.debug(f"Calling DELETE '{url}'")
|
||||
error_message = None
|
||||
request_span: _RequestSpan | None = None
|
||||
|
||||
try:
|
||||
response = await client.delete(
|
||||
url=url,
|
||||
with _request_scope(
|
||||
"DELETE",
|
||||
client_name=client_name,
|
||||
operation=operation,
|
||||
path_template=path_template,
|
||||
params=params,
|
||||
headers=headers,
|
||||
cookies=cookies,
|
||||
auth=auth,
|
||||
follow_redirects=follow_redirects,
|
||||
timeout=timeout,
|
||||
extensions=extensions,
|
||||
)
|
||||
) as request_span:
|
||||
response = await client.delete(
|
||||
url=url,
|
||||
params=params,
|
||||
headers=headers,
|
||||
cookies=cookies,
|
||||
auth=auth,
|
||||
follow_redirects=follow_redirects,
|
||||
timeout=timeout,
|
||||
extensions=extensions,
|
||||
)
|
||||
request_span.record_response(response)
|
||||
|
||||
if response.is_success:
|
||||
return response
|
||||
@@ -573,3 +731,7 @@ async def call_delete(
|
||||
|
||||
except HTTPStatusError as e:
|
||||
raise ToolError(error_message) from e
|
||||
except Exception as e:
|
||||
if request_span is not None:
|
||||
request_span.record_transport_error(e)
|
||||
raise
|
||||
|
||||
@@ -6,6 +6,7 @@ from typing import Annotated, List, Union, Optional, Literal
|
||||
from loguru import logger
|
||||
from pydantic import BeforeValidator
|
||||
|
||||
from basic_memory import telemetry
|
||||
from basic_memory.config import ConfigManager
|
||||
from basic_memory.mcp.project_context import get_project_client, add_project_metadata
|
||||
from basic_memory.mcp.server import mcp
|
||||
@@ -148,161 +149,180 @@ async def write_note(
|
||||
overwrite if overwrite is not None else ConfigManager().config.write_note_overwrite_default
|
||||
)
|
||||
|
||||
async with get_project_client(project, workspace, context) as (client, active_project):
|
||||
logger.info(
|
||||
f"MCP tool call tool=write_note project={active_project.name} directory={directory}, title={title}, tags={tags}"
|
||||
)
|
||||
|
||||
# Normalize "/" to empty string for root directory (must happen before validation)
|
||||
if directory == "/":
|
||||
directory = ""
|
||||
|
||||
# Validate directory path to prevent path traversal attacks
|
||||
project_path = active_project.home
|
||||
if directory and not validate_project_path(directory, project_path):
|
||||
logger.warning(
|
||||
"Attempted path traversal attack blocked",
|
||||
directory=directory,
|
||||
project=active_project.name,
|
||||
)
|
||||
if output_format == "json":
|
||||
return {
|
||||
"title": title,
|
||||
"permalink": None,
|
||||
"file_path": None,
|
||||
"checksum": None,
|
||||
"action": "created",
|
||||
"error": "SECURITY_VALIDATION_ERROR",
|
||||
}
|
||||
return f"# Error\n\nDirectory path '{directory}' is not allowed - paths must stay within project boundaries"
|
||||
|
||||
# Process tags using the helper function
|
||||
tag_list = parse_tags(tags)
|
||||
|
||||
# Build entity_metadata from optional metadata, then explicit tags on top
|
||||
# Order matters: explicit tags parameter takes precedence over metadata["tags"]
|
||||
entity_metadata = {}
|
||||
if metadata:
|
||||
entity_metadata.update(metadata)
|
||||
if tag_list:
|
||||
entity_metadata["tags"] = tag_list
|
||||
|
||||
entity = Entity(
|
||||
title=title,
|
||||
directory=directory,
|
||||
note_type=note_type,
|
||||
content_type="text/markdown",
|
||||
content=content,
|
||||
entity_metadata=entity_metadata or None,
|
||||
)
|
||||
|
||||
# Import here to avoid circular import
|
||||
from basic_memory.mcp.clients import KnowledgeClient
|
||||
|
||||
# Use typed KnowledgeClient for API calls
|
||||
knowledge_client = KnowledgeClient(client, active_project.external_id)
|
||||
|
||||
# Try to create the entity first (optimistic create)
|
||||
logger.debug(f"Attempting to create entity permalink={entity.permalink}")
|
||||
action = "Created" # Default to created
|
||||
try:
|
||||
result = await knowledge_client.create_entity(entity.model_dump(), fast=False)
|
||||
action = "Created"
|
||||
except Exception as e:
|
||||
# If creation failed due to conflict (already exists), try to update
|
||||
if (
|
||||
"409" in str(e)
|
||||
or "conflict" in str(e).lower()
|
||||
or "already exists" in str(e).lower()
|
||||
with telemetry.operation(
|
||||
"mcp.tool.write_note",
|
||||
entrypoint="mcp",
|
||||
tool_name="write_note",
|
||||
requested_project=project,
|
||||
workspace_id=workspace,
|
||||
note_type=note_type,
|
||||
overwrite=effective_overwrite,
|
||||
output_format=output_format,
|
||||
):
|
||||
async with get_project_client(project, workspace, context) as (client, active_project):
|
||||
with telemetry.contextualize(
|
||||
project_name=active_project.name,
|
||||
workspace_id=workspace,
|
||||
tool_name="write_note",
|
||||
):
|
||||
# Guard: block overwrite unless explicitly enabled
|
||||
if not effective_overwrite:
|
||||
logger.info(
|
||||
f"MCP tool call tool=write_note project={active_project.name} directory={directory}, title={title}, tags={tags}"
|
||||
)
|
||||
|
||||
# Normalize "/" to empty string for root directory (must happen before validation)
|
||||
if directory == "/":
|
||||
directory = ""
|
||||
|
||||
# Validate directory path to prevent path traversal attacks
|
||||
project_path = active_project.home
|
||||
if directory and not validate_project_path(directory, project_path):
|
||||
logger.warning(
|
||||
f"write_note blocked: note already exists (overwrite not enabled) "
|
||||
f"permalink={entity.permalink}"
|
||||
"Attempted path traversal attack blocked",
|
||||
directory=directory,
|
||||
project=active_project.name,
|
||||
)
|
||||
if output_format == "json":
|
||||
return {
|
||||
"title": title,
|
||||
"permalink": entity.permalink,
|
||||
"permalink": None,
|
||||
"file_path": None,
|
||||
"checksum": None,
|
||||
"action": "conflict",
|
||||
"error": "NOTE_ALREADY_EXISTS",
|
||||
"action": "created",
|
||||
"error": "SECURITY_VALIDATION_ERROR",
|
||||
}
|
||||
return _format_overwrite_error(title, entity.permalink, active_project.name)
|
||||
return f"# Error\n\nDirectory path '{directory}' is not allowed - paths must stay within project boundaries"
|
||||
|
||||
logger.debug(f"Entity exists, updating instead permalink={entity.permalink}")
|
||||
# Process tags using the helper function
|
||||
tag_list = parse_tags(tags)
|
||||
|
||||
# Build entity_metadata from optional metadata, then explicit tags on top
|
||||
# Order matters: explicit tags parameter takes precedence over metadata["tags"]
|
||||
entity_metadata = {}
|
||||
if metadata:
|
||||
entity_metadata.update(metadata)
|
||||
if tag_list:
|
||||
entity_metadata["tags"] = tag_list
|
||||
|
||||
entity = Entity(
|
||||
title=title,
|
||||
directory=directory,
|
||||
note_type=note_type,
|
||||
content_type="text/markdown",
|
||||
content=content,
|
||||
entity_metadata=entity_metadata or None,
|
||||
)
|
||||
|
||||
# Import here to avoid circular import
|
||||
from basic_memory.mcp.clients import KnowledgeClient
|
||||
|
||||
# Use typed KnowledgeClient for API calls
|
||||
knowledge_client = KnowledgeClient(client, active_project.external_id)
|
||||
|
||||
# Try to create the entity first (optimistic create)
|
||||
logger.debug(f"Attempting to create entity permalink={entity.permalink}")
|
||||
action = "Created" # Default to created
|
||||
try:
|
||||
if not entity.permalink:
|
||||
raise ValueError(
|
||||
"Entity permalink is required for updates"
|
||||
) # pragma: no cover
|
||||
entity_id = await knowledge_client.resolve_entity(entity.permalink)
|
||||
result = await knowledge_client.update_entity(
|
||||
entity_id, entity.model_dump(), fast=False
|
||||
)
|
||||
action = "Updated"
|
||||
except Exception as update_error: # pragma: no cover
|
||||
# Re-raise the original error if update also fails
|
||||
raise e from update_error # pragma: no cover
|
||||
else:
|
||||
# Re-raise if it's not a conflict error
|
||||
raise # pragma: no cover
|
||||
summary = [
|
||||
f"# {action} note",
|
||||
f"project: {active_project.name}",
|
||||
f"file_path: {result.file_path}",
|
||||
f"permalink: {result.permalink}",
|
||||
f"checksum: {result.checksum[:8] if result.checksum else 'unknown'}",
|
||||
]
|
||||
result = await knowledge_client.create_entity(entity.model_dump(), fast=False)
|
||||
action = "Created"
|
||||
except Exception as e:
|
||||
# If creation failed due to conflict (already exists), try to update
|
||||
if (
|
||||
"409" in str(e)
|
||||
or "conflict" in str(e).lower()
|
||||
or "already exists" in str(e).lower()
|
||||
):
|
||||
# Guard: block overwrite unless explicitly enabled
|
||||
if not effective_overwrite:
|
||||
logger.warning(
|
||||
f"write_note blocked: note already exists (overwrite not enabled) "
|
||||
f"permalink={entity.permalink}"
|
||||
)
|
||||
if output_format == "json":
|
||||
return {
|
||||
"title": title,
|
||||
"permalink": entity.permalink,
|
||||
"file_path": None,
|
||||
"checksum": None,
|
||||
"action": "conflict",
|
||||
"error": "NOTE_ALREADY_EXISTS",
|
||||
}
|
||||
return _format_overwrite_error(
|
||||
title, entity.permalink, active_project.name
|
||||
)
|
||||
|
||||
# Count observations by category
|
||||
categories = {}
|
||||
if result.observations:
|
||||
for obs in result.observations:
|
||||
categories[obs.category] = categories.get(obs.category, 0) + 1
|
||||
logger.debug(
|
||||
f"Entity exists, updating instead permalink={entity.permalink}"
|
||||
)
|
||||
try:
|
||||
if not entity.permalink:
|
||||
raise ValueError(
|
||||
"Entity permalink is required for updates"
|
||||
) # pragma: no cover
|
||||
entity_id = await knowledge_client.resolve_entity(entity.permalink)
|
||||
result = await knowledge_client.update_entity(
|
||||
entity_id, entity.model_dump(), fast=False
|
||||
)
|
||||
action = "Updated"
|
||||
except Exception as update_error: # pragma: no cover
|
||||
# Re-raise the original error if update also fails
|
||||
raise e from update_error # pragma: no cover
|
||||
else:
|
||||
# Re-raise if it's not a conflict error
|
||||
raise # pragma: no cover
|
||||
summary = [
|
||||
f"# {action} note",
|
||||
f"project: {active_project.name}",
|
||||
f"file_path: {result.file_path}",
|
||||
f"permalink: {result.permalink}",
|
||||
f"checksum: {result.checksum[:8] if result.checksum else 'unknown'}",
|
||||
]
|
||||
|
||||
summary.append("\n## Observations")
|
||||
for category, count in sorted(categories.items()):
|
||||
summary.append(f"- {category}: {count}")
|
||||
# Count observations by category
|
||||
categories = {}
|
||||
if result.observations:
|
||||
for obs in result.observations:
|
||||
categories[obs.category] = categories.get(obs.category, 0) + 1
|
||||
|
||||
# Count resolved/unresolved relations
|
||||
unresolved = 0
|
||||
resolved = 0
|
||||
if result.relations:
|
||||
unresolved = sum(1 for r in result.relations if not r.to_id)
|
||||
resolved = len(result.relations) - unresolved
|
||||
summary.append("\n## Observations")
|
||||
for category, count in sorted(categories.items()):
|
||||
summary.append(f"- {category}: {count}")
|
||||
|
||||
summary.append("\n## Relations")
|
||||
summary.append(f"- Resolved: {resolved}")
|
||||
if unresolved:
|
||||
summary.append(f"- Unresolved: {unresolved}")
|
||||
summary.append(
|
||||
"\nNote: Unresolved relations point to entities that don't exist yet."
|
||||
)
|
||||
summary.append(
|
||||
"They will be automatically resolved when target entities are created or during sync operations."
|
||||
# Count resolved/unresolved relations
|
||||
unresolved = 0
|
||||
resolved = 0
|
||||
if result.relations:
|
||||
unresolved = sum(1 for r in result.relations if not r.to_id)
|
||||
resolved = len(result.relations) - unresolved
|
||||
|
||||
summary.append("\n## Relations")
|
||||
summary.append(f"- Resolved: {resolved}")
|
||||
if unresolved:
|
||||
summary.append(f"- Unresolved: {unresolved}")
|
||||
summary.append(
|
||||
"\nNote: Unresolved relations point to entities that don't exist yet."
|
||||
)
|
||||
summary.append(
|
||||
"They will be automatically resolved when target entities are created or during sync operations."
|
||||
)
|
||||
|
||||
if tag_list:
|
||||
summary.append(f"\n## Tags\n- {', '.join(tag_list)}")
|
||||
|
||||
# Log the response with structured data
|
||||
logger.info(
|
||||
f"MCP tool response: tool=write_note project={active_project.name} action={action} permalink={result.permalink} observations_count={len(result.observations)} relations_count={len(result.relations)} resolved_relations={resolved} unresolved_relations={unresolved}"
|
||||
)
|
||||
if output_format == "json":
|
||||
return {
|
||||
"title": result.title,
|
||||
"permalink": result.permalink,
|
||||
"file_path": result.file_path,
|
||||
"checksum": result.checksum,
|
||||
"action": action.lower(),
|
||||
}
|
||||
|
||||
if tag_list:
|
||||
summary.append(f"\n## Tags\n- {', '.join(tag_list)}")
|
||||
|
||||
# Log the response with structured data
|
||||
logger.info(
|
||||
f"MCP tool response: tool=write_note project={active_project.name} action={action} permalink={result.permalink} observations_count={len(result.observations)} relations_count={len(result.relations)} resolved_relations={resolved} unresolved_relations={unresolved}"
|
||||
)
|
||||
if output_format == "json":
|
||||
return {
|
||||
"title": result.title,
|
||||
"permalink": result.permalink,
|
||||
"file_path": result.file_path,
|
||||
"checksum": result.checksum,
|
||||
"action": action.lower(),
|
||||
}
|
||||
|
||||
summary_result = "\n".join(summary)
|
||||
return add_project_metadata(summary_result, active_project.name)
|
||||
summary_result = "\n".join(summary)
|
||||
return add_project_metadata(summary_result, active_project.name)
|
||||
|
||||
|
||||
def _format_overwrite_error(title: str, permalink: str | None, project_name: str) -> str:
|
||||
|
||||
@@ -59,16 +59,27 @@ class EntityRepository(Repository[Entity]):
|
||||
)
|
||||
return await self.find_one(query)
|
||||
|
||||
async def get_by_permalink(self, permalink: str) -> Optional[Entity]:
|
||||
async def _find_one_by_query(self, query, *, load_relations: bool) -> Optional[Entity]:
|
||||
"""Return one entity row with optional eager loading."""
|
||||
if load_relations:
|
||||
query = query.options(*self.get_load_options())
|
||||
return await self.find_one(query)
|
||||
|
||||
result = await self.execute_query(query, use_query_options=False)
|
||||
return result.scalars().one_or_none()
|
||||
|
||||
async def get_by_permalink(
|
||||
self, permalink: str, *, load_relations: bool = True
|
||||
) -> Optional[Entity]:
|
||||
"""Get entity by permalink.
|
||||
|
||||
Args:
|
||||
permalink: Unique identifier for the entity
|
||||
"""
|
||||
query = self.select().where(Entity.permalink == permalink).options(*self.get_load_options())
|
||||
return await self.find_one(query)
|
||||
query = self.select().where(Entity.permalink == permalink)
|
||||
return await self._find_one_by_query(query, load_relations=load_relations)
|
||||
|
||||
async def get_by_title(self, title: str) -> Sequence[Entity]:
|
||||
async def get_by_title(self, title: str, *, load_relations: bool = True) -> Sequence[Entity]:
|
||||
"""Get entities by title, ordered by shortest path first.
|
||||
|
||||
When multiple entities share the same title (in different folders),
|
||||
@@ -82,23 +93,20 @@ class EntityRepository(Repository[Entity]):
|
||||
self.select()
|
||||
.where(Entity.title == title)
|
||||
.order_by(func.length(Entity.file_path), Entity.file_path)
|
||||
.options(*self.get_load_options())
|
||||
)
|
||||
result = await self.execute_query(query)
|
||||
result = await self.execute_query(query, use_query_options=load_relations)
|
||||
return list(result.scalars().all())
|
||||
|
||||
async def get_by_file_path(self, file_path: Union[Path, str]) -> Optional[Entity]:
|
||||
async def get_by_file_path(
|
||||
self, file_path: Union[Path, str], *, load_relations: bool = True
|
||||
) -> Optional[Entity]:
|
||||
"""Get entity by file_path.
|
||||
|
||||
Args:
|
||||
file_path: Path to the entity file (will be converted to string internally)
|
||||
"""
|
||||
query = (
|
||||
self.select()
|
||||
.where(Entity.file_path == Path(file_path).as_posix())
|
||||
.options(*self.get_load_options())
|
||||
)
|
||||
return await self.find_one(query)
|
||||
query = self.select().where(Entity.file_path == Path(file_path).as_posix())
|
||||
return await self._find_one_by_query(query, load_relations=load_relations)
|
||||
|
||||
# -------------------------------------------------------------------------
|
||||
# Lightweight methods for permalink resolution (no eager loading)
|
||||
@@ -306,7 +314,7 @@ class EntityRepository(Repository[Entity]):
|
||||
result = await self.execute_query(query)
|
||||
return list(result.scalars().all())
|
||||
|
||||
async def upsert_entity(self, entity: Entity) -> Entity:
|
||||
async def upsert_entity(self, entity: Entity, *, reload: bool = True) -> Entity:
|
||||
"""Insert or update entity using simple try/catch with database-level conflict resolution.
|
||||
|
||||
Handles file_path race conditions by checking for existing entity on IntegrityError.
|
||||
@@ -327,6 +335,9 @@ class EntityRepository(Repository[Entity]):
|
||||
session.add(entity)
|
||||
await session.flush()
|
||||
|
||||
if not reload:
|
||||
return entity
|
||||
|
||||
# Return with relationships loaded
|
||||
query = (
|
||||
self.select()
|
||||
@@ -363,13 +374,12 @@ class EntityRepository(Repository[Entity]):
|
||||
await session.rollback()
|
||||
|
||||
# Re-query after rollback to get a fresh, attached entity
|
||||
existing_result = await session.execute(
|
||||
select(Entity)
|
||||
.where(
|
||||
Entity.file_path == entity.file_path, Entity.project_id == entity.project_id
|
||||
)
|
||||
.options(*self.get_load_options())
|
||||
existing_query = select(Entity).where(
|
||||
Entity.file_path == entity.file_path, Entity.project_id == entity.project_id
|
||||
)
|
||||
if reload:
|
||||
existing_query = existing_query.options(*self.get_load_options())
|
||||
existing_result = await session.execute(existing_query)
|
||||
existing_entity = existing_result.scalar_one_or_none()
|
||||
|
||||
if existing_entity:
|
||||
@@ -393,6 +403,9 @@ class EntityRepository(Repository[Entity]):
|
||||
|
||||
await session.commit()
|
||||
|
||||
if not reload:
|
||||
return merged_entity
|
||||
|
||||
# Re-query to get proper relationships loaded
|
||||
final_result = await session.execute(
|
||||
select(Entity)
|
||||
|
||||
@@ -268,8 +268,21 @@ class Repository[T: Base]:
|
||||
|
||||
return await self.select_by_ids(session, [model.id for model in model_list]) # pyright: ignore [reportAttributeAccessIssue]
|
||||
|
||||
async def update(self, entity_id: int, entity_data: dict | T) -> Optional[T]:
|
||||
"""Update an entity with the given data."""
|
||||
async def update(
|
||||
self,
|
||||
entity_id: int,
|
||||
entity_data: dict | T,
|
||||
*,
|
||||
reload: bool = True,
|
||||
) -> Optional[T]:
|
||||
"""Update an entity with the given data.
|
||||
|
||||
Args:
|
||||
entity_id: Primary key to update
|
||||
entity_data: Column values or a model instance to copy from
|
||||
reload: When True, re-select the entity with repository load options.
|
||||
When False, return the attached row after flush/refresh.
|
||||
"""
|
||||
logger.debug(f"Updating {self.Model.__name__} {entity_id} with data: {entity_data}")
|
||||
async with db.scoped_session(self.session_maker) as session:
|
||||
try:
|
||||
@@ -291,6 +304,8 @@ class Repository[T: Base]:
|
||||
await session.refresh(entity) # Refresh
|
||||
|
||||
logger.debug(f"Updated {self.Model.__name__}: {entity_id}")
|
||||
if not reload:
|
||||
return entity
|
||||
return await self.select_by_id(session, entity.id) # pyright: ignore [reportAttributeAccessIssue]
|
||||
|
||||
except NoResultFound:
|
||||
|
||||
@@ -7,6 +7,7 @@ Composition roots (containers) read ConfigManager and use this module
|
||||
to resolve the runtime mode, then pass the result downstream.
|
||||
"""
|
||||
|
||||
import os
|
||||
from enum import Enum, auto
|
||||
|
||||
|
||||
@@ -44,10 +45,16 @@ def resolve_runtime_mode(
|
||||
Returns:
|
||||
The resolved RuntimeMode
|
||||
"""
|
||||
# Trigger: test environment is detected
|
||||
# Why: tests need special handling (no file sync, isolated DB)
|
||||
# Outcome: returns TEST mode, skipping cloud mode check
|
||||
if is_test_env:
|
||||
return RuntimeMode.TEST
|
||||
|
||||
# Trigger: BASIC_MEMORY_CLOUD_MODE env var is set
|
||||
# Why: cloud deployments must not start local file sync — cloud handles
|
||||
# file storage via S3/Tigris, and the local sync tries to open a
|
||||
# SQLite/Postgres DB that doesn't exist in the cloud container
|
||||
# Outcome: returns CLOUD mode, skipping file sync initialization
|
||||
cloud_mode = os.getenv("BASIC_MEMORY_CLOUD_MODE", "").lower() in ("1", "true")
|
||||
if cloud_mode:
|
||||
return RuntimeMode.CLOUD
|
||||
|
||||
return RuntimeMode.LOCAL
|
||||
|
||||
@@ -178,8 +178,13 @@ ContentType = Annotated[
|
||||
]
|
||||
|
||||
|
||||
RelationType = Annotated[str, MinLen(1), MaxLen(200)]
|
||||
"""Type of relationship between entities. Always use active voice present tense."""
|
||||
RelationType = Annotated[str, MinLen(1)]
|
||||
"""Type of relationship between entities. Always use active voice present tense.
|
||||
|
||||
The database stores relation_type as an unrestricted string, and response models
|
||||
need to tolerate existing long-form values written by LLMs. Keeping an API-only
|
||||
200-character cap here causes reads to fail for valid stored data.
|
||||
"""
|
||||
|
||||
ObservationStr = Annotated[
|
||||
str,
|
||||
|
||||
@@ -1,7 +1,11 @@
|
||||
"""Schemas for cloud-related API responses."""
|
||||
|
||||
from typing import Literal
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
type ProjectVisibility = Literal["workspace", "shared", "private"]
|
||||
|
||||
|
||||
class TenantMountInfo(BaseModel):
|
||||
"""Response from /tenant/mount/info endpoint."""
|
||||
@@ -36,6 +40,10 @@ class CloudProjectCreateRequest(BaseModel):
|
||||
name: str = Field(..., description="Project name")
|
||||
path: str = Field(..., description="Project path (permalink)")
|
||||
set_default: bool = Field(default=False, description="Set as default project")
|
||||
visibility: ProjectVisibility = Field(
|
||||
default="workspace",
|
||||
description="Project visibility for team workspaces",
|
||||
)
|
||||
|
||||
|
||||
class CloudProjectCreateResponse(BaseModel):
|
||||
|
||||
@@ -10,6 +10,7 @@ from typing import List, Optional, Tuple, TYPE_CHECKING
|
||||
from loguru import logger
|
||||
from sqlalchemy import text
|
||||
|
||||
from basic_memory import telemetry
|
||||
from basic_memory.repository.entity_repository import EntityRepository
|
||||
from basic_memory.repository.observation_repository import ObservationRepository
|
||||
from basic_memory.repository.postgres_search_repository import PostgresSearchRepository
|
||||
@@ -110,146 +111,162 @@ class ContextService:
|
||||
f"Building context for URI: '{memory_url}' depth: '{depth}' since: '{since}' limit: '{limit}' offset: '{offset}' max_related: '{max_related}'"
|
||||
)
|
||||
|
||||
# Fetch one extra item to detect whether more pages exist (N+1 trick)
|
||||
fetch_limit = limit + 1
|
||||
with telemetry.scope(
|
||||
"memory.build_context",
|
||||
domain="memory",
|
||||
action="build_context",
|
||||
phase="build_context",
|
||||
limit=limit,
|
||||
offset=offset,
|
||||
):
|
||||
fetch_limit = limit + 1
|
||||
|
||||
normalized_path: Optional[str] = None
|
||||
if memory_url:
|
||||
path = memory_url_path(memory_url)
|
||||
# Check for wildcards before normalization
|
||||
has_wildcard = "*" in path
|
||||
normalized_path: Optional[str] = None
|
||||
with telemetry.scope(
|
||||
"memory.build_context.resolve_primary",
|
||||
domain="memory",
|
||||
action="build_context",
|
||||
phase="resolve_primary",
|
||||
):
|
||||
if memory_url:
|
||||
path = memory_url_path(memory_url)
|
||||
has_wildcard = "*" in path
|
||||
|
||||
if has_wildcard:
|
||||
# For wildcard patterns, normalize each segment separately to preserve the *
|
||||
parts = path.split("*")
|
||||
normalized_parts = [
|
||||
generate_permalink(part, split_extension=False) if part else ""
|
||||
for part in parts
|
||||
]
|
||||
normalized_path = "*".join(normalized_parts)
|
||||
logger.debug(f"Pattern search for '{normalized_path}'")
|
||||
primary = await self.search_repository.search(
|
||||
permalink_match=normalized_path, limit=fetch_limit, offset=offset
|
||||
)
|
||||
else:
|
||||
# For exact paths, normalize the whole thing
|
||||
normalized_path = generate_permalink(path, split_extension=False)
|
||||
logger.debug(f"Direct lookup for '{normalized_path}'")
|
||||
primary = await self.search_repository.search(
|
||||
permalink=normalized_path, limit=fetch_limit, offset=offset
|
||||
)
|
||||
|
||||
# Trigger: exact permalink lookup returned no results
|
||||
# Why: the identifier may be valid but not an exact permalink match
|
||||
# (e.g., missing project prefix, title instead of permalink)
|
||||
# Outcome: use LinkResolver's multi-strategy resolution to find the entity,
|
||||
# then retry search with its actual permalink
|
||||
if not primary and self.link_resolver:
|
||||
entity = await self.link_resolver.resolve_link(
|
||||
path, use_search=True, strict=False
|
||||
)
|
||||
if entity:
|
||||
logger.debug(
|
||||
f"LinkResolver resolved '{path}' to permalink '{entity.permalink}'"
|
||||
)
|
||||
normalized_path = entity.permalink
|
||||
if has_wildcard:
|
||||
parts = path.split("*")
|
||||
normalized_parts = [
|
||||
generate_permalink(part, split_extension=False) if part else ""
|
||||
for part in parts
|
||||
]
|
||||
normalized_path = "*".join(normalized_parts)
|
||||
logger.debug(f"Pattern search for '{normalized_path}'")
|
||||
primary = await self.search_repository.search(
|
||||
permalink=entity.permalink, limit=fetch_limit, offset=offset
|
||||
permalink_match=normalized_path, limit=fetch_limit, offset=offset
|
||||
)
|
||||
else:
|
||||
logger.debug(f"Build context for '{types}'")
|
||||
primary = await self.search_repository.search(
|
||||
search_item_types=types, after_date=since, limit=fetch_limit, offset=offset
|
||||
else:
|
||||
normalized_path = generate_permalink(path, split_extension=False)
|
||||
logger.debug(f"Direct lookup for '{normalized_path}'")
|
||||
primary = await self.search_repository.search(
|
||||
permalink=normalized_path, limit=fetch_limit, offset=offset
|
||||
)
|
||||
|
||||
if not primary and self.link_resolver:
|
||||
entity = await self.link_resolver.resolve_link(
|
||||
path, use_search=True, strict=False
|
||||
)
|
||||
if entity:
|
||||
logger.debug(
|
||||
f"LinkResolver resolved '{path}' to permalink '{entity.permalink}'"
|
||||
)
|
||||
normalized_path = entity.permalink
|
||||
primary = await self.search_repository.search(
|
||||
permalink=entity.permalink,
|
||||
limit=fetch_limit,
|
||||
offset=offset,
|
||||
)
|
||||
else:
|
||||
logger.debug(f"Build context for '{types}'")
|
||||
primary = await self.search_repository.search(
|
||||
search_item_types=types,
|
||||
after_date=since,
|
||||
limit=fetch_limit,
|
||||
offset=offset,
|
||||
)
|
||||
|
||||
has_more = len(primary) > limit
|
||||
if has_more:
|
||||
primary = primary[:limit]
|
||||
|
||||
type_id_pairs = [(r.type, r.id) for r in primary] if primary else []
|
||||
logger.debug(f"found primary type_id_pairs: {len(type_id_pairs)}")
|
||||
|
||||
with telemetry.scope(
|
||||
"memory.build_context.find_related",
|
||||
domain="memory",
|
||||
action="build_context",
|
||||
phase="find_related",
|
||||
):
|
||||
related = await self.find_related(
|
||||
type_id_pairs, max_depth=depth, since=since, max_results=max_related
|
||||
)
|
||||
logger.debug(f"Found {len(related)} related results")
|
||||
|
||||
entity_ids = []
|
||||
for result in primary:
|
||||
if result.type == SearchItemType.ENTITY.value:
|
||||
entity_ids.append(result.id)
|
||||
|
||||
for result in related:
|
||||
if result.type == SearchItemType.ENTITY.value:
|
||||
entity_ids.append(result.id)
|
||||
|
||||
observations_by_entity = {}
|
||||
if include_observations and entity_ids:
|
||||
with telemetry.scope(
|
||||
"memory.build_context.load_observations",
|
||||
domain="memory",
|
||||
action="build_context",
|
||||
phase="load_observations",
|
||||
result_count=len(entity_ids),
|
||||
):
|
||||
observations_by_entity = await self.observation_repository.find_by_entities(
|
||||
entity_ids
|
||||
)
|
||||
logger.debug(f"Found observations for {len(observations_by_entity)} entities")
|
||||
|
||||
metadata = ContextMetadata(
|
||||
uri=normalized_path if memory_url else None,
|
||||
types=types,
|
||||
depth=depth,
|
||||
timeframe=since.isoformat() if since else None,
|
||||
primary_count=len(primary),
|
||||
related_count=len(related),
|
||||
total_observations=sum(len(obs) for obs in observations_by_entity.values()),
|
||||
total_relations=sum(1 for r in related if r.type == SearchItemType.RELATION),
|
||||
has_more=has_more,
|
||||
)
|
||||
|
||||
# Trim to requested limit and set has_more flag
|
||||
has_more = len(primary) > limit
|
||||
if has_more:
|
||||
primary = primary[:limit]
|
||||
with telemetry.scope(
|
||||
"memory.build_context.shape_results",
|
||||
domain="memory",
|
||||
action="build_context",
|
||||
phase="shape_results",
|
||||
result_count=len(primary),
|
||||
):
|
||||
context_results = []
|
||||
for primary_item in primary:
|
||||
related_to_primary = [r for r in related if r.root_id == primary_item.id]
|
||||
|
||||
# Get type_id pairs for traversal
|
||||
item_observations = []
|
||||
if primary_item.type == SearchItemType.ENTITY.value and include_observations:
|
||||
for obs in observations_by_entity.get(primary_item.id, []):
|
||||
item_observations.append(
|
||||
ContextResultRow(
|
||||
type="observation",
|
||||
id=obs.id,
|
||||
title=f"{obs.category}: {obs.content[:50]}...",
|
||||
permalink=generate_permalink(
|
||||
f"{primary_item.permalink}/observations/{obs.category}/{obs.content}"
|
||||
),
|
||||
file_path=primary_item.file_path,
|
||||
content=obs.content,
|
||||
category=obs.category,
|
||||
entity_id=primary_item.id,
|
||||
depth=0,
|
||||
root_id=primary_item.id,
|
||||
created_at=primary_item.created_at,
|
||||
)
|
||||
)
|
||||
|
||||
type_id_pairs = [(r.type, r.id) for r in primary] if primary else []
|
||||
logger.debug(f"found primary type_id_pairs: {len(type_id_pairs)}")
|
||||
|
||||
# Find related content
|
||||
related = await self.find_related(
|
||||
type_id_pairs, max_depth=depth, since=since, max_results=max_related
|
||||
)
|
||||
logger.debug(f"Found {len(related)} related results")
|
||||
|
||||
# Collect entity IDs from primary and related results
|
||||
entity_ids = []
|
||||
for result in primary:
|
||||
if result.type == SearchItemType.ENTITY.value:
|
||||
entity_ids.append(result.id)
|
||||
|
||||
for result in related:
|
||||
if result.type == SearchItemType.ENTITY.value:
|
||||
entity_ids.append(result.id)
|
||||
|
||||
# Fetch observations for all entities if requested
|
||||
observations_by_entity = {}
|
||||
if include_observations and entity_ids:
|
||||
# Use our observation repository to get observations for all entities at once
|
||||
observations_by_entity = await self.observation_repository.find_by_entities(entity_ids)
|
||||
logger.debug(f"Found observations for {len(observations_by_entity)} entities")
|
||||
|
||||
# Create metadata dataclass
|
||||
metadata = ContextMetadata(
|
||||
uri=normalized_path if memory_url else None,
|
||||
types=types,
|
||||
depth=depth,
|
||||
timeframe=since.isoformat() if since else None,
|
||||
primary_count=len(primary),
|
||||
related_count=len(related),
|
||||
total_observations=sum(len(obs) for obs in observations_by_entity.values()),
|
||||
total_relations=sum(1 for r in related if r.type == SearchItemType.RELATION),
|
||||
has_more=has_more,
|
||||
)
|
||||
|
||||
# Build context results list directly with ContextResultItem objects
|
||||
context_results = []
|
||||
|
||||
# For each primary result
|
||||
for primary_item in primary:
|
||||
# Find all related items with this primary item as root
|
||||
related_to_primary = [r for r in related if r.root_id == primary_item.id]
|
||||
|
||||
# Get observations for this item if it's an entity
|
||||
item_observations = []
|
||||
if primary_item.type == SearchItemType.ENTITY.value and include_observations:
|
||||
# Convert Observation models to ContextResultRows
|
||||
for obs in observations_by_entity.get(primary_item.id, []):
|
||||
item_observations.append(
|
||||
ContextResultRow(
|
||||
type="observation",
|
||||
id=obs.id,
|
||||
title=f"{obs.category}: {obs.content[:50]}...",
|
||||
permalink=generate_permalink(
|
||||
f"{primary_item.permalink}/observations/{obs.category}/{obs.content}"
|
||||
),
|
||||
file_path=primary_item.file_path,
|
||||
content=obs.content,
|
||||
category=obs.category,
|
||||
entity_id=primary_item.id,
|
||||
depth=0,
|
||||
root_id=primary_item.id,
|
||||
created_at=primary_item.created_at, # created_at time from entity
|
||||
context_results.append(
|
||||
ContextResultItem(
|
||||
primary_result=primary_item,
|
||||
observations=item_observations,
|
||||
related_results=related_to_primary,
|
||||
)
|
||||
)
|
||||
|
||||
# Create ContextResultItem directly
|
||||
context_item = ContextResultItem(
|
||||
primary_result=primary_item,
|
||||
observations=item_observations,
|
||||
related_results=related_to_primary,
|
||||
)
|
||||
|
||||
context_results.append(context_item)
|
||||
|
||||
# Return the structured ContextResult
|
||||
return ContextResult(results=context_results, metadata=metadata)
|
||||
return ContextResult(results=context_results, metadata=metadata)
|
||||
|
||||
async def find_related(
|
||||
self,
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
"""Service for managing entities in the database."""
|
||||
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import List, Optional, Sequence, Tuple, Union
|
||||
@@ -10,7 +11,7 @@ import yaml
|
||||
from loguru import logger
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
|
||||
|
||||
from basic_memory import telemetry
|
||||
from basic_memory.config import ProjectConfig, BasicMemoryConfig
|
||||
from basic_memory.file_utils import (
|
||||
has_frontmatter,
|
||||
@@ -50,6 +51,15 @@ from basic_memory.services.search_service import SearchService
|
||||
from basic_memory.utils import build_canonical_permalink
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class EntityWriteResult:
|
||||
"""Persisted entity plus the response/search content produced during this call."""
|
||||
|
||||
entity: EntityModel
|
||||
content: str
|
||||
search_content: str
|
||||
|
||||
|
||||
class EntityService(BaseService[EntityModel]):
|
||||
"""Service for managing entities in the database."""
|
||||
|
||||
@@ -79,7 +89,7 @@ class EntityService(BaseService[EntityModel]):
|
||||
|
||||
async def detect_file_path_conflicts(
|
||||
self, file_path: str, skip_check: bool = False
|
||||
) -> List[Entity]:
|
||||
) -> List[str]:
|
||||
"""Detect potential file path conflicts for a given file path.
|
||||
|
||||
This checks for entities with similar file paths that might cause conflicts:
|
||||
@@ -93,28 +103,19 @@ class EntityService(BaseService[EntityModel]):
|
||||
skip_check: If True, skip the check and return empty list (optimization for bulk operations)
|
||||
|
||||
Returns:
|
||||
List of entities that might conflict with the given file path
|
||||
List of file paths that might conflict with the given file path
|
||||
"""
|
||||
if skip_check:
|
||||
return []
|
||||
|
||||
from basic_memory.utils import detect_potential_file_conflicts
|
||||
|
||||
conflicts = []
|
||||
|
||||
# Get all existing file paths
|
||||
all_entities = await self.repository.find_all()
|
||||
existing_paths = [entity.file_path for entity in all_entities]
|
||||
# Load only file paths. Conflict detection is on the hot write path and
|
||||
# does not need observations or relations.
|
||||
existing_paths = await self.repository.get_all_file_paths()
|
||||
|
||||
# Use the enhanced conflict detection utility
|
||||
conflicting_paths = detect_potential_file_conflicts(file_path, existing_paths)
|
||||
|
||||
# Find the entities corresponding to conflicting paths
|
||||
for entity in all_entities:
|
||||
if entity.file_path in conflicting_paths:
|
||||
conflicts.append(entity)
|
||||
|
||||
return conflicts
|
||||
return detect_potential_file_conflicts(file_path, existing_paths)
|
||||
|
||||
async def resolve_permalink(
|
||||
self,
|
||||
@@ -143,8 +144,7 @@ class EntityService(BaseService[EntityModel]):
|
||||
)
|
||||
if conflicts:
|
||||
logger.warning(
|
||||
f"Detected potential file path conflicts for '{file_path_str}': "
|
||||
f"{[entity.file_path for entity in conflicts]}"
|
||||
f"Detected potential file path conflicts for '{file_path_str}': {conflicts}"
|
||||
)
|
||||
|
||||
# If markdown has explicit permalink, try to validate it
|
||||
@@ -242,9 +242,17 @@ class EntityService(BaseService[EntityModel]):
|
||||
|
||||
# Try to find existing entity using strict resolution (no fuzzy search)
|
||||
# This prevents incorrectly matching similar file paths like "Node A.md" and "Node C.md"
|
||||
existing = await self.link_resolver.resolve_link(schema.file_path, strict=True)
|
||||
existing = await self.link_resolver.resolve_link(
|
||||
schema.file_path,
|
||||
strict=True,
|
||||
load_relations=False,
|
||||
)
|
||||
if not existing and schema.permalink:
|
||||
existing = await self.link_resolver.resolve_link(schema.permalink, strict=True)
|
||||
existing = await self.link_resolver.resolve_link(
|
||||
schema.permalink,
|
||||
strict=True,
|
||||
load_relations=False,
|
||||
)
|
||||
|
||||
if existing:
|
||||
logger.debug(f"Found existing entity: {existing.file_path}")
|
||||
@@ -255,6 +263,10 @@ class EntityService(BaseService[EntityModel]):
|
||||
|
||||
async def create_entity(self, schema: EntitySchema) -> EntityModel:
|
||||
"""Create a new entity and write to filesystem."""
|
||||
return (await self.create_entity_with_content(schema)).entity
|
||||
|
||||
async def create_entity_with_content(self, schema: EntitySchema) -> EntityWriteResult:
|
||||
"""Create a new entity and return both the entity row and written markdown."""
|
||||
logger.debug(f"Creating entity: {schema.title}")
|
||||
|
||||
# Get file path and ensure it's a Path object
|
||||
@@ -281,34 +293,67 @@ class EntityService(BaseService[EntityModel]):
|
||||
|
||||
# Get unique permalink (prioritizing content frontmatter) unless disabled
|
||||
if self.app_config and self.app_config.disable_permalinks:
|
||||
# Use empty string as sentinel to indicate permalinks are disabled
|
||||
# The permalink property will return None when it sees empty string
|
||||
schema._permalink = ""
|
||||
else:
|
||||
# Generate and set permalink
|
||||
permalink = await self.resolve_permalink(file_path, content_markdown)
|
||||
with telemetry.scope(
|
||||
"entity_service.create.resolve_permalink",
|
||||
domain="entity_service",
|
||||
action="create",
|
||||
phase="resolve_permalink",
|
||||
):
|
||||
permalink = await self.resolve_permalink(file_path, content_markdown)
|
||||
schema._permalink = permalink
|
||||
|
||||
post = await schema_to_markdown(schema)
|
||||
|
||||
# write file
|
||||
final_content = dump_frontmatter(post)
|
||||
checksum = await self.file_service.write_file(file_path, final_content)
|
||||
with telemetry.scope(
|
||||
"entity_service.create.write_file",
|
||||
domain="entity_service",
|
||||
action="create",
|
||||
phase="write_file",
|
||||
):
|
||||
checksum = await self.file_service.write_file(file_path, final_content)
|
||||
|
||||
# parse entity from content we just wrote (avoids re-reading file for cloud compatibility)
|
||||
entity_markdown = await self.entity_parser.parse_markdown_content(
|
||||
file_path=file_path,
|
||||
with telemetry.scope(
|
||||
"entity_service.create.parse_markdown",
|
||||
domain="entity_service",
|
||||
action="create",
|
||||
phase="parse_markdown",
|
||||
):
|
||||
entity_markdown = await self.entity_parser.parse_markdown_content(
|
||||
file_path=file_path,
|
||||
content=final_content,
|
||||
)
|
||||
|
||||
with telemetry.scope(
|
||||
"entity_service.create.upsert_entity",
|
||||
domain="entity_service",
|
||||
action="create",
|
||||
phase="upsert_entity",
|
||||
):
|
||||
updated = await self.upsert_entity_from_markdown(
|
||||
file_path,
|
||||
entity_markdown,
|
||||
is_new=True,
|
||||
checksum=checksum,
|
||||
)
|
||||
if not updated: # pragma: no cover
|
||||
raise ValueError(f"Failed to persist entity after create: {file_path}")
|
||||
return EntityWriteResult(
|
||||
entity=updated,
|
||||
content=final_content,
|
||||
search_content=remove_frontmatter(final_content),
|
||||
)
|
||||
|
||||
# create entity and relations
|
||||
entity = await self.upsert_entity_from_markdown(file_path, entity_markdown, is_new=True)
|
||||
|
||||
# Set final checksum to mark complete
|
||||
return await self.repository.update(entity.id, {"checksum": checksum})
|
||||
|
||||
async def update_entity(self, entity: EntityModel, schema: EntitySchema) -> EntityModel:
|
||||
"""Update an entity's content and metadata."""
|
||||
return (await self.update_entity_with_content(entity, schema)).entity
|
||||
|
||||
async def update_entity_with_content(
|
||||
self, entity: EntityModel, schema: EntitySchema
|
||||
) -> EntityWriteResult:
|
||||
"""Update an entity and return both the entity row and written markdown."""
|
||||
logger.debug(
|
||||
f"Updating entity with permalink: {entity.permalink} content-type: {schema.content_type}"
|
||||
)
|
||||
@@ -316,12 +361,23 @@ class EntityService(BaseService[EntityModel]):
|
||||
# Convert file path string to Path
|
||||
file_path = Path(entity.file_path)
|
||||
|
||||
# Read existing content via file_service (for cloud compatibility)
|
||||
existing_content = await self.file_service.read_file_content(file_path)
|
||||
existing_markdown = await self.entity_parser.parse_markdown_content(
|
||||
file_path=file_path,
|
||||
content=existing_content,
|
||||
)
|
||||
with telemetry.scope(
|
||||
"entity_service.update.read_file",
|
||||
domain="entity_service",
|
||||
action="update",
|
||||
phase="read_file",
|
||||
):
|
||||
existing_content = await self.file_service.read_file_content(file_path)
|
||||
with telemetry.scope(
|
||||
"entity_service.update.parse_markdown",
|
||||
domain="entity_service",
|
||||
action="update",
|
||||
phase="parse_markdown",
|
||||
):
|
||||
existing_markdown = await self.entity_parser.parse_markdown_content(
|
||||
file_path=file_path,
|
||||
content=existing_content,
|
||||
)
|
||||
|
||||
# Parse content frontmatter to check for user-specified permalink and note_type
|
||||
content_markdown = None
|
||||
@@ -342,7 +398,13 @@ class EntityService(BaseService[EntityModel]):
|
||||
if self.app_config and not self.app_config.disable_permalinks:
|
||||
if content_markdown and content_markdown.frontmatter.permalink:
|
||||
# Resolve permalink with the new content frontmatter
|
||||
resolved_permalink = await self.resolve_permalink(file_path, content_markdown)
|
||||
with telemetry.scope(
|
||||
"entity_service.update.resolve_permalink",
|
||||
domain="entity_service",
|
||||
action="update",
|
||||
phase="resolve_permalink",
|
||||
):
|
||||
resolved_permalink = await self.resolve_permalink(file_path, content_markdown)
|
||||
if resolved_permalink != entity.permalink:
|
||||
new_permalink = resolved_permalink
|
||||
# Update the schema to use the new permalink
|
||||
@@ -367,24 +429,47 @@ class EntityService(BaseService[EntityModel]):
|
||||
merged_post = frontmatter.Post(post.content)
|
||||
merged_post.metadata.update(existing_markdown.frontmatter.metadata)
|
||||
|
||||
# write file
|
||||
final_content = dump_frontmatter(merged_post)
|
||||
checksum = await self.file_service.write_file(file_path, final_content)
|
||||
with telemetry.scope(
|
||||
"entity_service.update.write_file",
|
||||
domain="entity_service",
|
||||
action="update",
|
||||
phase="write_file",
|
||||
):
|
||||
checksum = await self.file_service.write_file(file_path, final_content)
|
||||
|
||||
# parse entity from content we just wrote (avoids re-reading file for cloud compatibility)
|
||||
entity_markdown = await self.entity_parser.parse_markdown_content(
|
||||
file_path=file_path,
|
||||
with telemetry.scope(
|
||||
"entity_service.update.parse_markdown",
|
||||
domain="entity_service",
|
||||
action="update",
|
||||
phase="parse_markdown",
|
||||
):
|
||||
entity_markdown = await self.entity_parser.parse_markdown_content(
|
||||
file_path=file_path,
|
||||
content=final_content,
|
||||
)
|
||||
|
||||
with telemetry.scope(
|
||||
"entity_service.update.upsert_entity",
|
||||
domain="entity_service",
|
||||
action="update",
|
||||
phase="upsert_entity",
|
||||
):
|
||||
entity = await self.upsert_entity_from_markdown(
|
||||
file_path,
|
||||
entity_markdown,
|
||||
is_new=False,
|
||||
checksum=checksum,
|
||||
)
|
||||
if not entity: # pragma: no cover
|
||||
raise ValueError(f"Failed to persist entity after update: {file_path}")
|
||||
|
||||
return EntityWriteResult(
|
||||
entity=entity,
|
||||
content=final_content,
|
||||
search_content=remove_frontmatter(final_content),
|
||||
)
|
||||
|
||||
# update entity and relations
|
||||
entity = await self.upsert_entity_from_markdown(file_path, entity_markdown, is_new=False)
|
||||
|
||||
# Set final checksum to match file
|
||||
entity = await self.repository.update(entity.id, {"checksum": checksum})
|
||||
|
||||
return entity
|
||||
|
||||
async def fast_write_entity(
|
||||
self,
|
||||
schema: EntitySchema,
|
||||
@@ -399,7 +484,15 @@ class EntityService(BaseService[EntityModel]):
|
||||
)
|
||||
|
||||
# --- Identity & File Path ---
|
||||
existing = await self.repository.get_by_external_id(external_id) if external_id else None
|
||||
with telemetry.scope(
|
||||
"entity_service.fast_write.resolve_entity",
|
||||
domain="entity_service",
|
||||
action="fast_write",
|
||||
phase="resolve_entity",
|
||||
):
|
||||
existing = (
|
||||
await self.repository.get_by_external_id(external_id) if external_id else None
|
||||
)
|
||||
|
||||
# Trigger: external_id already exists
|
||||
# Why: avoid duplicate entities when title-derived paths change
|
||||
@@ -429,18 +522,35 @@ class EntityService(BaseService[EntityModel]):
|
||||
schema._permalink = ""
|
||||
else:
|
||||
if existing and not (content_markdown and content_markdown.frontmatter.permalink):
|
||||
schema._permalink = existing.permalink or await self.resolve_permalink(
|
||||
file_path, skip_conflict_check=True
|
||||
)
|
||||
with telemetry.scope(
|
||||
"entity_service.fast_write.resolve_permalink",
|
||||
domain="entity_service",
|
||||
action="fast_write",
|
||||
phase="resolve_permalink",
|
||||
):
|
||||
schema._permalink = existing.permalink or await self.resolve_permalink(
|
||||
file_path, skip_conflict_check=True
|
||||
)
|
||||
else:
|
||||
schema._permalink = await self.resolve_permalink(
|
||||
file_path, content_markdown, skip_conflict_check=True
|
||||
)
|
||||
with telemetry.scope(
|
||||
"entity_service.fast_write.resolve_permalink",
|
||||
domain="entity_service",
|
||||
action="fast_write",
|
||||
phase="resolve_permalink",
|
||||
):
|
||||
schema._permalink = await self.resolve_permalink(
|
||||
file_path, content_markdown, skip_conflict_check=True
|
||||
)
|
||||
|
||||
# --- File Write ---
|
||||
post = await schema_to_markdown(schema)
|
||||
final_content = dump_frontmatter(post)
|
||||
checksum = await self.file_service.write_file(file_path, final_content)
|
||||
with telemetry.scope(
|
||||
"entity_service.fast_write.write_file",
|
||||
domain="entity_service",
|
||||
action="fast_write",
|
||||
phase="write_file",
|
||||
):
|
||||
checksum = await self.file_service.write_file(file_path, final_content)
|
||||
|
||||
# --- Minimal DB Upsert ---
|
||||
metadata = normalize_frontmatter_metadata(post.metadata or {})
|
||||
@@ -462,7 +572,13 @@ class EntityService(BaseService[EntityModel]):
|
||||
# Preserve existing created_by; only update last_updated_by
|
||||
if user_id is not None:
|
||||
update_data["last_updated_by"] = user_id
|
||||
updated = await self.repository.update(existing.id, update_data)
|
||||
with telemetry.scope(
|
||||
"entity_service.fast_write.upsert_entity",
|
||||
domain="entity_service",
|
||||
action="fast_write",
|
||||
phase="upsert_entity",
|
||||
):
|
||||
updated = await self.repository.update(existing.id, update_data)
|
||||
if not updated:
|
||||
raise ValueError(f"Failed to update entity in database: {existing.id}")
|
||||
return updated
|
||||
@@ -473,7 +589,13 @@ class EntityService(BaseService[EntityModel]):
|
||||
if user_id is not None:
|
||||
create_data["created_by"] = user_id
|
||||
create_data["last_updated_by"] = user_id
|
||||
return await self.repository.create(create_data)
|
||||
with telemetry.scope(
|
||||
"entity_service.fast_write.upsert_entity",
|
||||
domain="entity_service",
|
||||
action="fast_write",
|
||||
phase="upsert_entity",
|
||||
):
|
||||
return await self.repository.create(create_data)
|
||||
|
||||
async def fast_edit_entity(
|
||||
self,
|
||||
@@ -487,13 +609,30 @@ class EntityService(BaseService[EntityModel]):
|
||||
"""Edit an entity quickly and defer full indexing to background."""
|
||||
logger.debug(f"Fast editing entity: {entity.external_id}, operation: {operation}")
|
||||
|
||||
# --- File Edit ---
|
||||
file_path = Path(entity.file_path)
|
||||
current_content, _ = await self.file_service.read_file(file_path)
|
||||
new_content = self.apply_edit_operation(
|
||||
current_content, operation, content, section, find_text, expected_replacements
|
||||
)
|
||||
checksum = await self.file_service.write_file(file_path, new_content)
|
||||
with telemetry.scope(
|
||||
"entity_service.fast_edit.read_file",
|
||||
domain="entity_service",
|
||||
action="fast_edit",
|
||||
phase="read_file",
|
||||
):
|
||||
current_content, _ = await self.file_service.read_file(file_path)
|
||||
with telemetry.scope(
|
||||
"entity_service.fast_edit.apply_operation",
|
||||
domain="entity_service",
|
||||
action="fast_edit",
|
||||
phase="apply_operation",
|
||||
):
|
||||
new_content = self.apply_edit_operation(
|
||||
current_content, operation, content, section, find_text, expected_replacements
|
||||
)
|
||||
with telemetry.scope(
|
||||
"entity_service.fast_edit.write_file",
|
||||
domain="entity_service",
|
||||
action="fast_edit",
|
||||
phase="write_file",
|
||||
):
|
||||
checksum = await self.file_service.write_file(file_path, new_content)
|
||||
|
||||
# --- Frontmatter Overrides ---
|
||||
update_data = {
|
||||
@@ -528,39 +667,86 @@ class EntityService(BaseService[EntityModel]):
|
||||
if self.app_config and self.app_config.disable_permalinks:
|
||||
update_data["permalink"] = None
|
||||
elif content_markdown and content_markdown.frontmatter.permalink:
|
||||
update_data["permalink"] = await self.resolve_permalink(
|
||||
file_path, content_markdown, skip_conflict_check=True
|
||||
)
|
||||
with telemetry.scope(
|
||||
"entity_service.fast_edit.resolve_permalink",
|
||||
domain="entity_service",
|
||||
action="fast_edit",
|
||||
phase="resolve_permalink",
|
||||
):
|
||||
update_data["permalink"] = await self.resolve_permalink(
|
||||
file_path, content_markdown, skip_conflict_check=True
|
||||
)
|
||||
|
||||
updated = await self.repository.update(entity.id, update_data)
|
||||
with telemetry.scope(
|
||||
"entity_service.fast_edit.update_entity",
|
||||
domain="entity_service",
|
||||
action="fast_edit",
|
||||
phase="update_entity",
|
||||
):
|
||||
updated = await self.repository.update(entity.id, update_data)
|
||||
if not updated:
|
||||
raise ValueError(f"Failed to update entity in database: {entity.id}")
|
||||
return updated
|
||||
|
||||
async def reindex_entity(self, entity_id: int) -> None:
|
||||
"""Parse file content and rebuild observations/relations/search for an entity."""
|
||||
entity = await self.repository.find_by_id(entity_id)
|
||||
with telemetry.scope(
|
||||
"entity_service.reindex.load_entity",
|
||||
domain="entity_service",
|
||||
action="reindex",
|
||||
phase="load_entity",
|
||||
):
|
||||
entity = await self.repository.find_by_id(entity_id)
|
||||
if not entity:
|
||||
raise EntityNotFoundError(f"Entity not found: {entity_id}")
|
||||
|
||||
# --- Full Parse ---
|
||||
file_path = Path(entity.file_path)
|
||||
content = await self.file_service.read_file_content(file_path)
|
||||
entity_markdown = await self.entity_parser.parse_markdown_content(
|
||||
file_path=file_path,
|
||||
content=content,
|
||||
)
|
||||
with telemetry.scope(
|
||||
"entity_service.reindex.read_file",
|
||||
domain="entity_service",
|
||||
action="reindex",
|
||||
phase="read_file",
|
||||
):
|
||||
content = await self.file_service.read_file_content(file_path)
|
||||
with telemetry.scope(
|
||||
"entity_service.reindex.parse_markdown",
|
||||
domain="entity_service",
|
||||
action="reindex",
|
||||
phase="parse_markdown",
|
||||
):
|
||||
entity_markdown = await self.entity_parser.parse_markdown_content(
|
||||
file_path=file_path,
|
||||
content=content,
|
||||
)
|
||||
|
||||
# --- DB Reindex ---
|
||||
updated = await self.upsert_entity_from_markdown(file_path, entity_markdown, is_new=False)
|
||||
checksum = await self.file_service.compute_checksum(file_path)
|
||||
updated = await self.repository.update(updated.id, {"checksum": checksum})
|
||||
with telemetry.scope(
|
||||
"entity_service.reindex.upsert_entity",
|
||||
domain="entity_service",
|
||||
action="reindex",
|
||||
phase="upsert_entity",
|
||||
):
|
||||
updated = await self.upsert_entity_from_markdown(
|
||||
file_path, entity_markdown, is_new=False
|
||||
)
|
||||
with telemetry.scope(
|
||||
"entity_service.reindex.update_checksum",
|
||||
domain="entity_service",
|
||||
action="reindex",
|
||||
phase="update_checksum",
|
||||
):
|
||||
checksum = await self.file_service.compute_checksum(file_path)
|
||||
updated = await self.repository.update(updated.id, {"checksum": checksum})
|
||||
if not updated:
|
||||
raise ValueError(f"Failed to update entity in database: {entity.id}")
|
||||
|
||||
# --- Search Reindex ---
|
||||
if self.search_service:
|
||||
await self.search_service.index_entity_data(updated, content=content)
|
||||
with telemetry.scope(
|
||||
"entity_service.reindex.search_index",
|
||||
domain="entity_service",
|
||||
action="reindex",
|
||||
phase="search_index",
|
||||
):
|
||||
await self.search_service.index_entity_data(updated, content=content)
|
||||
|
||||
async def delete_entity(self, permalink_or_id: str | int) -> bool:
|
||||
"""Delete entity and its file."""
|
||||
@@ -572,6 +758,10 @@ class EntityService(BaseService[EntityModel]):
|
||||
entity = await self.get_by_permalink(permalink_or_id)
|
||||
else:
|
||||
entities = await self.get_entities_by_id([permalink_or_id])
|
||||
if len(entities) == 0:
|
||||
# Entity already deleted (concurrent delete or race condition)
|
||||
logger.info("Entity already deleted", entity_id=permalink_or_id)
|
||||
return True
|
||||
if len(entities) != 1: # pragma: no cover
|
||||
logger.error(
|
||||
"Entity lookup error", entity_id=permalink_or_id, found_count=len(entities)
|
||||
@@ -583,13 +773,28 @@ class EntityService(BaseService[EntityModel]):
|
||||
|
||||
# Delete from search index first (if search_service is available)
|
||||
if self.search_service:
|
||||
await self.search_service.handle_delete(entity)
|
||||
try:
|
||||
await self.search_service.handle_delete(entity)
|
||||
except Exception:
|
||||
# Search cleanup is best-effort during concurrent deletes.
|
||||
# Relationships may have been cascade-deleted by a concurrent request.
|
||||
logger.warning(
|
||||
"Search cleanup failed for entity (likely concurrent delete)",
|
||||
permalink_or_id=permalink_or_id,
|
||||
exc_info=True,
|
||||
)
|
||||
|
||||
# Delete file
|
||||
await self.file_service.delete_entity_file(entity)
|
||||
|
||||
# Delete from DB (this will cascade to observations/relations)
|
||||
return await self.repository.delete(entity.id)
|
||||
# Trigger: repository.delete returns False when entity is already gone (NoResultFound)
|
||||
# Why: concurrent delete_directory requests can race to delete the same entity
|
||||
# Outcome: treat as success since the entity is deleted either way
|
||||
deleted = await self.repository.delete(entity.id)
|
||||
if not deleted:
|
||||
logger.info("Entity already removed from DB", entity_id=permalink_or_id)
|
||||
return True
|
||||
|
||||
except EntityNotFoundError:
|
||||
logger.info(f"Entity not found: {permalink_or_id}")
|
||||
@@ -643,7 +848,7 @@ class EntityService(BaseService[EntityModel]):
|
||||
|
||||
# Use UPSERT to handle conflicts cleanly
|
||||
try:
|
||||
return await self.repository.upsert_entity(model)
|
||||
return await self.repository.upsert_entity(model, reload=False)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to upsert entity for {file_path}: {e}")
|
||||
raise EntityCreationError(f"Failed to create entity: {str(e)}") from e
|
||||
@@ -658,7 +863,12 @@ class EntityService(BaseService[EntityModel]):
|
||||
"""
|
||||
logger.debug(f"Updating entity and observations: {file_path}")
|
||||
|
||||
db_entity = await self.repository.get_by_file_path(file_path.as_posix())
|
||||
db_entity = await self.repository.get_by_file_path(
|
||||
file_path.as_posix(),
|
||||
load_relations=False,
|
||||
)
|
||||
if not db_entity: # pragma: no cover
|
||||
raise EntityNotFoundError(f"Entity not found: {file_path}")
|
||||
|
||||
# Clear observations for entity
|
||||
await self.observation_repository.delete_by_fields(entity_id=db_entity.id)
|
||||
@@ -675,23 +885,37 @@ class EntityService(BaseService[EntityModel]):
|
||||
)
|
||||
for obs in markdown.observations
|
||||
]
|
||||
await self.observation_repository.add_all(observations)
|
||||
if observations:
|
||||
await self.observation_repository.add_all(observations)
|
||||
|
||||
# update values from markdown
|
||||
db_entity = entity_model_from_markdown(file_path, markdown, db_entity)
|
||||
# Trigger: the lightweight lookup above returns a detached row without loaded collections
|
||||
# Why: assigning a new observation list onto that detached ORM object would trigger lazy loads
|
||||
# Outcome: rebuild a fresh model from markdown, then copy over stable identity fields
|
||||
db_entity_data = entity_model_from_markdown(
|
||||
file_path,
|
||||
markdown,
|
||||
project_id=self.repository.project_id,
|
||||
)
|
||||
db_entity_data.id = db_entity.id
|
||||
db_entity_data.project_id = db_entity.project_id
|
||||
db_entity_data.external_id = db_entity.external_id
|
||||
db_entity_data.created_by = db_entity.created_by
|
||||
|
||||
# checksum value is None == not finished with sync
|
||||
db_entity.checksum = None
|
||||
db_entity_data.checksum = None
|
||||
|
||||
# Set last_updated_by for cloud usage (preserve existing created_by)
|
||||
user_id = self.get_user_id()
|
||||
if user_id is not None:
|
||||
db_entity.last_updated_by = user_id
|
||||
db_entity_data.last_updated_by = user_id
|
||||
else:
|
||||
db_entity_data.last_updated_by = db_entity.last_updated_by
|
||||
|
||||
# update entity
|
||||
return await self.repository.update(
|
||||
db_entity.id,
|
||||
db_entity,
|
||||
db_entity_data,
|
||||
reload=False,
|
||||
)
|
||||
|
||||
async def upsert_entity_from_markdown(
|
||||
@@ -700,26 +924,76 @@ class EntityService(BaseService[EntityModel]):
|
||||
markdown: EntityMarkdown,
|
||||
*,
|
||||
is_new: bool,
|
||||
checksum: Optional[str] = None,
|
||||
) -> EntityModel:
|
||||
"""Create/update entity and relations from parsed markdown."""
|
||||
if is_new:
|
||||
created = await self.create_entity_from_markdown(file_path, markdown)
|
||||
else:
|
||||
created = await self.update_entity_and_observations(file_path, markdown)
|
||||
return await self.update_entity_relations(created.file_path, markdown)
|
||||
# --- Base Entity Row ---
|
||||
# Trigger: writes rebuild the entity row before touching relation edges
|
||||
# Why: relations need a stable source entity ID, but not a fully hydrated graph
|
||||
# Outcome: create/update the row with a lightweight return value
|
||||
with telemetry.scope(
|
||||
"entity_service.upsert.base_entity",
|
||||
domain="entity_service",
|
||||
action="upsert",
|
||||
phase="base_entity",
|
||||
):
|
||||
if is_new:
|
||||
created = await self.create_entity_from_markdown(file_path, markdown)
|
||||
else:
|
||||
created = await self.update_entity_and_observations(file_path, markdown)
|
||||
|
||||
# --- Relation Edges ---
|
||||
with telemetry.scope(
|
||||
"entity_service.upsert.relations",
|
||||
domain="entity_service",
|
||||
action="upsert",
|
||||
phase="relations",
|
||||
):
|
||||
await self.update_entity_relations(created, markdown)
|
||||
|
||||
# --- Final Entity State ---
|
||||
# Trigger: create/update/edit already computed the final file checksum
|
||||
# Why: fold the checksum write into the upsert flow so callers do one hydrated read
|
||||
# Outcome: the write path returns the final entity state without an extra checksum step
|
||||
if checksum is not None:
|
||||
with telemetry.scope(
|
||||
"entity_service.upsert.persist_checksum",
|
||||
domain="entity_service",
|
||||
action="upsert",
|
||||
phase="persist_checksum",
|
||||
):
|
||||
updated = await self.repository.update(created.id, {"checksum": checksum})
|
||||
if not updated: # pragma: no cover
|
||||
raise ValueError(f"Failed to update entity checksum after upsert: {file_path}")
|
||||
return updated
|
||||
|
||||
with telemetry.scope(
|
||||
"entity_service.upsert.hydrate_entity",
|
||||
domain="entity_service",
|
||||
action="upsert",
|
||||
phase="hydrate_entity",
|
||||
):
|
||||
hydrated = await self.repository.get_by_file_path(created.file_path)
|
||||
if not hydrated: # pragma: no cover
|
||||
raise EntityNotFoundError(f"Entity not found after upsert: {created.file_path}")
|
||||
return hydrated
|
||||
|
||||
async def update_entity_relations(
|
||||
self,
|
||||
path: str,
|
||||
db_entity: EntityModel,
|
||||
markdown: EntityMarkdown,
|
||||
) -> EntityModel:
|
||||
) -> None:
|
||||
"""Update relations for entity"""
|
||||
logger.debug(f"Updating relations for entity: {path}")
|
||||
|
||||
db_entity = await self.repository.get_by_file_path(path)
|
||||
logger.debug(f"Updating relations for entity: {db_entity.file_path}")
|
||||
|
||||
# Clear existing relations first
|
||||
await self.relation_repository.delete_outgoing_relations_from_entity(db_entity.id)
|
||||
with telemetry.scope(
|
||||
"entity_service.upsert.delete_relations",
|
||||
domain="entity_service",
|
||||
action="upsert",
|
||||
phase="delete_relations",
|
||||
):
|
||||
await self.relation_repository.delete_outgoing_relations_from_entity(db_entity.id)
|
||||
|
||||
# Batch resolve all relation targets in parallel
|
||||
if markdown.relations:
|
||||
@@ -728,13 +1002,23 @@ class EntityService(BaseService[EntityModel]):
|
||||
# Create tasks for all relation lookups
|
||||
# Use strict=True to disable fuzzy search - only exact matches should create resolved relations
|
||||
# This ensures forward references (links to non-existent entities) remain unresolved (to_id=NULL)
|
||||
lookup_tasks = [
|
||||
self.link_resolver.resolve_link(rel.target, strict=True)
|
||||
for rel in markdown.relations
|
||||
]
|
||||
with telemetry.scope(
|
||||
"entity_service.upsert.resolve_relation_targets",
|
||||
domain="entity_service",
|
||||
action="upsert",
|
||||
phase="resolve_relation_targets",
|
||||
):
|
||||
lookup_tasks = [
|
||||
self.link_resolver.resolve_link(
|
||||
rel.target,
|
||||
strict=True,
|
||||
load_relations=False,
|
||||
)
|
||||
for rel in markdown.relations
|
||||
]
|
||||
|
||||
# Execute all lookups in parallel
|
||||
resolved_entities = await asyncio.gather(*lookup_tasks, return_exceptions=True)
|
||||
# Execute all lookups in parallel
|
||||
resolved_entities = await asyncio.gather(*lookup_tasks, return_exceptions=True)
|
||||
|
||||
# Process results and create relation records
|
||||
relations_to_add = []
|
||||
@@ -763,22 +1047,26 @@ class EntityService(BaseService[EntityModel]):
|
||||
|
||||
# Batch insert all relations
|
||||
if relations_to_add:
|
||||
try:
|
||||
await self.relation_repository.add_all(relations_to_add)
|
||||
except IntegrityError:
|
||||
# Some relations might be duplicates - fall back to individual inserts
|
||||
logger.debug("Batch relation insert failed, trying individual inserts")
|
||||
for relation in relations_to_add:
|
||||
try:
|
||||
await self.relation_repository.add(relation)
|
||||
except IntegrityError:
|
||||
# Unique constraint violation - relation already exists
|
||||
logger.debug(
|
||||
f"Skipping duplicate relation {relation.relation_type} from {db_entity.permalink}"
|
||||
)
|
||||
continue
|
||||
|
||||
return await self.repository.get_by_file_path(path)
|
||||
with telemetry.scope(
|
||||
"entity_service.upsert.insert_relations",
|
||||
domain="entity_service",
|
||||
action="upsert",
|
||||
phase="insert_relations",
|
||||
):
|
||||
try:
|
||||
await self.relation_repository.add_all(relations_to_add)
|
||||
except IntegrityError:
|
||||
# Some relations might be duplicates - fall back to individual inserts
|
||||
logger.debug("Batch relation insert failed, trying individual inserts")
|
||||
for relation in relations_to_add:
|
||||
try:
|
||||
await self.relation_repository.add(relation)
|
||||
except IntegrityError:
|
||||
# Unique constraint violation - relation already exists
|
||||
logger.debug(
|
||||
f"Skipping duplicate relation {relation.relation_type} from {db_entity.permalink}"
|
||||
)
|
||||
continue
|
||||
|
||||
async def edit_entity(
|
||||
self,
|
||||
@@ -806,39 +1094,102 @@ class EntityService(BaseService[EntityModel]):
|
||||
EntityNotFoundError: If the entity cannot be found
|
||||
ValueError: If required parameters are missing for the operation or replacement count doesn't match expected
|
||||
"""
|
||||
return (
|
||||
await self.edit_entity_with_content(
|
||||
identifier=identifier,
|
||||
operation=operation,
|
||||
content=content,
|
||||
section=section,
|
||||
find_text=find_text,
|
||||
expected_replacements=expected_replacements,
|
||||
)
|
||||
).entity
|
||||
|
||||
async def edit_entity_with_content(
|
||||
self,
|
||||
identifier: str,
|
||||
operation: str,
|
||||
content: str,
|
||||
section: Optional[str] = None,
|
||||
find_text: Optional[str] = None,
|
||||
expected_replacements: int = 1,
|
||||
) -> EntityWriteResult:
|
||||
"""Edit an entity and return both the entity row and written markdown."""
|
||||
logger.debug(f"Editing entity: {identifier}, operation: {operation}")
|
||||
|
||||
# Find the entity using the link resolver with strict mode for destructive operations
|
||||
entity = await self.link_resolver.resolve_link(identifier, strict=True)
|
||||
with telemetry.scope(
|
||||
"entity_service.edit.resolve_entity",
|
||||
domain="entity_service",
|
||||
action="edit",
|
||||
phase="resolve_entity",
|
||||
):
|
||||
entity = await self.link_resolver.resolve_link(
|
||||
identifier,
|
||||
strict=True,
|
||||
load_relations=False,
|
||||
)
|
||||
if not entity:
|
||||
raise EntityNotFoundError(f"Entity not found: {identifier}")
|
||||
|
||||
# Read the current file content
|
||||
file_path = Path(entity.file_path)
|
||||
current_content, _ = await self.file_service.read_file(file_path)
|
||||
with telemetry.scope(
|
||||
"entity_service.edit.read_file",
|
||||
domain="entity_service",
|
||||
action="edit",
|
||||
phase="read_file",
|
||||
):
|
||||
current_content, _ = await self.file_service.read_file(file_path)
|
||||
|
||||
# Apply the edit operation
|
||||
new_content = self.apply_edit_operation(
|
||||
current_content, operation, content, section, find_text, expected_replacements
|
||||
)
|
||||
with telemetry.scope(
|
||||
"entity_service.edit.apply_operation",
|
||||
domain="entity_service",
|
||||
action="edit",
|
||||
phase="apply_operation",
|
||||
):
|
||||
new_content = self.apply_edit_operation(
|
||||
current_content, operation, content, section, find_text, expected_replacements
|
||||
)
|
||||
|
||||
# Write the updated content back to the file
|
||||
checksum = await self.file_service.write_file(file_path, new_content)
|
||||
with telemetry.scope(
|
||||
"entity_service.edit.write_file",
|
||||
domain="entity_service",
|
||||
action="edit",
|
||||
phase="write_file",
|
||||
):
|
||||
checksum = await self.file_service.write_file(file_path, new_content)
|
||||
|
||||
# Parse the content we just wrote (avoids re-reading file for cloud compatibility)
|
||||
entity_markdown = await self.entity_parser.parse_markdown_content(
|
||||
file_path=file_path,
|
||||
with telemetry.scope(
|
||||
"entity_service.edit.parse_markdown",
|
||||
domain="entity_service",
|
||||
action="edit",
|
||||
phase="parse_markdown",
|
||||
):
|
||||
entity_markdown = await self.entity_parser.parse_markdown_content(
|
||||
file_path=file_path,
|
||||
content=new_content,
|
||||
)
|
||||
|
||||
with telemetry.scope(
|
||||
"entity_service.edit.upsert_entity",
|
||||
domain="entity_service",
|
||||
action="edit",
|
||||
phase="upsert_entity",
|
||||
):
|
||||
entity = await self.upsert_entity_from_markdown(
|
||||
file_path,
|
||||
entity_markdown,
|
||||
is_new=False,
|
||||
checksum=checksum,
|
||||
)
|
||||
if not entity: # pragma: no cover
|
||||
raise ValueError(f"Failed to persist entity after edit: {file_path}")
|
||||
|
||||
return EntityWriteResult(
|
||||
entity=entity,
|
||||
content=new_content,
|
||||
search_content=remove_frontmatter(new_content),
|
||||
)
|
||||
|
||||
# Update entity and its relationships
|
||||
entity = await self.upsert_entity_from_markdown(file_path, entity_markdown, is_new=False)
|
||||
|
||||
# Set final checksum to match file
|
||||
entity = await self.repository.update(entity.id, {"checksum": checksum})
|
||||
|
||||
return entity
|
||||
|
||||
def apply_edit_operation(
|
||||
self,
|
||||
current_content: str,
|
||||
|
||||
@@ -11,6 +11,7 @@ import aiofiles
|
||||
|
||||
import yaml
|
||||
|
||||
from basic_memory import telemetry
|
||||
from basic_memory import file_utils
|
||||
|
||||
if TYPE_CHECKING: # pragma: no cover
|
||||
@@ -79,13 +80,18 @@ class FileService:
|
||||
"""
|
||||
logger.debug(f"Reading entity content, entity_id={entity.id}, permalink={entity.permalink}")
|
||||
|
||||
# markdown_processor is required for entity content reads — fail fast if not configured
|
||||
if self.markdown_processor is None:
|
||||
raise ValueError("markdown_processor is required for read_entity_content")
|
||||
with telemetry.scope(
|
||||
"file_service.read_content",
|
||||
domain="file_service",
|
||||
action="read_content",
|
||||
phase="read_content",
|
||||
):
|
||||
if self.markdown_processor is None:
|
||||
raise ValueError("markdown_processor is required for read_entity_content")
|
||||
|
||||
file_path = self.get_entity_path(entity)
|
||||
markdown = await self.markdown_processor.read_file(file_path)
|
||||
return markdown.content or ""
|
||||
file_path = self.get_entity_path(entity)
|
||||
markdown = await self.markdown_processor.read_file(file_path)
|
||||
return markdown.content or ""
|
||||
|
||||
async def delete_entity_file(self, entity: EntityModel) -> None:
|
||||
"""Delete entity file from filesystem.
|
||||
@@ -176,32 +182,34 @@ class FileService:
|
||||
full_path = path_obj if path_obj.is_absolute() else self.base_path / path_obj
|
||||
|
||||
try:
|
||||
# Ensure parent directory exists
|
||||
await self.ensure_directory(full_path.parent)
|
||||
with telemetry.scope(
|
||||
"file_service.write",
|
||||
domain="file_service",
|
||||
action="write",
|
||||
phase="write",
|
||||
):
|
||||
await self.ensure_directory(full_path.parent)
|
||||
|
||||
# Write content atomically
|
||||
logger.info(
|
||||
"Writing file: "
|
||||
f"path={path_obj}, "
|
||||
f"content_length={len(content)}, "
|
||||
f"is_markdown={full_path.suffix.lower() == '.md'}"
|
||||
)
|
||||
|
||||
await file_utils.write_file_atomic(full_path, content)
|
||||
|
||||
# Format file if configured
|
||||
final_content = content
|
||||
if self.app_config:
|
||||
formatted_content = await file_utils.format_file(
|
||||
full_path, self.app_config, is_markdown=self.is_markdown(path)
|
||||
logger.info(
|
||||
"Writing file: "
|
||||
f"path={path_obj}, "
|
||||
f"content_length={len(content)}, "
|
||||
f"is_markdown={full_path.suffix.lower() == '.md'}"
|
||||
)
|
||||
if formatted_content is not None:
|
||||
final_content = formatted_content # pragma: no cover
|
||||
|
||||
# Compute and return checksum of final content
|
||||
checksum = await file_utils.compute_checksum(final_content)
|
||||
logger.debug(f"File write completed path={full_path}, {checksum=}")
|
||||
return checksum
|
||||
await file_utils.write_file_atomic(full_path, content)
|
||||
|
||||
final_content = content
|
||||
if self.app_config:
|
||||
formatted_content = await file_utils.format_file(
|
||||
full_path, self.app_config, is_markdown=self.is_markdown(path)
|
||||
)
|
||||
if formatted_content is not None:
|
||||
final_content = formatted_content # pragma: no cover
|
||||
|
||||
checksum = await file_utils.compute_checksum(final_content)
|
||||
logger.debug(f"File write completed path={full_path}, {checksum=}")
|
||||
return checksum
|
||||
|
||||
except Exception as e:
|
||||
logger.exception("File write error", path=str(full_path), error=str(e))
|
||||
@@ -227,16 +235,24 @@ class FileService:
|
||||
full_path = path_obj if path_obj.is_absolute() else self.base_path / path_obj
|
||||
|
||||
try:
|
||||
logger.debug("Reading file content", operation="read_file_content", path=str(full_path))
|
||||
async with aiofiles.open(full_path, mode="r", encoding="utf-8") as f:
|
||||
content = await f.read()
|
||||
with telemetry.scope(
|
||||
"file_service.read_content",
|
||||
domain="file_service",
|
||||
action="read_content",
|
||||
phase="read_content",
|
||||
):
|
||||
logger.debug(
|
||||
"Reading file content", operation="read_file_content", path=str(full_path)
|
||||
)
|
||||
async with aiofiles.open(full_path, mode="r", encoding="utf-8") as f:
|
||||
content = await f.read()
|
||||
|
||||
logger.debug(
|
||||
"File read completed",
|
||||
path=str(full_path),
|
||||
content_length=len(content),
|
||||
)
|
||||
return content
|
||||
logger.debug(
|
||||
"File read completed",
|
||||
path=str(full_path),
|
||||
content_length=len(content),
|
||||
)
|
||||
return content
|
||||
|
||||
except FileNotFoundError:
|
||||
# Preserve FileNotFoundError so callers (e.g. sync) can treat it as deletion.
|
||||
@@ -266,16 +282,22 @@ class FileService:
|
||||
full_path = path_obj if path_obj.is_absolute() else self.base_path / path_obj
|
||||
|
||||
try:
|
||||
logger.debug("Reading file bytes", operation="read_file_bytes", path=str(full_path))
|
||||
async with aiofiles.open(full_path, mode="rb") as f:
|
||||
content = await f.read()
|
||||
with telemetry.scope(
|
||||
"file_service.read_content",
|
||||
domain="file_service",
|
||||
action="read_content",
|
||||
phase="read_content",
|
||||
):
|
||||
logger.debug("Reading file bytes", operation="read_file_bytes", path=str(full_path))
|
||||
async with aiofiles.open(full_path, mode="rb") as f:
|
||||
content = await f.read()
|
||||
|
||||
logger.debug(
|
||||
"File read completed",
|
||||
path=str(full_path),
|
||||
content_length=len(content),
|
||||
)
|
||||
return content
|
||||
logger.debug(
|
||||
"File read completed",
|
||||
path=str(full_path),
|
||||
content_length=len(content),
|
||||
)
|
||||
return content
|
||||
|
||||
except Exception as e:
|
||||
logger.exception("File read error", path=str(full_path), error=str(e))
|
||||
@@ -303,21 +325,26 @@ class FileService:
|
||||
full_path = path_obj if path_obj.is_absolute() else self.base_path / path_obj
|
||||
|
||||
try:
|
||||
logger.debug("Reading file", operation="read_file", path=str(full_path))
|
||||
with telemetry.scope(
|
||||
"file_service.read",
|
||||
domain="file_service",
|
||||
action="read",
|
||||
phase="read",
|
||||
):
|
||||
logger.debug("Reading file", operation="read_file", path=str(full_path))
|
||||
|
||||
# Use aiofiles for non-blocking read
|
||||
async with aiofiles.open(full_path, mode="r", encoding="utf-8") as f:
|
||||
content = await f.read()
|
||||
async with aiofiles.open(full_path, mode="r", encoding="utf-8") as f:
|
||||
content = await f.read()
|
||||
|
||||
checksum = await file_utils.compute_checksum(content)
|
||||
checksum = await file_utils.compute_checksum(content)
|
||||
|
||||
logger.debug(
|
||||
"File read completed",
|
||||
path=str(full_path),
|
||||
checksum=checksum,
|
||||
content_length=len(content),
|
||||
)
|
||||
return content, checksum
|
||||
logger.debug(
|
||||
"File read completed",
|
||||
path=str(full_path),
|
||||
checksum=checksum,
|
||||
content_length=len(content),
|
||||
)
|
||||
return content, checksum
|
||||
|
||||
except Exception as e:
|
||||
logger.exception("File read error", path=str(full_path), error=str(e))
|
||||
|
||||
@@ -47,6 +47,7 @@ class LinkResolver:
|
||||
use_search: bool = True,
|
||||
strict: bool = False,
|
||||
source_path: Optional[str] = None,
|
||||
load_relations: bool = True,
|
||||
) -> Optional[Entity]:
|
||||
"""Resolve a markdown link to a permalink.
|
||||
|
||||
@@ -56,6 +57,7 @@ class LinkResolver:
|
||||
strict: If True, only exact matches are allowed (no fuzzy search fallback)
|
||||
source_path: Optional path of the source file containing the link.
|
||||
Used to prefer notes closer to the source (context-aware resolution).
|
||||
load_relations: When False, skip eager loading and return a lightweight entity row.
|
||||
"""
|
||||
logger.trace(f"Resolving link: {link_text} (source: {source_path})")
|
||||
|
||||
@@ -98,6 +100,7 @@ class LinkResolver:
|
||||
strict=strict,
|
||||
source_path=None,
|
||||
project_permalink=project.permalink,
|
||||
load_relations=load_relations,
|
||||
)
|
||||
|
||||
current_project_permalink = await self._get_current_project_permalink()
|
||||
@@ -109,6 +112,7 @@ class LinkResolver:
|
||||
strict=strict,
|
||||
source_path=source_path,
|
||||
project_permalink=current_project_permalink,
|
||||
load_relations=load_relations,
|
||||
)
|
||||
if resolved:
|
||||
return resolved
|
||||
@@ -136,6 +140,7 @@ class LinkResolver:
|
||||
strict=strict,
|
||||
source_path=None,
|
||||
project_permalink=project.permalink,
|
||||
load_relations=load_relations,
|
||||
)
|
||||
|
||||
def _normalize_link_text(self, link_text: str) -> Tuple[str, Optional[str]]:
|
||||
@@ -176,6 +181,7 @@ class LinkResolver:
|
||||
strict: bool,
|
||||
source_path: Optional[str],
|
||||
project_permalink: Optional[str],
|
||||
load_relations: bool,
|
||||
) -> Optional[Entity]:
|
||||
"""Resolve a link within a specific project scope."""
|
||||
clean_text = link_text
|
||||
@@ -223,12 +229,18 @@ class LinkResolver:
|
||||
# Try with .md extension
|
||||
if not relative_path.endswith(".md"):
|
||||
relative_path_md = f"{relative_path}.md"
|
||||
entity = await entity_repository.get_by_file_path(relative_path_md)
|
||||
entity = await entity_repository.get_by_file_path(
|
||||
relative_path_md,
|
||||
load_relations=load_relations,
|
||||
)
|
||||
if entity:
|
||||
return entity
|
||||
|
||||
# Try as-is (already has extension or is a permalink)
|
||||
entity = await entity_repository.get_by_file_path(relative_path)
|
||||
entity = await entity_repository.get_by_file_path(
|
||||
relative_path,
|
||||
load_relations=load_relations,
|
||||
)
|
||||
if entity:
|
||||
return entity
|
||||
|
||||
@@ -242,12 +254,18 @@ class LinkResolver:
|
||||
|
||||
# Check permalink match
|
||||
for candidate_permalink in permalink_candidates:
|
||||
permalink_entity = await entity_repository.get_by_permalink(candidate_permalink)
|
||||
permalink_entity = await entity_repository.get_by_permalink(
|
||||
candidate_permalink,
|
||||
load_relations=load_relations,
|
||||
)
|
||||
if permalink_entity and permalink_entity.id not in [c.id for c in candidates]:
|
||||
candidates.append(permalink_entity)
|
||||
|
||||
# Check title matches
|
||||
title_entities = await entity_repository.get_by_title(clean_text)
|
||||
title_entities = await entity_repository.get_by_title(
|
||||
clean_text,
|
||||
load_relations=load_relations,
|
||||
)
|
||||
for entity in title_entities:
|
||||
# Avoid duplicates (permalink match might also be in title matches)
|
||||
if entity.id not in [c.id for c in candidates]:
|
||||
@@ -263,13 +281,19 @@ class LinkResolver:
|
||||
# Standard resolution (no source context): permalink first, then title
|
||||
# 1. Try exact permalink match first (most efficient)
|
||||
for candidate_permalink in permalink_candidates:
|
||||
entity = await entity_repository.get_by_permalink(candidate_permalink)
|
||||
entity = await entity_repository.get_by_permalink(
|
||||
candidate_permalink,
|
||||
load_relations=load_relations,
|
||||
)
|
||||
if entity:
|
||||
logger.debug(f"Found exact permalink match: {entity.permalink}")
|
||||
return entity
|
||||
|
||||
# 2. Try exact title match
|
||||
found = await entity_repository.get_by_title(clean_text)
|
||||
found = await entity_repository.get_by_title(
|
||||
clean_text,
|
||||
load_relations=load_relations,
|
||||
)
|
||||
if found:
|
||||
# Return first match (shortest path) if no source context
|
||||
entity = found[0]
|
||||
@@ -277,7 +301,10 @@ class LinkResolver:
|
||||
return entity
|
||||
|
||||
# 3. Try file path
|
||||
found_path = await entity_repository.get_by_file_path(clean_text)
|
||||
found_path = await entity_repository.get_by_file_path(
|
||||
clean_text,
|
||||
load_relations=load_relations,
|
||||
)
|
||||
if found_path:
|
||||
logger.debug(f"Found entity with path: {found_path.file_path}")
|
||||
return found_path
|
||||
@@ -285,7 +312,10 @@ class LinkResolver:
|
||||
# 4. Try file path with .md extension if not already present
|
||||
if not clean_text.endswith(".md") and "/" in clean_text:
|
||||
file_path_with_md = f"{clean_text}.md"
|
||||
found_path_md = await entity_repository.get_by_file_path(file_path_with_md)
|
||||
found_path_md = await entity_repository.get_by_file_path(
|
||||
file_path_with_md,
|
||||
load_relations=load_relations,
|
||||
)
|
||||
if found_path_md:
|
||||
logger.debug(f"Found entity with path (with .md): {found_path_md.file_path}")
|
||||
return found_path_md
|
||||
@@ -309,7 +339,10 @@ class LinkResolver:
|
||||
f"Selected best match from {len(results)} results: {best_match.permalink}"
|
||||
)
|
||||
if best_match.permalink:
|
||||
return await entity_repository.get_by_permalink(best_match.permalink)
|
||||
return await entity_repository.get_by_permalink(
|
||||
best_match.permalink,
|
||||
load_relations=load_relations,
|
||||
)
|
||||
|
||||
# if we couldn't find anything then return None
|
||||
return None
|
||||
|
||||
@@ -997,9 +997,18 @@ class ProjectService:
|
||||
)
|
||||
|
||||
# --- Count queries (tables exist) ---
|
||||
# Filter by entity existence to exclude stale rows from deleted entities
|
||||
# that remain in derived search tables (search_index, search_vector_chunks)
|
||||
entity_exists = "AND entity_id IN (SELECT id FROM entity WHERE project_id = :project_id)"
|
||||
# Same filter for aliased chunks table (used in JOIN queries below)
|
||||
chunk_entity_exists = (
|
||||
"AND c.entity_id IN (SELECT id FROM entity WHERE project_id = :project_id)"
|
||||
)
|
||||
|
||||
si_result = await self.repository.execute_query(
|
||||
text(
|
||||
"SELECT COUNT(DISTINCT entity_id) FROM search_index WHERE project_id = :project_id"
|
||||
"SELECT COUNT(DISTINCT entity_id) FROM search_index "
|
||||
f"WHERE project_id = :project_id {entity_exists}"
|
||||
),
|
||||
{"project_id": project_id},
|
||||
)
|
||||
@@ -1007,7 +1016,10 @@ class ProjectService:
|
||||
|
||||
try:
|
||||
chunks_result = await self.repository.execute_query(
|
||||
text("SELECT COUNT(*) FROM search_vector_chunks WHERE project_id = :project_id"),
|
||||
text(
|
||||
"SELECT COUNT(*) FROM search_vector_chunks "
|
||||
f"WHERE project_id = :project_id {entity_exists}"
|
||||
),
|
||||
{"project_id": project_id},
|
||||
)
|
||||
total_chunks = chunks_result.scalar() or 0
|
||||
@@ -1015,7 +1027,7 @@ class ProjectService:
|
||||
entities_with_chunks_result = await self.repository.execute_query(
|
||||
text(
|
||||
"SELECT COUNT(DISTINCT entity_id) FROM search_vector_chunks "
|
||||
"WHERE project_id = :project_id"
|
||||
f"WHERE project_id = :project_id {entity_exists}"
|
||||
),
|
||||
{"project_id": project_id},
|
||||
)
|
||||
@@ -1026,13 +1038,13 @@ class ProjectService:
|
||||
embeddings_sql = text(
|
||||
"SELECT COUNT(*) FROM search_vector_chunks c "
|
||||
"JOIN search_vector_embeddings e ON e.chunk_id = c.id "
|
||||
"WHERE c.project_id = :project_id"
|
||||
f"WHERE c.project_id = :project_id {chunk_entity_exists}"
|
||||
)
|
||||
else:
|
||||
embeddings_sql = text(
|
||||
"SELECT COUNT(*) FROM search_vector_chunks c "
|
||||
"JOIN search_vector_embeddings e ON e.rowid = c.id "
|
||||
"WHERE c.project_id = :project_id"
|
||||
f"WHERE c.project_id = :project_id {chunk_entity_exists}"
|
||||
)
|
||||
|
||||
embeddings_result = await self.repository.execute_query(
|
||||
@@ -1045,13 +1057,13 @@ class ProjectService:
|
||||
orphan_sql = text(
|
||||
"SELECT COUNT(*) FROM search_vector_chunks c "
|
||||
"LEFT JOIN search_vector_embeddings e ON e.chunk_id = c.id "
|
||||
"WHERE c.project_id = :project_id AND e.chunk_id IS NULL"
|
||||
f"WHERE c.project_id = :project_id AND e.chunk_id IS NULL {chunk_entity_exists}"
|
||||
)
|
||||
else:
|
||||
orphan_sql = text(
|
||||
"SELECT COUNT(*) FROM search_vector_chunks c "
|
||||
"LEFT JOIN search_vector_embeddings e ON e.rowid = c.id "
|
||||
"WHERE c.project_id = :project_id AND e.rowid IS NULL"
|
||||
f"WHERE c.project_id = :project_id AND e.rowid IS NULL {chunk_entity_exists}"
|
||||
)
|
||||
|
||||
orphan_result = await self.repository.execute_query(
|
||||
|
||||
@@ -5,12 +5,12 @@ import re
|
||||
from datetime import datetime
|
||||
from typing import List, Optional, Set, Dict, Any
|
||||
|
||||
|
||||
from dateparser import parse
|
||||
from fastapi import BackgroundTasks
|
||||
from loguru import logger
|
||||
from sqlalchemy import text
|
||||
|
||||
from basic_memory import telemetry
|
||||
from basic_memory.models import Entity
|
||||
from basic_memory.repository import EntityRepository
|
||||
from basic_memory.repository.search_repository import (
|
||||
@@ -152,8 +152,6 @@ class SearchService:
|
||||
logger.debug("no criteria passed to query")
|
||||
return []
|
||||
|
||||
logger.trace(f"Searching with query: {query}")
|
||||
|
||||
after_date = (
|
||||
(
|
||||
query.after_date
|
||||
@@ -175,22 +173,49 @@ class SearchService:
|
||||
|
||||
retrieval_mode = query.retrieval_mode or SearchRetrievalMode.FTS
|
||||
strict_search_text = query.text
|
||||
has_query = bool(
|
||||
strict_search_text or query.title or query.permalink or query.permalink_match
|
||||
)
|
||||
has_filters = bool(
|
||||
metadata_filters
|
||||
or query.note_types
|
||||
or query.entity_types
|
||||
or after_date
|
||||
or query.tags
|
||||
or query.status
|
||||
)
|
||||
|
||||
# First pass: preserve existing strict search behavior.
|
||||
results = await self.repository.search(
|
||||
search_text=strict_search_text,
|
||||
permalink=query.permalink,
|
||||
permalink_match=query.permalink_match,
|
||||
title=query.title,
|
||||
note_types=query.note_types,
|
||||
search_item_types=query.entity_types,
|
||||
after_date=after_date,
|
||||
metadata_filters=metadata_filters,
|
||||
retrieval_mode=retrieval_mode,
|
||||
min_similarity=query.min_similarity,
|
||||
with telemetry.scope(
|
||||
"search.execute",
|
||||
retrieval_mode=retrieval_mode.value,
|
||||
has_query=has_query,
|
||||
has_filters=has_filters,
|
||||
limit=limit,
|
||||
offset=offset,
|
||||
)
|
||||
):
|
||||
logger.trace(f"Searching with query: {query}")
|
||||
with telemetry.scope(
|
||||
"search.repository_query",
|
||||
retrieval_mode=retrieval_mode.value,
|
||||
phase="repository_query",
|
||||
has_query=has_query,
|
||||
has_filters=has_filters,
|
||||
):
|
||||
# First pass: preserve existing strict search behavior.
|
||||
results = await self.repository.search(
|
||||
search_text=strict_search_text,
|
||||
permalink=query.permalink,
|
||||
permalink_match=query.permalink_match,
|
||||
title=query.title,
|
||||
note_types=query.note_types,
|
||||
search_item_types=query.entity_types,
|
||||
after_date=after_date,
|
||||
metadata_filters=metadata_filters,
|
||||
retrieval_mode=retrieval_mode,
|
||||
min_similarity=query.min_similarity,
|
||||
limit=limit,
|
||||
offset=offset,
|
||||
)
|
||||
|
||||
# Trigger: strict FTS with plain multi-term text returned no results.
|
||||
# Why: natural-language queries often include stopwords that over-constrain implicit AND.
|
||||
@@ -209,20 +234,34 @@ class SearchService:
|
||||
"Strict FTS returned 0 results; retrying relaxed FTS query "
|
||||
f"strict='{strict_search_text}' relaxed='{relaxed_search_text}'"
|
||||
)
|
||||
return await self.repository.search(
|
||||
search_text=relaxed_search_text,
|
||||
permalink=query.permalink,
|
||||
permalink_match=query.permalink_match,
|
||||
title=query.title,
|
||||
note_types=query.note_types,
|
||||
search_item_types=query.entity_types,
|
||||
after_date=after_date,
|
||||
metadata_filters=metadata_filters,
|
||||
retrieval_mode=retrieval_mode,
|
||||
min_similarity=query.min_similarity,
|
||||
with telemetry.scope(
|
||||
"search.relaxed_fts_retry",
|
||||
retrieval_mode=retrieval_mode.value,
|
||||
token_count=len(self._tokenize_fts_text(strict_search_text)),
|
||||
limit=limit,
|
||||
offset=offset,
|
||||
)
|
||||
):
|
||||
with telemetry.scope(
|
||||
"search.repository_query",
|
||||
retrieval_mode=retrieval_mode.value,
|
||||
phase="repository_query",
|
||||
has_query=has_query,
|
||||
has_filters=has_filters,
|
||||
):
|
||||
return await self.repository.search(
|
||||
search_text=relaxed_search_text,
|
||||
permalink=query.permalink,
|
||||
permalink_match=query.permalink_match,
|
||||
title=query.title,
|
||||
note_types=query.note_types,
|
||||
search_item_types=query.entity_types,
|
||||
after_date=after_date,
|
||||
metadata_filters=metadata_filters,
|
||||
retrieval_mode=retrieval_mode,
|
||||
min_similarity=query.min_similarity,
|
||||
limit=limit,
|
||||
offset=offset,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _tokenize_fts_text(search_text: str) -> list[str]:
|
||||
@@ -356,13 +395,22 @@ class SearchService:
|
||||
f"permalink={entity.permalink} project_id={entity.project_id}"
|
||||
)
|
||||
try:
|
||||
# delete all search index data associated with entity
|
||||
await self.repository.delete_by_entity_id(entity_id=entity.id)
|
||||
with telemetry.scope(
|
||||
"search.index_entity_data",
|
||||
phase="index_entity_data",
|
||||
result_count=1,
|
||||
):
|
||||
with telemetry.scope(
|
||||
"search.index.delete_existing",
|
||||
phase="delete_existing",
|
||||
result_count=1,
|
||||
):
|
||||
await self.repository.delete_by_entity_id(entity_id=entity.id)
|
||||
|
||||
# reindex
|
||||
await self.index_entity_markdown(
|
||||
entity, content
|
||||
) if entity.is_markdown else await self.index_entity_file(entity)
|
||||
if entity.is_markdown:
|
||||
await self.index_entity_markdown(entity, content)
|
||||
else:
|
||||
await self.index_entity_file(entity)
|
||||
|
||||
logger.debug(
|
||||
f"[BackgroundTask] Completed search index for entity_id={entity.id} "
|
||||
@@ -403,6 +451,11 @@ class SearchService:
|
||||
"""
|
||||
entities = await self.entity_repository.find_all()
|
||||
entity_ids = [entity.id for entity in entities]
|
||||
|
||||
# Clean up stale rows in search_index and search_vector_chunks
|
||||
# that reference entity_ids no longer in the entity table
|
||||
await self._purge_stale_search_rows()
|
||||
|
||||
batch_result = await self.repository.sync_entity_vectors_batch(
|
||||
entity_ids,
|
||||
progress_callback=progress_callback,
|
||||
@@ -419,27 +472,78 @@ class SearchService:
|
||||
|
||||
return stats
|
||||
|
||||
async def _purge_stale_search_rows(self) -> None:
|
||||
"""Remove rows from search_index and search_vector_chunks for deleted entities.
|
||||
|
||||
Trigger: entities are deleted but their derived search rows remain
|
||||
Why: stale rows inflate embedding coverage stats in project info
|
||||
Outcome: search tables only contain rows for entities that still exist
|
||||
"""
|
||||
from basic_memory.repository.sqlite_search_repository import SQLiteSearchRepository
|
||||
from sqlalchemy import text
|
||||
|
||||
project_id = self.repository.project_id
|
||||
stale_entity_filter = (
|
||||
"entity_id NOT IN (SELECT id FROM entity WHERE project_id = :project_id)"
|
||||
)
|
||||
params = {"project_id": project_id}
|
||||
|
||||
# Delete stale search_index rows
|
||||
await self.repository.execute_query(
|
||||
text(
|
||||
f"DELETE FROM search_index WHERE project_id = :project_id AND {stale_entity_filter}"
|
||||
),
|
||||
params,
|
||||
)
|
||||
|
||||
# SQLite vec has no CASCADE — must delete embeddings before chunks
|
||||
if isinstance(self.repository, SQLiteSearchRepository):
|
||||
await self.repository.execute_query(
|
||||
text(
|
||||
"DELETE FROM search_vector_embeddings WHERE rowid IN ("
|
||||
"SELECT id FROM search_vector_chunks "
|
||||
f"WHERE project_id = :project_id AND {stale_entity_filter})"
|
||||
),
|
||||
params,
|
||||
)
|
||||
|
||||
# Postgres CASCADE handles embedding deletion automatically
|
||||
await self.repository.execute_query(
|
||||
text(
|
||||
f"DELETE FROM search_vector_chunks "
|
||||
f"WHERE project_id = :project_id AND {stale_entity_filter}"
|
||||
),
|
||||
params,
|
||||
)
|
||||
|
||||
logger.info("Purged stale search rows for deleted entities", project_id=project_id)
|
||||
|
||||
async def index_entity_file(
|
||||
self,
|
||||
entity: Entity,
|
||||
) -> None:
|
||||
# Index entity file with no content
|
||||
await self.repository.index_item(
|
||||
SearchIndexRow(
|
||||
id=entity.id,
|
||||
entity_id=entity.id,
|
||||
type=SearchItemType.ENTITY.value,
|
||||
title=_strip_nul(entity.title),
|
||||
permalink=entity.permalink, # Required for Postgres NOT NULL constraint
|
||||
file_path=entity.file_path,
|
||||
metadata={
|
||||
"note_type": entity.note_type,
|
||||
},
|
||||
created_at=entity.created_at,
|
||||
updated_at=_mtime_to_datetime(entity),
|
||||
project_id=entity.project_id,
|
||||
with telemetry.scope(
|
||||
"search.index_file",
|
||||
phase="index_file",
|
||||
result_count=1,
|
||||
):
|
||||
# Index entity file with no content
|
||||
await self.repository.index_item(
|
||||
SearchIndexRow(
|
||||
id=entity.id,
|
||||
entity_id=entity.id,
|
||||
type=SearchItemType.ENTITY.value,
|
||||
title=_strip_nul(entity.title),
|
||||
permalink=entity.permalink, # Required for Postgres NOT NULL constraint
|
||||
file_path=entity.file_path,
|
||||
metadata={
|
||||
"note_type": entity.note_type,
|
||||
},
|
||||
created_at=entity.created_at,
|
||||
updated_at=_mtime_to_datetime(entity),
|
||||
project_id=entity.project_id,
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
async def index_entity_markdown(
|
||||
self,
|
||||
@@ -472,129 +576,144 @@ class SearchService:
|
||||
The project_id is automatically added by the repository when indexing.
|
||||
"""
|
||||
|
||||
# Collect all search index rows to batch insert at the end
|
||||
rows_to_index = []
|
||||
with telemetry.scope(
|
||||
"search.index_markdown",
|
||||
phase="index_markdown",
|
||||
result_count=1,
|
||||
):
|
||||
rows_to_index = []
|
||||
|
||||
content_stems = []
|
||||
content_snippet = ""
|
||||
title_variants = self._generate_variants(entity.title)
|
||||
content_stems.extend(title_variants)
|
||||
content_stems = []
|
||||
content_snippet = ""
|
||||
title_variants = self._generate_variants(entity.title)
|
||||
content_stems.extend(title_variants)
|
||||
|
||||
# Use provided content or read from file
|
||||
if content is None:
|
||||
content = await self.file_service.read_entity_content(entity)
|
||||
if content:
|
||||
content_stems.append(content)
|
||||
# Store full content for vector embedding quality.
|
||||
# The chunker in the vector pipeline splits this into
|
||||
# appropriately-sized pieces for embedding.
|
||||
content_snippet = _strip_nul(content)
|
||||
if content is None:
|
||||
with telemetry.scope(
|
||||
"search.index.read_content",
|
||||
phase="read_content",
|
||||
result_count=1,
|
||||
):
|
||||
content = await self.file_service.read_entity_content(entity)
|
||||
if content:
|
||||
content_stems.append(content)
|
||||
content_snippet = _strip_nul(content)
|
||||
|
||||
if entity.permalink:
|
||||
content_stems.extend(self._generate_variants(entity.permalink))
|
||||
with telemetry.scope(
|
||||
"search.index.build_rows",
|
||||
phase="build_rows",
|
||||
result_count=1,
|
||||
):
|
||||
if entity.permalink:
|
||||
content_stems.extend(self._generate_variants(entity.permalink))
|
||||
|
||||
content_stems.extend(self._generate_variants(entity.file_path))
|
||||
content_stems.extend(self._generate_variants(entity.file_path))
|
||||
|
||||
# Add entity tags from frontmatter to search content
|
||||
entity_tags = self._extract_entity_tags(entity)
|
||||
if entity_tags:
|
||||
content_stems.extend(entity_tags)
|
||||
entity_tags = self._extract_entity_tags(entity)
|
||||
if entity_tags:
|
||||
content_stems.extend(entity_tags)
|
||||
|
||||
entity_content_stems = _strip_nul("\n".join(p for p in content_stems if p and p.strip()))
|
||||
|
||||
# Truncate to stay under Postgres's 8KB index row limit
|
||||
if len(entity_content_stems) > MAX_CONTENT_STEMS_SIZE: # pragma: no cover
|
||||
entity_content_stems = entity_content_stems[:MAX_CONTENT_STEMS_SIZE] # pragma: no cover
|
||||
|
||||
# Add entity row
|
||||
rows_to_index.append(
|
||||
SearchIndexRow(
|
||||
id=entity.id,
|
||||
type=SearchItemType.ENTITY.value,
|
||||
title=_strip_nul(entity.title),
|
||||
content_stems=entity_content_stems,
|
||||
content_snippet=content_snippet,
|
||||
permalink=entity.permalink,
|
||||
file_path=entity.file_path,
|
||||
entity_id=entity.id,
|
||||
metadata={
|
||||
"note_type": entity.note_type,
|
||||
},
|
||||
created_at=entity.created_at,
|
||||
updated_at=_mtime_to_datetime(entity),
|
||||
project_id=entity.project_id,
|
||||
)
|
||||
)
|
||||
|
||||
# Add observation rows - dedupe by permalink to avoid unique constraint violations
|
||||
# Two observations with same entity/category/content generate identical permalinks
|
||||
seen_permalinks: set[str] = {entity.permalink} if entity.permalink else set()
|
||||
for obs in entity.observations:
|
||||
obs_permalink = obs.permalink
|
||||
if obs_permalink in seen_permalinks:
|
||||
logger.debug(f"Skipping duplicate observation permalink: {obs_permalink}")
|
||||
continue
|
||||
seen_permalinks.add(obs_permalink)
|
||||
|
||||
# Index with parent entity's file path since that's where it's defined
|
||||
obs_content_stems = _strip_nul(
|
||||
"\n".join(p for p in self._generate_variants(obs.content) if p and p.strip())
|
||||
)
|
||||
# Truncate to stay under Postgres's 8KB index row limit
|
||||
if len(obs_content_stems) > MAX_CONTENT_STEMS_SIZE: # pragma: no cover
|
||||
obs_content_stems = obs_content_stems[:MAX_CONTENT_STEMS_SIZE] # pragma: no cover
|
||||
rows_to_index.append(
|
||||
SearchIndexRow(
|
||||
id=obs.id,
|
||||
type=SearchItemType.OBSERVATION.value,
|
||||
title=_strip_nul(f"{obs.category}: {obs.content[:100]}..."),
|
||||
content_stems=obs_content_stems,
|
||||
content_snippet=_strip_nul(obs.content),
|
||||
permalink=obs_permalink,
|
||||
file_path=entity.file_path,
|
||||
category=obs.category,
|
||||
entity_id=entity.id,
|
||||
metadata={
|
||||
"tags": obs.tags,
|
||||
},
|
||||
created_at=entity.created_at,
|
||||
updated_at=_mtime_to_datetime(entity),
|
||||
project_id=entity.project_id,
|
||||
entity_content_stems = _strip_nul(
|
||||
"\n".join(p for p in content_stems if p and p.strip())
|
||||
)
|
||||
)
|
||||
|
||||
# Add relation rows (only outgoing relations defined in this file)
|
||||
for rel in entity.outgoing_relations:
|
||||
# Create descriptive title showing the relationship
|
||||
relation_title = _strip_nul(
|
||||
f"{rel.from_entity.title} → {rel.to_entity.title}"
|
||||
if rel.to_entity
|
||||
else f"{rel.from_entity.title}"
|
||||
)
|
||||
if len(entity_content_stems) > MAX_CONTENT_STEMS_SIZE: # pragma: no cover
|
||||
entity_content_stems = entity_content_stems[
|
||||
:MAX_CONTENT_STEMS_SIZE
|
||||
] # pragma: no cover
|
||||
|
||||
rel_content_stems = _strip_nul(
|
||||
"\n".join(p for p in self._generate_variants(relation_title) if p and p.strip())
|
||||
)
|
||||
rows_to_index.append(
|
||||
SearchIndexRow(
|
||||
id=rel.id,
|
||||
title=relation_title,
|
||||
permalink=rel.permalink,
|
||||
content_stems=rel_content_stems,
|
||||
file_path=entity.file_path,
|
||||
type=SearchItemType.RELATION.value,
|
||||
entity_id=entity.id,
|
||||
from_id=rel.from_id,
|
||||
to_id=rel.to_id,
|
||||
relation_type=rel.relation_type,
|
||||
created_at=entity.created_at,
|
||||
updated_at=_mtime_to_datetime(entity),
|
||||
project_id=entity.project_id,
|
||||
rows_to_index.append(
|
||||
SearchIndexRow(
|
||||
id=entity.id,
|
||||
type=SearchItemType.ENTITY.value,
|
||||
title=_strip_nul(entity.title),
|
||||
content_stems=entity_content_stems,
|
||||
content_snippet=content_snippet,
|
||||
permalink=entity.permalink,
|
||||
file_path=entity.file_path,
|
||||
entity_id=entity.id,
|
||||
metadata={
|
||||
"note_type": entity.note_type,
|
||||
},
|
||||
created_at=entity.created_at,
|
||||
updated_at=_mtime_to_datetime(entity),
|
||||
project_id=entity.project_id,
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
# Batch insert all rows at once
|
||||
await self.repository.bulk_index_items(rows_to_index)
|
||||
seen_permalinks: set[str] = {entity.permalink} if entity.permalink else set()
|
||||
for obs in entity.observations:
|
||||
obs_permalink = obs.permalink
|
||||
if obs_permalink in seen_permalinks:
|
||||
logger.debug(f"Skipping duplicate observation permalink: {obs_permalink}")
|
||||
continue
|
||||
seen_permalinks.add(obs_permalink)
|
||||
|
||||
obs_content_stems = _strip_nul(
|
||||
"\n".join(
|
||||
p for p in self._generate_variants(obs.content) if p and p.strip()
|
||||
)
|
||||
)
|
||||
if len(obs_content_stems) > MAX_CONTENT_STEMS_SIZE: # pragma: no cover
|
||||
obs_content_stems = obs_content_stems[
|
||||
:MAX_CONTENT_STEMS_SIZE
|
||||
] # pragma: no cover
|
||||
rows_to_index.append(
|
||||
SearchIndexRow(
|
||||
id=obs.id,
|
||||
type=SearchItemType.OBSERVATION.value,
|
||||
title=_strip_nul(f"{obs.category}: {obs.content[:100]}..."),
|
||||
content_stems=obs_content_stems,
|
||||
content_snippet=_strip_nul(obs.content),
|
||||
permalink=obs_permalink,
|
||||
file_path=entity.file_path,
|
||||
category=obs.category,
|
||||
entity_id=entity.id,
|
||||
metadata={
|
||||
"tags": obs.tags,
|
||||
},
|
||||
created_at=entity.created_at,
|
||||
updated_at=_mtime_to_datetime(entity),
|
||||
project_id=entity.project_id,
|
||||
)
|
||||
)
|
||||
|
||||
for rel in entity.outgoing_relations:
|
||||
relation_title = _strip_nul(
|
||||
f"{rel.from_entity.title} -> {rel.to_entity.title}"
|
||||
if rel.to_entity
|
||||
else f"{rel.from_entity.title}"
|
||||
)
|
||||
|
||||
rel_content_stems = _strip_nul(
|
||||
"\n".join(
|
||||
p for p in self._generate_variants(relation_title) if p and p.strip()
|
||||
)
|
||||
)
|
||||
rows_to_index.append(
|
||||
SearchIndexRow(
|
||||
id=rel.id,
|
||||
title=relation_title,
|
||||
permalink=rel.permalink,
|
||||
content_stems=rel_content_stems,
|
||||
file_path=entity.file_path,
|
||||
type=SearchItemType.RELATION.value,
|
||||
entity_id=entity.id,
|
||||
from_id=rel.from_id,
|
||||
to_id=rel.to_id,
|
||||
relation_type=rel.relation_type,
|
||||
created_at=entity.created_at,
|
||||
updated_at=_mtime_to_datetime(entity),
|
||||
project_id=entity.project_id,
|
||||
)
|
||||
)
|
||||
|
||||
with telemetry.scope(
|
||||
"search.index.bulk_upsert",
|
||||
phase="bulk_upsert",
|
||||
result_count=len(rows_to_index),
|
||||
):
|
||||
await self.repository.bulk_index_items(rows_to_index)
|
||||
|
||||
async def delete_by_permalink(self, permalink: str):
|
||||
"""Delete an item from the search index."""
|
||||
|
||||
@@ -15,6 +15,7 @@ import aiofiles.os
|
||||
from loguru import logger
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
|
||||
from basic_memory import telemetry
|
||||
from basic_memory import db
|
||||
from basic_memory.config import BasicMemoryConfig, ConfigManager
|
||||
from basic_memory.file_utils import has_frontmatter
|
||||
@@ -36,6 +37,7 @@ from basic_memory.services.search_service import SearchService
|
||||
|
||||
# Circuit breaker configuration
|
||||
MAX_CONSECUTIVE_FAILURES = 3
|
||||
SLOW_FILE_SYNC_WARNING_MS = 500
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -265,138 +267,163 @@ class SyncService:
|
||||
|
||||
start_time = time.time()
|
||||
sync_start_timestamp = time.time() # Capture at start for watermark
|
||||
logger.info(f"Sync operation started for directory: {directory} (force_full={force_full})")
|
||||
|
||||
# initial paths from db to sync
|
||||
# path -> checksum
|
||||
report = await self.scan(directory, force_full=force_full)
|
||||
|
||||
# order of sync matters to resolve relations effectively
|
||||
logger.info(
|
||||
f"Sync changes detected: new_files={len(report.new)}, modified_files={len(report.modified)}, "
|
||||
+ f"deleted_files={len(report.deleted)}, moved_files={len(report.moves)}"
|
||||
)
|
||||
|
||||
# sync moves first
|
||||
for old_path, new_path in report.moves.items():
|
||||
# in the case where a file has been deleted and replaced by another file
|
||||
# it will show up in the move and modified lists, so handle it in modified
|
||||
if new_path in report.modified:
|
||||
report.modified.remove(new_path)
|
||||
logger.debug(
|
||||
f"File marked as moved and modified: old_path={old_path}, new_path={new_path}"
|
||||
)
|
||||
else:
|
||||
await self.handle_move(old_path, new_path)
|
||||
|
||||
# deleted next
|
||||
for path in report.deleted:
|
||||
await self.handle_delete(path)
|
||||
|
||||
# then new and modified — collect entity IDs for batch vector embedding
|
||||
synced_entity_ids: list[int] = []
|
||||
|
||||
for path in report.new:
|
||||
entity, _ = await self.sync_file(path, new=True)
|
||||
|
||||
if entity is not None:
|
||||
synced_entity_ids.append(entity.id)
|
||||
# Track if file was skipped
|
||||
elif await self._should_skip_file(path):
|
||||
failure_info = self._file_failures[path]
|
||||
report.skipped_files.append(
|
||||
SkippedFile(
|
||||
path=path,
|
||||
reason=failure_info.last_error,
|
||||
failure_count=failure_info.count,
|
||||
first_failed=failure_info.first_failure,
|
||||
)
|
||||
)
|
||||
|
||||
for path in report.modified:
|
||||
entity, _ = await self.sync_file(path, new=False)
|
||||
|
||||
if entity is not None:
|
||||
synced_entity_ids.append(entity.id)
|
||||
# Track if file was skipped
|
||||
elif await self._should_skip_file(path):
|
||||
failure_info = self._file_failures[path]
|
||||
report.skipped_files.append(
|
||||
SkippedFile(
|
||||
path=path,
|
||||
reason=failure_info.last_error,
|
||||
failure_count=failure_info.count,
|
||||
first_failed=failure_info.first_failure,
|
||||
)
|
||||
)
|
||||
|
||||
# Only resolve relations if there were actual changes
|
||||
# If no files changed, no new unresolved relations could have been created
|
||||
if report.total > 0:
|
||||
await self.resolve_relations()
|
||||
else:
|
||||
logger.info("Skipping relation resolution - no file changes detected")
|
||||
|
||||
# Batch-generate vector embeddings for all synced entities
|
||||
if synced_entity_ids and self.app_config.semantic_search_enabled:
|
||||
try:
|
||||
logger.info(
|
||||
f"Generating semantic embeddings for {len(synced_entity_ids)} entities..."
|
||||
)
|
||||
batch_result = await self.search_service.sync_entity_vectors_batch(
|
||||
synced_entity_ids
|
||||
)
|
||||
logger.info(
|
||||
f"Semantic embeddings complete: "
|
||||
f"synced={batch_result.entities_synced}, "
|
||||
f"failed={batch_result.entities_failed}"
|
||||
)
|
||||
except SemanticDependenciesMissingError:
|
||||
logger.warning(
|
||||
"Semantic search dependencies missing — vector embeddings skipped. "
|
||||
"Run 'bm reindex --embeddings' after resolving the dependency issue."
|
||||
)
|
||||
|
||||
# Update scan watermark after successful sync
|
||||
# Use the timestamp from sync start (not end) to ensure we catch files
|
||||
# created during the sync on the next iteration
|
||||
current_file_count = await self._quick_count_files(directory)
|
||||
if self.entity_repository.project_id is not None:
|
||||
project = await self.project_repository.find_by_id(self.entity_repository.project_id)
|
||||
if project:
|
||||
await self.project_repository.update(
|
||||
project.id,
|
||||
{
|
||||
"last_scan_timestamp": sync_start_timestamp,
|
||||
"last_file_count": current_file_count,
|
||||
},
|
||||
)
|
||||
logger.debug(
|
||||
f"Updated scan watermark: timestamp={sync_start_timestamp}, "
|
||||
f"file_count={current_file_count}"
|
||||
)
|
||||
|
||||
duration_ms = int((time.time() - start_time) * 1000)
|
||||
|
||||
# Log summary with skipped files if any
|
||||
if report.skipped_files:
|
||||
logger.warning(
|
||||
f"Sync completed with {len(report.skipped_files)} skipped files: "
|
||||
f"directory={directory}, total_changes={report.total}, "
|
||||
f"skipped={len(report.skipped_files)}, duration_ms={duration_ms}"
|
||||
)
|
||||
for skipped in report.skipped_files:
|
||||
logger.warning(
|
||||
f"Skipped file: path={skipped.path}, "
|
||||
f"failures={skipped.failure_count}, reason={skipped.reason}"
|
||||
)
|
||||
else:
|
||||
with telemetry.operation(
|
||||
"sync.project.run",
|
||||
project_name=project_name,
|
||||
force_full=force_full,
|
||||
):
|
||||
logger.info(
|
||||
f"Sync operation completed: directory={directory}, "
|
||||
f"total_changes={report.total}, duration_ms={duration_ms}"
|
||||
f"Sync operation started for directory: {directory} (force_full={force_full})"
|
||||
)
|
||||
|
||||
return report
|
||||
# initial paths from db to sync
|
||||
# path -> checksum
|
||||
with telemetry.scope("sync.project.scan", force_full=force_full):
|
||||
report = await self.scan(directory, force_full=force_full)
|
||||
|
||||
# order of sync matters to resolve relations effectively
|
||||
logger.info(
|
||||
f"Sync changes detected: new_files={len(report.new)}, modified_files={len(report.modified)}, "
|
||||
+ f"deleted_files={len(report.deleted)}, moved_files={len(report.moves)}"
|
||||
)
|
||||
|
||||
with telemetry.scope(
|
||||
"sync.project.apply_changes",
|
||||
new_count=len(report.new),
|
||||
modified_count=len(report.modified),
|
||||
deleted_count=len(report.deleted),
|
||||
move_count=len(report.moves),
|
||||
):
|
||||
# sync moves first
|
||||
for old_path, new_path in report.moves.items():
|
||||
# in the case where a file has been deleted and replaced by another file
|
||||
# it will show up in the move and modified lists, so handle it in modified
|
||||
if new_path in report.modified:
|
||||
report.modified.remove(new_path)
|
||||
logger.debug(
|
||||
f"File marked as moved and modified: old_path={old_path}, new_path={new_path}"
|
||||
)
|
||||
else:
|
||||
await self.handle_move(old_path, new_path)
|
||||
|
||||
# deleted next
|
||||
for path in report.deleted:
|
||||
await self.handle_delete(path)
|
||||
|
||||
# then new and modified — collect entity IDs for batch vector embedding
|
||||
synced_entity_ids: list[int] = []
|
||||
|
||||
for path in report.new:
|
||||
entity, _ = await self.sync_file(path, new=True)
|
||||
|
||||
if entity is not None:
|
||||
synced_entity_ids.append(entity.id)
|
||||
# Track if file was skipped
|
||||
elif await self._should_skip_file(path):
|
||||
failure_info = self._file_failures[path]
|
||||
report.skipped_files.append(
|
||||
SkippedFile(
|
||||
path=path,
|
||||
reason=failure_info.last_error,
|
||||
failure_count=failure_info.count,
|
||||
first_failed=failure_info.first_failure,
|
||||
)
|
||||
)
|
||||
|
||||
for path in report.modified:
|
||||
entity, _ = await self.sync_file(path, new=False)
|
||||
|
||||
if entity is not None:
|
||||
synced_entity_ids.append(entity.id)
|
||||
# Track if file was skipped
|
||||
elif await self._should_skip_file(path):
|
||||
failure_info = self._file_failures[path]
|
||||
report.skipped_files.append(
|
||||
SkippedFile(
|
||||
path=path,
|
||||
reason=failure_info.last_error,
|
||||
failure_count=failure_info.count,
|
||||
first_failed=failure_info.first_failure,
|
||||
)
|
||||
)
|
||||
|
||||
# Only resolve relations if there were actual changes
|
||||
# If no files changed, no new unresolved relations could have been created
|
||||
if report.total > 0:
|
||||
with telemetry.scope(
|
||||
"sync.project.resolve_relations", relation_scope="all_pending"
|
||||
):
|
||||
await self.resolve_relations()
|
||||
else:
|
||||
logger.info("Skipping relation resolution - no file changes detected")
|
||||
|
||||
# Batch-generate vector embeddings for all synced entities
|
||||
if synced_entity_ids and self.app_config.semantic_search_enabled:
|
||||
try:
|
||||
with telemetry.scope(
|
||||
"sync.project.sync_embeddings",
|
||||
entity_count=len(synced_entity_ids),
|
||||
):
|
||||
logger.info(
|
||||
f"Generating semantic embeddings for {len(synced_entity_ids)} entities..."
|
||||
)
|
||||
batch_result = await self.search_service.sync_entity_vectors_batch(
|
||||
synced_entity_ids
|
||||
)
|
||||
logger.info(
|
||||
f"Semantic embeddings complete: "
|
||||
f"synced={batch_result.entities_synced}, "
|
||||
f"failed={batch_result.entities_failed}"
|
||||
)
|
||||
except SemanticDependenciesMissingError:
|
||||
logger.warning(
|
||||
"Semantic search dependencies missing — vector embeddings skipped. "
|
||||
"Run 'bm reindex --embeddings' after resolving the dependency issue."
|
||||
)
|
||||
|
||||
# Update scan watermark after successful sync
|
||||
# Use the timestamp from sync start (not end) to ensure we catch files
|
||||
# created during the sync on the next iteration
|
||||
with telemetry.scope("sync.project.update_watermark"):
|
||||
current_file_count = await self._quick_count_files(directory)
|
||||
if self.entity_repository.project_id is not None:
|
||||
project = await self.project_repository.find_by_id(
|
||||
self.entity_repository.project_id
|
||||
)
|
||||
if project:
|
||||
await self.project_repository.update(
|
||||
project.id,
|
||||
{
|
||||
"last_scan_timestamp": sync_start_timestamp,
|
||||
"last_file_count": current_file_count,
|
||||
},
|
||||
)
|
||||
logger.debug(
|
||||
f"Updated scan watermark: timestamp={sync_start_timestamp}, "
|
||||
f"file_count={current_file_count}"
|
||||
)
|
||||
|
||||
duration_ms = int((time.time() - start_time) * 1000)
|
||||
|
||||
# Log summary with skipped files if any
|
||||
if report.skipped_files:
|
||||
logger.warning(
|
||||
f"Sync completed with {len(report.skipped_files)} skipped files: "
|
||||
f"directory={directory}, total_changes={report.total}, "
|
||||
f"skipped={len(report.skipped_files)}, duration_ms={duration_ms}"
|
||||
)
|
||||
for skipped in report.skipped_files:
|
||||
logger.warning(
|
||||
f"Skipped file: path={skipped.path}, "
|
||||
f"failures={skipped.failure_count}, reason={skipped.reason}"
|
||||
)
|
||||
else:
|
||||
logger.info(
|
||||
f"Sync operation completed: directory={directory}, "
|
||||
f"total_changes={report.total}, duration_ms={duration_ms}"
|
||||
)
|
||||
|
||||
return report
|
||||
|
||||
async def scan(self, directory, force_full: bool = False):
|
||||
"""Smart scan using watermark and file count for large project optimization.
|
||||
@@ -433,171 +460,180 @@ class SyncService:
|
||||
if project is None:
|
||||
raise ValueError(f"Project not found: {self.entity_repository.project_id}")
|
||||
|
||||
# Step 1: Quick file count
|
||||
logger.debug("Counting files in directory")
|
||||
current_count = await self._quick_count_files(directory)
|
||||
logger.debug(f"Found {current_count} files in directory")
|
||||
with telemetry.scope("sync.project.select_scan_strategy", force_full=force_full):
|
||||
# Step 1: Quick file count
|
||||
logger.debug("Counting files in directory")
|
||||
current_count = await self._quick_count_files(directory)
|
||||
logger.debug(f"Found {current_count} files in directory")
|
||||
|
||||
# Step 2: Determine scan strategy based on watermark and file count
|
||||
if force_full:
|
||||
# User explicitly requested full scan → bypass watermark optimization
|
||||
scan_type = "full_forced"
|
||||
logger.info("Force full scan requested, bypassing watermark optimization")
|
||||
file_paths_to_scan = await self._scan_directory_full(directory)
|
||||
# Step 2: Determine scan strategy based on watermark and file count
|
||||
if force_full:
|
||||
# User explicitly requested full scan → bypass watermark optimization
|
||||
scan_type = "full_forced"
|
||||
logger.info("Force full scan requested, bypassing watermark optimization")
|
||||
scan_coro = self._scan_directory_full(directory)
|
||||
|
||||
elif project.last_file_count is None:
|
||||
# First sync ever → full scan
|
||||
scan_type = "full_initial"
|
||||
logger.info("First sync for this project, performing full scan")
|
||||
file_paths_to_scan = await self._scan_directory_full(directory)
|
||||
elif project.last_file_count is None:
|
||||
# First sync ever → full scan
|
||||
scan_type = "full_initial"
|
||||
logger.info("First sync for this project, performing full scan")
|
||||
scan_coro = self._scan_directory_full(directory)
|
||||
|
||||
elif current_count < project.last_file_count:
|
||||
# Files deleted → need full scan to detect which ones
|
||||
scan_type = "full_deletions"
|
||||
logger.info(
|
||||
f"File count decreased ({project.last_file_count} → {current_count}), "
|
||||
f"running full scan to detect deletions"
|
||||
)
|
||||
file_paths_to_scan = await self._scan_directory_full(directory)
|
||||
elif current_count < project.last_file_count:
|
||||
# Files deleted → need full scan to detect which ones
|
||||
scan_type = "full_deletions"
|
||||
logger.info(
|
||||
f"File count decreased ({project.last_file_count} → {current_count}), "
|
||||
f"running full scan to detect deletions"
|
||||
)
|
||||
scan_coro = self._scan_directory_full(directory)
|
||||
|
||||
elif project.last_scan_timestamp is not None:
|
||||
# Incremental scan: only files modified since last scan
|
||||
scan_type = "incremental"
|
||||
logger.debug(
|
||||
f"Running incremental scan for files modified since {project.last_scan_timestamp}"
|
||||
)
|
||||
file_paths_to_scan = await self._scan_directory_modified_since(
|
||||
directory, project.last_scan_timestamp
|
||||
)
|
||||
logger.debug(
|
||||
f"Incremental scan found {len(file_paths_to_scan)} potentially changed files"
|
||||
)
|
||||
elif project.last_scan_timestamp is not None:
|
||||
# Incremental scan: only files modified since last scan
|
||||
scan_type = "incremental"
|
||||
logger.debug(
|
||||
f"Running incremental scan for files modified since {project.last_scan_timestamp}"
|
||||
)
|
||||
scan_coro = self._scan_directory_modified_since(
|
||||
directory, project.last_scan_timestamp
|
||||
)
|
||||
|
||||
else:
|
||||
# Fallback to full scan (no watermark available)
|
||||
scan_type = "full_fallback"
|
||||
logger.warning("No scan watermark available, falling back to full scan")
|
||||
file_paths_to_scan = await self._scan_directory_full(directory)
|
||||
|
||||
# Step 3: Process each file with mtime-based comparison
|
||||
scanned_paths: Set[str] = set()
|
||||
changed_checksums: Dict[str, str] = {}
|
||||
|
||||
logger.debug(f"Processing {len(file_paths_to_scan)} files with mtime-based comparison")
|
||||
|
||||
for rel_path in file_paths_to_scan:
|
||||
scanned_paths.add(rel_path)
|
||||
|
||||
# Get file stats
|
||||
abs_path = directory / rel_path
|
||||
if not abs_path.exists():
|
||||
# File was deleted between scan and now (race condition)
|
||||
continue
|
||||
|
||||
stat_info = abs_path.stat()
|
||||
|
||||
# Indexed lookup - single file query (not full table scan)
|
||||
db_entity = await self.entity_repository.get_by_file_path(rel_path)
|
||||
|
||||
if db_entity is None:
|
||||
# New file - need checksum for move detection
|
||||
checksum = await self.file_service.compute_checksum(rel_path)
|
||||
report.new.add(rel_path)
|
||||
changed_checksums[rel_path] = checksum
|
||||
logger.trace(f"New file detected: {rel_path}")
|
||||
continue
|
||||
|
||||
# File exists in DB - check if mtime/size changed
|
||||
db_mtime = db_entity.mtime
|
||||
db_size = db_entity.size
|
||||
fs_mtime = stat_info.st_mtime
|
||||
fs_size = stat_info.st_size
|
||||
|
||||
# Compare mtime and size (like rsync/rclone)
|
||||
# Allow small epsilon for float comparison (0.01s = 10ms)
|
||||
mtime_changed = db_mtime is None or abs(fs_mtime - db_mtime) > 0.01
|
||||
size_changed = db_size is None or fs_size != db_size
|
||||
|
||||
if mtime_changed or size_changed:
|
||||
# File modified - compute checksum
|
||||
checksum = await self.file_service.compute_checksum(rel_path)
|
||||
db_checksum = db_entity.checksum
|
||||
|
||||
# Only mark as modified if checksum actually differs
|
||||
# (handles cases where mtime changed but content didn't, e.g., git operations)
|
||||
if checksum != db_checksum:
|
||||
report.modified.add(rel_path)
|
||||
changed_checksums[rel_path] = checksum
|
||||
logger.trace(
|
||||
f"Modified file detected: {rel_path}, "
|
||||
f"mtime_changed={mtime_changed}, size_changed={size_changed}"
|
||||
)
|
||||
else:
|
||||
# File unchanged - no checksum needed
|
||||
logger.trace(f"File unchanged (mtime/size match): {rel_path}")
|
||||
# Fallback to full scan (no watermark available)
|
||||
scan_type = "full_fallback"
|
||||
logger.warning("No scan watermark available, falling back to full scan")
|
||||
scan_coro = self._scan_directory_full(directory)
|
||||
|
||||
# Step 4: Detect moves (for both full and incremental scans)
|
||||
# Check if any "new" files are actually moves by matching checksums
|
||||
for new_path in list(report.new): # Use list() to allow modification during iteration
|
||||
new_checksum = changed_checksums.get(new_path)
|
||||
if not new_checksum:
|
||||
continue
|
||||
with telemetry.scope("sync.project.filesystem_scan", scan_type=scan_type):
|
||||
file_paths_to_scan = await scan_coro
|
||||
if scan_type == "incremental":
|
||||
logger.debug(
|
||||
f"Incremental scan found {len(file_paths_to_scan)} potentially changed files"
|
||||
)
|
||||
|
||||
# Look for existing entity with same checksum but different path
|
||||
# This could be a move or a copy
|
||||
existing_entities = await self.entity_repository.find_by_checksum(new_checksum)
|
||||
# Step 3: Process each file with mtime-based comparison
|
||||
scanned_paths: Set[str] = set()
|
||||
changed_checksums: Dict[str, str] = {}
|
||||
|
||||
for candidate in existing_entities:
|
||||
if candidate.file_path == new_path:
|
||||
# Same path, skip (shouldn't happen for "new" files but be safe)
|
||||
logger.debug(f"Processing {len(file_paths_to_scan)} files with mtime-based comparison")
|
||||
|
||||
for rel_path in file_paths_to_scan:
|
||||
scanned_paths.add(rel_path)
|
||||
|
||||
# Get file stats
|
||||
abs_path = directory / rel_path
|
||||
if not abs_path.exists():
|
||||
# File was deleted between scan and now (race condition)
|
||||
continue
|
||||
|
||||
# Check if the old path still exists on disk
|
||||
old_path_abs = directory / candidate.file_path
|
||||
if old_path_abs.exists():
|
||||
# Original still exists → this is a copy, not a move
|
||||
logger.trace(
|
||||
f"File copy detected (not move): {candidate.file_path} copied to {new_path}"
|
||||
)
|
||||
stat_info = abs_path.stat()
|
||||
|
||||
# Indexed lookup - single file query (not full table scan)
|
||||
db_entity = await self.entity_repository.get_by_file_path(rel_path)
|
||||
|
||||
if db_entity is None:
|
||||
# New file - need checksum for move detection
|
||||
checksum = await self.file_service.compute_checksum(rel_path)
|
||||
report.new.add(rel_path)
|
||||
changed_checksums[rel_path] = checksum
|
||||
logger.trace(f"New file detected: {rel_path}")
|
||||
continue
|
||||
|
||||
# Original doesn't exist → this is a move!
|
||||
report.moves[candidate.file_path] = new_path
|
||||
report.new.remove(new_path)
|
||||
logger.trace(f"Move detected: {candidate.file_path} -> {new_path}")
|
||||
break # Only match first candidate
|
||||
# File exists in DB - check if mtime/size changed
|
||||
db_mtime = db_entity.mtime
|
||||
db_size = db_entity.size
|
||||
fs_mtime = stat_info.st_mtime
|
||||
fs_size = stat_info.st_size
|
||||
|
||||
# Step 5: Detect deletions (only for full scans)
|
||||
# Incremental scans can't reliably detect deletions since they only see modified files
|
||||
if scan_type in ("full_initial", "full_deletions", "full_fallback", "full_forced"):
|
||||
# Use optimized query for just file paths (not full entities)
|
||||
db_file_paths = await self.entity_repository.get_all_file_paths()
|
||||
logger.debug(f"Found {len(db_file_paths)} db paths for deletion detection")
|
||||
# Compare mtime and size (like rsync/rclone)
|
||||
# Allow small epsilon for float comparison (0.01s = 10ms)
|
||||
mtime_changed = db_mtime is None or abs(fs_mtime - db_mtime) > 0.01
|
||||
size_changed = db_size is None or fs_size != db_size
|
||||
|
||||
for db_path in db_file_paths:
|
||||
if db_path not in scanned_paths:
|
||||
# File in DB but not on filesystem
|
||||
# Check if it was already detected as a move
|
||||
if db_path in report.moves:
|
||||
# Already handled as a move, skip
|
||||
if mtime_changed or size_changed:
|
||||
# File modified - compute checksum
|
||||
checksum = await self.file_service.compute_checksum(rel_path)
|
||||
db_checksum = db_entity.checksum
|
||||
|
||||
# Only mark as modified if checksum actually differs
|
||||
# (handles cases where mtime changed but content didn't, e.g., git operations)
|
||||
if checksum != db_checksum:
|
||||
report.modified.add(rel_path)
|
||||
changed_checksums[rel_path] = checksum
|
||||
logger.trace(
|
||||
f"Modified file detected: {rel_path}, "
|
||||
f"mtime_changed={mtime_changed}, size_changed={size_changed}"
|
||||
)
|
||||
else:
|
||||
# File unchanged - no checksum needed
|
||||
logger.trace(f"File unchanged (mtime/size match): {rel_path}")
|
||||
|
||||
# Step 4: Detect moves (for both full and incremental scans)
|
||||
# Check if any "new" files are actually moves by matching checksums
|
||||
with telemetry.scope("sync.project.detect_moves", new_count=len(report.new)):
|
||||
for new_path in list(
|
||||
report.new
|
||||
): # Use list() to allow modification during iteration
|
||||
new_checksum = changed_checksums.get(new_path)
|
||||
if not new_checksum:
|
||||
continue
|
||||
|
||||
# File was deleted
|
||||
report.deleted.add(db_path)
|
||||
logger.trace(f"Deleted file detected: {db_path}")
|
||||
# Look for existing entity with same checksum but different path
|
||||
# This could be a move or a copy
|
||||
existing_entities = await self.entity_repository.find_by_checksum(new_checksum)
|
||||
|
||||
# Store checksums for files that need syncing
|
||||
report.checksums = changed_checksums
|
||||
for candidate in existing_entities:
|
||||
if candidate.file_path == new_path:
|
||||
# Same path, skip (shouldn't happen for "new" files but be safe)
|
||||
continue
|
||||
|
||||
scan_duration_ms = int((time.time() - scan_start_time) * 1000)
|
||||
# Check if the old path still exists on disk
|
||||
old_path_abs = directory / candidate.file_path
|
||||
if old_path_abs.exists():
|
||||
# Original still exists → this is a copy, not a move
|
||||
logger.trace(
|
||||
f"File copy detected (not move): {candidate.file_path} copied to {new_path}"
|
||||
)
|
||||
continue
|
||||
|
||||
logger.info(
|
||||
f"Completed {scan_type} scan for directory {directory} in {scan_duration_ms}ms, "
|
||||
f"found {report.total} changes (new={len(report.new)}, "
|
||||
f"modified={len(report.modified)}, deleted={len(report.deleted)}, "
|
||||
f"moves={len(report.moves)})"
|
||||
)
|
||||
return report
|
||||
# Original doesn't exist → this is a move!
|
||||
report.moves[candidate.file_path] = new_path
|
||||
report.new.remove(new_path)
|
||||
logger.trace(f"Move detected: {candidate.file_path} -> {new_path}")
|
||||
break # Only match first candidate
|
||||
|
||||
# Step 5: Detect deletions (only for full scans)
|
||||
# Incremental scans can't reliably detect deletions since they only see modified files
|
||||
if scan_type in ("full_initial", "full_deletions", "full_fallback", "full_forced"):
|
||||
with telemetry.scope("sync.project.detect_deletions", scan_type=scan_type):
|
||||
# Use optimized query for just file paths (not full entities)
|
||||
db_file_paths = await self.entity_repository.get_all_file_paths()
|
||||
logger.debug(f"Found {len(db_file_paths)} db paths for deletion detection")
|
||||
|
||||
for db_path in db_file_paths:
|
||||
if db_path not in scanned_paths:
|
||||
# File in DB but not on filesystem
|
||||
# Check if it was already detected as a move
|
||||
if db_path in report.moves:
|
||||
# Already handled as a move, skip
|
||||
continue
|
||||
|
||||
# File was deleted
|
||||
report.deleted.add(db_path)
|
||||
logger.trace(f"Deleted file detected: {db_path}")
|
||||
|
||||
# Store checksums for files that need syncing
|
||||
report.checksums = changed_checksums
|
||||
|
||||
scan_duration_ms = int((time.time() - scan_start_time) * 1000)
|
||||
|
||||
logger.info(
|
||||
f"Completed {scan_type} scan for directory {directory} in {scan_duration_ms}ms, "
|
||||
f"found {report.total} changes (new={len(report.new)}, "
|
||||
f"modified={len(report.modified)}, deleted={len(report.deleted)}, "
|
||||
f"moves={len(report.moves)})"
|
||||
)
|
||||
return report
|
||||
|
||||
async def sync_file(
|
||||
self, path: str, new: bool = True
|
||||
@@ -616,12 +652,14 @@ class SyncService:
|
||||
logger.warning(f"Skipping file due to repeated failures: {path}")
|
||||
return None, None
|
||||
|
||||
try:
|
||||
logger.debug(
|
||||
f"Syncing file path={path} is_new={new} is_markdown={self.file_service.is_markdown(path)}"
|
||||
)
|
||||
start_time = time.time()
|
||||
is_markdown = self.file_service.is_markdown(path)
|
||||
file_kind = "markdown" if is_markdown else "regular"
|
||||
|
||||
if self.file_service.is_markdown(path):
|
||||
try:
|
||||
logger.debug(f"Syncing file path={path} is_new={new} is_markdown={is_markdown}")
|
||||
|
||||
if is_markdown:
|
||||
entity, checksum = await self.sync_markdown_file(path, new)
|
||||
else:
|
||||
entity, checksum = await self.sync_regular_file(path, new)
|
||||
@@ -646,33 +684,63 @@ class SyncService:
|
||||
logger.debug(
|
||||
f"File sync completed, path={path}, entity_id={entity.id}, checksum={checksum[:8]}"
|
||||
)
|
||||
duration_ms = int((time.time() - start_time) * 1000)
|
||||
if duration_ms >= SLOW_FILE_SYNC_WARNING_MS:
|
||||
logger.warning(
|
||||
f"Slow file sync detected: path={path}, file_kind={file_kind}, duration_ms={duration_ms}"
|
||||
)
|
||||
return entity, checksum
|
||||
|
||||
except FileNotFoundError:
|
||||
# File exists in database but not on filesystem
|
||||
# This indicates a database/filesystem inconsistency - treat as deletion
|
||||
logger.warning(
|
||||
f"File not found during sync, treating as deletion: path={path}. "
|
||||
"This may indicate a race condition or manual file deletion."
|
||||
)
|
||||
await self.handle_delete(path)
|
||||
with telemetry.scope(
|
||||
"sync.file.failure",
|
||||
failure_type="file_not_found",
|
||||
path=path,
|
||||
file_kind=file_kind,
|
||||
is_new=new,
|
||||
is_fatal=False,
|
||||
):
|
||||
logger.warning(
|
||||
f"File not found during sync, treating as deletion: path={path}. "
|
||||
"This may indicate a race condition or manual file deletion."
|
||||
)
|
||||
await self.handle_delete(path)
|
||||
return None, None
|
||||
|
||||
except Exception as e:
|
||||
failure_type = type(e).__name__
|
||||
# Check if this is a fatal error (or caused by one)
|
||||
# Fatal errors like project deletion should terminate sync immediately
|
||||
if isinstance(e, SyncFatalError) or isinstance(
|
||||
e.__cause__, SyncFatalError
|
||||
): # pragma: no cover
|
||||
logger.error(f"Fatal sync error encountered, terminating sync: path={path}")
|
||||
with telemetry.scope(
|
||||
"sync.file.failure",
|
||||
failure_type=failure_type,
|
||||
path=path,
|
||||
file_kind=file_kind,
|
||||
is_new=new,
|
||||
is_fatal=True,
|
||||
):
|
||||
logger.error(f"Fatal sync error encountered, terminating sync: path={path}")
|
||||
raise
|
||||
|
||||
# Otherwise treat as recoverable file-level error
|
||||
error_msg = str(e)
|
||||
logger.error(f"Failed to sync file: path={path}, error={error_msg}")
|
||||
with telemetry.scope(
|
||||
"sync.file.failure",
|
||||
failure_type=failure_type,
|
||||
path=path,
|
||||
file_kind=file_kind,
|
||||
is_new=new,
|
||||
is_fatal=False,
|
||||
):
|
||||
logger.error(f"Failed to sync file: path={path}, error={error_msg}")
|
||||
|
||||
# Record failure for circuit breaker
|
||||
await self._record_failure(path, error_msg)
|
||||
# Record failure for circuit breaker
|
||||
await self._record_failure(path, error_msg)
|
||||
|
||||
return None, None
|
||||
|
||||
@@ -1066,24 +1134,36 @@ class SyncService:
|
||||
# update search index only on successful resolution
|
||||
await self.search_service.index_entity(resolved_entity)
|
||||
except IntegrityError:
|
||||
# IntegrityError means a relation with this (from_id, to_id, relation_type)
|
||||
# already exists. The UPDATE was rolled back, so our unresolved relation
|
||||
# (to_id=NULL) still exists in the database. We delete it because:
|
||||
# 1. It's redundant - a resolved relation already captures this relationship
|
||||
# 2. If we don't delete it, future syncs will try to resolve it again
|
||||
# and get the same IntegrityError
|
||||
logger.debug(
|
||||
"Deleting duplicate unresolved relation "
|
||||
f"relation_id={relation.id} "
|
||||
f"from_id={relation.from_id} "
|
||||
f"to_name={relation.to_name} "
|
||||
f"resolved_to_id={resolved_entity.id}"
|
||||
)
|
||||
try:
|
||||
await self.relation_repository.delete(relation.id)
|
||||
except Exception as e:
|
||||
# Log but don't fail - the relation may have been deleted already
|
||||
logger.debug(f"Could not delete duplicate relation {relation.id}: {e}")
|
||||
with telemetry.scope(
|
||||
"sync.relation.resolve_conflict",
|
||||
relation_id=relation.id,
|
||||
relation_type=relation.relation_type,
|
||||
):
|
||||
# IntegrityError means a relation with this (from_id, to_id, relation_type)
|
||||
# already exists. The UPDATE was rolled back, so our unresolved relation
|
||||
# (to_id=NULL) still exists in the database. We delete it because:
|
||||
# 1. It's redundant - a resolved version already captures this relationship
|
||||
# 2. If we don't delete it, future syncs will try to resolve it again
|
||||
# and get the same IntegrityError
|
||||
logger.debug(
|
||||
"Deleting duplicate unresolved relation "
|
||||
f"relation_id={relation.id} "
|
||||
f"from_id={relation.from_id} "
|
||||
f"to_name={relation.to_name} "
|
||||
f"resolved_to_id={resolved_entity.id}"
|
||||
)
|
||||
try:
|
||||
await self.relation_repository.delete(relation.id)
|
||||
except Exception as e:
|
||||
with telemetry.scope(
|
||||
"sync.relation.cleanup_failure",
|
||||
relation_id=relation.id,
|
||||
relation_type=relation.relation_type,
|
||||
):
|
||||
# Log but don't fail - the relation may have been deleted already
|
||||
logger.debug(
|
||||
f"Could not delete duplicate relation {relation.id}: {e}"
|
||||
)
|
||||
|
||||
async def _quick_count_files(self, directory: Path) -> int:
|
||||
"""Fast file count using find command.
|
||||
|
||||
@@ -0,0 +1,189 @@
|
||||
"""Optional Logfire telemetry helpers for Basic Memory.
|
||||
|
||||
Telemetry is disabled by default. When enabled, this module configures Logfire,
|
||||
exposes a `loguru` handler for trace-aware logging, and provides lightweight
|
||||
helpers for manual spans and logger context binding.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Iterator
|
||||
|
||||
from loguru import logger
|
||||
|
||||
REPOSITORY_URL = "https://github.com/basicmachines-co/basic-memory"
|
||||
ROOT_PATH = "src/basic_memory"
|
||||
|
||||
|
||||
def _load_logfire() -> Any | None:
|
||||
"""Load the optional logfire dependency lazily."""
|
||||
try:
|
||||
import logfire
|
||||
except ImportError:
|
||||
return None
|
||||
return logfire
|
||||
|
||||
|
||||
@dataclass
|
||||
class TelemetryState:
|
||||
"""Process-local Logfire configuration state."""
|
||||
|
||||
enabled: bool = False
|
||||
configured: bool = False
|
||||
service_name: str | None = None
|
||||
environment: str | None = None
|
||||
send_to_logfire: bool = False
|
||||
warnings: list[str] = field(default_factory=list)
|
||||
|
||||
|
||||
_STATE = TelemetryState()
|
||||
_LOGFIRE_HANDLER: dict[str, Any] | None = None
|
||||
|
||||
|
||||
def reset_telemetry_state() -> None:
|
||||
"""Reset process-local telemetry state.
|
||||
|
||||
Primarily used by tests.
|
||||
"""
|
||||
global _LOGFIRE_HANDLER
|
||||
_STATE.enabled = False
|
||||
_STATE.configured = False
|
||||
_STATE.service_name = None
|
||||
_STATE.environment = None
|
||||
_STATE.send_to_logfire = False
|
||||
_STATE.warnings.clear()
|
||||
_LOGFIRE_HANDLER = None
|
||||
|
||||
|
||||
def _filter_attributes(attrs: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Drop null attributes so span and log payloads stay compact."""
|
||||
return {key: value for key, value in attrs.items() if value is not None}
|
||||
|
||||
|
||||
def configure_telemetry(
|
||||
service_name: str,
|
||||
*,
|
||||
environment: str,
|
||||
service_version: str | None = None,
|
||||
enable_logfire: bool = False,
|
||||
send_to_logfire: bool = False,
|
||||
log_level: str = "INFO",
|
||||
) -> bool:
|
||||
"""Configure optional Logfire instrumentation for the current process."""
|
||||
global _LOGFIRE_HANDLER
|
||||
|
||||
reset_telemetry_state()
|
||||
_STATE.service_name = service_name
|
||||
_STATE.environment = environment
|
||||
_STATE.send_to_logfire = send_to_logfire
|
||||
_STATE.enabled = enable_logfire
|
||||
|
||||
if not enable_logfire:
|
||||
return False
|
||||
|
||||
logfire = _load_logfire()
|
||||
if logfire is None:
|
||||
_STATE.enabled = False
|
||||
_STATE.warnings.append(
|
||||
"Logfire telemetry was enabled but the 'logfire' package is not installed. "
|
||||
"Telemetry remains disabled."
|
||||
)
|
||||
return False
|
||||
|
||||
configure_kwargs = {
|
||||
"service_name": service_name,
|
||||
"environment": environment,
|
||||
"code_source": logfire.CodeSource(
|
||||
repository=REPOSITORY_URL,
|
||||
revision=service_version or "",
|
||||
root_path=ROOT_PATH,
|
||||
),
|
||||
"min_level": log_level.lower(),
|
||||
"send_to_logfire": send_to_logfire,
|
||||
}
|
||||
|
||||
try:
|
||||
logfire.configure(**configure_kwargs)
|
||||
except TypeError:
|
||||
configure_kwargs.pop("send_to_logfire", None)
|
||||
logfire.configure(**configure_kwargs)
|
||||
except Exception as exc: # pragma: no cover
|
||||
_STATE.enabled = False # pragma: no cover
|
||||
_STATE.warnings.append(f"Failed to configure Logfire telemetry: {exc}") # pragma: no cover
|
||||
return False # pragma: no cover
|
||||
|
||||
_LOGFIRE_HANDLER = logfire.loguru_handler()
|
||||
_STATE.configured = True
|
||||
return True
|
||||
|
||||
|
||||
def telemetry_enabled() -> bool:
|
||||
"""Return True when telemetry is both enabled and configured."""
|
||||
return _STATE.enabled and _STATE.configured
|
||||
|
||||
|
||||
def get_logfire_handler() -> dict[str, Any] | None:
|
||||
"""Return the active Logfire `loguru` handler, if any."""
|
||||
return _LOGFIRE_HANDLER
|
||||
|
||||
|
||||
def pop_telemetry_warnings() -> list[str]:
|
||||
"""Return and clear pending telemetry warnings."""
|
||||
warnings = list(_STATE.warnings)
|
||||
_STATE.warnings.clear()
|
||||
return warnings
|
||||
|
||||
|
||||
@contextmanager
|
||||
def contextualize(**attrs: Any) -> Iterator[None]:
|
||||
"""Apply filtered telemetry attributes to Loguru calls in this scope."""
|
||||
with logger.contextualize(**_filter_attributes(attrs)):
|
||||
yield
|
||||
|
||||
|
||||
@contextmanager
|
||||
def scope(name: str, **attrs: Any) -> Iterator[None]:
|
||||
"""Create a span and bind the same stable attributes into Loguru context."""
|
||||
with contextualize(**attrs):
|
||||
with span(name, **attrs):
|
||||
yield
|
||||
|
||||
|
||||
# Alias: `operation` signals a root-level boundary (entrypoint, tool invocation),
|
||||
# while `scope` signals a nested phase. The distinction is convention only.
|
||||
operation = scope
|
||||
|
||||
|
||||
@contextmanager
|
||||
def span(name: str, **attrs: Any) -> Iterator[None]:
|
||||
"""Create a manual Logfire span when telemetry is enabled."""
|
||||
with started_span(name, **attrs):
|
||||
yield
|
||||
|
||||
|
||||
@contextmanager
|
||||
def started_span(name: str, **attrs: Any) -> Iterator[Any | None]:
|
||||
"""Create a manual Logfire span and expose the active span handle when available."""
|
||||
logfire = _load_logfire()
|
||||
if logfire is None or not _STATE.configured: # pragma: no cover
|
||||
yield # pragma: no cover
|
||||
return # pragma: no cover
|
||||
|
||||
with logfire.span(name, **_filter_attributes(attrs)) as active_span:
|
||||
yield active_span
|
||||
|
||||
|
||||
__all__ = [
|
||||
"contextualize",
|
||||
"configure_telemetry",
|
||||
"get_logfire_handler",
|
||||
"operation",
|
||||
"pop_telemetry_warnings",
|
||||
"reset_telemetry_state",
|
||||
"scope",
|
||||
"span",
|
||||
"started_span",
|
||||
"telemetry_enabled",
|
||||
]
|
||||
@@ -13,6 +13,8 @@ from typing import Any, Protocol, Union, runtime_checkable, List, Optional
|
||||
from loguru import logger
|
||||
from unidecode import unidecode
|
||||
|
||||
from basic_memory import telemetry
|
||||
|
||||
|
||||
def normalize_project_path(path: str) -> str:
|
||||
"""Normalize project path by stripping mount point prefix.
|
||||
@@ -300,6 +302,11 @@ def setup_logging(
|
||||
if log_to_stdout:
|
||||
logger.add(sys.stderr, level=log_level, backtrace=True, diagnose=True, colorize=True)
|
||||
|
||||
# Add Logfire sink when telemetry bootstrap enabled it for this process.
|
||||
logfire_handler = telemetry.get_logfire_handler()
|
||||
if logfire_handler is not None:
|
||||
logger.add(**logfire_handler)
|
||||
|
||||
# Bind structured context for cloud observability
|
||||
if structured_context:
|
||||
logger.configure(
|
||||
@@ -315,6 +322,9 @@ def setup_logging(
|
||||
logging.getLogger("httpx").setLevel(logging.WARNING)
|
||||
logging.getLogger("watchfiles.main").setLevel(logging.WARNING)
|
||||
|
||||
for warning_message in telemetry.pop_telemetry_warnings():
|
||||
logger.warning(warning_message)
|
||||
|
||||
|
||||
def _cleanup_windows_log_files(log_dir: Path, current_log_name: str) -> None:
|
||||
"""Trim stale per-process Windows log files so the directory stays bounded."""
|
||||
@@ -503,12 +513,23 @@ def valid_project_path_value(path: str):
|
||||
if not path:
|
||||
return True
|
||||
|
||||
# Check for obvious path traversal patterns first
|
||||
if ".." in path or "~" in path:
|
||||
# Check for tilde (home directory expansion)
|
||||
if "~" in path:
|
||||
return False
|
||||
|
||||
# Check for Windows-style path traversal (even on Unix systems)
|
||||
if "\\.." in path or path.startswith("\\"):
|
||||
# Check for ".." as a path segment (path traversal), not as a substring.
|
||||
# Filenames like "hi-everyone..md" are legitimate and must not be blocked.
|
||||
# Also block segments like ".. " and ".. ." because Windows normalizes
|
||||
# trailing dots and spaces away, making them equivalent to "..".
|
||||
segments = path.replace("\\", "/").split("/")
|
||||
if any(
|
||||
seg == ".." or (len(seg) > 2 and seg[:2] == ".." and all(c in ". " for c in seg[2:]))
|
||||
for seg in segments
|
||||
):
|
||||
return False
|
||||
|
||||
# Check for Windows-style leading backslash
|
||||
if path.startswith("\\"):
|
||||
return False
|
||||
|
||||
# Block absolute paths (Unix-style starting with / or Windows-style with drive letters)
|
||||
|
||||
+86
-25
@@ -51,7 +51,7 @@ The `app` fixture ensures FastAPI dependency overrides are active, and
|
||||
"""
|
||||
|
||||
import os
|
||||
from typing import AsyncGenerator, Literal
|
||||
from typing import AsyncGenerator, Generator, Literal
|
||||
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
@@ -63,7 +63,13 @@ from testcontainers.postgres import PostgresContainer
|
||||
|
||||
from httpx import AsyncClient, ASGITransport
|
||||
|
||||
from basic_memory.config import BasicMemoryConfig, ProjectConfig, ConfigManager, DatabaseBackend
|
||||
from basic_memory.config import (
|
||||
BasicMemoryConfig,
|
||||
ProjectConfig,
|
||||
ProjectEntry,
|
||||
ConfigManager,
|
||||
DatabaseBackend,
|
||||
)
|
||||
from basic_memory.db import engine_session_factory, DatabaseType
|
||||
from basic_memory.models import Project
|
||||
from basic_memory.models.base import Base
|
||||
@@ -103,7 +109,7 @@ def postgres_container(db_backend):
|
||||
Uses testcontainers to spin up a real Postgres instance.
|
||||
Only starts if db_backend is "postgres".
|
||||
"""
|
||||
if db_backend != "postgres":
|
||||
if db_backend != "postgres" or _configured_postgres_sync_url():
|
||||
yield None
|
||||
return
|
||||
|
||||
@@ -112,6 +118,70 @@ def postgres_container(db_backend):
|
||||
yield postgres
|
||||
|
||||
|
||||
POSTGRES_EPHEMERAL_TABLES = [
|
||||
"search_vector_embeddings",
|
||||
"search_vector_chunks",
|
||||
"search_vector_index",
|
||||
]
|
||||
|
||||
|
||||
def _configured_postgres_sync_url() -> str | None:
|
||||
"""Prefer an externally managed Postgres server when CI provides one."""
|
||||
configured_url = os.environ.get("BASIC_MEMORY_TEST_POSTGRES_URL") or os.environ.get(
|
||||
"POSTGRES_TEST_URL"
|
||||
)
|
||||
if not configured_url:
|
||||
return None
|
||||
|
||||
return (
|
||||
configured_url.replace("postgresql+asyncpg://", "postgresql+psycopg2://", 1)
|
||||
.replace("postgresql://", "postgresql+psycopg2://", 1)
|
||||
.replace("postgres://", "postgresql+psycopg2://", 1)
|
||||
)
|
||||
|
||||
|
||||
def _postgres_reset_tables() -> list[str]:
|
||||
"""Resolve the current ORM table set at reset time."""
|
||||
return [table.name for table in Base.metadata.sorted_tables] + ["search_index"]
|
||||
|
||||
|
||||
def _resolve_postgres_sync_url(postgres_container) -> str:
|
||||
"""Use CI's shared service when configured, otherwise fall back to testcontainers."""
|
||||
configured_url = _configured_postgres_sync_url()
|
||||
if configured_url:
|
||||
return configured_url
|
||||
assert postgres_container is not None
|
||||
return postgres_container.get_connection_url()
|
||||
|
||||
|
||||
async def _reset_postgres_integration_schema(engine) -> None:
|
||||
"""Restore the shared Postgres integration schema to a clean baseline."""
|
||||
from basic_memory.models.search import (
|
||||
CREATE_POSTGRES_SEARCH_INDEX_FTS,
|
||||
CREATE_POSTGRES_SEARCH_INDEX_METADATA,
|
||||
CREATE_POSTGRES_SEARCH_INDEX_PERMALINK,
|
||||
CREATE_POSTGRES_SEARCH_INDEX_TABLE,
|
||||
)
|
||||
|
||||
async with engine.begin() as conn:
|
||||
# Trigger: integration tests may leave behind temporary search/vector tables while
|
||||
# exercising full-stack recovery paths.
|
||||
# Why: recreating only the missing schema is much cheaper than dropping every table.
|
||||
# Outcome: each integration test gets the same baseline without paying repeated full DDL cost.
|
||||
await conn.run_sync(Base.metadata.create_all)
|
||||
await conn.execute(CREATE_POSTGRES_SEARCH_INDEX_TABLE)
|
||||
await conn.execute(CREATE_POSTGRES_SEARCH_INDEX_FTS)
|
||||
await conn.execute(CREATE_POSTGRES_SEARCH_INDEX_METADATA)
|
||||
await conn.execute(CREATE_POSTGRES_SEARCH_INDEX_PERMALINK)
|
||||
|
||||
for table_name in POSTGRES_EPHEMERAL_TABLES:
|
||||
await conn.execute(text(f"DROP TABLE IF EXISTS {table_name} CASCADE"))
|
||||
|
||||
await conn.execute(
|
||||
text(f"TRUNCATE TABLE {', '.join(_postgres_reset_tables())} RESTART IDENTITY CASCADE")
|
||||
)
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def engine_factory(
|
||||
app_config,
|
||||
@@ -121,18 +191,12 @@ async def engine_factory(
|
||||
tmp_path,
|
||||
) -> AsyncGenerator[tuple, None]:
|
||||
"""Create engine and session factory for the configured database backend."""
|
||||
from basic_memory.models.search import (
|
||||
CREATE_SEARCH_INDEX,
|
||||
CREATE_POSTGRES_SEARCH_INDEX_TABLE,
|
||||
CREATE_POSTGRES_SEARCH_INDEX_FTS,
|
||||
CREATE_POSTGRES_SEARCH_INDEX_METADATA,
|
||||
CREATE_POSTGRES_SEARCH_INDEX_PERMALINK,
|
||||
)
|
||||
from basic_memory.models.search import CREATE_SEARCH_INDEX
|
||||
from basic_memory import db
|
||||
|
||||
if db_backend == "postgres":
|
||||
# Postgres mode using testcontainers
|
||||
sync_url = postgres_container.get_connection_url()
|
||||
sync_url = _resolve_postgres_sync_url(postgres_container)
|
||||
async_url = sync_url.replace("postgresql+psycopg2", "postgresql+asyncpg")
|
||||
|
||||
engine = create_async_engine(
|
||||
@@ -153,16 +217,7 @@ async def engine_factory(
|
||||
db._engine = engine
|
||||
db._session_maker = session_maker
|
||||
|
||||
# Drop and recreate all tables for test isolation
|
||||
async with engine.begin() as conn:
|
||||
await conn.execute(text("DROP TABLE IF EXISTS search_index CASCADE"))
|
||||
await conn.run_sync(Base.metadata.drop_all)
|
||||
await conn.run_sync(Base.metadata.create_all)
|
||||
# asyncpg requires separate execute calls for each statement
|
||||
await conn.execute(CREATE_POSTGRES_SEARCH_INDEX_TABLE)
|
||||
await conn.execute(CREATE_POSTGRES_SEARCH_INDEX_FTS)
|
||||
await conn.execute(CREATE_POSTGRES_SEARCH_INDEX_METADATA)
|
||||
await conn.execute(CREATE_POSTGRES_SEARCH_INDEX_PERMALINK)
|
||||
await _reset_postgres_integration_schema(engine)
|
||||
|
||||
yield engine, session_maker
|
||||
|
||||
@@ -228,13 +283,15 @@ def app_config(
|
||||
monkeypatch.setenv("BASIC_MEMORY_CLOUD_MODE", "false")
|
||||
|
||||
# Create a basic config with test-project like unit tests do
|
||||
projects = {"test-project": str(config_home)}
|
||||
projects = {"test-project": ProjectEntry(path=str(config_home))}
|
||||
|
||||
# Configure database backend based on env var
|
||||
if db_backend == "postgres":
|
||||
database_backend = DatabaseBackend.POSTGRES
|
||||
# Get URL from testcontainer and convert to asyncpg driver
|
||||
sync_url = postgres_container.get_connection_url()
|
||||
# Trigger: CI jobs can provide a shared Postgres service instead of per-session containers.
|
||||
# Why: reusing one pgvector-enabled server avoids Docker startup churn on every job.
|
||||
# Outcome: local runs keep using testcontainers, while CI injects a stable service URL.
|
||||
sync_url = _resolve_postgres_sync_url(postgres_container)
|
||||
database_url = sync_url.replace("postgresql+psycopg2", "postgresql+asyncpg")
|
||||
else:
|
||||
database_backend = DatabaseBackend.SQLITE
|
||||
@@ -258,6 +315,8 @@ def config_manager(app_config: BasicMemoryConfig, config_home) -> ConfigManager:
|
||||
from basic_memory import config as config_module
|
||||
|
||||
config_module._CONFIG_CACHE = None
|
||||
config_module._CONFIG_MTIME = None
|
||||
config_module._CONFIG_SIZE = None
|
||||
|
||||
config_manager = ConfigManager()
|
||||
# Update its paths to use the test directory
|
||||
@@ -283,7 +342,9 @@ def project_config(test_project):
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def app(app_config, project_config, engine_factory, test_project, config_manager) -> FastAPI:
|
||||
def app(
|
||||
app_config, project_config, engine_factory, test_project, config_manager
|
||||
) -> Generator[FastAPI, None, None]:
|
||||
"""Create test FastAPI application with single project."""
|
||||
|
||||
# Import the FastAPI app AFTER the config_manager has written the test config to disk
|
||||
|
||||
@@ -148,9 +148,7 @@ async def test_canvas_nodes_edges_as_string(mcp_server, app, test_project):
|
||||
"height": 100,
|
||||
}
|
||||
]
|
||||
edges = [
|
||||
{"id": "e1", "fromNode": "n1", "toNode": "n1", "label": "self"}
|
||||
]
|
||||
edges = [{"id": "e1", "fromNode": "n1", "toNode": "n1", "label": "self"}]
|
||||
|
||||
async with Client(mcp_server) as client:
|
||||
result = await client.call_tool(
|
||||
|
||||
@@ -137,6 +137,58 @@ async def test_get_entity_by_id(client: AsyncClient, test_graph, v2_project_url,
|
||||
assert entity.api_version == "v2"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_entity_by_id_allows_long_relation_type(
|
||||
client: AsyncClient,
|
||||
v2_project_url,
|
||||
relation_repository,
|
||||
):
|
||||
"""GET entity should not fail when stored relation_type exceeds 200 characters."""
|
||||
source_response = await client.post(
|
||||
f"{v2_project_url}/knowledge/entities",
|
||||
json={
|
||||
"title": "Long Relation Source",
|
||||
"directory": "test",
|
||||
"content": "Source entity content",
|
||||
},
|
||||
)
|
||||
assert source_response.status_code == 200
|
||||
source_entity = EntityResponseV2.model_validate(source_response.json())
|
||||
|
||||
target_response = await client.post(
|
||||
f"{v2_project_url}/knowledge/entities",
|
||||
json={
|
||||
"title": "Long Relation Target",
|
||||
"directory": "test",
|
||||
"content": "Target entity content",
|
||||
},
|
||||
)
|
||||
assert target_response.status_code == 200
|
||||
target_entity = EntityResponseV2.model_validate(target_response.json())
|
||||
|
||||
long_relation_type = (
|
||||
"**Architecture/efficiency concern:** "
|
||||
"the orchestration prompt expanded a short edge label into a full descriptive note "
|
||||
"that is much longer than 200 characters but should still serialize cleanly."
|
||||
)
|
||||
|
||||
await relation_repository.create(
|
||||
{
|
||||
"from_id": source_entity.id,
|
||||
"to_id": target_entity.id,
|
||||
"to_name": target_entity.title,
|
||||
"relation_type": long_relation_type,
|
||||
}
|
||||
)
|
||||
|
||||
response = await client.get(f"{v2_project_url}/knowledge/entities/{source_entity.external_id}")
|
||||
|
||||
assert response.status_code == 200
|
||||
entity = EntityResponseV2.model_validate(response.json())
|
||||
assert len(entity.relations) == 1
|
||||
assert entity.relations[0].relation_type == long_relation_type
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_entity_by_id_not_found(client: AsyncClient, v2_project_url):
|
||||
"""Test getting a non-existent entity by external_id returns 404."""
|
||||
@@ -303,6 +355,7 @@ async def test_update_entity_by_id(
|
||||
response = await client.put(
|
||||
f"{v2_project_url}/knowledge/entities/{original_external_id}",
|
||||
json=update_data,
|
||||
params={"fast": False},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
@@ -311,6 +364,8 @@ async def test_update_entity_by_id(
|
||||
# V2 update must return external_id field
|
||||
assert updated_entity.external_id is not None
|
||||
assert updated_entity.api_version == "v2"
|
||||
assert updated_entity.content is not None
|
||||
assert "Updated content via V2" in updated_entity.content
|
||||
|
||||
# Verify file was updated
|
||||
file_path = file_service.get_entity_path(updated_entity)
|
||||
@@ -480,6 +535,7 @@ async def test_edit_entity_by_id_append(
|
||||
response = await client.patch(
|
||||
f"{v2_project_url}/knowledge/entities/{original_external_id}",
|
||||
json=edit_data,
|
||||
params={"fast": False},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
@@ -488,6 +544,8 @@ async def test_edit_entity_by_id_append(
|
||||
# V2 patch must return external_id field
|
||||
assert edited_entity.external_id is not None
|
||||
assert edited_entity.api_version == "v2"
|
||||
assert edited_entity.content is not None
|
||||
assert "Appended content" in edited_entity.content
|
||||
|
||||
# Verify file has both original and appended content
|
||||
file_path = file_service.get_entity_path(edited_entity)
|
||||
|
||||
@@ -0,0 +1,245 @@
|
||||
"""Telemetry coverage for the v2 knowledge router."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
from contextlib import contextmanager
|
||||
from datetime import datetime, timezone
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
from fastapi import BackgroundTasks, Response
|
||||
|
||||
from basic_memory.schemas.base import Entity
|
||||
from basic_memory.schemas.request import EditEntityRequest
|
||||
|
||||
knowledge_router_module = importlib.import_module("basic_memory.api.v2.routers.knowledge_router")
|
||||
|
||||
|
||||
def _capture_spans():
|
||||
spans: list[tuple[str, dict]] = []
|
||||
|
||||
@contextmanager
|
||||
def fake_span(name: str, **attrs):
|
||||
spans.append((name, attrs))
|
||||
yield
|
||||
|
||||
return spans, fake_span
|
||||
|
||||
|
||||
def _fake_entity(*, external_id: str = "entity-123", file_path: str = "notes/test.md"):
|
||||
now = datetime.now(timezone.utc)
|
||||
return SimpleNamespace(
|
||||
external_id=external_id,
|
||||
id=1,
|
||||
title="Telemetry Entity",
|
||||
note_type="note",
|
||||
content_type="text/markdown",
|
||||
permalink="notes/test",
|
||||
file_path=file_path,
|
||||
entity_metadata=None,
|
||||
observations=[],
|
||||
relations=[],
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
created_by=None,
|
||||
last_updated_by=None,
|
||||
)
|
||||
|
||||
|
||||
def _assert_names_in_order(names: list[str], expected: list[str]) -> None:
|
||||
cursor = 0
|
||||
for expected_name in expected:
|
||||
cursor = names.index(expected_name, cursor) + 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_create_entity_emits_root_and_nested_spans(monkeypatch) -> None:
|
||||
spans, fake_span = _capture_spans()
|
||||
monkeypatch.setattr(knowledge_router_module.telemetry, "span", fake_span)
|
||||
|
||||
entity = _fake_entity()
|
||||
response_content = (
|
||||
"---\ntitle: Telemetry Entity\ntype: note\npermalink: notes/test\n---\n\ntelemetry content"
|
||||
)
|
||||
|
||||
class FakeEntityService:
|
||||
async def create_entity_with_content(self, data):
|
||||
return SimpleNamespace(
|
||||
entity=entity,
|
||||
content=response_content,
|
||||
search_content="telemetry content",
|
||||
)
|
||||
|
||||
class FakeSearchService:
|
||||
async def index_entity(self, entity, content=None):
|
||||
assert content == "telemetry content"
|
||||
return None
|
||||
|
||||
class FakeTaskScheduler:
|
||||
def schedule(self, *args, **kwargs):
|
||||
return None
|
||||
|
||||
class FakeFileService:
|
||||
async def read_file_content(self, path):
|
||||
raise AssertionError("non-fast create should not re-read file content")
|
||||
|
||||
result = await knowledge_router_module.create_entity(
|
||||
project_id="project-123",
|
||||
data=Entity(
|
||||
title="Telemetry Entity",
|
||||
directory="notes",
|
||||
note_type="note",
|
||||
content_type="text/markdown",
|
||||
content="telemetry content",
|
||||
),
|
||||
background_tasks=BackgroundTasks(),
|
||||
entity_service=FakeEntityService(),
|
||||
search_service=FakeSearchService(),
|
||||
task_scheduler=FakeTaskScheduler(),
|
||||
file_service=FakeFileService(),
|
||||
app_config=SimpleNamespace(semantic_search_enabled=False),
|
||||
fast=False,
|
||||
)
|
||||
|
||||
assert result.content == response_content
|
||||
_assert_names_in_order(
|
||||
[name for name, _ in spans],
|
||||
[
|
||||
"api.request.knowledge.create_entity",
|
||||
"api.knowledge.create_entity.write_entity",
|
||||
"api.knowledge.create_entity.search_index",
|
||||
"api.knowledge.create_entity.vector_sync",
|
||||
"api.knowledge.create_entity.read_content",
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_update_entity_emits_root_and_nested_spans(monkeypatch) -> None:
|
||||
spans, fake_span = _capture_spans()
|
||||
monkeypatch.setattr(knowledge_router_module.telemetry, "span", fake_span)
|
||||
|
||||
entity = _fake_entity()
|
||||
response_content = "---\ntitle: Telemetry Entity\ntype: note\npermalink: notes/test\n---\n\nupdated telemetry content"
|
||||
|
||||
class FakeEntityService:
|
||||
async def update_entity_with_content(self, existing, data):
|
||||
return SimpleNamespace(
|
||||
entity=entity,
|
||||
content=response_content,
|
||||
search_content="updated telemetry content",
|
||||
)
|
||||
|
||||
class FakeSearchService:
|
||||
async def index_entity(self, entity, content=None):
|
||||
assert content == "updated telemetry content"
|
||||
return None
|
||||
|
||||
class FakeEntityRepository:
|
||||
async def get_by_external_id(self, external_id):
|
||||
return entity
|
||||
|
||||
class FakeTaskScheduler:
|
||||
def schedule(self, *args, **kwargs):
|
||||
return None
|
||||
|
||||
class FakeFileService:
|
||||
async def read_file_content(self, path):
|
||||
raise AssertionError("non-fast update should not re-read file content")
|
||||
|
||||
response = Response()
|
||||
result = await knowledge_router_module.update_entity_by_id(
|
||||
data=Entity(
|
||||
title="Telemetry Entity",
|
||||
directory="notes",
|
||||
note_type="note",
|
||||
content_type="text/markdown",
|
||||
content="updated telemetry content",
|
||||
),
|
||||
response=response,
|
||||
background_tasks=BackgroundTasks(),
|
||||
project_id="project-123",
|
||||
entity_service=FakeEntityService(),
|
||||
search_service=FakeSearchService(),
|
||||
entity_repository=FakeEntityRepository(),
|
||||
task_scheduler=FakeTaskScheduler(),
|
||||
file_service=FakeFileService(),
|
||||
app_config=SimpleNamespace(semantic_search_enabled=False),
|
||||
entity_id=entity.external_id,
|
||||
fast=False,
|
||||
)
|
||||
|
||||
assert result.content == response_content
|
||||
_assert_names_in_order(
|
||||
[name for name, _ in spans],
|
||||
[
|
||||
"api.request.knowledge.update_entity",
|
||||
"api.knowledge.update_entity.load_entity",
|
||||
"api.knowledge.update_entity.write_entity",
|
||||
"api.knowledge.update_entity.search_index",
|
||||
"api.knowledge.update_entity.vector_sync",
|
||||
"api.knowledge.update_entity.read_content",
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_edit_entity_emits_root_and_nested_spans(monkeypatch) -> None:
|
||||
spans, fake_span = _capture_spans()
|
||||
monkeypatch.setattr(knowledge_router_module.telemetry, "span", fake_span)
|
||||
|
||||
entity = _fake_entity()
|
||||
response_content = "---\ntitle: Telemetry Entity\ntype: note\npermalink: notes/test\n---\n\nedited telemetry content"
|
||||
|
||||
class FakeEntityService:
|
||||
async def edit_entity_with_content(self, **kwargs):
|
||||
return SimpleNamespace(
|
||||
entity=entity,
|
||||
content=response_content,
|
||||
search_content="edited telemetry content",
|
||||
)
|
||||
|
||||
class FakeSearchService:
|
||||
async def index_entity(self, entity, content=None):
|
||||
assert content == "edited telemetry content"
|
||||
return None
|
||||
|
||||
class FakeEntityRepository:
|
||||
async def get_by_external_id(self, external_id):
|
||||
return entity
|
||||
|
||||
class FakeTaskScheduler:
|
||||
def schedule(self, *args, **kwargs):
|
||||
return None
|
||||
|
||||
class FakeFileService:
|
||||
async def read_file_content(self, path):
|
||||
raise AssertionError("non-fast edit should not re-read file content")
|
||||
|
||||
result = await knowledge_router_module.edit_entity_by_id(
|
||||
data=EditEntityRequest(operation="append", content="edited telemetry content"),
|
||||
background_tasks=BackgroundTasks(),
|
||||
project_id="project-123",
|
||||
entity_service=FakeEntityService(),
|
||||
search_service=FakeSearchService(),
|
||||
entity_repository=FakeEntityRepository(),
|
||||
task_scheduler=FakeTaskScheduler(),
|
||||
file_service=FakeFileService(),
|
||||
app_config=SimpleNamespace(semantic_search_enabled=False),
|
||||
entity_id=entity.external_id,
|
||||
fast=False,
|
||||
)
|
||||
|
||||
assert result.content == response_content
|
||||
_assert_names_in_order(
|
||||
[name for name, _ in spans],
|
||||
[
|
||||
"api.request.knowledge.edit_entity",
|
||||
"api.knowledge.edit_entity.load_entity",
|
||||
"api.knowledge.edit_entity.write_entity",
|
||||
"api.knowledge.edit_entity.search_index",
|
||||
"api.knowledge.edit_entity.vector_sync",
|
||||
"api.knowledge.edit_entity.read_content",
|
||||
],
|
||||
)
|
||||
@@ -0,0 +1,187 @@
|
||||
"""Tests for graph context hydration in to_graph_context().
|
||||
|
||||
Proves that recent-activity/build-context hydration batches entity lookups
|
||||
for entities, observations, and relations in a single repository call.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from basic_memory.api.v2.utils import to_graph_context
|
||||
from basic_memory.schemas.search import SearchItemType
|
||||
from basic_memory.services.context_service import (
|
||||
ContextMetadata,
|
||||
ContextResult as ServiceContextResult,
|
||||
ContextResultItem,
|
||||
ContextResultRow,
|
||||
)
|
||||
|
||||
|
||||
# --- Helpers ---
|
||||
|
||||
|
||||
def _make_entity(id: int, title: str, external_id: str) -> SimpleNamespace:
|
||||
return SimpleNamespace(id=id, title=title, external_id=external_id)
|
||||
|
||||
|
||||
def _make_row(*, type: str, id: int, root_id: int, **kwargs) -> ContextResultRow:
|
||||
now = kwargs.pop("created_at", datetime.now(timezone.utc))
|
||||
defaults = dict(
|
||||
title=f"Item {id}",
|
||||
permalink=f"notes/{id}",
|
||||
file_path=f"notes/{id}.md",
|
||||
depth=0,
|
||||
root_id=root_id,
|
||||
created_at=now,
|
||||
)
|
||||
defaults.update(kwargs)
|
||||
return ContextResultRow(type=type, id=id, **defaults)
|
||||
|
||||
|
||||
class SpyEntityRepository:
|
||||
"""Tracks batched ID lookups and returns entities from a preset map."""
|
||||
|
||||
def __init__(self, entities_by_id: dict[int, SimpleNamespace]):
|
||||
self.entities_by_id = entities_by_id
|
||||
self.calls: list[list[int]] = []
|
||||
|
||||
async def find_by_ids(self, ids: list[int]):
|
||||
self.calls.append(ids)
|
||||
return [self.entities_by_id[i] for i in ids if i in self.entities_by_id]
|
||||
|
||||
|
||||
# --- Single batch fetch (N+1 elimination) ---
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_to_graph_context_batches_entity_hydration_for_recent_activity():
|
||||
"""Mixed entity, observation, and relation items must hydrate in one lookup."""
|
||||
repo = SpyEntityRepository(
|
||||
{
|
||||
1: _make_entity(1, "Root", "ext-root"),
|
||||
2: _make_entity(2, "Child", "ext-child"),
|
||||
3: _make_entity(3, "Peer", "ext-peer"),
|
||||
}
|
||||
)
|
||||
now = datetime.now(timezone.utc)
|
||||
|
||||
root_entity = _make_row(
|
||||
type="entity",
|
||||
id=1,
|
||||
root_id=1,
|
||||
title="Root",
|
||||
permalink="notes/root",
|
||||
file_path="notes/root.md",
|
||||
created_at=now,
|
||||
)
|
||||
root_observation = _make_row(
|
||||
type="observation",
|
||||
id=10,
|
||||
root_id=1,
|
||||
title="fact: observed",
|
||||
permalink="notes/root/observations/fact/observed",
|
||||
file_path="notes/root.md",
|
||||
category="fact",
|
||||
content="observed",
|
||||
entity_id=1,
|
||||
created_at=now,
|
||||
)
|
||||
root_relation = _make_row(
|
||||
type="relation",
|
||||
id=20,
|
||||
root_id=1,
|
||||
title="links_to: Child",
|
||||
permalink="notes/root",
|
||||
file_path="notes/root.md",
|
||||
relation_type="links_to",
|
||||
from_id=1,
|
||||
to_id=2,
|
||||
depth=1,
|
||||
created_at=now,
|
||||
)
|
||||
child_observation = _make_row(
|
||||
type="observation",
|
||||
id=11,
|
||||
root_id=11,
|
||||
title="note: child update",
|
||||
permalink="notes/child/observations/note/update",
|
||||
file_path="notes/child.md",
|
||||
category="note",
|
||||
content="child update",
|
||||
entity_id=2,
|
||||
created_at=now,
|
||||
)
|
||||
peer_entity = _make_row(
|
||||
type="entity",
|
||||
id=3,
|
||||
root_id=11,
|
||||
title="Peer",
|
||||
permalink="notes/peer",
|
||||
file_path="notes/peer.md",
|
||||
depth=1,
|
||||
created_at=now,
|
||||
)
|
||||
|
||||
context = ServiceContextResult(
|
||||
results=[
|
||||
ContextResultItem(
|
||||
primary_result=root_entity,
|
||||
observations=[root_observation],
|
||||
related_results=[root_relation],
|
||||
),
|
||||
ContextResultItem(
|
||||
primary_result=child_observation,
|
||||
observations=[],
|
||||
related_results=[peer_entity],
|
||||
),
|
||||
],
|
||||
metadata=ContextMetadata(
|
||||
types=[
|
||||
SearchItemType.ENTITY,
|
||||
SearchItemType.OBSERVATION,
|
||||
SearchItemType.RELATION,
|
||||
],
|
||||
depth=1,
|
||||
primary_count=2,
|
||||
related_count=2,
|
||||
total_relations=1,
|
||||
total_observations=1,
|
||||
),
|
||||
)
|
||||
|
||||
graph = await to_graph_context(context, entity_repository=repo, page=1, page_size=10)
|
||||
|
||||
assert len(repo.calls) == 1, f"Expected 1 entity lookup, got {len(repo.calls)}"
|
||||
assert set(repo.calls[0]) == {1, 2, 3}
|
||||
|
||||
first_result = graph.results[0]
|
||||
assert first_result.primary_result.external_id == "ext-root"
|
||||
assert first_result.observations[0].entity_external_id == "ext-root"
|
||||
assert first_result.observations[0].title == "Root"
|
||||
|
||||
relation = first_result.related_results[0]
|
||||
assert relation.from_entity == "Root"
|
||||
assert relation.from_entity_external_id == "ext-root"
|
||||
assert relation.to_entity == "Child"
|
||||
assert relation.to_entity_external_id == "ext-child"
|
||||
|
||||
second_result = graph.results[1]
|
||||
assert second_result.primary_result.entity_external_id == "ext-child"
|
||||
assert second_result.primary_result.title == "Child"
|
||||
assert second_result.related_results[0].external_id == "ext-peer"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_to_graph_context_empty_results_skip_entity_lookup():
|
||||
"""An empty context result should not perform any entity hydration lookup."""
|
||||
repo = SpyEntityRepository({})
|
||||
context = ServiceContextResult(results=[], metadata=ContextMetadata(depth=1))
|
||||
|
||||
graph = await to_graph_context(context, entity_repository=repo)
|
||||
|
||||
assert repo.calls == []
|
||||
assert list(graph.results) == []
|
||||
@@ -0,0 +1,305 @@
|
||||
"""Tests for search result hydration in to_search_results().
|
||||
|
||||
Proves that the batch fetch eliminates N+1 queries and that
|
||||
entity ID lookups are correct across all result types.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from basic_memory.api.v2.utils import to_search_results
|
||||
from basic_memory.repository.search_index_row import SearchIndexRow
|
||||
|
||||
|
||||
# --- Helpers ---
|
||||
|
||||
|
||||
def _make_entity(id: int, permalink: str) -> SimpleNamespace:
|
||||
return SimpleNamespace(id=id, permalink=permalink)
|
||||
|
||||
|
||||
def _make_row(*, type: str, id: int, **kwargs) -> SearchIndexRow:
|
||||
now = datetime.now(timezone.utc)
|
||||
defaults = dict(
|
||||
project_id=1,
|
||||
file_path=f"notes/{id}.md",
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
score=1.0,
|
||||
title=f"Item {id}",
|
||||
permalink=f"notes/{id}",
|
||||
)
|
||||
defaults.update(kwargs)
|
||||
return SearchIndexRow(type=type, id=id, **defaults)
|
||||
|
||||
|
||||
class SpyEntityService:
|
||||
"""Tracks calls to get_entities_by_id and returns from a preset lookup."""
|
||||
|
||||
def __init__(self, entities_by_id: dict[int, SimpleNamespace]):
|
||||
self.entities_by_id = entities_by_id
|
||||
self.calls: list[list[int]] = []
|
||||
|
||||
async def get_entities_by_id(self, ids: list[int]):
|
||||
self.calls.append(ids)
|
||||
return [self.entities_by_id[i] for i in ids if i in self.entities_by_id]
|
||||
|
||||
|
||||
# --- Single batch fetch (N+1 elimination) ---
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_single_db_call_for_multiple_results():
|
||||
"""Multiple search results must trigger exactly one get_entities_by_id call."""
|
||||
service = SpyEntityService(
|
||||
{
|
||||
1: _make_entity(1, "notes/a"),
|
||||
2: _make_entity(2, "notes/b"),
|
||||
3: _make_entity(3, "notes/c"),
|
||||
}
|
||||
)
|
||||
results = [
|
||||
_make_row(type="entity", id=1, entity_id=1),
|
||||
_make_row(type="entity", id=2, entity_id=2),
|
||||
_make_row(type="entity", id=3, entity_id=3),
|
||||
]
|
||||
|
||||
await to_search_results(service, results)
|
||||
|
||||
assert len(service.calls) == 1, f"Expected 1 DB call, got {len(service.calls)}"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_no_db_call_for_empty_results():
|
||||
"""Empty result list should not make any DB call."""
|
||||
service = SpyEntityService({})
|
||||
|
||||
search_results = await to_search_results(service, [])
|
||||
|
||||
assert len(service.calls) == 0
|
||||
assert search_results == []
|
||||
|
||||
|
||||
# --- ID deduplication ---
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_deduplicates_entity_ids():
|
||||
"""Shared entity IDs across results should be fetched once, not per-result."""
|
||||
# entity_id=1 appears in all three results, from_id=1 overlaps with entity_id
|
||||
service = SpyEntityService(
|
||||
{
|
||||
1: _make_entity(1, "notes/shared"),
|
||||
2: _make_entity(2, "notes/target-a"),
|
||||
3: _make_entity(3, "notes/target-b"),
|
||||
}
|
||||
)
|
||||
results = [
|
||||
_make_row(type="relation", id=10, entity_id=1, from_id=1, to_id=2, relation_type="links"),
|
||||
_make_row(type="relation", id=11, entity_id=1, from_id=1, to_id=3, relation_type="links"),
|
||||
]
|
||||
|
||||
await to_search_results(service, results)
|
||||
|
||||
# Single call with deduplicated IDs: {1, 2, 3}
|
||||
assert len(service.calls) == 1
|
||||
fetched_ids = set(service.calls[0])
|
||||
assert fetched_ids == {1, 2, 3}
|
||||
|
||||
|
||||
# --- Correct entity-to-field mapping ---
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_entity_result_maps_permalink():
|
||||
"""Entity results should populate the 'entity' field with the entity's permalink."""
|
||||
service = SpyEntityService({5: _make_entity(5, "notes/my-entity")})
|
||||
results = [_make_row(type="entity", id=5, entity_id=5)]
|
||||
|
||||
search_results = await to_search_results(service, results)
|
||||
|
||||
assert len(search_results) == 1
|
||||
r = search_results[0]
|
||||
assert r.entity == "notes/my-entity"
|
||||
assert r.entity_id == 5
|
||||
assert r.from_entity is None
|
||||
assert r.to_entity is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_observation_result_maps_parent_entity():
|
||||
"""Observation results should populate 'entity' with the parent entity's permalink."""
|
||||
service = SpyEntityService({10: _make_entity(10, "notes/parent")})
|
||||
results = [_make_row(type="observation", id=20, entity_id=10)]
|
||||
|
||||
search_results = await to_search_results(service, results)
|
||||
|
||||
r = search_results[0]
|
||||
assert r.entity == "notes/parent"
|
||||
assert r.entity_id == 10
|
||||
assert r.observation_id == 20
|
||||
assert r.from_entity is None
|
||||
assert r.to_entity is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_relation_result_maps_from_and_to():
|
||||
"""Relation results should populate entity, from_entity, and to_entity correctly."""
|
||||
service = SpyEntityService(
|
||||
{
|
||||
1: _make_entity(1, "notes/parent"),
|
||||
2: _make_entity(2, "notes/source"),
|
||||
3: _make_entity(3, "notes/target"),
|
||||
}
|
||||
)
|
||||
results = [
|
||||
_make_row(
|
||||
type="relation",
|
||||
id=99,
|
||||
entity_id=1,
|
||||
from_id=2,
|
||||
to_id=3,
|
||||
relation_type="references",
|
||||
)
|
||||
]
|
||||
|
||||
search_results = await to_search_results(service, results)
|
||||
|
||||
r = search_results[0]
|
||||
assert r.entity == "notes/parent"
|
||||
assert r.from_entity == "notes/source"
|
||||
assert r.to_entity == "notes/target"
|
||||
assert r.relation_id == 99
|
||||
assert r.relation_type == "references"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_relation_with_distinct_entity_and_from_ids():
|
||||
"""When entity_id != from_id, from_entity must use from_id's permalink, not entity_id's.
|
||||
|
||||
This was a bug in the old positional-index code: entities[0] was used for both
|
||||
'entity' and 'from_entity', which was wrong when entity_id != from_id.
|
||||
"""
|
||||
service = SpyEntityService(
|
||||
{
|
||||
10: _make_entity(10, "notes/parent-entity"),
|
||||
20: _make_entity(20, "notes/actual-source"),
|
||||
30: _make_entity(30, "notes/target"),
|
||||
}
|
||||
)
|
||||
results = [
|
||||
_make_row(
|
||||
type="relation",
|
||||
id=50,
|
||||
entity_id=10,
|
||||
from_id=20,
|
||||
to_id=30,
|
||||
relation_type="derived_from",
|
||||
)
|
||||
]
|
||||
|
||||
search_results = await to_search_results(service, results)
|
||||
|
||||
r = search_results[0]
|
||||
# entity should be the parent entity (entity_id=10)
|
||||
assert r.entity == "notes/parent-entity"
|
||||
# from_entity must be from_id=20, NOT entity_id=10
|
||||
assert r.from_entity == "notes/actual-source"
|
||||
assert r.to_entity == "notes/target"
|
||||
|
||||
|
||||
# --- Mixed result types ---
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_mixed_result_types_single_fetch():
|
||||
"""A mix of entity, observation, and relation results should all hydrate in one fetch."""
|
||||
service = SpyEntityService(
|
||||
{
|
||||
1: _make_entity(1, "notes/entity-one"),
|
||||
2: _make_entity(2, "notes/entity-two"),
|
||||
3: _make_entity(3, "notes/entity-three"),
|
||||
}
|
||||
)
|
||||
results = [
|
||||
_make_row(type="entity", id=1, entity_id=1),
|
||||
_make_row(type="observation", id=10, entity_id=2, category="fact"),
|
||||
_make_row(type="relation", id=20, entity_id=1, from_id=1, to_id=3, relation_type="links"),
|
||||
]
|
||||
|
||||
search_results = await to_search_results(service, results)
|
||||
|
||||
# Single DB call
|
||||
assert len(service.calls) == 1
|
||||
|
||||
# Entity result
|
||||
assert search_results[0].entity == "notes/entity-one"
|
||||
assert search_results[0].entity_id == 1
|
||||
|
||||
# Observation result
|
||||
assert search_results[1].entity == "notes/entity-two"
|
||||
assert search_results[1].observation_id == 10
|
||||
|
||||
# Relation result
|
||||
assert search_results[2].from_entity == "notes/entity-one"
|
||||
assert search_results[2].to_entity == "notes/entity-three"
|
||||
|
||||
|
||||
# --- Graceful handling of missing entities ---
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_missing_entity_returns_none_permalink():
|
||||
"""If an entity ID isn't found in the DB, permalink fields should be None."""
|
||||
# Only entity 1 exists; entity 99 (to_id) is missing
|
||||
service = SpyEntityService({1: _make_entity(1, "notes/source")})
|
||||
results = [
|
||||
_make_row(type="relation", id=5, entity_id=1, from_id=1, to_id=99, relation_type="links")
|
||||
]
|
||||
|
||||
search_results = await to_search_results(service, results)
|
||||
|
||||
r = search_results[0]
|
||||
assert r.entity == "notes/source"
|
||||
assert r.from_entity == "notes/source"
|
||||
assert r.to_entity is None # entity 99 not found
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_null_ids_handled_gracefully():
|
||||
"""Results with None entity_id/from_id/to_id should not cause errors."""
|
||||
service = SpyEntityService({})
|
||||
# Entity result: entity_id is the row id itself, from_id/to_id are None
|
||||
results = [_make_row(type="entity", id=1)]
|
||||
|
||||
search_results = await to_search_results(service, results)
|
||||
|
||||
# No entity_id on the row means no fetch needed, all fields None
|
||||
r = search_results[0]
|
||||
assert r.entity is None
|
||||
assert r.from_entity is None
|
||||
assert r.to_entity is None
|
||||
|
||||
|
||||
# --- Scaling: prove O(1) DB calls ---
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_single_db_call_scales_to_many_results():
|
||||
"""Even with many results, only one DB call should be made."""
|
||||
n = 50
|
||||
entities = {i: _make_entity(i, f"notes/e-{i}") for i in range(1, n + 1)}
|
||||
service = SpyEntityService(entities)
|
||||
results = [_make_row(type="entity", id=i, entity_id=i) for i in range(1, n + 1)]
|
||||
|
||||
search_results = await to_search_results(service, results)
|
||||
|
||||
assert len(service.calls) == 1, f"Expected 1 DB call for {n} results, got {len(service.calls)}"
|
||||
assert len(search_results) == n
|
||||
# Every result got its permalink
|
||||
for i, r in enumerate(search_results, start=1):
|
||||
assert r.entity == f"notes/e-{i}"
|
||||
@@ -0,0 +1,63 @@
|
||||
"""Telemetry coverage for the v2 search router."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
from contextlib import contextmanager
|
||||
|
||||
import pytest
|
||||
|
||||
from basic_memory.schemas.search import SearchQuery
|
||||
|
||||
search_router_module = importlib.import_module("basic_memory.api.v2.routers.search_router")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_search_router_wraps_request_in_manual_operation() -> None:
|
||||
operations: list[tuple[str, dict]] = []
|
||||
|
||||
class FakeSearchService:
|
||||
async def search(self, query, *, limit, offset):
|
||||
return []
|
||||
|
||||
@contextmanager
|
||||
def fake_operation(name: str, **attrs):
|
||||
operations.append((name, attrs))
|
||||
yield
|
||||
|
||||
async def fake_to_search_results(entity_service, results):
|
||||
return []
|
||||
|
||||
original_operation = search_router_module.telemetry.operation
|
||||
original_to_search_results = search_router_module.to_search_results
|
||||
search_router_module.telemetry.operation = fake_operation
|
||||
search_router_module.to_search_results = fake_to_search_results
|
||||
try:
|
||||
response = await search_router_module.search(
|
||||
SearchQuery(text="hello world"),
|
||||
FakeSearchService(),
|
||||
object(),
|
||||
project_id="project-123",
|
||||
page=2,
|
||||
page_size=5,
|
||||
)
|
||||
finally:
|
||||
search_router_module.telemetry.operation = original_operation
|
||||
search_router_module.to_search_results = original_to_search_results
|
||||
|
||||
assert response.current_page == 2
|
||||
assert operations == [
|
||||
(
|
||||
"api.request.search",
|
||||
{
|
||||
"entrypoint": "api",
|
||||
"domain": "search",
|
||||
"action": "search",
|
||||
"page": 2,
|
||||
"page_size": 5,
|
||||
"retrieval_mode": "fts",
|
||||
"has_query": True,
|
||||
"has_filters": False,
|
||||
},
|
||||
)
|
||||
]
|
||||
@@ -0,0 +1,66 @@
|
||||
"""Telemetry coverage for API v2 hydration utilities."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
from contextlib import contextmanager
|
||||
from datetime import datetime, timezone
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from basic_memory.repository.search_index_row import SearchIndexRow
|
||||
|
||||
utils_module = importlib.import_module("basic_memory.api.v2.utils")
|
||||
|
||||
|
||||
def _capture_spans():
|
||||
spans: list[tuple[str, dict]] = []
|
||||
|
||||
@contextmanager
|
||||
def fake_span(name: str, **attrs):
|
||||
spans.append((name, attrs))
|
||||
yield
|
||||
|
||||
return spans, fake_span
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_to_search_results_emits_hydration_spans(monkeypatch) -> None:
|
||||
spans, fake_span = _capture_spans()
|
||||
monkeypatch.setattr(utils_module.telemetry, "span", fake_span)
|
||||
|
||||
class FakeEntityService:
|
||||
async def get_entities_by_id(self, ids):
|
||||
return [
|
||||
SimpleNamespace(id=1, permalink="notes/root"),
|
||||
SimpleNamespace(id=2, permalink="notes/child"),
|
||||
]
|
||||
|
||||
now = datetime.now(timezone.utc)
|
||||
results = [
|
||||
SearchIndexRow(
|
||||
project_id=1,
|
||||
id=1,
|
||||
type="relation",
|
||||
file_path="notes/root.md",
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
permalink="notes/root/relates_to/notes/child",
|
||||
entity_id=1,
|
||||
from_id=1,
|
||||
to_id=2,
|
||||
relation_type="relates_to",
|
||||
title="Root relates to Child",
|
||||
score=1.0,
|
||||
)
|
||||
]
|
||||
|
||||
search_results = await utils_module.to_search_results(FakeEntityService(), results)
|
||||
|
||||
assert search_results[0].relation_type == "relates_to"
|
||||
assert [name for name, _ in spans] == [
|
||||
"search.hydrate_results",
|
||||
"search.hydrate_results.fetch_entities",
|
||||
"search.hydrate_results.shape_results",
|
||||
]
|
||||
@@ -10,10 +10,12 @@ from basic_memory.cli.commands.cloud.api_client import (
|
||||
make_api_request,
|
||||
)
|
||||
from basic_memory.cli.commands.cloud.cloud_utils import (
|
||||
CloudUtilsError,
|
||||
create_cloud_project,
|
||||
fetch_cloud_projects,
|
||||
project_exists,
|
||||
)
|
||||
from basic_memory.config import ProjectMode
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -163,3 +165,174 @@ async def test_cloud_utils_fetch_and_exists_and_create_project(
|
||||
assert created.new_project["name"] == "My Project"
|
||||
# Path should be permalink-like (kebab)
|
||||
assert seen["create_payload"]["path"] == "my-project"
|
||||
assert seen["create_payload"]["visibility"] == "workspace"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_create_cloud_project_accepts_visibility_override(config_home, config_manager):
|
||||
"""Shared cloud helper should pass explicit visibility through to the API payload."""
|
||||
config = config_manager.load_config()
|
||||
config.cloud_host = "https://cloud.example.test"
|
||||
config_manager.save_config(config)
|
||||
|
||||
seen_payload: dict | None = None
|
||||
|
||||
async def api_request(**kwargs):
|
||||
nonlocal seen_payload
|
||||
seen_payload = kwargs["json_data"]
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"message": "created",
|
||||
"status": "success",
|
||||
"default": False,
|
||||
"old_project": None,
|
||||
"new_project": {"name": "shared-project", "path": "shared-project"},
|
||||
},
|
||||
)
|
||||
|
||||
created = await create_cloud_project(
|
||||
"Shared Project",
|
||||
visibility="shared",
|
||||
api_request=api_request,
|
||||
)
|
||||
|
||||
assert created.new_project is not None
|
||||
assert seen_payload == {
|
||||
"name": "Shared Project",
|
||||
"path": "shared-project",
|
||||
"set_default": False,
|
||||
"visibility": "shared",
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cloud_utils_use_configured_workspace_headers(config_home, config_manager):
|
||||
"""Workspace-aware cloud helpers should prefer project workspace over global default."""
|
||||
config = config_manager.load_config()
|
||||
config.cloud_host = "https://cloud.example.test"
|
||||
config.default_workspace = "default-workspace"
|
||||
config.set_project_mode("alpha", ProjectMode.CLOUD)
|
||||
config.projects["alpha"].workspace_id = "project-workspace"
|
||||
config_manager.save_config(config)
|
||||
|
||||
seen: list[tuple[str, str | None]] = []
|
||||
|
||||
async def api_request(**kwargs):
|
||||
seen.append(
|
||||
(
|
||||
kwargs["method"],
|
||||
(kwargs.get("headers") or {}).get("X-Workspace-ID"),
|
||||
)
|
||||
)
|
||||
|
||||
if kwargs["method"] == "GET":
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"projects": [{"id": 1, "name": "alpha", "path": "alpha", "is_default": True}]
|
||||
},
|
||||
)
|
||||
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"message": "created",
|
||||
"status": "success",
|
||||
"default": False,
|
||||
"old_project": None,
|
||||
"new_project": {"name": "alpha", "path": "alpha"},
|
||||
},
|
||||
)
|
||||
|
||||
assert await project_exists("alpha", api_request=api_request) is True
|
||||
await create_cloud_project("alpha", api_request=api_request)
|
||||
await fetch_cloud_projects(project_name="missing", api_request=api_request)
|
||||
|
||||
assert seen == [
|
||||
("GET", "project-workspace"),
|
||||
("POST", "project-workspace"),
|
||||
("GET", "default-workspace"),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_project_exists_surfaces_cloud_lookup_failures(config_home, config_manager):
|
||||
"""project_exists should surface lookup failures instead of pretending the project is missing."""
|
||||
config = config_manager.load_config()
|
||||
config.cloud_host = "https://cloud.example.test"
|
||||
config_manager.save_config(config)
|
||||
|
||||
async def api_request(**_kwargs):
|
||||
raise httpx.ConnectError("boom")
|
||||
|
||||
with pytest.raises(CloudUtilsError, match="Failed to fetch cloud projects"):
|
||||
await project_exists("alpha", api_request=api_request)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_make_api_request_prefers_api_key_over_oauth(config_home, config_manager):
|
||||
"""API key in config should be used without needing an OAuth token on disk."""
|
||||
# Arrange: set an API key in config, no OAuth token on disk
|
||||
config = config_manager.load_config()
|
||||
config.cloud_api_key = "bmc_test_key_12345"
|
||||
config_manager.save_config(config)
|
||||
|
||||
async def handler(request: httpx.Request) -> httpx.Response:
|
||||
# Verify the API key is sent as the Bearer token
|
||||
assert request.headers.get("authorization") == "Bearer bmc_test_key_12345"
|
||||
return httpx.Response(200, json={"ok": True})
|
||||
|
||||
transport = httpx.MockTransport(handler)
|
||||
|
||||
@asynccontextmanager
|
||||
async def http_client_factory():
|
||||
async with httpx.AsyncClient(transport=transport) as client:
|
||||
yield client
|
||||
|
||||
# Act — no auth= parameter, no OAuth token file; should use API key from config
|
||||
resp = await make_api_request(
|
||||
method="GET",
|
||||
url="https://cloud.example.test/proxy/health",
|
||||
http_client_factory=http_client_factory,
|
||||
)
|
||||
|
||||
# Assert
|
||||
assert resp.json()["ok"] is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_make_api_request_falls_back_to_oauth_when_no_api_key(config_home, config_manager):
|
||||
"""When no API key is configured, should fall back to OAuth token."""
|
||||
# Arrange: no API key, but OAuth token on disk
|
||||
config = config_manager.load_config()
|
||||
config.cloud_api_key = None
|
||||
config_manager.save_config(config)
|
||||
|
||||
auth = CLIAuth(client_id="cid", authkit_domain="https://auth.example.test")
|
||||
auth.token_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
auth.token_file.write_text(
|
||||
'{"access_token":"oauth-token-456","refresh_token":null,'
|
||||
'"expires_at":9999999999,"token_type":"Bearer"}',
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
async def handler(request: httpx.Request) -> httpx.Response:
|
||||
assert request.headers.get("authorization") == "Bearer oauth-token-456"
|
||||
return httpx.Response(200, json={"ok": True})
|
||||
|
||||
transport = httpx.MockTransport(handler)
|
||||
|
||||
@asynccontextmanager
|
||||
async def http_client_factory():
|
||||
async with httpx.AsyncClient(transport=transport) as client:
|
||||
yield client
|
||||
|
||||
resp = await make_api_request(
|
||||
method="GET",
|
||||
url="https://cloud.example.test/proxy/health",
|
||||
auth=auth,
|
||||
http_client_factory=http_client_factory,
|
||||
)
|
||||
|
||||
assert resp.json()["ok"] is True
|
||||
|
||||
@@ -0,0 +1,114 @@
|
||||
"""Tests for cloud sync and bisync command behavior."""
|
||||
|
||||
import importlib
|
||||
from contextlib import asynccontextmanager
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
from typer.testing import CliRunner
|
||||
|
||||
from basic_memory.cli.app import app
|
||||
from basic_memory.config import ProjectMode
|
||||
|
||||
runner = CliRunner()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"argv",
|
||||
[
|
||||
["cloud", "sync", "--name", "research"],
|
||||
["cloud", "bisync", "--name", "research"],
|
||||
],
|
||||
)
|
||||
def test_cloud_sync_commands_use_incremental_db_sync(monkeypatch, argv, config_manager):
|
||||
"""Cloud sync commands should not force a full database re-index after file sync."""
|
||||
project_sync_command = importlib.import_module("basic_memory.cli.commands.cloud.project_sync")
|
||||
|
||||
seen: dict[str, object] = {}
|
||||
config = config_manager.load_config()
|
||||
config.set_project_mode("research", ProjectMode.CLOUD)
|
||||
config_manager.save_config(config)
|
||||
|
||||
monkeypatch.setattr(project_sync_command, "_require_cloud_credentials", lambda _config: None)
|
||||
monkeypatch.setattr(
|
||||
project_sync_command,
|
||||
"get_mount_info",
|
||||
lambda: _async_value(SimpleNamespace(bucket_name="tenant-bucket")),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
project_sync_command,
|
||||
"_get_cloud_project",
|
||||
lambda _name: _async_value(
|
||||
SimpleNamespace(name="research", external_id="external-project-id", path="research")
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
project_sync_command,
|
||||
"_get_sync_project",
|
||||
lambda _name, _config, _project_data: (SimpleNamespace(name="research"), "/tmp/research"),
|
||||
)
|
||||
monkeypatch.setattr(project_sync_command, "project_sync", lambda *args, **kwargs: True)
|
||||
monkeypatch.setattr(project_sync_command, "project_bisync", lambda *args, **kwargs: True)
|
||||
|
||||
@asynccontextmanager
|
||||
async def fake_get_client(*, project_name=None, workspace=None):
|
||||
seen["project_name"] = project_name
|
||||
seen["workspace"] = workspace
|
||||
yield object()
|
||||
|
||||
class FakeProjectClient:
|
||||
def __init__(self, _client):
|
||||
pass
|
||||
|
||||
async def sync(self, external_id: str, force_full: bool = False):
|
||||
seen["external_id"] = external_id
|
||||
seen["force_full"] = force_full
|
||||
return {"message": "queued"}
|
||||
|
||||
monkeypatch.setattr(project_sync_command, "get_client", fake_get_client)
|
||||
monkeypatch.setattr(project_sync_command, "ProjectClient", FakeProjectClient)
|
||||
|
||||
result = runner.invoke(app, argv)
|
||||
|
||||
assert result.exit_code == 0, result.output
|
||||
assert seen["project_name"] == "research"
|
||||
assert seen["external_id"] == "external-project-id"
|
||||
assert seen["force_full"] is False
|
||||
|
||||
|
||||
def test_cloud_bisync_fails_fast_when_sync_entry_disappears(monkeypatch, config_manager):
|
||||
"""Bisync should raise a runtime error when validated sync config vanishes before persistence."""
|
||||
project_sync_command = importlib.import_module("basic_memory.cli.commands.cloud.project_sync")
|
||||
|
||||
config = config_manager.load_config()
|
||||
config.projects.pop("research", None)
|
||||
config_manager.save_config(config)
|
||||
|
||||
monkeypatch.setattr(project_sync_command, "_require_cloud_credentials", lambda _config: None)
|
||||
monkeypatch.setattr(
|
||||
project_sync_command,
|
||||
"get_mount_info",
|
||||
lambda: _async_value(SimpleNamespace(bucket_name="tenant-bucket")),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
project_sync_command,
|
||||
"_get_cloud_project",
|
||||
lambda _name: _async_value(
|
||||
SimpleNamespace(name="research", external_id="external-project-id", path="research")
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
project_sync_command,
|
||||
"_get_sync_project",
|
||||
lambda _name, _config, _project_data: (SimpleNamespace(name="research"), "/tmp/research"),
|
||||
)
|
||||
monkeypatch.setattr(project_sync_command, "project_bisync", lambda *args, **kwargs: True)
|
||||
|
||||
result = runner.invoke(app, ["cloud", "bisync", "--name", "research"])
|
||||
|
||||
assert result.exit_code == 1, result.output
|
||||
assert "unexpectedly missing after validation" in result.output
|
||||
|
||||
|
||||
async def _async_value(value):
|
||||
return value
|
||||
@@ -6,6 +6,8 @@ import httpx
|
||||
from typer.testing import CliRunner
|
||||
|
||||
from basic_memory.cli.app import app
|
||||
from basic_memory.cli.commands.cloud.cloud_utils import CloudUtilsError
|
||||
from basic_memory.config import ProjectMode
|
||||
|
||||
runner = CliRunner()
|
||||
|
||||
@@ -20,11 +22,11 @@ def test_cloud_upload_uses_control_plane_client(monkeypatch, tmp_path):
|
||||
|
||||
seen: dict[str, str] = {}
|
||||
|
||||
async def fake_project_exists(_project_name: str) -> bool:
|
||||
async def fake_project_exists(_project_name: str, workspace: str | None = None) -> bool:
|
||||
return True
|
||||
|
||||
@asynccontextmanager
|
||||
async def fake_get_client():
|
||||
async def fake_get_client(workspace: str | None = None):
|
||||
async with httpx.AsyncClient(base_url="https://cloud.example.test") as client:
|
||||
yield client
|
||||
|
||||
@@ -53,3 +55,89 @@ def test_cloud_upload_uses_control_plane_client(monkeypatch, tmp_path):
|
||||
|
||||
assert result.exit_code == 0, result.output
|
||||
assert seen["base_url"] == "https://cloud.example.test"
|
||||
|
||||
|
||||
def test_cloud_upload_uses_project_workspace_for_api_and_webdav(
|
||||
monkeypatch, tmp_path, config_manager
|
||||
):
|
||||
"""Upload command should reuse the configured workspace across API and WebDAV calls."""
|
||||
import basic_memory.cli.commands.cloud.upload_command as upload_command
|
||||
|
||||
config = config_manager.load_config()
|
||||
config.default_workspace = "default-workspace"
|
||||
config.set_project_mode("routing-test", ProjectMode.CLOUD)
|
||||
config.projects["routing-test"].workspace_id = "project-workspace"
|
||||
config_manager.save_config(config)
|
||||
|
||||
upload_dir = tmp_path / "upload"
|
||||
upload_dir.mkdir()
|
||||
(upload_dir / "note.md").write_text("hello", encoding="utf-8")
|
||||
|
||||
seen: dict[str, str | None] = {}
|
||||
|
||||
async def fake_project_exists(_project_name: str, workspace: str | None = None) -> bool:
|
||||
seen["project_exists_workspace"] = workspace
|
||||
return True
|
||||
|
||||
@asynccontextmanager
|
||||
async def fake_get_client(workspace: str | None = None):
|
||||
seen["control_plane_workspace"] = workspace
|
||||
async with httpx.AsyncClient(base_url="https://cloud.example.test") as client:
|
||||
yield client
|
||||
|
||||
async def fake_upload_path(*args, **kwargs):
|
||||
client_cm_factory = kwargs.get("client_cm_factory")
|
||||
assert client_cm_factory is not None
|
||||
async with client_cm_factory() as client:
|
||||
seen["base_url"] = str(client.base_url).rstrip("/")
|
||||
return True
|
||||
|
||||
monkeypatch.setattr(upload_command, "project_exists", fake_project_exists)
|
||||
monkeypatch.setattr(upload_command, "get_cloud_control_plane_client", fake_get_client)
|
||||
monkeypatch.setattr(upload_command, "upload_path", fake_upload_path)
|
||||
|
||||
result = runner.invoke(
|
||||
app,
|
||||
[
|
||||
"cloud",
|
||||
"upload",
|
||||
str(upload_dir),
|
||||
"--project",
|
||||
"routing-test",
|
||||
"--no-sync",
|
||||
],
|
||||
)
|
||||
|
||||
assert result.exit_code == 0, result.output
|
||||
assert seen["project_exists_workspace"] == "project-workspace"
|
||||
assert seen["control_plane_workspace"] == "project-workspace"
|
||||
assert seen["base_url"] == "https://cloud.example.test"
|
||||
|
||||
|
||||
def test_cloud_upload_exits_when_project_lookup_fails(monkeypatch, tmp_path):
|
||||
"""Upload command should fail fast when cloud project lookup cannot reach the API."""
|
||||
import basic_memory.cli.commands.cloud.upload_command as upload_command
|
||||
|
||||
upload_dir = tmp_path / "upload"
|
||||
upload_dir.mkdir()
|
||||
(upload_dir / "note.md").write_text("hello", encoding="utf-8")
|
||||
|
||||
async def fake_project_exists(_project_name: str, workspace: str | None = None) -> bool:
|
||||
raise CloudUtilsError("lookup failed")
|
||||
|
||||
monkeypatch.setattr(upload_command, "project_exists", fake_project_exists)
|
||||
|
||||
result = runner.invoke(
|
||||
app,
|
||||
[
|
||||
"cloud",
|
||||
"upload",
|
||||
str(upload_dir),
|
||||
"--project",
|
||||
"routing-test",
|
||||
"--no-sync",
|
||||
],
|
||||
)
|
||||
|
||||
assert result.exit_code == 1, result.output
|
||||
assert "Failed to check cloud project 'routing-test'" in result.output
|
||||
|
||||
@@ -25,6 +25,8 @@ def isolated_home(tmp_path, monkeypatch) -> Path:
|
||||
from basic_memory import config as config_module
|
||||
|
||||
config_module._CONFIG_CACHE = None
|
||||
config_module._CONFIG_MTIME = None
|
||||
config_module._CONFIG_SIZE = None
|
||||
|
||||
monkeypatch.setenv("HOME", str(tmp_path))
|
||||
if os.name == "nt":
|
||||
|
||||
@@ -0,0 +1,399 @@
|
||||
"""Tests for CLI auto-update behavior."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import subprocess
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from io import StringIO
|
||||
|
||||
from rich.console import Console
|
||||
|
||||
from basic_memory.cli.auto_update import (
|
||||
AutoUpdateResult,
|
||||
AutoUpdateStatus,
|
||||
InstallSource,
|
||||
_check_homebrew_update_available,
|
||||
_is_interactive_session,
|
||||
detect_install_source,
|
||||
maybe_run_periodic_auto_update,
|
||||
run_auto_update,
|
||||
)
|
||||
from basic_memory.config import BasicMemoryConfig
|
||||
|
||||
|
||||
class StubConfigManager:
|
||||
"""Simple in-memory ConfigManager stub for updater tests."""
|
||||
|
||||
def __init__(self, config: BasicMemoryConfig):
|
||||
self._config = config
|
||||
self.save_calls = 0
|
||||
|
||||
def load_config(self) -> BasicMemoryConfig:
|
||||
return self._config
|
||||
|
||||
def save_config(self, config: BasicMemoryConfig) -> None:
|
||||
self._config = config
|
||||
self.save_calls += 1
|
||||
|
||||
|
||||
def _capture_console() -> tuple[Console, StringIO]:
|
||||
"""Create a Console that writes to an in-memory buffer."""
|
||||
buf = StringIO()
|
||||
return Console(file=buf, force_terminal=True), buf
|
||||
|
||||
|
||||
def _base_config(tmp_path) -> BasicMemoryConfig:
|
||||
return BasicMemoryConfig(projects={"main": {"path": str(tmp_path / "main")}})
|
||||
|
||||
|
||||
def _result(
|
||||
status: AutoUpdateStatus,
|
||||
*,
|
||||
message: str | None,
|
||||
error: str | None = None,
|
||||
) -> AutoUpdateResult:
|
||||
return AutoUpdateResult(
|
||||
status=status,
|
||||
source=InstallSource.UV_TOOL,
|
||||
checked=True,
|
||||
update_available=status in {AutoUpdateStatus.UPDATE_AVAILABLE, AutoUpdateStatus.UPDATED},
|
||||
updated=status == AutoUpdateStatus.UPDATED,
|
||||
latest_version="9.9.9",
|
||||
message=message,
|
||||
error=error,
|
||||
restart_recommended=status == AutoUpdateStatus.UPDATED,
|
||||
)
|
||||
|
||||
|
||||
def test_detect_install_source_variants():
|
||||
assert (
|
||||
detect_install_source("/opt/homebrew/Cellar/basic-memory/0.18.0/bin/python")
|
||||
== InstallSource.HOMEBREW
|
||||
)
|
||||
assert (
|
||||
detect_install_source("/Users/me/.local/share/uv/tools/basic-memory/bin/python")
|
||||
== InstallSource.UV_TOOL
|
||||
)
|
||||
assert (
|
||||
detect_install_source("/Users/me/.cache/uv/archive-v0/abc123/bin/python")
|
||||
== InstallSource.UVX
|
||||
)
|
||||
assert (
|
||||
detect_install_source("/Users/me/Library/Caches/uv/archive-v0/abc123/bin/python")
|
||||
== InstallSource.UVX
|
||||
)
|
||||
assert detect_install_source("/usr/local/bin/python3") == InstallSource.UNKNOWN
|
||||
|
||||
|
||||
def test_interval_gate_skips_check_when_recent(tmp_path):
|
||||
config = _base_config(tmp_path)
|
||||
config.auto_update_last_checked_at = datetime.now() - timedelta(seconds=30)
|
||||
config.update_check_interval = 3600
|
||||
manager = StubConfigManager(config)
|
||||
|
||||
result = run_auto_update(config_manager=manager)
|
||||
|
||||
assert result.status == AutoUpdateStatus.SKIPPED
|
||||
assert result.checked is False
|
||||
assert manager.save_calls == 0
|
||||
|
||||
|
||||
def test_auto_update_disabled_skips_periodic(tmp_path):
|
||||
config = _base_config(tmp_path)
|
||||
config.auto_update = False
|
||||
manager = StubConfigManager(config)
|
||||
|
||||
result = run_auto_update(config_manager=manager)
|
||||
|
||||
assert result.status == AutoUpdateStatus.SKIPPED
|
||||
assert result.checked is False
|
||||
|
||||
|
||||
def test_force_bypasses_auto_update_disabled(monkeypatch, tmp_path):
|
||||
config = _base_config(tmp_path)
|
||||
config.auto_update = False
|
||||
manager = StubConfigManager(config)
|
||||
|
||||
monkeypatch.setattr(
|
||||
"basic_memory.cli.auto_update._check_pypi_update_available",
|
||||
lambda: (False, "0.0.0"),
|
||||
)
|
||||
|
||||
result = run_auto_update(
|
||||
force=True,
|
||||
config_manager=manager,
|
||||
executable="/Users/me/.local/share/uv/tools/basic-memory/bin/python",
|
||||
)
|
||||
|
||||
assert result.status == AutoUpdateStatus.UP_TO_DATE
|
||||
assert result.checked is True
|
||||
assert manager.save_calls == 1
|
||||
|
||||
|
||||
def test_check_homebrew_update_available_exit_code_1_means_outdated(monkeypatch):
|
||||
"""brew outdated exits 1 when the formula is outdated, not on error."""
|
||||
|
||||
def _fake_run(command, **kwargs):
|
||||
return subprocess.CompletedProcess(
|
||||
command, 1, stdout="basicmachines-co/basic-memory/basic-memory\n", stderr=""
|
||||
)
|
||||
|
||||
monkeypatch.setattr("basic_memory.cli.auto_update._run_subprocess", _fake_run)
|
||||
is_outdated, _ = _check_homebrew_update_available(silent=False)
|
||||
assert is_outdated is True
|
||||
|
||||
|
||||
def test_check_homebrew_update_available_exit_code_0_means_up_to_date(monkeypatch):
|
||||
"""brew outdated exits 0 when the formula is up to date."""
|
||||
|
||||
def _fake_run(command, **kwargs):
|
||||
return subprocess.CompletedProcess(command, 0, stdout="", stderr="")
|
||||
|
||||
monkeypatch.setattr("basic_memory.cli.auto_update._run_subprocess", _fake_run)
|
||||
is_outdated, _ = _check_homebrew_update_available(silent=False)
|
||||
assert is_outdated is False
|
||||
|
||||
|
||||
def test_homebrew_outdated_triggers_upgrade(monkeypatch, tmp_path):
|
||||
config = _base_config(tmp_path)
|
||||
manager = StubConfigManager(config)
|
||||
|
||||
monkeypatch.setattr(
|
||||
"basic_memory.cli.auto_update._check_homebrew_update_available",
|
||||
lambda silent: (True, None),
|
||||
)
|
||||
calls: list[list[str]] = []
|
||||
|
||||
def _fake_run_subprocess(command, **kwargs):
|
||||
calls.append(command)
|
||||
return subprocess.CompletedProcess(command, 0, stdout="", stderr="")
|
||||
|
||||
monkeypatch.setattr("basic_memory.cli.auto_update._run_subprocess", _fake_run_subprocess)
|
||||
|
||||
result = run_auto_update(
|
||||
config_manager=manager,
|
||||
executable="/opt/homebrew/Cellar/basic-memory/0.18.0/bin/python",
|
||||
)
|
||||
|
||||
assert result.status == AutoUpdateStatus.UPDATED
|
||||
assert calls == [["brew", "upgrade", "basic-memory"]]
|
||||
|
||||
|
||||
def test_uv_tool_pypi_check_triggers_upgrade(monkeypatch, tmp_path):
|
||||
config = _base_config(tmp_path)
|
||||
manager = StubConfigManager(config)
|
||||
|
||||
monkeypatch.setattr(
|
||||
"basic_memory.cli.auto_update._check_pypi_update_available",
|
||||
lambda: (True, "9.9.9"),
|
||||
)
|
||||
calls: list[list[str]] = []
|
||||
|
||||
def _fake_run_subprocess(command, **kwargs):
|
||||
calls.append(command)
|
||||
return subprocess.CompletedProcess(command, 0, stdout="", stderr="")
|
||||
|
||||
monkeypatch.setattr("basic_memory.cli.auto_update._run_subprocess", _fake_run_subprocess)
|
||||
|
||||
result = run_auto_update(
|
||||
config_manager=manager,
|
||||
executable="/Users/me/.local/share/uv/tools/basic-memory/bin/python",
|
||||
)
|
||||
|
||||
assert result.status == AutoUpdateStatus.UPDATED
|
||||
assert result.latest_version == "9.9.9"
|
||||
assert calls == [["uv", "tool", "upgrade", "basic-memory"]]
|
||||
|
||||
|
||||
def test_unknown_manager_returns_manual_update_guidance(monkeypatch, tmp_path):
|
||||
config = _base_config(tmp_path)
|
||||
manager = StubConfigManager(config)
|
||||
monkeypatch.setattr(
|
||||
"basic_memory.cli.auto_update._check_pypi_update_available",
|
||||
lambda: (True, "9.9.9"),
|
||||
)
|
||||
|
||||
result = run_auto_update(
|
||||
force=True,
|
||||
config_manager=manager,
|
||||
executable="/usr/local/bin/python3",
|
||||
)
|
||||
|
||||
assert result.status == AutoUpdateStatus.UPDATE_AVAILABLE
|
||||
assert result.updated is False
|
||||
assert "Automatic install is not supported" in (result.message or "")
|
||||
|
||||
|
||||
def test_uvx_runtime_is_skipped(monkeypatch, tmp_path):
|
||||
config = _base_config(tmp_path)
|
||||
manager = StubConfigManager(config)
|
||||
|
||||
result = run_auto_update(
|
||||
config_manager=manager,
|
||||
executable="/Users/me/.cache/uv/archive-v0/abc123/bin/python",
|
||||
)
|
||||
|
||||
assert result.status == AutoUpdateStatus.SKIPPED
|
||||
assert result.source == InstallSource.UVX
|
||||
assert result.checked is False
|
||||
assert manager.save_calls == 0
|
||||
|
||||
|
||||
def test_mcp_silent_mode_suppresses_subprocess_output(monkeypatch, tmp_path):
|
||||
config = _base_config(tmp_path)
|
||||
manager = StubConfigManager(config)
|
||||
monkeypatch.setattr(
|
||||
"basic_memory.cli.auto_update._check_pypi_update_available",
|
||||
lambda: (True, "9.9.9"),
|
||||
)
|
||||
|
||||
captured_kwargs: list[dict] = []
|
||||
|
||||
def _fake_run_subprocess(command, **kwargs):
|
||||
captured_kwargs.append(kwargs)
|
||||
return subprocess.CompletedProcess(command, 0, stdout="", stderr="")
|
||||
|
||||
monkeypatch.setattr("basic_memory.cli.auto_update._run_subprocess", _fake_run_subprocess)
|
||||
|
||||
result = run_auto_update(
|
||||
config_manager=manager,
|
||||
executable="/Users/me/.local/share/uv/tools/basic-memory/bin/python",
|
||||
silent=True,
|
||||
)
|
||||
|
||||
assert result.status == AutoUpdateStatus.UPDATED
|
||||
assert captured_kwargs
|
||||
assert captured_kwargs[0]["silent"] is True
|
||||
assert captured_kwargs[0]["capture_output"] is False
|
||||
|
||||
|
||||
def test_subprocess_oserror_is_non_fatal(monkeypatch, tmp_path):
|
||||
config = _base_config(tmp_path)
|
||||
manager = StubConfigManager(config)
|
||||
monkeypatch.setattr(
|
||||
"basic_memory.cli.auto_update._check_pypi_update_available",
|
||||
lambda: (True, "9.9.9"),
|
||||
)
|
||||
|
||||
def _raise_oserror(command, **kwargs):
|
||||
raise FileNotFoundError(command[0])
|
||||
|
||||
monkeypatch.setattr("basic_memory.cli.auto_update._run_subprocess", _raise_oserror)
|
||||
|
||||
result = run_auto_update(
|
||||
config_manager=manager,
|
||||
executable="/Users/me/.local/share/uv/tools/basic-memory/bin/python",
|
||||
)
|
||||
|
||||
assert result.status == AutoUpdateStatus.FAILED
|
||||
assert result.checked is True
|
||||
|
||||
|
||||
def test_mixed_timezone_timestamp_does_not_crash_interval_gate(monkeypatch, tmp_path):
|
||||
config = _base_config(tmp_path)
|
||||
config.auto_update_last_checked_at = datetime.now(timezone.utc)
|
||||
manager = StubConfigManager(config)
|
||||
|
||||
monkeypatch.setattr(
|
||||
"basic_memory.cli.auto_update._check_pypi_update_available",
|
||||
lambda: (False, "0.0.0"),
|
||||
)
|
||||
|
||||
result = run_auto_update(
|
||||
config_manager=manager,
|
||||
executable="/Users/me/.local/share/uv/tools/basic-memory/bin/python",
|
||||
)
|
||||
|
||||
assert result.status == AutoUpdateStatus.UP_TO_DATE
|
||||
assert result.checked is True
|
||||
|
||||
|
||||
def test_maybe_run_periodic_auto_update_non_interactive_has_no_console_output():
|
||||
console, buf = _capture_console()
|
||||
result = maybe_run_periodic_auto_update(
|
||||
"status",
|
||||
is_interactive=False,
|
||||
console=console,
|
||||
)
|
||||
assert result is None
|
||||
assert buf.getvalue() == ""
|
||||
|
||||
|
||||
def test_maybe_run_periodic_auto_update_prints_updated(monkeypatch):
|
||||
console, buf = _capture_console()
|
||||
monkeypatch.setattr(
|
||||
"basic_memory.cli.auto_update.run_auto_update",
|
||||
lambda **kwargs: _result(
|
||||
AutoUpdateStatus.UPDATED,
|
||||
message="Basic Memory was updated successfully.",
|
||||
),
|
||||
)
|
||||
|
||||
result = maybe_run_periodic_auto_update("status", is_interactive=True, console=console)
|
||||
assert result is not None
|
||||
assert result.status == AutoUpdateStatus.UPDATED
|
||||
assert "updated successfully" in buf.getvalue().lower()
|
||||
|
||||
|
||||
def test_maybe_run_periodic_auto_update_prints_available(monkeypatch):
|
||||
console, buf = _capture_console()
|
||||
monkeypatch.setattr(
|
||||
"basic_memory.cli.auto_update.run_auto_update",
|
||||
lambda **kwargs: _result(
|
||||
AutoUpdateStatus.UPDATE_AVAILABLE,
|
||||
message="Update available (latest: 9.9.9).",
|
||||
),
|
||||
)
|
||||
|
||||
result = maybe_run_periodic_auto_update("status", is_interactive=True, console=console)
|
||||
assert result is not None
|
||||
assert result.status == AutoUpdateStatus.UPDATE_AVAILABLE
|
||||
assert "update available" in buf.getvalue().lower()
|
||||
|
||||
|
||||
def test_maybe_run_periodic_auto_update_prints_failed_with_error(monkeypatch):
|
||||
console, buf = _capture_console()
|
||||
monkeypatch.setattr(
|
||||
"basic_memory.cli.auto_update.run_auto_update",
|
||||
lambda **kwargs: _result(
|
||||
AutoUpdateStatus.FAILED,
|
||||
message="Automatic update check failed.",
|
||||
error="network timeout",
|
||||
),
|
||||
)
|
||||
|
||||
result = maybe_run_periodic_auto_update("status", is_interactive=True, console=console)
|
||||
assert result is not None
|
||||
assert result.status == AutoUpdateStatus.FAILED
|
||||
output = buf.getvalue().lower()
|
||||
assert "automatic update check failed" in output
|
||||
assert "network timeout" in output
|
||||
|
||||
|
||||
def test_maybe_run_periodic_auto_update_uses_interactive_probe_when_not_overridden(monkeypatch):
|
||||
console, buf = _capture_console()
|
||||
monkeypatch.setattr("basic_memory.cli.auto_update._is_interactive_session", lambda: True)
|
||||
monkeypatch.setattr(
|
||||
"basic_memory.cli.auto_update.run_auto_update",
|
||||
lambda **kwargs: _result(
|
||||
AutoUpdateStatus.UP_TO_DATE,
|
||||
message="Basic Memory is up to date.",
|
||||
),
|
||||
)
|
||||
|
||||
result = maybe_run_periodic_auto_update("status", console=console)
|
||||
assert result is not None
|
||||
assert result.status == AutoUpdateStatus.UP_TO_DATE
|
||||
# UP_TO_DATE is intentionally silent for periodic checks.
|
||||
assert buf.getvalue() == ""
|
||||
|
||||
|
||||
def test_is_interactive_session_handles_closed_stdio(monkeypatch):
|
||||
class _BrokenStream:
|
||||
def isatty(self) -> bool:
|
||||
raise ValueError("I/O operation on closed file")
|
||||
|
||||
monkeypatch.setattr("basic_memory.cli.auto_update.sys.stdin", _BrokenStream())
|
||||
monkeypatch.setattr("basic_memory.cli.auto_update.sys.stdout", _BrokenStream())
|
||||
|
||||
assert _is_interactive_session() is False
|
||||
@@ -0,0 +1,49 @@
|
||||
"""Telemetry coverage for CLI command boundaries."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from basic_memory.cli import app as cli_app
|
||||
|
||||
|
||||
class FakeContext:
|
||||
"""Small Typer-like context for callback testing."""
|
||||
|
||||
def __init__(self, invoked_subcommand: str | None) -> None:
|
||||
self.invoked_subcommand = invoked_subcommand
|
||||
self.resources: list[object] = []
|
||||
self.close_callbacks: list[object] = []
|
||||
|
||||
def with_resource(self, resource: object) -> None:
|
||||
self.resources.append(resource)
|
||||
|
||||
def call_on_close(self, callback) -> None:
|
||||
self.close_callbacks.append(callback)
|
||||
|
||||
|
||||
def test_app_callback_registers_command_operation(monkeypatch) -> None:
|
||||
operations: list[tuple[str, dict]] = []
|
||||
resource = object()
|
||||
|
||||
monkeypatch.setattr(cli_app, "init_cli_logging", lambda: None)
|
||||
monkeypatch.setattr(cli_app.CliContainer, "create", staticmethod(lambda: object()))
|
||||
monkeypatch.setattr(cli_app, "set_container", lambda container: None)
|
||||
monkeypatch.setattr(cli_app, "maybe_show_init_line", lambda command_name: None)
|
||||
monkeypatch.setattr(cli_app, "maybe_show_cloud_promo", lambda command_name: None)
|
||||
monkeypatch.setattr(cli_app, "maybe_run_periodic_auto_update", lambda command_name: None)
|
||||
|
||||
def fake_operation(name: str, **attrs):
|
||||
operations.append((name, attrs))
|
||||
return resource
|
||||
|
||||
monkeypatch.setattr(cli_app.telemetry, "operation", fake_operation)
|
||||
|
||||
ctx = FakeContext(invoked_subcommand="status")
|
||||
cli_app.app_callback(ctx, version=None)
|
||||
|
||||
assert ctx.resources == [resource]
|
||||
assert operations == [
|
||||
(
|
||||
"cli.command.status",
|
||||
{"entrypoint": "cli", "command_name": "status"},
|
||||
)
|
||||
]
|
||||
@@ -350,6 +350,8 @@ def write_config(tmp_path, monkeypatch):
|
||||
from basic_memory import config as config_module
|
||||
|
||||
config_module._CONFIG_CACHE = None
|
||||
config_module._CONFIG_MTIME = None
|
||||
config_module._CONFIG_SIZE = None
|
||||
|
||||
config_dir = tmp_path / ".basic-memory"
|
||||
config_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
@@ -27,6 +27,8 @@ def mock_config(tmp_path, monkeypatch):
|
||||
from basic_memory import config as config_module
|
||||
|
||||
config_module._CONFIG_CACHE = None
|
||||
config_module._CONFIG_MTIME = None
|
||||
config_module._CONFIG_SIZE = None
|
||||
|
||||
config_dir = tmp_path / ".basic-memory"
|
||||
config_dir.mkdir(parents=True, exist_ok=True)
|
||||
@@ -50,9 +52,11 @@ def mock_config(tmp_path, monkeypatch):
|
||||
@pytest.fixture
|
||||
def mock_api_client(monkeypatch):
|
||||
"""Stub the API client for project add without stdlib mocks."""
|
||||
seen_workspaces: list[str | None] = []
|
||||
|
||||
@asynccontextmanager
|
||||
async def fake_get_client():
|
||||
async def fake_get_client(*, workspace=None):
|
||||
seen_workspaces.append(workspace)
|
||||
yield object()
|
||||
|
||||
_response_data = {
|
||||
@@ -78,7 +82,7 @@ def mock_api_client(monkeypatch):
|
||||
monkeypatch.setattr(project_cmd, "get_client", fake_get_client)
|
||||
monkeypatch.setattr(ProjectClient, "create_project", fake_create_project)
|
||||
|
||||
return calls
|
||||
return {"calls": calls, "workspaces": seen_workspaces}
|
||||
|
||||
|
||||
def test_project_add_with_local_path_saves_to_config(
|
||||
@@ -111,6 +115,7 @@ def test_project_add_with_local_path_saves_to_config(
|
||||
assert "test-project" in config_data["projects"]
|
||||
entry = config_data["projects"]["test-project"]
|
||||
# Use as_posix() for cross-platform compatibility (Windows uses backslashes)
|
||||
assert entry["mode"] == "cloud"
|
||||
assert entry["local_sync_path"] == local_sync_dir.as_posix()
|
||||
assert entry.get("last_sync") is None
|
||||
assert entry.get("bisync_initialized", False) is False
|
||||
@@ -171,3 +176,162 @@ def test_project_add_local_path_creates_nested_directories(
|
||||
assert result.exit_code == 0
|
||||
assert nested_path.exists()
|
||||
assert nested_path.is_dir()
|
||||
|
||||
|
||||
def test_project_add_cloud_visibility_passes_payload(runner, mock_config, mock_api_client):
|
||||
"""Cloud project creation should forward visibility to the API payload."""
|
||||
result = runner.invoke(
|
||||
app,
|
||||
["project", "add", "test-project", "--cloud", "--visibility", "shared"],
|
||||
)
|
||||
|
||||
assert result.exit_code == 0
|
||||
assert mock_api_client["workspaces"] == [None]
|
||||
assert mock_api_client["calls"] == [
|
||||
{
|
||||
"name": "test-project",
|
||||
"path": "test-project",
|
||||
"local_sync_path": None,
|
||||
"set_default": False,
|
||||
"visibility": "shared",
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
def test_project_add_cloud_workspace_resolves_and_persists(
|
||||
runner, mock_config, mock_api_client, monkeypatch, tmp_path
|
||||
):
|
||||
"""Cloud project add should resolve workspace names to tenant IDs."""
|
||||
from basic_memory.schemas.cloud import WorkspaceInfo
|
||||
|
||||
local_sync_dir = tmp_path / "sync" / "team-notes"
|
||||
|
||||
async def fake_get_available_workspaces():
|
||||
return [
|
||||
WorkspaceInfo(
|
||||
tenant_id="11111111-1111-1111-1111-111111111111",
|
||||
workspace_type="organization",
|
||||
name="Basic Memory",
|
||||
role="owner",
|
||||
),
|
||||
]
|
||||
|
||||
monkeypatch.setattr(
|
||||
"basic_memory.mcp.project_context.get_available_workspaces",
|
||||
fake_get_available_workspaces,
|
||||
)
|
||||
|
||||
result = runner.invoke(
|
||||
app,
|
||||
[
|
||||
"project",
|
||||
"add",
|
||||
"team-notes",
|
||||
"--cloud",
|
||||
"--workspace",
|
||||
"Basic Memory",
|
||||
"--local-path",
|
||||
str(local_sync_dir),
|
||||
],
|
||||
)
|
||||
|
||||
assert result.exit_code == 0
|
||||
assert mock_api_client["workspaces"] == ["11111111-1111-1111-1111-111111111111"]
|
||||
assert mock_api_client["calls"] == [
|
||||
{
|
||||
"name": "team-notes",
|
||||
"path": "team-notes",
|
||||
"local_sync_path": local_sync_dir.as_posix(),
|
||||
"set_default": False,
|
||||
"visibility": "workspace",
|
||||
}
|
||||
]
|
||||
|
||||
config_data = json.loads(mock_config.read_text())
|
||||
entry = config_data["projects"]["team-notes"]
|
||||
assert entry["mode"] == "cloud"
|
||||
assert entry["workspace_id"] == "11111111-1111-1111-1111-111111111111"
|
||||
|
||||
|
||||
def test_project_add_cloud_workspace_persists_without_local_path(
|
||||
runner, mock_config, mock_api_client, monkeypatch
|
||||
):
|
||||
"""Cloud project add should persist workspace routing even without local sync."""
|
||||
from basic_memory.schemas.cloud import WorkspaceInfo
|
||||
|
||||
async def fake_get_available_workspaces():
|
||||
return [
|
||||
WorkspaceInfo(
|
||||
tenant_id="11111111-1111-1111-1111-111111111111",
|
||||
workspace_type="organization",
|
||||
name="Basic Memory",
|
||||
role="owner",
|
||||
),
|
||||
]
|
||||
|
||||
monkeypatch.setattr(
|
||||
"basic_memory.mcp.project_context.get_available_workspaces",
|
||||
fake_get_available_workspaces,
|
||||
)
|
||||
|
||||
result = runner.invoke(
|
||||
app,
|
||||
[
|
||||
"project",
|
||||
"add",
|
||||
"team-notes",
|
||||
"--cloud",
|
||||
"--workspace",
|
||||
"Basic Memory",
|
||||
],
|
||||
)
|
||||
|
||||
assert result.exit_code == 0
|
||||
assert mock_api_client["workspaces"] == ["11111111-1111-1111-1111-111111111111"]
|
||||
assert mock_api_client["calls"] == [
|
||||
{
|
||||
"name": "team-notes",
|
||||
"path": "team-notes",
|
||||
"local_sync_path": None,
|
||||
"set_default": False,
|
||||
"visibility": "workspace",
|
||||
}
|
||||
]
|
||||
|
||||
config_data = json.loads(mock_config.read_text())
|
||||
entry = config_data["projects"]["team-notes"]
|
||||
assert entry["path"] == ""
|
||||
assert entry["mode"] == "cloud"
|
||||
assert entry["workspace_id"] == "11111111-1111-1111-1111-111111111111"
|
||||
assert entry["local_sync_path"] is None
|
||||
|
||||
|
||||
def test_project_add_visibility_requires_cloud_mode(runner, mock_config, tmp_path):
|
||||
"""Visibility is a cloud-only option."""
|
||||
project_path = tmp_path / "local-project"
|
||||
|
||||
result = runner.invoke(
|
||||
app,
|
||||
[
|
||||
"project",
|
||||
"add",
|
||||
"local-project",
|
||||
str(project_path),
|
||||
"--visibility",
|
||||
"shared",
|
||||
],
|
||||
)
|
||||
|
||||
assert result.exit_code == 1
|
||||
assert "--visibility is only supported in cloud mode" in result.stdout
|
||||
|
||||
|
||||
def test_project_add_rejects_invalid_visibility(runner, mock_config):
|
||||
"""Invalid visibility values should fail fast before the API call."""
|
||||
result = runner.invoke(
|
||||
app,
|
||||
["project", "add", "test-project", "--cloud", "--visibility", "team-only"],
|
||||
)
|
||||
|
||||
assert result.exit_code == 1
|
||||
assert "Invalid visibility" in result.stdout
|
||||
|
||||
@@ -29,6 +29,8 @@ def write_config(tmp_path, monkeypatch):
|
||||
from basic_memory import config as config_module
|
||||
|
||||
config_module._CONFIG_CACHE = None
|
||||
config_module._CONFIG_MTIME = None
|
||||
config_module._CONFIG_SIZE = None
|
||||
|
||||
config_dir = tmp_path / ".basic-memory"
|
||||
config_dir.mkdir(parents=True, exist_ok=True)
|
||||
@@ -122,15 +124,16 @@ def test_project_list_shows_local_cloud_presence_and_routes(
|
||||
assert "Local Path" in result.stdout
|
||||
assert "Cloud Path" in result.stdout
|
||||
assert "CLI Route" in result.stdout
|
||||
assert "MCP (stdio)" in result.stdout
|
||||
assert "MCP" in result.stdout
|
||||
|
||||
lines = result.stdout.splitlines()
|
||||
alpha_line = next(line for line in lines if "│ alpha" in line)
|
||||
beta_line = next(line for line in lines if "│ beta" in line)
|
||||
|
||||
assert "local" in alpha_line # CLI route for alpha
|
||||
assert "stdio" in alpha_line # Local projects use stdio transport
|
||||
assert "cloud" in beta_line # CLI route for beta
|
||||
assert "n/a" in beta_line # MCP stdio route is unavailable for cloud-only projects
|
||||
assert "https" in beta_line # Cloud projects use HTTPS transport
|
||||
assert "alpha-local" in result.stdout
|
||||
assert "/alpha" in result.stdout
|
||||
assert "/beta" in result.stdout
|
||||
|
||||
@@ -22,6 +22,8 @@ def mock_config(tmp_path, monkeypatch):
|
||||
from basic_memory import config as config_module
|
||||
|
||||
config_module._CONFIG_CACHE = None
|
||||
config_module._CONFIG_MTIME = None
|
||||
config_module._CONFIG_SIZE = None
|
||||
|
||||
config_dir = tmp_path / ".basic-memory"
|
||||
config_dir.mkdir(parents=True, exist_ok=True)
|
||||
@@ -68,6 +70,8 @@ class TestSetCloud:
|
||||
from basic_memory import config as config_module
|
||||
|
||||
config_module._CONFIG_CACHE = None
|
||||
config_module._CONFIG_MTIME = None
|
||||
config_module._CONFIG_SIZE = None
|
||||
|
||||
config_dir = tmp_path / ".basic-memory"
|
||||
config_dir.mkdir(parents=True, exist_ok=True)
|
||||
@@ -91,6 +95,8 @@ class TestSetCloud:
|
||||
from basic_memory import config as config_module
|
||||
|
||||
config_module._CONFIG_CACHE = None
|
||||
config_module._CONFIG_MTIME = None
|
||||
config_module._CONFIG_SIZE = None
|
||||
|
||||
config_dir = tmp_path / ".basic-memory"
|
||||
config_dir.mkdir(parents=True, exist_ok=True)
|
||||
@@ -161,11 +167,15 @@ class TestSetLocal:
|
||||
|
||||
# Manually set workspace_id on the project
|
||||
config_module._CONFIG_CACHE = None
|
||||
config_module._CONFIG_MTIME = None
|
||||
config_module._CONFIG_SIZE = None
|
||||
config_data = json.loads(mock_config.read_text())
|
||||
config_data["projects"]["research"]["mode"] = "cloud"
|
||||
config_data["projects"]["research"]["workspace_id"] = "11111111-1111-1111-1111-111111111111"
|
||||
mock_config.write_text(json.dumps(config_data, indent=2))
|
||||
config_module._CONFIG_CACHE = None
|
||||
config_module._CONFIG_MTIME = None
|
||||
config_module._CONFIG_SIZE = None
|
||||
|
||||
# Set back to local
|
||||
result = runner.invoke(app, ["project", "set-local", "research"])
|
||||
@@ -173,6 +183,8 @@ class TestSetLocal:
|
||||
|
||||
# Verify workspace_id was cleared
|
||||
config_module._CONFIG_CACHE = None
|
||||
config_module._CONFIG_MTIME = None
|
||||
config_module._CONFIG_SIZE = None
|
||||
updated_data = json.loads(mock_config.read_text())
|
||||
assert updated_data["projects"]["research"]["workspace_id"] is None
|
||||
assert updated_data["projects"]["research"]["mode"] == "local"
|
||||
@@ -187,6 +199,8 @@ class TestSetCloudWithWorkspace:
|
||||
from basic_memory.schemas.cloud import WorkspaceInfo
|
||||
|
||||
config_module._CONFIG_CACHE = None
|
||||
config_module._CONFIG_MTIME = None
|
||||
config_module._CONFIG_SIZE = None
|
||||
|
||||
async def fake_get_available_workspaces():
|
||||
return [
|
||||
@@ -210,6 +224,8 @@ class TestSetCloudWithWorkspace:
|
||||
|
||||
# Verify workspace_id was persisted
|
||||
config_module._CONFIG_CACHE = None
|
||||
config_module._CONFIG_MTIME = None
|
||||
config_module._CONFIG_SIZE = None
|
||||
updated_data = json.loads(mock_config.read_text())
|
||||
assert (
|
||||
updated_data["projects"]["research"]["workspace_id"]
|
||||
@@ -222,6 +238,8 @@ class TestSetCloudWithWorkspace:
|
||||
from basic_memory.schemas.cloud import WorkspaceInfo
|
||||
|
||||
config_module._CONFIG_CACHE = None
|
||||
config_module._CONFIG_MTIME = None
|
||||
config_module._CONFIG_SIZE = None
|
||||
|
||||
async def fake_get_available_workspaces():
|
||||
return [
|
||||
@@ -249,17 +267,23 @@ class TestSetCloudWithWorkspace:
|
||||
from basic_memory import config as config_module
|
||||
|
||||
config_module._CONFIG_CACHE = None
|
||||
config_module._CONFIG_MTIME = None
|
||||
config_module._CONFIG_SIZE = None
|
||||
|
||||
# Set default_workspace in config
|
||||
config_data = json.loads(mock_config.read_text())
|
||||
config_data["default_workspace"] = "global-default-tenant-id"
|
||||
mock_config.write_text(json.dumps(config_data, indent=2))
|
||||
config_module._CONFIG_CACHE = None
|
||||
config_module._CONFIG_MTIME = None
|
||||
config_module._CONFIG_SIZE = None
|
||||
|
||||
result = runner.invoke(app, ["project", "set-cloud", "research"])
|
||||
assert result.exit_code == 0
|
||||
|
||||
# Verify workspace_id was set from default
|
||||
config_module._CONFIG_CACHE = None
|
||||
config_module._CONFIG_MTIME = None
|
||||
config_module._CONFIG_SIZE = None
|
||||
updated_data = json.loads(mock_config.read_text())
|
||||
assert updated_data["projects"]["research"]["workspace_id"] == "global-default-tenant-id"
|
||||
|
||||
@@ -0,0 +1,90 @@
|
||||
"""Tests for `bm update` command."""
|
||||
|
||||
from typer.testing import CliRunner
|
||||
|
||||
from basic_memory.cli.app import app
|
||||
from basic_memory.cli.auto_update import AutoUpdateResult, AutoUpdateStatus, InstallSource
|
||||
|
||||
|
||||
def _result(
|
||||
status: AutoUpdateStatus,
|
||||
*,
|
||||
message: str | None,
|
||||
error: str | None = None,
|
||||
) -> AutoUpdateResult:
|
||||
return AutoUpdateResult(
|
||||
status=status,
|
||||
source=InstallSource.UV_TOOL,
|
||||
checked=True,
|
||||
update_available=status in {AutoUpdateStatus.UPDATE_AVAILABLE, AutoUpdateStatus.UPDATED},
|
||||
updated=status == AutoUpdateStatus.UPDATED,
|
||||
latest_version="9.9.9",
|
||||
message=message,
|
||||
error=error,
|
||||
restart_recommended=status == AutoUpdateStatus.UPDATED,
|
||||
)
|
||||
|
||||
|
||||
def test_update_command_applies_upgrade(monkeypatch):
|
||||
runner = CliRunner()
|
||||
|
||||
monkeypatch.setattr(
|
||||
"basic_memory.cli.commands.update.run_auto_update",
|
||||
lambda **kwargs: _result(
|
||||
AutoUpdateStatus.UPDATED,
|
||||
message="Basic Memory was updated successfully.",
|
||||
),
|
||||
)
|
||||
|
||||
result = runner.invoke(app, ["update"])
|
||||
assert result.exit_code == 0
|
||||
assert "updated successfully" in result.stdout.lower()
|
||||
|
||||
|
||||
def test_update_command_check_only_shows_available(monkeypatch):
|
||||
runner = CliRunner()
|
||||
|
||||
monkeypatch.setattr(
|
||||
"basic_memory.cli.commands.update.run_auto_update",
|
||||
lambda **kwargs: _result(
|
||||
AutoUpdateStatus.UPDATE_AVAILABLE,
|
||||
message="Update available (latest: 9.9.9). Run `uv tool upgrade basic-memory`.",
|
||||
),
|
||||
)
|
||||
|
||||
result = runner.invoke(app, ["update", "--check"])
|
||||
assert result.exit_code == 0
|
||||
assert "update available" in result.stdout.lower()
|
||||
|
||||
|
||||
def test_update_command_reports_up_to_date(monkeypatch):
|
||||
runner = CliRunner()
|
||||
|
||||
monkeypatch.setattr(
|
||||
"basic_memory.cli.commands.update.run_auto_update",
|
||||
lambda **kwargs: _result(
|
||||
AutoUpdateStatus.UP_TO_DATE,
|
||||
message="Basic Memory is up to date.",
|
||||
),
|
||||
)
|
||||
|
||||
result = runner.invoke(app, ["update"])
|
||||
assert result.exit_code == 0
|
||||
assert "up to date" in result.stdout.lower()
|
||||
|
||||
|
||||
def test_update_command_failure_exits_nonzero(monkeypatch):
|
||||
runner = CliRunner()
|
||||
|
||||
monkeypatch.setattr(
|
||||
"basic_memory.cli.commands.update.run_auto_update",
|
||||
lambda **kwargs: _result(
|
||||
AutoUpdateStatus.FAILED,
|
||||
message="Automatic update failed.",
|
||||
error="network timeout",
|
||||
),
|
||||
)
|
||||
|
||||
result = runner.invoke(app, ["update"])
|
||||
assert result.exit_code == 1
|
||||
assert "automatic update failed" in result.stdout.lower()
|
||||
@@ -76,6 +76,8 @@ class TestWorkspaceSetDefault:
|
||||
monkeypatch.setenv("HOME", str(temp_path))
|
||||
monkeypatch.setenv("BASIC_MEMORY_CONFIG_DIR", str(config_dir))
|
||||
basic_memory.config._CONFIG_CACHE = None
|
||||
basic_memory.config._CONFIG_MTIME = None
|
||||
basic_memory.config._CONFIG_SIZE = None
|
||||
|
||||
config_manager = ConfigManager()
|
||||
test_config = BasicMemoryConfig(
|
||||
@@ -106,6 +108,8 @@ class TestWorkspaceSetDefault:
|
||||
|
||||
# Verify config was updated
|
||||
basic_memory.config._CONFIG_CACHE = None
|
||||
basic_memory.config._CONFIG_MTIME = None
|
||||
basic_memory.config._CONFIG_SIZE = None
|
||||
config = ConfigManager().config
|
||||
assert config.default_workspace == "11111111-1111-1111-1111-111111111111"
|
||||
|
||||
|
||||
+117
-46
@@ -22,7 +22,13 @@ from sqlalchemy.pool import NullPool
|
||||
from testcontainers.postgres import PostgresContainer
|
||||
|
||||
from basic_memory import db
|
||||
from basic_memory.config import ProjectConfig, BasicMemoryConfig, ConfigManager, DatabaseBackend
|
||||
from basic_memory.config import (
|
||||
ProjectConfig,
|
||||
ProjectEntry,
|
||||
BasicMemoryConfig,
|
||||
ConfigManager,
|
||||
DatabaseBackend,
|
||||
)
|
||||
from basic_memory.db import DatabaseType
|
||||
from basic_memory.markdown import EntityParser
|
||||
from basic_memory.markdown.markdown_processor import MarkdownProcessor
|
||||
@@ -74,7 +80,7 @@ def postgres_container(db_backend):
|
||||
The container is started once per test session and shared across all tests.
|
||||
Only starts if db_backend is "postgres".
|
||||
"""
|
||||
if db_backend != "postgres":
|
||||
if db_backend != "postgres" or _configured_postgres_sync_url():
|
||||
yield None
|
||||
return
|
||||
|
||||
@@ -83,11 +89,111 @@ def postgres_container(db_backend):
|
||||
yield postgres
|
||||
|
||||
|
||||
POSTGRES_EPHEMERAL_TABLES = [
|
||||
"search_vector_embeddings",
|
||||
"search_vector_index",
|
||||
]
|
||||
|
||||
|
||||
def _configured_postgres_sync_url() -> str | None:
|
||||
"""Prefer an externally managed Postgres server when CI provides one."""
|
||||
configured_url = os.environ.get("BASIC_MEMORY_TEST_POSTGRES_URL") or os.environ.get(
|
||||
"POSTGRES_TEST_URL"
|
||||
)
|
||||
if not configured_url:
|
||||
return None
|
||||
|
||||
return (
|
||||
configured_url.replace("postgresql+asyncpg://", "postgresql+psycopg2://", 1)
|
||||
.replace("postgresql://", "postgresql+psycopg2://", 1)
|
||||
.replace("postgres://", "postgresql+psycopg2://", 1)
|
||||
)
|
||||
|
||||
|
||||
def _postgres_alembic_config(async_url: str) -> Config:
|
||||
"""Build Alembic config for stamping the shared Postgres test schema."""
|
||||
alembic_dir = Path(db.__file__).parent / "alembic"
|
||||
cfg = Config()
|
||||
cfg.set_main_option("script_location", str(alembic_dir))
|
||||
cfg.set_main_option(
|
||||
"file_template",
|
||||
"%%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d-%%(rev)s_%%(slug)s",
|
||||
)
|
||||
cfg.set_main_option("timezone", "UTC")
|
||||
cfg.set_main_option("revision_environment", "false")
|
||||
cfg.set_main_option("sqlalchemy.url", async_url)
|
||||
return cfg
|
||||
|
||||
|
||||
def _postgres_reset_tables() -> list[str]:
|
||||
"""Resolve the current ORM table set at reset time.
|
||||
|
||||
Some tests declare models after conftest import, so the list must stay dynamic.
|
||||
"""
|
||||
return [table.name for table in Base.metadata.sorted_tables] + [
|
||||
"search_index",
|
||||
"search_vector_chunks",
|
||||
]
|
||||
|
||||
|
||||
def _resolve_postgres_sync_url(postgres_container) -> str:
|
||||
"""Use CI's shared service when configured, otherwise fall back to testcontainers."""
|
||||
configured_url = _configured_postgres_sync_url()
|
||||
if configured_url:
|
||||
return configured_url
|
||||
assert postgres_container is not None
|
||||
return postgres_container.get_connection_url()
|
||||
|
||||
|
||||
async def _reset_postgres_test_schema(engine: AsyncEngine, async_url: str) -> None:
|
||||
"""Restore the shared Postgres schema to a clean baseline before each test."""
|
||||
from basic_memory.models.search import (
|
||||
CREATE_POSTGRES_SEARCH_INDEX_FTS,
|
||||
CREATE_POSTGRES_SEARCH_INDEX_METADATA,
|
||||
CREATE_POSTGRES_SEARCH_INDEX_PERMALINK,
|
||||
CREATE_POSTGRES_SEARCH_INDEX_TABLE,
|
||||
CREATE_POSTGRES_SEARCH_VECTOR_CHUNKS_INDEX,
|
||||
CREATE_POSTGRES_SEARCH_VECTOR_CHUNKS_TABLE,
|
||||
)
|
||||
|
||||
async with engine.begin() as conn:
|
||||
# Trigger: several tests intentionally drop or stub search tables to exercise recovery code.
|
||||
# Why: TRUNCATE is much cheaper than drop_all/create_all, but it only works when the schema exists.
|
||||
# Outcome: we recreate any missing core tables once, then clear rows for deterministic test setup.
|
||||
await conn.run_sync(Base.metadata.create_all)
|
||||
await conn.execute(CREATE_POSTGRES_SEARCH_INDEX_TABLE)
|
||||
await conn.execute(CREATE_POSTGRES_SEARCH_INDEX_FTS)
|
||||
await conn.execute(CREATE_POSTGRES_SEARCH_INDEX_METADATA)
|
||||
await conn.execute(CREATE_POSTGRES_SEARCH_INDEX_PERMALINK)
|
||||
await conn.execute(CREATE_POSTGRES_SEARCH_VECTOR_CHUNKS_TABLE)
|
||||
await conn.execute(CREATE_POSTGRES_SEARCH_VECTOR_CHUNKS_INDEX)
|
||||
|
||||
for table_name in POSTGRES_EPHEMERAL_TABLES:
|
||||
await conn.execute(text(f"DROP TABLE IF EXISTS {table_name} CASCADE"))
|
||||
|
||||
await conn.execute(
|
||||
text(f"TRUNCATE TABLE {', '.join(_postgres_reset_tables())} RESTART IDENTITY CASCADE")
|
||||
)
|
||||
|
||||
alembic_version_exists = (
|
||||
await conn.execute(text("SELECT to_regclass('public.alembic_version')"))
|
||||
).scalar() is not None
|
||||
|
||||
if not alembic_version_exists:
|
||||
command.stamp(_postgres_alembic_config(async_url), "head")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def anyio_backend():
|
||||
return "asyncio"
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def suppress_logfire_no_config_warning(monkeypatch) -> None:
|
||||
"""Keep tests focused on behavior instead of Logfire bootstrap warnings."""
|
||||
monkeypatch.setenv("LOGFIRE_IGNORE_NO_CONFIG", "1")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def project_root() -> Path:
|
||||
return Path(__file__).parent.parent
|
||||
@@ -108,13 +214,15 @@ def config_home(tmp_path, monkeypatch) -> Path:
|
||||
@pytest.fixture(scope="function")
|
||||
def app_config(config_home, db_backend, postgres_container, monkeypatch) -> BasicMemoryConfig:
|
||||
"""Create test app configuration for the appropriate backend."""
|
||||
projects = {"test-project": str(config_home)}
|
||||
projects = {"test-project": ProjectEntry(path=str(config_home))}
|
||||
|
||||
# Set backend based on parameterized db_backend fixture
|
||||
if db_backend == "postgres":
|
||||
backend = DatabaseBackend.POSTGRES
|
||||
# Get URL from testcontainer and convert to asyncpg driver
|
||||
sync_url = postgres_container.get_connection_url()
|
||||
# Trigger: CI jobs can provide a shared Postgres service instead of per-session containers.
|
||||
# Why: reusing one pgvector-enabled server avoids Docker startup churn on every job.
|
||||
# Outcome: local runs keep using testcontainers, while CI injects a stable service URL.
|
||||
sync_url = _resolve_postgres_sync_url(postgres_container)
|
||||
database_url = sync_url.replace("postgresql+psycopg2", "postgresql+asyncpg")
|
||||
else:
|
||||
backend = DatabaseBackend.SQLITE
|
||||
@@ -138,6 +246,8 @@ def config_manager(app_config: BasicMemoryConfig, config_home: Path, monkeypatch
|
||||
from basic_memory import config as config_module
|
||||
|
||||
config_module._CONFIG_CACHE = None
|
||||
config_module._CONFIG_MTIME = None
|
||||
config_module._CONFIG_SIZE = None
|
||||
|
||||
# Create a new ConfigManager that uses the test home directory
|
||||
config_manager = ConfigManager()
|
||||
@@ -198,7 +308,7 @@ async def engine_factory(
|
||||
if db_backend == "postgres":
|
||||
# Postgres mode using testcontainers
|
||||
# Get async connection URL (asyncpg driver - same as production)
|
||||
sync_url = postgres_container.get_connection_url()
|
||||
sync_url = _resolve_postgres_sync_url(postgres_container)
|
||||
async_url = sync_url.replace("postgresql+psycopg2", "postgresql+asyncpg")
|
||||
|
||||
engine = create_async_engine(
|
||||
@@ -221,46 +331,7 @@ async def engine_factory(
|
||||
db._engine = engine
|
||||
db._session_maker = session_maker
|
||||
|
||||
from basic_memory.models.search import (
|
||||
CREATE_POSTGRES_SEARCH_INDEX_TABLE,
|
||||
CREATE_POSTGRES_SEARCH_INDEX_FTS,
|
||||
CREATE_POSTGRES_SEARCH_INDEX_METADATA,
|
||||
CREATE_POSTGRES_SEARCH_INDEX_PERMALINK,
|
||||
CREATE_POSTGRES_SEARCH_VECTOR_CHUNKS_TABLE,
|
||||
CREATE_POSTGRES_SEARCH_VECTOR_CHUNKS_INDEX,
|
||||
)
|
||||
|
||||
# Drop and recreate all tables for test isolation
|
||||
async with engine.begin() as conn:
|
||||
# Must drop search_index first (has FK to project, blocks drop_all)
|
||||
await conn.execute(text("DROP TABLE IF EXISTS search_index CASCADE"))
|
||||
await conn.run_sync(Base.metadata.drop_all)
|
||||
await conn.run_sync(Base.metadata.create_all)
|
||||
# Create search_index via DDL (not ORM - uses composite PK + tsvector)
|
||||
# asyncpg requires separate execute calls for each statement
|
||||
await conn.execute(CREATE_POSTGRES_SEARCH_INDEX_TABLE)
|
||||
await conn.execute(CREATE_POSTGRES_SEARCH_INDEX_FTS)
|
||||
await conn.execute(CREATE_POSTGRES_SEARCH_INDEX_METADATA)
|
||||
await conn.execute(CREATE_POSTGRES_SEARCH_INDEX_PERMALINK)
|
||||
await conn.execute(CREATE_POSTGRES_SEARCH_VECTOR_CHUNKS_TABLE)
|
||||
await conn.execute(CREATE_POSTGRES_SEARCH_VECTOR_CHUNKS_INDEX)
|
||||
|
||||
# Mark migrations as already applied for this test-created schema.
|
||||
#
|
||||
# Some codepaths (e.g. ensure_initialization()) invoke Alembic migrations.
|
||||
# If we create tables via ORM directly, alembic_version is missing and migrations
|
||||
# will try to create tables again, causing DuplicateTableError.
|
||||
alembic_dir = Path(db.__file__).parent / "alembic"
|
||||
cfg = Config()
|
||||
cfg.set_main_option("script_location", str(alembic_dir))
|
||||
cfg.set_main_option(
|
||||
"file_template",
|
||||
"%%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d-%%(rev)s_%%(slug)s",
|
||||
)
|
||||
cfg.set_main_option("timezone", "UTC")
|
||||
cfg.set_main_option("revision_environment", "false")
|
||||
cfg.set_main_option("sqlalchemy.url", async_url)
|
||||
command.stamp(cfg, "head")
|
||||
await _reset_postgres_test_schema(engine, async_url)
|
||||
|
||||
yield engine, session_maker
|
||||
|
||||
|
||||
@@ -79,6 +79,35 @@ async def test_get_client_cloud_adds_workspace_header(config_manager):
|
||||
assert client.headers.get("X-Workspace-ID") == "tenant-123"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_client_cloud_uses_project_workspace_when_not_explicit(config_manager):
|
||||
cfg = config_manager.load_config()
|
||||
cfg.cloud_host = "https://cloud.example.test"
|
||||
cfg.cloud_api_key = "bmc_test_key_123"
|
||||
cfg.default_workspace = "default-tenant"
|
||||
cfg.set_project_mode("research", ProjectMode.CLOUD)
|
||||
cfg.projects["research"].workspace_id = "project-tenant"
|
||||
config_manager.save_config(cfg)
|
||||
|
||||
async with get_client(project_name="research") as client:
|
||||
assert str(client.base_url).rstrip("/") == "https://cloud.example.test/proxy"
|
||||
assert client.headers.get("X-Workspace-ID") == "project-tenant"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_client_cloud_uses_default_workspace_when_project_has_none(config_manager):
|
||||
cfg = config_manager.load_config()
|
||||
cfg.cloud_host = "https://cloud.example.test"
|
||||
cfg.cloud_api_key = "bmc_test_key_123"
|
||||
cfg.default_workspace = "default-tenant"
|
||||
cfg.set_project_mode("research", ProjectMode.CLOUD)
|
||||
config_manager.save_config(cfg)
|
||||
|
||||
async with get_client(project_name="research") as client:
|
||||
assert str(client.base_url).rstrip("/") == "https://cloud.example.test/proxy"
|
||||
assert client.headers.get("X-Workspace-ID") == "default-tenant"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_client_explicit_cloud_raises_without_credentials(config_manager, monkeypatch):
|
||||
cfg = config_manager.load_config()
|
||||
@@ -253,6 +282,19 @@ async def test_get_cloud_control_plane_client_uses_api_key_when_available(config
|
||||
assert client.headers.get("Authorization") == "Bearer bmc_test_key_123"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_cloud_control_plane_client_adds_workspace_header(config_manager):
|
||||
cfg = config_manager.load_config()
|
||||
cfg.cloud_host = "https://cloud.example.test"
|
||||
cfg.cloud_api_key = "bmc_test_key_123"
|
||||
config_manager.save_config(cfg)
|
||||
|
||||
async with get_cloud_control_plane_client(workspace="tenant-123") as client:
|
||||
assert str(client.base_url).rstrip("/") == "https://cloud.example.test"
|
||||
assert client.headers.get("Authorization") == "Bearer bmc_test_key_123"
|
||||
assert client.headers.get("X-Workspace-ID") == "tenant-123"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_cloud_control_plane_client_uses_oauth_token(config_manager):
|
||||
cfg = config_manager.load_config()
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
"""Telemetry coverage for async client auth failures."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
from contextlib import contextmanager
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
async_client_module = importlib.import_module("basic_memory.mcp.async_client")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_resolve_cloud_token_emits_failure_span(monkeypatch) -> None:
|
||||
spans: list[tuple[str, dict]] = []
|
||||
error_messages: list[str] = []
|
||||
|
||||
class FakeAuth:
|
||||
def __init__(self, client_id: str, authkit_domain: str) -> None:
|
||||
self.client_id = client_id
|
||||
self.authkit_domain = authkit_domain
|
||||
|
||||
async def get_valid_token(self):
|
||||
return None
|
||||
|
||||
@contextmanager
|
||||
def fake_span(name: str, **attrs):
|
||||
spans.append((name, attrs))
|
||||
yield
|
||||
|
||||
monkeypatch.setattr(async_client_module.telemetry, "span", fake_span)
|
||||
monkeypatch.setattr("basic_memory.cli.auth.CLIAuth", FakeAuth)
|
||||
monkeypatch.setattr(async_client_module.logger, "error", error_messages.append)
|
||||
|
||||
config = SimpleNamespace(
|
||||
cloud_api_key=None,
|
||||
cloud_client_id="client-123",
|
||||
cloud_domain="auth.example.com",
|
||||
)
|
||||
|
||||
with pytest.raises(RuntimeError, match="no credentials found"):
|
||||
await async_client_module._resolve_cloud_token(config)
|
||||
|
||||
assert spans == [("routing.resolve_cloud_credentials", {"has_api_key": False})]
|
||||
assert error_messages == ["Cloud routing requested but no credentials were available"]
|
||||
@@ -0,0 +1,191 @@
|
||||
"""Telemetry coverage for typed MCP clients and shared HTTP helpers."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
from contextlib import contextmanager
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from mcp.server.fastmcp.exceptions import ToolError
|
||||
|
||||
knowledge_client_module = importlib.import_module("basic_memory.mcp.clients.knowledge")
|
||||
search_client_module = importlib.import_module("basic_memory.mcp.clients.search")
|
||||
utils_module = importlib.import_module("basic_memory.mcp.tools.utils")
|
||||
|
||||
|
||||
def _capture_spans():
|
||||
spans: list[tuple[str, dict]] = []
|
||||
|
||||
@contextmanager
|
||||
def fake_span(name: str, **attrs):
|
||||
spans.append((name, attrs))
|
||||
yield
|
||||
|
||||
@contextmanager
|
||||
def fake_started_span(name: str, **attrs):
|
||||
spans.append((name, attrs))
|
||||
|
||||
class FakeStartedSpan:
|
||||
def set_attribute(self, key: str, value) -> None:
|
||||
attrs[key] = value
|
||||
|
||||
def set_attributes(self, new_attrs: dict) -> None:
|
||||
attrs.update(new_attrs)
|
||||
|
||||
yield FakeStartedSpan()
|
||||
|
||||
return spans, fake_span, fake_started_span
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_knowledge_client_resolve_entity_emits_client_and_http_spans(monkeypatch) -> None:
|
||||
spans, fake_span, fake_started_span = _capture_spans()
|
||||
monkeypatch.setattr(knowledge_client_module.telemetry, "span", fake_span)
|
||||
monkeypatch.setattr(utils_module.telemetry, "started_span", fake_started_span)
|
||||
|
||||
async def handler(request: httpx.Request) -> httpx.Response:
|
||||
assert request.method == "POST"
|
||||
return httpx.Response(200, json={"external_id": "entity-123"})
|
||||
|
||||
transport = httpx.MockTransport(handler)
|
||||
async with httpx.AsyncClient(transport=transport, base_url="https://example.test") as client:
|
||||
knowledge_client = knowledge_client_module.KnowledgeClient(client, "project-123")
|
||||
resolved = await knowledge_client.resolve_entity("notes/root", strict=True)
|
||||
|
||||
assert resolved == "entity-123"
|
||||
assert [name for name, _ in spans] == [
|
||||
"mcp.client.knowledge.resolve_entity",
|
||||
"mcp.http.request",
|
||||
]
|
||||
assert spans[1][1] == {
|
||||
"method": "POST",
|
||||
"client_name": "knowledge",
|
||||
"operation": "resolve_entity",
|
||||
"path_template": "/v2/projects/{project_id}/knowledge/resolve",
|
||||
"phase": "request",
|
||||
"has_query": False,
|
||||
"has_body": True,
|
||||
"status_code": 200,
|
||||
"is_success": True,
|
||||
"outcome": "success",
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_search_client_emits_client_and_http_spans(monkeypatch) -> None:
|
||||
spans, fake_span, fake_started_span = _capture_spans()
|
||||
monkeypatch.setattr(search_client_module.telemetry, "span", fake_span)
|
||||
monkeypatch.setattr(utils_module.telemetry, "started_span", fake_started_span)
|
||||
|
||||
async def handler(request: httpx.Request) -> httpx.Response:
|
||||
assert request.method == "POST"
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"results": [],
|
||||
"current_page": 2,
|
||||
"page_size": 5,
|
||||
"has_more": False,
|
||||
},
|
||||
)
|
||||
|
||||
transport = httpx.MockTransport(handler)
|
||||
async with httpx.AsyncClient(transport=transport, base_url="https://example.test") as client:
|
||||
search_client = search_client_module.SearchClient(client, "project-123")
|
||||
response = await search_client.search({"text": "telemetry"}, page=2, page_size=5)
|
||||
|
||||
assert response.current_page == 2
|
||||
assert [name for name, _ in spans] == [
|
||||
"mcp.client.search.search",
|
||||
"mcp.http.request",
|
||||
]
|
||||
assert spans[1][1] == {
|
||||
"method": "POST",
|
||||
"client_name": "search",
|
||||
"operation": "search",
|
||||
"path_template": "/v2/projects/{project_id}/search/",
|
||||
"phase": "request",
|
||||
"has_query": True,
|
||||
"has_body": True,
|
||||
"status_code": 200,
|
||||
"is_success": True,
|
||||
"outcome": "success",
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_call_get_emits_http_outcome_for_client_errors(monkeypatch) -> None:
|
||||
spans, _, fake_started_span = _capture_spans()
|
||||
monkeypatch.setattr(utils_module.telemetry, "started_span", fake_started_span)
|
||||
|
||||
async def handler(request: httpx.Request) -> httpx.Response:
|
||||
assert request.method == "GET"
|
||||
return httpx.Response(404, json={"detail": "missing"})
|
||||
|
||||
transport = httpx.MockTransport(handler)
|
||||
async with httpx.AsyncClient(transport=transport, base_url="https://example.test") as client:
|
||||
with pytest.raises(ToolError, match="missing"):
|
||||
await utils_module.call_get(
|
||||
client,
|
||||
"/missing",
|
||||
client_name="knowledge",
|
||||
operation="get_entity",
|
||||
path_template="/v2/projects/{project_id}/knowledge/entities/{entity_id}",
|
||||
)
|
||||
|
||||
assert spans == [
|
||||
(
|
||||
"mcp.http.request",
|
||||
{
|
||||
"method": "GET",
|
||||
"client_name": "knowledge",
|
||||
"operation": "get_entity",
|
||||
"path_template": "/v2/projects/{project_id}/knowledge/entities/{entity_id}",
|
||||
"phase": "request",
|
||||
"has_query": False,
|
||||
"has_body": False,
|
||||
"status_code": 404,
|
||||
"is_success": False,
|
||||
"outcome": "client_error",
|
||||
},
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_call_get_emits_transport_error_outcome(monkeypatch) -> None:
|
||||
spans, _, fake_started_span = _capture_spans()
|
||||
monkeypatch.setattr(utils_module.telemetry, "started_span", fake_started_span)
|
||||
|
||||
async def handler(request: httpx.Request) -> httpx.Response:
|
||||
raise httpx.ConnectError("boom", request=request)
|
||||
|
||||
transport = httpx.MockTransport(handler)
|
||||
async with httpx.AsyncClient(transport=transport, base_url="https://example.test") as client:
|
||||
with pytest.raises(httpx.ConnectError, match="boom"):
|
||||
await utils_module.call_get(
|
||||
client,
|
||||
"/boom",
|
||||
client_name="knowledge",
|
||||
operation="get_entity",
|
||||
path_template="/v2/projects/{project_id}/knowledge/entities/{entity_id}",
|
||||
)
|
||||
|
||||
assert spans == [
|
||||
(
|
||||
"mcp.http.request",
|
||||
{
|
||||
"method": "GET",
|
||||
"client_name": "knowledge",
|
||||
"operation": "get_entity",
|
||||
"path_template": "/v2/projects/{project_id}/knowledge/entities/{entity_id}",
|
||||
"phase": "request",
|
||||
"has_query": False,
|
||||
"has_body": False,
|
||||
"is_success": False,
|
||||
"outcome": "transport_error",
|
||||
"error_type": "ConnectError",
|
||||
},
|
||||
)
|
||||
]
|
||||
@@ -94,6 +94,26 @@ async def test_uses_explicit_project_when_no_env(config_manager, monkeypatch):
|
||||
assert await resolve_project_parameter(project="explicit-project") == "explicit-project"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_canonicalizes_case_insensitive_project_reference(
|
||||
config_manager, config_home, monkeypatch
|
||||
):
|
||||
from basic_memory.config import ProjectEntry
|
||||
from basic_memory.mcp.project_context import resolve_project_parameter
|
||||
|
||||
cfg = config_manager.load_config()
|
||||
project_name = "Personal-Project"
|
||||
project_path = config_home / "personal-project"
|
||||
project_path.mkdir(parents=True, exist_ok=True)
|
||||
cfg.projects[project_name] = ProjectEntry(path=str(project_path))
|
||||
config_manager.save_config(cfg)
|
||||
|
||||
monkeypatch.delenv("BASIC_MEMORY_MCP_PROJECT", raising=False)
|
||||
|
||||
assert await resolve_project_parameter(project="personal-project") == project_name
|
||||
assert await resolve_project_parameter(project="PERSONAL-PROJECT") == project_name
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_uses_default_project(config_manager, config_home, monkeypatch):
|
||||
from basic_memory.mcp.project_context import resolve_project_parameter
|
||||
@@ -291,6 +311,169 @@ async def test_workspace_uses_cached_workspace_without_fetch(monkeypatch):
|
||||
assert resolved.tenant_id == cached_workspace.tenant_id
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_resolve_project_parameter_uses_cached_active_project_before_api_default_lookup(
|
||||
config_manager, monkeypatch
|
||||
):
|
||||
from basic_memory.mcp.project_context import resolve_project_parameter
|
||||
from basic_memory.schemas.project_info import ProjectItem
|
||||
|
||||
config = config_manager.load_config()
|
||||
config.default_project = None
|
||||
config_manager.save_config(config)
|
||||
|
||||
context = _ContextState()
|
||||
cached_project = ProjectItem(
|
||||
id=1,
|
||||
external_id="11111111-1111-1111-1111-111111111111",
|
||||
name="Cached Project",
|
||||
path="/tmp/cached-project",
|
||||
is_default=True,
|
||||
)
|
||||
await context.set_state("active_project", cached_project.model_dump())
|
||||
|
||||
async def fail_if_called(): # pragma: no cover
|
||||
raise AssertionError("Default project API lookup should not run when project is cached")
|
||||
|
||||
monkeypatch.setattr(
|
||||
"basic_memory.mcp.project_context._resolve_default_project_from_api",
|
||||
fail_if_called,
|
||||
)
|
||||
|
||||
resolved = await resolve_project_parameter(project=None, context=context)
|
||||
assert resolved == cached_project.name
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_resolve_project_parameter_caches_api_default_project_name(
|
||||
config_manager, monkeypatch
|
||||
):
|
||||
from basic_memory.mcp.project_context import resolve_project_parameter
|
||||
|
||||
config = config_manager.load_config()
|
||||
config.default_project = None
|
||||
config_manager.save_config(config)
|
||||
|
||||
context = _ContextState()
|
||||
api_calls = {"count": 0}
|
||||
|
||||
async def fake_default_lookup():
|
||||
api_calls["count"] += 1
|
||||
return "cloud-default"
|
||||
|
||||
monkeypatch.setattr(
|
||||
"basic_memory.mcp.project_context._resolve_default_project_from_api",
|
||||
fake_default_lookup,
|
||||
)
|
||||
|
||||
first = await resolve_project_parameter(project=None, context=context)
|
||||
second = await resolve_project_parameter(project=None, context=context)
|
||||
|
||||
assert first == "cloud-default"
|
||||
assert second == "cloud-default"
|
||||
assert api_calls["count"] == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_active_project_uses_cached_project_before_resolution(monkeypatch):
|
||||
from basic_memory.mcp.project_context import get_active_project
|
||||
from basic_memory.schemas.project_info import ProjectItem
|
||||
|
||||
context = _ContextState()
|
||||
cached_project = ProjectItem(
|
||||
id=1,
|
||||
external_id="11111111-1111-1111-1111-111111111111",
|
||||
name="Cached Project",
|
||||
path="/tmp/cached-project",
|
||||
is_default=True,
|
||||
)
|
||||
await context.set_state("active_project", cached_project.model_dump())
|
||||
|
||||
async def fail_if_called(*args, **kwargs): # pragma: no cover
|
||||
raise AssertionError("Project resolution should not run when cache matches")
|
||||
|
||||
monkeypatch.setattr(
|
||||
"basic_memory.mcp.project_context.resolve_project_parameter",
|
||||
fail_if_called,
|
||||
)
|
||||
|
||||
resolved = await get_active_project(client=None, context=context)
|
||||
assert resolved == cached_project
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_active_project_uses_cached_project_for_explicit_permalink(monkeypatch):
|
||||
from basic_memory.mcp.project_context import get_active_project
|
||||
from basic_memory.schemas.project_info import ProjectItem
|
||||
|
||||
context = _ContextState()
|
||||
cached_project = ProjectItem(
|
||||
id=1,
|
||||
external_id="11111111-1111-1111-1111-111111111111",
|
||||
name="My Research",
|
||||
path="/tmp/my-research",
|
||||
is_default=False,
|
||||
)
|
||||
await context.set_state("active_project", cached_project.model_dump())
|
||||
|
||||
async def fail_if_called(*args, **kwargs): # pragma: no cover
|
||||
raise AssertionError(
|
||||
"Project resolution should not run when explicit project matches cache"
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
"basic_memory.mcp.project_context.resolve_project_parameter",
|
||||
fail_if_called,
|
||||
)
|
||||
|
||||
resolved = await get_active_project(client=None, project="my-research", context=context)
|
||||
assert resolved == cached_project
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_resolve_project_and_path_uses_cached_project_for_memory_url_prefix(
|
||||
config_manager, monkeypatch
|
||||
):
|
||||
from basic_memory.mcp.project_context import resolve_project_and_path
|
||||
from basic_memory.schemas.project_info import ProjectItem
|
||||
|
||||
config = config_manager.load_config()
|
||||
config.permalinks_include_project = False
|
||||
config_manager.save_config(config)
|
||||
|
||||
context = _ContextState()
|
||||
cached_project = ProjectItem(
|
||||
id=1,
|
||||
external_id="11111111-1111-1111-1111-111111111111",
|
||||
name="My Research",
|
||||
path="/tmp/my-research",
|
||||
is_default=False,
|
||||
)
|
||||
await context.set_state("active_project", cached_project.model_dump())
|
||||
|
||||
async def fail_if_called(*args, **kwargs): # pragma: no cover
|
||||
raise AssertionError("Project resolve API should not run when memory URL matches cache")
|
||||
|
||||
async def fake_resolve_project_parameter(project=None, **kwargs):
|
||||
return cached_project.name if project else cached_project.name
|
||||
|
||||
monkeypatch.setattr("basic_memory.mcp.tools.utils.call_post", fail_if_called)
|
||||
monkeypatch.setattr(
|
||||
"basic_memory.mcp.project_context.resolve_project_parameter",
|
||||
fake_resolve_project_parameter,
|
||||
)
|
||||
|
||||
active_project, resolved_path, is_memory_url = await resolve_project_and_path(
|
||||
client=None,
|
||||
identifier="memory://my-research/notes/roadmap.md",
|
||||
context=context,
|
||||
)
|
||||
|
||||
assert active_project == cached_project
|
||||
assert resolved_path == "notes/roadmap.md"
|
||||
assert is_memory_url is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_project_client_rejects_workspace_for_local_project(config_manager):
|
||||
from basic_memory.mcp.project_context import get_project_client
|
||||
|
||||
@@ -0,0 +1,94 @@
|
||||
"""Telemetry coverage for project routing helpers."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
from contextlib import contextmanager
|
||||
|
||||
import pytest
|
||||
|
||||
from basic_memory.config import ProjectEntry
|
||||
from basic_memory.schemas.cloud import WorkspaceInfo
|
||||
|
||||
project_context = importlib.import_module("basic_memory.mcp.project_context")
|
||||
|
||||
|
||||
class _ContextState:
|
||||
"""Minimal FastMCP context-state stub for unit tests."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._state: dict[str, object] = {}
|
||||
|
||||
async def get_state(self, key: str):
|
||||
return self._state.get(key)
|
||||
|
||||
async def set_state(self, key: str, value: object, **kwargs) -> None:
|
||||
self._state[key] = value
|
||||
|
||||
|
||||
def _capture_telemetry():
|
||||
spans: list[tuple[str, dict]] = []
|
||||
contexts: list[dict] = []
|
||||
|
||||
@contextmanager
|
||||
def fake_span(name: str, **attrs):
|
||||
spans.append((name, attrs))
|
||||
yield
|
||||
|
||||
@contextmanager
|
||||
def fake_contextualize(**attrs):
|
||||
contexts.append(attrs)
|
||||
yield
|
||||
|
||||
return spans, contexts, fake_span, fake_contextualize
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_resolve_workspace_parameter_emits_routing_span(monkeypatch) -> None:
|
||||
spans, _, fake_span, _ = _capture_telemetry()
|
||||
context = _ContextState()
|
||||
workspace = WorkspaceInfo(
|
||||
tenant_id="11111111-1111-1111-1111-111111111111",
|
||||
workspace_type="personal",
|
||||
name="Personal",
|
||||
role="owner",
|
||||
)
|
||||
|
||||
async def fake_get_available_workspaces(context=None):
|
||||
return [workspace]
|
||||
|
||||
monkeypatch.setattr(project_context.telemetry, "span", fake_span)
|
||||
monkeypatch.setattr(project_context, "get_available_workspaces", fake_get_available_workspaces)
|
||||
|
||||
resolved = await project_context.resolve_workspace_parameter(context=context)
|
||||
|
||||
assert resolved.tenant_id == workspace.tenant_id
|
||||
assert spans == [
|
||||
(
|
||||
"routing.resolve_workspace",
|
||||
{"workspace_requested": False, "has_context": True},
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_project_client_contextualizes_route_mode(config_manager, monkeypatch) -> None:
|
||||
spans, contexts, fake_span, fake_contextualize = _capture_telemetry()
|
||||
|
||||
config = config_manager.load_config()
|
||||
(config_manager.config_dir.parent / "main").mkdir(parents=True, exist_ok=True)
|
||||
config.projects["main"] = ProjectEntry(path=str(config_manager.config_dir.parent / "main"))
|
||||
config_manager.save_config(config)
|
||||
|
||||
monkeypatch.setattr(project_context.telemetry, "span", fake_span)
|
||||
monkeypatch.setattr(project_context.telemetry, "contextualize", fake_contextualize)
|
||||
|
||||
with pytest.raises(Exception):
|
||||
async with project_context.get_project_client(project="main"):
|
||||
pass
|
||||
|
||||
span_names = [name for name, _ in spans]
|
||||
assert "routing.resolve_project" in span_names
|
||||
assert "routing.client_session" in span_names
|
||||
assert "routing.validate_project" in span_names
|
||||
assert {"project_name": "main", "route_mode": "local_asgi"} in contexts
|
||||
@@ -0,0 +1,33 @@
|
||||
"""Telemetry coverage for MCP server lifecycle spans."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from contextlib import contextmanager
|
||||
|
||||
import pytest
|
||||
|
||||
from basic_memory.mcp.server import lifespan, mcp
|
||||
import basic_memory.mcp.server as server_module
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_mcp_lifespan_wraps_startup_and_shutdown(config_manager) -> None:
|
||||
operations: list[tuple[str, dict]] = []
|
||||
|
||||
@contextmanager
|
||||
def fake_operation(name: str, **attrs):
|
||||
operations.append((name, attrs))
|
||||
yield
|
||||
|
||||
original_operation = server_module.telemetry.operation
|
||||
server_module.telemetry.operation = fake_operation
|
||||
try:
|
||||
async with lifespan(mcp):
|
||||
pass
|
||||
finally:
|
||||
server_module.telemetry.operation = original_operation
|
||||
|
||||
assert [name for name, _ in operations] == [
|
||||
"mcp.lifecycle.startup",
|
||||
"mcp.lifecycle.shutdown",
|
||||
]
|
||||
@@ -1,5 +1,7 @@
|
||||
"""Tests for delete_note MCP tool."""
|
||||
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
from basic_memory.mcp.tools.delete_note import delete_note, _format_delete_error_response
|
||||
@@ -120,3 +122,56 @@ async def test_delete_note_rejects_fuzzy_match(client, test_project):
|
||||
# Verify the existing note was NOT deleted
|
||||
content = await read_note("Delete Target Note", project=test_project.name)
|
||||
assert "Should not be deleted" in content
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_delete_note_detects_project_from_memory_url(client, test_project):
|
||||
"""delete_note should detect project from memory:// URL prefix when project=None."""
|
||||
# Create a note to delete
|
||||
await write_note(
|
||||
project=test_project.name,
|
||||
title="Delete URL Note",
|
||||
directory="test",
|
||||
content="# Delete URL Note\nContent to delete.",
|
||||
)
|
||||
|
||||
# Delete using memory:// URL with project=None — should auto-detect project
|
||||
# The note may or may not be found (depends on URL resolution), but the key
|
||||
# assertion is that routing goes to the correct project
|
||||
result = await delete_note(
|
||||
identifier=f"memory://{test_project.name}/test/delete-url-note",
|
||||
project=None,
|
||||
)
|
||||
|
||||
# Result is True (deleted) or False (not found by that URL) — either is acceptable.
|
||||
# The important thing is it didn't error and routed to the correct project.
|
||||
assert isinstance(result, bool)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_delete_note_skips_detection_for_plain_path(client, test_project):
|
||||
"""delete_note should NOT call detect_project_from_url_prefix for plain path identifiers.
|
||||
|
||||
A plain path like 'research/note' should not be misrouted to a project
|
||||
named 'research' — the 'research' segment is a directory, not a project.
|
||||
"""
|
||||
with patch("basic_memory.mcp.tools.delete_note.detect_project_from_url_prefix") as mock_detect:
|
||||
# Use a plain path (no memory:// prefix) — detection should not be called
|
||||
await delete_note(
|
||||
identifier="test/nonexistent-note",
|
||||
project=None,
|
||||
)
|
||||
|
||||
mock_detect.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_delete_note_skips_detection_when_project_provided(client, test_project):
|
||||
"""delete_note should skip URL detection when project is explicitly provided."""
|
||||
with patch("basic_memory.mcp.tools.delete_note.detect_project_from_url_prefix") as mock_detect:
|
||||
await delete_note(
|
||||
identifier=f"memory://{test_project.name}/test/some-note",
|
||||
project=test_project.name,
|
||||
)
|
||||
|
||||
mock_detect.assert_not_called()
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
"""Tests for the edit_note MCP tool."""
|
||||
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
@@ -770,3 +771,61 @@ async def test_edit_note_insert_before_section_not_found(client, test_project):
|
||||
|
||||
assert isinstance(result, str)
|
||||
assert "# Edit Failed" in result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_edit_note_detects_project_from_memory_url(client, test_project):
|
||||
"""edit_note should detect project from memory:// URL prefix when project=None."""
|
||||
# Create a note first
|
||||
await write_note(
|
||||
project=test_project.name,
|
||||
title="URL Detection Note",
|
||||
directory="test",
|
||||
content="# URL Detection Note\nOriginal content.",
|
||||
)
|
||||
|
||||
# Edit using memory:// URL with project=None — should auto-detect project
|
||||
# The memory URL uses the permalink (which includes project prefix)
|
||||
result = await edit_note(
|
||||
identifier=f"memory://{test_project.name}/test/url-detection-note",
|
||||
operation="append",
|
||||
content="\nAppended via memory URL.",
|
||||
project=None,
|
||||
)
|
||||
|
||||
assert isinstance(result, str)
|
||||
# Should route to the correct project and succeed (either edit or create)
|
||||
assert f"project: {test_project.name}" in result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_edit_note_skips_detection_for_plain_path(client, test_project):
|
||||
"""edit_note should NOT call detect_project_from_url_prefix for plain path identifiers.
|
||||
|
||||
A plain path like 'research/note' should not be misrouted to a project
|
||||
named 'research' — the 'research' segment is a directory, not a project.
|
||||
"""
|
||||
with patch("basic_memory.mcp.tools.edit_note.detect_project_from_url_prefix") as mock_detect:
|
||||
# Use a plain path (no memory:// prefix) — detection should not be called
|
||||
await edit_note(
|
||||
identifier="test/some-note",
|
||||
operation="append",
|
||||
content="content",
|
||||
project=None,
|
||||
)
|
||||
|
||||
mock_detect.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_edit_note_skips_detection_when_project_provided(client, test_project):
|
||||
"""edit_note should skip URL detection when project is explicitly provided."""
|
||||
with patch("basic_memory.mcp.tools.edit_note.detect_project_from_url_prefix") as mock_detect:
|
||||
await edit_note(
|
||||
identifier=f"memory://{test_project.name}/test/some-note",
|
||||
operation="append",
|
||||
content="content",
|
||||
project=test_project.name,
|
||||
)
|
||||
|
||||
mock_detect.assert_not_called()
|
||||
|
||||
@@ -264,6 +264,63 @@ async def test_read_note_memory_url_with_project_prefix(app, test_project):
|
||||
assert "Testing memory:// URL handling with project prefix" in content
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_read_note_memory_url_fallback_uses_search_tool_normalization(
|
||||
monkeypatch, app, test_project
|
||||
):
|
||||
"""Fallback search should go back through search_notes for memory:// normalization."""
|
||||
await write_note(
|
||||
project=test_project.name,
|
||||
title="Memory URL Fallback Note",
|
||||
directory="test",
|
||||
content="Fallback note content",
|
||||
)
|
||||
|
||||
import importlib
|
||||
|
||||
read_note_module = importlib.import_module("basic_memory.mcp.tools.read_note")
|
||||
clients_mod = importlib.import_module("basic_memory.mcp.clients")
|
||||
OriginalKnowledgeClient = clients_mod.KnowledgeClient
|
||||
|
||||
fallback_memory_url = f"memory://{test_project.name}/test/memory-url-fallback-note"
|
||||
search_calls: list[tuple[str, str, str | None]] = []
|
||||
|
||||
class SelectiveKnowledgeClient(OriginalKnowledgeClient):
|
||||
async def resolve_entity(self, identifier: str, *, strict: bool = False) -> int:
|
||||
if strict and identifier.endswith("test/memory-url-fallback-note"):
|
||||
raise RuntimeError("force direct lookup failure")
|
||||
return await super().resolve_entity(identifier, strict=strict)
|
||||
|
||||
async def fake_search_notes_fn(*, query, search_type, project, **kwargs):
|
||||
search_calls.append((search_type, query, project))
|
||||
return {
|
||||
"results": [
|
||||
{
|
||||
"title": "Memory URL Fallback Note",
|
||||
"permalink": "test/memory-url-fallback-note",
|
||||
"content": "",
|
||||
"type": "entity",
|
||||
"score": 1.0,
|
||||
"file_path": "test/Memory URL Fallback Note.md",
|
||||
}
|
||||
],
|
||||
"current_page": 1,
|
||||
"page_size": 10,
|
||||
}
|
||||
|
||||
monkeypatch.setattr(clients_mod, "KnowledgeClient", SelectiveKnowledgeClient)
|
||||
monkeypatch.setattr(read_note_module, "search_notes", fake_search_notes_fn)
|
||||
|
||||
result = await read_note(fallback_memory_url)
|
||||
|
||||
assert search_calls == [
|
||||
("title", fallback_memory_url, test_project.name),
|
||||
("text", fallback_memory_url, test_project.name),
|
||||
]
|
||||
assert "I couldn't find an exact match" in result
|
||||
assert "Memory URL Fallback Note" in result
|
||||
|
||||
|
||||
class TestReadNoteSecurityValidation:
|
||||
"""Test read_note security validation features."""
|
||||
|
||||
|
||||
@@ -788,6 +788,7 @@ async def test_search_notes_defaults_to_hybrid_when_semantic_enabled(monkeypatch
|
||||
@dataclass
|
||||
class StubConfig:
|
||||
semantic_search_enabled: bool = True
|
||||
default_search_type: str | None = None
|
||||
|
||||
@dataclass
|
||||
class StubContainer:
|
||||
@@ -849,6 +850,7 @@ async def test_search_notes_defaults_to_fts_when_semantic_disabled(monkeypatch):
|
||||
@dataclass
|
||||
class StubConfig:
|
||||
semantic_search_enabled: bool = False
|
||||
default_search_type: str | None = None
|
||||
|
||||
@dataclass
|
||||
class StubContainer:
|
||||
@@ -909,6 +911,7 @@ async def test_search_notes_explicit_text_stays_fts_when_semantic_enabled(monkey
|
||||
@dataclass
|
||||
class StubConfig:
|
||||
semantic_search_enabled: bool = True
|
||||
default_search_type: str | None = None
|
||||
|
||||
@dataclass
|
||||
class StubContainer:
|
||||
@@ -976,7 +979,11 @@ async def test_search_notes_defaults_to_hybrid_when_container_not_initialized(mo
|
||||
lambda: type(
|
||||
"StubConfigManager",
|
||||
(),
|
||||
{"config": type("Cfg", (), {"semantic_search_enabled": True})()},
|
||||
{
|
||||
"config": type(
|
||||
"Cfg", (), {"semantic_search_enabled": True, "default_search_type": None}
|
||||
)()
|
||||
},
|
||||
)(),
|
||||
)
|
||||
|
||||
@@ -1037,7 +1044,11 @@ async def test_search_notes_defaults_to_fts_when_container_not_initialized_and_s
|
||||
lambda: type(
|
||||
"StubConfigManager",
|
||||
(),
|
||||
{"config": type("Cfg", (), {"semantic_search_enabled": False})()},
|
||||
{
|
||||
"config": type(
|
||||
"Cfg", (), {"semantic_search_enabled": False, "default_search_type": None}
|
||||
)()
|
||||
},
|
||||
)(),
|
||||
)
|
||||
|
||||
@@ -1567,3 +1578,54 @@ async def test_search_notes_metadata_filters_preserves_non_aliased_keys(monkeypa
|
||||
|
||||
# "note_type" aliased to "type", "priority" passes through unchanged
|
||||
assert captured_payload["metadata_filters"] == {"type": "spec", "priority": "high"}
|
||||
|
||||
|
||||
def test_default_search_type_uses_config_value():
|
||||
"""_default_search_type should return config.default_search_type when set."""
|
||||
import sys
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
search_module = sys.modules["basic_memory.mcp.tools.search"]
|
||||
|
||||
mock_config = MagicMock()
|
||||
mock_config.default_search_type = "vector"
|
||||
mock_config.semantic_search_enabled = True
|
||||
mock_container = MagicMock()
|
||||
mock_container.config = mock_config
|
||||
|
||||
with patch.object(search_module, "get_container", return_value=mock_container):
|
||||
assert search_module._default_search_type() == "vector"
|
||||
|
||||
|
||||
def test_default_search_type_falls_back_to_hybrid_when_semantic_enabled():
|
||||
"""When default_search_type is None and semantic is enabled, default to hybrid."""
|
||||
import sys
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
search_module = sys.modules["basic_memory.mcp.tools.search"]
|
||||
|
||||
mock_config = MagicMock()
|
||||
mock_config.default_search_type = None
|
||||
mock_config.semantic_search_enabled = True
|
||||
mock_container = MagicMock()
|
||||
mock_container.config = mock_config
|
||||
|
||||
with patch.object(search_module, "get_container", return_value=mock_container):
|
||||
assert search_module._default_search_type() == "hybrid"
|
||||
|
||||
|
||||
def test_default_search_type_falls_back_to_text_when_semantic_disabled():
|
||||
"""When default_search_type is None and semantic is disabled, default to text."""
|
||||
import sys
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
search_module = sys.modules["basic_memory.mcp.tools.search"]
|
||||
|
||||
mock_config = MagicMock()
|
||||
mock_config.default_search_type = None
|
||||
mock_config.semantic_search_enabled = False
|
||||
mock_container = MagicMock()
|
||||
mock_container.config = mock_config
|
||||
|
||||
with patch.object(search_module, "get_container", return_value=mock_container):
|
||||
assert search_module._default_search_type() == "text"
|
||||
|
||||
@@ -0,0 +1,310 @@
|
||||
"""Telemetry coverage for MCP tool root spans."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
from contextlib import contextmanager
|
||||
|
||||
import pytest
|
||||
|
||||
build_context_module = importlib.import_module("basic_memory.mcp.tools.build_context")
|
||||
edit_note_module = importlib.import_module("basic_memory.mcp.tools.edit_note")
|
||||
read_note_module = importlib.import_module("basic_memory.mcp.tools.read_note")
|
||||
search_module = importlib.import_module("basic_memory.mcp.tools.search")
|
||||
write_note_module = importlib.import_module("basic_memory.mcp.tools.write_note")
|
||||
|
||||
|
||||
def _recording_contexts():
|
||||
operations: list[tuple[str, dict]] = []
|
||||
contexts: list[dict] = []
|
||||
|
||||
@contextmanager
|
||||
def fake_operation(name: str, **attrs):
|
||||
operations.append((name, attrs))
|
||||
yield
|
||||
|
||||
@contextmanager
|
||||
def fake_contextualize(**attrs):
|
||||
contexts.append(attrs)
|
||||
yield
|
||||
|
||||
return operations, contexts, fake_operation, fake_contextualize
|
||||
|
||||
|
||||
def _contains_context(contexts: list[dict], expected: dict) -> bool:
|
||||
return any(expected.items() <= context.items() for context in contexts)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_write_note_emits_root_operation_and_project_context(
|
||||
app, test_project, monkeypatch
|
||||
) -> None:
|
||||
operations, contexts, fake_operation, fake_contextualize = _recording_contexts()
|
||||
monkeypatch.setattr(write_note_module.telemetry, "operation", fake_operation)
|
||||
monkeypatch.setattr(write_note_module.telemetry, "contextualize", fake_contextualize)
|
||||
|
||||
await write_note_module.write_note(
|
||||
project=test_project.name,
|
||||
title="Telemetry Note",
|
||||
directory="notes",
|
||||
content="Telemetry content",
|
||||
output_format="json",
|
||||
)
|
||||
|
||||
assert operations[0] == (
|
||||
"mcp.tool.write_note",
|
||||
{
|
||||
"entrypoint": "mcp",
|
||||
"tool_name": "write_note",
|
||||
"requested_project": test_project.name,
|
||||
"workspace_id": None,
|
||||
"note_type": "note",
|
||||
"overwrite": False,
|
||||
"output_format": "json",
|
||||
},
|
||||
)
|
||||
assert "api.request.knowledge.create_entity" in [name for name, _ in operations]
|
||||
assert _contains_context(
|
||||
contexts,
|
||||
{
|
||||
"project_name": test_project.name,
|
||||
"route_mode": "local_asgi",
|
||||
},
|
||||
)
|
||||
assert _contains_context(
|
||||
contexts,
|
||||
{
|
||||
"project_name": test_project.name,
|
||||
"tool_name": "write_note",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_read_note_emits_root_operation_and_project_context(
|
||||
app, test_project, monkeypatch
|
||||
) -> None:
|
||||
await write_note_module.write_note(
|
||||
project=test_project.name,
|
||||
title="Readable Telemetry Note",
|
||||
directory="notes",
|
||||
content="Readable telemetry content",
|
||||
)
|
||||
|
||||
operations, contexts, fake_operation, fake_contextualize = _recording_contexts()
|
||||
monkeypatch.setattr(read_note_module.telemetry, "operation", fake_operation)
|
||||
monkeypatch.setattr(read_note_module.telemetry, "contextualize", fake_contextualize)
|
||||
|
||||
await read_note_module.read_note(
|
||||
"notes/readable-telemetry-note",
|
||||
project=test_project.name,
|
||||
output_format="json",
|
||||
include_frontmatter=True,
|
||||
)
|
||||
|
||||
assert operations[0] == (
|
||||
"mcp.tool.read_note",
|
||||
{
|
||||
"entrypoint": "mcp",
|
||||
"tool_name": "read_note",
|
||||
"requested_project": test_project.name,
|
||||
"workspace_id": None,
|
||||
"output_format": "json",
|
||||
"page": 1,
|
||||
"page_size": 10,
|
||||
"include_frontmatter": True,
|
||||
},
|
||||
)
|
||||
operation_names = [name for name, _ in operations]
|
||||
assert "api.request.knowledge.resolve_entity" in operation_names
|
||||
assert "api.request.resource.get_content" in operation_names
|
||||
assert "api.request.knowledge.get_entity" in operation_names
|
||||
assert _contains_context(
|
||||
contexts,
|
||||
{
|
||||
"project_name": test_project.name,
|
||||
"route_mode": "local_asgi",
|
||||
},
|
||||
)
|
||||
assert _contains_context(
|
||||
contexts,
|
||||
{
|
||||
"project_name": test_project.name,
|
||||
"tool_name": "read_note",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_search_notes_emits_root_operation_and_project_context(
|
||||
app, test_project, monkeypatch
|
||||
) -> None:
|
||||
await write_note_module.write_note(
|
||||
project=test_project.name,
|
||||
title="Searchable Telemetry Note",
|
||||
directory="notes",
|
||||
content="Telemetry search content",
|
||||
tags=["telemetry"],
|
||||
)
|
||||
|
||||
operations, contexts, fake_operation, fake_contextualize = _recording_contexts()
|
||||
monkeypatch.setattr(search_module.telemetry, "operation", fake_operation)
|
||||
monkeypatch.setattr(search_module.telemetry, "contextualize", fake_contextualize)
|
||||
|
||||
await search_module.search_notes(
|
||||
project=test_project.name,
|
||||
query="telemetry",
|
||||
search_type="text",
|
||||
tags=["telemetry"],
|
||||
output_format="json",
|
||||
)
|
||||
|
||||
assert operations[0] == (
|
||||
"mcp.tool.search_notes",
|
||||
{
|
||||
"entrypoint": "mcp",
|
||||
"tool_name": "search_notes",
|
||||
"requested_project": test_project.name,
|
||||
"workspace_id": None,
|
||||
"search_type": "text",
|
||||
"output_format": "json",
|
||||
"page": 1,
|
||||
"page_size": 10,
|
||||
"has_query": True,
|
||||
"note_type_filter_count": 0,
|
||||
"entity_type_filter_count": 0,
|
||||
"has_filters": True,
|
||||
"has_tags_filter": True,
|
||||
"has_status_filter": False,
|
||||
},
|
||||
)
|
||||
assert ("api.request.search",) == (operations[1][0],)
|
||||
assert _contains_context(
|
||||
contexts,
|
||||
{
|
||||
"project_name": test_project.name,
|
||||
"route_mode": "local_asgi",
|
||||
},
|
||||
)
|
||||
assert _contains_context(
|
||||
contexts,
|
||||
{
|
||||
"project_name": test_project.name,
|
||||
"tool_name": "search_notes",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_edit_note_emits_root_operation_and_project_context(
|
||||
app, test_project, monkeypatch
|
||||
) -> None:
|
||||
await write_note_module.write_note(
|
||||
project=test_project.name,
|
||||
title="Editable Telemetry Note",
|
||||
directory="notes",
|
||||
content="Original telemetry content",
|
||||
)
|
||||
|
||||
operations, contexts, fake_operation, fake_contextualize = _recording_contexts()
|
||||
monkeypatch.setattr(edit_note_module.telemetry, "operation", fake_operation)
|
||||
monkeypatch.setattr(edit_note_module.telemetry, "contextualize", fake_contextualize)
|
||||
|
||||
await edit_note_module.edit_note(
|
||||
"notes/editable-telemetry-note",
|
||||
operation="append",
|
||||
content="\n\nAppended telemetry content",
|
||||
project=test_project.name,
|
||||
output_format="json",
|
||||
)
|
||||
|
||||
assert operations[0] == (
|
||||
"mcp.tool.edit_note",
|
||||
{
|
||||
"entrypoint": "mcp",
|
||||
"tool_name": "edit_note",
|
||||
"requested_project": test_project.name,
|
||||
"workspace_id": None,
|
||||
"edit_operation": "append",
|
||||
"output_format": "json",
|
||||
"has_section": False,
|
||||
"has_find_text": False,
|
||||
"expected_replacements": 1,
|
||||
},
|
||||
)
|
||||
operation_names = [name for name, _ in operations]
|
||||
assert "api.request.knowledge.resolve_entity" in operation_names
|
||||
assert "api.request.knowledge.edit_entity" in operation_names
|
||||
assert _contains_context(
|
||||
contexts,
|
||||
{
|
||||
"project_name": test_project.name,
|
||||
"route_mode": "local_asgi",
|
||||
},
|
||||
)
|
||||
assert _contains_context(
|
||||
contexts,
|
||||
{
|
||||
"project_name": test_project.name,
|
||||
"tool_name": "edit_note",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_build_context_emits_root_operation_and_project_context(
|
||||
app, test_project, monkeypatch
|
||||
) -> None:
|
||||
await write_note_module.write_note(
|
||||
project=test_project.name,
|
||||
title="Context Telemetry Note",
|
||||
directory="notes",
|
||||
content="Context telemetry content",
|
||||
)
|
||||
|
||||
operations, contexts, fake_operation, fake_contextualize = _recording_contexts()
|
||||
monkeypatch.setattr(build_context_module.telemetry, "operation", fake_operation)
|
||||
monkeypatch.setattr(build_context_module.telemetry, "contextualize", fake_contextualize)
|
||||
|
||||
await build_context_module.build_context(
|
||||
url="memory://notes/context-telemetry-note",
|
||||
project=test_project.name,
|
||||
depth=2,
|
||||
timeframe="7d",
|
||||
page=1,
|
||||
page_size=5,
|
||||
max_related=3,
|
||||
output_format="json",
|
||||
)
|
||||
|
||||
assert operations[0] == (
|
||||
"mcp.tool.build_context",
|
||||
{
|
||||
"entrypoint": "mcp",
|
||||
"tool_name": "build_context",
|
||||
"requested_project": test_project.name,
|
||||
"workspace_id": None,
|
||||
"depth": 2,
|
||||
"timeframe": "7d",
|
||||
"page": 1,
|
||||
"page_size": 5,
|
||||
"max_related": 3,
|
||||
"output_format": "json",
|
||||
"is_memory_url": True,
|
||||
},
|
||||
)
|
||||
assert "api.request.memory.build_context" in [name for name, _ in operations]
|
||||
assert _contains_context(
|
||||
contexts,
|
||||
{
|
||||
"project_name": test_project.name,
|
||||
"route_mode": "local_asgi",
|
||||
},
|
||||
)
|
||||
assert _contains_context(
|
||||
contexts,
|
||||
{
|
||||
"project_name": test_project.name,
|
||||
"tool_name": "build_context",
|
||||
},
|
||||
)
|
||||
@@ -1257,6 +1257,11 @@ class TestWriteNoteOverwriteGuard:
|
||||
# Set config to allow overwrites by default
|
||||
app_config.write_note_overwrite_default = True
|
||||
config_module._CONFIG_CACHE = app_config
|
||||
# Pin mtime+size to the on-disk file so the cache guard sees a match
|
||||
# and keeps our injected config instead of re-reading from disk.
|
||||
_st = config_manager.config_file.stat()
|
||||
config_module._CONFIG_MTIME = _st.st_mtime
|
||||
config_module._CONFIG_SIZE = _st.st_size
|
||||
|
||||
try:
|
||||
await write_note(
|
||||
@@ -1281,6 +1286,9 @@ class TestWriteNoteOverwriteGuard:
|
||||
# Restore config
|
||||
app_config.write_note_overwrite_default = False
|
||||
config_module._CONFIG_CACHE = app_config
|
||||
_st = config_manager.config_file.stat()
|
||||
config_module._CONFIG_MTIME = _st.st_mtime
|
||||
config_module._CONFIG_SIZE = _st.st_size
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_write_note_new_note_unaffected(self, app, test_project):
|
||||
|
||||
@@ -91,6 +91,26 @@ def test_relation_response():
|
||||
assert relation.context is None
|
||||
|
||||
|
||||
def test_relation_response_allows_long_relation_type():
|
||||
"""Long relation labels should round-trip because stored data has no DB length cap."""
|
||||
long_relation_type = (
|
||||
"**Architecture/efficiency concern:** "
|
||||
"the orchestration prompt expanded a short edge label into a full descriptive note "
|
||||
"that is much longer than 200 characters but still represents the stored relation type."
|
||||
)
|
||||
data = {
|
||||
"permalink": "test/123/long/test/456",
|
||||
"from_id": "test/123",
|
||||
"to_id": "test/456",
|
||||
"relation_type": long_relation_type,
|
||||
"from_entity": {"permalink": "test/123"},
|
||||
"to_entity": {"permalink": "test/456"},
|
||||
}
|
||||
|
||||
relation = RelationResponse.model_validate(data)
|
||||
assert relation.relation_type == long_relation_type
|
||||
|
||||
|
||||
def test_relation_response_with_null_permalink():
|
||||
"""Test RelationResponse handles null permalinks by falling back to file_path (fixes issue #483).
|
||||
|
||||
|
||||
@@ -2426,3 +2426,116 @@ async def test_fast_write_entity_null_user_id(entity_service: EntityService):
|
||||
entity = await entity_service.fast_write_entity(schema, external_id=str(uuid.uuid4()))
|
||||
assert entity.created_by is None
|
||||
assert entity.last_updated_by is None
|
||||
|
||||
|
||||
# --- Concurrent Delete Resilience ---
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_delete_entity_by_id_already_deleted(entity_service: EntityService):
|
||||
"""delete_entity returns True when entity was already deleted (concurrent delete)."""
|
||||
entity_data = EntitySchema(
|
||||
title="ConcurrentDeleteTarget",
|
||||
directory="test",
|
||||
note_type="test",
|
||||
)
|
||||
created = await entity_service.create_entity(entity_data)
|
||||
entity_id = created.id
|
||||
|
||||
# Delete once - should succeed
|
||||
assert await entity_service.delete_entity(entity_id) is True
|
||||
|
||||
# Delete again by ID - should return True (already deleted), not raise ValueError
|
||||
assert await entity_service.delete_entity(entity_id) is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_delete_entity_by_permalink_already_deleted(entity_service: EntityService):
|
||||
"""delete_entity returns True when entity was already deleted by permalink."""
|
||||
entity_data = EntitySchema(
|
||||
title="ConcurrentDeleteByPermalink",
|
||||
directory="test",
|
||||
note_type="test",
|
||||
)
|
||||
created = await entity_service.create_entity(entity_data)
|
||||
|
||||
# Delete once
|
||||
assert await entity_service.delete_entity(created.id) is True
|
||||
|
||||
# Delete again by permalink - should return True (EntityNotFoundError caught)
|
||||
assert await entity_service.delete_entity(entity_data.permalink) is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_delete_directory_concurrent_resilience(entity_service: EntityService):
|
||||
"""delete_directory succeeds even when entities are concurrently deleted."""
|
||||
# Create entities in a directory
|
||||
entities = []
|
||||
for i in range(5):
|
||||
entity_data = EntitySchema(
|
||||
title=f"DirEntity{i}",
|
||||
directory="concurrent-test",
|
||||
note_type="test",
|
||||
)
|
||||
created = await entity_service.create_entity(entity_data)
|
||||
entities.append(created)
|
||||
|
||||
# Delete some entities directly to simulate concurrent delete
|
||||
await entity_service.delete_entity(entities[0].id)
|
||||
await entity_service.delete_entity(entities[2].id)
|
||||
|
||||
# Now delete the directory - should handle already-deleted entities gracefully
|
||||
result = await entity_service.delete_directory("concurrent-test")
|
||||
|
||||
# Only 3 remain in DB (2 were already deleted before the directory query)
|
||||
assert result.total_files == 3
|
||||
assert result.successful_deletes == 3
|
||||
assert result.failed_deletes == 0
|
||||
assert len(result.errors) == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_delete_directory_all_already_deleted(entity_service: EntityService):
|
||||
"""delete_directory handles case where all entities were concurrently deleted."""
|
||||
# Create entities
|
||||
created_entities = []
|
||||
for i in range(3):
|
||||
entity_data = EntitySchema(
|
||||
title=f"AllGone{i}",
|
||||
directory="all-deleted",
|
||||
note_type="test",
|
||||
)
|
||||
created = await entity_service.create_entity(entity_data)
|
||||
created_entities.append(created)
|
||||
|
||||
# Delete all entities directly
|
||||
for e in created_entities:
|
||||
await entity_service.delete_entity(e.id)
|
||||
|
||||
# Directory delete should report empty (entities no longer found in prefix query)
|
||||
result = await entity_service.delete_directory("all-deleted")
|
||||
assert result.total_files == 0
|
||||
assert result.successful_deletes == 0
|
||||
assert result.failed_deletes == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_delete_directory_entity_deleted_between_query_and_delete(
|
||||
entity_service: EntityService,
|
||||
):
|
||||
"""Simulates the real race condition: entity exists in prefix query but is deleted
|
||||
by a concurrent request before delete_entity is called."""
|
||||
# Create entities
|
||||
entity_data = EntitySchema(title="RaceTarget", directory="race-dir", note_type="test")
|
||||
created = await entity_service.create_entity(entity_data)
|
||||
|
||||
# Get the entities via prefix query (as delete_directory does)
|
||||
entities = await entity_service.repository.find_by_directory_prefix("race-dir")
|
||||
assert len(entities) == 1
|
||||
|
||||
# Now delete the entity behind the scenes (simulating a concurrent request)
|
||||
await entity_service.delete_entity(created.id)
|
||||
|
||||
# Call delete_entity with the stale entity ID - should return True, not raise
|
||||
result = await entity_service.delete_entity(entities[0].id)
|
||||
assert result is True
|
||||
|
||||
@@ -0,0 +1,138 @@
|
||||
"""Telemetry coverage for entity service write/edit/reindex paths."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
from contextlib import contextmanager
|
||||
|
||||
import pytest
|
||||
|
||||
from basic_memory.schemas import Entity as EntitySchema
|
||||
|
||||
entity_service_module = importlib.import_module("basic_memory.services.entity_service")
|
||||
|
||||
|
||||
def _capture_spans():
|
||||
spans: list[tuple[str, dict]] = []
|
||||
|
||||
@contextmanager
|
||||
def fake_span(name: str, **attrs):
|
||||
spans.append((name, attrs))
|
||||
yield
|
||||
|
||||
return spans, fake_span
|
||||
|
||||
|
||||
def _assert_names_in_order(names: list[str], expected: list[str]) -> None:
|
||||
cursor = 0
|
||||
for expected_name in expected:
|
||||
cursor = names.index(expected_name, cursor) + 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_create_entity_emits_expected_phase_spans(entity_service, monkeypatch) -> None:
|
||||
spans, fake_span = _capture_spans()
|
||||
monkeypatch.setattr(entity_service_module.telemetry, "span", fake_span)
|
||||
|
||||
schema = EntitySchema(
|
||||
title="Telemetry Create",
|
||||
directory="notes",
|
||||
note_type="note",
|
||||
content_type="text/markdown",
|
||||
content="Create telemetry content",
|
||||
)
|
||||
|
||||
entity = await entity_service.create_entity(schema)
|
||||
|
||||
assert entity.title == "Telemetry Create"
|
||||
span_names = [name for name, _ in spans]
|
||||
_assert_names_in_order(
|
||||
span_names,
|
||||
[
|
||||
"entity_service.create.resolve_permalink",
|
||||
"entity_service.create.write_file",
|
||||
"file_service.write",
|
||||
"entity_service.create.parse_markdown",
|
||||
"entity_service.create.upsert_entity",
|
||||
"entity_service.upsert.base_entity",
|
||||
"entity_service.upsert.relations",
|
||||
"entity_service.upsert.persist_checksum",
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_edit_entity_emits_expected_phase_spans(entity_service, monkeypatch) -> None:
|
||||
created = await entity_service.create_entity(
|
||||
EntitySchema(
|
||||
title="Telemetry Edit",
|
||||
directory="notes",
|
||||
note_type="note",
|
||||
content_type="text/markdown",
|
||||
content="Before edit",
|
||||
)
|
||||
)
|
||||
|
||||
spans, fake_span = _capture_spans()
|
||||
monkeypatch.setattr(entity_service_module.telemetry, "span", fake_span)
|
||||
|
||||
updated = await entity_service.edit_entity(
|
||||
created.file_path,
|
||||
operation="append",
|
||||
content="\n\nAfter edit",
|
||||
)
|
||||
|
||||
assert updated.id == created.id
|
||||
span_names = [name for name, _ in spans]
|
||||
_assert_names_in_order(
|
||||
span_names,
|
||||
[
|
||||
"entity_service.edit.resolve_entity",
|
||||
"entity_service.edit.read_file",
|
||||
"file_service.read",
|
||||
"entity_service.edit.apply_operation",
|
||||
"entity_service.edit.write_file",
|
||||
"file_service.write",
|
||||
"entity_service.edit.parse_markdown",
|
||||
"entity_service.edit.upsert_entity",
|
||||
"entity_service.upsert.base_entity",
|
||||
"entity_service.upsert.relations",
|
||||
"entity_service.upsert.persist_checksum",
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_reindex_entity_emits_expected_phase_spans(entity_service, monkeypatch) -> None:
|
||||
created = await entity_service.create_entity(
|
||||
EntitySchema(
|
||||
title="Telemetry Reindex",
|
||||
directory="notes",
|
||||
note_type="note",
|
||||
content_type="text/markdown",
|
||||
content="Reindex telemetry content",
|
||||
)
|
||||
)
|
||||
|
||||
spans, fake_span = _capture_spans()
|
||||
monkeypatch.setattr(entity_service_module.telemetry, "span", fake_span)
|
||||
|
||||
await entity_service.reindex_entity(created.id)
|
||||
|
||||
span_names = [name for name, _ in spans]
|
||||
_assert_names_in_order(
|
||||
span_names,
|
||||
[
|
||||
"entity_service.reindex.load_entity",
|
||||
"entity_service.reindex.read_file",
|
||||
"file_service.read_content",
|
||||
"entity_service.reindex.parse_markdown",
|
||||
"entity_service.reindex.upsert_entity",
|
||||
"entity_service.upsert.base_entity",
|
||||
"entity_service.upsert.relations",
|
||||
"entity_service.upsert.hydrate_entity",
|
||||
"entity_service.reindex.update_checksum",
|
||||
],
|
||||
)
|
||||
if entity_service.search_service is not None:
|
||||
assert "entity_service.reindex.search_index" in span_names
|
||||
@@ -0,0 +1,86 @@
|
||||
"""Tests for EntityWriteResult content variants."""
|
||||
|
||||
import pytest
|
||||
|
||||
from basic_memory.file_utils import remove_frontmatter
|
||||
from basic_memory.schemas import Entity as EntitySchema
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_create_entity_with_content_returns_full_and_search_content(
|
||||
entity_service, file_service
|
||||
) -> None:
|
||||
result = await entity_service.create_entity_with_content(
|
||||
EntitySchema(
|
||||
title="Create Write Result",
|
||||
directory="notes",
|
||||
note_type="note",
|
||||
content="Create body content",
|
||||
)
|
||||
)
|
||||
|
||||
file_path = file_service.get_entity_path(result.entity)
|
||||
file_content, _ = await file_service.read_file(file_path)
|
||||
|
||||
assert result.content == file_content
|
||||
assert result.search_content == remove_frontmatter(file_content)
|
||||
assert result.search_content == "Create body content"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_update_entity_with_content_returns_full_and_search_content(
|
||||
entity_service, file_service
|
||||
) -> None:
|
||||
created = await entity_service.create_entity(
|
||||
EntitySchema(
|
||||
title="Update Write Result",
|
||||
directory="notes",
|
||||
note_type="note",
|
||||
content="Original body content",
|
||||
)
|
||||
)
|
||||
|
||||
result = await entity_service.update_entity_with_content(
|
||||
created,
|
||||
EntitySchema(
|
||||
title="Update Write Result",
|
||||
directory="notes",
|
||||
note_type="note",
|
||||
content="Updated body content",
|
||||
),
|
||||
)
|
||||
|
||||
file_path = file_service.get_entity_path(result.entity)
|
||||
file_content, _ = await file_service.read_file(file_path)
|
||||
|
||||
assert result.content == file_content
|
||||
assert result.search_content == remove_frontmatter(file_content)
|
||||
assert result.search_content == "Updated body content"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_edit_entity_with_content_returns_full_and_search_content(
|
||||
entity_service, file_service
|
||||
) -> None:
|
||||
created = await entity_service.create_entity(
|
||||
EntitySchema(
|
||||
title="Edit Write Result",
|
||||
directory="notes",
|
||||
note_type="note",
|
||||
content="Original body content",
|
||||
)
|
||||
)
|
||||
|
||||
result = await entity_service.edit_entity_with_content(
|
||||
identifier=created.permalink,
|
||||
operation="find_replace",
|
||||
content="Edited body content",
|
||||
find_text="Original body content",
|
||||
)
|
||||
|
||||
file_path = file_service.get_entity_path(result.entity)
|
||||
file_content, _ = await file_service.read_file(file_path)
|
||||
|
||||
assert result.content == file_content
|
||||
assert result.search_content == remove_frontmatter(file_content)
|
||||
assert result.search_content == "Edited body content"
|
||||
@@ -778,6 +778,8 @@ async def test_add_project_with_project_root_sanitizes_paths(
|
||||
from basic_memory import config as config_module
|
||||
|
||||
config_module._CONFIG_CACHE = None
|
||||
config_module._CONFIG_MTIME = None
|
||||
config_module._CONFIG_SIZE = None
|
||||
|
||||
test_cases = [
|
||||
# (project_name, user_path, expected_sanitized_name)
|
||||
@@ -845,6 +847,8 @@ async def test_add_project_with_project_root_rejects_escape_attempts(
|
||||
from basic_memory import config as config_module
|
||||
|
||||
config_module._CONFIG_CACHE = None
|
||||
config_module._CONFIG_MTIME = None
|
||||
config_module._CONFIG_SIZE = None
|
||||
|
||||
# All of these should succeed by being sanitized to paths under project_root
|
||||
# The sanitization removes dangerous patterns, so they don't escape
|
||||
@@ -931,6 +935,8 @@ async def test_add_project_with_project_root_normalizes_case(
|
||||
from basic_memory import config as config_module
|
||||
|
||||
config_module._CONFIG_CACHE = None
|
||||
config_module._CONFIG_MTIME = None
|
||||
config_module._CONFIG_SIZE = None
|
||||
|
||||
test_cases = [
|
||||
# (input_path, expected_normalized_path)
|
||||
@@ -985,6 +991,8 @@ async def test_add_project_with_project_root_detects_case_collisions(
|
||||
from basic_memory import config as config_module
|
||||
|
||||
config_module._CONFIG_CACHE = None
|
||||
config_module._CONFIG_MTIME = None
|
||||
config_module._CONFIG_SIZE = None
|
||||
|
||||
# First, create a project with lowercase path
|
||||
first_project = "documents-project"
|
||||
@@ -1159,6 +1167,8 @@ async def test_add_project_nested_validation_with_project_root(
|
||||
from basic_memory import config as config_module
|
||||
|
||||
config_module._CONFIG_CACHE = None
|
||||
config_module._CONFIG_MTIME = None
|
||||
config_module._CONFIG_SIZE = None
|
||||
|
||||
parent_project_name = f"cloud-parent-{os.urandom(4).hex()}"
|
||||
child_project_name = f"cloud-child-{os.urandom(4).hex()}"
|
||||
|
||||
@@ -251,6 +251,55 @@ async def test_embedding_status_healthy(project_service: ProjectService, test_gr
|
||||
assert status.reindex_reason is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_embedding_status_excludes_stale_entity_ids(
|
||||
project_service: ProjectService, test_graph, test_project
|
||||
):
|
||||
"""Stale rows in search_index for deleted entities should not inflate counts.
|
||||
|
||||
Regression test for #670: after reindex, project info reported missing embeddings
|
||||
because stale entity_ids in search_index/search_vector_chunks inflated total_indexed_entities.
|
||||
"""
|
||||
# Insert a stale search_index row for an entity_id that doesn't exist in the entity table.
|
||||
# Include 'id' column — required NOT NULL on Postgres (regular table),
|
||||
# ignored on SQLite (FTS5 virtual table where id is UNINDEXED).
|
||||
stale_entity_id = 999999
|
||||
await project_service.repository.execute_query(
|
||||
text(
|
||||
"INSERT INTO search_index "
|
||||
"(id, entity_id, project_id, type, title, permalink, content_stems, "
|
||||
"content_snippet, file_path, metadata) "
|
||||
"VALUES (:id, :eid, :pid, 'entity', 'Stale Note', 'stale-note', "
|
||||
"'stale content', 'stale snippet', 'stale.md', '{}')"
|
||||
),
|
||||
{"id": stale_entity_id, "eid": stale_entity_id, "pid": test_project.id},
|
||||
)
|
||||
|
||||
with patch.object(
|
||||
type(project_service),
|
||||
"config_manager",
|
||||
new_callable=lambda: property(
|
||||
lambda self: _config_manager_with(semantic_search_enabled=True)
|
||||
),
|
||||
):
|
||||
status = await project_service.get_embedding_status(test_project.id)
|
||||
|
||||
# The stale entity_id should NOT be counted in total_indexed_entities.
|
||||
# Count real entities that have search_index rows (the stale one should be excluded).
|
||||
real_indexed_result = await project_service.repository.execute_query(
|
||||
text(
|
||||
"SELECT COUNT(DISTINCT si.entity_id) FROM search_index si "
|
||||
"JOIN entity e ON e.id = si.entity_id "
|
||||
"WHERE si.project_id = :pid"
|
||||
),
|
||||
{"pid": test_project.id},
|
||||
)
|
||||
real_indexed_count = real_indexed_result.scalar() or 0
|
||||
|
||||
# Exact match — stale entity_id must not inflate the count
|
||||
assert status.total_indexed_entities == real_indexed_count
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_project_info_includes_embedding_status(
|
||||
project_service: ProjectService, test_graph, test_project
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
"""Telemetry coverage for search execution phases."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
from contextlib import contextmanager
|
||||
|
||||
import pytest
|
||||
|
||||
from basic_memory.schemas.search import SearchQuery
|
||||
|
||||
search_service_module = importlib.import_module("basic_memory.services.search_service")
|
||||
|
||||
|
||||
def _capture_spans():
|
||||
spans: list[tuple[str, dict]] = []
|
||||
|
||||
@contextmanager
|
||||
def fake_span(name: str, **attrs):
|
||||
spans.append((name, attrs))
|
||||
yield
|
||||
|
||||
return spans, fake_span
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_search_service_wraps_repository_search(search_service, monkeypatch) -> None:
|
||||
spans, fake_span = _capture_spans()
|
||||
monkeypatch.setattr(search_service_module.telemetry, "span", fake_span)
|
||||
|
||||
await search_service.search(SearchQuery(text="Root Entity"))
|
||||
|
||||
assert spans[0] == (
|
||||
"search.execute",
|
||||
{
|
||||
"retrieval_mode": "fts",
|
||||
"has_query": True,
|
||||
"has_filters": False,
|
||||
"limit": 10,
|
||||
"offset": 0,
|
||||
},
|
||||
)
|
||||
assert spans[1] == (
|
||||
"search.repository_query",
|
||||
{
|
||||
"retrieval_mode": "fts",
|
||||
"phase": "repository_query",
|
||||
"has_query": True,
|
||||
"has_filters": False,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_search_service_emits_relaxed_retry_span(search_service, monkeypatch) -> None:
|
||||
spans, fake_span = _capture_spans()
|
||||
calls: list[dict] = []
|
||||
|
||||
async def fake_repository_search(**kwargs):
|
||||
calls.append(kwargs)
|
||||
return []
|
||||
|
||||
monkeypatch.setattr(search_service_module.telemetry, "span", fake_span)
|
||||
monkeypatch.setattr(search_service.repository, "search", fake_repository_search)
|
||||
|
||||
await search_service.search(SearchQuery(text="who are our main competitors and partners"))
|
||||
|
||||
assert [name for name, _ in spans] == [
|
||||
"search.execute",
|
||||
"search.repository_query",
|
||||
"search.relaxed_fts_retry",
|
||||
"search.repository_query",
|
||||
]
|
||||
assert spans[2] == (
|
||||
"search.relaxed_fts_retry",
|
||||
{
|
||||
"retrieval_mode": "fts",
|
||||
"token_count": 7,
|
||||
"limit": 10,
|
||||
"offset": 0,
|
||||
},
|
||||
)
|
||||
assert len(calls) == 2
|
||||
assert calls[0]["search_text"] == "who are our main competitors and partners"
|
||||
assert calls[1]["search_text"] == "main OR competitors OR partners"
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user