mirror of
https://github.com/basicmachines-co/basic-memory
synced 2026-06-21 13:47:35 +00:00
Compare commits
9 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 449645b452 | |||
| ec9b2c4046 | |||
| de7f15b7a2 | |||
| 630eeb94ab | |||
| 0eae0e1678 | |||
| 88a5b07b89 | |||
| 59e8a937ee | |||
| c07465d904 | |||
| dfb89e841c |
@@ -37,7 +37,17 @@ jobs:
|
||||
run: |
|
||||
pip install uv
|
||||
|
||||
- uses: extractions/setup-just@v3
|
||||
- name: Install just (Linux/macOS)
|
||||
if: runner.os != 'Windows'
|
||||
run: |
|
||||
curl --proto '=https' --tlsv1.2 -sSf https://just.systems/install.sh | bash -s -- --to /usr/local/bin
|
||||
|
||||
- name: Install just (Windows)
|
||||
if: runner.os == 'Windows'
|
||||
run: |
|
||||
# Install just using Chocolatey (pre-installed on GitHub Actions Windows runners)
|
||||
choco install just --yes
|
||||
shell: pwsh
|
||||
|
||||
- name: Create virtual env
|
||||
run: |
|
||||
@@ -45,7 +55,7 @@ jobs:
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
uv pip install -e ".[dev,semantic]"
|
||||
uv pip install -e .[dev]
|
||||
|
||||
- name: Run type checks
|
||||
run: |
|
||||
@@ -85,7 +95,9 @@ jobs:
|
||||
run: |
|
||||
pip install uv
|
||||
|
||||
- uses: extractions/setup-just@v3
|
||||
- name: Install just
|
||||
run: |
|
||||
curl --proto '=https' --tlsv1.2 -sSf https://just.systems/install.sh | bash -s -- --to /usr/local/bin
|
||||
|
||||
- name: Create virtual env
|
||||
run: |
|
||||
@@ -93,7 +105,7 @@ jobs:
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
uv pip install -e ".[dev,semantic]"
|
||||
uv pip install -e .[dev]
|
||||
|
||||
- name: Run tests (Postgres via testcontainers)
|
||||
run: |
|
||||
@@ -119,7 +131,9 @@ jobs:
|
||||
run: |
|
||||
pip install uv
|
||||
|
||||
- uses: extractions/setup-just@v3
|
||||
- name: Install just
|
||||
run: |
|
||||
curl --proto '=https' --tlsv1.2 -sSf https://just.systems/install.sh | bash -s -- --to /usr/local/bin
|
||||
|
||||
- name: Create virtual env
|
||||
run: |
|
||||
@@ -127,7 +141,7 @@ jobs:
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
uv pip install -e ".[dev,semantic]"
|
||||
uv pip install -e .[dev]
|
||||
|
||||
- name: Run combined coverage (SQLite + Postgres)
|
||||
run: |
|
||||
|
||||
@@ -57,4 +57,3 @@ claude-output
|
||||
.mcp.json
|
||||
.mcpregistry_*
|
||||
/.testmondata
|
||||
.benchmarks/
|
||||
|
||||
@@ -189,46 +189,27 @@ Flow: MCP Tool → Typed Client → HTTP API → Router → Service → Reposito
|
||||
|
||||
### Async Client Pattern (Important!)
|
||||
|
||||
**MCP tools use `get_project_client()` for per-project routing:**
|
||||
|
||||
```python
|
||||
from basic_memory.mcp.project_context import get_project_client
|
||||
|
||||
@mcp.tool()
|
||||
async def my_tool(project: str | None = None, context: Context | None = None):
|
||||
async with get_project_client(project, context) as (client, active_project):
|
||||
# client is routed based on project's mode (local ASGI or cloud HTTP)
|
||||
response = await call_get(client, "/path")
|
||||
return response
|
||||
```
|
||||
|
||||
**CLI commands and non-project-scoped code use `get_client()` directly:**
|
||||
**All MCP tools and CLI commands use the context manager pattern for HTTP clients:**
|
||||
|
||||
```python
|
||||
from basic_memory.mcp.async_client import get_client
|
||||
|
||||
async def my_cli_command():
|
||||
async def my_mcp_tool():
|
||||
async with get_client() as client:
|
||||
# Use client for API calls
|
||||
response = await call_get(client, "/path")
|
||||
return response
|
||||
|
||||
# Per-project routing (when project name is known):
|
||||
async with get_client(project_name="research") as client:
|
||||
...
|
||||
```
|
||||
|
||||
**Do NOT use:**
|
||||
- ❌ `from basic_memory.mcp.async_client import client` (deprecated module-level client)
|
||||
- ❌ Manual auth header management
|
||||
- ❌ `inject_auth_header()` (deleted)
|
||||
- ❌ Separate `get_client()` + `get_active_project()` in MCP tools (use `get_project_client()` instead)
|
||||
|
||||
**Key principles:**
|
||||
- Auth happens at client creation, not per-request
|
||||
- Proper resource management via context managers
|
||||
- Per-project routing: each project can be LOCAL or CLOUD independently
|
||||
- Cloud projects use API key (`cloud_api_key` in config) as Bearer token
|
||||
- Routing priority: factory injection > force-local > per-project cloud > global cloud > local ASGI
|
||||
- Supports three modes: Local (ASGI), CLI cloud (HTTP + auth), Cloud app (factory injection)
|
||||
- Factory pattern enables dependency injection for cloud consolidation
|
||||
|
||||
**For cloud app integration:**
|
||||
@@ -269,19 +250,15 @@ See SPEC-16 for full context manager refactor details.
|
||||
- List projects: `basic-memory project list`
|
||||
- Add project: `basic-memory project add "name" ~/path`
|
||||
- Project info: `basic-memory project info`
|
||||
- Set cloud mode: `basic-memory project set-cloud "name"`
|
||||
- Set local mode: `basic-memory project set-local "name"`
|
||||
- One-way sync (local -> cloud): `basic-memory project sync`
|
||||
- Bidirectional sync: `basic-memory project bisync`
|
||||
- Integrity check: `basic-memory project check`
|
||||
|
||||
**Cloud Commands (requires subscription):**
|
||||
- Authenticate (global): `basic-memory cloud login`
|
||||
- Logout (global): `basic-memory cloud logout`
|
||||
- Authenticate: `basic-memory cloud login`
|
||||
- Logout: `basic-memory cloud logout`
|
||||
- Check cloud status: `basic-memory cloud status`
|
||||
- Setup cloud sync: `basic-memory cloud setup`
|
||||
- Save API key: `basic-memory cloud set-key bmc_...`
|
||||
- Create API key: `basic-memory cloud create-key "name"`
|
||||
- Manage snapshots: `basic-memory cloud snapshot [create|list|delete|show|browse]`
|
||||
- Restore from snapshot: `basic-memory cloud restore <path> --snapshot <id>`
|
||||
|
||||
@@ -352,22 +329,9 @@ Basic Memory now supports cloud synchronization and storage (requires active sub
|
||||
- Background relation resolution (non-blocking startup)
|
||||
- API performance optimizations (SPEC-11)
|
||||
|
||||
**Per-Project Cloud Routing:**
|
||||
**CLI Routing Flags:**
|
||||
|
||||
Individual projects can be routed through the cloud while others stay local, using an API key:
|
||||
|
||||
```bash
|
||||
# Save API key and set project to cloud mode
|
||||
basic-memory cloud set-key bmc_abc123...
|
||||
basic-memory project set-cloud research # route through cloud
|
||||
basic-memory project set-local research # revert to local
|
||||
```
|
||||
|
||||
MCP tools use `get_project_client()` which automatically routes based on the project's mode. Cloud projects use the `cloud_api_key` from config as Bearer token.
|
||||
|
||||
**CLI Routing Flags (Global Cloud Mode):**
|
||||
|
||||
When global cloud mode is enabled, CLI commands route to the cloud API by default. Use `--local` and `--cloud` flags to override:
|
||||
When cloud mode is enabled, CLI commands route to the cloud API by default. Use `--local` and `--cloud` flags to override:
|
||||
|
||||
```bash
|
||||
# Force local routing (ignore cloud mode)
|
||||
@@ -384,7 +348,6 @@ Key behaviors:
|
||||
- This allows simultaneous use of local Claude Desktop and cloud-based clients
|
||||
- Some commands (like `project default`, `project sync-config`, `project move`) require `--local` in cloud mode since they modify local configuration
|
||||
- Environment variable `BASIC_MEMORY_FORCE_LOCAL=true` forces local routing globally
|
||||
- Per-project cloud routing via API key works independently of global cloud mode
|
||||
|
||||
## AI-Human Collaborative Development
|
||||
|
||||
|
||||
+2
-2
@@ -1,11 +1,11 @@
|
||||
# CHANGELOG
|
||||
|
||||
## v0.18.3 (2026-02-12)
|
||||
## v0.18.4 (2026-02-12)
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- Use global `--header` flag for Tigris consistency on all rclone transactions
|
||||
([`7fcf587`](https://github.com/basicmachines-co/basic-memory/commit/7fcf587))
|
||||
([`0eae0e1`](https://github.com/basicmachines-co/basic-memory/commit/0eae0e1))
|
||||
- `--header-download` / `--header-upload` only apply to GET/PUT requests, missing S3
|
||||
ListObjectsV2 calls that bisync issues first. Non-US users saw stale edge-cached metadata.
|
||||
- `--header` applies to ALL HTTP transactions (list, download, upload), fixing bisync for
|
||||
|
||||
-494
@@ -1,494 +0,0 @@
|
||||
# Note Format Reference
|
||||
|
||||
Every document in Basic Memory is a plain Markdown file. Files are the source of truth — changes to files automatically update the knowledge graph in the database. You maintain complete ownership, files work with git, and knowledge persists independently of any AI conversation.
|
||||
|
||||
## Document Structure
|
||||
|
||||
A note has three parts: YAML frontmatter, content (observations), and relations.
|
||||
|
||||
```markdown
|
||||
---
|
||||
title: Coffee Brewing Methods
|
||||
type: note
|
||||
tags: [coffee, brewing]
|
||||
permalink: coffee-brewing-methods
|
||||
---
|
||||
|
||||
# Coffee Brewing Methods
|
||||
|
||||
## Observations
|
||||
- [method] Pour over provides more flavor clarity than French press
|
||||
- [technique] Water temperature at 205°F extracts optimal compounds #brewing
|
||||
- [preference] Ethiopian beans work well with lighter roasts (personal experience)
|
||||
|
||||
## Relations
|
||||
- relates_to [[Coffee Bean Origins]]
|
||||
- requires [[Proper Grinding Technique]]
|
||||
- contrasts_with [[Tea Brewing Methods]]
|
||||
```
|
||||
|
||||
The `## Observations` and `## Relations` headings are conventional but not required — the parser detects observations and relations by their syntax patterns anywhere in the document.
|
||||
|
||||
## Frontmatter
|
||||
|
||||
YAML metadata between `---` fences at the top of the file.
|
||||
|
||||
| Field | Required | Default | Description |
|
||||
|-------|----------|---------|-------------|
|
||||
| `title` | No | filename stem | Used for linking and references. Auto-set from filename if missing. |
|
||||
| `type` | No | `note` | Entity type. Used for schema resolution and filtering. |
|
||||
| `tags` | No | `[]` | List or comma-separated string. Used for organization and search. |
|
||||
| `permalink` | No | generated from title | Stable identifier. Persists even if the file moves. |
|
||||
| `schema` | No | none | Schema attachment — dict (inline), string (reference), or omitted (implicit). |
|
||||
|
||||
Custom fields are allowed. Any key not in the standard set is stored as `entity_metadata` and indexed for search and filtering.
|
||||
|
||||
```yaml
|
||||
---
|
||||
title: Paul Graham
|
||||
type: Person
|
||||
tags: [startups, essays, lisp]
|
||||
permalink: paul-graham
|
||||
status: active
|
||||
source: wikipedia
|
||||
---
|
||||
```
|
||||
|
||||
Here `status` and `source` are custom fields stored in `entity_metadata`.
|
||||
|
||||
### Frontmatter Value Handling
|
||||
|
||||
YAML automatically converts some values to native types. Basic Memory normalizes them:
|
||||
|
||||
- Date strings (`2025-10-24`) → kept as ISO format strings
|
||||
- Numbers (`1.0`) → converted to strings
|
||||
- Booleans (`true`) → converted to strings (`"True"`)
|
||||
- Lists and dicts → preserved, items normalized recursively
|
||||
|
||||
This prevents errors when downstream code expects string values.
|
||||
|
||||
## Observations
|
||||
|
||||
An observation is a categorized fact about the entity. Written as a Markdown list item.
|
||||
|
||||
**Syntax:**
|
||||
|
||||
```
|
||||
- [category] content text #tag1 #tag2 (context)
|
||||
```
|
||||
|
||||
| Part | Required | Description |
|
||||
|------|----------|-------------|
|
||||
| `[category]` | Yes | Classification in square brackets. Any text except `[]()` chars. |
|
||||
| content | Yes | The fact or statement. |
|
||||
| `#tags` | No | Inline tags. Space-separated, each starting with `#`. |
|
||||
| `(context)` | No | Parenthesized text at end of line. Supporting details or source. |
|
||||
|
||||
### Examples
|
||||
|
||||
```markdown
|
||||
- [tech] Uses SQLite for storage #database
|
||||
- [design] Follows local-first architecture #architecture
|
||||
- [decision] Selected bcrypt for passwords #security (based on OWASP audit)
|
||||
- [name] Paul Graham
|
||||
- [expertise] Startups
|
||||
- [expertise] Lisp
|
||||
- [expertise] Essay writing
|
||||
```
|
||||
|
||||
Array-like fields use repeated categories — multiple `[expertise]` observations above.
|
||||
|
||||
### What Is Not an Observation
|
||||
|
||||
The parser excludes these list item patterns:
|
||||
|
||||
| Pattern | Example | Reason |
|
||||
|---------|---------|--------|
|
||||
| Checkboxes | `- [ ] Todo item`, `- [x] Done`, `- [-] Cancelled` | Task list syntax |
|
||||
| Markdown links | `- [text](url)` | URL link syntax |
|
||||
| Bare wiki links | `- [[Target]]` | Treated as a relation instead |
|
||||
|
||||
A list item with `#tags` but no `[category]` is still parsed — the tags are extracted and the category defaults to `Note`.
|
||||
|
||||
## Relations
|
||||
|
||||
Relations connect documents to form the knowledge graph. There are two kinds.
|
||||
|
||||
### Explicit Relations
|
||||
|
||||
Written as list items with a relation type and a `[[wiki link]]` target.
|
||||
|
||||
**Syntax:**
|
||||
|
||||
```
|
||||
- relation_type [[Target Entity]] (context)
|
||||
```
|
||||
|
||||
| Part | Required | Description |
|
||||
|------|----------|-------------|
|
||||
| `relation_type` | No | Text before `[[`. Defaults to `relates_to` if omitted. |
|
||||
| `[[Target]]` | Yes | Wiki link to the target entity. Matched by title or permalink. |
|
||||
| `(context)` | No | Parenthesized text after `]]`. Supporting details. |
|
||||
|
||||
### Examples
|
||||
|
||||
```markdown
|
||||
- implements [[Search Design]]
|
||||
- depends_on [[Database Schema]]
|
||||
- works_at [[Y Combinator]] (co-founder)
|
||||
- [[Some Entity]]
|
||||
```
|
||||
|
||||
The last example — a bare `[[wiki link]]` in a list item — gets relation type `relates_to`.
|
||||
|
||||
Common relation types:
|
||||
- `implements`, `depends_on`, `relates_to`, `inspired_by`
|
||||
- `extends`, `part_of`, `contains`, `pairs_with`
|
||||
- `works_at`, `authored`, `collaborated_with`
|
||||
|
||||
Any text works as a relation type. These are conventions, not a fixed set.
|
||||
|
||||
### Inline References
|
||||
|
||||
Wiki links appearing in regular prose (not as list items) create implicit `links_to` relations.
|
||||
|
||||
```markdown
|
||||
This builds on [[Core Design]] and uses [[Utility Functions]].
|
||||
```
|
||||
|
||||
This creates two relations: `links_to [[Core Design]]` and `links_to [[Utility Functions]]`.
|
||||
|
||||
### Forward References
|
||||
|
||||
Relations can link to entities that don't exist yet. Basic Memory resolves them when the target is created.
|
||||
|
||||
## Permalinks and memory:// URLs
|
||||
|
||||
Every document has a unique **permalink** — a stable identifier derived from its title. You can set one explicitly in frontmatter, or let the system generate it.
|
||||
|
||||
```yaml
|
||||
permalink: auth-approaches-2024
|
||||
```
|
||||
|
||||
Permalinks form the basis of `memory://` URLs:
|
||||
|
||||
```
|
||||
memory://auth-approaches-2024 # By permalink
|
||||
memory://Authentication Approaches # By title (auto-resolves)
|
||||
memory://project/auth-approaches # By path
|
||||
```
|
||||
|
||||
Pattern matching is supported:
|
||||
|
||||
```
|
||||
memory://auth* # Starts with "auth"
|
||||
memory://*/approaches # Ends with "approaches"
|
||||
memory://project/*/requirements # Nested wildcard
|
||||
```
|
||||
|
||||
## Schemas
|
||||
|
||||
Schemas declare the expected structure of a note — which observation categories and relation types a well-formed note should have. They use Picoschema, a compact notation from Google's Dotprompt that fits naturally in YAML frontmatter.
|
||||
|
||||
### Picoschema Syntax
|
||||
|
||||
```yaml
|
||||
schema:
|
||||
name: string, full name # required field with description
|
||||
email?: string, contact email # ? = optional
|
||||
role?: string, job title
|
||||
works_at?: Organization, employer # capitalized type = entity reference
|
||||
tags?(array): string, categories # array of type
|
||||
status?(enum): [active, inactive] # enum with allowed values
|
||||
metadata?(object): # nested object
|
||||
updated_at?: string
|
||||
source?: string
|
||||
```
|
||||
|
||||
| Notation | Meaning | Example |
|
||||
|----------|---------|---------|
|
||||
| `field: type` | Required field | `name: string` |
|
||||
| `field?: type` | Optional field | `role?: string` |
|
||||
| `field(array): type` | Array of values | `expertise(array): string` |
|
||||
| `field?(enum): [vals]` | Enum with allowed values | `status?(enum): [active, inactive]` |
|
||||
| `field?(object):` | Nested object with sub-fields | `metadata?(object):` |
|
||||
| `, description` | Description after comma | `name: string, full name` |
|
||||
| `EntityName` | Capitalized type = entity reference | `works_at?: Organization` |
|
||||
|
||||
**Scalar types:** `string`, `integer`, `number`, `boolean`, `any`
|
||||
|
||||
Any type not in that set whose first letter is uppercase is treated as an entity reference (a relation target).
|
||||
|
||||
### Schema-to-Note Mapping
|
||||
|
||||
Schemas validate against existing observation/relation syntax. Note authors don't learn new syntax.
|
||||
|
||||
| Schema Declaration | Maps To | Example in Note |
|
||||
|--------------------|---------|-----------------|
|
||||
| `field: string` | Observation `[field] value` | `- [name] Paul Graham` |
|
||||
| `field?(array): string` | Multiple `[field]` observations | `- [expertise] Lisp` (repeated) |
|
||||
| `field?: EntityType` | Relation `field [[Target]]` | `- works_at [[Y Combinator]]` |
|
||||
| `field?(array): EntityType` | Multiple `field` relations | `- authored [[Book]]` (repeated) |
|
||||
| `tags` | Frontmatter `tags` array | `tags: [startups, essays]` |
|
||||
| `field?(enum): [vals]` | Observation `[field] value` where value is in the set | `- [status] active` |
|
||||
|
||||
Observations and relations not covered by the schema are valid — schemas describe a subset, not a straitjacket.
|
||||
|
||||
### Schema Attachment
|
||||
|
||||
Three ways to attach a schema to a note, resolved in priority order:
|
||||
|
||||
**1. Inline schema** — `schema` is a dict in frontmatter:
|
||||
|
||||
```yaml
|
||||
---
|
||||
title: Team Standup 2024-01-15
|
||||
type: meeting
|
||||
schema:
|
||||
attendees(array): string, who was there
|
||||
decisions(array): string, what was decided
|
||||
action_items(array): string, follow-ups
|
||||
blockers?(array): string, anything stuck
|
||||
---
|
||||
```
|
||||
|
||||
Good for one-off structured notes or prototyping a schema before extracting it.
|
||||
|
||||
**2. Explicit reference** — `schema` is a string naming a schema note:
|
||||
|
||||
```yaml
|
||||
---
|
||||
title: Basic Memory
|
||||
schema: SoftwareProject
|
||||
---
|
||||
```
|
||||
|
||||
or by permalink:
|
||||
|
||||
```yaml
|
||||
---
|
||||
title: LLM Memory Patterns
|
||||
schema: schema/research-project
|
||||
---
|
||||
```
|
||||
|
||||
Use when the note's `type` differs from the schema it should validate against, or when multiple schema variants exist.
|
||||
|
||||
**3. Implicit by type** — no `schema` field, resolved by matching `type`:
|
||||
|
||||
```yaml
|
||||
---
|
||||
title: Paul Graham
|
||||
type: Person
|
||||
---
|
||||
```
|
||||
|
||||
The system looks up a schema note where `entity: Person`. If found, it applies. If not, no validation occurs.
|
||||
|
||||
**4. No schema** — perfectly fine. Most notes don't need one.
|
||||
|
||||
### Schema Notes
|
||||
|
||||
A schema is itself a Basic Memory note with `type: schema`. It lives anywhere (though `schema/` is the conventional directory).
|
||||
|
||||
```yaml
|
||||
# schema/Person.md
|
||||
---
|
||||
title: Person
|
||||
type: schema
|
||||
entity: Person
|
||||
version: 1
|
||||
schema:
|
||||
name: string, full name
|
||||
role?: string, job title or position
|
||||
works_at?: Organization, employer
|
||||
expertise?(array): string, areas of knowledge
|
||||
email?: string, contact email
|
||||
settings:
|
||||
validation: warn
|
||||
---
|
||||
|
||||
# Person
|
||||
|
||||
A human individual in the knowledge graph.
|
||||
```
|
||||
|
||||
| Field | Required | Description |
|
||||
|-------|----------|-------------|
|
||||
| `type` | Yes | Must be `schema` |
|
||||
| `entity` | Yes | The entity type this schema describes (e.g., `Person`) |
|
||||
| `version` | No | Schema version number (default: `1`) |
|
||||
| `schema` | Yes | Picoschema dict defining the fields |
|
||||
| `settings.validation` | No | Validation mode (default: `warn`) |
|
||||
|
||||
Schema notes are regular notes — they show up in search, can have observations and relations, and participate in the knowledge graph.
|
||||
|
||||
### Validation Modes
|
||||
|
||||
| Mode | Behavior |
|
||||
|------|----------|
|
||||
| `warn` | Warnings in output, doesn't block (default) |
|
||||
| `strict` | Errors that block sync, for CI/CD enforcement |
|
||||
| `off` | No validation |
|
||||
|
||||
### Validation Output
|
||||
|
||||
```
|
||||
$ bm schema validate people/ada-lovelace.md
|
||||
|
||||
⚠ Person schema validation:
|
||||
- Missing required field: name (expected [name] observation)
|
||||
- Missing optional field: role
|
||||
- Missing optional field: works_at (no relation found)
|
||||
|
||||
ℹ Unmatched observations: [fact] ×2, [born] ×1
|
||||
ℹ Unmatched relations: collaborated_with
|
||||
```
|
||||
|
||||
"Unmatched" items are informational — observations and relations the schema doesn't cover.
|
||||
|
||||
### Schema Inference
|
||||
|
||||
Generate schemas from existing notes by analyzing observation and relation frequency:
|
||||
|
||||
```
|
||||
$ bm schema infer Person
|
||||
|
||||
Analyzing 30 notes with type: Person...
|
||||
|
||||
Observations found:
|
||||
[name] 30/30 100% → name: string
|
||||
[role] 27/30 90% → role?: string
|
||||
[expertise] 18/30 60% → expertise?(array): string
|
||||
[email] 8/30 27% → email?: string
|
||||
|
||||
Relations found:
|
||||
works_at 22/30 73% → works_at?: Organization
|
||||
|
||||
Suggested schema:
|
||||
name: string, full name
|
||||
role?: string, job title
|
||||
expertise?(array): string, areas of knowledge
|
||||
email?: string, contact email
|
||||
works_at?: Organization, employer
|
||||
|
||||
Save to schema/Person.md? [y/n]
|
||||
```
|
||||
|
||||
Frequency thresholds:
|
||||
- **100% present** → required field
|
||||
- **25%+ present** → optional field
|
||||
- **Below 25%** → excluded from suggestion
|
||||
|
||||
### Schema Drift Detection
|
||||
|
||||
Track how usage patterns shift over time:
|
||||
|
||||
```
|
||||
$ bm schema diff Person
|
||||
|
||||
Schema drift detected:
|
||||
|
||||
+ expertise: now in 81% of notes (was 12%)
|
||||
- department: dropped to 3% of notes
|
||||
~ works_at: cardinality changed (one → many)
|
||||
|
||||
Update schema? [y/n/review]
|
||||
```
|
||||
|
||||
## Complete Examples
|
||||
|
||||
### Simple Note (No Schema)
|
||||
|
||||
```markdown
|
||||
---
|
||||
title: Project Ideas
|
||||
type: note
|
||||
tags: [ideas, brainstorm]
|
||||
---
|
||||
|
||||
# Project Ideas
|
||||
|
||||
## Observations
|
||||
- [idea] Build a CLI tool for markdown linting #tooling
|
||||
- [idea] Create a recipe knowledge base #cooking
|
||||
- [priority] Focus on developer tools first (Q1 goal)
|
||||
|
||||
## Relations
|
||||
- inspired_by [[Developer Workflow Research]]
|
||||
- part_of [[Q1 Planning]]
|
||||
```
|
||||
|
||||
### Schema-Validated Note
|
||||
|
||||
Schema at `schema/Person.md`:
|
||||
|
||||
```yaml
|
||||
---
|
||||
title: Person
|
||||
type: schema
|
||||
entity: Person
|
||||
version: 1
|
||||
schema:
|
||||
name: string, full name
|
||||
role?: string, job title or position
|
||||
works_at?: Organization, employer
|
||||
expertise?(array): string, areas of knowledge
|
||||
email?: string, contact email
|
||||
settings:
|
||||
validation: warn
|
||||
---
|
||||
|
||||
# Person
|
||||
|
||||
A human individual in the knowledge graph.
|
||||
```
|
||||
|
||||
Note at `people/paul-graham.md`:
|
||||
|
||||
```markdown
|
||||
---
|
||||
title: Paul Graham
|
||||
type: Person
|
||||
tags: [startups, essays, lisp]
|
||||
---
|
||||
|
||||
# Paul Graham
|
||||
|
||||
## Observations
|
||||
- [name] Paul Graham
|
||||
- [role] Essayist and investor
|
||||
- [expertise] Startups
|
||||
- [expertise] Lisp
|
||||
- [expertise] Essay writing
|
||||
- [fact] Created Viaweb, the first web app
|
||||
|
||||
## Relations
|
||||
- works_at [[Y Combinator]]
|
||||
- authored [[Hackers and Painters]]
|
||||
```
|
||||
|
||||
The `[fact]` observation and `authored` relation are not in the schema — they're valid, just unmatched. The schema only checks that `[name]` exists (required) and looks for optional fields like `[role]`, `[expertise]`, and `works_at`.
|
||||
|
||||
### Inline Schema Note
|
||||
|
||||
```markdown
|
||||
---
|
||||
title: Team Standup 2024-01-15
|
||||
type: meeting
|
||||
schema:
|
||||
attendees(array): string, who was there
|
||||
decisions(array): string, what was decided
|
||||
action_items(array): string, follow-ups
|
||||
blockers?(array): string, anything stuck
|
||||
---
|
||||
|
||||
# Team Standup 2024-01-15
|
||||
|
||||
## Observations
|
||||
- [attendees] Paul
|
||||
- [attendees] Sarah
|
||||
- [decisions] Ship v2 by Friday
|
||||
- [action_items] Paul to review PR #42
|
||||
- [blockers] Waiting on API credentials
|
||||
```
|
||||
@@ -9,9 +9,9 @@
|
||||
|
||||
## 🚀 Basic Memory Cloud is Live!
|
||||
|
||||
- **Cross-device and multi-platform support is here.** Your knowledge graph now works on desktop, web, and mobile.
|
||||
- **Cloud is optional.** The local-first open-source workflow continues as always.
|
||||
- **OSS discount:** use code `{{OSS_DISCOUNT_CODE}}` for 20% off for 3 months.
|
||||
- **Cross-device and multi-platform support is here.** Your knowledge graph now works on desktop, web, and mobile - seamlessly synced across all your AI tools (Claude, ChatGPT, Gemini, Claude Code, and Codex)
|
||||
- **Early Supporter Pricing:** Early users get 25% off forever.
|
||||
The open source project continues as always. Cloud just makes it work everywhere.
|
||||
|
||||
[Sign up now →](https://basicmemory.com)
|
||||
|
||||
@@ -344,7 +344,7 @@ basic-memory sync --watch
|
||||
3. Cloud features (optional, requires subscription):
|
||||
|
||||
```bash
|
||||
# Authenticate with cloud (global cloud mode via OAuth)
|
||||
# Authenticate with cloud
|
||||
basic-memory cloud login
|
||||
|
||||
# Bidirectional sync with cloud
|
||||
@@ -357,29 +357,9 @@ basic-memory cloud check
|
||||
basic-memory cloud mount
|
||||
```
|
||||
|
||||
**Per-Project Cloud Routing** (API key based):
|
||||
**Routing Flags** (for users with cloud subscriptions):
|
||||
|
||||
Individual projects can be routed through the cloud while others stay local. This uses an API key instead of OAuth:
|
||||
|
||||
```bash
|
||||
# Save an API key (create one in the web app or via CLI)
|
||||
basic-memory cloud set-key bmc_abc123...
|
||||
# Or create one via CLI (requires OAuth login first)
|
||||
basic-memory cloud create-key "my-laptop"
|
||||
|
||||
# Set a project to route through cloud
|
||||
basic-memory project set-cloud research
|
||||
|
||||
# Revert a project to local mode
|
||||
basic-memory project set-local research
|
||||
|
||||
# List projects with mode column (local/cloud)
|
||||
basic-memory project list
|
||||
```
|
||||
|
||||
**Routing Flags** (for users with global cloud mode):
|
||||
|
||||
When global cloud mode is enabled, CLI commands communicate with the cloud API by default. Use routing flags to override this:
|
||||
When cloud mode is enabled, CLI commands communicate with the cloud API by default. Use routing flags to override this:
|
||||
|
||||
```bash
|
||||
# Force local routing (useful for local MCP server while cloud mode is enabled)
|
||||
@@ -428,12 +408,6 @@ get_current_project() - Show current project stats
|
||||
sync_status() - Check synchronization status
|
||||
```
|
||||
|
||||
**Cloud Discovery (opt-in):**
|
||||
```
|
||||
cloud_info() - Show optional Cloud overview and setup guidance
|
||||
release_notes() - Show latest release notes
|
||||
```
|
||||
|
||||
**Visualization:**
|
||||
```
|
||||
canvas(nodes, edges, title, folder) - Generate knowledge visualizations
|
||||
|
||||
+3
-22
@@ -110,8 +110,6 @@ def resolve_runtime_mode(cloud_mode_enabled: bool, is_test_env: bool) -> Runtime
|
||||
return RuntimeMode.LOCAL
|
||||
```
|
||||
|
||||
**Note**: `RuntimeMode` determines global behavior (e.g., whether to start file sync). Per-project routing is orthogonal — individual projects can be set to `cloud` mode via `ProjectMode` in config, which affects client routing in `get_client(project_name=...)` without changing the global runtime mode.
|
||||
|
||||
## Dependencies Package
|
||||
|
||||
### Structure
|
||||
@@ -223,7 +221,9 @@ async def search_notes(
|
||||
tags: list[str] | None = None,
|
||||
status: str | None = None,
|
||||
) -> SearchResponse:
|
||||
async with get_project_client(project, context) as (client, active_project):
|
||||
async with get_client() as client:
|
||||
active_project = await get_active_project(client, project)
|
||||
|
||||
# Import client inside function to avoid circular imports
|
||||
from basic_memory.mcp.clients import SearchClient
|
||||
from basic_memory.schemas.search import SearchQuery
|
||||
@@ -238,25 +238,6 @@ async def search_notes(
|
||||
return await search_client.search(search_query.model_dump())
|
||||
```
|
||||
|
||||
### Per-Project Client Routing
|
||||
|
||||
`get_project_client()` from `mcp/project_context.py` is an async context manager that:
|
||||
1. Resolves the project name from config (no network call)
|
||||
2. Creates the correctly-routed client based on the project's mode (local ASGI or cloud HTTP with API key)
|
||||
3. Validates the project via the API
|
||||
4. Yields `(client, active_project)` tuple
|
||||
|
||||
This solves the bootstrap problem: you need the project name to choose the right client (local vs cloud), but you need the client to validate the project exists.
|
||||
|
||||
```python
|
||||
from basic_memory.mcp.project_context import get_project_client
|
||||
|
||||
async with get_project_client(project, context) as (client, active_project):
|
||||
# client is routed based on project's mode (local or cloud)
|
||||
# active_project is validated via the API
|
||||
...
|
||||
```
|
||||
|
||||
## Sync Coordination
|
||||
|
||||
### SyncCoordinator
|
||||
|
||||
@@ -1,175 +0,0 @@
|
||||
# Per-Project Local/Cloud Routing
|
||||
|
||||
## Context
|
||||
|
||||
basic-memory's cloud/local mode is currently a global toggle (`cloud_mode: bool`). When enabled, ALL projects route through the cloud proxy via OAuth. This is too coarse — users should be able to keep some projects local and route others through cloud.
|
||||
|
||||
The cloud API already supports API key auth (`bmc_`-prefixed keys, `POST /api/keys` to create, `HybridTokenVerifier` routes them automatically). API keys are per-tenant (account-level), not per-project — there are no per-project permissions in the cloud yet.
|
||||
|
||||
**Goal**: Users can set each project to `local` or `cloud` mode. Local projects use the existing ASGI in-process transport. Cloud projects use the cloud API with a single account-level API key. No OAuth dance needed for cloud project access.
|
||||
|
||||
## UX Flow
|
||||
|
||||
**Option A — Create key in web app:**
|
||||
1. User creates API key in cloud web app (already supported)
|
||||
2. Copies the key
|
||||
3. Runs `bm cloud set-key bmc_abc123...` → saves to config.json
|
||||
|
||||
**Option B — Create key via CLI:**
|
||||
1. User is already logged in via OAuth (`bm cloud login`)
|
||||
2. Runs `bm cloud create-key "my-laptop"` → calls `POST /api/keys` with JWT auth → gets key back → saves to config.json
|
||||
3. OAuth login is no longer needed for day-to-day use — the API key handles auth
|
||||
|
||||
**Setting project mode:**
|
||||
```bash
|
||||
bm project set-cloud research # route "research" project through cloud
|
||||
bm project set-local research # revert to local
|
||||
bm project list # shows mode column (local/cloud)
|
||||
```
|
||||
|
||||
## Implementation Plan
|
||||
|
||||
### Step 1: Config model changes
|
||||
|
||||
**File: `src/basic_memory/config.py`**
|
||||
|
||||
- Add `ProjectMode` enum: `LOCAL = "local"`, `CLOUD = "cloud"`
|
||||
- Add `ProjectConfigEntry` Pydantic model: `path: str`, `mode: ProjectMode = LOCAL`
|
||||
- Evolve `BasicMemoryConfig.projects` from `Dict[str, str]` to `Dict[str, ProjectConfigEntry]`
|
||||
- Add `model_validator(mode="before")` to auto-migrate old `{"name": "/path"}` format to `{"name": {"path": "/path", "mode": "local"}}`
|
||||
- Add `cloud_api_key: Optional[str] = None` field to `BasicMemoryConfig` (account-level, not per-project)
|
||||
- Update `ProjectConfig` dataclass to carry `mode` from config entry
|
||||
- Add helpers: `get_project_entry(name)`, `get_project_mode(name)`
|
||||
- Keep global `cloud_mode` as deprecated fallback
|
||||
- Update all code that reads `config.projects` as `Dict[str, str]` to handle `ProjectConfigEntry`
|
||||
|
||||
### Step 2: Client routing
|
||||
|
||||
**File: `src/basic_memory/mcp/async_client.py`**
|
||||
|
||||
- Add optional `project_name: Optional[str] = None` parameter to `get_client()`
|
||||
- Routing logic (priority order):
|
||||
1. Factory injection (`_client_factory`) — unchanged
|
||||
2. Force-local (`_force_local_mode()`) — unchanged
|
||||
3. **New**: If `project_name` provided and project's mode is `CLOUD` → HTTP client with `cloud_api_key` as Bearer token, hitting `cloud_host/proxy`
|
||||
4. Global `cloud_mode_enabled` fallback — existing OAuth flow (deprecated)
|
||||
5. Default: local ASGI transport
|
||||
- Error if cloud project but no `cloud_api_key` in config — actionable message pointing to `bm cloud set-key` or `bm cloud create-key`
|
||||
|
||||
### Step 3: Project-aware client helper
|
||||
|
||||
**File: `src/basic_memory/mcp/project_context.py`**
|
||||
|
||||
- Add `get_project_client(project, context)` async context manager
|
||||
- Combines `resolve_project_parameter()` (config-only, no network) + `get_client(project_name=resolved)` + `get_active_project(client, resolved, context)`
|
||||
- Returns `(client, active_project)` tuple
|
||||
- Solves bootstrap problem: resolve project name first, create correct client, then validate
|
||||
|
||||
### Step 4: Simplify ProjectResolver
|
||||
|
||||
**File: `src/basic_memory/project_resolver.py`**
|
||||
|
||||
- Remove global `cloud_mode` parameter — routing mode is orthogonal to project resolution
|
||||
- Resolution becomes purely: constrained env var → explicit param → default project
|
||||
- Update `resolve_project_parameter()` in `project_context.py` to drop `cloud_mode` param
|
||||
|
||||
### Step 5: Update MCP tools
|
||||
|
||||
**Files: `src/basic_memory/mcp/tools/*.py` (~15 files)**
|
||||
|
||||
Mechanical change per tool:
|
||||
```python
|
||||
# Before
|
||||
async with get_client() as client:
|
||||
active_project = await get_active_project(client, project, context)
|
||||
|
||||
# After
|
||||
async with get_project_client(project, context) as (client, active_project):
|
||||
```
|
||||
|
||||
Special handling for `recent_activity.py` discovery mode: iterate projects, create per-project client for each.
|
||||
|
||||
### Step 6: Sync coordinator
|
||||
|
||||
**Files: `src/basic_memory/sync/coordinator.py`, `src/basic_memory/mcp/container.py`**
|
||||
|
||||
- Filter file watchers to local-mode projects only
|
||||
- Cloud projects skip sync
|
||||
|
||||
### Step 7: CLI commands
|
||||
|
||||
**File: `src/basic_memory/cli/commands/cloud/core_commands.py`**
|
||||
|
||||
- `bm cloud set-key <api-key>` — saves API key to config.json
|
||||
- `bm cloud create-key <name>` — calls `POST {cloud_host}/api/keys` using existing JWT auth (from `make_api_request`), saves returned key to config. Uses existing `api_client.py:make_api_request()` for the authenticated call.
|
||||
|
||||
**File: `src/basic_memory/cli/commands/project.py`**
|
||||
|
||||
- `bm project set-cloud <name>` — sets project mode to cloud (validates API key exists in config)
|
||||
- `bm project set-local <name>` — reverts project to local mode
|
||||
- Extend `bm project list` / `bm project info` to show mode column
|
||||
|
||||
### Step 8: RuntimeMode simplification
|
||||
|
||||
**File: `src/basic_memory/runtime.py`**
|
||||
|
||||
- `resolve_runtime_mode()` drops `cloud_mode_enabled` parameter
|
||||
- Simplifies to: TEST if test env, otherwise LOCAL
|
||||
- `RuntimeMode.CLOUD` kept for backward compat but not used in global resolution
|
||||
|
||||
### Step 9: Tests
|
||||
|
||||
- Config: migration from old format, round-trip serialization, `get_project_mode()`
|
||||
- `get_client()`: local project → ASGI, cloud project → HTTP+API key, missing key → error
|
||||
- `get_project_client()`: resolve + route combined
|
||||
- MCP tools: representative sample with new helper
|
||||
- Sync: cloud projects skipped, local projects synced
|
||||
- CLI: `set-key`, `create-key`, `set-cloud`, `set-local`
|
||||
|
||||
## Key Files
|
||||
|
||||
| File | Change |
|
||||
|------|--------|
|
||||
| `src/basic_memory/config.py` | `ProjectMode`, `ProjectConfigEntry`, migration, `cloud_api_key` field |
|
||||
| `src/basic_memory/mcp/async_client.py` | `get_client(project_name=)` per-project routing |
|
||||
| `src/basic_memory/mcp/project_context.py` | `get_project_client()` helper |
|
||||
| `src/basic_memory/project_resolver.py` | Remove global `cloud_mode` concern |
|
||||
| `src/basic_memory/mcp/tools/*.py` | Mechanical swap to `get_project_client()` |
|
||||
| `src/basic_memory/sync/coordinator.py` | Filter to local-mode projects |
|
||||
| `src/basic_memory/mcp/container.py` | Update should_sync logic |
|
||||
| `src/basic_memory/cli/commands/cloud/core_commands.py` | `set-key`, `create-key` commands |
|
||||
| `src/basic_memory/cli/commands/project.py` | `set-cloud`, `set-local` commands |
|
||||
| `src/basic_memory/runtime.py` | Drop cloud_mode from global resolution |
|
||||
|
||||
## Config Example
|
||||
|
||||
```json
|
||||
{
|
||||
"projects": {
|
||||
"personal": {"path": "/Users/me/notes", "mode": "local"},
|
||||
"research": {"path": "/Users/me/research", "mode": "cloud"}
|
||||
},
|
||||
"cloud_api_key": "bmc_abc123...",
|
||||
"cloud_host": "https://cloud.basicmemory.com",
|
||||
"default_project": "personal"
|
||||
}
|
||||
```
|
||||
|
||||
## Edge Cases
|
||||
|
||||
| Case | Handling |
|
||||
|------|----------|
|
||||
| No API key + cloud project | `get_client()` raises error: "Run `bm cloud set-key` first" |
|
||||
| Old config format loaded | `model_validator` auto-migrates `Dict[str,str]` to new format |
|
||||
| Default project is cloud | Works — resolver returns name, routing uses API key |
|
||||
| Global `cloud_mode=true` (legacy) | Deprecated fallback still works via OAuth |
|
||||
| Factory-injected client (cloud app) | Factory takes priority, unaffected |
|
||||
| `--local` CLI flag on cloud project | Force-local override still works |
|
||||
|
||||
## Verification
|
||||
|
||||
1. `just fast-check` — lint/format/typecheck + impacted tests
|
||||
2. `just test` — full suite (SQLite + Postgres)
|
||||
3. Manual: `bm cloud set-key bmc_...`, `bm project set-cloud test`, run MCP tools against it
|
||||
4. Manual: verify local projects work unchanged
|
||||
5. Manual: `bm project list` shows mode column
|
||||
+7
-102
@@ -17,8 +17,6 @@ Before using Basic Memory Cloud, you need:
|
||||
|
||||
- **Active Subscription**: An active Basic Memory Cloud subscription is required to access cloud features
|
||||
- **Subscribe**: Visit [https://basicmemory.com/subscribe](https://basicmemory.com/subscribe) to sign up
|
||||
- **Optional**: Cloud is optional. Local-first open-source usage continues without cloud.
|
||||
- **OSS Discount**: Use code `{{OSS_DISCOUNT_CODE}}` for 20% off for 3 months.
|
||||
|
||||
If you attempt to log in without an active subscription, you'll receive a "Subscription Required" error with a link to subscribe.
|
||||
|
||||
@@ -83,7 +81,6 @@ bm cloud login
|
||||
4. Validates your subscription status
|
||||
|
||||
**Result:** All `bm project`, `bm tools` commands now work with cloud.
|
||||
Apply OSS discount code `{{OSS_DISCOUNT_CODE}}` during checkout to receive 20% off for 3 months.
|
||||
|
||||
### 2. Set Up Sync
|
||||
|
||||
@@ -436,97 +433,20 @@ bm project bisync --name work
|
||||
|
||||
**Result:** Fine-grained control over what syncs.
|
||||
|
||||
## Per-Project Cloud Routing (API Key)
|
||||
|
||||
Instead of toggling global cloud mode, you can route individual projects through the cloud using an API key. This lets you keep some projects local while others route through the cloud.
|
||||
|
||||
### Setting Up API Key Auth
|
||||
|
||||
**Option A: Create a key in the web app, then save it locally:**
|
||||
|
||||
```bash
|
||||
bm cloud set-key bmc_abc123...
|
||||
```
|
||||
|
||||
**Option B: Create a key via CLI (requires OAuth login first):**
|
||||
|
||||
```bash
|
||||
bm cloud login # One-time OAuth login
|
||||
bm cloud create-key "my-laptop" # Creates key and saves it locally
|
||||
```
|
||||
|
||||
The API key is account-level — it grants access to all your cloud projects. It's stored in `~/.basic-memory/config.json` as `cloud_api_key`.
|
||||
|
||||
### Setting Project Modes
|
||||
|
||||
```bash
|
||||
# Route a project through cloud
|
||||
bm project set-cloud research
|
||||
|
||||
# Revert to local mode
|
||||
bm project set-local research
|
||||
|
||||
# View project modes
|
||||
bm project list
|
||||
```
|
||||
|
||||
**What happens:**
|
||||
- `set-cloud`: validates the API key exists, then sets the project mode to `cloud` in config
|
||||
- `set-local`: reverts the project to local mode (removes the mode entry from config)
|
||||
- MCP tools and CLI commands for that project will route to `cloud_host/proxy` with the API key as Bearer token
|
||||
|
||||
### How It Works
|
||||
|
||||
When an MCP tool or CLI command runs for a cloud-mode project:
|
||||
|
||||
1. `get_client(project_name="research")` checks the project's mode in config
|
||||
2. If mode is `cloud`, creates an HTTP client pointed at `cloud_host/proxy` with `Authorization: Bearer bmc_...`
|
||||
3. If mode is `local` (default), uses the in-process ASGI transport as usual
|
||||
|
||||
**Routing priority** (highest to lowest):
|
||||
1. Factory injection (cloud app, tests)
|
||||
2. `BASIC_MEMORY_FORCE_LOCAL` env var
|
||||
3. Per-project cloud mode (API key)
|
||||
4. Global cloud mode (OAuth — deprecated fallback)
|
||||
5. Local ASGI transport (default)
|
||||
|
||||
### Configuration Example
|
||||
|
||||
```json
|
||||
{
|
||||
"projects": {
|
||||
"personal": "/Users/me/notes",
|
||||
"research": "/Users/me/research"
|
||||
},
|
||||
"project_modes": {
|
||||
"research": "cloud"
|
||||
},
|
||||
"cloud_api_key": "bmc_abc123...",
|
||||
"cloud_host": "https://cloud.basicmemory.com",
|
||||
"default_project": "personal"
|
||||
}
|
||||
```
|
||||
|
||||
In this example, `personal` stays local and `research` routes through cloud. Projects not listed in `project_modes` default to local.
|
||||
|
||||
### Sync Behavior
|
||||
|
||||
Cloud-mode projects are automatically skipped during local file sync (background sync and file watching). Their files live on the cloud instance, not locally.
|
||||
|
||||
## Disable Cloud Mode
|
||||
|
||||
Return to local mode (global):
|
||||
Return to local mode:
|
||||
|
||||
```bash
|
||||
bm cloud logout
|
||||
```
|
||||
|
||||
**What this does:**
|
||||
1. Disables global cloud mode in config
|
||||
2. All commands now work locally (unless individual projects are set to cloud via `set-cloud`)
|
||||
1. Disables cloud mode in config
|
||||
2. All commands now work locally
|
||||
3. Auth token remains (can re-enable with login)
|
||||
|
||||
**Result:** All `bm` commands work with local projects again. Per-project cloud routing via API key continues to work independently of global cloud mode.
|
||||
**Result:** All `bm` commands work with local projects again.
|
||||
|
||||
## Filter Configuration
|
||||
|
||||
@@ -736,17 +656,9 @@ If instance is down, wait a few minutes and retry.
|
||||
### Cloud Mode Management
|
||||
|
||||
```bash
|
||||
bm cloud login # Authenticate and enable global cloud mode (OAuth)
|
||||
bm cloud logout # Disable global cloud mode
|
||||
bm cloud login # Authenticate and enable cloud mode
|
||||
bm cloud logout # Disable cloud mode
|
||||
bm cloud status # Check cloud mode and instance health
|
||||
bm cloud promo --off # Disable CLI cloud promo notices
|
||||
```
|
||||
|
||||
### API Key Management
|
||||
|
||||
```bash
|
||||
bm cloud set-key <key> # Save a cloud API key (bmc_ prefixed)
|
||||
bm cloud create-key <name> # Create API key via cloud API (requires OAuth login)
|
||||
```
|
||||
|
||||
### Setup
|
||||
@@ -760,20 +672,13 @@ bm cloud setup # Install rclone and configure credentials
|
||||
When cloud mode is enabled:
|
||||
|
||||
```bash
|
||||
bm project list # List projects with mode column
|
||||
bm project list # List cloud projects
|
||||
bm project add <name> # Create cloud project (no sync)
|
||||
bm project add <name> --local-path <path> # Create with local sync
|
||||
bm project sync-setup <name> <path> # Add sync to existing project
|
||||
bm project rm <name> # Delete project
|
||||
```
|
||||
|
||||
### Per-Project Routing
|
||||
|
||||
```bash
|
||||
bm project set-cloud <name> # Route project through cloud (requires API key)
|
||||
bm project set-local <name> # Revert project to local mode
|
||||
```
|
||||
|
||||
### File Synchronization
|
||||
|
||||
```bash
|
||||
|
||||
@@ -1,91 +0,0 @@
|
||||
# Cloud Semantic Search Value (Customer-Facing Technical Story)
|
||||
|
||||
This document explains why teams should buy cloud semantic search even when local search exists.
|
||||
|
||||
## Core Promise
|
||||
|
||||
Markdown files remain the source of truth in both local and cloud modes.
|
||||
|
||||
- Files are portable.
|
||||
- Search indexes are derived and rebuildable.
|
||||
- You never get locked into proprietary document storage.
|
||||
|
||||
## The Customer Problem
|
||||
|
||||
Teams paying for cloud are usually not optimizing for "can this run locally." They are optimizing for:
|
||||
|
||||
- finding the right note the first time,
|
||||
- keeping retrieval quality high as note volume grows,
|
||||
- avoiding search slowdowns while content is actively changing,
|
||||
- getting consistent results across users, agents, and sessions.
|
||||
|
||||
## Why Cloud Is the Aspirin
|
||||
|
||||
Cloud semantic search is the immediate pain reliever because it fixes the problems users feel right now.
|
||||
|
||||
### 1) Better hit rate on real queries
|
||||
|
||||
Cloud uses stronger managed embeddings than the default local model, which improves semantic recall for paraphrases and vague questions.
|
||||
|
||||
Customer outcome:
|
||||
|
||||
- fewer "I know this exists but search missed it" moments,
|
||||
- less query rewording,
|
||||
- faster time to answer.
|
||||
|
||||
### 2) Better behavior under active workloads
|
||||
|
||||
Cloud indexing runs out of band in workers, so indexing does not compete with interactive read/write traffic.
|
||||
|
||||
Customer outcome:
|
||||
|
||||
- stable search responsiveness during heavy updates,
|
||||
- fresher semantic results shortly after edits,
|
||||
- less user-visible performance variance.
|
||||
|
||||
### 3) Better consistency for shared knowledge
|
||||
|
||||
Cloud retrieval runs against a centralized tenant index, so teams and agents resolve against the same semantic state.
|
||||
|
||||
Customer outcome:
|
||||
|
||||
- fewer "works on my machine" search differences,
|
||||
- more predictable agent behavior across environments,
|
||||
- easier cross-user collaboration on large knowledge bases.
|
||||
|
||||
### 4) Better quality at higher scale
|
||||
|
||||
With Postgres + `pgvector` per tenant, cloud can sustain larger note collections and higher query volumes than typical local setups.
|
||||
|
||||
Customer outcome:
|
||||
|
||||
- confidence as repositories grow to tens of thousands of notes,
|
||||
- less need for user-side tuning,
|
||||
- fewer quality regressions as usage increases.
|
||||
|
||||
## Local Is the Vitamin
|
||||
|
||||
Local semantic search still matters and should stay strong.
|
||||
|
||||
- offline use,
|
||||
- privacy-first operation,
|
||||
- no cloud dependency,
|
||||
- user-controlled runtime.
|
||||
|
||||
It compounds long-term ownership and resilience, but does not remove the immediate pain points cloud solves for teams at scale.
|
||||
|
||||
## Recommended Messaging
|
||||
|
||||
One-liner:
|
||||
|
||||
"Cloud semantic search is the aspirin: it fixes retrieval quality and performance pain now. Local semantic search is the vitamin: it builds long-term control and resilience."
|
||||
|
||||
Long form:
|
||||
|
||||
"Basic Memory keeps markdown as the source of truth everywhere. Local gives privacy and offline control. Cloud adds immediate, measurable improvements in search quality, consistency, and responsiveness for teams and agents running at scale."
|
||||
|
||||
## Packaging Guidance
|
||||
|
||||
- Base: local FTS plus optional local semantic search.
|
||||
- Cloud value: higher semantic quality, stable performance under load, and consistent team-wide retrieval.
|
||||
- Keep interfaces pluggable (`EmbeddingProvider`, vector backend protocol) so implementation can evolve without changing user workflows.
|
||||
@@ -1,130 +0,0 @@
|
||||
# MCP UI Bakeoff - Instructions & Test Plan
|
||||
|
||||
Last updated: 2026-02-02
|
||||
|
||||
## Scope
|
||||
|
||||
Compare three presentation paths for Basic Memory MCP tools:
|
||||
|
||||
1. **Tool‑UI (React)** via MCP App resources.
|
||||
2. **MCP‑UI Python SDK** embedded UI resources (legacy host path).
|
||||
3. **ASCII/ANSI** output for TUI clients.
|
||||
|
||||
This doc is the running instruction set and test plan. Update as implementation progresses.
|
||||
|
||||
---
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- Repo: `basic-memory` (worktree: `basic-memory-mcp-ui-poc`)
|
||||
- Node for tool‑ui build (already used for POC)
|
||||
- Python 3.12+ with `uv`
|
||||
|
||||
Optional (for MCP‑UI Python SDK path):
|
||||
|
||||
- Local repo: `/Users/phernandez/dev/mcp-ui`
|
||||
- Install the server SDK into the Basic Memory venv:
|
||||
- `uv pip install -e /Users/phernandez/dev/mcp-ui/sdks/python/server`
|
||||
|
||||
---
|
||||
|
||||
## Build / Refresh Steps
|
||||
|
||||
### Tool‑UI React bundle
|
||||
|
||||
```bash
|
||||
cd ui/tool-ui-react
|
||||
npm install
|
||||
npm run build
|
||||
```
|
||||
|
||||
This regenerates:
|
||||
|
||||
- `src/basic_memory/mcp/ui/html/search-results-tool-ui.html`
|
||||
- `src/basic_memory/mcp/ui/html/note-preview-tool-ui.html`
|
||||
|
||||
---
|
||||
|
||||
## How to Run the MCP Server
|
||||
|
||||
```bash
|
||||
basic-memory mcp --transport stdio
|
||||
```
|
||||
|
||||
Optional to pick UI variant for MCP App resources:
|
||||
|
||||
```bash
|
||||
export BASIC_MEMORY_MCP_UI_VARIANT=tool-ui # or vanilla | mcp-ui
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Test Cases
|
||||
|
||||
### 1) MCP App Resource UI (tool‑ui / vanilla / mcp‑ui)
|
||||
|
||||
Tools:
|
||||
- `search_notes`
|
||||
- `read_note`
|
||||
|
||||
Expect:
|
||||
- Tool meta points to `ui://basic-memory/search-results` and `ui://basic-memory/note-preview`
|
||||
- Resource content differs by `BASIC_MEMORY_MCP_UI_VARIANT`
|
||||
- Variant‑specific URIs also available:
|
||||
- `ui://basic-memory/search-results/vanilla`
|
||||
- `ui://basic-memory/search-results/tool-ui`
|
||||
- `ui://basic-memory/search-results/mcp-ui`
|
||||
- `ui://basic-memory/note-preview/vanilla`
|
||||
- `ui://basic-memory/note-preview/tool-ui`
|
||||
- `ui://basic-memory/note-preview/mcp-ui`
|
||||
|
||||
Manual check:
|
||||
- Trigger tool in MCP‑App‑capable host and confirm UI renders.
|
||||
|
||||
---
|
||||
|
||||
### 2) ASCII / ANSI TUI Output
|
||||
|
||||
Tools:
|
||||
- `search_notes(output_format="ascii" | "ansi")`
|
||||
- `read_note(output_format="ascii" | "ansi")`
|
||||
|
||||
Expect:
|
||||
- ASCII table for search, header + content preview for note.
|
||||
- ANSI variants include color escape codes.
|
||||
|
||||
Automated:
|
||||
- `uv run pytest test-int/mcp/test_output_format_ascii_integration.py`
|
||||
|
||||
---
|
||||
|
||||
### 3) MCP‑UI Python SDK (embedded UI resource)
|
||||
|
||||
Tools (embedded resource responses):
|
||||
- `search_notes_ui` (MCP‑UI SDK)
|
||||
- `read_note_ui` (MCP‑UI SDK)
|
||||
|
||||
Expected output:
|
||||
- Tool response content contains an EmbeddedResource (`type: "resource"`)
|
||||
- `mimeType` is `text/html`
|
||||
- `_meta` includes:
|
||||
- `mcpui.dev/ui-preferred-frame-size`
|
||||
- `mcpui.dev/ui-initial-render-data`
|
||||
|
||||
Manual check:
|
||||
- Render tool responses using `UIResourceRenderer` (legacy host flow).
|
||||
|
||||
Automated (if SDK installed):
|
||||
- `uv run pytest test-int/mcp/test_ui_sdk_integration.py`
|
||||
|
||||
---
|
||||
|
||||
## Bakeoff Notes Template
|
||||
|
||||
Fill in after running:
|
||||
|
||||
- Tool‑UI (React): __
|
||||
- MCP‑UI SDK (embedded): __
|
||||
- ASCII/ANSI: __
|
||||
|
||||
Decision + rationale: __
|
||||
@@ -1,265 +0,0 @@
|
||||
# Semantic Search
|
||||
|
||||
This guide covers Basic Memory's optional semantic (vector) search feature, which adds meaning-based retrieval alongside the existing full-text search.
|
||||
|
||||
## Overview
|
||||
|
||||
Basic Memory's default search uses full-text search (FTS) — keyword matching with boolean operators. Semantic search adds vector embeddings that capture the *meaning* of your content, enabling:
|
||||
|
||||
- **Paraphrase matching**: Find "authentication flow" when searching for "login process"
|
||||
- **Conceptual queries**: Search for "ways to improve performance" and find notes about caching, indexing, and optimization
|
||||
- **Hybrid retrieval**: Combine the precision of keyword search with the recall of semantic similarity
|
||||
|
||||
Semantic search is **opt-in** — existing behavior is completely unchanged unless you enable it. It works on both SQLite (local) and Postgres (cloud) backends.
|
||||
|
||||
## Installation
|
||||
|
||||
Semantic search dependencies (fastembed, sqlite-vec, openai) are **optional extras** — they are not installed with the base `basic-memory` package. Install them with:
|
||||
|
||||
```bash
|
||||
pip install 'basic-memory[semantic]'
|
||||
```
|
||||
|
||||
This keeps the base install lightweight and avoids platform-specific issues with ONNX Runtime wheels.
|
||||
|
||||
### Platform Compatibility
|
||||
|
||||
| Platform | FastEmbed (local) | OpenAI (API) |
|
||||
|---|---|---|
|
||||
| macOS ARM64 (Apple Silicon) | Yes | Yes |
|
||||
| macOS x86_64 (Intel Mac) | No — see workaround below | Yes |
|
||||
| Linux x86_64 | Yes | Yes |
|
||||
| Linux ARM64 | Yes | Yes |
|
||||
| Windows x86_64 | Yes | Yes |
|
||||
|
||||
#### Intel Mac Workaround
|
||||
|
||||
The default FastEmbed provider uses ONNX Runtime, which dropped Intel Mac (x86_64) wheels starting in v1.24. Intel Mac users have two options:
|
||||
|
||||
**Option 1: Use OpenAI embeddings (recommended)**
|
||||
|
||||
Install only the OpenAI dependency manually — no ONNX Runtime or FastEmbed needed:
|
||||
|
||||
```bash
|
||||
pip install openai sqlite-vec
|
||||
export BASIC_MEMORY_SEMANTIC_SEARCH_ENABLED=true
|
||||
export BASIC_MEMORY_SEMANTIC_EMBEDDING_PROVIDER=openai
|
||||
export OPENAI_API_KEY=sk-...
|
||||
```
|
||||
|
||||
**Option 2: Pin an older ONNX Runtime**
|
||||
|
||||
FastEmbed's ONNX Runtime dependency is unpinned, so you can constrain it to an older version that still ships Intel Mac wheels by passing both requirements in the same install command:
|
||||
|
||||
```bash
|
||||
pip install 'basic-memory[semantic]' 'onnxruntime<1.24'
|
||||
```
|
||||
|
||||
## Quick Start
|
||||
|
||||
1. Install semantic extras:
|
||||
|
||||
```bash
|
||||
pip install 'basic-memory[semantic]'
|
||||
```
|
||||
|
||||
2. Enable semantic search:
|
||||
|
||||
```bash
|
||||
export BASIC_MEMORY_SEMANTIC_SEARCH_ENABLED=true
|
||||
```
|
||||
|
||||
3. Build vector embeddings for your existing content:
|
||||
|
||||
```bash
|
||||
bm reindex --embeddings
|
||||
```
|
||||
|
||||
4. Search using semantic modes:
|
||||
|
||||
```python
|
||||
# Pure vector similarity
|
||||
search_notes("login process", search_type="vector")
|
||||
|
||||
# Hybrid: combines FTS precision with vector recall (recommended)
|
||||
search_notes("login process", search_type="hybrid")
|
||||
|
||||
# Traditional full-text search (still the default)
|
||||
search_notes("login process", search_type="text")
|
||||
```
|
||||
|
||||
## Configuration Reference
|
||||
|
||||
All settings are fields on `BasicMemoryConfig` and can be set via environment variables (prefixed with `BASIC_MEMORY_`).
|
||||
|
||||
| Config Field | Env Var | Default | Description |
|
||||
|---|---|---|---|
|
||||
| `semantic_search_enabled` | `BASIC_MEMORY_SEMANTIC_SEARCH_ENABLED` | `false` | Enable semantic search. Required before vector/hybrid modes work. |
|
||||
| `semantic_embedding_provider` | `BASIC_MEMORY_SEMANTIC_EMBEDDING_PROVIDER` | `"fastembed"` | Embedding provider: `"fastembed"` (local) or `"openai"` (API). |
|
||||
| `semantic_embedding_model` | `BASIC_MEMORY_SEMANTIC_EMBEDDING_MODEL` | `"bge-small-en-v1.5"` | Model identifier. Auto-adjusted per provider if left at default. |
|
||||
| `semantic_embedding_dimensions` | `BASIC_MEMORY_SEMANTIC_EMBEDDING_DIMENSIONS` | Auto-detected | Vector dimensions. 384 for FastEmbed, 1536 for OpenAI. Override only if using a non-default model. |
|
||||
| `semantic_embedding_batch_size` | `BASIC_MEMORY_SEMANTIC_EMBEDDING_BATCH_SIZE` | `64` | Number of texts to embed per batch. |
|
||||
| `semantic_vector_k` | `BASIC_MEMORY_SEMANTIC_VECTOR_K` | `100` | Candidate count for vector nearest-neighbour retrieval. Higher values improve recall at the cost of latency. |
|
||||
|
||||
## Embedding Providers
|
||||
|
||||
### FastEmbed (default)
|
||||
|
||||
FastEmbed runs entirely locally using ONNX models — no API key, no network calls, no cost.
|
||||
|
||||
- **Model**: `BAAI/bge-small-en-v1.5`
|
||||
- **Dimensions**: 384
|
||||
- **Tradeoff**: Smaller model, fast inference, good quality for most use cases
|
||||
|
||||
```bash
|
||||
# Install semantic extras and enable
|
||||
pip install 'basic-memory[semantic]'
|
||||
export BASIC_MEMORY_SEMANTIC_SEARCH_ENABLED=true
|
||||
```
|
||||
|
||||
### OpenAI
|
||||
|
||||
Uses OpenAI's embeddings API for higher-dimensional vectors. Requires an API key.
|
||||
|
||||
- **Model**: `text-embedding-3-small`
|
||||
- **Dimensions**: 1536
|
||||
- **Tradeoff**: Higher quality embeddings, requires API calls and an OpenAI key
|
||||
|
||||
```bash
|
||||
export BASIC_MEMORY_SEMANTIC_SEARCH_ENABLED=true
|
||||
export BASIC_MEMORY_SEMANTIC_EMBEDDING_PROVIDER=openai
|
||||
export OPENAI_API_KEY=sk-...
|
||||
```
|
||||
|
||||
When switching from FastEmbed to OpenAI (or vice versa), you must rebuild embeddings since the vector dimensions differ:
|
||||
|
||||
```bash
|
||||
bm reindex --embeddings
|
||||
```
|
||||
|
||||
## Search Modes
|
||||
|
||||
### `text` (default)
|
||||
|
||||
Full-text keyword search using FTS5 (SQLite) or tsvector (Postgres). Supports boolean operators (`AND`, `OR`, `NOT`), phrase matching, and prefix wildcards.
|
||||
|
||||
```python
|
||||
search_notes("project AND planning", search_type="text")
|
||||
```
|
||||
|
||||
This is the existing default and does not require semantic search to be enabled.
|
||||
|
||||
### `vector`
|
||||
|
||||
Pure semantic similarity search. Embeds your query and finds the nearest content vectors. Good for conceptual or paraphrase queries where exact keywords may not appear in the content.
|
||||
|
||||
```python
|
||||
search_notes("how to speed up the app", search_type="vector")
|
||||
```
|
||||
|
||||
Returns results ranked by cosine similarity. Individual observations and relations surface as first-class results, not collapsed into parent entities.
|
||||
|
||||
### `hybrid`
|
||||
|
||||
Combines FTS and vector results using reciprocal rank fusion (RRF). This is generally the best mode when you want both keyword precision and semantic recall.
|
||||
|
||||
```python
|
||||
search_notes("authentication security", search_type="hybrid")
|
||||
```
|
||||
|
||||
RRF merges the two ranked lists so that items appearing in both get a score boost, while items found by only one method still appear.
|
||||
|
||||
### When to Use Which
|
||||
|
||||
| Mode | Best For |
|
||||
|---|---|
|
||||
| `text` | Exact keyword matching, boolean queries, tag/category searches |
|
||||
| `vector` | Conceptual queries, paraphrase matching, exploratory searches |
|
||||
| `hybrid` | General-purpose search combining precision and recall |
|
||||
|
||||
## The Reindex Command
|
||||
|
||||
The `bm reindex` command rebuilds search indexes without dropping the database.
|
||||
|
||||
```bash
|
||||
# Rebuild everything (FTS + embeddings if semantic is enabled)
|
||||
bm reindex
|
||||
|
||||
# Only rebuild vector embeddings
|
||||
bm reindex --embeddings
|
||||
|
||||
# Only rebuild the full-text search index
|
||||
bm reindex --search
|
||||
|
||||
# Target a specific project
|
||||
bm reindex -p my-project
|
||||
```
|
||||
|
||||
### When You Need to Reindex
|
||||
|
||||
- **First enable**: After turning on `semantic_search_enabled` for the first time
|
||||
- **Provider change**: After switching between `fastembed` and `openai`
|
||||
- **Model change**: After changing `semantic_embedding_model`
|
||||
- **Dimension change**: After changing `semantic_embedding_dimensions`
|
||||
|
||||
The reindex command shows progress with embedded/skipped/error counts:
|
||||
|
||||
```
|
||||
Project: main
|
||||
Building vector embeddings...
|
||||
✓ Embeddings complete: 142 entities embedded, 0 skipped, 0 errors
|
||||
|
||||
Reindex complete!
|
||||
```
|
||||
|
||||
## How It Works
|
||||
|
||||
### Chunking
|
||||
|
||||
Each entity in the search index is split into semantic chunks before embedding:
|
||||
|
||||
- **Headers**: Markdown headers (`#`, `##`, etc.) start new chunks
|
||||
- **Bullets**: Each bullet item (`-`, `*`) becomes its own chunk for granular fact retrieval
|
||||
- **Prose sections**: Non-bullet text is merged up to ~900 characters per chunk
|
||||
- **Long sections**: Oversized content is split with ~120 character overlap to preserve context at boundaries
|
||||
|
||||
Each search index item type (entity, observation, relation) is chunked independently, so observations and relations are embeddable as discrete facts.
|
||||
|
||||
### Deduplication
|
||||
|
||||
Each chunk has a `source_hash` (SHA-256 of the chunk text). On re-sync, unchanged chunks skip re-embedding entirely. This makes incremental updates fast — only modified content triggers API calls or model inference.
|
||||
|
||||
### Hybrid Fusion
|
||||
|
||||
Hybrid search uses reciprocal rank fusion (RRF) to merge FTS and vector results:
|
||||
|
||||
1. Run FTS search to get keyword-ranked results
|
||||
2. Run vector search to get similarity-ranked results
|
||||
3. For each result, compute: `score = 1/(k + fts_rank) + 1/(k + vector_rank)` where `k = 60`
|
||||
4. Sort by fused score
|
||||
|
||||
Items found by both methods get a natural score boost. Items found by only one method still appear but rank lower.
|
||||
|
||||
### Observation-Level Results
|
||||
|
||||
Vector and hybrid modes return individual observations and relations as first-class search results, not just parent entities. This means a search for "water temperature for brewing" can surface the specific observation about 205°F without returning the entire "Coffee Brewing Methods" entity.
|
||||
|
||||
## Database Backends
|
||||
|
||||
### SQLite (local)
|
||||
|
||||
- **Vector storage**: [sqlite-vec](https://github.com/asg017/sqlite-vec) virtual table
|
||||
- **Table creation**: At runtime when semantic search is first used — no migration needed
|
||||
- **Embedding table**: `search_vector_embeddings` using `vec0(embedding float[N])` where N is the configured dimensions
|
||||
- **Chunk metadata**: `search_vector_chunks` table stores chunk text, keys, and source hashes
|
||||
|
||||
The sqlite-vec extension is loaded per-connection. Vector tables are created lazily on first use.
|
||||
|
||||
### Postgres (cloud)
|
||||
|
||||
- **Vector storage**: [pgvector](https://github.com/pgvector/pgvector) with HNSW indexing
|
||||
- **Chunk metadata table**: Created via Alembic migration (`search_vector_chunks` with `BIGSERIAL` primary key)
|
||||
- **Embedding table**: `search_vector_embeddings` created at runtime (dimension-dependent, same pattern as SQLite)
|
||||
- **Index**: HNSW index on the embedding column for fast approximate nearest-neighbour queries
|
||||
|
||||
The Alembic migration creates the dimension-independent chunks table. The embeddings table and HNSW index are deferred to runtime because they depend on the configured vector dimensions.
|
||||
@@ -1,365 +0,0 @@
|
||||
# SPEC-SCHEMA-IMPL: Schema System Implementation Plan
|
||||
|
||||
**Status:** Draft
|
||||
**Created:** 2025-02-06
|
||||
**Branch:** `feature/schema-system`
|
||||
**Depends on:** [SPEC-SCHEMA](SPEC-SCHEMA.md)
|
||||
|
||||
## Overview
|
||||
|
||||
Implementation plan for the Basic Memory Schema System. The system is entirely programmatic —
|
||||
no LLM agent runtime or API key required. The LLM already in the user's session (Claude Code,
|
||||
Claude Desktop, etc.) provides the intelligence layer by reading schema notes via existing
|
||||
MCP tools.
|
||||
|
||||
## Architecture
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────────────┐
|
||||
│ Entry Points │
|
||||
│ CLI (bm schema ...) │ MCP (schema_validate) │
|
||||
└──────────┬────────────┴──────────┬──────────────┘
|
||||
│ │
|
||||
▼ ▼
|
||||
┌─────────────────────────────────────────────────┐
|
||||
│ Schema Service Layer │
|
||||
│ resolve_schema · validate · infer · diff │
|
||||
└──────────┬────────────────────────┬──────────────┘
|
||||
│ │
|
||||
▼ ▼
|
||||
┌──────────────────────┐ ┌────────────────────────┐
|
||||
│ Picoschema Parser │ │ Note/Entity Access │
|
||||
│ YAML → SchemaModel │ │ (existing repository) │
|
||||
└──────────────────────┘ └────────────────────────┘
|
||||
```
|
||||
|
||||
No new database tables. Schemas are notes with `type: schema` — they're already indexed.
|
||||
Validation reads observations and relations from existing data.
|
||||
|
||||
## Components
|
||||
|
||||
### 1. Picoschema Parser
|
||||
|
||||
**Location:** `src/basic_memory/schema/parser.py`
|
||||
|
||||
Parses Picoschema YAML into an internal representation.
|
||||
|
||||
```python
|
||||
@dataclass
|
||||
class SchemaField:
|
||||
name: str
|
||||
type: str # string, integer, number, boolean, any, or EntityName
|
||||
required: bool # True unless field name ends with ?
|
||||
is_array: bool # True if (array) notation
|
||||
is_enum: bool # True if (enum) notation
|
||||
enum_values: list[str] # Populated for enums
|
||||
description: str | None # Text after comma
|
||||
is_entity_ref: bool # True if type is capitalized (entity reference)
|
||||
children: list[SchemaField] # For (object) types
|
||||
|
||||
|
||||
@dataclass
|
||||
class SchemaDefinition:
|
||||
entity: str # The entity type this schema describes
|
||||
version: int # Schema version
|
||||
fields: list[SchemaField] # Parsed fields
|
||||
validation_mode: str # "warn" | "strict" | "off"
|
||||
|
||||
|
||||
def parse_picoschema(yaml_dict: dict) -> list[SchemaField]:
|
||||
"""Parse a Picoschema YAML dict into a list of SchemaField objects."""
|
||||
|
||||
|
||||
def parse_schema_note(frontmatter: dict) -> SchemaDefinition:
|
||||
"""Parse a full schema note's frontmatter into a SchemaDefinition."""
|
||||
```
|
||||
|
||||
**Input/Output:**
|
||||
```yaml
|
||||
# Input (YAML dict from frontmatter)
|
||||
schema:
|
||||
name: string, full name
|
||||
role?: string, job title
|
||||
works_at?: Organization, employer
|
||||
expertise?(array): string, areas of knowledge
|
||||
```
|
||||
|
||||
```python
|
||||
# Output
|
||||
[
|
||||
SchemaField(name="name", type="string", required=True, description="full name", ...),
|
||||
SchemaField(name="role", type="string", required=False, description="job title", ...),
|
||||
SchemaField(name="works_at", type="Organization", required=False, is_entity_ref=True, ...),
|
||||
SchemaField(name="expertise", type="string", required=False, is_array=True, ...),
|
||||
]
|
||||
```
|
||||
|
||||
### 2. Schema Resolver
|
||||
|
||||
**Location:** `src/basic_memory/schema/resolver.py`
|
||||
|
||||
Finds the applicable schema for a note using the resolution order.
|
||||
|
||||
```python
|
||||
async def resolve_schema(
|
||||
note_frontmatter: dict,
|
||||
search_fn: Callable, # injected search capability
|
||||
) -> SchemaDefinition | None:
|
||||
"""Resolve schema for a note.
|
||||
|
||||
Resolution order:
|
||||
1. Inline schema (frontmatter['schema'] is a dict)
|
||||
2. Explicit reference (frontmatter['schema'] is a string)
|
||||
3. Implicit by type (frontmatter['type'] → schema note with matching entity)
|
||||
4. No schema (returns None)
|
||||
"""
|
||||
```
|
||||
|
||||
### 3. Schema Validator
|
||||
|
||||
**Location:** `src/basic_memory/schema/validator.py`
|
||||
|
||||
Validates a note's observations and relations against a resolved schema.
|
||||
|
||||
```python
|
||||
@dataclass
|
||||
class FieldResult:
|
||||
field: SchemaField
|
||||
status: str # "present" | "missing" | "type_mismatch"
|
||||
values: list[str] # Matched observation values or relation targets
|
||||
message: str | None # Human-readable detail
|
||||
|
||||
|
||||
@dataclass
|
||||
class ValidationResult:
|
||||
note_identifier: str
|
||||
schema_entity: str
|
||||
passed: bool # True if no errors (warnings are OK)
|
||||
field_results: list[FieldResult]
|
||||
unmatched_observations: dict[str, int] # category → count
|
||||
unmatched_relations: list[str] # relation types not in schema
|
||||
warnings: list[str]
|
||||
errors: list[str]
|
||||
|
||||
|
||||
async def validate_note(
|
||||
note: Note,
|
||||
schema: SchemaDefinition,
|
||||
) -> ValidationResult:
|
||||
"""Validate a note against a schema definition.
|
||||
|
||||
Mapping rules:
|
||||
- field: string → observation [field] exists
|
||||
- field?(array): type → multiple [field] observations
|
||||
- field?: EntityType → relation 'field [[...]]' exists
|
||||
- field?(enum): [v] → observation [field] value ∈ enum values
|
||||
"""
|
||||
```
|
||||
|
||||
### 4. Schema Inference Engine
|
||||
|
||||
**Location:** `src/basic_memory/schema/inference.py`
|
||||
|
||||
Analyzes notes of a given type and suggests a schema based on usage frequency.
|
||||
|
||||
```python
|
||||
@dataclass
|
||||
class FieldFrequency:
|
||||
name: str
|
||||
source: str # "observation" | "relation"
|
||||
count: int # notes containing this field
|
||||
total: int # total notes analyzed
|
||||
percentage: float
|
||||
sample_values: list[str] # representative values
|
||||
is_array: bool # True if typically appears multiple times per note
|
||||
target_type: str | None # For relations, the most common target entity type
|
||||
|
||||
|
||||
@dataclass
|
||||
class InferenceResult:
|
||||
entity_type: str
|
||||
notes_analyzed: int
|
||||
field_frequencies: list[FieldFrequency]
|
||||
suggested_schema: dict # Ready-to-use Picoschema YAML dict
|
||||
suggested_required: list[str]
|
||||
suggested_optional: list[str]
|
||||
excluded: list[str] # Below threshold
|
||||
|
||||
|
||||
async def infer_schema(
|
||||
entity_type: str,
|
||||
notes: list[Note],
|
||||
required_threshold: float = 0.95, # 95%+ = required
|
||||
optional_threshold: float = 0.25, # 25%+ = optional
|
||||
) -> InferenceResult:
|
||||
"""Analyze notes and suggest a Picoschema definition."""
|
||||
```
|
||||
|
||||
### 5. Schema Diff
|
||||
|
||||
**Location:** `src/basic_memory/schema/diff.py`
|
||||
|
||||
Compares current note usage against an existing schema definition.
|
||||
|
||||
```python
|
||||
@dataclass
|
||||
class SchemaDrift:
|
||||
new_fields: list[FieldFrequency] # Fields not in schema but common in notes
|
||||
dropped_fields: list[FieldFrequency] # Fields in schema but rare in notes
|
||||
cardinality_changes: list[str] # one → many or many → one
|
||||
type_mismatches: list[str] # observation values don't match declared type
|
||||
|
||||
|
||||
async def diff_schema(
|
||||
schema: SchemaDefinition,
|
||||
notes: list[Note],
|
||||
) -> SchemaDrift:
|
||||
"""Compare a schema against actual note usage to detect drift."""
|
||||
```
|
||||
|
||||
## Entry Points
|
||||
|
||||
### CLI Commands
|
||||
|
||||
**Location:** `src/basic_memory/cli/schema.py`
|
||||
|
||||
```python
|
||||
import typer
|
||||
|
||||
schema_app = typer.Typer(name="schema", help="Schema management commands")
|
||||
|
||||
@schema_app.command()
|
||||
async def validate(
|
||||
target: str = typer.Argument(None, help="Note path or entity type"),
|
||||
strict: bool = typer.Option(False, help="Override to strict mode"),
|
||||
):
|
||||
"""Validate notes against their schemas."""
|
||||
|
||||
@schema_app.command()
|
||||
async def infer(
|
||||
entity_type: str = typer.Argument(..., help="Entity type to analyze"),
|
||||
threshold: float = typer.Option(0.25, help="Minimum frequency for optional fields"),
|
||||
save: bool = typer.Option(False, help="Save to schema/ directory"),
|
||||
):
|
||||
"""Infer schema from existing notes of a type."""
|
||||
|
||||
@schema_app.command()
|
||||
async def diff(
|
||||
entity_type: str = typer.Argument(..., help="Entity type to diff"),
|
||||
):
|
||||
"""Show drift between schema and actual usage."""
|
||||
```
|
||||
|
||||
Registered as subcommand: `bm schema validate`, `bm schema infer`, `bm schema diff`.
|
||||
|
||||
### MCP Tools
|
||||
|
||||
**Location:** `src/basic_memory/mcp/tools/schema.py`
|
||||
|
||||
```python
|
||||
@mcp_tool
|
||||
async def schema_validate(
|
||||
entity_type: str | None = None,
|
||||
identifier: str | None = None,
|
||||
project: str | None = None,
|
||||
) -> str:
|
||||
"""Validate notes against their resolved schema."""
|
||||
|
||||
@mcp_tool
|
||||
async def schema_infer(
|
||||
entity_type: str,
|
||||
threshold: float = 0.25,
|
||||
project: str | None = None,
|
||||
) -> str:
|
||||
"""Analyze existing notes and suggest a schema definition."""
|
||||
```
|
||||
|
||||
### API Endpoints
|
||||
|
||||
**Location:** `src/basic_memory/api/schema_router.py`
|
||||
|
||||
```python
|
||||
router = APIRouter(prefix="/schema", tags=["schema"])
|
||||
|
||||
@router.post("/validate")
|
||||
async def validate_schema(...) -> ValidationReport: ...
|
||||
|
||||
@router.post("/infer")
|
||||
async def infer_schema(...) -> InferenceResult: ...
|
||||
|
||||
@router.get("/diff/{entity_type}")
|
||||
async def diff_schema(...) -> SchemaDrift: ...
|
||||
```
|
||||
|
||||
MCP tools call these endpoints via the typed client pattern (consistent with existing
|
||||
architecture).
|
||||
|
||||
## Implementation Phases
|
||||
|
||||
### Phase 1: Parser + Resolver
|
||||
|
||||
Build the foundation — can parse Picoschema and find schemas for notes.
|
||||
|
||||
**Deliverables:**
|
||||
- `schema/parser.py` — Picoschema YAML → `SchemaDefinition`
|
||||
- `schema/resolver.py` — Resolution order (inline → explicit ref → implicit by type → none)
|
||||
- Unit tests for all Picoschema syntax variations
|
||||
- Unit tests for resolution order
|
||||
|
||||
**No external dependencies.** Pure Python parsing of YAML dicts. Can develop and test
|
||||
in isolation.
|
||||
|
||||
### Phase 2: Validator
|
||||
|
||||
Connect schemas to notes and produce validation results.
|
||||
|
||||
**Deliverables:**
|
||||
- `schema/validator.py` — Validate note observations/relations against schema fields
|
||||
- API endpoint: `POST /schema/validate`
|
||||
- MCP tool: `schema_validate`
|
||||
- CLI command: `bm schema validate`
|
||||
- Integration tests with real notes and schemas
|
||||
|
||||
**Depends on:** Phase 1 (parser + resolver)
|
||||
|
||||
### Phase 3: Inference
|
||||
|
||||
Analyze existing notes to suggest schemas.
|
||||
|
||||
**Deliverables:**
|
||||
- `schema/inference.py` — Frequency analysis across notes of a type
|
||||
- API endpoint: `POST /schema/infer`
|
||||
- MCP tool: `schema_infer`
|
||||
- CLI command: `bm schema infer`
|
||||
- Option to save inferred schema as a note via `write_note`
|
||||
|
||||
**Depends on:** Phase 1 (parser for output format)
|
||||
|
||||
### Phase 4: Diff
|
||||
|
||||
Compare schemas against current usage.
|
||||
|
||||
**Deliverables:**
|
||||
- `schema/diff.py` — Drift detection between schema and actual notes
|
||||
- API endpoint: `GET /schema/diff/{entity_type}`
|
||||
- CLI command: `bm schema diff`
|
||||
|
||||
**Depends on:** Phase 1 (parser), Phase 3 (inference, for frequency analysis)
|
||||
|
||||
## Testing Strategy
|
||||
|
||||
- **Unit tests** (`tests/schema/`): Parser edge cases, resolution logic, validation mapping,
|
||||
inference thresholds
|
||||
- **Integration tests** (`test-int/schema/`): End-to-end with real markdown files, schema notes
|
||||
on disk, CLI invocation
|
||||
- Coverage target: 100% (consistent with project standard)
|
||||
|
||||
## What This Does NOT Include
|
||||
|
||||
- No new database tables or migrations
|
||||
- No new markdown syntax (schemas validate existing observations/relations)
|
||||
- No LLM agent runtime or API key management
|
||||
- No hook integration (deferred)
|
||||
- No schema composition/inheritance (deferred)
|
||||
- No OWL/RDF export (deferred)
|
||||
- No built-in templates (deferred)
|
||||
@@ -1,462 +0,0 @@
|
||||
# SPEC-SCHEMA: Basic Memory Schema System
|
||||
|
||||
**Status:** Draft
|
||||
**Created:** 2025-02-06
|
||||
**Branch:** `feature/schema-system`
|
||||
|
||||
## Summary
|
||||
|
||||
A schema system for Basic Memory that uses [Picoschema](https://genkit.dev/docs/dotprompt/)
|
||||
syntax in YAML frontmatter. Schemas validate notes against their existing observation/relation
|
||||
structure — no new data model, no migration, just a declarative lens over what's already there.
|
||||
|
||||
## Core Principles
|
||||
|
||||
1. **Schemas are just notes** — A schema is a note with `type: schema`, lives anywhere
|
||||
2. **Use prior art** — Picoschema syntax in YAML frontmatter, no custom notation
|
||||
3. **Validation maps to existing format** — Observations and relations, not a parallel data model
|
||||
4. **Validation is soft** — Warnings by default, not blocking errors
|
||||
5. **Inference over prescription** — Schemas describe reality, emerge from usage
|
||||
6. **No built-in agent** — Programmatic core; the LLM already in the session provides intelligence
|
||||
|
||||
## Picoschema Syntax
|
||||
|
||||
Picoschema is a compact schema notation from Google's Dotprompt that fits naturally in YAML
|
||||
frontmatter.
|
||||
|
||||
### Supported Types
|
||||
|
||||
| Type | Description |
|
||||
|------|-------------|
|
||||
| `string` | Text value |
|
||||
| `integer` | Whole number |
|
||||
| `number` | Decimal number |
|
||||
| `boolean` | True/false |
|
||||
| `any` | Any scalar type |
|
||||
| `EntityName` | Reference to another entity (capitalized = entity reference) |
|
||||
|
||||
### Syntax Rules
|
||||
|
||||
```yaml
|
||||
schema:
|
||||
name: string, full name # required field with description
|
||||
email?: string, contact email # ? = optional
|
||||
role?: string, job title
|
||||
works_at?: Organization, employer # capitalized type = entity reference
|
||||
tags?(array): string, categories # array of type
|
||||
status?(enum): [active, inactive] # enum with allowed values
|
||||
metadata?(object): # nested object
|
||||
updated_at?: string
|
||||
source?: string
|
||||
```
|
||||
|
||||
- `field: type` — required field
|
||||
- `field?: type` — optional field
|
||||
- `field(array): type` — array of values
|
||||
- `field?(enum): [values]` — enumeration
|
||||
- `field?(object):` — nested object with sub-fields
|
||||
- `, description` — description after comma
|
||||
- `EntityName` as type (capitalized) — reference to another entity
|
||||
|
||||
## Schema-to-Note Mapping
|
||||
|
||||
Schemas validate against the existing Basic Memory note format. No new syntax for note
|
||||
authors to learn.
|
||||
|
||||
### Mapping Rules
|
||||
|
||||
| Schema Declaration | Grounded In | Example Match |
|
||||
|--------------------|-------------|---------------|
|
||||
| `field: string` | Observation `[field] value` | `- [name] Paul Graham` |
|
||||
| `field?(array): string` | Multiple `[field]` observations | `- [expertise] Lisp` (×N) |
|
||||
| `field?: EntityType` | Relation `field [[Target]]` | `- works_at [[Y Combinator]]` |
|
||||
| `field?(array): EntityType` | Multiple `field` relations | `- authored [[Book]]` (×N) |
|
||||
| `tags` | Frontmatter `tags` array | `tags: [startups, essays]` |
|
||||
| `field?(enum): [values]` | Observation `[field] value` where value ∈ set | `- [status] active` |
|
||||
|
||||
### Key Insight
|
||||
|
||||
Schemas don't introduce a new way to store data. They describe the patterns already present
|
||||
in observations and relations. A note doesn't have to change how it's written — the schema
|
||||
just says "a good Person note has a `[name]` observation and a `works_at` relation."
|
||||
|
||||
## Schema Definition
|
||||
|
||||
### As a Dedicated Schema Note
|
||||
|
||||
```yaml
|
||||
# schema/Person.md
|
||||
---
|
||||
title: Person
|
||||
type: schema
|
||||
entity: Person
|
||||
version: 1
|
||||
schema:
|
||||
name: string, full name
|
||||
email?: string, contact email
|
||||
role?: string, job title
|
||||
works_at?: Organization, employer
|
||||
expertise?(array): string, areas of knowledge
|
||||
settings:
|
||||
validation: warn # warn | strict | off
|
||||
---
|
||||
|
||||
# Person
|
||||
|
||||
A human individual in the knowledge graph.
|
||||
|
||||
Any documentation about this entity type goes here as prose.
|
||||
```
|
||||
|
||||
Schema notes are regular Basic Memory notes. They show up in search, can have their own
|
||||
observations and relations, and can be organized in any folder (though `schema/` is
|
||||
the suggested convention).
|
||||
|
||||
### Inline Schema in a Note
|
||||
|
||||
Notes can carry their own schema directly:
|
||||
|
||||
```yaml
|
||||
# meetings/2024-01-15-standup.md
|
||||
---
|
||||
title: Team Standup 2024-01-15
|
||||
type: meeting
|
||||
schema:
|
||||
attendees(array): string, who was there
|
||||
decisions(array): string, what was decided
|
||||
action_items(array): string, follow-ups
|
||||
blockers?(array): string, anything stuck
|
||||
---
|
||||
|
||||
# Team Standup 2024-01-15
|
||||
|
||||
## Observations
|
||||
- [attendees] Paul
|
||||
- [attendees] Sarah
|
||||
- [decisions] Ship v2 by Friday
|
||||
- [action_items] Paul to review PR #42
|
||||
- [blockers] Waiting on API credentials
|
||||
```
|
||||
|
||||
Good for one-off structured notes or prototyping a schema before extracting it.
|
||||
|
||||
### Explicit Schema Reference
|
||||
|
||||
A note can reference a schema by entity name or permalink:
|
||||
|
||||
```yaml
|
||||
# projects/basic-memory.md
|
||||
---
|
||||
title: Basic Memory
|
||||
schema: SoftwareProject # by entity name
|
||||
---
|
||||
|
||||
# research/llm-memory-patterns.md
|
||||
---
|
||||
title: LLM Memory Patterns
|
||||
schema: schema/research-project # by permalink
|
||||
---
|
||||
```
|
||||
|
||||
Use cases:
|
||||
- Note's `type` differs from the schema it should validate against
|
||||
- Multiple schema variants exist for the same domain
|
||||
- Applying structure to existing notes without changing their type
|
||||
|
||||
## Schema Resolution
|
||||
|
||||
When validating a note, schemas resolve in priority order:
|
||||
|
||||
```
|
||||
1. Inline schema → schema: { ... } (dict in frontmatter)
|
||||
2. Explicit ref → schema: Person (string in frontmatter)
|
||||
3. Implicit by type → type: Person (lookup schema note with entity: Person)
|
||||
4. No schema → no validation (perfectly fine)
|
||||
```
|
||||
|
||||
```python
|
||||
async def resolve_schema(note: Note) -> Schema | None:
|
||||
schema_value = note.frontmatter.get('schema')
|
||||
|
||||
# 1. Inline schema (dict)
|
||||
if isinstance(schema_value, dict):
|
||||
return parse_picoschema(schema_value)
|
||||
|
||||
# 2. Explicit reference (string)
|
||||
if isinstance(schema_value, str):
|
||||
schema_note = await find_schema_note(schema_value)
|
||||
if schema_note:
|
||||
return parse_picoschema(schema_note.frontmatter['schema'])
|
||||
|
||||
# 3. Implicit by type
|
||||
note_type = note.frontmatter.get('type')
|
||||
if note_type:
|
||||
results = await search_notes(f"type:schema entity:{note_type}")
|
||||
if results:
|
||||
return parse_picoschema(results[0].frontmatter['schema'])
|
||||
|
||||
# 4. No schema
|
||||
return None
|
||||
```
|
||||
|
||||
## Validation
|
||||
|
||||
### Modes
|
||||
|
||||
Configured in the schema's `settings.validation`:
|
||||
|
||||
| Mode | Behavior |
|
||||
|------|----------|
|
||||
| `off` | No validation |
|
||||
| `warn` | Warnings in output, doesn't block (default) |
|
||||
| `strict` | Errors that block sync, for CI/CD enforcement |
|
||||
|
||||
### Validation Output
|
||||
|
||||
For a note missing required fields:
|
||||
|
||||
```
|
||||
$ bm schema validate people/ada-lovelace.md
|
||||
|
||||
⚠ Person schema validation:
|
||||
- Missing required field: name (expected [name] observation)
|
||||
- Missing optional field: role
|
||||
- Missing optional field: works_at (no relation found)
|
||||
|
||||
ℹ Unmatched observations: [fact] ×2, [born] ×1
|
||||
ℹ Unmatched relations: collaborated_with
|
||||
```
|
||||
|
||||
"Unmatched" items are informational — observations and relations the schema doesn't cover.
|
||||
They're valid. Schemas are a subset, not a straitjacket.
|
||||
|
||||
### Batch Validation
|
||||
|
||||
```
|
||||
$ bm schema validate Person
|
||||
|
||||
Validating 30 notes against Person schema...
|
||||
|
||||
✓ people/paul-graham.md — all fields present
|
||||
✓ people/rich-hickey.md — all fields present
|
||||
⚠ people/ada-lovelace.md — missing: name
|
||||
⚠ people/alan-kay.md — missing: name, role
|
||||
✓ people/linus-torvalds.md — all fields present
|
||||
...
|
||||
|
||||
Summary: 22/30 valid, 8 warnings, 0 errors
|
||||
```
|
||||
|
||||
## Emerging Schemas
|
||||
|
||||
### The Problem with Traditional Schemas
|
||||
|
||||
Most schema systems require: define schema → create conforming content → fight the schema
|
||||
when reality doesn't match. This is backwards. Knowledge grows organically.
|
||||
|
||||
### The Basic Memory Approach
|
||||
|
||||
```
|
||||
Write notes freely → Patterns emerge → Crystallize into schema → Validate future notes
|
||||
```
|
||||
|
||||
### Schema Inference
|
||||
|
||||
Generate schemas from existing notes by analyzing observation and relation frequency:
|
||||
|
||||
```
|
||||
$ bm schema infer Person
|
||||
|
||||
Analyzing 30 notes with type: Person...
|
||||
|
||||
Observations found:
|
||||
[name] 30/30 100% → name: string
|
||||
[role] 27/30 90% → role?: string
|
||||
[fact] 25/30 83% (generic — no single field)
|
||||
[expertise] 18/30 60% → expertise?(array): string
|
||||
[email] 8/30 27% → email?: string
|
||||
[born] 6/30 20% (below threshold)
|
||||
|
||||
Relations found:
|
||||
works_at 22/30 73% → works_at?: Organization
|
||||
authored 11/30 37% → authored?(array): string
|
||||
|
||||
Suggested schema:
|
||||
name: string, full name
|
||||
role?: string, job title
|
||||
expertise?(array): string, areas of knowledge
|
||||
email?: string, contact email
|
||||
works_at?: Organization, employer
|
||||
|
||||
Save to schema/Person.md? [y/n]
|
||||
```
|
||||
|
||||
Frequency thresholds:
|
||||
- 100% present → required field
|
||||
- 25%+ present → optional field
|
||||
- Below 25% → excluded from suggestion (but noted)
|
||||
|
||||
### Schema Drift Detection
|
||||
|
||||
Track how usage patterns shift over time:
|
||||
|
||||
```
|
||||
$ bm schema diff Person
|
||||
|
||||
Schema drift detected:
|
||||
|
||||
+ expertise: now in 81% of notes (was 12%)
|
||||
- department: dropped to 3% of notes
|
||||
~ works_at: cardinality changed (one → many)
|
||||
|
||||
Update schema? [y/n/review]
|
||||
```
|
||||
|
||||
## LLM Integration (AI Guidance)
|
||||
|
||||
No agent runtime or API key required. The LLM already in the session uses schemas as
|
||||
context for note creation.
|
||||
|
||||
### Flow
|
||||
|
||||
1. User asks LLM to "write a note about Rich Hickey"
|
||||
2. LLM determines `type: Person` is appropriate
|
||||
3. LLM calls `search_notes("type:schema entity:Person")` → finds schema
|
||||
4. LLM reads schema fields: required `name`, optional `role`, `works_at`, `expertise`
|
||||
5. LLM calls `write_note` with observations and relations that satisfy the schema
|
||||
|
||||
The schema acts as a creation template. The LLM knows what a "complete" note looks like
|
||||
without any custom agent infrastructure.
|
||||
|
||||
### MCP Tools
|
||||
|
||||
```python
|
||||
@mcp_tool
|
||||
async def schema_validate(
|
||||
entity_type: str | None = None,
|
||||
identifier: str | None = None,
|
||||
project: str | None = None,
|
||||
) -> ValidationReport:
|
||||
"""Validate notes against their resolved schema.
|
||||
|
||||
Validates a specific note (by identifier) or all notes of a given type.
|
||||
Returns warnings/errors based on the schema's validation mode.
|
||||
"""
|
||||
|
||||
@mcp_tool
|
||||
async def schema_infer(
|
||||
entity_type: str,
|
||||
threshold: float = 0.25,
|
||||
project: str | None = None,
|
||||
) -> SuggestedSchema:
|
||||
"""Analyze existing notes and suggest a schema definition.
|
||||
|
||||
Examines observation categories and relation types across all notes
|
||||
of the given type. Returns frequency analysis and suggested Picoschema.
|
||||
"""
|
||||
```
|
||||
|
||||
## CLI Commands
|
||||
|
||||
```bash
|
||||
# Validate a specific note
|
||||
bm schema validate people/ada-lovelace.md
|
||||
|
||||
# Validate all notes of a type
|
||||
bm schema validate Person
|
||||
|
||||
# Validate everything with a schema
|
||||
bm schema validate
|
||||
|
||||
# Infer schema from existing notes
|
||||
bm schema infer Person
|
||||
|
||||
# Show schema drift from current definition
|
||||
bm schema diff Person
|
||||
|
||||
# List all schema notes
|
||||
bm search "type:schema"
|
||||
```
|
||||
|
||||
## Examples
|
||||
|
||||
### Complete Person Workflow
|
||||
|
||||
**Schema:**
|
||||
```yaml
|
||||
# schema/Person.md
|
||||
---
|
||||
title: Person
|
||||
type: schema
|
||||
entity: Person
|
||||
version: 1
|
||||
schema:
|
||||
name: string, full name
|
||||
role?: string, job title or position
|
||||
works_at?: Organization, employer
|
||||
expertise?(array): string, areas of knowledge
|
||||
email?: string, contact email
|
||||
settings:
|
||||
validation: warn
|
||||
---
|
||||
|
||||
# Person
|
||||
|
||||
A human individual in the knowledge graph.
|
||||
```
|
||||
|
||||
**Valid note:**
|
||||
```yaml
|
||||
# people/paul-graham.md
|
||||
---
|
||||
title: Paul Graham
|
||||
type: Person
|
||||
tags: [startups, essays, lisp]
|
||||
---
|
||||
|
||||
# Paul Graham
|
||||
|
||||
## Observations
|
||||
- [name] Paul Graham
|
||||
- [role] Essayist and investor
|
||||
- [expertise] Startups
|
||||
- [expertise] Lisp
|
||||
- [expertise] Essay writing
|
||||
- [fact] Created Viaweb, the first web app
|
||||
|
||||
## Relations
|
||||
- works_at [[Y Combinator]]
|
||||
- authored [[Hackers and Painters]]
|
||||
```
|
||||
|
||||
**Note with warnings:**
|
||||
```yaml
|
||||
# people/ada-lovelace.md
|
||||
---
|
||||
title: Ada Lovelace
|
||||
type: Person
|
||||
---
|
||||
|
||||
# Ada Lovelace
|
||||
|
||||
## Observations
|
||||
- [fact] Wrote the first computer program
|
||||
- [born] 1815
|
||||
|
||||
## Relations
|
||||
- collaborated_with [[Charles Babbage]]
|
||||
```
|
||||
|
||||
Validation: warns about missing required `[name]` observation. Everything else is optional
|
||||
or unmatched (which is fine).
|
||||
|
||||
## Future Considerations (Deferred)
|
||||
|
||||
These are interesting but out of scope for the initial implementation:
|
||||
|
||||
- **Multiple schema inheritance** — `schema: [Person, Author]`
|
||||
- **Hook integration** — Pre-write validation via the hooks system
|
||||
- **OWL/RDF export** — `bm schema export --format owl`
|
||||
- **SPARQL queries** — Schema-aware graph queries
|
||||
- **Built-in templates** — `bm schema use gtd`, `bm schema use zettelkasten`
|
||||
- **Schema versioning/migration** — Tracking breaking changes across versions
|
||||
@@ -107,13 +107,6 @@ test-windows:
|
||||
test-benchmark:
|
||||
uv run pytest -p pytest_mock -v --no-cov -m benchmark tests test-int
|
||||
|
||||
# Compare two search benchmark JSONL outputs
|
||||
# Usage:
|
||||
# just benchmark-compare .benchmarks/search-baseline.jsonl .benchmarks/search-candidate.jsonl
|
||||
# just benchmark-compare .benchmarks/search-baseline.jsonl .benchmarks/search-candidate.jsonl --format markdown --show-missing
|
||||
benchmark-compare baseline candidate *args:
|
||||
uv run python test-int/compare_search_benchmarks.py "{{baseline}}" "{{candidate}}" --format table {{args}}
|
||||
|
||||
# Run all tests including Windows, Postgres, and Benchmarks (for CI/comprehensive testing)
|
||||
# Use this before releasing to ensure everything works across all backends and platforms
|
||||
test-all:
|
||||
|
||||
@@ -46,12 +46,6 @@ dependencies = [
|
||||
"httpx>=0.28.0",
|
||||
]
|
||||
|
||||
[project.optional-dependencies]
|
||||
semantic = [
|
||||
"fastembed>=0.7.4",
|
||||
"sqlite-vec>=0.1.6",
|
||||
"openai>=1.100.2",
|
||||
]
|
||||
|
||||
[project.urls]
|
||||
Homepage = "https://github.com/basicmachines-co/basic-memory"
|
||||
|
||||
+2
-2
@@ -6,12 +6,12 @@
|
||||
"url": "https://github.com/basicmachines-co/basic-memory.git",
|
||||
"source": "github"
|
||||
},
|
||||
"version": "0.18.3",
|
||||
"version": "0.18.5",
|
||||
"packages": [
|
||||
{
|
||||
"registryType": "pypi",
|
||||
"identifier": "basic-memory",
|
||||
"version": "0.18.3",
|
||||
"version": "0.18.5",
|
||||
"runtimeHint": "uvx",
|
||||
"runtimeArguments": [
|
||||
{"type": "positional", "value": "basic-memory"},
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
"""basic-memory - Local-first knowledge management combining Zettelkasten with knowledge graphs"""
|
||||
|
||||
# Package version - updated by release automation
|
||||
__version__ = "0.18.3"
|
||||
__version__ = "0.18.5"
|
||||
|
||||
# API version for FastAPI - independent of package version
|
||||
__api_version__ = "v0"
|
||||
|
||||
@@ -1,68 +0,0 @@
|
||||
"""Add Postgres semantic vector search tables (pgvector-aware, optional)
|
||||
|
||||
Revision ID: h1b2c3d4e5f6
|
||||
Revises: d7e8f9a0b1c2
|
||||
Create Date: 2026-02-07 00:00:00.000000
|
||||
|
||||
"""
|
||||
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = "h1b2c3d4e5f6"
|
||||
down_revision: Union[str, None] = "d7e8f9a0b1c2"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Create Postgres vector chunk metadata table.
|
||||
|
||||
Trigger: database backend is PostgreSQL.
|
||||
Why: search_vector_chunks stores text metadata with no vector-dimension
|
||||
dependency, so it's safe in a migration. search_vector_embeddings (which
|
||||
requires pgvector and a provider-specific dimension) is created at runtime
|
||||
by PostgresSearchRepository._ensure_vector_tables(), mirroring the SQLite
|
||||
pattern where vector tables are created dynamically.
|
||||
Outcome: creates the dimension-independent chunks table. The embeddings
|
||||
table + HNSW index are deferred to runtime.
|
||||
"""
|
||||
connection = op.get_bind()
|
||||
if connection.dialect.name != "postgresql":
|
||||
return
|
||||
|
||||
op.execute(
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS search_vector_chunks (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
entity_id INTEGER NOT NULL,
|
||||
project_id INTEGER NOT NULL,
|
||||
chunk_key TEXT NOT NULL,
|
||||
chunk_text TEXT NOT NULL,
|
||||
source_hash TEXT NOT NULL,
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
UNIQUE (project_id, entity_id, chunk_key)
|
||||
)
|
||||
"""
|
||||
)
|
||||
op.execute(
|
||||
"""
|
||||
CREATE INDEX IF NOT EXISTS idx_search_vector_chunks_project_entity
|
||||
ON search_vector_chunks (project_id, entity_id)
|
||||
"""
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Remove Postgres vector chunk/embedding tables.
|
||||
|
||||
Does not drop pgvector extension because other schema objects may depend on it.
|
||||
"""
|
||||
connection = op.get_bind()
|
||||
if connection.dialect.name != "postgresql":
|
||||
return
|
||||
|
||||
op.execute("DROP TABLE IF EXISTS search_vector_embeddings")
|
||||
op.execute("DROP TABLE IF EXISTS search_vector_chunks")
|
||||
@@ -18,7 +18,6 @@ from basic_memory.api.v2.routers import (
|
||||
directory_router as v2_directory,
|
||||
prompt_router as v2_prompt,
|
||||
importer_router as v2_importer,
|
||||
schema_router as v2_schema,
|
||||
)
|
||||
from basic_memory.api.v2.routers.project_router import (
|
||||
add_project,
|
||||
@@ -85,7 +84,6 @@ app.include_router(v2_resource, prefix="/v2/projects/{project_id}")
|
||||
app.include_router(v2_directory, prefix="/v2/projects/{project_id}")
|
||||
app.include_router(v2_prompt, prefix="/v2/projects/{project_id}")
|
||||
app.include_router(v2_importer, prefix="/v2/projects/{project_id}")
|
||||
app.include_router(v2_schema, prefix="/v2/projects/{project_id}")
|
||||
app.include_router(v2_project, prefix="/v2")
|
||||
|
||||
# Legacy web app proxy paths (compat with /proxy/projects/projects)
|
||||
|
||||
@@ -8,7 +8,6 @@ from basic_memory.api.v2.routers.resource_router import router as resource_route
|
||||
from basic_memory.api.v2.routers.directory_router import router as directory_router
|
||||
from basic_memory.api.v2.routers.prompt_router import router as prompt_router
|
||||
from basic_memory.api.v2.routers.importer_router import router as importer_router
|
||||
from basic_memory.api.v2.routers.schema_router import router as schema_router
|
||||
|
||||
__all__ = [
|
||||
"knowledge_router",
|
||||
@@ -19,5 +18,4 @@ __all__ = [
|
||||
"directory_router",
|
||||
"prompt_router",
|
||||
"importer_router",
|
||||
"schema_router",
|
||||
]
|
||||
|
||||
@@ -39,23 +39,6 @@ from basic_memory.schemas.response import DirectoryMoveResult, DirectoryDeleteRe
|
||||
|
||||
router = APIRouter(prefix="/knowledge", tags=["knowledge-v2"])
|
||||
|
||||
|
||||
def _schedule_vector_sync_if_enabled(
|
||||
*,
|
||||
task_scheduler,
|
||||
app_config,
|
||||
entity_id: int,
|
||||
project_id: int,
|
||||
) -> None:
|
||||
"""Schedule out-of-band vector sync only when semantic search is enabled."""
|
||||
if app_config.semantic_search_enabled:
|
||||
task_scheduler.schedule(
|
||||
"sync_entity_vectors",
|
||||
entity_id=entity_id,
|
||||
project_id=project_id,
|
||||
)
|
||||
|
||||
|
||||
## Resolution endpoint
|
||||
|
||||
|
||||
@@ -186,7 +169,6 @@ async def create_entity(
|
||||
search_service: SearchServiceV2ExternalDep,
|
||||
task_scheduler: TaskSchedulerDep,
|
||||
file_service: FileServiceV2ExternalDep,
|
||||
app_config: AppConfigDep,
|
||||
fast: bool = Query(
|
||||
True, description="If true, write quickly and defer indexing to background tasks."
|
||||
),
|
||||
@@ -213,13 +195,7 @@ async def create_entity(
|
||||
)
|
||||
else:
|
||||
entity = await entity_service.create_entity(data)
|
||||
await search_service.index_entity(entity)
|
||||
_schedule_vector_sync_if_enabled(
|
||||
task_scheduler=task_scheduler,
|
||||
app_config=app_config,
|
||||
entity_id=entity.id,
|
||||
project_id=project_id,
|
||||
)
|
||||
await search_service.index_entity(entity, background_tasks=background_tasks)
|
||||
|
||||
result = EntityResponseV2.model_validate(entity)
|
||||
if fast:
|
||||
@@ -249,7 +225,6 @@ async def update_entity_by_id(
|
||||
entity_repository: EntityRepositoryV2ExternalDep,
|
||||
task_scheduler: TaskSchedulerDep,
|
||||
file_service: FileServiceV2ExternalDep,
|
||||
app_config: AppConfigDep,
|
||||
entity_id: str = Path(..., description="Entity external ID (UUID)"),
|
||||
fast: bool = Query(
|
||||
True, description="If true, write quickly and defer indexing to background tasks."
|
||||
@@ -302,13 +277,7 @@ async def update_entity_by_id(
|
||||
)
|
||||
response.status_code = 201
|
||||
|
||||
await search_service.index_entity(entity)
|
||||
_schedule_vector_sync_if_enabled(
|
||||
task_scheduler=task_scheduler,
|
||||
app_config=app_config,
|
||||
entity_id=entity.id,
|
||||
project_id=project_id,
|
||||
)
|
||||
await search_service.index_entity(entity, background_tasks=background_tasks)
|
||||
|
||||
result = EntityResponseV2.model_validate(entity)
|
||||
if fast:
|
||||
@@ -334,7 +303,6 @@ async def edit_entity_by_id(
|
||||
entity_repository: EntityRepositoryV2ExternalDep,
|
||||
task_scheduler: TaskSchedulerDep,
|
||||
file_service: FileServiceV2ExternalDep,
|
||||
app_config: AppConfigDep,
|
||||
entity_id: str = Path(..., description="Entity external ID (UUID)"),
|
||||
fast: bool = Query(
|
||||
True, description="If true, write quickly and defer indexing to background tasks."
|
||||
@@ -391,13 +359,7 @@ async def edit_entity_by_id(
|
||||
expected_replacements=data.expected_replacements,
|
||||
)
|
||||
|
||||
await search_service.index_entity(updated_entity)
|
||||
_schedule_vector_sync_if_enabled(
|
||||
task_scheduler=task_scheduler,
|
||||
app_config=app_config,
|
||||
entity_id=updated_entity.id,
|
||||
project_id=project_id,
|
||||
)
|
||||
await search_service.index_entity(updated_entity, background_tasks=background_tasks)
|
||||
|
||||
result = EntityResponseV2.model_validate(updated_entity)
|
||||
if fast:
|
||||
@@ -472,7 +434,6 @@ async def move_entity(
|
||||
project_config: ProjectConfigV2ExternalDep,
|
||||
app_config: AppConfigDep,
|
||||
search_service: SearchServiceV2ExternalDep,
|
||||
task_scheduler: TaskSchedulerDep,
|
||||
entity_id: str = Path(..., description="Entity external ID (UUID)"),
|
||||
) -> EntityResponseV2:
|
||||
"""Move an entity to a new file location.
|
||||
@@ -511,13 +472,7 @@ async def move_entity(
|
||||
# Reindex at new location
|
||||
reindexed_entity = await entity_service.link_resolver.resolve_link(data.destination_path)
|
||||
if reindexed_entity:
|
||||
await search_service.index_entity(reindexed_entity)
|
||||
_schedule_vector_sync_if_enabled(
|
||||
task_scheduler=task_scheduler,
|
||||
app_config=app_config,
|
||||
entity_id=reindexed_entity.id,
|
||||
project_id=project_id,
|
||||
)
|
||||
await search_service.index_entity(reindexed_entity, background_tasks=background_tasks)
|
||||
|
||||
result = EntityResponseV2.model_validate(moved_entity)
|
||||
|
||||
@@ -544,7 +499,6 @@ async def move_directory(
|
||||
project_config: ProjectConfigV2ExternalDep,
|
||||
app_config: AppConfigDep,
|
||||
search_service: SearchServiceV2ExternalDep,
|
||||
task_scheduler: TaskSchedulerDep,
|
||||
) -> DirectoryMoveResult:
|
||||
"""Move all entities in a directory to a new location.
|
||||
|
||||
@@ -576,13 +530,7 @@ async def move_directory(
|
||||
for file_path in result.moved_files:
|
||||
entity = await entity_service.link_resolver.resolve_link(file_path)
|
||||
if entity:
|
||||
await search_service.index_entity(entity)
|
||||
_schedule_vector_sync_if_enabled(
|
||||
task_scheduler=task_scheduler,
|
||||
app_config=app_config,
|
||||
entity_id=entity.id,
|
||||
project_id=project_id,
|
||||
)
|
||||
await search_service.index_entity(entity, background_tasks=background_tasks)
|
||||
|
||||
logger.info(
|
||||
f"API v2 response: move_directory "
|
||||
|
||||
@@ -1,305 +0,0 @@
|
||||
"""V2 router for schema operations.
|
||||
|
||||
Provides endpoints for schema validation, inference, and drift detection.
|
||||
The schema system validates notes against Picoschema definitions without
|
||||
introducing any new data model -- it works entirely with existing
|
||||
observations and relations.
|
||||
|
||||
Flow: Entity loaded with eager observations/relations -> convert to tuples -> core functions.
|
||||
"""
|
||||
|
||||
from fastapi import APIRouter, Path, Query
|
||||
|
||||
from basic_memory.deps import (
|
||||
SearchServiceV2ExternalDep,
|
||||
EntityRepositoryV2ExternalDep,
|
||||
)
|
||||
from basic_memory.models.knowledge import Entity
|
||||
from basic_memory.schemas.schema import (
|
||||
ValidationReport,
|
||||
InferenceReport,
|
||||
DriftReport,
|
||||
NoteValidationResponse,
|
||||
FieldResultResponse,
|
||||
FieldFrequencyResponse,
|
||||
DriftFieldResponse,
|
||||
)
|
||||
from basic_memory.schemas.search import SearchQuery
|
||||
from basic_memory.schema.resolver import resolve_schema
|
||||
from basic_memory.schema.validator import validate_note
|
||||
from basic_memory.schema.inference import infer_schema, NoteData, ObservationData, RelationData
|
||||
from basic_memory.schema.diff import diff_schema
|
||||
|
||||
# Note: No prefix here -- it's added during registration as /v2/{project_id}/schema
|
||||
router = APIRouter(tags=["schema"])
|
||||
|
||||
|
||||
# --- ORM to core data conversion ---
|
||||
|
||||
|
||||
def _entity_observations(entity: Entity) -> list[ObservationData]:
|
||||
"""Extract ObservationData from an entity's observations."""
|
||||
return [ObservationData(obs.category, obs.content) for obs in entity.observations]
|
||||
|
||||
|
||||
def _entity_relations(entity: Entity) -> list[RelationData]:
|
||||
"""Extract RelationData from an entity's outgoing relations.
|
||||
|
||||
Carries the target entity's type on each relation so the inference engine
|
||||
can suggest correct types (e.g. works_at -> Organization, not the source type).
|
||||
"""
|
||||
return [
|
||||
RelationData(
|
||||
relation_type=rel.relation_type,
|
||||
target_name=rel.to_name,
|
||||
target_entity_type=rel.to_entity.entity_type if rel.to_entity else None,
|
||||
)
|
||||
for rel in entity.outgoing_relations
|
||||
]
|
||||
|
||||
|
||||
def _entity_to_note_data(entity: Entity) -> NoteData:
|
||||
"""Convert an ORM Entity to a NoteData for inference/diff analysis."""
|
||||
return NoteData(
|
||||
identifier=entity.permalink or entity.file_path,
|
||||
observations=_entity_observations(entity),
|
||||
relations=_entity_relations(entity),
|
||||
)
|
||||
|
||||
|
||||
def _entity_frontmatter(entity: Entity) -> dict:
|
||||
"""Build a frontmatter dict from an entity for schema resolution."""
|
||||
frontmatter = dict(entity.entity_metadata) if entity.entity_metadata else {}
|
||||
if entity.entity_type:
|
||||
frontmatter.setdefault("type", entity.entity_type)
|
||||
return frontmatter
|
||||
|
||||
|
||||
# --- Validation ---
|
||||
|
||||
|
||||
@router.post("/schema/validate", response_model=ValidationReport)
|
||||
async def validate_schema(
|
||||
entity_repository: EntityRepositoryV2ExternalDep,
|
||||
search_service: SearchServiceV2ExternalDep,
|
||||
project_id: str = Path(..., description="Project external UUID"),
|
||||
entity_type: str | None = Query(None, description="Entity type to validate"),
|
||||
identifier: str | None = Query(None, description="Specific note identifier"),
|
||||
):
|
||||
"""Validate notes against their resolved schemas.
|
||||
|
||||
Validates a specific note (by identifier) or all notes of a given type.
|
||||
Returns warnings/errors based on the schema's validation mode.
|
||||
"""
|
||||
results: list[NoteValidationResponse] = []
|
||||
|
||||
async def search_fn(query: str) -> list:
|
||||
# Search for schema notes, then load full entity_metadata from the entity table.
|
||||
# The search index only stores minimal metadata (e.g., {"entity_type": "schema"}),
|
||||
# but parse_schema_note needs the full frontmatter with entity/schema/version keys.
|
||||
results = await search_service.search(SearchQuery(text=query, types=["schema"]), limit=5)
|
||||
frontmatters = []
|
||||
for row in results:
|
||||
if row.permalink:
|
||||
entity = await entity_repository.get_by_permalink(row.permalink)
|
||||
if entity:
|
||||
frontmatters.append(_entity_frontmatter(entity))
|
||||
return frontmatters
|
||||
|
||||
# --- Single note validation ---
|
||||
if identifier:
|
||||
entity = await entity_repository.get_by_permalink(identifier)
|
||||
if not entity:
|
||||
return ValidationReport(entity_type=entity_type, total_notes=0, results=[])
|
||||
|
||||
schema_def = await resolve_schema(_entity_frontmatter(entity), search_fn)
|
||||
if schema_def:
|
||||
result = validate_note(
|
||||
entity.permalink or identifier,
|
||||
schema_def,
|
||||
_entity_observations(entity),
|
||||
_entity_relations(entity),
|
||||
)
|
||||
results.append(_to_note_validation_response(result))
|
||||
|
||||
return ValidationReport(
|
||||
entity_type=entity_type or entity.entity_type,
|
||||
total_notes=1,
|
||||
valid_count=1 if (results and results[0].passed) else 0,
|
||||
warning_count=sum(len(r.warnings) for r in results),
|
||||
error_count=sum(len(r.errors) for r in results),
|
||||
results=results,
|
||||
)
|
||||
|
||||
# --- Batch validation by entity type ---
|
||||
entities = await _find_by_entity_type(entity_repository, entity_type) if entity_type else []
|
||||
|
||||
for entity in entities:
|
||||
schema_def = await resolve_schema(_entity_frontmatter(entity), search_fn)
|
||||
if schema_def:
|
||||
result = validate_note(
|
||||
entity.permalink or entity.file_path,
|
||||
schema_def,
|
||||
_entity_observations(entity),
|
||||
_entity_relations(entity),
|
||||
)
|
||||
results.append(_to_note_validation_response(result))
|
||||
|
||||
valid = sum(1 for r in results if r.passed)
|
||||
return ValidationReport(
|
||||
entity_type=entity_type,
|
||||
total_notes=len(results),
|
||||
valid_count=valid,
|
||||
warning_count=sum(len(r.warnings) for r in results),
|
||||
error_count=sum(len(r.errors) for r in results),
|
||||
results=results,
|
||||
)
|
||||
|
||||
|
||||
# --- Inference ---
|
||||
|
||||
|
||||
@router.post("/schema/infer", response_model=InferenceReport)
|
||||
async def infer_schema_endpoint(
|
||||
entity_repository: EntityRepositoryV2ExternalDep,
|
||||
project_id: str = Path(..., description="Project external UUID"),
|
||||
entity_type: str = Query(..., description="Entity type to analyze"),
|
||||
threshold: float = Query(0.25, description="Minimum frequency for optional fields"),
|
||||
):
|
||||
"""Infer a schema from existing notes of a given type.
|
||||
|
||||
Examines observation categories and relation types across all notes
|
||||
of the given type. Returns frequency analysis and suggested Picoschema.
|
||||
"""
|
||||
entities = await _find_by_entity_type(entity_repository, entity_type)
|
||||
notes_data = [_entity_to_note_data(entity) for entity in entities]
|
||||
|
||||
result = infer_schema(entity_type, notes_data, optional_threshold=threshold)
|
||||
|
||||
return InferenceReport(
|
||||
entity_type=result.entity_type,
|
||||
notes_analyzed=result.notes_analyzed,
|
||||
field_frequencies=[
|
||||
FieldFrequencyResponse(
|
||||
name=f.name,
|
||||
source=f.source,
|
||||
count=f.count,
|
||||
total=f.total,
|
||||
percentage=f.percentage,
|
||||
sample_values=f.sample_values,
|
||||
is_array=f.is_array,
|
||||
target_type=f.target_type,
|
||||
)
|
||||
for f in result.field_frequencies
|
||||
],
|
||||
suggested_schema=result.suggested_schema,
|
||||
suggested_required=result.suggested_required,
|
||||
suggested_optional=result.suggested_optional,
|
||||
excluded=result.excluded,
|
||||
)
|
||||
|
||||
|
||||
# --- Drift Detection ---
|
||||
|
||||
|
||||
@router.get("/schema/diff/{entity_type}", response_model=DriftReport)
|
||||
async def diff_schema_endpoint(
|
||||
entity_repository: EntityRepositoryV2ExternalDep,
|
||||
search_service: SearchServiceV2ExternalDep,
|
||||
entity_type: str = Path(..., description="Entity type to check for drift"),
|
||||
project_id: str = Path(..., description="Project external UUID"),
|
||||
):
|
||||
"""Show drift between a schema definition and actual note usage.
|
||||
|
||||
Compares the existing schema for an entity type against how notes
|
||||
of that type are actually structured. Identifies new fields, dropped
|
||||
fields, and cardinality changes.
|
||||
"""
|
||||
|
||||
async def search_fn(query: str) -> list:
|
||||
# Search for schema notes, then load full entity_metadata from the entity table.
|
||||
# The search index only stores minimal metadata (e.g., {"entity_type": "schema"}),
|
||||
# but parse_schema_note needs the full frontmatter with entity/schema/version keys.
|
||||
results = await search_service.search(SearchQuery(text=query, types=["schema"]), limit=5)
|
||||
frontmatters = []
|
||||
for row in results:
|
||||
if row.permalink:
|
||||
entity = await entity_repository.get_by_permalink(row.permalink)
|
||||
if entity:
|
||||
frontmatters.append(_entity_frontmatter(entity))
|
||||
return frontmatters
|
||||
|
||||
# Resolve schema by entity type
|
||||
schema_frontmatter = {"type": entity_type}
|
||||
schema_def = await resolve_schema(schema_frontmatter, search_fn)
|
||||
|
||||
if not schema_def:
|
||||
return DriftReport(entity_type=entity_type)
|
||||
|
||||
# Collect all notes of this type
|
||||
entities = await _find_by_entity_type(entity_repository, entity_type)
|
||||
notes_data = [_entity_to_note_data(entity) for entity in entities]
|
||||
|
||||
result = diff_schema(schema_def, notes_data)
|
||||
|
||||
return DriftReport(
|
||||
entity_type=entity_type,
|
||||
new_fields=[
|
||||
DriftFieldResponse(
|
||||
name=f.name,
|
||||
source=f.source,
|
||||
count=f.count,
|
||||
total=f.total,
|
||||
percentage=f.percentage,
|
||||
)
|
||||
for f in result.new_fields
|
||||
],
|
||||
dropped_fields=[
|
||||
DriftFieldResponse(
|
||||
name=f.name,
|
||||
source=f.source,
|
||||
count=f.count,
|
||||
total=f.total,
|
||||
percentage=f.percentage,
|
||||
)
|
||||
for f in result.dropped_fields
|
||||
],
|
||||
cardinality_changes=result.cardinality_changes,
|
||||
)
|
||||
|
||||
|
||||
# --- Helpers ---
|
||||
|
||||
|
||||
async def _find_by_entity_type(
|
||||
entity_repository: EntityRepositoryV2ExternalDep,
|
||||
entity_type: str,
|
||||
) -> list[Entity]:
|
||||
"""Find all entities of a given type using the repository's select pattern."""
|
||||
query = entity_repository.select().where(Entity.entity_type == entity_type)
|
||||
result = await entity_repository.execute_query(query)
|
||||
return list(result.scalars().all())
|
||||
|
||||
|
||||
def _to_note_validation_response(result) -> NoteValidationResponse:
|
||||
"""Convert a core ValidationResult to a Pydantic response model."""
|
||||
return NoteValidationResponse(
|
||||
note_identifier=result.note_identifier,
|
||||
schema_entity=result.schema_entity,
|
||||
passed=result.passed,
|
||||
field_results=[
|
||||
FieldResultResponse(
|
||||
field_name=fr.field.name,
|
||||
field_type=fr.field.type,
|
||||
required=fr.field.required,
|
||||
status=fr.status,
|
||||
values=fr.values,
|
||||
message=fr.message,
|
||||
)
|
||||
for fr in result.field_results
|
||||
],
|
||||
unmatched_observations=result.unmatched_observations,
|
||||
unmatched_relations=result.unmatched_relations,
|
||||
warnings=result.warnings,
|
||||
errors=result.errors,
|
||||
)
|
||||
@@ -4,13 +4,9 @@ This router uses external_id UUIDs for stable, API-friendly routing.
|
||||
V1 uses string-based project names which are less efficient and less stable.
|
||||
"""
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Path
|
||||
from fastapi import APIRouter, Path
|
||||
|
||||
from basic_memory.api.v2.utils import to_search_results
|
||||
from basic_memory.repository.semantic_errors import (
|
||||
SemanticDependenciesMissingError,
|
||||
SemanticSearchDisabledError,
|
||||
)
|
||||
from basic_memory.schemas.search import SearchQuery, SearchResponse
|
||||
from basic_memory.deps import (
|
||||
SearchServiceV2ExternalDep,
|
||||
@@ -49,14 +45,7 @@ async def search(
|
||||
"""
|
||||
limit = page_size
|
||||
offset = (page - 1) * page_size
|
||||
try:
|
||||
results = await search_service.search(query, limit=limit, offset=offset)
|
||||
except SemanticSearchDisabledError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
except SemanticDependenciesMissingError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
results = await search_service.search(query, limit=limit, offset=offset)
|
||||
search_results = await to_search_results(entity_service, results)
|
||||
return SearchResponse(
|
||||
results=search_results,
|
||||
|
||||
@@ -9,7 +9,6 @@ from typing import Optional # noqa: E402
|
||||
import typer # noqa: E402
|
||||
|
||||
from basic_memory.cli.container import CliContainer, set_container # noqa: E402
|
||||
from basic_memory.cli.promo import maybe_show_cloud_promo # noqa: E402
|
||||
from basic_memory.config import init_cli_logging # noqa: E402
|
||||
|
||||
|
||||
@@ -47,23 +46,11 @@ def app_callback(
|
||||
container = CliContainer.create()
|
||||
set_container(container)
|
||||
|
||||
maybe_show_cloud_promo(ctx.invoked_subcommand)
|
||||
|
||||
# Run initialization for commands that don't use the API
|
||||
# Skip for 'mcp' command - it has its own lifespan that handles initialization
|
||||
# Skip for API-using commands (status, sync, etc.) - they handle initialization via deps.py
|
||||
# Skip for 'reset' command - it manages its own database lifecycle
|
||||
skip_init_commands = {
|
||||
"doctor",
|
||||
"mcp",
|
||||
"status",
|
||||
"sync",
|
||||
"project",
|
||||
"tool",
|
||||
"reset",
|
||||
"reindex",
|
||||
"watch",
|
||||
}
|
||||
skip_init_commands = {"doctor", "mcp", "status", "sync", "project", "tool", "reset"}
|
||||
if (
|
||||
not version
|
||||
and ctx.invoked_subcommand is not None
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
"""CLI commands for basic-memory."""
|
||||
|
||||
from . import status, db, doctor, import_memory_json, mcp, import_claude_conversations
|
||||
from . import import_claude_projects, import_chatgpt, tool, project, format, schema, watch
|
||||
from . import import_claude_projects, import_chatgpt, tool, project, format
|
||||
|
||||
__all__ = [
|
||||
"status",
|
||||
@@ -15,6 +15,4 @@ __all__ = [
|
||||
"tool",
|
||||
"project",
|
||||
"format",
|
||||
"schema",
|
||||
"watch",
|
||||
]
|
||||
|
||||
@@ -6,7 +6,6 @@ from rich.console import Console
|
||||
from basic_memory.cli.app import cloud_app
|
||||
from basic_memory.cli.commands.command_utils import run_with_cleanup
|
||||
from basic_memory.cli.auth import CLIAuth
|
||||
from basic_memory.cli.promo import OSS_DISCOUNT_CODE
|
||||
from basic_memory.config import ConfigManager
|
||||
from basic_memory.cli.commands.cloud.api_client import (
|
||||
CloudAPIError,
|
||||
@@ -58,10 +57,6 @@ def login():
|
||||
except SubscriptionRequiredError as e:
|
||||
console.print("\n[red]Subscription Required[/red]\n")
|
||||
console.print(f"[yellow]{e.args[0]}[/yellow]\n")
|
||||
console.print(
|
||||
f"OSS discount code: [bold]{OSS_DISCOUNT_CODE}[/bold] "
|
||||
"(20% off for 3 months)\n"
|
||||
)
|
||||
console.print(f"Subscribe at: [blue underline]{e.subscribe_url}[/blue underline]\n")
|
||||
console.print(
|
||||
"[dim]Once you have an active subscription, run [bold]bm cloud login[/bold] again.[/dim]"
|
||||
@@ -196,93 +191,3 @@ def setup() -> None:
|
||||
except Exception as e:
|
||||
console.print(f"\n[red]Unexpected error during setup: {e}[/red]")
|
||||
raise typer.Exit(1)
|
||||
|
||||
|
||||
@cloud_app.command("promo")
|
||||
def promo(enabled: bool = typer.Option(True, "--on/--off", help="Enable or disable CLI promos.")):
|
||||
"""Enable or disable CLI cloud promo messages."""
|
||||
config_manager = ConfigManager()
|
||||
config = config_manager.load_config()
|
||||
config.cloud_promo_opt_out = not enabled
|
||||
config_manager.save_config(config)
|
||||
|
||||
if enabled:
|
||||
console.print("[green]Cloud promo messages enabled[/green]")
|
||||
else:
|
||||
console.print("[yellow]Cloud promo messages disabled[/yellow]")
|
||||
|
||||
|
||||
@cloud_app.command("set-key")
|
||||
def set_key(
|
||||
api_key: str = typer.Argument(..., help="API key (bmc_ prefixed) for cloud access"),
|
||||
) -> None:
|
||||
"""Save a cloud API key for per-project cloud routing.
|
||||
|
||||
The API key is account-level and used by projects set to cloud mode.
|
||||
Create a key in the web app or use 'bm cloud create-key'.
|
||||
|
||||
Example:
|
||||
bm cloud set-key bmc_abc123...
|
||||
"""
|
||||
if not api_key.startswith("bmc_"):
|
||||
console.print("[red]Error: API key must start with 'bmc_'[/red]")
|
||||
raise typer.Exit(1)
|
||||
|
||||
config_manager = ConfigManager()
|
||||
config = config_manager.load_config()
|
||||
config.cloud_api_key = api_key
|
||||
config_manager.save_config(config)
|
||||
|
||||
console.print("[green]API key saved[/green]")
|
||||
console.print("[dim]Projects set to cloud mode will use this key for authentication[/dim]")
|
||||
console.print("[dim]Set a project to cloud mode: bm project set-cloud <name>[/dim]")
|
||||
|
||||
|
||||
@cloud_app.command("create-key")
|
||||
def create_key(
|
||||
name: str = typer.Argument(..., help="Human-readable name for the API key"),
|
||||
) -> None:
|
||||
"""Create a new cloud API key and save it locally.
|
||||
|
||||
Requires active OAuth session (run 'bm cloud login' first).
|
||||
The key is created via the cloud API and saved to local config.
|
||||
|
||||
Example:
|
||||
bm cloud create-key "my-laptop"
|
||||
"""
|
||||
|
||||
async def _create_key():
|
||||
_, _, host_url = get_cloud_config()
|
||||
host_url = host_url.rstrip("/")
|
||||
|
||||
console.print(f"[dim]Creating API key '{name}'...[/dim]")
|
||||
response = await make_api_request(
|
||||
method="POST",
|
||||
url=f"{host_url}/api/keys",
|
||||
json_data={"name": name},
|
||||
)
|
||||
|
||||
key_data = response.json()
|
||||
api_key = key_data.get("key")
|
||||
if not api_key:
|
||||
console.print("[red]Error: No key returned from API[/red]")
|
||||
raise typer.Exit(1)
|
||||
|
||||
# Save to config
|
||||
config_manager = ConfigManager()
|
||||
config = config_manager.load_config()
|
||||
config.cloud_api_key = api_key
|
||||
config_manager.save_config(config)
|
||||
|
||||
console.print(f"[green]API key '{name}' created and saved[/green]")
|
||||
console.print("[dim]Projects set to cloud mode will use this key for authentication[/dim]")
|
||||
console.print("[dim]Set a project to cloud mode: bm project set-cloud <name>[/dim]")
|
||||
|
||||
try:
|
||||
run_with_cleanup(_create_key())
|
||||
except CloudAPIError as e:
|
||||
console.print(f"[red]Error creating API key: {e}[/red]")
|
||||
raise typer.Exit(1)
|
||||
except Exception as e:
|
||||
console.print(f"[red]Unexpected error: {e}[/red]")
|
||||
raise typer.Exit(1)
|
||||
|
||||
@@ -223,6 +223,9 @@ def project_sync(
|
||||
*TIGRIS_CONSISTENCY_HEADERS,
|
||||
"--filter-from",
|
||||
str(filter_path),
|
||||
# Prevent NUL byte padding on virtual filesystems (e.g. Google Drive File Stream)
|
||||
# See: rclone/rclone#6801
|
||||
"--local-no-preallocate",
|
||||
]
|
||||
|
||||
if verbose:
|
||||
@@ -299,6 +302,9 @@ def project_bisync(
|
||||
str(filter_path),
|
||||
"--workdir",
|
||||
str(state_path),
|
||||
# Prevent NUL byte padding on virtual filesystems (e.g. Google Drive File Stream)
|
||||
# See: rclone/rclone#6801
|
||||
"--local-no-preallocate",
|
||||
]
|
||||
|
||||
# Add --create-empty-src-dirs if rclone version supports it (v1.64+)
|
||||
|
||||
@@ -5,7 +5,6 @@ from pathlib import Path
|
||||
import typer
|
||||
from loguru import logger
|
||||
from rich.console import Console
|
||||
from rich.progress import Progress, SpinnerColumn, TextColumn, BarColumn, TaskProgressColumn
|
||||
from sqlalchemy.exc import OperationalError
|
||||
|
||||
from basic_memory import db
|
||||
@@ -104,118 +103,3 @@ def reset(
|
||||
# ensures db.shutdown_db() is called even if _reindex_projects changes
|
||||
run_with_cleanup(_reindex_projects(app_config))
|
||||
console.print("[green]Reindex complete[/green]")
|
||||
|
||||
|
||||
@app.command()
|
||||
def reindex(
|
||||
embeddings: bool = typer.Option(
|
||||
False, "--embeddings", "-e", help="Rebuild vector embeddings (requires semantic search)"
|
||||
),
|
||||
search: bool = typer.Option(False, "--search", "-s", help="Rebuild full-text search index"),
|
||||
project: str = typer.Option(
|
||||
None, "--project", "-p", help="Reindex a specific project (default: all)"
|
||||
),
|
||||
): # pragma: no cover
|
||||
"""Rebuild search indexes and/or vector embeddings without dropping the database.
|
||||
|
||||
By default rebuilds everything (search + embeddings if semantic is enabled).
|
||||
Use --search or --embeddings to rebuild only one.
|
||||
|
||||
Examples:
|
||||
bm reindex # Rebuild everything
|
||||
bm reindex --embeddings # Only rebuild vector embeddings
|
||||
bm reindex --search # Only rebuild FTS index
|
||||
bm reindex -p claw # Reindex only the 'claw' project
|
||||
"""
|
||||
# If neither flag is set, do both
|
||||
if not embeddings and not search:
|
||||
embeddings = True
|
||||
search = True
|
||||
|
||||
config_manager = ConfigManager()
|
||||
app_config = config_manager.config
|
||||
|
||||
if embeddings and not app_config.semantic_search_enabled:
|
||||
console.print(
|
||||
"[yellow]Semantic search is not enabled.[/yellow] "
|
||||
"Set [cyan]semantic_search_enabled: true[/cyan] in config to use embeddings."
|
||||
)
|
||||
embeddings = False
|
||||
if not search:
|
||||
raise typer.Exit(0)
|
||||
|
||||
run_with_cleanup(_reindex(app_config, search=search, embeddings=embeddings, project=project))
|
||||
|
||||
|
||||
async def _reindex(app_config, search: bool, embeddings: bool, project: str | None):
|
||||
"""Run reindex operations."""
|
||||
from basic_memory.repository import EntityRepository
|
||||
from basic_memory.repository.search_repository import create_search_repository
|
||||
from basic_memory.services.search_service import SearchService
|
||||
from basic_memory.services.file_service import FileService
|
||||
from basic_memory.markdown.markdown_processor import MarkdownProcessor
|
||||
from basic_memory.markdown.entity_parser import EntityParser
|
||||
|
||||
try:
|
||||
await reconcile_projects_with_config(app_config)
|
||||
|
||||
_, session_maker = await db.get_or_create_db(
|
||||
db_path=app_config.database_path,
|
||||
db_type=db.DatabaseType.FILESYSTEM,
|
||||
)
|
||||
project_repository = ProjectRepository(session_maker)
|
||||
projects = await project_repository.get_active_projects()
|
||||
|
||||
if project:
|
||||
projects = [p for p in projects if p.name == project]
|
||||
if not projects:
|
||||
console.print(f"[red]Project '{project}' not found.[/red]")
|
||||
raise typer.Exit(1)
|
||||
|
||||
for proj in projects:
|
||||
console.print(f"\n[bold]Project: [cyan]{proj.name}[/cyan][/bold]")
|
||||
|
||||
if search:
|
||||
console.print(" Rebuilding full-text search index...")
|
||||
sync_service = await get_sync_service(proj)
|
||||
sync_dir = Path(proj.path)
|
||||
await sync_service.sync(sync_dir, project_name=proj.name)
|
||||
console.print(" [green]✓[/green] Full-text search index rebuilt")
|
||||
|
||||
if embeddings:
|
||||
console.print(" Building vector embeddings...")
|
||||
entity_repository = EntityRepository(session_maker, project_id=proj.id)
|
||||
search_repository = create_search_repository(
|
||||
session_maker, project_id=proj.id, app_config=app_config
|
||||
)
|
||||
project_path = Path(proj.path)
|
||||
entity_parser = EntityParser(project_path)
|
||||
markdown_processor = MarkdownProcessor(entity_parser, app_config=app_config)
|
||||
file_service = FileService(project_path, markdown_processor, app_config=app_config)
|
||||
search_service = SearchService(search_repository, entity_repository, file_service)
|
||||
|
||||
with Progress(
|
||||
SpinnerColumn(),
|
||||
TextColumn("[progress.description]{task.description}"),
|
||||
BarColumn(),
|
||||
TaskProgressColumn(),
|
||||
console=console,
|
||||
) as progress:
|
||||
task = progress.add_task(" Embedding entities...", total=None)
|
||||
|
||||
def on_progress(entity_id, index, total):
|
||||
progress.update(task, total=total, completed=index)
|
||||
|
||||
stats = await search_service.reindex_vectors(progress_callback=on_progress)
|
||||
progress.update(task, completed=stats["total_entities"])
|
||||
|
||||
console.print(
|
||||
f" [green]✓[/green] Embeddings complete: "
|
||||
f"{stats['embedded']} entities embedded, "
|
||||
f"{stats['skipped']} skipped, "
|
||||
f"{stats['errors']} errors"
|
||||
)
|
||||
|
||||
console.print("\n[green]Reindex complete![/green]")
|
||||
finally:
|
||||
await db.shutdown_db()
|
||||
|
||||
@@ -60,9 +60,7 @@ def import_chatgpt(
|
||||
console.print(f"\nImporting chats from {conversations_json}...writing to {base_path}")
|
||||
|
||||
# Create importer and run import
|
||||
importer = ChatGPTImporter(
|
||||
config.home, markdown_processor, file_service, project_name=config.name
|
||||
)
|
||||
importer = ChatGPTImporter(config.home, markdown_processor, file_service)
|
||||
with conversations_json.open("r", encoding="utf-8") as file:
|
||||
json_data = json.load(file)
|
||||
result = run_with_cleanup(importer.import_data(json_data, folder))
|
||||
|
||||
@@ -57,9 +57,7 @@ def import_claude(
|
||||
markdown_processor, file_service = run_with_cleanup(get_importer_dependencies())
|
||||
|
||||
# Create the importer
|
||||
importer = ClaudeConversationsImporter(
|
||||
config.home, markdown_processor, file_service, project_name=config.name
|
||||
)
|
||||
importer = ClaudeConversationsImporter(config.home, markdown_processor, file_service)
|
||||
|
||||
# Process the file
|
||||
base_path = config.home / folder
|
||||
|
||||
@@ -56,9 +56,7 @@ def import_projects(
|
||||
markdown_processor, file_service = run_with_cleanup(get_importer_dependencies())
|
||||
|
||||
# Create the importer
|
||||
importer = ClaudeProjectsImporter(
|
||||
config.home, markdown_processor, file_service, project_name=config.name
|
||||
)
|
||||
importer = ClaudeProjectsImporter(config.home, markdown_processor, file_service)
|
||||
|
||||
# Process the file
|
||||
base_path = config.home / base_folder if base_folder else config.home
|
||||
|
||||
@@ -55,9 +55,7 @@ def memory_json(
|
||||
markdown_processor, file_service = run_with_cleanup(get_importer_dependencies())
|
||||
|
||||
# Create the importer
|
||||
importer = MemoryJsonImporter(
|
||||
config.home, markdown_processor, file_service, project_name=config.name
|
||||
)
|
||||
importer = MemoryJsonImporter(config.home, markdown_processor, file_service)
|
||||
|
||||
# Process the file
|
||||
base_path = config.home if not destination_folder else config.home / destination_folder
|
||||
|
||||
@@ -1,24 +1,21 @@
|
||||
"""MCP server command with streamable HTTP transport."""
|
||||
|
||||
import os
|
||||
from typing import Any, Optional
|
||||
|
||||
import typer
|
||||
from loguru import logger
|
||||
from typing import Optional
|
||||
|
||||
from basic_memory.cli.app import app
|
||||
from basic_memory.config import ConfigManager, init_mcp_logging
|
||||
|
||||
# Import mcp instance (has lifespan that handles initialization and file sync)
|
||||
from basic_memory.mcp.server import mcp as mcp_server # pragma: no cover
|
||||
|
||||
class _DeferredMcpServer:
|
||||
def run(self, *args: Any, **kwargs: Any) -> None: # pragma: no cover
|
||||
from basic_memory.mcp.server import mcp as live_mcp_server
|
||||
# Import mcp tools to register them
|
||||
import basic_memory.mcp.tools # noqa: F401 # pragma: no cover
|
||||
|
||||
live_mcp_server.run(*args, **kwargs)
|
||||
|
||||
|
||||
# Keep module-level attribute for tests/monkeypatching while deferring heavy import.
|
||||
mcp_server = _DeferredMcpServer()
|
||||
# Import prompts to register them
|
||||
import basic_memory.mcp.prompts # noqa: F401 # pragma: no cover
|
||||
from loguru import logger
|
||||
|
||||
|
||||
@app.command()
|
||||
@@ -50,11 +47,6 @@ def mcp(
|
||||
# Even when cloud_mode_enabled is True, stdio MCP runs locally and needs local API access.
|
||||
os.environ["BASIC_MEMORY_FORCE_LOCAL"] = "true"
|
||||
|
||||
# Import mcp tools/prompts to register them with the server
|
||||
import basic_memory.mcp.tools # noqa: F401 # pragma: no cover
|
||||
import basic_memory.mcp.prompts # noqa: F401 # pragma: no cover
|
||||
import basic_memory.mcp.resources # noqa: F401 # pragma: no cover
|
||||
|
||||
# Initialize logging for MCP (file only, stdout breaks protocol)
|
||||
init_mcp_logging()
|
||||
|
||||
|
||||
@@ -13,7 +13,7 @@ from rich.table import Table
|
||||
from basic_memory.cli.app import app
|
||||
from basic_memory.cli.commands.command_utils import get_project_info, run_with_cleanup
|
||||
from basic_memory.cli.commands.routing import force_routing, validate_routing_flags
|
||||
from basic_memory.config import ConfigManager, ProjectMode
|
||||
from basic_memory.config import ConfigManager
|
||||
from basic_memory.mcp.async_client import get_client
|
||||
from basic_memory.mcp.tools.utils import call_delete, call_get, call_patch, call_post, call_put
|
||||
from basic_memory.schemas.project_info import ProjectList, ProjectStatusResponse
|
||||
@@ -77,7 +77,6 @@ def list_projects(
|
||||
table = Table(title="Basic Memory Projects")
|
||||
table.add_column("Name", style="cyan")
|
||||
table.add_column("Path", style="green")
|
||||
table.add_column("Mode", style="blue")
|
||||
|
||||
# Add Local Path column if in cloud mode and not forcing local
|
||||
if config.cloud_mode_enabled and not local:
|
||||
@@ -91,10 +90,9 @@ def list_projects(
|
||||
for project in result.projects:
|
||||
is_default = "[X]" if project.is_default else ""
|
||||
normalized_path = normalize_project_path(project.path)
|
||||
project_mode = config.get_project_mode(project.name).value
|
||||
|
||||
# Build row based on mode
|
||||
row = [project.name, format_path(normalized_path), project_mode]
|
||||
row = [project.name, format_path(normalized_path)]
|
||||
|
||||
# Add local path if in cloud mode and not forcing local
|
||||
if config.cloud_mode_enabled and not local:
|
||||
@@ -513,74 +511,6 @@ def move_project(
|
||||
raise typer.Exit(1)
|
||||
|
||||
|
||||
@project_app.command("set-cloud")
|
||||
def set_cloud(
|
||||
name: str = typer.Argument(..., help="Name of the project to route through cloud"),
|
||||
) -> None:
|
||||
"""Set a project to cloud mode (route through cloud API).
|
||||
|
||||
Requires either an API key or an active OAuth session.
|
||||
|
||||
Examples:
|
||||
bm cloud set-key bmc_abc123... # save API key, then:
|
||||
bm project set-cloud research # route "research" through cloud
|
||||
|
||||
bm cloud login # OAuth login, then:
|
||||
bm project set-cloud research # route "research" through cloud
|
||||
"""
|
||||
from basic_memory.cli.auth import CLIAuth
|
||||
|
||||
config_manager = ConfigManager()
|
||||
config = config_manager.config
|
||||
|
||||
# Validate project exists in config
|
||||
if name not in config.projects:
|
||||
console.print(f"[red]Error: Project '{name}' not found in config[/red]")
|
||||
raise typer.Exit(1)
|
||||
|
||||
# Validate credentials: API key or OAuth session
|
||||
has_api_key = bool(config.cloud_api_key)
|
||||
has_oauth = False
|
||||
if not has_api_key:
|
||||
auth = CLIAuth(client_id=config.cloud_client_id, authkit_domain=config.cloud_domain)
|
||||
has_oauth = auth.load_tokens() is not None
|
||||
|
||||
if not has_api_key and not has_oauth:
|
||||
console.print("[red]Error: No cloud credentials found[/red]")
|
||||
console.print("[dim]Run 'bm cloud set-key <key>' or 'bm cloud login' first[/dim]")
|
||||
raise typer.Exit(1)
|
||||
|
||||
config.set_project_mode(name, ProjectMode.CLOUD)
|
||||
config_manager.save_config(config)
|
||||
|
||||
console.print(f"[green]Project '{name}' set to cloud mode[/green]")
|
||||
console.print("[dim]MCP tools and CLI commands for this project will route through cloud[/dim]")
|
||||
|
||||
|
||||
@project_app.command("set-local")
|
||||
def set_local(
|
||||
name: str = typer.Argument(..., help="Name of the project to revert to local mode"),
|
||||
) -> None:
|
||||
"""Revert a project to local mode (use in-process ASGI transport).
|
||||
|
||||
Example:
|
||||
bm project set-local research
|
||||
"""
|
||||
config_manager = ConfigManager()
|
||||
config = config_manager.config
|
||||
|
||||
# Validate project exists in config
|
||||
if name not in config.projects:
|
||||
console.print(f"[red]Error: Project '{name}' not found in config[/red]")
|
||||
raise typer.Exit(1)
|
||||
|
||||
config.set_project_mode(name, ProjectMode.LOCAL)
|
||||
config_manager.save_config(config)
|
||||
|
||||
console.print(f"[green]Project '{name}' set to local mode[/green]")
|
||||
console.print("[dim]MCP tools and CLI commands for this project will use local transport[/dim]")
|
||||
|
||||
|
||||
@project_app.command("sync")
|
||||
def sync_project_command(
|
||||
name: str = typer.Option(..., "--name", help="Project name to sync"),
|
||||
|
||||
@@ -1,336 +0,0 @@
|
||||
"""Schema management CLI commands for Basic Memory.
|
||||
|
||||
Provides CLI access to schema validation, inference, and drift detection.
|
||||
Registered as a subcommand group: `bm schema validate`, `bm schema infer`, `bm schema diff`.
|
||||
"""
|
||||
|
||||
import json
|
||||
from typing import Annotated, Optional
|
||||
|
||||
import typer
|
||||
from loguru import logger
|
||||
from rich.console import Console
|
||||
from rich.panel import Panel
|
||||
from rich.table import Table
|
||||
|
||||
from basic_memory.cli.app import app
|
||||
from basic_memory.cli.commands.command_utils import run_with_cleanup
|
||||
from basic_memory.cli.commands.routing import force_routing, validate_routing_flags
|
||||
from basic_memory.config import ConfigManager
|
||||
from basic_memory.mcp.async_client import get_client
|
||||
from basic_memory.mcp.project_context import get_active_project
|
||||
|
||||
console = Console()
|
||||
|
||||
schema_app = typer.Typer(help="Schema management commands")
|
||||
app.add_typer(schema_app, name="schema")
|
||||
|
||||
|
||||
def _resolve_project_name(project: Optional[str]) -> Optional[str]:
|
||||
"""Resolve project name from CLI argument or config default."""
|
||||
config_manager = ConfigManager()
|
||||
if project is not None:
|
||||
project_name, _ = config_manager.get_project(project)
|
||||
if not project_name:
|
||||
typer.echo(f"No project found named: {project}", err=True)
|
||||
raise typer.Exit(1)
|
||||
return project_name
|
||||
return config_manager.default_project
|
||||
|
||||
|
||||
# --- Validate ---
|
||||
|
||||
|
||||
async def _run_validate(
|
||||
target: Optional[str] = None,
|
||||
project: Optional[str] = None,
|
||||
strict: bool = False,
|
||||
):
|
||||
"""Run schema validation via the API."""
|
||||
from basic_memory.mcp.clients.schema import SchemaClient
|
||||
|
||||
async with get_client() as client:
|
||||
active_project = await get_active_project(client, project, None)
|
||||
schema_client = SchemaClient(client, active_project.external_id)
|
||||
|
||||
# Determine if target is a note identifier or entity type
|
||||
# Heuristic: if target contains / or ., treat as identifier
|
||||
entity_type = None
|
||||
identifier = None
|
||||
if target:
|
||||
if "/" in target or "." in target:
|
||||
identifier = target
|
||||
else:
|
||||
entity_type = target
|
||||
|
||||
report = await schema_client.validate(
|
||||
entity_type=entity_type,
|
||||
identifier=identifier,
|
||||
)
|
||||
|
||||
# --- Display results ---
|
||||
if report.total_notes == 0:
|
||||
console.print("[yellow]No notes matched for validation.[/yellow]")
|
||||
return
|
||||
|
||||
table = Table(title=f"Schema Validation: {entity_type or identifier or 'all'}")
|
||||
table.add_column("Note", style="cyan")
|
||||
table.add_column("Status", justify="center")
|
||||
table.add_column("Warnings", justify="right")
|
||||
table.add_column("Errors", justify="right")
|
||||
|
||||
for result in report.results:
|
||||
if result.passed and not result.warnings:
|
||||
status = "[green]pass[/green]"
|
||||
elif result.passed:
|
||||
status = "[yellow]warn[/yellow]"
|
||||
else:
|
||||
status = "[red]fail[/red]"
|
||||
|
||||
table.add_row(
|
||||
result.note_identifier,
|
||||
status,
|
||||
str(len(result.warnings)),
|
||||
str(len(result.errors)),
|
||||
)
|
||||
|
||||
console.print(table)
|
||||
console.print(
|
||||
f"\nSummary: {report.valid_count}/{report.total_notes} valid, "
|
||||
f"{report.warning_count} warnings, {report.error_count} errors"
|
||||
)
|
||||
|
||||
# Exit with error code in strict mode if there are failures
|
||||
if strict and report.error_count > 0:
|
||||
raise typer.Exit(1)
|
||||
|
||||
|
||||
@schema_app.command()
|
||||
def validate(
|
||||
target: Annotated[
|
||||
Optional[str],
|
||||
typer.Argument(help="Note path or entity type to validate"),
|
||||
] = None,
|
||||
project: Annotated[
|
||||
Optional[str],
|
||||
typer.Option(help="The project name."),
|
||||
] = None,
|
||||
strict: bool = typer.Option(False, "--strict", help="Exit with error on validation failures"),
|
||||
local: bool = typer.Option(
|
||||
False, "--local", help="Force local API routing (ignore cloud mode)"
|
||||
),
|
||||
cloud: bool = typer.Option(False, "--cloud", help="Force cloud API routing"),
|
||||
):
|
||||
"""Validate notes against their schemas.
|
||||
|
||||
TARGET can be a note path (e.g., people/ada-lovelace.md) or an entity type
|
||||
(e.g., Person). If omitted, validates all notes that have schemas.
|
||||
|
||||
Use --strict to exit with error code 1 if any validation errors are found.
|
||||
Use --local to force local routing when cloud mode is enabled.
|
||||
Use --cloud to force cloud routing when cloud mode is disabled.
|
||||
"""
|
||||
try:
|
||||
validate_routing_flags(local, cloud)
|
||||
project_name = _resolve_project_name(project)
|
||||
with force_routing(local=local, cloud=cloud):
|
||||
run_with_cleanup(_run_validate(target, project_name, strict))
|
||||
except ValueError as e:
|
||||
console.print(f"[red]Error: {e}[/red]")
|
||||
raise typer.Exit(1)
|
||||
except Exception as e:
|
||||
if not isinstance(e, typer.Exit):
|
||||
logger.error(f"Error during schema validate: {e}")
|
||||
typer.echo(f"Error during schema validate: {e}", err=True)
|
||||
raise typer.Exit(1)
|
||||
raise
|
||||
|
||||
|
||||
# --- Infer ---
|
||||
|
||||
|
||||
async def _run_infer(
|
||||
entity_type: str,
|
||||
project: Optional[str] = None,
|
||||
threshold: float = 0.25,
|
||||
save: bool = False,
|
||||
):
|
||||
"""Run schema inference via the API."""
|
||||
from basic_memory.mcp.clients.schema import SchemaClient
|
||||
|
||||
async with get_client() as client:
|
||||
active_project = await get_active_project(client, project, None)
|
||||
schema_client = SchemaClient(client, active_project.external_id)
|
||||
|
||||
report = await schema_client.infer(entity_type, threshold=threshold)
|
||||
|
||||
if report.notes_analyzed == 0:
|
||||
console.print(f"[yellow]No notes found with type: {entity_type}[/yellow]")
|
||||
return
|
||||
|
||||
# --- Display frequency analysis ---
|
||||
console.print(
|
||||
f"\n[bold]Analyzing {report.notes_analyzed} notes with type: {entity_type}...[/bold]\n"
|
||||
)
|
||||
|
||||
table = Table(title="Field Frequencies")
|
||||
table.add_column("Field", style="cyan")
|
||||
table.add_column("Source")
|
||||
table.add_column("Count", justify="right")
|
||||
table.add_column("Percentage", justify="right")
|
||||
table.add_column("Suggested")
|
||||
|
||||
for freq in report.field_frequencies:
|
||||
pct = f"{freq.percentage:.0%}"
|
||||
if freq.name in report.suggested_required:
|
||||
suggested = "[green]required[/green]"
|
||||
elif freq.name in report.suggested_optional:
|
||||
suggested = "[yellow]optional[/yellow]"
|
||||
else:
|
||||
suggested = "[dim]excluded[/dim]"
|
||||
|
||||
table.add_row(
|
||||
freq.name,
|
||||
freq.source,
|
||||
str(freq.count),
|
||||
pct,
|
||||
suggested,
|
||||
)
|
||||
|
||||
console.print(table)
|
||||
|
||||
# --- Display suggested schema ---
|
||||
console.print("\n[bold]Suggested schema:[/bold]")
|
||||
console.print(Panel(json.dumps(report.suggested_schema, indent=2), title="Picoschema"))
|
||||
|
||||
if save:
|
||||
console.print(
|
||||
f"\n[yellow]--save not yet implemented. "
|
||||
f"Copy the schema above into schema/{entity_type}.md[/yellow]"
|
||||
)
|
||||
|
||||
|
||||
@schema_app.command()
|
||||
def infer(
|
||||
entity_type: Annotated[
|
||||
str,
|
||||
typer.Argument(help="Entity type to analyze (e.g., Person, meeting)"),
|
||||
],
|
||||
project: Annotated[
|
||||
Optional[str],
|
||||
typer.Option(help="The project name."),
|
||||
] = None,
|
||||
threshold: float = typer.Option(
|
||||
0.25, "--threshold", help="Minimum frequency for optional fields (0-1)"
|
||||
),
|
||||
save: bool = typer.Option(False, "--save", help="Save inferred schema to schema/ directory"),
|
||||
local: bool = typer.Option(
|
||||
False, "--local", help="Force local API routing (ignore cloud mode)"
|
||||
),
|
||||
cloud: bool = typer.Option(False, "--cloud", help="Force cloud API routing"),
|
||||
):
|
||||
"""Infer schema from existing notes of a type.
|
||||
|
||||
Analyzes all notes with the given entity type and suggests a Picoschema
|
||||
definition based on observation and relation frequency.
|
||||
|
||||
Fields present in 95%+ of notes become required. Fields above the
|
||||
threshold (default 25%) become optional. Fields below threshold are excluded.
|
||||
|
||||
Use --local to force local routing when cloud mode is enabled.
|
||||
Use --cloud to force cloud routing when cloud mode is disabled.
|
||||
"""
|
||||
try:
|
||||
validate_routing_flags(local, cloud)
|
||||
project_name = _resolve_project_name(project)
|
||||
with force_routing(local=local, cloud=cloud):
|
||||
run_with_cleanup(_run_infer(entity_type, project_name, threshold, save))
|
||||
except ValueError as e:
|
||||
console.print(f"[red]Error: {e}[/red]")
|
||||
raise typer.Exit(1)
|
||||
except Exception as e:
|
||||
if not isinstance(e, typer.Exit):
|
||||
logger.error(f"Error during schema infer: {e}")
|
||||
typer.echo(f"Error during schema infer: {e}", err=True)
|
||||
raise typer.Exit(1)
|
||||
raise
|
||||
|
||||
|
||||
# --- Diff ---
|
||||
|
||||
|
||||
async def _run_diff(
|
||||
entity_type: str,
|
||||
project: Optional[str] = None,
|
||||
):
|
||||
"""Run schema drift detection via the API."""
|
||||
from basic_memory.mcp.clients.schema import SchemaClient
|
||||
|
||||
async with get_client() as client:
|
||||
active_project = await get_active_project(client, project, None)
|
||||
schema_client = SchemaClient(client, active_project.external_id)
|
||||
|
||||
report = await schema_client.diff(entity_type)
|
||||
|
||||
has_drift = report.new_fields or report.dropped_fields or report.cardinality_changes
|
||||
|
||||
if not has_drift:
|
||||
console.print(f"[green]No drift detected for {entity_type} schema.[/green]")
|
||||
return
|
||||
|
||||
console.print(f"\n[bold]Schema drift detected for {entity_type}:[/bold]\n")
|
||||
|
||||
if report.new_fields:
|
||||
console.print("[green]+ New fields (common in notes, not in schema):[/green]")
|
||||
for f in report.new_fields:
|
||||
console.print(f" + {f.name}: {f.percentage:.0%} of notes ({f.source})")
|
||||
|
||||
if report.dropped_fields:
|
||||
console.print("[red]- Dropped fields (in schema, rare in notes):[/red]")
|
||||
for f in report.dropped_fields:
|
||||
console.print(f" - {f.name}: {f.percentage:.0%} of notes ({f.source})")
|
||||
|
||||
if report.cardinality_changes:
|
||||
console.print("[yellow]~ Cardinality changes:[/yellow]")
|
||||
for change in report.cardinality_changes:
|
||||
console.print(f" ~ {change}")
|
||||
|
||||
|
||||
@schema_app.command()
|
||||
def diff(
|
||||
entity_type: Annotated[
|
||||
str,
|
||||
typer.Argument(help="Entity type to check for drift"),
|
||||
],
|
||||
project: Annotated[
|
||||
Optional[str],
|
||||
typer.Option(help="The project name."),
|
||||
] = None,
|
||||
local: bool = typer.Option(
|
||||
False, "--local", help="Force local API routing (ignore cloud mode)"
|
||||
),
|
||||
cloud: bool = typer.Option(False, "--cloud", help="Force cloud API routing"),
|
||||
):
|
||||
"""Show drift between schema and actual usage.
|
||||
|
||||
Compares the existing schema definition for an entity type against
|
||||
how notes of that type are actually structured. Identifies new fields,
|
||||
dropped fields, and cardinality changes.
|
||||
|
||||
Use --local to force local routing when cloud mode is enabled.
|
||||
Use --cloud to force cloud routing when cloud mode is disabled.
|
||||
"""
|
||||
try:
|
||||
validate_routing_flags(local, cloud)
|
||||
project_name = _resolve_project_name(project)
|
||||
with force_routing(local=local, cloud=cloud):
|
||||
run_with_cleanup(_run_diff(entity_type, project_name))
|
||||
except ValueError as e:
|
||||
console.print(f"[red]Error: {e}[/red]")
|
||||
raise typer.Exit(1)
|
||||
except Exception as e:
|
||||
if not isinstance(e, typer.Exit):
|
||||
logger.error(f"Error during schema diff: {e}")
|
||||
typer.echo(f"Error during schema diff: {e}", err=True)
|
||||
raise typer.Exit(1)
|
||||
raise
|
||||
@@ -459,8 +459,6 @@ def search_notes(
|
||||
] = "",
|
||||
permalink: Annotated[bool, typer.Option("--permalink", help="Search permalink values")] = False,
|
||||
title: Annotated[bool, typer.Option("--title", help="Search title values")] = False,
|
||||
vector: Annotated[bool, typer.Option("--vector", help="Use vector retrieval")] = False,
|
||||
hybrid: Annotated[bool, typer.Option("--hybrid", help="Use hybrid retrieval")] = False,
|
||||
project: Annotated[
|
||||
Optional[str],
|
||||
typer.Option(
|
||||
@@ -483,13 +481,6 @@ def search_notes(
|
||||
Optional[List[str]],
|
||||
typer.Option("--type", help="Filter by frontmatter type (repeatable)"),
|
||||
] = None,
|
||||
entity_types: Annotated[
|
||||
Optional[List[str]],
|
||||
typer.Option(
|
||||
"--entity-type",
|
||||
help="Filter by search item type: entity, observation, relation (repeatable)",
|
||||
),
|
||||
] = None,
|
||||
meta: Annotated[
|
||||
Optional[List[str]],
|
||||
typer.Option("--meta", help="Filter by frontmatter key=value (repeatable)"),
|
||||
@@ -525,10 +516,9 @@ def search_notes(
|
||||
# use the project name, or the default from the config
|
||||
project_name = project_name or config_manager.default_project
|
||||
|
||||
mode_flags = [permalink, title, vector, hybrid]
|
||||
if sum(1 for enabled in mode_flags if enabled) > 1: # pragma: no cover
|
||||
if permalink and title: # pragma: no cover
|
||||
typer.echo(
|
||||
"Use only one mode flag: --permalink, --title, --vector, or --hybrid. Exiting.",
|
||||
"Use either --permalink or --title, not both. Exiting.",
|
||||
err=True,
|
||||
)
|
||||
raise typer.Exit(1)
|
||||
@@ -570,10 +560,6 @@ def search_notes(
|
||||
search_type = "permalink"
|
||||
if title:
|
||||
search_type = "title"
|
||||
if vector:
|
||||
search_type = "vector"
|
||||
if hybrid:
|
||||
search_type = "hybrid"
|
||||
|
||||
with force_routing(local=local, cloud=cloud):
|
||||
results = run_with_cleanup(
|
||||
@@ -585,16 +571,11 @@ def search_notes(
|
||||
after_date=after_date,
|
||||
page_size=page_size,
|
||||
types=note_types,
|
||||
entity_types=entity_types,
|
||||
metadata_filters=metadata_filters,
|
||||
tags=tags,
|
||||
status=status,
|
||||
)
|
||||
)
|
||||
if isinstance(results, str):
|
||||
print(results)
|
||||
raise typer.Exit(1)
|
||||
|
||||
results_dict = results.model_dump(exclude_none=True)
|
||||
print(json.dumps(results_dict, indent=2, ensure_ascii=True, default=str))
|
||||
except ValueError as e:
|
||||
|
||||
@@ -1,97 +0,0 @@
|
||||
"""Watch command - run file watcher as a standalone long-running process."""
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import signal
|
||||
import sys
|
||||
from typing import Optional
|
||||
|
||||
import typer
|
||||
from loguru import logger
|
||||
|
||||
from basic_memory import db
|
||||
from basic_memory.cli.app import app
|
||||
from basic_memory.cli.container import get_container
|
||||
from basic_memory.config import ConfigManager
|
||||
from basic_memory.services.initialization import initialize_app
|
||||
from basic_memory.sync.coordinator import SyncCoordinator
|
||||
|
||||
|
||||
async def run_watch(project: Optional[str] = None) -> None:
|
||||
"""Run the file watcher as a long-running process.
|
||||
|
||||
This is the async core of the watch command. It:
|
||||
1. Initializes the app (DB migrations + project reconciliation)
|
||||
2. Validates and sets project constraint if --project given
|
||||
3. Creates a SyncCoordinator with quiet=False for Rich console output
|
||||
4. Blocks until SIGINT/SIGTERM, then shuts down cleanly
|
||||
"""
|
||||
container = get_container()
|
||||
config = container.config
|
||||
|
||||
# --- Initialization ---
|
||||
# Wrapped in try/finally so DB resources are cleaned up on all exit paths,
|
||||
# including early exits from invalid --project names.
|
||||
await initialize_app(config)
|
||||
sync_coordinator = None
|
||||
|
||||
try:
|
||||
# --- Project constraint ---
|
||||
if project:
|
||||
config_manager = ConfigManager()
|
||||
project_name, _ = config_manager.get_project(project)
|
||||
if not project_name:
|
||||
typer.echo(f"No project found named: {project}", err=True)
|
||||
raise typer.Exit(1)
|
||||
|
||||
os.environ["BASIC_MEMORY_MCP_PROJECT"] = project_name
|
||||
logger.info(f"Watch constrained to project: {project_name}")
|
||||
|
||||
# --- Sync coordinator ---
|
||||
# quiet=False so file change events are printed to the terminal
|
||||
sync_coordinator = SyncCoordinator(config=config, should_sync=True, quiet=False)
|
||||
|
||||
# --- Signal handling ---
|
||||
shutdown_event = asyncio.Event()
|
||||
|
||||
def _signal_handler() -> None:
|
||||
logger.info("Shutdown signal received")
|
||||
shutdown_event.set()
|
||||
|
||||
loop = asyncio.get_running_loop()
|
||||
|
||||
# Windows ProactorEventLoop does not support add_signal_handler;
|
||||
# fall back to the stdlib signal module which works cross-platform.
|
||||
try:
|
||||
for sig in (signal.SIGINT, signal.SIGTERM):
|
||||
loop.add_signal_handler(sig, _signal_handler)
|
||||
except NotImplementedError:
|
||||
for sig in (signal.SIGINT, signal.SIGTERM):
|
||||
signal.signal(sig, lambda _signum, _frame: _signal_handler())
|
||||
|
||||
# --- Run ---
|
||||
await sync_coordinator.start()
|
||||
logger.info("Watch service running, press Ctrl+C to stop")
|
||||
await shutdown_event.wait()
|
||||
finally:
|
||||
if sync_coordinator is not None:
|
||||
await sync_coordinator.stop()
|
||||
await db.shutdown_db()
|
||||
logger.info("Watch service stopped")
|
||||
|
||||
|
||||
@app.command()
|
||||
def watch(
|
||||
project: Optional[str] = typer.Option(None, help="Restrict watcher to a single project"),
|
||||
) -> None:
|
||||
"""Run file watcher as a long-running process (no MCP server).
|
||||
|
||||
Watches for file changes in project directories and syncs them to the
|
||||
database. Useful for running Basic Memory sync alongside external tools
|
||||
that don't use the MCP server.
|
||||
"""
|
||||
# On Windows, use SelectorEventLoop to avoid ProactorEventLoop cleanup issues
|
||||
if sys.platform == "win32": # pragma: no cover
|
||||
asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
|
||||
|
||||
asyncio.run(run_watch(project=project))
|
||||
@@ -1,33 +1,25 @@
|
||||
"""Main CLI entry point for basic-memory.""" # pragma: no cover
|
||||
|
||||
import sys
|
||||
import warnings
|
||||
|
||||
from basic_memory.cli.app import app # pragma: no cover
|
||||
|
||||
def _version_only_invocation(argv: list[str]) -> bool:
|
||||
# Trigger: invocation is exactly `bm --version` or `bm -v`
|
||||
# Why: avoid importing command modules on the hot version path
|
||||
# Outcome: eager version callback exits quickly with minimal startup work
|
||||
return len(argv) == 1 and argv[0] in {"--version", "-v"}
|
||||
# Register commands
|
||||
from basic_memory.cli.commands import ( # noqa: F401 # pragma: no cover
|
||||
cloud,
|
||||
db,
|
||||
doctor,
|
||||
import_chatgpt,
|
||||
import_claude_conversations,
|
||||
import_claude_projects,
|
||||
import_memory_json,
|
||||
mcp,
|
||||
project,
|
||||
status,
|
||||
tool,
|
||||
)
|
||||
|
||||
|
||||
if not _version_only_invocation(sys.argv[1:]):
|
||||
# Register commands only when not short-circuiting for --version
|
||||
from basic_memory.cli.commands import ( # noqa: F401 # pragma: no cover
|
||||
cloud,
|
||||
db,
|
||||
doctor,
|
||||
import_chatgpt,
|
||||
import_claude_conversations,
|
||||
import_claude_projects,
|
||||
import_memory_json,
|
||||
mcp,
|
||||
project,
|
||||
schema,
|
||||
status,
|
||||
tool,
|
||||
)
|
||||
# Re-apply warning filter AFTER all imports
|
||||
# (authlib adds a DeprecationWarning filter that overrides ours)
|
||||
import warnings # pragma: no cover
|
||||
|
||||
warnings.filterwarnings("ignore") # pragma: no cover
|
||||
|
||||
|
||||
@@ -1,84 +0,0 @@
|
||||
"""Cloud promo messaging for CLI entrypoint."""
|
||||
|
||||
import os
|
||||
import sys
|
||||
from collections.abc import Callable
|
||||
|
||||
import typer
|
||||
|
||||
from basic_memory.config import ConfigManager
|
||||
|
||||
CLOUD_PROMO_VERSION = "2026-02-06"
|
||||
OSS_DISCOUNT_CODE = "{{OSS_DISCOUNT_CODE}}"
|
||||
|
||||
|
||||
def _promos_disabled_by_env() -> bool:
|
||||
"""Check environment-level kill switch for promo output."""
|
||||
value = os.getenv("BASIC_MEMORY_NO_PROMOS", "").strip().lower()
|
||||
return value in {"1", "true", "yes"}
|
||||
|
||||
|
||||
def _is_interactive_session() -> bool:
|
||||
"""Return whether stdin/stdout are interactive terminals."""
|
||||
return sys.stdin.isatty() and sys.stdout.isatty()
|
||||
|
||||
|
||||
def _build_first_run_message() -> str:
|
||||
"""Build first-run cloud promo copy."""
|
||||
return (
|
||||
"Basic Memory initialized (local mode).\n"
|
||||
"Cloud is optional and keeps your workflow local-first.\n"
|
||||
"Cloud adds cross-device sync + mobile/web access.\n"
|
||||
f"OSS discount: {OSS_DISCOUNT_CODE} (20% off for 3 months).\n"
|
||||
"Run `bm cloud login` to enable."
|
||||
)
|
||||
|
||||
|
||||
def _build_version_message() -> str:
|
||||
"""Build cloud promo copy shown after promo-version bumps."""
|
||||
return (
|
||||
"New in Basic Memory Cloud: cross-device sync + mobile/web access.\n"
|
||||
f"OSS discount: {OSS_DISCOUNT_CODE} (20% off for 3 months).\n"
|
||||
"Run `bm cloud login` to enable."
|
||||
)
|
||||
|
||||
|
||||
def maybe_show_cloud_promo(
|
||||
invoked_subcommand: str | None,
|
||||
*,
|
||||
config_manager: ConfigManager | None = None,
|
||||
is_interactive: bool | None = None,
|
||||
echo: Callable[[str], None] = typer.echo,
|
||||
) -> None:
|
||||
"""Show cloud promo copy when discovery gates are satisfied."""
|
||||
manager = config_manager or ConfigManager()
|
||||
config = manager.load_config()
|
||||
|
||||
interactive = _is_interactive_session() if is_interactive is None else is_interactive
|
||||
|
||||
# Trigger: environment-level promo suppression or non-interactive execution.
|
||||
# Why: avoid polluting scripts/CI output and support a hard opt-out.
|
||||
# Outcome: skip all promo copy for this invocation.
|
||||
if _promos_disabled_by_env() or not interactive:
|
||||
return
|
||||
|
||||
# Trigger: command context where cloud promo is not actionable.
|
||||
# Why: mcp/stdin protocol and root help flows should stay noise-free.
|
||||
# Outcome: command continues without promo messaging.
|
||||
if invoked_subcommand in {None, "mcp"}:
|
||||
return
|
||||
|
||||
if config.cloud_mode_enabled or config.cloud_promo_opt_out:
|
||||
return
|
||||
|
||||
show_first_run = not config.cloud_promo_first_run_shown
|
||||
show_version_notice = config.cloud_promo_last_version_shown != CLOUD_PROMO_VERSION
|
||||
if not show_first_run and not show_version_notice:
|
||||
return
|
||||
|
||||
message = _build_first_run_message() if show_first_run else _build_version_message()
|
||||
echo(message)
|
||||
|
||||
config.cloud_promo_first_run_shown = True
|
||||
config.cloud_promo_last_version_shown = CLOUD_PROMO_VERSION
|
||||
manager.save_config(config)
|
||||
@@ -24,13 +24,6 @@ WATCH_STATUS_JSON = "watch-status.json"
|
||||
Environment = Literal["test", "dev", "user"]
|
||||
|
||||
|
||||
class ProjectMode(str, Enum):
|
||||
"""Per-project routing mode."""
|
||||
|
||||
LOCAL = "local"
|
||||
CLOUD = "cloud"
|
||||
|
||||
|
||||
class DatabaseBackend(str, Enum):
|
||||
"""Supported database backends."""
|
||||
|
||||
@@ -44,7 +37,6 @@ class ProjectConfig:
|
||||
|
||||
name: str
|
||||
home: Path
|
||||
mode: ProjectMode = ProjectMode.LOCAL
|
||||
|
||||
@property
|
||||
def project(self):
|
||||
@@ -89,7 +81,7 @@ class BasicMemoryConfig(BaseSettings):
|
||||
description="Name of the default project to use",
|
||||
)
|
||||
default_project_mode: bool = Field(
|
||||
default=True,
|
||||
default=False,
|
||||
description="When True, MCP tools automatically use default_project when no project parameter is specified. Enables simplified UX for single-project workflows.",
|
||||
)
|
||||
|
||||
@@ -107,34 +99,6 @@ class BasicMemoryConfig(BaseSettings):
|
||||
description="Database connection URL. For Postgres, use postgresql+asyncpg://user:pass@host:port/db. If not set, SQLite will use default path.",
|
||||
)
|
||||
|
||||
# Semantic search configuration
|
||||
semantic_search_enabled: bool = Field(
|
||||
default=False,
|
||||
description="Enable semantic search (vector/hybrid retrieval). Works on both SQLite and Postgres backends. Requires semantic extras.",
|
||||
)
|
||||
semantic_embedding_provider: str = Field(
|
||||
default="fastembed",
|
||||
description="Embedding provider for local semantic indexing/search.",
|
||||
)
|
||||
semantic_embedding_model: str = Field(
|
||||
default="bge-small-en-v1.5",
|
||||
description="Embedding model identifier used by the local provider.",
|
||||
)
|
||||
semantic_embedding_dimensions: int | None = Field(
|
||||
default=None,
|
||||
description="Embedding vector dimensions. Auto-detected from provider if not set (384 for FastEmbed, 1536 for OpenAI).",
|
||||
)
|
||||
semantic_embedding_batch_size: int = Field(
|
||||
default=64,
|
||||
description="Batch size for embedding generation.",
|
||||
gt=0,
|
||||
)
|
||||
semantic_vector_k: int = Field(
|
||||
default=100,
|
||||
description="Vector candidate count for vector and hybrid retrieval.",
|
||||
gt=0,
|
||||
)
|
||||
|
||||
# Database connection pool configuration (Postgres only)
|
||||
db_pool_size: int = Field(
|
||||
default=20,
|
||||
@@ -196,11 +160,6 @@ class BasicMemoryConfig(BaseSettings):
|
||||
description="Disable automatic permalink generation in frontmatter. When enabled, new notes won't have permalinks added and sync won't update permalinks. Existing permalinks will still work for reading.",
|
||||
)
|
||||
|
||||
permalinks_include_project: bool = Field(
|
||||
default=True,
|
||||
description="When True, generated permalinks are prefixed with the project slug (e.g., 'specs/search'). Existing permalinks remain unchanged unless explicitly updated.",
|
||||
)
|
||||
|
||||
skip_initialization_sync: bool = Field(
|
||||
default=False,
|
||||
description="Skip expensive initialization synchronization. Useful for cloud/stateless deployments where project reconciliation is not needed.",
|
||||
@@ -262,31 +221,6 @@ class BasicMemoryConfig(BaseSettings):
|
||||
description="Cloud project sync configuration mapping project names to their local paths and sync state",
|
||||
)
|
||||
|
||||
cloud_promo_opt_out: bool = Field(
|
||||
default=False,
|
||||
description="Disable CLI cloud promo messages when true.",
|
||||
)
|
||||
|
||||
cloud_promo_first_run_shown: bool = Field(
|
||||
default=False,
|
||||
description="Tracks whether the first-run cloud promo message has been shown.",
|
||||
)
|
||||
|
||||
cloud_promo_last_version_shown: Optional[str] = Field(
|
||||
default=None,
|
||||
description="Most recent cloud promo version shown in CLI.",
|
||||
)
|
||||
|
||||
cloud_api_key: Optional[str] = Field(
|
||||
default=None,
|
||||
description="API key for cloud access (bmc_ prefixed). Account-level, not per-project.",
|
||||
)
|
||||
|
||||
project_modes: Dict[str, ProjectMode] = Field(
|
||||
default_factory=dict,
|
||||
description="Per-project routing mode. Projects not listed default to LOCAL.",
|
||||
)
|
||||
|
||||
@property
|
||||
def is_test_env(self) -> bool:
|
||||
"""Check if running in a test environment.
|
||||
@@ -320,21 +254,6 @@ class BasicMemoryConfig(BaseSettings):
|
||||
# Fall back to config file value
|
||||
return self.cloud_mode
|
||||
|
||||
def get_project_mode(self, project_name: str) -> ProjectMode:
|
||||
"""Get the routing mode for a project.
|
||||
|
||||
Returns the per-project mode if set, otherwise LOCAL.
|
||||
"""
|
||||
return self.project_modes.get(project_name, ProjectMode.LOCAL)
|
||||
|
||||
def set_project_mode(self, project_name: str, mode: ProjectMode) -> None:
|
||||
"""Set the routing mode for a project."""
|
||||
if mode == ProjectMode.LOCAL:
|
||||
# Remove from dict to keep config clean — LOCAL is the default
|
||||
self.project_modes.pop(project_name, None)
|
||||
else:
|
||||
self.project_modes[project_name] = mode
|
||||
|
||||
@classmethod
|
||||
def for_cloud_tenant(
|
||||
cls,
|
||||
@@ -402,11 +321,8 @@ class BasicMemoryConfig(BaseSettings):
|
||||
|
||||
This is the single database that will store all knowledge data
|
||||
across all projects.
|
||||
|
||||
Uses BASIC_MEMORY_CONFIG_DIR when set so each process/worktree can
|
||||
isolate both config and database state.
|
||||
"""
|
||||
database_path = self.data_dir_path / APP_DATABASE_NAME
|
||||
database_path = Path.home() / DATA_DIR_NAME / APP_DATABASE_NAME
|
||||
if not database_path.exists(): # pragma: no cover
|
||||
database_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
database_path.touch()
|
||||
@@ -428,10 +344,7 @@ class BasicMemoryConfig(BaseSettings):
|
||||
@property
|
||||
def project_list(self) -> List[ProjectConfig]: # pragma: no cover
|
||||
"""Get all configured projects as ProjectConfig objects."""
|
||||
return [
|
||||
ProjectConfig(name=name, home=Path(path), mode=self.get_project_mode(name))
|
||||
for name, path in self.projects.items()
|
||||
]
|
||||
return [ProjectConfig(name=name, home=Path(path)) for name, path in self.projects.items()]
|
||||
|
||||
@model_validator(mode="after")
|
||||
def ensure_project_paths_exists(self) -> "BasicMemoryConfig": # pragma: no cover
|
||||
@@ -455,13 +368,8 @@ class BasicMemoryConfig(BaseSettings):
|
||||
return self
|
||||
|
||||
@property
|
||||
def data_dir_path(self) -> Path:
|
||||
"""Get app state directory for config and default SQLite database."""
|
||||
if config_dir := os.getenv("BASIC_MEMORY_CONFIG_DIR"):
|
||||
return Path(config_dir)
|
||||
|
||||
home = os.getenv("HOME", Path.home())
|
||||
return Path(home) / DATA_DIR_NAME
|
||||
def data_dir_path(self):
|
||||
return Path.home() / DATA_DIR_NAME
|
||||
|
||||
|
||||
# Module-level cache for configuration
|
||||
|
||||
@@ -41,12 +41,7 @@ async def get_chatgpt_importer(
|
||||
file_service: FileServiceDep,
|
||||
) -> ChatGPTImporter:
|
||||
"""Create ChatGPTImporter with dependencies."""
|
||||
return ChatGPTImporter(
|
||||
project_config.home,
|
||||
markdown_processor,
|
||||
file_service,
|
||||
project_name=project_config.name,
|
||||
)
|
||||
return ChatGPTImporter(project_config.home, markdown_processor, file_service)
|
||||
|
||||
|
||||
ChatGPTImporterDep = Annotated[ChatGPTImporter, Depends(get_chatgpt_importer)]
|
||||
@@ -58,12 +53,7 @@ async def get_chatgpt_importer_v2( # pragma: no cover
|
||||
file_service: FileServiceV2Dep,
|
||||
) -> ChatGPTImporter:
|
||||
"""Create ChatGPTImporter with v2 dependencies."""
|
||||
return ChatGPTImporter(
|
||||
project_config.home,
|
||||
markdown_processor,
|
||||
file_service,
|
||||
project_name=project_config.name,
|
||||
)
|
||||
return ChatGPTImporter(project_config.home, markdown_processor, file_service)
|
||||
|
||||
|
||||
ChatGPTImporterV2Dep = Annotated[ChatGPTImporter, Depends(get_chatgpt_importer_v2)]
|
||||
@@ -75,12 +65,7 @@ async def get_chatgpt_importer_v2_external(
|
||||
file_service: FileServiceV2ExternalDep,
|
||||
) -> ChatGPTImporter:
|
||||
"""Create ChatGPTImporter with v2 external_id dependencies."""
|
||||
return ChatGPTImporter(
|
||||
project_config.home,
|
||||
markdown_processor,
|
||||
file_service,
|
||||
project_name=project_config.name,
|
||||
)
|
||||
return ChatGPTImporter(project_config.home, markdown_processor, file_service)
|
||||
|
||||
|
||||
ChatGPTImporterV2ExternalDep = Annotated[ChatGPTImporter, Depends(get_chatgpt_importer_v2_external)]
|
||||
@@ -95,12 +80,7 @@ async def get_claude_conversations_importer(
|
||||
file_service: FileServiceDep,
|
||||
) -> ClaudeConversationsImporter:
|
||||
"""Create ClaudeConversationsImporter with dependencies."""
|
||||
return ClaudeConversationsImporter(
|
||||
project_config.home,
|
||||
markdown_processor,
|
||||
file_service,
|
||||
project_name=project_config.name,
|
||||
)
|
||||
return ClaudeConversationsImporter(project_config.home, markdown_processor, file_service)
|
||||
|
||||
|
||||
ClaudeConversationsImporterDep = Annotated[
|
||||
@@ -114,12 +94,7 @@ async def get_claude_conversations_importer_v2( # pragma: no cover
|
||||
file_service: FileServiceV2Dep,
|
||||
) -> ClaudeConversationsImporter:
|
||||
"""Create ClaudeConversationsImporter with v2 dependencies."""
|
||||
return ClaudeConversationsImporter(
|
||||
project_config.home,
|
||||
markdown_processor,
|
||||
file_service,
|
||||
project_name=project_config.name,
|
||||
)
|
||||
return ClaudeConversationsImporter(project_config.home, markdown_processor, file_service)
|
||||
|
||||
|
||||
ClaudeConversationsImporterV2Dep = Annotated[
|
||||
@@ -133,12 +108,7 @@ async def get_claude_conversations_importer_v2_external(
|
||||
file_service: FileServiceV2ExternalDep,
|
||||
) -> ClaudeConversationsImporter:
|
||||
"""Create ClaudeConversationsImporter with v2 external_id dependencies."""
|
||||
return ClaudeConversationsImporter(
|
||||
project_config.home,
|
||||
markdown_processor,
|
||||
file_service,
|
||||
project_name=project_config.name,
|
||||
)
|
||||
return ClaudeConversationsImporter(project_config.home, markdown_processor, file_service)
|
||||
|
||||
|
||||
ClaudeConversationsImporterV2ExternalDep = Annotated[
|
||||
@@ -155,12 +125,7 @@ async def get_claude_projects_importer(
|
||||
file_service: FileServiceDep,
|
||||
) -> ClaudeProjectsImporter:
|
||||
"""Create ClaudeProjectsImporter with dependencies."""
|
||||
return ClaudeProjectsImporter(
|
||||
project_config.home,
|
||||
markdown_processor,
|
||||
file_service,
|
||||
project_name=project_config.name,
|
||||
)
|
||||
return ClaudeProjectsImporter(project_config.home, markdown_processor, file_service)
|
||||
|
||||
|
||||
ClaudeProjectsImporterDep = Annotated[ClaudeProjectsImporter, Depends(get_claude_projects_importer)]
|
||||
@@ -172,12 +137,7 @@ async def get_claude_projects_importer_v2( # pragma: no cover
|
||||
file_service: FileServiceV2Dep,
|
||||
) -> ClaudeProjectsImporter:
|
||||
"""Create ClaudeProjectsImporter with v2 dependencies."""
|
||||
return ClaudeProjectsImporter(
|
||||
project_config.home,
|
||||
markdown_processor,
|
||||
file_service,
|
||||
project_name=project_config.name,
|
||||
)
|
||||
return ClaudeProjectsImporter(project_config.home, markdown_processor, file_service)
|
||||
|
||||
|
||||
ClaudeProjectsImporterV2Dep = Annotated[
|
||||
@@ -191,12 +151,7 @@ async def get_claude_projects_importer_v2_external(
|
||||
file_service: FileServiceV2ExternalDep,
|
||||
) -> ClaudeProjectsImporter:
|
||||
"""Create ClaudeProjectsImporter with v2 external_id dependencies."""
|
||||
return ClaudeProjectsImporter(
|
||||
project_config.home,
|
||||
markdown_processor,
|
||||
file_service,
|
||||
project_name=project_config.name,
|
||||
)
|
||||
return ClaudeProjectsImporter(project_config.home, markdown_processor, file_service)
|
||||
|
||||
|
||||
ClaudeProjectsImporterV2ExternalDep = Annotated[
|
||||
@@ -213,12 +168,7 @@ async def get_memory_json_importer(
|
||||
file_service: FileServiceDep,
|
||||
) -> MemoryJsonImporter:
|
||||
"""Create MemoryJsonImporter with dependencies."""
|
||||
return MemoryJsonImporter(
|
||||
project_config.home,
|
||||
markdown_processor,
|
||||
file_service,
|
||||
project_name=project_config.name,
|
||||
)
|
||||
return MemoryJsonImporter(project_config.home, markdown_processor, file_service)
|
||||
|
||||
|
||||
MemoryJsonImporterDep = Annotated[MemoryJsonImporter, Depends(get_memory_json_importer)]
|
||||
@@ -230,12 +180,7 @@ async def get_memory_json_importer_v2( # pragma: no cover
|
||||
file_service: FileServiceV2Dep,
|
||||
) -> MemoryJsonImporter:
|
||||
"""Create MemoryJsonImporter with v2 dependencies."""
|
||||
return MemoryJsonImporter(
|
||||
project_config.home,
|
||||
markdown_processor,
|
||||
file_service,
|
||||
project_name=project_config.name,
|
||||
)
|
||||
return MemoryJsonImporter(project_config.home, markdown_processor, file_service)
|
||||
|
||||
|
||||
MemoryJsonImporterV2Dep = Annotated[MemoryJsonImporter, Depends(get_memory_json_importer_v2)]
|
||||
@@ -247,12 +192,7 @@ async def get_memory_json_importer_v2_external(
|
||||
file_service: FileServiceV2ExternalDep,
|
||||
) -> MemoryJsonImporter:
|
||||
"""Create MemoryJsonImporter with v2 external_id dependencies."""
|
||||
return MemoryJsonImporter(
|
||||
project_config.home,
|
||||
markdown_processor,
|
||||
file_service,
|
||||
project_name=project_config.name,
|
||||
)
|
||||
return MemoryJsonImporter(project_config.home, markdown_processor, file_service)
|
||||
|
||||
|
||||
MemoryJsonImporterV2ExternalDep = Annotated[
|
||||
|
||||
@@ -13,7 +13,6 @@ from typing import Annotated
|
||||
|
||||
from fastapi import Depends
|
||||
|
||||
from basic_memory.deps.config import AppConfigDep
|
||||
from basic_memory.deps.db import SessionMakerDep
|
||||
from basic_memory.deps.projects import (
|
||||
ProjectIdDep,
|
||||
@@ -148,14 +147,13 @@ RelationRepositoryV2ExternalDep = Annotated[
|
||||
async def get_search_repository(
|
||||
session_maker: SessionMakerDep,
|
||||
project_id: ProjectIdDep,
|
||||
app_config: AppConfigDep,
|
||||
) -> SearchRepository:
|
||||
"""Create a backend-specific SearchRepository instance for the current project.
|
||||
|
||||
Uses factory function to return SQLiteSearchRepository or PostgresSearchRepository
|
||||
based on database backend configuration.
|
||||
"""
|
||||
return create_search_repository(session_maker, project_id=project_id, app_config=app_config)
|
||||
return create_search_repository(session_maker, project_id=project_id)
|
||||
|
||||
|
||||
SearchRepositoryDep = Annotated[SearchRepository, Depends(get_search_repository)]
|
||||
@@ -164,10 +162,9 @@ SearchRepositoryDep = Annotated[SearchRepository, Depends(get_search_repository)
|
||||
async def get_search_repository_v2( # pragma: no cover
|
||||
session_maker: SessionMakerDep,
|
||||
project_id: ProjectIdPathDep,
|
||||
app_config: AppConfigDep,
|
||||
) -> SearchRepository:
|
||||
"""Create a SearchRepository instance for v2 API."""
|
||||
return create_search_repository(session_maker, project_id=project_id, app_config=app_config)
|
||||
return create_search_repository(session_maker, project_id=project_id)
|
||||
|
||||
|
||||
SearchRepositoryV2Dep = Annotated[SearchRepository, Depends(get_search_repository_v2)]
|
||||
@@ -176,10 +173,9 @@ SearchRepositoryV2Dep = Annotated[SearchRepository, Depends(get_search_repositor
|
||||
async def get_search_repository_v2_external(
|
||||
session_maker: SessionMakerDep,
|
||||
project_id: ProjectExternalIdPathDep,
|
||||
app_config: AppConfigDep,
|
||||
) -> SearchRepository:
|
||||
"""Create a SearchRepository instance for v2 API (uses external_id)."""
|
||||
return create_search_repository(session_maker, project_id=project_id, app_config=app_config)
|
||||
return create_search_repository(session_maker, project_id=project_id)
|
||||
|
||||
|
||||
SearchRepositoryV2ExternalDep = Annotated[
|
||||
|
||||
@@ -470,12 +470,9 @@ async def get_task_scheduler(
|
||||
sync_service: SyncServiceV2ExternalDep,
|
||||
search_service: SearchServiceV2ExternalDep,
|
||||
project_config: ProjectConfigV2ExternalDep,
|
||||
app_config: AppConfigDep,
|
||||
) -> TaskScheduler:
|
||||
"""Create a scheduler that maps task specs to coroutines."""
|
||||
|
||||
scheduler: LocalTaskScheduler | None = None
|
||||
|
||||
async def _reindex_entity(
|
||||
entity_id: int,
|
||||
resolve_relations: bool = False,
|
||||
@@ -487,18 +484,10 @@ async def get_task_scheduler(
|
||||
# Outcome: updates unresolved relations pointing to this entity
|
||||
if resolve_relations:
|
||||
await sync_service.resolve_relations(entity_id=entity_id)
|
||||
# Trigger: semantic search enabled in local config.
|
||||
# Why: vector chunks are derived and should refresh after canonical reindex completes.
|
||||
# Outcome: schedules out-of-band vector sync without extending write latency.
|
||||
if app_config.semantic_search_enabled and scheduler is not None:
|
||||
scheduler.schedule("sync_entity_vectors", entity_id=entity_id)
|
||||
|
||||
async def _resolve_relations(entity_id: int, **_: Any) -> None:
|
||||
await sync_service.resolve_relations(entity_id=entity_id)
|
||||
|
||||
async def _sync_entity_vectors(entity_id: int, **_: Any) -> None:
|
||||
await search_service.sync_entity_vectors(entity_id)
|
||||
|
||||
async def _sync_project(force_full: bool = False, **_: Any) -> None:
|
||||
await sync_service.sync(
|
||||
project_config.home,
|
||||
@@ -509,16 +498,14 @@ async def get_task_scheduler(
|
||||
async def _reindex_project(**_: Any) -> None:
|
||||
await search_service.reindex_all()
|
||||
|
||||
scheduler = LocalTaskScheduler(
|
||||
return LocalTaskScheduler(
|
||||
{
|
||||
"reindex_entity": _reindex_entity,
|
||||
"resolve_relations": _resolve_relations,
|
||||
"sync_entity_vectors": _sync_entity_vectors,
|
||||
"sync_project": _sync_project,
|
||||
"reindex_project": _reindex_project,
|
||||
}
|
||||
)
|
||||
return scheduler
|
||||
|
||||
|
||||
TaskSchedulerDep = Annotated[TaskScheduler, Depends(get_task_scheduler)]
|
||||
|
||||
@@ -8,7 +8,6 @@ from typing import TYPE_CHECKING, Any, Optional, TypeVar
|
||||
from basic_memory.markdown.markdown_processor import MarkdownProcessor
|
||||
from basic_memory.markdown.schemas import EntityMarkdown
|
||||
from basic_memory.schemas.importer import ImportResult
|
||||
from basic_memory.utils import build_canonical_permalink, generate_permalink
|
||||
|
||||
if TYPE_CHECKING: # pragma: no cover
|
||||
from basic_memory.services.file_service import FileService
|
||||
@@ -30,7 +29,6 @@ class Importer[T: ImportResult]:
|
||||
base_path: Path,
|
||||
markdown_processor: MarkdownProcessor,
|
||||
file_service: "FileService",
|
||||
project_name: Optional[str] = None,
|
||||
):
|
||||
"""Initialize the import service.
|
||||
|
||||
@@ -42,8 +40,6 @@ class Importer[T: ImportResult]:
|
||||
self.base_path = base_path.resolve() # Get absolute path
|
||||
self.markdown_processor = markdown_processor
|
||||
self.file_service = file_service
|
||||
self.project_name = project_name
|
||||
self.project_permalink = generate_permalink(project_name) if project_name else None
|
||||
|
||||
@abstractmethod
|
||||
async def import_data(self, source_data, destination_folder: str, **kwargs: Any) -> T:
|
||||
@@ -77,26 +73,6 @@ class Importer[T: ImportResult]:
|
||||
# FileService.write_file handles directory creation and returns checksum
|
||||
return await self.file_service.write_file(file_path, content)
|
||||
|
||||
def canonical_permalink(self, path: str) -> str:
|
||||
"""Build a canonical permalink for imported content."""
|
||||
include_project = True
|
||||
# Trigger: importer has app config with permalink prefixing flag
|
||||
# Why: imported notes should align with canonical permalink format
|
||||
# Outcome: include project prefix when enabled
|
||||
if self.file_service.app_config is not None:
|
||||
include_project = self.file_service.app_config.permalinks_include_project
|
||||
|
||||
return build_canonical_permalink(
|
||||
self.project_permalink,
|
||||
path,
|
||||
include_project=include_project,
|
||||
)
|
||||
|
||||
def build_import_paths(self, path: str) -> tuple[str, str]:
|
||||
"""Return (permalink, file_path) for an imported entity."""
|
||||
permalink = self.canonical_permalink(path)
|
||||
return permalink, f"{path}.md"
|
||||
|
||||
async def ensure_folder_exists(self, folder: str) -> None:
|
||||
"""Ensure folder exists using FileService.
|
||||
|
||||
|
||||
@@ -51,20 +51,11 @@ class ChatGPTImporter(Importer[ChatImportResult]):
|
||||
chats_imported = 0
|
||||
|
||||
for chat in conversations:
|
||||
created_at = chat["create_time"]
|
||||
date_prefix = datetime.fromtimestamp(created_at).astimezone().strftime("%Y%m%d")
|
||||
clean_title = clean_filename(chat["title"])
|
||||
relative_path = (
|
||||
f"{destination_folder}/{date_prefix}-{clean_title}"
|
||||
if destination_folder
|
||||
else f"{date_prefix}-{clean_title}"
|
||||
)
|
||||
permalink, file_path = self.build_import_paths(relative_path)
|
||||
|
||||
# Convert to entity
|
||||
entity = self._format_chat_content(chat, permalink)
|
||||
entity = self._format_chat_content(destination_folder, chat)
|
||||
|
||||
# Write file using relative path - FileService handles base_path
|
||||
file_path = f"{entity.frontmatter.metadata['permalink']}.md"
|
||||
await self.write_entity(entity, file_path)
|
||||
|
||||
# Count messages
|
||||
@@ -92,7 +83,7 @@ class ChatGPTImporter(Importer[ChatImportResult]):
|
||||
return self.handle_error("Failed to import ChatGPT conversations", e)
|
||||
|
||||
def _format_chat_content(
|
||||
self, conversation: Dict[str, Any], permalink: str
|
||||
self, folder: str, conversation: Dict[str, Any]
|
||||
) -> EntityMarkdown: # pragma: no cover
|
||||
"""Convert chat conversation to Basic Memory entity.
|
||||
|
||||
@@ -114,6 +105,10 @@ class ChatGPTImporter(Importer[ChatImportResult]):
|
||||
root_id = node_id
|
||||
break
|
||||
|
||||
# Generate permalink
|
||||
date_prefix = datetime.fromtimestamp(created_at).astimezone().strftime("%Y%m%d")
|
||||
clean_title = clean_filename(conversation["title"])
|
||||
|
||||
# Format content
|
||||
content = self._format_chat_markdown(
|
||||
title=conversation["title"],
|
||||
@@ -131,7 +126,7 @@ class ChatGPTImporter(Importer[ChatImportResult]):
|
||||
"title": conversation["title"],
|
||||
"created": format_timestamp(created_at),
|
||||
"modified": format_timestamp(modified_at),
|
||||
"permalink": permalink,
|
||||
"permalink": f"{folder}/{date_prefix}-{clean_title}",
|
||||
}
|
||||
),
|
||||
content=content,
|
||||
|
||||
@@ -54,27 +54,18 @@ class ClaudeConversationsImporter(Importer[ChatImportResult]):
|
||||
for chat in conversations:
|
||||
# Get name, providing default for unnamed conversations
|
||||
chat_name = chat.get("name") or f"Conversation {chat.get('uuid', 'untitled')}"
|
||||
date_prefix = datetime.fromisoformat(chat["created_at"].replace("Z", "+00:00")).strftime(
|
||||
"%Y%m%d"
|
||||
)
|
||||
clean_title = clean_filename(chat_name)
|
||||
relative_path = (
|
||||
f"{destination_folder}/{date_prefix}-{clean_title}"
|
||||
if destination_folder
|
||||
else f"{date_prefix}-{clean_title}"
|
||||
)
|
||||
permalink, file_path = self.build_import_paths(relative_path)
|
||||
|
||||
# Convert to entity
|
||||
entity = self._format_chat_content(
|
||||
folder=destination_folder,
|
||||
name=chat_name,
|
||||
messages=chat["chat_messages"],
|
||||
created_at=chat["created_at"],
|
||||
modified_at=chat["updated_at"],
|
||||
permalink=permalink,
|
||||
)
|
||||
|
||||
# Write file using relative path - FileService handles base_path
|
||||
file_path = f"{entity.frontmatter.metadata['permalink']}.md"
|
||||
await self.write_entity(entity, file_path)
|
||||
|
||||
chats_imported += 1
|
||||
@@ -93,11 +84,11 @@ class ClaudeConversationsImporter(Importer[ChatImportResult]):
|
||||
|
||||
def _format_chat_content(
|
||||
self,
|
||||
folder: str,
|
||||
name: str,
|
||||
messages: List[Dict[str, Any]],
|
||||
created_at: str,
|
||||
modified_at: str,
|
||||
permalink: str,
|
||||
) -> EntityMarkdown:
|
||||
"""Convert chat messages to Basic Memory entity format.
|
||||
|
||||
@@ -111,6 +102,11 @@ class ClaudeConversationsImporter(Importer[ChatImportResult]):
|
||||
Returns:
|
||||
EntityMarkdown instance representing the conversation.
|
||||
"""
|
||||
# Generate permalink using folder name (relative path)
|
||||
date_prefix = datetime.fromisoformat(created_at.replace("Z", "+00:00")).strftime("%Y%m%d")
|
||||
clean_title = clean_filename(name)
|
||||
permalink = f"{folder}/{date_prefix}-{clean_title}"
|
||||
|
||||
# Format content
|
||||
content = self._format_chat_markdown(
|
||||
name=name,
|
||||
|
||||
@@ -63,28 +63,17 @@ class ClaudeProjectsImporter(Importer[ProjectImportResult]):
|
||||
await self.file_service.ensure_directory(docs_dir)
|
||||
|
||||
# Import prompt template if it exists
|
||||
if project.get("prompt_template"):
|
||||
prompt_path = (
|
||||
f"{destination_folder}/{project_dir}/prompt-template"
|
||||
if destination_folder
|
||||
else f"{project_dir}/prompt-template"
|
||||
)
|
||||
permalink, file_path = self.build_import_paths(prompt_path)
|
||||
prompt_entity = self._format_prompt_markdown(project, permalink)
|
||||
if prompt_entity:
|
||||
await self.write_entity(prompt_entity, file_path)
|
||||
if prompt_entity := self._format_prompt_markdown(project, destination_folder):
|
||||
# Write file using relative path - FileService handles base_path
|
||||
file_path = f"{prompt_entity.frontmatter.metadata['permalink']}.md"
|
||||
await self.write_entity(prompt_entity, file_path)
|
||||
prompts_imported += 1
|
||||
|
||||
# Import project documents
|
||||
for doc in project.get("docs", []):
|
||||
doc_file = clean_filename(doc["filename"])
|
||||
doc_path = (
|
||||
f"{destination_folder}/{project_dir}/docs/{doc_file}"
|
||||
if destination_folder
|
||||
else f"{project_dir}/docs/{doc_file}"
|
||||
)
|
||||
permalink, file_path = self.build_import_paths(doc_path)
|
||||
entity = self._format_project_markdown(project, doc, permalink)
|
||||
entity = self._format_project_markdown(project, doc, destination_folder)
|
||||
# Write file using relative path - FileService handles base_path
|
||||
file_path = f"{entity.frontmatter.metadata['permalink']}.md"
|
||||
await self.write_entity(entity, file_path)
|
||||
docs_imported += 1
|
||||
|
||||
@@ -100,7 +89,7 @@ class ClaudeProjectsImporter(Importer[ProjectImportResult]):
|
||||
return self.handle_error("Failed to import Claude projects", e)
|
||||
|
||||
def _format_project_markdown(
|
||||
self, project: Dict[str, Any], doc: Dict[str, Any], permalink: str
|
||||
self, project: Dict[str, Any], doc: Dict[str, Any], destination_folder: str = ""
|
||||
) -> EntityMarkdown:
|
||||
"""Format a project document as a Basic Memory entity.
|
||||
|
||||
@@ -116,6 +105,17 @@ class ClaudeProjectsImporter(Importer[ProjectImportResult]):
|
||||
created_at = doc.get("created_at") or project["created_at"]
|
||||
modified_at = project["updated_at"]
|
||||
|
||||
# Generate clean names for organization
|
||||
project_dir = clean_filename(project["name"])
|
||||
doc_file = clean_filename(doc["filename"])
|
||||
|
||||
# Build permalink with optional destination folder prefix
|
||||
permalink = (
|
||||
f"{destination_folder}/{project_dir}/docs/{doc_file}"
|
||||
if destination_folder
|
||||
else f"{project_dir}/docs/{doc_file}"
|
||||
)
|
||||
|
||||
# Create entity
|
||||
entity = EntityMarkdown(
|
||||
frontmatter=EntityFrontmatter(
|
||||
@@ -136,7 +136,7 @@ class ClaudeProjectsImporter(Importer[ProjectImportResult]):
|
||||
return entity
|
||||
|
||||
def _format_prompt_markdown(
|
||||
self, project: Dict[str, Any], permalink: str
|
||||
self, project: Dict[str, Any], destination_folder: str = ""
|
||||
) -> Optional[EntityMarkdown]:
|
||||
"""Format project prompt template as a Basic Memory entity.
|
||||
|
||||
@@ -155,6 +155,16 @@ class ClaudeProjectsImporter(Importer[ProjectImportResult]):
|
||||
created_at = project["created_at"]
|
||||
modified_at = project["updated_at"]
|
||||
|
||||
# Generate clean project directory name
|
||||
project_dir = clean_filename(project["name"])
|
||||
|
||||
# Build permalink with optional destination folder prefix
|
||||
permalink = (
|
||||
f"{destination_folder}/{project_dir}/prompt-template"
|
||||
if destination_folder
|
||||
else f"{project_dir}/prompt-template"
|
||||
)
|
||||
|
||||
# Create entity
|
||||
entity = EntityMarkdown(
|
||||
frontmatter=EntityFrontmatter(
|
||||
|
||||
@@ -80,12 +80,11 @@ class MemoryJsonImporter(Importer[EntityImportResult]):
|
||||
entity_type = entity_data.get("entityType") or entity_data.get("type") or "entity"
|
||||
|
||||
# Build permalink with optional destination folder prefix
|
||||
relative_path = (
|
||||
permalink = (
|
||||
f"{destination_folder}/{entity_type}/{name}"
|
||||
if destination_folder
|
||||
else f"{entity_type}/{name}"
|
||||
)
|
||||
permalink, file_path = self.build_import_paths(relative_path)
|
||||
|
||||
# Ensure entity type directory exists using FileService with relative path
|
||||
entity_type_dir = (
|
||||
@@ -110,6 +109,7 @@ class MemoryJsonImporter(Importer[EntityImportResult]):
|
||||
)
|
||||
|
||||
# Write file using relative path - FileService handles base_path
|
||||
file_path = f"{entity.frontmatter.metadata['permalink']}.md"
|
||||
await self.write_entity(entity, file_path)
|
||||
entities_created += 1
|
||||
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
"""Markdown-it plugins for Basic Memory markdown parsing."""
|
||||
|
||||
from typing import List, Any, Dict
|
||||
|
||||
from basic_memory.utils import normalize_project_reference
|
||||
from markdown_it import MarkdownIt
|
||||
from markdown_it.token import Token
|
||||
|
||||
@@ -116,7 +114,7 @@ def parse_relation(token: Token) -> Dict[str, Any] | None:
|
||||
rel_type = before
|
||||
|
||||
# Get target
|
||||
target = normalize_project_reference(content[start + 2 : end].strip())
|
||||
target = content[start + 2 : end].strip()
|
||||
|
||||
# Look for context after
|
||||
after = content[end + 2 :].strip()
|
||||
@@ -162,7 +160,7 @@ def parse_inline_relations(content: str) -> List[Dict[str, Any]]:
|
||||
# No matching ]] found
|
||||
break
|
||||
|
||||
target = normalize_project_reference(content[start + 2 : end].strip())
|
||||
target = content[start + 2 : end].strip()
|
||||
if target:
|
||||
relations.append({"type": "links_to", "target": target, "context": None})
|
||||
|
||||
|
||||
@@ -6,7 +6,7 @@ from httpx import ASGITransport, AsyncClient, Timeout
|
||||
from loguru import logger
|
||||
|
||||
from basic_memory.api.app import app as fastapi_app
|
||||
from basic_memory.config import ConfigManager, ProjectMode
|
||||
from basic_memory.config import ConfigManager
|
||||
|
||||
|
||||
def _force_local_mode() -> bool:
|
||||
@@ -45,54 +45,31 @@ def set_client_factory(factory: Callable[[], AbstractAsyncContextManager[AsyncCl
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def get_client(
|
||||
project_name: Optional[str] = None,
|
||||
) -> AsyncIterator[AsyncClient]:
|
||||
async def get_client() -> AsyncIterator[AsyncClient]:
|
||||
"""Get an AsyncClient as a context manager.
|
||||
|
||||
This function provides proper resource management for HTTP clients,
|
||||
ensuring connections are closed after use. Routing priority:
|
||||
ensuring connections are closed after use. It supports three modes:
|
||||
|
||||
1. **Factory injection** (cloud app, tests):
|
||||
If a custom factory is set via set_client_factory(), use that.
|
||||
|
||||
2. **Per-project cloud mode** (project_name provided):
|
||||
If the project's mode is CLOUD, routes to cloud using API key or
|
||||
OAuth token. Honored even when FORCE_LOCAL is set, because the user
|
||||
explicitly declared this project as cloud.
|
||||
2. **CLI cloud mode**:
|
||||
When cloud_mode_enabled is True, create HTTP client with auth
|
||||
token from CLIAuth for requests to cloud proxy endpoint.
|
||||
|
||||
3. **Per-project local mode** (project_name provided):
|
||||
If the project's mode is LOCAL (or unspecified, default LOCAL), route
|
||||
to local ASGI transport. This allows mixed local/cloud routing even when
|
||||
global cloud mode is enabled.
|
||||
|
||||
4. **Force-local** (BASIC_MEMORY_FORCE_LOCAL env var):
|
||||
Routes to local ASGI transport, ignoring global cloud settings.
|
||||
|
||||
5. **Global cloud mode** (deprecated fallback):
|
||||
When cloud_mode_enabled is True, uses OAuth JWT token.
|
||||
|
||||
6. **Local mode** (default):
|
||||
3. **Local mode** (default):
|
||||
Use ASGI transport for in-process requests to local FastAPI app.
|
||||
|
||||
Args:
|
||||
project_name: Optional project name for per-project routing.
|
||||
If provided and the project's mode is CLOUD, routes to cloud
|
||||
using the API key or OAuth token.
|
||||
|
||||
Usage:
|
||||
async with get_client() as client:
|
||||
response = await client.get("/path")
|
||||
|
||||
# Per-project routing
|
||||
async with get_client(project_name="research") as client:
|
||||
response = await client.get("/path")
|
||||
|
||||
Yields:
|
||||
AsyncClient: Configured HTTP client for the current mode
|
||||
|
||||
Raises:
|
||||
RuntimeError: If cloud routing needed but no API key / not authenticated
|
||||
RuntimeError: If cloud mode is enabled but user is not authenticated
|
||||
"""
|
||||
if _client_factory:
|
||||
# Use injected factory (cloud app, tests)
|
||||
@@ -108,61 +85,18 @@ async def get_client(
|
||||
pool=30.0, # 30 seconds for connection pool
|
||||
)
|
||||
|
||||
# Trigger: project has per-project cloud mode set
|
||||
# Why: per-project CLOUD is an explicit user declaration that should be
|
||||
# honored even from the MCP server (which sets FORCE_LOCAL)
|
||||
# Outcome: HTTP client with API key or OAuth auth to cloud proxy
|
||||
if project_name and config.get_project_mode(project_name) == ProjectMode.CLOUD:
|
||||
# Try API key first (explicit, no network)
|
||||
token = config.cloud_api_key
|
||||
if not token:
|
||||
# Fall back to OAuth session (may refresh token)
|
||||
from basic_memory.cli.auth import CLIAuth
|
||||
|
||||
auth = CLIAuth(client_id=config.cloud_client_id, authkit_domain=config.cloud_domain)
|
||||
token = await auth.get_valid_token()
|
||||
|
||||
if not token:
|
||||
raise RuntimeError(
|
||||
f"Project '{project_name}' is set to cloud mode but no credentials found. "
|
||||
"Run 'bm cloud set-key <key>' or 'bm cloud login' first."
|
||||
)
|
||||
|
||||
proxy_base_url = f"{config.cloud_host}/proxy"
|
||||
logger.info(
|
||||
f"Creating HTTP client for cloud project '{project_name}' at: {proxy_base_url}"
|
||||
)
|
||||
async with AsyncClient(
|
||||
base_url=proxy_base_url,
|
||||
headers={"Authorization": f"Bearer {token}"},
|
||||
timeout=timeout,
|
||||
) as client:
|
||||
yield client
|
||||
|
||||
# Trigger: project is not explicitly cloud (LOCAL is the default)
|
||||
# Why: project-scoped routing should honor local mode even when global
|
||||
# cloud mode is enabled for backward compatibility
|
||||
# Outcome: uses ASGI transport for in-process local API calls
|
||||
elif project_name and config.get_project_mode(project_name) == ProjectMode.LOCAL:
|
||||
logger.info(f"Project '{project_name}' is set to local mode - using ASGI transport")
|
||||
async with AsyncClient(
|
||||
transport=ASGITransport(app=fastapi_app), base_url="http://test", timeout=timeout
|
||||
) as client:
|
||||
yield client
|
||||
|
||||
# Trigger: BASIC_MEMORY_FORCE_LOCAL env var is set
|
||||
# Why: allows local MCP server and CLI commands to route locally
|
||||
# even when cloud_mode_enabled is True
|
||||
# Outcome: uses ASGI transport for in-process local API calls
|
||||
elif _force_local_mode():
|
||||
if _force_local_mode():
|
||||
logger.info("Force local mode enabled - using ASGI client for local Basic Memory API")
|
||||
async with AsyncClient(
|
||||
transport=ASGITransport(app=fastapi_app), base_url="http://test", timeout=timeout
|
||||
) as client:
|
||||
yield client
|
||||
|
||||
elif config.cloud_mode_enabled:
|
||||
# Global cloud mode (deprecated fallback): inject OAuth auth when creating client
|
||||
# CLI cloud mode: inject auth when creating client
|
||||
from basic_memory.cli.auth import CLIAuth
|
||||
|
||||
auth = CLIAuth(client_id=config.cloud_client_id, authkit_domain=config.cloud_domain)
|
||||
|
||||
@@ -17,7 +17,6 @@ from basic_memory.mcp.clients.memory import MemoryClient
|
||||
from basic_memory.mcp.clients.directory import DirectoryClient
|
||||
from basic_memory.mcp.clients.resource import ResourceClient
|
||||
from basic_memory.mcp.clients.project import ProjectClient
|
||||
from basic_memory.mcp.clients.schema import SchemaClient
|
||||
|
||||
__all__ = [
|
||||
"KnowledgeClient",
|
||||
@@ -26,5 +25,4 @@ __all__ = [
|
||||
"DirectoryClient",
|
||||
"ResourceClient",
|
||||
"ProjectClient",
|
||||
"SchemaClient",
|
||||
]
|
||||
|
||||
@@ -1,113 +0,0 @@
|
||||
"""Typed client for schema API operations.
|
||||
|
||||
Encapsulates all /v2/projects/{project_id}/schema/* endpoints.
|
||||
"""
|
||||
|
||||
from httpx import AsyncClient
|
||||
|
||||
from basic_memory.mcp.tools.utils import call_post, call_get
|
||||
from basic_memory.schemas.schema import (
|
||||
ValidationReport,
|
||||
InferenceReport,
|
||||
DriftReport,
|
||||
)
|
||||
|
||||
|
||||
class SchemaClient:
|
||||
"""Typed client for schema operations.
|
||||
|
||||
Centralizes:
|
||||
- API path construction for /v2/projects/{project_id}/schema/*
|
||||
- Response validation via Pydantic models
|
||||
- Consistent error handling through call_* utilities
|
||||
|
||||
Usage:
|
||||
async with get_client() as http_client:
|
||||
client = SchemaClient(http_client, project_id)
|
||||
report = await client.validate(entity_type="Person")
|
||||
"""
|
||||
|
||||
def __init__(self, http_client: AsyncClient, project_id: str):
|
||||
"""Initialize the schema client.
|
||||
|
||||
Args:
|
||||
http_client: HTTPX AsyncClient for making requests
|
||||
project_id: Project external_id (UUID) for API calls
|
||||
"""
|
||||
self.http_client = http_client
|
||||
self.project_id = project_id
|
||||
self._base_path = f"/v2/projects/{project_id}/schema"
|
||||
|
||||
async def validate(
|
||||
self,
|
||||
*,
|
||||
entity_type: str | None = None,
|
||||
identifier: str | None = None,
|
||||
) -> ValidationReport:
|
||||
"""Validate notes against their resolved schemas.
|
||||
|
||||
Args:
|
||||
entity_type: Optional entity type to batch-validate
|
||||
identifier: Optional specific note to validate
|
||||
|
||||
Returns:
|
||||
ValidationReport with per-note results
|
||||
|
||||
Raises:
|
||||
ToolError: If the request fails
|
||||
"""
|
||||
params: dict[str, str] = {}
|
||||
if entity_type:
|
||||
params["entity_type"] = entity_type
|
||||
if identifier:
|
||||
params["identifier"] = identifier
|
||||
|
||||
response = await call_post(
|
||||
self.http_client,
|
||||
f"{self._base_path}/validate",
|
||||
params=params,
|
||||
)
|
||||
return ValidationReport.model_validate(response.json())
|
||||
|
||||
async def infer(
|
||||
self,
|
||||
entity_type: str,
|
||||
*,
|
||||
threshold: float = 0.25,
|
||||
) -> InferenceReport:
|
||||
"""Infer a schema from existing notes of a given type.
|
||||
|
||||
Args:
|
||||
entity_type: The entity type to analyze
|
||||
threshold: Minimum frequency for optional fields (0-1)
|
||||
|
||||
Returns:
|
||||
InferenceReport with frequency data and suggested schema
|
||||
|
||||
Raises:
|
||||
ToolError: If the request fails
|
||||
"""
|
||||
response = await call_post(
|
||||
self.http_client,
|
||||
f"{self._base_path}/infer",
|
||||
params={"entity_type": entity_type, "threshold": threshold},
|
||||
)
|
||||
return InferenceReport.model_validate(response.json())
|
||||
|
||||
async def diff(self, entity_type: str) -> DriftReport:
|
||||
"""Show drift between schema definition and actual usage.
|
||||
|
||||
Args:
|
||||
entity_type: The entity type to check for drift
|
||||
|
||||
Returns:
|
||||
DriftReport with detected differences
|
||||
|
||||
Raises:
|
||||
ToolError: If the request fails
|
||||
"""
|
||||
response = await call_get(
|
||||
self.http_client,
|
||||
f"{self._base_path}/diff/{entity_type}",
|
||||
)
|
||||
return DriftReport.model_validate(response.json())
|
||||
@@ -1,160 +0,0 @@
|
||||
"""Formatting helpers for MCP tool outputs."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Iterable, Sequence
|
||||
|
||||
from basic_memory.schemas.search import SearchResponse, SearchResult
|
||||
|
||||
ANSI_RESET = "\x1b[0m"
|
||||
ANSI_BOLD = "\x1b[1m"
|
||||
ANSI_DIM = "\x1b[2m"
|
||||
ANSI_CYAN = "\x1b[36m"
|
||||
|
||||
|
||||
def _apply_style(text: str, style: str, enabled: bool) -> str:
|
||||
if not enabled:
|
||||
return text
|
||||
return f"{style}{text}{ANSI_RESET}"
|
||||
|
||||
|
||||
def _strip_frontmatter(text: str) -> str:
|
||||
lines = text.splitlines()
|
||||
if not lines or lines[0].strip() != "---":
|
||||
return text
|
||||
|
||||
for idx in range(1, len(lines)):
|
||||
if lines[idx].strip() == "---":
|
||||
return "\n".join(lines[idx + 1 :]).lstrip()
|
||||
return text
|
||||
|
||||
|
||||
def _parse_title(text: str) -> str | None:
|
||||
for line in text.splitlines():
|
||||
if line.startswith("# "):
|
||||
return line[2:].strip()
|
||||
return None
|
||||
|
||||
|
||||
def _truncate(text: str, width: int) -> str:
|
||||
if width <= 0:
|
||||
return ""
|
||||
if len(text) <= width:
|
||||
return text
|
||||
if width <= 3:
|
||||
return text[:width]
|
||||
return text[: width - 3] + "..."
|
||||
|
||||
|
||||
def _make_separator(widths: Sequence[int]) -> str:
|
||||
return "+" + "+".join("-" * (width + 2) for width in widths) + "+"
|
||||
|
||||
|
||||
def _format_row(values: Sequence[str], widths: Sequence[int]) -> str:
|
||||
cells = []
|
||||
for value, width in zip(values, widths, strict=True):
|
||||
cells.append(f" {_truncate(value, width).ljust(width)} ")
|
||||
return "|" + "|".join(cells) + "|"
|
||||
|
||||
|
||||
def _get_result_tags(result: SearchResult) -> str:
|
||||
metadata = result.metadata or {}
|
||||
if isinstance(metadata, dict):
|
||||
tags = metadata.get("tags")
|
||||
if isinstance(tags, list):
|
||||
return ", ".join(str(tag) for tag in tags if tag)
|
||||
return ""
|
||||
|
||||
|
||||
def _get_result_path(result: SearchResult) -> str:
|
||||
return result.permalink or result.file_path or ""
|
||||
|
||||
|
||||
def format_search_results_ascii(
|
||||
result: SearchResponse,
|
||||
query: str | None = None,
|
||||
color: bool = False,
|
||||
) -> str:
|
||||
"""Format search results as an ASCII table for TUI clients."""
|
||||
|
||||
results = result.results or []
|
||||
header_line = _apply_style("Search results", f"{ANSI_BOLD}{ANSI_CYAN}", color)
|
||||
lines = [header_line]
|
||||
|
||||
if query:
|
||||
lines.append(f"Query: {query}")
|
||||
|
||||
summary = f"Results: {len(results)} | Page: {result.current_page} | Page size: {result.page_size}"
|
||||
lines.append(_apply_style(summary, ANSI_DIM, color))
|
||||
|
||||
if not results:
|
||||
lines.append("No results.")
|
||||
return "\n".join(lines).strip()
|
||||
|
||||
headers = ["#", "Title", "Type", "Score", "Path", "Tags"]
|
||||
rows = []
|
||||
for idx, item in enumerate(results, start=1):
|
||||
rows.append(
|
||||
[
|
||||
str(idx),
|
||||
item.title or "Untitled",
|
||||
item.type.value if hasattr(item.type, "value") else str(item.type),
|
||||
f"{item.score:.2f}" if isinstance(item.score, (int, float)) else "",
|
||||
_get_result_path(item),
|
||||
_get_result_tags(item),
|
||||
]
|
||||
)
|
||||
|
||||
max_widths = [3, 32, 10, 7, 36, 24]
|
||||
widths = []
|
||||
for index, header in enumerate(headers):
|
||||
column_values = [header] + [row[index] for row in rows]
|
||||
max_len = max(len(value) for value in column_values)
|
||||
widths.append(min(max_widths[index], max_len))
|
||||
|
||||
table = [_make_separator(widths)]
|
||||
header_row = _format_row(headers, widths)
|
||||
if color:
|
||||
header_cells = []
|
||||
for value, width in zip(headers, widths, strict=True):
|
||||
padded = f" {_truncate(value, width).ljust(width)} "
|
||||
header_cells.append(_apply_style(padded, f"{ANSI_BOLD}{ANSI_CYAN}", color))
|
||||
header_row = "|" + "|".join(header_cells) + "|"
|
||||
table.append(header_row)
|
||||
table.append(_make_separator(widths))
|
||||
|
||||
for row in rows:
|
||||
table.append(_format_row(row, widths))
|
||||
|
||||
table.append(_make_separator(widths))
|
||||
|
||||
lines.append("")
|
||||
lines.extend(table)
|
||||
return "\n".join(lines).rstrip()
|
||||
|
||||
|
||||
def format_note_preview_ascii(
|
||||
content: str,
|
||||
identifier: str | None = None,
|
||||
color: bool = False,
|
||||
) -> str:
|
||||
"""Format note content for ASCII/TUI display."""
|
||||
|
||||
identifier = identifier or ""
|
||||
cleaned = _strip_frontmatter(content)
|
||||
title = _parse_title(cleaned) or identifier or "Note Preview"
|
||||
|
||||
header = _apply_style("Note preview", f"{ANSI_BOLD}{ANSI_CYAN}", color)
|
||||
lines = [header, f"Title: {title}"]
|
||||
|
||||
if identifier:
|
||||
lines.append(f"Identifier: {identifier}")
|
||||
|
||||
lines.append(_apply_style("-" * 72, ANSI_DIM, color))
|
||||
|
||||
if content.strip():
|
||||
lines.append(content.rstrip())
|
||||
else:
|
||||
lines.append("(empty note)")
|
||||
|
||||
return "\n".join(lines).rstrip()
|
||||
@@ -8,23 +8,18 @@ The resolve_project_parameter function is a thin wrapper for backwards
|
||||
compatibility with existing MCP tools.
|
||||
"""
|
||||
|
||||
from contextlib import asynccontextmanager
|
||||
from typing import AsyncIterator, Optional, List, Tuple
|
||||
|
||||
from typing import Optional, List
|
||||
from httpx import AsyncClient
|
||||
from httpx._types import (
|
||||
HeaderTypes,
|
||||
)
|
||||
from loguru import logger
|
||||
from fastmcp import Context
|
||||
from mcp.server.fastmcp.exceptions import ToolError
|
||||
|
||||
from basic_memory.config import ConfigManager
|
||||
from basic_memory.project_resolver import ProjectResolver
|
||||
from basic_memory.schemas.project_info import ProjectItem, ProjectList
|
||||
from basic_memory.schemas.v2 import ProjectResolveResponse
|
||||
from basic_memory.schemas.memory import memory_url_path
|
||||
from basic_memory.utils import generate_permalink, normalize_project_reference
|
||||
|
||||
|
||||
async def resolve_project_parameter(
|
||||
@@ -34,17 +29,19 @@ async def resolve_project_parameter(
|
||||
default_project_mode: Optional[bool] = None,
|
||||
default_project: Optional[str] = None,
|
||||
) -> Optional[str]:
|
||||
"""Resolve project parameter using unified linear priority chain.
|
||||
"""Resolve project parameter using three-tier hierarchy.
|
||||
|
||||
This is a thin wrapper around ProjectResolver for backwards compatibility.
|
||||
New code should consider using ProjectResolver directly for more detailed
|
||||
resolution information.
|
||||
|
||||
Resolution order (same for local and cloud modes):
|
||||
1. ENV_CONSTRAINT: BASIC_MEMORY_MCP_PROJECT env var (highest priority)
|
||||
2. EXPLICIT: project parameter passed directly
|
||||
3. DEFAULT: default project when default_project_mode=true
|
||||
4. Fallback: cloud → CLOUD_DISCOVERY or ValueError; local → NONE
|
||||
if cloud_mode:
|
||||
project is required (unless allow_discovery=True for tools that support discovery mode)
|
||||
else:
|
||||
Resolution order:
|
||||
1. Single Project Mode (--project cli arg, or BASIC_MEMORY_MCP_PROJECT env var) - highest priority
|
||||
2. Explicit project parameter - medium priority
|
||||
3. Default project if default_project_mode=true - lowest priority
|
||||
|
||||
Args:
|
||||
project: Optional explicit project parameter
|
||||
@@ -153,98 +150,6 @@ async def get_active_project(
|
||||
return active_project
|
||||
|
||||
|
||||
def _split_project_prefix(path: str) -> tuple[Optional[str], str]:
|
||||
"""Split a possible project prefix from a memory URL path."""
|
||||
if "/" not in path:
|
||||
return None, path
|
||||
|
||||
project_prefix, remainder = path.split("/", 1)
|
||||
if not project_prefix or not remainder:
|
||||
return None, path
|
||||
|
||||
if "*" in project_prefix:
|
||||
return None, path
|
||||
|
||||
return project_prefix, remainder
|
||||
|
||||
|
||||
async def resolve_project_and_path(
|
||||
client: AsyncClient,
|
||||
identifier: str,
|
||||
project: Optional[str] = None,
|
||||
context: Optional[Context] = None,
|
||||
headers: HeaderTypes | None = None,
|
||||
) -> tuple[ProjectItem, str, bool]:
|
||||
"""Resolve project and normalized path for memory:// identifiers.
|
||||
|
||||
Returns:
|
||||
Tuple of (active_project, normalized_path, is_memory_url)
|
||||
"""
|
||||
is_memory_url = identifier.strip().startswith("memory://")
|
||||
if not is_memory_url:
|
||||
active_project = await get_active_project(client, project, context, headers)
|
||||
return active_project, identifier, False
|
||||
|
||||
normalized_path = normalize_project_reference(memory_url_path(identifier))
|
||||
project_prefix, remainder = _split_project_prefix(normalized_path)
|
||||
include_project = ConfigManager().config.permalinks_include_project
|
||||
|
||||
# Trigger: memory URL begins with a potential project segment
|
||||
# Why: allow project-scoped memory URLs without requiring a separate project parameter
|
||||
# Outcome: attempt to resolve the prefix as a project and route to it
|
||||
if project_prefix:
|
||||
try:
|
||||
from basic_memory.mcp.tools.utils import call_post
|
||||
|
||||
response = await call_post(
|
||||
client,
|
||||
"/v2/projects/resolve",
|
||||
json={"identifier": project_prefix},
|
||||
headers=headers,
|
||||
)
|
||||
resolved = ProjectResolveResponse.model_validate(response.json())
|
||||
except ToolError as exc:
|
||||
if "project not found" not in str(exc).lower():
|
||||
raise
|
||||
else:
|
||||
resolved_project = await resolve_project_parameter(project_prefix)
|
||||
if resolved_project and generate_permalink(resolved_project) != generate_permalink(
|
||||
project_prefix
|
||||
):
|
||||
raise ValueError(
|
||||
f"Project is constrained to '{resolved_project}', cannot use '{project_prefix}'."
|
||||
)
|
||||
|
||||
active_project = ProjectItem(
|
||||
id=resolved.project_id,
|
||||
external_id=resolved.external_id,
|
||||
name=resolved.name,
|
||||
path=resolved.path,
|
||||
is_default=resolved.is_default,
|
||||
)
|
||||
if context:
|
||||
context.set_state("active_project", active_project)
|
||||
|
||||
resolved_path = (
|
||||
f"{resolved.permalink}/{remainder}" if include_project else remainder
|
||||
)
|
||||
return active_project, resolved_path, True
|
||||
|
||||
# Trigger: no resolvable project prefix in the memory URL
|
||||
# Why: preserve existing memory URL behavior within the active project
|
||||
# Outcome: use the active project and normalize the path for lookup
|
||||
active_project = await get_active_project(client, project, context, headers)
|
||||
resolved_path = normalized_path
|
||||
if include_project:
|
||||
# Trigger: project-prefixed permalinks are enabled and the path lacks a prefix
|
||||
# Why: ensure memory URL lookups align with canonical permalinks
|
||||
# Outcome: prefix the path with the active project's permalink
|
||||
project_prefix = active_project.permalink
|
||||
if resolved_path != project_prefix and not resolved_path.startswith(f"{project_prefix}/"):
|
||||
resolved_path = f"{project_prefix}/{resolved_path}"
|
||||
return active_project, resolved_path, True
|
||||
|
||||
|
||||
def add_project_metadata(result: str, project_name: str) -> str:
|
||||
"""Add project context as metadata footer for assistant session tracking.
|
||||
|
||||
@@ -259,48 +164,3 @@ def add_project_metadata(result: str, project_name: str) -> str:
|
||||
Result with project session tracking metadata
|
||||
"""
|
||||
return f"{result}\n\n[Session: Using project '{project_name}']"
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def get_project_client(
|
||||
project: Optional[str] = None,
|
||||
context: Optional[Context] = None,
|
||||
) -> AsyncIterator[Tuple[AsyncClient, ProjectItem]]:
|
||||
"""Resolve project, create correctly-routed client, and validate project.
|
||||
|
||||
Solves the bootstrap problem: we need to know the project name to choose
|
||||
the right client (local vs cloud), but we need the client to validate
|
||||
the project. This helper resolves the project from config first (no
|
||||
network), creates the correctly-routed client, then validates via API.
|
||||
|
||||
Args:
|
||||
project: Optional explicit project parameter
|
||||
context: Optional FastMCP context for caching
|
||||
|
||||
Yields:
|
||||
Tuple of (client, active_project)
|
||||
|
||||
Raises:
|
||||
ValueError: If no project can be resolved
|
||||
RuntimeError: If cloud project but no API key configured
|
||||
"""
|
||||
# Deferred import to avoid circular dependency
|
||||
from basic_memory.mcp.async_client import get_client
|
||||
|
||||
# Step 1: Resolve project name from config (no network call)
|
||||
resolved_project = await resolve_project_parameter(project)
|
||||
if not resolved_project:
|
||||
# Fall back to local client to discover projects and raise helpful error
|
||||
async with get_client() as client:
|
||||
project_names = await get_project_names(client)
|
||||
raise ValueError(
|
||||
"No project specified. "
|
||||
"Either set 'default_project_mode=true' in config, or use 'project' argument.\n"
|
||||
f"Available projects: {project_names}"
|
||||
)
|
||||
|
||||
# Step 2: Create client routed based on project's mode
|
||||
async with get_client(project_name=resolved_project) as client:
|
||||
# Step 3: Validate project exists via API
|
||||
active_project = await get_active_project(client, resolved_project, context)
|
||||
yield client, active_project
|
||||
|
||||
@@ -32,7 +32,7 @@ def ai_assistant_guide() -> str:
|
||||
|
||||
# Add mode-specific header
|
||||
mode_info = ""
|
||||
if config.default_project_mode:
|
||||
if config.default_project_mode: # pragma: no cover
|
||||
mode_info = f"""
|
||||
# 🎯 Default Project Mode Active
|
||||
|
||||
|
||||
@@ -1,25 +0,0 @@
|
||||
"""MCP resources for Basic Memory."""
|
||||
|
||||
from basic_memory.mcp.resources.project_info import project_info
|
||||
from basic_memory.mcp.resources.ui import (
|
||||
note_preview_ui,
|
||||
note_preview_ui_mcp_ui,
|
||||
note_preview_ui_tool_ui,
|
||||
note_preview_ui_vanilla,
|
||||
search_results_ui,
|
||||
search_results_ui_mcp_ui,
|
||||
search_results_ui_tool_ui,
|
||||
search_results_ui_vanilla,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"project_info",
|
||||
"note_preview_ui",
|
||||
"note_preview_ui_mcp_ui",
|
||||
"note_preview_ui_tool_ui",
|
||||
"note_preview_ui_vanilla",
|
||||
"search_results_ui",
|
||||
"search_results_ui_mcp_ui",
|
||||
"search_results_ui_tool_ui",
|
||||
"search_results_ui_vanilla",
|
||||
]
|
||||
@@ -43,7 +43,7 @@ await write_note("Note", "Content", "folder")
|
||||
await write_note("Note", "Content", "folder", project="main")
|
||||
```
|
||||
|
||||
When `default_project_mode=false`:
|
||||
When `default_project_mode=false` (default):
|
||||
```python
|
||||
# Project required:
|
||||
await write_note("Note", "Content", "folder", project="main") # ✓
|
||||
|
||||
@@ -1,16 +0,0 @@
|
||||
# Basic Memory Cloud (optional)
|
||||
|
||||
Basic Memory Cloud is an optional add-on for users who want hosted access and sync.
|
||||
|
||||
- Hosted access to your knowledge
|
||||
- Cross-device sync
|
||||
- Mobile and web access
|
||||
- Multi-client workflows (Claude, ChatGPT, Gemini, and others)
|
||||
|
||||
OSS discount: `{{OSS_DISCOUNT_CODE}}` (20% off for 3 months)
|
||||
|
||||
Get started:
|
||||
|
||||
```bash
|
||||
bm cloud login
|
||||
```
|
||||
@@ -1,16 +0,0 @@
|
||||
# Release Notes
|
||||
|
||||
## 2026-02-06
|
||||
|
||||
- Added optional cloud discovery copy to CLI first-run and promo-version notices.
|
||||
- Added MCP tools for opt-in cloud discovery: `cloud_info` and `release_notes`.
|
||||
- Updated docs and README copy to keep cloud messaging explicit and optional.
|
||||
|
||||
Cloud remains optional for open-source users.
|
||||
OSS discount: `{{OSS_DISCOUNT_CODE}}` (20% off for 3 months)
|
||||
|
||||
Get started:
|
||||
|
||||
```bash
|
||||
bm cloud login
|
||||
```
|
||||
@@ -1,89 +0,0 @@
|
||||
"""UI resources for MCP Apps integration."""
|
||||
|
||||
from basic_memory.mcp.server import mcp
|
||||
from basic_memory.mcp.ui import load_html, load_variant_html
|
||||
|
||||
# FastMCP's MIME type validator currently accepts only type/subtype, so we
|
||||
# use text/html here. MCP Apps hosts typically expect text/html;profile=mcp-app.
|
||||
MIME_TYPE = "text/html"
|
||||
|
||||
|
||||
@mcp.resource(
|
||||
uri="ui://basic-memory/search-results",
|
||||
name="Basic Memory Search Results",
|
||||
description="Search results UI for Basic Memory tools.",
|
||||
mime_type=MIME_TYPE,
|
||||
)
|
||||
def search_results_ui() -> str:
|
||||
return load_variant_html("search-results")
|
||||
|
||||
|
||||
@mcp.resource(
|
||||
uri="ui://basic-memory/note-preview",
|
||||
name="Basic Memory Note Preview",
|
||||
description="Note preview UI for Basic Memory read_note tool.",
|
||||
mime_type=MIME_TYPE,
|
||||
)
|
||||
def note_preview_ui() -> str:
|
||||
return load_variant_html("note-preview")
|
||||
|
||||
|
||||
# Variant-specific resource URIs for bakeoff comparisons.
|
||||
@mcp.resource(
|
||||
uri="ui://basic-memory/search-results/vanilla",
|
||||
name="Basic Memory Search Results (Vanilla)",
|
||||
description="Vanilla HTML search results UI.",
|
||||
mime_type=MIME_TYPE,
|
||||
)
|
||||
def search_results_ui_vanilla() -> str:
|
||||
return load_html("search-results-vanilla.html")
|
||||
|
||||
|
||||
@mcp.resource(
|
||||
uri="ui://basic-memory/search-results/tool-ui",
|
||||
name="Basic Memory Search Results (Tool UI)",
|
||||
description="Tool UI styled search results UI.",
|
||||
mime_type=MIME_TYPE,
|
||||
)
|
||||
def search_results_ui_tool_ui() -> str:
|
||||
return load_html("search-results-tool-ui.html")
|
||||
|
||||
|
||||
@mcp.resource(
|
||||
uri="ui://basic-memory/search-results/mcp-ui",
|
||||
name="Basic Memory Search Results (MCP UI)",
|
||||
description="MCP UI styled search results UI.",
|
||||
mime_type=MIME_TYPE,
|
||||
)
|
||||
def search_results_ui_mcp_ui() -> str:
|
||||
return load_html("search-results-mcp-ui.html")
|
||||
|
||||
|
||||
@mcp.resource(
|
||||
uri="ui://basic-memory/note-preview/vanilla",
|
||||
name="Basic Memory Note Preview (Vanilla)",
|
||||
description="Vanilla HTML note preview UI.",
|
||||
mime_type=MIME_TYPE,
|
||||
)
|
||||
def note_preview_ui_vanilla() -> str:
|
||||
return load_html("note-preview-vanilla.html")
|
||||
|
||||
|
||||
@mcp.resource(
|
||||
uri="ui://basic-memory/note-preview/tool-ui",
|
||||
name="Basic Memory Note Preview (Tool UI)",
|
||||
description="Tool UI styled note preview UI.",
|
||||
mime_type=MIME_TYPE,
|
||||
)
|
||||
def note_preview_ui_tool_ui() -> str:
|
||||
return load_html("note-preview-tool-ui.html")
|
||||
|
||||
|
||||
@mcp.resource(
|
||||
uri="ui://basic-memory/note-preview/mcp-ui",
|
||||
name="Basic Memory Note Preview (MCP UI)",
|
||||
description="MCP UI styled note preview UI.",
|
||||
mime_type=MIME_TYPE,
|
||||
)
|
||||
def note_preview_ui_mcp_ui() -> str:
|
||||
return load_html("note-preview-mcp-ui.html")
|
||||
@@ -11,11 +11,8 @@ from basic_memory.mcp.tools.read_content import read_content
|
||||
from basic_memory.mcp.tools.build_context import build_context
|
||||
from basic_memory.mcp.tools.recent_activity import recent_activity
|
||||
from basic_memory.mcp.tools.read_note import read_note
|
||||
from basic_memory.mcp.tools.ui_sdk import read_note_ui, search_notes_ui
|
||||
from basic_memory.mcp.tools.view_note import view_note
|
||||
from basic_memory.mcp.tools.write_note import write_note
|
||||
from basic_memory.mcp.tools.cloud_info import cloud_info
|
||||
from basic_memory.mcp.tools.release_notes import release_notes
|
||||
from basic_memory.mcp.tools.search import search_notes, search_by_metadata
|
||||
from basic_memory.mcp.tools.canvas import canvas
|
||||
from basic_memory.mcp.tools.list_directory import list_directory
|
||||
@@ -30,13 +27,9 @@ from basic_memory.mcp.tools.project_management import (
|
||||
# ChatGPT-compatible tools
|
||||
from basic_memory.mcp.tools.chatgpt_tools import search, fetch
|
||||
|
||||
# Schema tools
|
||||
from basic_memory.mcp.tools.schema import schema_validate, schema_infer, schema_diff
|
||||
|
||||
__all__ = [
|
||||
"build_context",
|
||||
"canvas",
|
||||
"cloud_info",
|
||||
"create_memory_project",
|
||||
"delete_note",
|
||||
"delete_project",
|
||||
@@ -47,16 +40,10 @@ __all__ = [
|
||||
"move_note",
|
||||
"read_content",
|
||||
"read_note",
|
||||
"release_notes",
|
||||
"read_note_ui",
|
||||
"recent_activity",
|
||||
"schema_diff",
|
||||
"schema_infer",
|
||||
"schema_validate",
|
||||
"search",
|
||||
"search_by_metadata",
|
||||
"search_notes",
|
||||
"search_notes_ui",
|
||||
"view_note",
|
||||
"write_note",
|
||||
]
|
||||
|
||||
@@ -5,10 +5,15 @@ from typing import Optional
|
||||
from loguru import logger
|
||||
from fastmcp import Context
|
||||
|
||||
from basic_memory.mcp.project_context import get_project_client, resolve_project_and_path
|
||||
from basic_memory.mcp.async_client import get_client
|
||||
from basic_memory.mcp.project_context import get_active_project
|
||||
from basic_memory.mcp.server import mcp
|
||||
from basic_memory.schemas.base import TimeFrame
|
||||
from basic_memory.schemas.memory import GraphContext, MemoryUrl, parse_memory_url
|
||||
from basic_memory.schemas.memory import (
|
||||
GraphContext,
|
||||
MemoryUrl,
|
||||
memory_url_path,
|
||||
)
|
||||
|
||||
|
||||
@mcp.tool(
|
||||
@@ -19,10 +24,9 @@ from basic_memory.schemas.memory import GraphContext, MemoryUrl, parse_memory_ur
|
||||
Memory URL Format:
|
||||
- Use paths like "folder/note" or "memory://folder/note"
|
||||
- Pattern matching: "folder/*" matches all notes in folder
|
||||
- Query parameters: "memory://folder/note?project=myproject" to target a specific project
|
||||
- Valid characters: letters, numbers, hyphens, underscores, forward slashes
|
||||
- Avoid: double slashes (//), angle brackets (<>), quotes, pipes (|)
|
||||
- Examples: "specs/search", "memory://specs/search?project=research", "notes/*"
|
||||
- Examples: "specs/search", "projects/basic-memory", "notes/*"
|
||||
|
||||
Timeframes support natural language like:
|
||||
- "2 days ago", "last week", "today", "3 months ago"
|
||||
@@ -46,16 +50,13 @@ async def build_context(
|
||||
a rich context graph of related information.
|
||||
|
||||
Project Resolution:
|
||||
Server resolves projects using a unified priority chain (same in local and cloud modes):
|
||||
Single Project Mode → project parameter → default project.
|
||||
Uses default project automatically. Specify `project` parameter to target a different project.
|
||||
Server resolves projects in this order: Single Project Mode → project parameter → default project.
|
||||
If project unknown, use list_memory_projects() or recent_activity() first.
|
||||
|
||||
Args:
|
||||
project: Project name to build context from. Optional - server will resolve using hierarchy.
|
||||
If unknown, use list_memory_projects() to discover available projects.
|
||||
Can also be specified via ?project= query parameter on the URL.
|
||||
url: memory:// URI pointing to discussion content (e.g. memory://specs/search)
|
||||
Supports ?project= query parameter (e.g. memory://specs/search?project=research)
|
||||
depth: How many relation hops to traverse (1-3 recommended for performance)
|
||||
timeframe: How far back to look. Supports natural language like "2 days ago", "last week"
|
||||
page: Page number of results to return (default: 1)
|
||||
@@ -85,11 +86,6 @@ async def build_context(
|
||||
Raises:
|
||||
ToolError: If project doesn't exist or depth parameter is invalid
|
||||
"""
|
||||
# Extract ?project= from URL when no explicit project parameter was provided
|
||||
url, url_params = parse_memory_url(url)
|
||||
if project is None and "project" in url_params:
|
||||
project = url_params["project"]
|
||||
|
||||
logger.info(f"Building context from {url} in project {project}")
|
||||
|
||||
# Convert string depth to integer if needed
|
||||
@@ -101,11 +97,11 @@ async def build_context(
|
||||
|
||||
raise ToolError(f"Invalid depth parameter: '{depth}' is not a valid integer")
|
||||
|
||||
async with get_project_client(project, context) as (client, active_project):
|
||||
# Resolve memory:// identifier with project-prefix awareness
|
||||
_, resolved_path, _ = await resolve_project_and_path(
|
||||
client, url, project, context
|
||||
)
|
||||
# URL is already validated and normalized by MemoryUrl type annotation
|
||||
|
||||
async with get_client() as client:
|
||||
# Get the active project using the new stateless approach
|
||||
active_project = await get_active_project(client, project, context)
|
||||
|
||||
# Import here to avoid circular import
|
||||
from basic_memory.mcp.clients import MemoryClient
|
||||
@@ -113,7 +109,7 @@ async def build_context(
|
||||
# Use typed MemoryClient for API calls
|
||||
memory_client = MemoryClient(client, active_project.external_id)
|
||||
return await memory_client.build_context(
|
||||
resolved_path,
|
||||
memory_url_path(url),
|
||||
depth=depth or 1,
|
||||
timeframe=timeframe,
|
||||
page=page,
|
||||
|
||||
@@ -9,7 +9,8 @@ from typing import Dict, List, Any, Optional
|
||||
from loguru import logger
|
||||
from fastmcp import Context
|
||||
|
||||
from basic_memory.mcp.project_context import get_project_client
|
||||
from basic_memory.mcp.async_client import get_client
|
||||
from basic_memory.mcp.project_context import get_active_project
|
||||
from basic_memory.mcp.server import mcp
|
||||
from basic_memory.mcp.tools.utils import call_put, call_post, resolve_entity_id
|
||||
|
||||
@@ -93,7 +94,9 @@ async def canvas(
|
||||
Raises:
|
||||
ToolError: If project doesn't exist or directory path is invalid
|
||||
"""
|
||||
async with get_project_client(project, context) as (client, active_project):
|
||||
async with get_client() as client:
|
||||
active_project = await get_active_project(client, project, context)
|
||||
|
||||
# Ensure path has .canvas extension
|
||||
file_title = title if title.endswith(".canvas") else f"{title}.canvas"
|
||||
file_path = f"{directory}/{file_title}"
|
||||
|
||||
@@ -1,12 +0,0 @@
|
||||
"""Cloud information MCP tool."""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from basic_memory.mcp.server import mcp
|
||||
|
||||
|
||||
@mcp.tool("cloud_info")
|
||||
def cloud_info() -> str:
|
||||
"""Return optional Basic Memory Cloud information and setup guidance."""
|
||||
content_path = Path(__file__).parent.parent / "resources" / "cloud_info.md"
|
||||
return content_path.read_text(encoding="utf-8")
|
||||
@@ -5,9 +5,9 @@ from loguru import logger
|
||||
from fastmcp import Context
|
||||
from mcp.server.fastmcp.exceptions import ToolError
|
||||
|
||||
from basic_memory.mcp.project_context import get_project_client
|
||||
from basic_memory.mcp.project_context import get_active_project
|
||||
from basic_memory.mcp.server import mcp
|
||||
from basic_memory.schemas.memory import parse_memory_url
|
||||
from basic_memory.mcp.async_client import get_client
|
||||
|
||||
|
||||
def _format_delete_error_response(project: str, error_message: str, identifier: str) -> str:
|
||||
@@ -216,17 +216,13 @@ async def delete_note(
|
||||
with suggestions for finding the correct identifier, including search
|
||||
commands and alternative formats to try.
|
||||
"""
|
||||
# Extract ?project= from identifier when no explicit project parameter was provided
|
||||
if identifier.startswith("memory://"):
|
||||
identifier, url_params = parse_memory_url(identifier)
|
||||
if project is None and "project" in url_params:
|
||||
project = url_params["project"]
|
||||
|
||||
async with get_project_client(project, context) as (client, active_project):
|
||||
async with get_client() as client:
|
||||
logger.debug(
|
||||
f"Deleting {'directory' if is_directory else 'note'}: {identifier} in project: {active_project.name}"
|
||||
f"Deleting {'directory' if is_directory else 'note'}: {identifier} in project: {project}"
|
||||
)
|
||||
|
||||
active_project = await get_active_project(client, project, context)
|
||||
|
||||
# Import here to avoid circular import
|
||||
from basic_memory.mcp.clients import KnowledgeClient
|
||||
|
||||
|
||||
@@ -5,9 +5,9 @@ from typing import Optional
|
||||
from loguru import logger
|
||||
from fastmcp import Context
|
||||
|
||||
from basic_memory.mcp.project_context import get_project_client, add_project_metadata
|
||||
from basic_memory.mcp.async_client import get_client
|
||||
from basic_memory.mcp.project_context import get_active_project, add_project_metadata
|
||||
from basic_memory.mcp.server import mcp
|
||||
from basic_memory.schemas.memory import parse_memory_url
|
||||
|
||||
|
||||
def _format_error_response(
|
||||
@@ -212,13 +212,9 @@ async def edit_note(
|
||||
search_notes() first to find the correct identifier. The tool provides detailed
|
||||
error messages with suggestions if operations fail.
|
||||
"""
|
||||
# Extract ?project= from identifier when no explicit project parameter was provided
|
||||
if identifier.startswith("memory://"):
|
||||
identifier, url_params = parse_memory_url(identifier)
|
||||
if project is None and "project" in url_params:
|
||||
project = url_params["project"]
|
||||
async with get_client() as client:
|
||||
active_project = await get_active_project(client, project, context)
|
||||
|
||||
async with get_project_client(project, context) as (client, active_project):
|
||||
logger.info("MCP tool call", tool="edit_note", identifier=identifier, operation=operation)
|
||||
|
||||
# Validate operation
|
||||
|
||||
@@ -5,7 +5,8 @@ from typing import Optional
|
||||
from loguru import logger
|
||||
from fastmcp import Context
|
||||
|
||||
from basic_memory.mcp.project_context import get_project_client
|
||||
from basic_memory.mcp.async_client import get_client
|
||||
from basic_memory.mcp.project_context import get_active_project
|
||||
from basic_memory.mcp.server import mcp
|
||||
|
||||
|
||||
@@ -61,7 +62,9 @@ async def list_directory(
|
||||
Raises:
|
||||
ToolError: If project doesn't exist or directory path is invalid
|
||||
"""
|
||||
async with get_project_client(project, context) as (client, active_project):
|
||||
async with get_client() as client:
|
||||
active_project = await get_active_project(client, project, context)
|
||||
|
||||
logger.debug(
|
||||
f"Listing directory '{dir_name}' in project {project} with depth={depth}, glob='{file_name_glob}'"
|
||||
)
|
||||
|
||||
@@ -6,9 +6,9 @@ from typing import Optional
|
||||
from loguru import logger
|
||||
from fastmcp import Context
|
||||
|
||||
from basic_memory.mcp.async_client import get_client
|
||||
from basic_memory.mcp.server import mcp
|
||||
from basic_memory.mcp.project_context import get_project_client
|
||||
from basic_memory.schemas.memory import parse_memory_url
|
||||
from basic_memory.mcp.project_context import get_active_project
|
||||
from basic_memory.utils import validate_project_path
|
||||
|
||||
|
||||
@@ -412,17 +412,13 @@ async def move_note(
|
||||
- Re-indexes the entity for search
|
||||
- Maintains all observations and relations
|
||||
"""
|
||||
# Extract ?project= from identifier when no explicit project parameter was provided
|
||||
if identifier.startswith("memory://"):
|
||||
identifier, url_params = parse_memory_url(identifier)
|
||||
if project is None and "project" in url_params:
|
||||
project = url_params["project"]
|
||||
|
||||
async with get_project_client(project, context) as (client, active_project):
|
||||
async with get_client() as client:
|
||||
logger.debug(
|
||||
f"Moving {'directory' if is_directory else 'note'}: {identifier} to {destination_path} in project: {active_project.name}"
|
||||
f"Moving {'directory' if is_directory else 'note'}: {identifier} to {destination_path} in project: {project}"
|
||||
)
|
||||
|
||||
active_project = await get_active_project(client, project, context)
|
||||
|
||||
# Validate destination path to prevent path traversal attacks
|
||||
project_path = active_project.home
|
||||
if not validate_project_path(destination_path, project_path):
|
||||
|
||||
@@ -15,10 +15,11 @@ from PIL import Image as PILImage
|
||||
from fastmcp import Context
|
||||
from mcp.server.fastmcp.exceptions import ToolError
|
||||
|
||||
from basic_memory.mcp.project_context import get_project_client, resolve_project_and_path
|
||||
from basic_memory.mcp.project_context import get_active_project
|
||||
from basic_memory.mcp.server import mcp
|
||||
from basic_memory.mcp.async_client import get_client
|
||||
from basic_memory.mcp.tools.utils import call_get, resolve_entity_id
|
||||
from basic_memory.schemas.memory import memory_url_path, parse_memory_url
|
||||
from basic_memory.schemas.memory import memory_url_path
|
||||
from basic_memory.utils import validate_project_path
|
||||
|
||||
|
||||
@@ -199,26 +200,16 @@ async def read_content(
|
||||
HTTPError: If project doesn't exist or is inaccessible
|
||||
SecurityError: If path attempts path traversal
|
||||
"""
|
||||
# Extract ?project= from path when no explicit project parameter was provided
|
||||
if path.startswith("memory://"):
|
||||
path, url_params = parse_memory_url(path)
|
||||
if project is None and "project" in url_params:
|
||||
project = url_params["project"]
|
||||
|
||||
logger.info("Reading file", path=path, project=project)
|
||||
|
||||
async with get_project_client(project, context) as (client, active_project):
|
||||
# Resolve path with project-prefix awareness for memory:// URLs
|
||||
_, url, _ = await resolve_project_and_path(client, path, project, context)
|
||||
async with get_client() as client:
|
||||
active_project = await get_active_project(client, project, context)
|
||||
|
||||
url = memory_url_path(path)
|
||||
|
||||
# Validate path to prevent path traversal attacks
|
||||
# For memory:// URLs, validate the extracted path (not the raw URL which
|
||||
# has a scheme prefix that confuses path validation)
|
||||
raw_path = memory_url_path(path) if path.startswith("memory://") else path
|
||||
project_path = active_project.home
|
||||
if not validate_project_path(raw_path, project_path) or not validate_project_path(
|
||||
url, project_path
|
||||
):
|
||||
if not validate_project_path(url, project_path):
|
||||
logger.warning(
|
||||
"Attempted path traversal attack blocked",
|
||||
path=path,
|
||||
|
||||
@@ -1,29 +1,27 @@
|
||||
"""Read note tool for Basic Memory MCP server."""
|
||||
|
||||
from textwrap import dedent
|
||||
from typing import Optional, Literal
|
||||
from typing import Optional
|
||||
|
||||
from loguru import logger
|
||||
from fastmcp import Context
|
||||
|
||||
from basic_memory.mcp.project_context import get_project_client, resolve_project_and_path
|
||||
from basic_memory.mcp.async_client import get_client
|
||||
from basic_memory.mcp.project_context import get_active_project
|
||||
from basic_memory.mcp.server import mcp
|
||||
from basic_memory.mcp.formatting import format_note_preview_ascii
|
||||
from basic_memory.mcp.tools.search import search_notes
|
||||
from basic_memory.schemas.memory import memory_url_path, parse_memory_url
|
||||
from basic_memory.schemas.memory import memory_url_path
|
||||
from basic_memory.utils import validate_project_path
|
||||
|
||||
|
||||
@mcp.tool(
|
||||
description="Read a markdown note by title or permalink.",
|
||||
meta={"ui/resourceUri": "ui://basic-memory/note-preview"},
|
||||
)
|
||||
async def read_note(
|
||||
identifier: str,
|
||||
project: Optional[str] = None,
|
||||
page: int = 1,
|
||||
page_size: int = 10,
|
||||
output_format: Literal["default", "ascii", "ansi"] = "default",
|
||||
context: Context | None = None,
|
||||
) -> str:
|
||||
"""Return the raw markdown for a note, or guidance text if no match is found.
|
||||
@@ -32,9 +30,8 @@ async def read_note(
|
||||
returning the raw markdown content including observations, relations, and metadata.
|
||||
|
||||
Project Resolution:
|
||||
Server resolves projects using a unified priority chain (same in local and cloud modes):
|
||||
Single Project Mode → project parameter → default project.
|
||||
Uses default project automatically. Specify `project` parameter to target a different project.
|
||||
Server resolves projects in this order: Single Project Mode → project parameter → default project.
|
||||
If project unknown, use list_memory_projects() or recent_activity() first.
|
||||
|
||||
This tool will try multiple lookup strategies to find the most relevant note:
|
||||
1. Direct permalink lookup
|
||||
@@ -49,8 +46,6 @@ async def read_note(
|
||||
Can be a full memory:// URL, a permalink, a title, or search text
|
||||
page: Page number for paginated results (default: 1)
|
||||
page_size: Number of items per page (default: 10)
|
||||
output_format: "default" returns markdown, "ascii" returns a plain text preview,
|
||||
"ansi" returns a colorized preview for TUI clients.
|
||||
context: Optional FastMCP context for performance caching.
|
||||
|
||||
Returns:
|
||||
@@ -81,26 +76,16 @@ async def read_note(
|
||||
If the exact note isn't found, this tool provides helpful suggestions
|
||||
including related notes, search commands, and note creation templates.
|
||||
"""
|
||||
# Extract ?project= from identifier when no explicit project parameter was provided
|
||||
if identifier.startswith("memory://"):
|
||||
identifier, url_params = parse_memory_url(identifier)
|
||||
if project is None and "project" in url_params:
|
||||
project = url_params["project"]
|
||||
|
||||
async with get_project_client(project, context) as (client, active_project):
|
||||
# Resolve identifier with project-prefix awareness for memory:// URLs
|
||||
_, entity_path, _ = await resolve_project_and_path(
|
||||
client, identifier, project, context
|
||||
)
|
||||
async with get_client() as client:
|
||||
# Get and validate the project
|
||||
active_project = await get_active_project(client, project, context)
|
||||
|
||||
# Validate identifier to prevent path traversal attacks
|
||||
# For memory:// URLs, validate the extracted path (not the raw URL which
|
||||
# has a scheme prefix that confuses path validation)
|
||||
raw_path = memory_url_path(identifier) if identifier.startswith("memory://") else identifier
|
||||
processed_path = entity_path
|
||||
# We need to check both the raw identifier and the processed path
|
||||
processed_path = memory_url_path(identifier)
|
||||
project_path = active_project.home
|
||||
|
||||
if not validate_project_path(raw_path, project_path) or not validate_project_path(
|
||||
if not validate_project_path(identifier, project_path) or not validate_project_path(
|
||||
processed_path, project_path
|
||||
):
|
||||
logger.warning(
|
||||
@@ -112,6 +97,7 @@ async def read_note(
|
||||
return f"# Error\n\nIdentifier '{identifier}' is not allowed - paths must stay within project boundaries"
|
||||
|
||||
# Get the file via REST API - first try direct identifier resolution
|
||||
entity_path = memory_url_path(identifier)
|
||||
logger.info(
|
||||
f"Attempting to read note from Project: {active_project.name} identifier: {entity_path}"
|
||||
)
|
||||
@@ -133,12 +119,6 @@ async def read_note(
|
||||
# If successful, return the content
|
||||
if response.status_code == 200:
|
||||
logger.info("Returning read_note result from resource: {path}", path=entity_path)
|
||||
if output_format in ("ascii", "ansi"):
|
||||
return format_note_preview_ascii(
|
||||
response.text,
|
||||
identifier=identifier,
|
||||
color=output_format == "ansi",
|
||||
)
|
||||
return response.text
|
||||
except Exception as e: # pragma: no cover
|
||||
logger.info(f"Direct lookup failed for '{entity_path}': {e}")
|
||||
@@ -147,10 +127,7 @@ async def read_note(
|
||||
# Fallback 1: Try title search via API
|
||||
logger.info(f"Search title for: {identifier}")
|
||||
title_results = await search_notes.fn(
|
||||
query=identifier,
|
||||
search_type="title",
|
||||
project=active_project.name,
|
||||
context=context,
|
||||
query=identifier, search_type="title", project=project, context=context
|
||||
)
|
||||
|
||||
# Handle both SearchResponse object and error strings
|
||||
@@ -166,12 +143,6 @@ async def read_note(
|
||||
|
||||
if response.status_code == 200:
|
||||
logger.info(f"Found note by title search: {result.permalink}")
|
||||
if output_format in ("ascii", "ansi"):
|
||||
return format_note_preview_ascii(
|
||||
response.text,
|
||||
identifier=identifier,
|
||||
color=output_format == "ansi",
|
||||
)
|
||||
return response.text
|
||||
except Exception as e: # pragma: no cover
|
||||
logger.info(
|
||||
@@ -185,10 +156,7 @@ async def read_note(
|
||||
# Fallback 2: Text search as a last resort
|
||||
logger.info(f"Title search failed, trying text search for: {identifier}")
|
||||
text_results = await search_notes.fn(
|
||||
query=identifier,
|
||||
search_type="text",
|
||||
project=active_project.name,
|
||||
context=context,
|
||||
query=identifier, search_type="text", project=project, context=context
|
||||
)
|
||||
|
||||
# We didn't find a direct match, construct a helpful error message
|
||||
|
||||
@@ -7,10 +7,7 @@ from loguru import logger
|
||||
from fastmcp import Context
|
||||
|
||||
from basic_memory.mcp.async_client import get_client
|
||||
from basic_memory.mcp.project_context import (
|
||||
get_project_client,
|
||||
resolve_project_parameter,
|
||||
)
|
||||
from basic_memory.mcp.project_context import get_active_project, resolve_project_parameter
|
||||
from basic_memory.mcp.server import mcp
|
||||
from basic_memory.mcp.tools.utils import call_get
|
||||
from basic_memory.schemas.base import TimeFrame
|
||||
@@ -102,51 +99,50 @@ async def recent_activity(
|
||||
- For focused queries, consider using build_context with a specific URI
|
||||
- Max timeframe is 1 year in the past
|
||||
"""
|
||||
# Build common parameters for API calls
|
||||
params: dict = {
|
||||
"page": 1,
|
||||
"page_size": 10,
|
||||
"max_related": 10,
|
||||
}
|
||||
if depth:
|
||||
params["depth"] = depth
|
||||
if timeframe:
|
||||
params["timeframe"] = timeframe # pyright: ignore
|
||||
async with get_client() as client:
|
||||
# Build common parameters for API calls
|
||||
params = {
|
||||
"page": 1,
|
||||
"page_size": 10,
|
||||
"max_related": 10,
|
||||
}
|
||||
if depth:
|
||||
params["depth"] = depth
|
||||
if timeframe:
|
||||
params["timeframe"] = timeframe # pyright: ignore
|
||||
|
||||
# Validate and convert type parameter
|
||||
if type:
|
||||
# Convert single string to list
|
||||
if isinstance(type, str):
|
||||
type_list = [type]
|
||||
else:
|
||||
type_list = type
|
||||
# Validate and convert type parameter
|
||||
if type:
|
||||
# Convert single string to list
|
||||
if isinstance(type, str):
|
||||
type_list = [type]
|
||||
else:
|
||||
type_list = type
|
||||
|
||||
# Validate each type against SearchItemType enum
|
||||
validated_types = []
|
||||
for t in type_list:
|
||||
try:
|
||||
# Try to convert string to enum
|
||||
if isinstance(t, str):
|
||||
validated_types.append(SearchItemType(t.lower()))
|
||||
except ValueError:
|
||||
valid_types = [t.value for t in SearchItemType]
|
||||
raise ValueError(f"Invalid type: {t}. Valid types are: {valid_types}")
|
||||
# Validate each type against SearchItemType enum
|
||||
validated_types = []
|
||||
for t in type_list:
|
||||
try:
|
||||
# Try to convert string to enum
|
||||
if isinstance(t, str):
|
||||
validated_types.append(SearchItemType(t.lower()))
|
||||
except ValueError:
|
||||
valid_types = [t.value for t in SearchItemType]
|
||||
raise ValueError(f"Invalid type: {t}. Valid types are: {valid_types}")
|
||||
|
||||
# Add validated types to params
|
||||
params["type"] = [t.value for t in validated_types] # pyright: ignore
|
||||
# Add validated types to params
|
||||
params["type"] = [t.value for t in validated_types] # pyright: ignore
|
||||
|
||||
# Resolve project parameter using the three-tier hierarchy
|
||||
# allow_discovery=True enables Discovery Mode, so a project is not required
|
||||
resolved_project = await resolve_project_parameter(project, allow_discovery=True)
|
||||
# Resolve project parameter using the three-tier hierarchy
|
||||
# allow_discovery=True enables Discovery Mode, so a project is not required
|
||||
resolved_project = await resolve_project_parameter(project, allow_discovery=True)
|
||||
|
||||
if resolved_project is None:
|
||||
# Discovery Mode: Get activity across all projects
|
||||
# Uses plain get_client() since we iterate across all projects (no single project routing)
|
||||
logger.info(
|
||||
f"Getting recent activity across all projects: type={type}, depth={depth}, timeframe={timeframe}"
|
||||
)
|
||||
if resolved_project is None:
|
||||
# Discovery Mode: Get activity across all projects
|
||||
logger.info(
|
||||
f"Getting recent activity across all projects: type={type}, depth={depth}, timeframe={timeframe}"
|
||||
)
|
||||
|
||||
async with get_client() as client:
|
||||
# Get list of all projects
|
||||
response = await call_get(client, "/v2/projects/")
|
||||
project_list = ProjectList.model_validate(response.json())
|
||||
@@ -185,68 +181,76 @@ async def recent_activity(
|
||||
most_active_count = item_count
|
||||
most_active_project = project_info.name
|
||||
|
||||
# Build summary stats
|
||||
summary = ActivityStats(
|
||||
total_projects=len(project_list.projects),
|
||||
active_projects=active_projects,
|
||||
most_active_project=most_active_project,
|
||||
total_items=total_items,
|
||||
total_entities=total_entities,
|
||||
total_relations=total_relations,
|
||||
total_observations=total_observations,
|
||||
)
|
||||
# Build summary stats
|
||||
summary = ActivityStats(
|
||||
total_projects=len(project_list.projects),
|
||||
active_projects=active_projects,
|
||||
most_active_project=most_active_project,
|
||||
total_items=total_items,
|
||||
total_entities=total_entities,
|
||||
total_relations=total_relations,
|
||||
total_observations=total_observations,
|
||||
)
|
||||
|
||||
# Generate guidance for the assistant
|
||||
guidance_lines = ["\n" + "─" * 40]
|
||||
# Generate guidance for the assistant
|
||||
guidance_lines = ["\n" + "─" * 40]
|
||||
|
||||
if active_projects == 0:
|
||||
# No recent activity
|
||||
guidance_lines.extend(
|
||||
[
|
||||
"No recent activity found in any project.",
|
||||
"Consider: Ask which project to use or if they want to create a new one.",
|
||||
]
|
||||
)
|
||||
else:
|
||||
# At least one project has activity: suggest the most active project.
|
||||
suggested_project = most_active_project or next(
|
||||
(
|
||||
name
|
||||
for name, activity in projects_activity.items()
|
||||
if activity.item_count > 0
|
||||
),
|
||||
None,
|
||||
)
|
||||
if suggested_project:
|
||||
suffix = (
|
||||
f"(most active with {most_active_count} items)"
|
||||
if most_active_count > 0
|
||||
else ""
|
||||
)
|
||||
guidance_lines.append(
|
||||
f"Suggested project: '{suggested_project}' {suffix}".strip()
|
||||
)
|
||||
if active_projects == 1:
|
||||
guidance_lines.append(
|
||||
f"Ask user: 'Should I use {suggested_project} for this task?'"
|
||||
)
|
||||
else:
|
||||
guidance_lines.append(
|
||||
f"Ask user: 'Should I use {suggested_project} for this task, or would you prefer a different project?'"
|
||||
)
|
||||
|
||||
if active_projects == 0:
|
||||
# No recent activity
|
||||
guidance_lines.extend(
|
||||
[
|
||||
"No recent activity found in any project.",
|
||||
"Consider: Ask which project to use or if they want to create a new one.",
|
||||
"",
|
||||
"Session reminder: Remember their project choice throughout this conversation.",
|
||||
]
|
||||
)
|
||||
|
||||
guidance = "\n".join(guidance_lines)
|
||||
|
||||
# Format discovery mode output
|
||||
return _format_discovery_output(projects_activity, summary, timeframe, guidance)
|
||||
|
||||
else:
|
||||
# At least one project has activity: suggest the most active project.
|
||||
suggested_project = most_active_project or next(
|
||||
(name for name, activity in projects_activity.items() if activity.item_count > 0),
|
||||
None,
|
||||
# Project-Specific Mode: Get activity for specific project
|
||||
logger.info(
|
||||
f"Getting recent activity from project {resolved_project}: type={type}, depth={depth}, timeframe={timeframe}"
|
||||
)
|
||||
if suggested_project:
|
||||
suffix = (
|
||||
f"(most active with {most_active_count} items)" if most_active_count > 0 else ""
|
||||
)
|
||||
guidance_lines.append(f"Suggested project: '{suggested_project}' {suffix}".strip())
|
||||
if active_projects == 1:
|
||||
guidance_lines.append(
|
||||
f"Ask user: 'Should I use {suggested_project} for this task?'"
|
||||
)
|
||||
else:
|
||||
guidance_lines.append(
|
||||
f"Ask user: 'Should I use {suggested_project} for this task, or would you prefer a different project?'"
|
||||
)
|
||||
|
||||
guidance_lines.extend(
|
||||
[
|
||||
"",
|
||||
"Session reminder: Remember their project choice throughout this conversation.",
|
||||
]
|
||||
)
|
||||
active_project = await get_active_project(client, resolved_project, context)
|
||||
|
||||
guidance = "\n".join(guidance_lines)
|
||||
|
||||
# Format discovery mode output
|
||||
return _format_discovery_output(projects_activity, summary, timeframe, guidance)
|
||||
|
||||
else:
|
||||
# Project-Specific Mode: Get activity for specific project
|
||||
# Uses get_project_client() for per-project routing (local vs cloud)
|
||||
logger.info(
|
||||
f"Getting recent activity from project {resolved_project}: type={type}, depth={depth}, timeframe={timeframe}"
|
||||
)
|
||||
|
||||
async with get_project_client(resolved_project, context) as (client, active_project):
|
||||
response = await call_get(
|
||||
client,
|
||||
f"/v2/projects/{active_project.external_id}/memory/recent",
|
||||
|
||||
@@ -1,12 +0,0 @@
|
||||
"""Release notes MCP tool."""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from basic_memory.mcp.server import mcp
|
||||
|
||||
|
||||
@mcp.tool("release_notes")
|
||||
def release_notes() -> str:
|
||||
"""Return the latest product release notes for optional user review."""
|
||||
content_path = Path(__file__).parent.parent / "resources" / "release_notes.md"
|
||||
return content_path.read_text(encoding="utf-8")
|
||||
@@ -1,244 +0,0 @@
|
||||
"""Schema tools for Basic Memory MCP server.
|
||||
|
||||
Provides tools for schema validation, inference, and drift detection through the MCP protocol.
|
||||
These tools call the schema API endpoints via the typed SchemaClient.
|
||||
"""
|
||||
|
||||
from typing import Optional
|
||||
|
||||
from loguru import logger
|
||||
from fastmcp import Context
|
||||
|
||||
from basic_memory.mcp.async_client import get_client
|
||||
from basic_memory.mcp.project_context import get_active_project
|
||||
from basic_memory.mcp.server import mcp
|
||||
from basic_memory.schemas.schema import ValidationReport, InferenceReport, DriftReport
|
||||
|
||||
|
||||
@mcp.tool(
|
||||
description="Validate notes against their Picoschema definitions.",
|
||||
)
|
||||
async def schema_validate(
|
||||
entity_type: Optional[str] = None,
|
||||
identifier: Optional[str] = None,
|
||||
project: Optional[str] = None,
|
||||
context: Context | None = None,
|
||||
) -> ValidationReport | str:
|
||||
"""Validate notes against their resolved schema.
|
||||
|
||||
Validates a specific note (by identifier) or all notes of a given type.
|
||||
Returns warnings/errors based on the schema's validation mode.
|
||||
|
||||
Schemas are resolved in priority order:
|
||||
1. Inline schema (dict in frontmatter)
|
||||
2. Explicit reference (string in frontmatter)
|
||||
3. Implicit by type (type field matches schema note entity field)
|
||||
4. No schema (no validation)
|
||||
|
||||
Project Resolution:
|
||||
Server resolves projects in this order: Single Project Mode -> project parameter -> default.
|
||||
If project unknown, use list_memory_projects() first.
|
||||
|
||||
Args:
|
||||
entity_type: Entity type to batch-validate (e.g., "Person").
|
||||
If provided, validates all notes of this type.
|
||||
identifier: Specific note to validate (permalink, title, or path).
|
||||
If provided, validates only this note.
|
||||
project: Project name. Optional -- server will resolve.
|
||||
context: Optional FastMCP context for performance caching.
|
||||
|
||||
Returns:
|
||||
ValidationReport with per-note results, or error guidance string
|
||||
|
||||
Examples:
|
||||
# Validate all Person notes
|
||||
schema_validate(entity_type="Person")
|
||||
|
||||
# Validate a specific note
|
||||
schema_validate(identifier="people/paul-graham")
|
||||
|
||||
# Validate in a specific project
|
||||
schema_validate(entity_type="Person", project="my-research")
|
||||
"""
|
||||
async with get_client() as client:
|
||||
active_project = await get_active_project(client, project, context)
|
||||
logger.info(
|
||||
f"MCP tool call tool=schema_validate project={active_project.name} "
|
||||
f"entity_type={entity_type} identifier={identifier}"
|
||||
)
|
||||
|
||||
try:
|
||||
from basic_memory.mcp.clients.schema import SchemaClient
|
||||
|
||||
schema_client = SchemaClient(client, active_project.external_id)
|
||||
result = await schema_client.validate(
|
||||
entity_type=entity_type,
|
||||
identifier=identifier,
|
||||
)
|
||||
|
||||
logger.info(
|
||||
f"MCP tool response: tool=schema_validate project={active_project.name} "
|
||||
f"total={result.total_notes} valid={result.valid_count} "
|
||||
f"warnings={result.warning_count} errors={result.error_count}"
|
||||
)
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Schema validation failed: {e}, project: {active_project.name}")
|
||||
return (
|
||||
f"# Schema Validation Failed\n\n"
|
||||
f"Error validating schemas: {e}\n\n"
|
||||
f"## Troubleshooting\n"
|
||||
f"1. Ensure schema notes exist (type: schema) for the target entity type\n"
|
||||
f"2. Check that notes have the correct type in frontmatter\n"
|
||||
f"3. Verify the project has been synced: `basic-memory status`\n"
|
||||
)
|
||||
|
||||
|
||||
@mcp.tool(
|
||||
description="Analyze existing notes and suggest a Picoschema definition.",
|
||||
)
|
||||
async def schema_infer(
|
||||
entity_type: str,
|
||||
threshold: float = 0.25,
|
||||
project: Optional[str] = None,
|
||||
context: Context | None = None,
|
||||
) -> InferenceReport | str:
|
||||
"""Analyze existing notes and suggest a schema definition.
|
||||
|
||||
Examines observation categories and relation types across all notes
|
||||
of the given type. Returns frequency analysis and suggested Picoschema
|
||||
YAML that can be saved as a schema note.
|
||||
|
||||
Frequency thresholds:
|
||||
- 95%+ present -> required field
|
||||
- threshold+ present -> optional field
|
||||
- Below threshold -> excluded (but noted)
|
||||
|
||||
Project Resolution:
|
||||
Server resolves projects in this order: Single Project Mode -> project parameter -> default.
|
||||
If project unknown, use list_memory_projects() first.
|
||||
|
||||
Args:
|
||||
entity_type: The entity type to analyze (e.g., "Person", "meeting").
|
||||
threshold: Minimum frequency (0-1) for a field to be suggested as optional.
|
||||
Default 0.25 (25%). Fields above 95% become required.
|
||||
project: Project name. Optional -- server will resolve.
|
||||
context: Optional FastMCP context for performance caching.
|
||||
|
||||
Returns:
|
||||
InferenceReport with frequency data and suggested schema, or error string
|
||||
|
||||
Examples:
|
||||
# Infer schema for Person notes
|
||||
schema_infer("Person")
|
||||
|
||||
# Use a higher threshold (50% minimum)
|
||||
schema_infer("meeting", threshold=0.5)
|
||||
|
||||
# Infer in a specific project
|
||||
schema_infer("Person", project="my-research")
|
||||
"""
|
||||
async with get_client() as client:
|
||||
active_project = await get_active_project(client, project, context)
|
||||
logger.info(
|
||||
f"MCP tool call tool=schema_infer project={active_project.name} "
|
||||
f"entity_type={entity_type} threshold={threshold}"
|
||||
)
|
||||
|
||||
try:
|
||||
from basic_memory.mcp.clients.schema import SchemaClient
|
||||
|
||||
schema_client = SchemaClient(client, active_project.external_id)
|
||||
result = await schema_client.infer(entity_type, threshold=threshold)
|
||||
|
||||
logger.info(
|
||||
f"MCP tool response: tool=schema_infer project={active_project.name} "
|
||||
f"entity_type={entity_type} notes_analyzed={result.notes_analyzed} "
|
||||
f"required={len(result.suggested_required)} "
|
||||
f"optional={len(result.suggested_optional)}"
|
||||
)
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Schema inference failed: {e}, project: {active_project.name}")
|
||||
return (
|
||||
f"# Schema Inference Failed\n\n"
|
||||
f"Error inferring schema for '{entity_type}': {e}\n\n"
|
||||
f"## Troubleshooting\n"
|
||||
f"1. Ensure notes of type '{entity_type}' exist in the project\n"
|
||||
f'2. Try searching: `search_notes("{entity_type}", types=["{entity_type}"])`\n'
|
||||
f"3. Verify the project has been synced: `basic-memory status`\n"
|
||||
)
|
||||
|
||||
|
||||
@mcp.tool(
|
||||
description="Detect drift between a schema definition and actual note usage.",
|
||||
)
|
||||
async def schema_diff(
|
||||
entity_type: str,
|
||||
project: Optional[str] = None,
|
||||
context: Context | None = None,
|
||||
) -> DriftReport | str:
|
||||
"""Detect drift between a schema definition and actual note usage.
|
||||
|
||||
Compares the existing schema for an entity type against how notes of
|
||||
that type are actually structured. Identifies new fields that have
|
||||
appeared, declared fields that are rarely used, and cardinality changes
|
||||
(single-value vs array).
|
||||
|
||||
Useful for evolving schemas as your knowledge base grows -- run
|
||||
periodically to see if your schema still matches reality.
|
||||
|
||||
Project Resolution:
|
||||
Server resolves projects in this order: Single Project Mode -> project parameter -> default.
|
||||
If project unknown, use list_memory_projects() first.
|
||||
|
||||
Args:
|
||||
entity_type: The entity type to check for drift (e.g., "Person").
|
||||
project: Project name. Optional -- server will resolve.
|
||||
context: Optional FastMCP context for performance caching.
|
||||
|
||||
Returns:
|
||||
DriftReport with new fields, dropped fields, and cardinality changes,
|
||||
or error guidance string
|
||||
|
||||
Examples:
|
||||
# Check drift for Person schema
|
||||
schema_diff("Person")
|
||||
|
||||
# Check drift in a specific project
|
||||
schema_diff("Person", project="my-research")
|
||||
"""
|
||||
async with get_client() as client:
|
||||
active_project = await get_active_project(client, project, context)
|
||||
logger.info(
|
||||
f"MCP tool call tool=schema_diff project={active_project.name} "
|
||||
f"entity_type={entity_type}"
|
||||
)
|
||||
|
||||
try:
|
||||
from basic_memory.mcp.clients.schema import SchemaClient
|
||||
|
||||
schema_client = SchemaClient(client, active_project.external_id)
|
||||
result = await schema_client.diff(entity_type)
|
||||
|
||||
logger.info(
|
||||
f"MCP tool response: tool=schema_diff project={active_project.name} "
|
||||
f"entity_type={entity_type} "
|
||||
f"new_fields={len(result.new_fields)} "
|
||||
f"dropped_fields={len(result.dropped_fields)} "
|
||||
f"cardinality_changes={len(result.cardinality_changes)}"
|
||||
)
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Schema diff failed: {e}, project: {active_project.name}")
|
||||
return (
|
||||
f"# Schema Diff Failed\n\n"
|
||||
f"Error detecting drift for '{entity_type}': {e}\n\n"
|
||||
f"## Troubleshooting\n"
|
||||
f"1. Ensure a schema note exists for entity type '{entity_type}'\n"
|
||||
f"2. Ensure notes of type '{entity_type}' exist in the project\n"
|
||||
f"3. Verify the project has been synced: `basic-memory status`\n"
|
||||
)
|
||||
@@ -1,21 +1,15 @@
|
||||
"""Search tools for Basic Memory MCP server."""
|
||||
|
||||
from textwrap import dedent
|
||||
from typing import List, Optional, Dict, Any, Literal
|
||||
from typing import List, Optional, Dict, Any
|
||||
|
||||
from loguru import logger
|
||||
from fastmcp import Context
|
||||
|
||||
from basic_memory.mcp.project_context import get_project_client, resolve_project_and_path
|
||||
from basic_memory.mcp.formatting import format_search_results_ascii
|
||||
from basic_memory.mcp.async_client import get_client
|
||||
from basic_memory.mcp.project_context import get_active_project
|
||||
from basic_memory.mcp.server import mcp
|
||||
from basic_memory.schemas.memory import parse_memory_url
|
||||
from basic_memory.schemas.search import (
|
||||
SearchItemType,
|
||||
SearchQuery,
|
||||
SearchResponse,
|
||||
SearchRetrievalMode,
|
||||
)
|
||||
from basic_memory.schemas.search import SearchItemType, SearchQuery, SearchResponse
|
||||
|
||||
|
||||
def _format_search_error_response(
|
||||
@@ -23,35 +17,6 @@ def _format_search_error_response(
|
||||
) -> str:
|
||||
"""Format helpful error responses for search failures that guide users to successful searches."""
|
||||
|
||||
# Semantic config/dependency errors
|
||||
if "semantic search is disabled" in error_message.lower():
|
||||
return dedent(f"""
|
||||
# Search Failed - Semantic Search Disabled
|
||||
|
||||
You requested `{search_type}` search for query '{query}', but semantic search is disabled.
|
||||
|
||||
## How to enable
|
||||
1. Set `BASIC_MEMORY_SEMANTIC_SEARCH_ENABLED=true`
|
||||
2. Restart the Basic Memory server/process
|
||||
|
||||
## Alternative now
|
||||
- Run FTS search instead:
|
||||
`search_notes("{project}", "{query}", search_type="text")`
|
||||
""").strip()
|
||||
|
||||
if "pip install" in error_message.lower() and "semantic" in error_message.lower():
|
||||
return dedent(f"""
|
||||
# Search Failed - Semantic Dependencies Missing
|
||||
|
||||
Semantic retrieval is enabled but required packages are not installed.
|
||||
|
||||
## Fix
|
||||
1. Install semantic extras: `pip install 'basic-memory[semantic]'`
|
||||
2. Restart Basic Memory
|
||||
3. Retry your query:
|
||||
`search_notes("{project}", "{query}", search_type="{search_type}")`
|
||||
""").strip()
|
||||
|
||||
# FTS5 syntax errors
|
||||
if "syntax error" in error_message.lower() or "fts5" in error_message.lower():
|
||||
clean_query = (
|
||||
@@ -232,7 +197,6 @@ Error searching for '{query}': {error_message}
|
||||
|
||||
@mcp.tool(
|
||||
description="Search across all content in the knowledge base with advanced syntax support.",
|
||||
meta={"ui/resourceUri": "ui://basic-memory/search-results"},
|
||||
)
|
||||
async def search_notes(
|
||||
query: str,
|
||||
@@ -240,7 +204,6 @@ async def search_notes(
|
||||
page: int = 1,
|
||||
page_size: int = 10,
|
||||
search_type: str = "text",
|
||||
output_format: Literal["default", "ascii", "ansi"] = "default",
|
||||
types: List[str] | None = None,
|
||||
entity_types: List[str] | None = None,
|
||||
after_date: Optional[str] = None,
|
||||
@@ -322,10 +285,7 @@ async def search_notes(
|
||||
If unknown, use list_memory_projects() to discover available projects.
|
||||
page: The page number of results to return (default 1)
|
||||
page_size: The number of results to return per page (default 10)
|
||||
search_type: Type of search to perform, one of:
|
||||
"text", "title", "permalink", "vector", "hybrid" (default: "text")
|
||||
output_format: "default" returns structured data, "ascii" returns a plain text table,
|
||||
"ansi" returns a colorized table for TUI clients.
|
||||
search_type: Type of search to perform, one of: "text", "title", "permalink" (default: "text")
|
||||
types: Optional list of note types to search (e.g., ["note", "person"])
|
||||
entity_types: Optional list of entity types to filter by (e.g., ["entity", "observation"])
|
||||
after_date: Optional date filter for recent content (e.g., "1 week", "2d", "2024-01-01")
|
||||
@@ -396,59 +356,41 @@ async def search_notes(
|
||||
# Explicit project specification
|
||||
results = await search_notes("project planning", project="my-project")
|
||||
"""
|
||||
# Extract ?project= from query when it's a memory:// URL
|
||||
if query.startswith("memory://"):
|
||||
query, url_params = parse_memory_url(query)
|
||||
if project is None and "project" in url_params:
|
||||
project = url_params["project"]
|
||||
|
||||
# Avoid mutable-default-argument footguns. Treat None as "no filter".
|
||||
types = types or []
|
||||
entity_types = entity_types or []
|
||||
|
||||
async with get_project_client(project, context) as (client, active_project):
|
||||
# Handle memory:// URLs by resolving to permalink search
|
||||
_, resolved_query, is_memory_url = await resolve_project_and_path(
|
||||
client, query, project, context
|
||||
)
|
||||
if is_memory_url:
|
||||
query = resolved_query
|
||||
search_type = "permalink"
|
||||
# Create a SearchQuery object based on the parameters
|
||||
search_query = SearchQuery()
|
||||
|
||||
# Create a SearchQuery object based on the parameters
|
||||
search_query = SearchQuery()
|
||||
# Set the appropriate search field based on search_type
|
||||
if search_type == "text":
|
||||
search_query.text = query
|
||||
elif search_type == "title":
|
||||
search_query.title = query
|
||||
elif search_type == "permalink" and "*" in query:
|
||||
search_query.permalink_match = query
|
||||
elif search_type == "permalink":
|
||||
search_query.permalink = query
|
||||
else: # pragma: no cover
|
||||
search_query.text = query # Default to text search
|
||||
|
||||
# Set the appropriate search field based on search_type
|
||||
if search_type == "text":
|
||||
search_query.text = query
|
||||
elif search_type == "vector":
|
||||
search_query.text = query
|
||||
search_query.retrieval_mode = SearchRetrievalMode.VECTOR
|
||||
elif search_type == "hybrid":
|
||||
search_query.text = query
|
||||
search_query.retrieval_mode = SearchRetrievalMode.HYBRID
|
||||
elif search_type == "title":
|
||||
search_query.title = query
|
||||
elif search_type == "permalink" and "*" in query:
|
||||
search_query.permalink_match = query
|
||||
elif search_type == "permalink":
|
||||
search_query.permalink = query
|
||||
else: # pragma: no cover
|
||||
search_query.text = query # Default to text search
|
||||
# Add optional filters if provided (empty lists are treated as no filter)
|
||||
if entity_types:
|
||||
search_query.entity_types = [SearchItemType(t) for t in entity_types]
|
||||
if types:
|
||||
search_query.types = types
|
||||
if after_date:
|
||||
search_query.after_date = after_date
|
||||
if metadata_filters:
|
||||
search_query.metadata_filters = metadata_filters
|
||||
if tags:
|
||||
search_query.tags = tags
|
||||
if status:
|
||||
search_query.status = status
|
||||
|
||||
# Add optional filters if provided (empty lists are treated as no filter)
|
||||
if entity_types:
|
||||
search_query.entity_types = [SearchItemType(t) for t in entity_types]
|
||||
if types:
|
||||
search_query.types = types
|
||||
if after_date:
|
||||
search_query.after_date = after_date
|
||||
if metadata_filters:
|
||||
search_query.metadata_filters = metadata_filters
|
||||
if tags:
|
||||
search_query.tags = tags
|
||||
if status:
|
||||
search_query.status = status
|
||||
async with get_client() as client:
|
||||
active_project = await get_active_project(client, project, context)
|
||||
|
||||
logger.info(f"Searching for {search_query} in project {active_project.name}")
|
||||
|
||||
@@ -472,13 +414,6 @@ async def search_notes(
|
||||
# Don't treat this as an error, but the user might want guidance
|
||||
# We return the empty result as normal - the user can decide if they need help
|
||||
|
||||
if output_format in ("ascii", "ansi"):
|
||||
return format_search_results_ascii(
|
||||
result,
|
||||
query=query,
|
||||
color=output_format == "ansi",
|
||||
)
|
||||
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
@@ -522,7 +457,8 @@ async def search_by_metadata(
|
||||
page = (offset // limit) + 1
|
||||
offset_within_page = offset % limit
|
||||
|
||||
async with get_project_client(project, context) as (client, active_project):
|
||||
async with get_client() as client:
|
||||
active_project = await get_active_project(client, project, context)
|
||||
logger.info(
|
||||
f"Structured search in project {active_project.name} filters={filters} limit={limit} offset={offset}"
|
||||
)
|
||||
|
||||
@@ -1,123 +0,0 @@
|
||||
"""Embedded UI tools using the MCP-UI Python SDK."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from fastmcp import Context
|
||||
from mcp.types import ContentBlock, TextContent
|
||||
|
||||
from basic_memory.mcp.server import mcp
|
||||
from basic_memory.mcp.tools.read_note import read_note
|
||||
from basic_memory.mcp.tools.search import search_notes
|
||||
from basic_memory.mcp.ui.sdk import MissingMCPUIServerError, build_embedded_ui_resource
|
||||
|
||||
|
||||
def _text_block(message: str) -> List[ContentBlock]:
|
||||
return [TextContent(type="text", text=message)]
|
||||
|
||||
|
||||
@mcp.tool(
|
||||
description="Search notes and return an embedded MCP-UI resource (raw HTML).",
|
||||
output_schema=None,
|
||||
)
|
||||
async def search_notes_ui(
|
||||
query: str,
|
||||
project: Optional[str] = None,
|
||||
page: int = 1,
|
||||
page_size: int = 10,
|
||||
search_type: str = "text",
|
||||
types: List[str] | None = None,
|
||||
entity_types: List[str] | None = None,
|
||||
after_date: Optional[str] = None,
|
||||
metadata_filters: Optional[Dict[str, Any]] = None,
|
||||
tags: Optional[List[str]] = None,
|
||||
status: Optional[str] = None,
|
||||
context: Context | None = None,
|
||||
) -> List[ContentBlock]:
|
||||
"""Return a search results UI as an embedded MCP-UI resource."""
|
||||
result = await search_notes.fn(
|
||||
query=query,
|
||||
project=project,
|
||||
page=page,
|
||||
page_size=page_size,
|
||||
search_type=search_type,
|
||||
output_format="default",
|
||||
types=types,
|
||||
entity_types=entity_types,
|
||||
after_date=after_date,
|
||||
metadata_filters=metadata_filters,
|
||||
tags=tags,
|
||||
status=status,
|
||||
context=context,
|
||||
)
|
||||
|
||||
if isinstance(result, str):
|
||||
return _text_block(result)
|
||||
|
||||
render_data = {
|
||||
"toolInput": {
|
||||
"query": query,
|
||||
"search_type": search_type,
|
||||
"page": page,
|
||||
"page_size": page_size,
|
||||
},
|
||||
"toolOutput": result.model_dump(),
|
||||
}
|
||||
|
||||
try:
|
||||
resource = build_embedded_ui_resource(
|
||||
uri="ui://basic-memory/search-results/mcp-ui-sdk",
|
||||
html_filename="search-results-mcp-ui.html",
|
||||
render_data=render_data,
|
||||
preferred_frame_size=["100%", "540px"],
|
||||
metadata={"basic-memory.ui-variant": "mcp-ui-sdk"},
|
||||
)
|
||||
except MissingMCPUIServerError as exc:
|
||||
return _text_block(str(exc))
|
||||
|
||||
return [resource]
|
||||
|
||||
|
||||
@mcp.tool(
|
||||
description="Read a note and return an embedded MCP-UI resource (raw HTML).",
|
||||
output_schema=None,
|
||||
)
|
||||
async def read_note_ui(
|
||||
identifier: str,
|
||||
project: Optional[str] = None,
|
||||
page: int = 1,
|
||||
page_size: int = 10,
|
||||
context: Context | None = None,
|
||||
) -> List[ContentBlock]:
|
||||
"""Return a note preview UI as an embedded MCP-UI resource."""
|
||||
content = await read_note.fn(
|
||||
identifier=identifier,
|
||||
project=project,
|
||||
page=page,
|
||||
page_size=page_size,
|
||||
output_format="default",
|
||||
context=context,
|
||||
)
|
||||
|
||||
render_data = {
|
||||
"toolInput": {
|
||||
"identifier": identifier,
|
||||
"page": page,
|
||||
"page_size": page_size,
|
||||
},
|
||||
"toolOutput": content,
|
||||
}
|
||||
|
||||
try:
|
||||
resource = build_embedded_ui_resource(
|
||||
uri="ui://basic-memory/note-preview/mcp-ui-sdk",
|
||||
html_filename="note-preview-mcp-ui.html",
|
||||
render_data=render_data,
|
||||
preferred_frame_size=["100%", "640px"],
|
||||
metadata={"basic-memory.ui-variant": "mcp-ui-sdk"},
|
||||
)
|
||||
except MissingMCPUIServerError as exc:
|
||||
return _text_block(str(exc))
|
||||
|
||||
return [resource]
|
||||
@@ -4,7 +4,8 @@ from typing import List, Union, Optional
|
||||
|
||||
from loguru import logger
|
||||
|
||||
from basic_memory.mcp.project_context import get_project_client, add_project_metadata
|
||||
from basic_memory.mcp.async_client import get_client
|
||||
from basic_memory.mcp.project_context import get_active_project, add_project_metadata
|
||||
from basic_memory.mcp.server import mcp
|
||||
from fastmcp import Context
|
||||
from basic_memory.schemas.base import Entity
|
||||
@@ -31,9 +32,8 @@ async def write_note(
|
||||
Creates or updates a markdown note with semantic observations and relations.
|
||||
|
||||
Project Resolution:
|
||||
Server resolves projects using a unified priority chain (same in local and cloud modes):
|
||||
Single Project Mode → project parameter → default project.
|
||||
Uses default project automatically. Specify `project` parameter to target a different project.
|
||||
Server resolves projects in this order: Single Project Mode → project parameter → default project.
|
||||
If project unknown, use list_memory_projects() or recent_activity() first.
|
||||
|
||||
The content can include semantic observations and relations using markdown syntax:
|
||||
|
||||
@@ -79,7 +79,12 @@ async def write_note(
|
||||
- Session tracking metadata for project awareness
|
||||
|
||||
Examples:
|
||||
# Create a simple note (uses default project automatically)
|
||||
# Assistant flow when project is unknown
|
||||
# 1. list_memory_projects() -> Ask user which project
|
||||
# 2. User: "Use my-research"
|
||||
# 3. write_note(...) and remember "my-research" for session
|
||||
|
||||
# Create a simple note
|
||||
write_note(
|
||||
project="my-research",
|
||||
title="Meeting Notes",
|
||||
@@ -109,11 +114,14 @@ async def write_note(
|
||||
HTTPError: If project doesn't exist or is inaccessible
|
||||
SecurityError: If directory path attempts path traversal
|
||||
"""
|
||||
async with get_project_client(project, context) as (client, active_project):
|
||||
async with get_client() as client:
|
||||
logger.info(
|
||||
f"MCP tool call tool=write_note project={active_project.name} directory={directory}, title={title}, tags={tags}"
|
||||
f"MCP tool call tool=write_note project={project} directory={directory}, title={title}, tags={tags}"
|
||||
)
|
||||
|
||||
# Get and validate the project (supports optional project parameter)
|
||||
active_project = await get_active_project(client, project, context)
|
||||
|
||||
# Normalize "/" to empty string for root directory (must happen before validation)
|
||||
if directory == "/":
|
||||
directory = ""
|
||||
|
||||
@@ -1,5 +0,0 @@
|
||||
"""UI helpers for MCP Apps resources."""
|
||||
|
||||
from basic_memory.mcp.ui.templates import get_ui_variant, load_html, load_variant_html
|
||||
|
||||
__all__ = ["get_ui_variant", "load_html", "load_variant_html"]
|
||||
@@ -1,176 +0,0 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
<title>Basic Memory Note Preview - MCP UI</title>
|
||||
<style>
|
||||
:root {
|
||||
color-scheme: light;
|
||||
--bg: #fdf2f2;
|
||||
--panel: #ffffff;
|
||||
--ink: #1a1a1d;
|
||||
--muted: #6b6770;
|
||||
--border: #ecd7dd;
|
||||
--accent: #b91c1c;
|
||||
}
|
||||
|
||||
* {
|
||||
box-sizing: border-box;
|
||||
}
|
||||
|
||||
body {
|
||||
margin: 0;
|
||||
padding: 24px;
|
||||
font-family: "IBM Plex Sans", "Space Grotesk", system-ui, sans-serif;
|
||||
background: radial-gradient(circle at top, #fff7f7 0%, #f7ecec 60%, #f0e3e4 100%);
|
||||
color: var(--ink);
|
||||
}
|
||||
|
||||
.header {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
margin-bottom: 12px;
|
||||
}
|
||||
|
||||
.header h1 {
|
||||
margin: 0;
|
||||
font-size: 19px;
|
||||
}
|
||||
|
||||
.badge {
|
||||
font-size: 11px;
|
||||
border-radius: 999px;
|
||||
padding: 4px 10px;
|
||||
border: 1px solid var(--border);
|
||||
background: #fee2e2;
|
||||
color: var(--accent);
|
||||
}
|
||||
|
||||
.subtitle {
|
||||
font-size: 12px;
|
||||
color: var(--muted);
|
||||
margin-bottom: 12px;
|
||||
}
|
||||
|
||||
.panel {
|
||||
background: var(--panel);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: 14px;
|
||||
padding: 16px;
|
||||
box-shadow: 0 14px 30px rgba(15, 23, 42, 0.1);
|
||||
}
|
||||
|
||||
pre {
|
||||
margin: 0;
|
||||
font-family: "IBM Plex Mono", "SFMono-Regular", ui-monospace, monospace;
|
||||
font-size: 12px;
|
||||
line-height: 1.5;
|
||||
white-space: pre-wrap;
|
||||
color: #2a2a2a;
|
||||
}
|
||||
|
||||
.empty {
|
||||
text-align: center;
|
||||
color: var(--muted);
|
||||
font-size: 13px;
|
||||
padding: 18px;
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="header">
|
||||
<h1 id="title">Note Preview</h1>
|
||||
<div class="badge">mcp-ui style</div>
|
||||
</div>
|
||||
<div class="subtitle" id="subtitle">Waiting for note content...</div>
|
||||
|
||||
<div class="panel">
|
||||
<div id="empty" class="empty" style="display: none;">No content available.</div>
|
||||
<pre id="content"></pre>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
(function () {
|
||||
const title = document.getElementById("title");
|
||||
const subtitle = document.getElementById("subtitle");
|
||||
const contentEl = document.getElementById("content");
|
||||
const emptyEl = document.getElementById("empty");
|
||||
let hasData = false;
|
||||
|
||||
function stripFrontmatter(text) {
|
||||
if (!text.startsWith("---")) return text;
|
||||
const end = text.indexOf("---", 3);
|
||||
if (end === -1) return text;
|
||||
return text.slice(end + 3).trim();
|
||||
}
|
||||
|
||||
function parseTitle(text) {
|
||||
const lines = text.split("\n");
|
||||
for (const line of lines) {
|
||||
if (line.startsWith("# ")) return line.replace("# ", "").trim();
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function extractText(toolOutput) {
|
||||
if (!toolOutput) return "";
|
||||
if (typeof toolOutput === "string") return toolOutput;
|
||||
|
||||
const content = toolOutput.content;
|
||||
if (Array.isArray(content)) {
|
||||
const textBlock = content.find(
|
||||
(block) => block && block.type === "text" && typeof block.text === "string"
|
||||
);
|
||||
if (textBlock) return textBlock.text;
|
||||
}
|
||||
|
||||
if (toolOutput.structuredContent) {
|
||||
if (typeof toolOutput.structuredContent === "string") {
|
||||
return toolOutput.structuredContent;
|
||||
}
|
||||
}
|
||||
|
||||
return "";
|
||||
}
|
||||
|
||||
function updateFromRenderData(renderData) {
|
||||
if (!renderData) return;
|
||||
const identifier = renderData.toolInput && renderData.toolInput.identifier;
|
||||
const rawText = extractText(renderData.toolOutput);
|
||||
if (!rawText) {
|
||||
emptyEl.style.display = "block";
|
||||
contentEl.textContent = "";
|
||||
return;
|
||||
}
|
||||
|
||||
emptyEl.style.display = "none";
|
||||
const text = stripFrontmatter(rawText);
|
||||
const previewTitle = parseTitle(text) || identifier || "Note Preview";
|
||||
|
||||
title.textContent = previewTitle;
|
||||
subtitle.textContent = identifier ? `Identifier: ${identifier}` : "Note content";
|
||||
contentEl.textContent = text.slice(0, 1400).trim();
|
||||
}
|
||||
|
||||
function handleMessage(event) {
|
||||
const message = event.data || {};
|
||||
if (message.type === "ui-lifecycle-iframe-render-data") {
|
||||
hasData = true;
|
||||
updateFromRenderData(message.payload && message.payload.renderData);
|
||||
}
|
||||
}
|
||||
|
||||
window.addEventListener("message", handleMessage);
|
||||
window.parent?.postMessage({ type: "ui-lifecycle-iframe-ready" }, "*");
|
||||
|
||||
setTimeout(() => {
|
||||
if (!hasData) {
|
||||
window.parent?.postMessage({ type: "ui-request-render-data" }, "*");
|
||||
}
|
||||
}, 200);
|
||||
})();
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
File diff suppressed because one or more lines are too long
@@ -1,160 +0,0 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
<title>Basic Memory Note Preview</title>
|
||||
<style>
|
||||
:root {
|
||||
color-scheme: light;
|
||||
--bg: #f5f1ea;
|
||||
--panel: #ffffff;
|
||||
--ink: #1d1d1f;
|
||||
--muted: #6d6d6d;
|
||||
--accent: #0f766e;
|
||||
--border: #e1d7cc;
|
||||
}
|
||||
|
||||
* {
|
||||
box-sizing: border-box;
|
||||
}
|
||||
|
||||
body {
|
||||
margin: 0;
|
||||
padding: 24px;
|
||||
font-family: "Space Grotesk", "Segoe UI", system-ui, sans-serif;
|
||||
background: linear-gradient(140deg, #f7f3ed, #efe8dd);
|
||||
color: var(--ink);
|
||||
}
|
||||
|
||||
h1 {
|
||||
margin: 0;
|
||||
font-size: 20px;
|
||||
font-weight: 600;
|
||||
}
|
||||
|
||||
.subtitle {
|
||||
font-size: 12px;
|
||||
color: var(--muted);
|
||||
margin-top: 4px;
|
||||
margin-bottom: 16px;
|
||||
}
|
||||
|
||||
.panel {
|
||||
background: var(--panel);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: 14px;
|
||||
padding: 16px;
|
||||
box-shadow: 0 12px 30px rgba(0, 0, 0, 0.08);
|
||||
}
|
||||
|
||||
pre {
|
||||
margin: 0;
|
||||
font-family: "IBM Plex Mono", "SFMono-Regular", ui-monospace, monospace;
|
||||
font-size: 12px;
|
||||
line-height: 1.5;
|
||||
white-space: pre-wrap;
|
||||
color: #2a2a2a;
|
||||
}
|
||||
|
||||
.empty {
|
||||
text-align: center;
|
||||
color: var(--muted);
|
||||
font-size: 13px;
|
||||
padding: 18px;
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<h1 id="title">Note Preview</h1>
|
||||
<div class="subtitle" id="subtitle">Waiting for note content...</div>
|
||||
|
||||
<div class="panel">
|
||||
<div id="empty" class="empty" style="display: none;">No content available.</div>
|
||||
<pre id="content"></pre>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
(function () {
|
||||
const title = document.getElementById("title");
|
||||
const subtitle = document.getElementById("subtitle");
|
||||
const contentEl = document.getElementById("content");
|
||||
const emptyEl = document.getElementById("empty");
|
||||
let hasData = false;
|
||||
|
||||
function stripFrontmatter(text) {
|
||||
if (!text.startsWith("---")) return text;
|
||||
const end = text.indexOf("---", 3);
|
||||
if (end === -1) return text;
|
||||
return text.slice(end + 3).trim();
|
||||
}
|
||||
|
||||
function parseTitle(text) {
|
||||
const lines = text.split("\n");
|
||||
for (const line of lines) {
|
||||
if (line.startsWith("# ")) return line.replace("# ", "").trim();
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function extractText(toolOutput) {
|
||||
if (!toolOutput) return "";
|
||||
|
||||
if (typeof toolOutput === "string") return toolOutput;
|
||||
|
||||
const content = toolOutput.content;
|
||||
if (Array.isArray(content)) {
|
||||
const textBlock = content.find(
|
||||
(block) => block && block.type === "text" && typeof block.text === "string"
|
||||
);
|
||||
if (textBlock) return textBlock.text;
|
||||
}
|
||||
|
||||
if (toolOutput.structuredContent) {
|
||||
if (typeof toolOutput.structuredContent === "string") {
|
||||
return toolOutput.structuredContent;
|
||||
}
|
||||
}
|
||||
|
||||
return "";
|
||||
}
|
||||
|
||||
function updateFromRenderData(renderData) {
|
||||
if (!renderData) return;
|
||||
const identifier = renderData.toolInput && renderData.toolInput.identifier;
|
||||
const rawText = extractText(renderData.toolOutput);
|
||||
if (!rawText) {
|
||||
emptyEl.style.display = "block";
|
||||
contentEl.textContent = "";
|
||||
return;
|
||||
}
|
||||
|
||||
emptyEl.style.display = "none";
|
||||
const text = stripFrontmatter(rawText);
|
||||
const previewTitle = parseTitle(text) || identifier || "Note Preview";
|
||||
|
||||
title.textContent = previewTitle;
|
||||
subtitle.textContent = identifier ? `Identifier: ${identifier}` : "Note content";
|
||||
contentEl.textContent = text.slice(0, 1400).trim();
|
||||
}
|
||||
|
||||
function handleMessage(event) {
|
||||
const message = event.data || {};
|
||||
if (message.type === "ui-lifecycle-iframe-render-data") {
|
||||
hasData = true;
|
||||
updateFromRenderData(message.payload && message.payload.renderData);
|
||||
}
|
||||
}
|
||||
|
||||
window.addEventListener("message", handleMessage);
|
||||
window.parent?.postMessage({ type: "ui-lifecycle-iframe-ready" }, "*");
|
||||
|
||||
setTimeout(() => {
|
||||
if (!hasData) {
|
||||
window.parent?.postMessage({ type: "ui-request-render-data" }, "*");
|
||||
}
|
||||
}, 200);
|
||||
})();
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -1,254 +0,0 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
<title>Basic Memory MCP UI Search</title>
|
||||
<style>
|
||||
:root {
|
||||
color-scheme: light;
|
||||
--bg: #f6f0f2;
|
||||
--panel: #ffffff;
|
||||
--ink: #1a1a1d;
|
||||
--muted: #6b6770;
|
||||
--accent: #b91c1c;
|
||||
--accent-soft: #fee2e2;
|
||||
--border: #ead7dc;
|
||||
}
|
||||
|
||||
* {
|
||||
box-sizing: border-box;
|
||||
}
|
||||
|
||||
body {
|
||||
margin: 0;
|
||||
padding: 24px;
|
||||
font-family: "IBM Plex Sans", "Space Grotesk", system-ui, sans-serif;
|
||||
background: radial-gradient(circle at top, #fdf7f8 0%, #f7eef1 60%, #f2e7ea 100%);
|
||||
color: var(--ink);
|
||||
}
|
||||
|
||||
.top {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
margin-bottom: 12px;
|
||||
}
|
||||
|
||||
h1 {
|
||||
margin: 0;
|
||||
font-size: 20px;
|
||||
font-weight: 600;
|
||||
}
|
||||
|
||||
.badge {
|
||||
font-size: 11px;
|
||||
border-radius: 999px;
|
||||
border: 1px solid var(--border);
|
||||
background: var(--accent-soft);
|
||||
color: var(--accent);
|
||||
padding: 4px 10px;
|
||||
}
|
||||
|
||||
#subtitle {
|
||||
font-size: 12px;
|
||||
color: var(--muted);
|
||||
margin-bottom: 14px;
|
||||
}
|
||||
|
||||
.panel {
|
||||
background: var(--panel);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: 14px;
|
||||
padding: 16px;
|
||||
box-shadow: 0 16px 40px rgba(15, 23, 42, 0.1);
|
||||
}
|
||||
|
||||
table {
|
||||
width: 100%;
|
||||
border-collapse: collapse;
|
||||
font-size: 13px;
|
||||
}
|
||||
|
||||
th,
|
||||
td {
|
||||
padding: 10px 8px;
|
||||
border-bottom: 1px solid var(--border);
|
||||
text-align: left;
|
||||
vertical-align: top;
|
||||
}
|
||||
|
||||
th {
|
||||
font-size: 11px;
|
||||
text-transform: uppercase;
|
||||
letter-spacing: 0.08em;
|
||||
color: var(--muted);
|
||||
}
|
||||
|
||||
tr:hover {
|
||||
background: #fff5f5;
|
||||
}
|
||||
|
||||
.title {
|
||||
font-weight: 600;
|
||||
}
|
||||
|
||||
.path {
|
||||
font-size: 11px;
|
||||
color: var(--muted);
|
||||
margin-top: 4px;
|
||||
}
|
||||
|
||||
.empty {
|
||||
text-align: center;
|
||||
padding: 18px;
|
||||
color: var(--muted);
|
||||
}
|
||||
|
||||
.footer {
|
||||
margin-top: 12px;
|
||||
font-size: 12px;
|
||||
color: var(--muted);
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="top">
|
||||
<h1>Search Results</h1>
|
||||
<div class="badge">mcp-ui style</div>
|
||||
</div>
|
||||
|
||||
<div id="subtitle">Waiting for results...</div>
|
||||
|
||||
<div class="panel">
|
||||
<div id="table"></div>
|
||||
<div id="empty" class="empty" style="display: none;">No results to show.</div>
|
||||
</div>
|
||||
|
||||
<div id="footer" class="footer"></div>
|
||||
|
||||
<script>
|
||||
(function () {
|
||||
const subtitle = document.getElementById("subtitle");
|
||||
const tableContainer = document.getElementById("table");
|
||||
const emptyState = document.getElementById("empty");
|
||||
const footer = document.getElementById("footer");
|
||||
let hasData = false;
|
||||
|
||||
function safeJsonParse(text) {
|
||||
try {
|
||||
return JSON.parse(text);
|
||||
} catch (error) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function extractStructured(toolOutput) {
|
||||
if (!toolOutput) return null;
|
||||
const structured =
|
||||
toolOutput.structuredContent ||
|
||||
toolOutput.structured_content ||
|
||||
toolOutput.structured ||
|
||||
null;
|
||||
if (structured) return structured;
|
||||
|
||||
const content = toolOutput.content;
|
||||
if (Array.isArray(content)) {
|
||||
const textBlock = content.find(
|
||||
(block) => block && block.type === "text" && typeof block.text === "string"
|
||||
);
|
||||
if (textBlock) {
|
||||
const parsed = safeJsonParse(textBlock.text);
|
||||
return parsed || { text: textBlock.text };
|
||||
}
|
||||
}
|
||||
|
||||
if (typeof toolOutput === "string") {
|
||||
return safeJsonParse(toolOutput) || { text: toolOutput };
|
||||
}
|
||||
|
||||
return toolOutput;
|
||||
}
|
||||
|
||||
function getResults(structured) {
|
||||
if (!structured) return [];
|
||||
if (Array.isArray(structured.results)) return structured.results;
|
||||
if (structured.result && Array.isArray(structured.result.results)) {
|
||||
return structured.result.results;
|
||||
}
|
||||
return [];
|
||||
}
|
||||
|
||||
function renderTable(results) {
|
||||
if (!results.length) {
|
||||
tableContainer.innerHTML = "";
|
||||
emptyState.style.display = "block";
|
||||
return;
|
||||
}
|
||||
|
||||
emptyState.style.display = "none";
|
||||
|
||||
const rows = results
|
||||
.map((result) => {
|
||||
return `
|
||||
<tr>
|
||||
<td>
|
||||
<div class=\"title\">${result.title || "Untitled"}</div>
|
||||
<div class=\"path\">${result.permalink || result.file_path || ""}</div>
|
||||
</td>
|
||||
<td>${result.type || ""}</td>
|
||||
<td>${Number(result.score || 0).toFixed(2)}</td>
|
||||
</tr>
|
||||
`;
|
||||
})
|
||||
.join("");
|
||||
|
||||
tableContainer.innerHTML = `
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Title</th>
|
||||
<th>Type</th>
|
||||
<th>Score</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
${rows}
|
||||
</tbody>
|
||||
</table>
|
||||
`;
|
||||
}
|
||||
|
||||
function updateFromRenderData(renderData) {
|
||||
if (!renderData) return;
|
||||
|
||||
const query = renderData.toolInput && renderData.toolInput.query;
|
||||
const structured = extractStructured(renderData.toolOutput);
|
||||
const results = getResults(structured);
|
||||
|
||||
subtitle.textContent = query ? `Query: ${query}` : "Search results";
|
||||
footer.textContent = `Results: ${results.length}`;
|
||||
|
||||
renderTable(results);
|
||||
}
|
||||
|
||||
function handleMessage(event) {
|
||||
const message = event.data || {};
|
||||
if (message.type === "ui-lifecycle-iframe-render-data") {
|
||||
hasData = true;
|
||||
updateFromRenderData(message.payload && message.payload.renderData);
|
||||
}
|
||||
}
|
||||
|
||||
window.addEventListener("message", handleMessage);
|
||||
window.parent?.postMessage({ type: "ui-lifecycle-iframe-ready" }, "*");
|
||||
|
||||
setTimeout(() => {
|
||||
if (!hasData) {
|
||||
window.parent?.postMessage({ type: "ui-request-render-data" }, "*");
|
||||
}
|
||||
}, 200);
|
||||
})();
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
File diff suppressed because one or more lines are too long
@@ -1,280 +0,0 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
<title>Basic Memory Search Results</title>
|
||||
<style>
|
||||
:root {
|
||||
color-scheme: light;
|
||||
--bg: #f7f4ef;
|
||||
--panel: #ffffff;
|
||||
--ink: #1f1f1f;
|
||||
--muted: #5b5b5b;
|
||||
--accent: #0f766e;
|
||||
--accent-2: #0a4f4b;
|
||||
--border: #e3dbd2;
|
||||
--row: #fbf9f5;
|
||||
--row-hover: #f2ece3;
|
||||
--tag: #f3efe7;
|
||||
}
|
||||
|
||||
* {
|
||||
box-sizing: border-box;
|
||||
}
|
||||
|
||||
body {
|
||||
margin: 0;
|
||||
padding: 24px;
|
||||
font-family: "Space Grotesk", "Segoe UI", system-ui, sans-serif;
|
||||
background: radial-gradient(circle at top, #faf6ef 0%, #f2ece2 45%, #efe7da 100%);
|
||||
color: var(--ink);
|
||||
}
|
||||
|
||||
header {
|
||||
display: flex;
|
||||
align-items: baseline;
|
||||
justify-content: space-between;
|
||||
gap: 12px;
|
||||
margin-bottom: 16px;
|
||||
}
|
||||
|
||||
h1 {
|
||||
margin: 0;
|
||||
font-size: 20px;
|
||||
font-weight: 600;
|
||||
letter-spacing: 0.3px;
|
||||
}
|
||||
|
||||
#subtitle {
|
||||
font-size: 13px;
|
||||
color: var(--muted);
|
||||
}
|
||||
|
||||
.panel {
|
||||
background: var(--panel);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: 14px;
|
||||
padding: 16px;
|
||||
box-shadow: 0 10px 30px rgba(0, 0, 0, 0.08);
|
||||
}
|
||||
|
||||
table {
|
||||
width: 100%;
|
||||
border-collapse: collapse;
|
||||
font-size: 14px;
|
||||
}
|
||||
|
||||
th,
|
||||
td {
|
||||
padding: 12px 10px;
|
||||
border-bottom: 1px solid var(--border);
|
||||
text-align: left;
|
||||
vertical-align: top;
|
||||
}
|
||||
|
||||
th {
|
||||
font-size: 12px;
|
||||
text-transform: uppercase;
|
||||
letter-spacing: 0.08em;
|
||||
color: var(--muted);
|
||||
}
|
||||
|
||||
tr:nth-child(even) {
|
||||
background: var(--row);
|
||||
}
|
||||
|
||||
tr:hover {
|
||||
background: var(--row-hover);
|
||||
}
|
||||
|
||||
.title {
|
||||
font-weight: 600;
|
||||
color: var(--accent-2);
|
||||
}
|
||||
|
||||
.meta {
|
||||
font-size: 12px;
|
||||
color: var(--muted);
|
||||
}
|
||||
|
||||
.tags {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
gap: 6px;
|
||||
}
|
||||
|
||||
.tag {
|
||||
font-size: 11px;
|
||||
padding: 2px 8px;
|
||||
background: var(--tag);
|
||||
border-radius: 999px;
|
||||
border: 1px solid var(--border);
|
||||
}
|
||||
|
||||
.empty {
|
||||
padding: 24px;
|
||||
text-align: center;
|
||||
color: var(--muted);
|
||||
font-size: 14px;
|
||||
}
|
||||
|
||||
.footer {
|
||||
margin-top: 12px;
|
||||
font-size: 12px;
|
||||
color: var(--muted);
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<header>
|
||||
<h1>Search Results</h1>
|
||||
<div id="subtitle">Waiting for results...</div>
|
||||
</header>
|
||||
|
||||
<div class="panel">
|
||||
<div id="table"></div>
|
||||
<div id="empty" class="empty" style="display: none;">No results to show.</div>
|
||||
</div>
|
||||
|
||||
<div class="footer" id="footer"></div>
|
||||
|
||||
<script>
|
||||
(function () {
|
||||
const subtitle = document.getElementById("subtitle");
|
||||
const tableContainer = document.getElementById("table");
|
||||
const emptyState = document.getElementById("empty");
|
||||
const footer = document.getElementById("footer");
|
||||
let hasData = false;
|
||||
|
||||
function safeJsonParse(text) {
|
||||
try {
|
||||
return JSON.parse(text);
|
||||
} catch (error) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function extractStructured(toolOutput) {
|
||||
if (!toolOutput) return null;
|
||||
|
||||
const structured =
|
||||
toolOutput.structuredContent ||
|
||||
toolOutput.structured_content ||
|
||||
toolOutput.structured ||
|
||||
null;
|
||||
if (structured) return structured;
|
||||
|
||||
const content = toolOutput.content;
|
||||
if (Array.isArray(content)) {
|
||||
const textBlock = content.find(
|
||||
(block) => block && block.type === "text" && typeof block.text === "string"
|
||||
);
|
||||
if (textBlock) {
|
||||
const parsed = safeJsonParse(textBlock.text);
|
||||
return parsed || { text: textBlock.text };
|
||||
}
|
||||
}
|
||||
|
||||
if (typeof toolOutput === "string") {
|
||||
return safeJsonParse(toolOutput) || { text: toolOutput };
|
||||
}
|
||||
|
||||
return toolOutput;
|
||||
}
|
||||
|
||||
function getResults(structured) {
|
||||
if (!structured) return [];
|
||||
if (Array.isArray(structured.results)) return structured.results;
|
||||
if (structured.result && Array.isArray(structured.result.results)) {
|
||||
return structured.result.results;
|
||||
}
|
||||
return [];
|
||||
}
|
||||
|
||||
function renderTable(results) {
|
||||
if (!results.length) {
|
||||
tableContainer.innerHTML = "";
|
||||
emptyState.style.display = "block";
|
||||
return;
|
||||
}
|
||||
|
||||
emptyState.style.display = "none";
|
||||
|
||||
const rows = results
|
||||
.map((result) => {
|
||||
const tags =
|
||||
(result.metadata && result.metadata.tags) ||
|
||||
result.tags ||
|
||||
result.metadata?.tag ||
|
||||
[];
|
||||
const tagsMarkup = Array.isArray(tags)
|
||||
? tags.map((tag) => `<span class=\"tag\">${tag}</span>`).join("")
|
||||
: "";
|
||||
|
||||
return `
|
||||
<tr>
|
||||
<td>
|
||||
<div class=\"title\">${result.title || "Untitled"}</div>
|
||||
<div class=\"meta\">${result.permalink || result.file_path || ""}</div>
|
||||
</td>
|
||||
<td>${result.type || ""}</td>
|
||||
<td>${Number(result.score || 0).toFixed(2)}</td>
|
||||
<td>
|
||||
<div class=\"tags\">${tagsMarkup}</div>
|
||||
</td>
|
||||
</tr>
|
||||
`;
|
||||
})
|
||||
.join("");
|
||||
|
||||
tableContainer.innerHTML = `
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Title</th>
|
||||
<th>Type</th>
|
||||
<th>Score</th>
|
||||
<th>Tags</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
${rows}
|
||||
</tbody>
|
||||
</table>
|
||||
`;
|
||||
}
|
||||
|
||||
function updateFromRenderData(renderData) {
|
||||
if (!renderData) return;
|
||||
|
||||
const query = renderData.toolInput && renderData.toolInput.query;
|
||||
const structured = extractStructured(renderData.toolOutput);
|
||||
const results = getResults(structured);
|
||||
|
||||
subtitle.textContent = query ? `Query: ${query}` : "Search results";
|
||||
footer.textContent = `Results: ${results.length}`;
|
||||
|
||||
renderTable(results);
|
||||
}
|
||||
|
||||
function handleMessage(event) {
|
||||
const message = event.data || {};
|
||||
if (message.type === "ui-lifecycle-iframe-render-data") {
|
||||
hasData = true;
|
||||
updateFromRenderData(message.payload && message.payload.renderData);
|
||||
}
|
||||
}
|
||||
|
||||
window.addEventListener("message", handleMessage);
|
||||
window.parent?.postMessage({ type: "ui-lifecycle-iframe-ready" }, "*");
|
||||
|
||||
setTimeout(() => {
|
||||
if (!hasData) {
|
||||
window.parent?.postMessage({ type: "ui-request-render-data" }, "*");
|
||||
}
|
||||
}, 200);
|
||||
})();
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -1,56 +0,0 @@
|
||||
"""Helpers for embedded MCP-UI resources (Python SDK)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
from typing import Any
|
||||
|
||||
from basic_memory.mcp.ui import load_html
|
||||
|
||||
try: # Optional dependency for MCP-UI embedded resources
|
||||
mcp_ui_server = importlib.import_module("mcp_ui_server")
|
||||
UIMetadataKey = mcp_ui_server.UIMetadataKey
|
||||
create_ui_resource = mcp_ui_server.create_ui_resource
|
||||
except ImportError: # pragma: no cover - handled by callers
|
||||
UIMetadataKey = None
|
||||
create_ui_resource = None
|
||||
|
||||
|
||||
class MissingMCPUIServerError(RuntimeError):
|
||||
"""Raised when the MCP-UI server SDK is not available."""
|
||||
|
||||
|
||||
def _ensure_sdk() -> tuple[Any, Any]:
|
||||
if create_ui_resource is None or UIMetadataKey is None:
|
||||
raise MissingMCPUIServerError(
|
||||
"mcp-ui-server is not installed. "
|
||||
"Install it with `uv pip install -e /Users/phernandez/dev/mcp-ui/sdks/python/server` "
|
||||
"or `pip install mcp-ui-server`."
|
||||
)
|
||||
return create_ui_resource, UIMetadataKey
|
||||
|
||||
|
||||
def build_embedded_ui_resource(
|
||||
*,
|
||||
uri: str,
|
||||
html_filename: str,
|
||||
render_data: dict[str, Any],
|
||||
preferred_frame_size: list[str],
|
||||
metadata: dict[str, Any] | None = None,
|
||||
):
|
||||
"""Create an embedded UI resource using the MCP-UI Python SDK."""
|
||||
create_resource, metadata_keys = _ensure_sdk()
|
||||
html = load_html(html_filename)
|
||||
|
||||
return create_resource(
|
||||
{
|
||||
"uri": uri,
|
||||
"content": {"type": "rawHtml", "htmlString": html},
|
||||
"encoding": "text",
|
||||
"uiMetadata": {
|
||||
metadata_keys.PREFERRED_FRAME_SIZE: preferred_frame_size,
|
||||
metadata_keys.INITIAL_RENDER_DATA: render_data,
|
||||
},
|
||||
"metadata": metadata or {},
|
||||
}
|
||||
)
|
||||
@@ -1,27 +0,0 @@
|
||||
"""Helpers for serving MCP UI HTML resources."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
DEFAULT_VARIANT = "vanilla"
|
||||
SUPPORTED_VARIANTS = {"vanilla", "tool-ui", "mcp-ui"}
|
||||
|
||||
|
||||
def get_ui_variant() -> str:
|
||||
"""Return the active UI variant from environment settings."""
|
||||
value = os.getenv("BASIC_MEMORY_MCP_UI_VARIANT", DEFAULT_VARIANT).strip().lower()
|
||||
return value if value in SUPPORTED_VARIANTS else DEFAULT_VARIANT
|
||||
|
||||
|
||||
def load_html(filename: str) -> str:
|
||||
"""Load a UI HTML template from disk."""
|
||||
path = Path(__file__).parent / "html" / filename
|
||||
return path.read_text(encoding="utf-8")
|
||||
|
||||
|
||||
def load_variant_html(base_name: str) -> str:
|
||||
"""Load a UI template for the current variant."""
|
||||
variant = get_ui_variant()
|
||||
return load_html(f"{base_name}-{variant}.html")
|
||||
@@ -92,37 +92,3 @@ CREATE VIRTUAL TABLE IF NOT EXISTS search_index USING fts5(
|
||||
prefix='1,2,3,4' -- Support longer prefixes for paths
|
||||
);
|
||||
""")
|
||||
|
||||
# Local semantic chunk metadata table for SQLite.
|
||||
# Embedding vectors live in sqlite-vec virtual table keyed by this table rowid.
|
||||
CREATE_SQLITE_SEARCH_VECTOR_CHUNKS = DDL("""
|
||||
CREATE TABLE IF NOT EXISTS search_vector_chunks (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
entity_id INTEGER NOT NULL,
|
||||
project_id INTEGER NOT NULL,
|
||||
chunk_key TEXT NOT NULL,
|
||||
chunk_text TEXT NOT NULL,
|
||||
source_hash TEXT NOT NULL,
|
||||
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
)
|
||||
""")
|
||||
|
||||
CREATE_SQLITE_SEARCH_VECTOR_CHUNKS_PROJECT_ENTITY = DDL("""
|
||||
CREATE INDEX IF NOT EXISTS idx_search_vector_chunks_project_entity
|
||||
ON search_vector_chunks (project_id, entity_id)
|
||||
""")
|
||||
|
||||
CREATE_SQLITE_SEARCH_VECTOR_CHUNKS_UNIQUE = DDL("""
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS uix_search_vector_chunks_entity_key
|
||||
ON search_vector_chunks (project_id, entity_id, chunk_key)
|
||||
""")
|
||||
|
||||
|
||||
def create_sqlite_search_vector_embeddings(dimensions: int) -> DDL:
|
||||
"""Build sqlite-vec virtual table DDL for the configured embedding dimension."""
|
||||
return DDL(
|
||||
f"""
|
||||
CREATE VIRTUAL TABLE IF NOT EXISTS search_vector_embeddings
|
||||
USING vec0(embedding float[{dimensions}])
|
||||
"""
|
||||
)
|
||||
|
||||
@@ -3,13 +3,12 @@
|
||||
This module provides a single canonical implementation of project resolution
|
||||
logic, eliminating duplicated decision trees across the codebase.
|
||||
|
||||
The resolution follows a unified linear priority chain that works
|
||||
identically in both local and cloud modes:
|
||||
The resolution follows a three-tier hierarchy:
|
||||
1. Constrained mode: BASIC_MEMORY_MCP_PROJECT env var (highest priority)
|
||||
2. Explicit parameter: Project passed directly to operation
|
||||
3. Default project: Used when default_project_mode=true (lowest priority)
|
||||
|
||||
1. ENV_CONSTRAINT: BASIC_MEMORY_MCP_PROJECT env var (highest priority)
|
||||
2. EXPLICIT: Project passed directly to operation
|
||||
3. DEFAULT: Default project when default_project_mode=true
|
||||
4. Fallback: cloud → CLOUD_DISCOVERY or ValueError; local → NONE
|
||||
In cloud mode, project is required unless discovery mode is explicitly allowed.
|
||||
"""
|
||||
|
||||
import os
|
||||
@@ -69,7 +68,7 @@ class ProjectResolver:
|
||||
used by MCP tools, API routes, and CLI commands.
|
||||
|
||||
Args:
|
||||
cloud_mode: Whether running in cloud mode
|
||||
cloud_mode: Whether running in cloud mode (project required)
|
||||
default_project_mode: Whether to use default project when not specified
|
||||
default_project: The default project name
|
||||
constrained_project: Optional env-constrained project override
|
||||
@@ -111,13 +110,13 @@ class ProjectResolver:
|
||||
project: Optional[str] = None,
|
||||
allow_discovery: bool = False,
|
||||
) -> ResolvedProject:
|
||||
"""Resolve project using a unified linear priority chain.
|
||||
"""Resolve project using the three-tier hierarchy.
|
||||
|
||||
The same resolution order applies in both local and cloud modes:
|
||||
1. ENV_CONSTRAINT — BASIC_MEMORY_MCP_PROJECT env var (highest priority)
|
||||
2. EXPLICIT — project parameter passed directly
|
||||
3. DEFAULT — default project when default_project_mode=true
|
||||
4. Fallback — cloud: CLOUD_DISCOVERY or ValueError; local: NONE
|
||||
Resolution order:
|
||||
1. Cloud mode check (project required unless discovery allowed)
|
||||
2. Constrained project from env var (highest priority in local mode)
|
||||
3. Explicit project parameter
|
||||
4. Default project if default_project_mode=true
|
||||
|
||||
Args:
|
||||
project: Optional explicit project parameter
|
||||
@@ -128,10 +127,31 @@ class ProjectResolver:
|
||||
ResolvedProject with project name, resolution mode, and reason
|
||||
|
||||
Raises:
|
||||
ValueError: If in cloud mode and no project could be resolved
|
||||
(unless allow_discovery=True)
|
||||
ValueError: If in cloud mode and no project specified (unless discovery allowed)
|
||||
"""
|
||||
# --- Priority 1: ENV constraint overrides everything ---
|
||||
# --- Cloud Mode Handling ---
|
||||
# In cloud mode, project is required unless discovery is explicitly allowed
|
||||
if self.cloud_mode:
|
||||
if project:
|
||||
logger.debug(f"Cloud mode: using explicit project '{project}'")
|
||||
return ResolvedProject(
|
||||
project=project,
|
||||
mode=ResolutionMode.CLOUD_EXPLICIT,
|
||||
reason=f"Explicit project in cloud mode: {project}",
|
||||
)
|
||||
elif allow_discovery:
|
||||
logger.debug("Cloud mode: discovery mode allowed, no project required")
|
||||
return ResolvedProject(
|
||||
project=None,
|
||||
mode=ResolutionMode.CLOUD_DISCOVERY,
|
||||
reason="Discovery mode enabled in cloud",
|
||||
)
|
||||
else:
|
||||
raise ValueError("No project specified. Project is required for cloud mode.")
|
||||
|
||||
# --- Local Mode: Three-Tier Hierarchy ---
|
||||
|
||||
# Priority 1: CLI constraint overrides everything
|
||||
if self.constrained_project:
|
||||
logger.debug(f"Using CLI constrained project: {self.constrained_project}")
|
||||
return ResolvedProject(
|
||||
@@ -140,17 +160,16 @@ class ProjectResolver:
|
||||
reason=f"Environment constraint: BASIC_MEMORY_MCP_PROJECT={self.constrained_project}",
|
||||
)
|
||||
|
||||
# --- Priority 2: Explicit project parameter ---
|
||||
# Priority 2: Explicit project parameter
|
||||
if project:
|
||||
mode = ResolutionMode.CLOUD_EXPLICIT if self.cloud_mode else ResolutionMode.EXPLICIT
|
||||
logger.debug(f"Using explicit project parameter: {project}")
|
||||
return ResolvedProject(
|
||||
project=project,
|
||||
mode=mode,
|
||||
mode=ResolutionMode.EXPLICIT,
|
||||
reason=f"Explicit parameter: {project}",
|
||||
)
|
||||
|
||||
# --- Priority 3: Default project mode ---
|
||||
# Priority 3: Default project mode
|
||||
if self.default_project_mode and self.default_project:
|
||||
logger.debug(f"Using default project from config: {self.default_project}")
|
||||
return ResolvedProject(
|
||||
@@ -159,23 +178,12 @@ class ProjectResolver:
|
||||
reason=f"Default project mode: {self.default_project}",
|
||||
)
|
||||
|
||||
# --- Fallback: mode-dependent behavior ---
|
||||
if self.cloud_mode:
|
||||
if allow_discovery:
|
||||
logger.debug("Cloud mode: discovery mode allowed, no project required")
|
||||
return ResolvedProject(
|
||||
project=None,
|
||||
mode=ResolutionMode.CLOUD_DISCOVERY,
|
||||
reason="Discovery mode enabled in cloud",
|
||||
)
|
||||
raise ValueError("No project specified. Project is required for cloud mode.")
|
||||
|
||||
# Local mode: no resolution possible
|
||||
# No resolution possible
|
||||
logger.debug("No project resolution possible")
|
||||
return ResolvedProject(
|
||||
project=None,
|
||||
mode=ResolutionMode.NONE,
|
||||
reason="No project specified and no default project configured",
|
||||
reason="No project specified and default_project_mode is disabled",
|
||||
)
|
||||
|
||||
def require_project(
|
||||
|
||||
@@ -1,18 +0,0 @@
|
||||
"""Embedding provider protocol for pluggable semantic backends."""
|
||||
|
||||
from typing import Protocol
|
||||
|
||||
|
||||
class EmbeddingProvider(Protocol):
|
||||
"""Contract for semantic embedding providers."""
|
||||
|
||||
model_name: str
|
||||
dimensions: int
|
||||
|
||||
async def embed_query(self, text: str) -> list[float]:
|
||||
"""Embed a single query string."""
|
||||
...
|
||||
|
||||
async def embed_documents(self, texts: list[str]) -> list[list[float]]:
|
||||
"""Embed a list of document chunks."""
|
||||
...
|
||||
@@ -1,41 +0,0 @@
|
||||
"""Factory for creating configured semantic embedding providers."""
|
||||
|
||||
from basic_memory.config import BasicMemoryConfig
|
||||
from basic_memory.repository.embedding_provider import EmbeddingProvider
|
||||
|
||||
|
||||
def create_embedding_provider(app_config: BasicMemoryConfig) -> EmbeddingProvider:
|
||||
"""Create an embedding provider based on semantic config.
|
||||
|
||||
When semantic_embedding_dimensions is set in config, it overrides
|
||||
the provider's default dimensions (384 for FastEmbed, 1536 for OpenAI).
|
||||
"""
|
||||
provider_name = app_config.semantic_embedding_provider.strip().lower()
|
||||
extra_kwargs: dict = {}
|
||||
if app_config.semantic_embedding_dimensions is not None:
|
||||
extra_kwargs["dimensions"] = app_config.semantic_embedding_dimensions
|
||||
|
||||
if provider_name == "fastembed":
|
||||
# Deferred import: fastembed (and its onnxruntime dep) may not be installed
|
||||
from basic_memory.repository.fastembed_provider import FastEmbedEmbeddingProvider
|
||||
|
||||
return FastEmbedEmbeddingProvider(
|
||||
model_name=app_config.semantic_embedding_model,
|
||||
batch_size=app_config.semantic_embedding_batch_size,
|
||||
**extra_kwargs,
|
||||
)
|
||||
|
||||
if provider_name == "openai":
|
||||
# Deferred import: openai may not be installed
|
||||
from basic_memory.repository.openai_provider import OpenAIEmbeddingProvider
|
||||
|
||||
model_name = app_config.semantic_embedding_model or "text-embedding-3-small"
|
||||
if model_name == "bge-small-en-v1.5":
|
||||
model_name = "text-embedding-3-small"
|
||||
return OpenAIEmbeddingProvider(
|
||||
model_name=model_name,
|
||||
batch_size=app_config.semantic_embedding_batch_size,
|
||||
**extra_kwargs,
|
||||
)
|
||||
|
||||
raise ValueError(f"Unsupported semantic embedding provider: {provider_name}")
|
||||
@@ -1,83 +0,0 @@
|
||||
"""FastEmbed-based local embedding provider."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from basic_memory.repository.embedding_provider import EmbeddingProvider
|
||||
from basic_memory.repository.semantic_errors import SemanticDependenciesMissingError
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from fastembed import TextEmbedding # pragma: no cover
|
||||
|
||||
|
||||
class FastEmbedEmbeddingProvider(EmbeddingProvider):
|
||||
"""Local ONNX embedding provider backed by FastEmbed."""
|
||||
|
||||
_MODEL_ALIASES = {
|
||||
"bge-small-en-v1.5": "BAAI/bge-small-en-v1.5",
|
||||
}
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
model_name: str = "bge-small-en-v1.5",
|
||||
*,
|
||||
batch_size: int = 64,
|
||||
dimensions: int = 384,
|
||||
) -> None:
|
||||
self.model_name = model_name
|
||||
self.dimensions = dimensions
|
||||
self.batch_size = batch_size
|
||||
self._model: TextEmbedding | None = None
|
||||
self._model_lock = asyncio.Lock()
|
||||
|
||||
async def _load_model(self) -> "TextEmbedding":
|
||||
if self._model is not None:
|
||||
return self._model
|
||||
|
||||
async with self._model_lock:
|
||||
if self._model is not None:
|
||||
return self._model
|
||||
|
||||
def _create_model() -> "TextEmbedding":
|
||||
try:
|
||||
from fastembed import TextEmbedding
|
||||
except (
|
||||
ImportError
|
||||
) as exc: # pragma: no cover - exercised via tests with monkeypatch
|
||||
raise SemanticDependenciesMissingError(
|
||||
"fastembed package is missing. "
|
||||
"Install semantic extras: pip install 'basic-memory[semantic]'"
|
||||
) from exc
|
||||
resolved_model_name = self._MODEL_ALIASES.get(self.model_name, self.model_name)
|
||||
return TextEmbedding(model_name=resolved_model_name)
|
||||
|
||||
self._model = await asyncio.to_thread(_create_model)
|
||||
return self._model
|
||||
|
||||
async def embed_documents(self, texts: list[str]) -> list[list[float]]:
|
||||
if not texts:
|
||||
return []
|
||||
|
||||
model = await self._load_model()
|
||||
|
||||
def _embed_batch() -> list[list[float]]:
|
||||
vectors = list(model.embed(texts, batch_size=self.batch_size))
|
||||
normalized: list[list[float]] = []
|
||||
for vector in vectors:
|
||||
values = vector.tolist() if hasattr(vector, "tolist") else vector
|
||||
normalized.append([float(value) for value in values])
|
||||
return normalized
|
||||
|
||||
vectors = await asyncio.to_thread(_embed_batch)
|
||||
if vectors and len(vectors[0]) != self.dimensions:
|
||||
raise RuntimeError(
|
||||
f"Embedding model returned {len(vectors[0])}-dimensional vectors "
|
||||
f"but provider was configured for {self.dimensions} dimensions."
|
||||
)
|
||||
return vectors
|
||||
|
||||
async def embed_query(self, text: str) -> list[float]:
|
||||
vectors = await self.embed_documents([text])
|
||||
return vectors[0] if vectors else [0.0] * self.dimensions
|
||||
@@ -1,98 +0,0 @@
|
||||
"""OpenAI-based embedding provider for cloud or API-backed semantic indexing."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
from typing import Any
|
||||
|
||||
from basic_memory.repository.embedding_provider import EmbeddingProvider
|
||||
from basic_memory.repository.semantic_errors import SemanticDependenciesMissingError
|
||||
|
||||
|
||||
class OpenAIEmbeddingProvider(EmbeddingProvider):
|
||||
"""Embedding provider backed by OpenAI's embeddings API."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
model_name: str = "text-embedding-3-small",
|
||||
*,
|
||||
batch_size: int = 64,
|
||||
dimensions: int = 1536,
|
||||
api_key: str | None = None,
|
||||
base_url: str | None = None,
|
||||
timeout: float = 30.0,
|
||||
) -> None:
|
||||
self.model_name = model_name
|
||||
self.dimensions = dimensions
|
||||
self.batch_size = batch_size
|
||||
self._api_key = api_key
|
||||
self._base_url = base_url
|
||||
self._timeout = timeout
|
||||
self._client: Any | None = None
|
||||
self._client_lock = asyncio.Lock()
|
||||
|
||||
async def _get_client(self) -> Any:
|
||||
if self._client is not None:
|
||||
return self._client
|
||||
|
||||
async with self._client_lock:
|
||||
if self._client is not None:
|
||||
return self._client
|
||||
|
||||
try:
|
||||
from openai import AsyncOpenAI
|
||||
except ImportError as exc: # pragma: no cover - covered via monkeypatch tests
|
||||
raise SemanticDependenciesMissingError(
|
||||
"OpenAI dependency is missing. "
|
||||
"Install semantic extras: pip install 'basic-memory[semantic]'"
|
||||
) from exc
|
||||
|
||||
api_key = self._api_key or os.getenv("OPENAI_API_KEY")
|
||||
if not api_key:
|
||||
raise SemanticDependenciesMissingError(
|
||||
"OpenAI embedding provider requires OPENAI_API_KEY."
|
||||
)
|
||||
|
||||
self._client = AsyncOpenAI(
|
||||
api_key=api_key,
|
||||
base_url=self._base_url,
|
||||
timeout=self._timeout,
|
||||
)
|
||||
return self._client
|
||||
|
||||
async def embed_documents(self, texts: list[str]) -> list[list[float]]:
|
||||
if not texts:
|
||||
return []
|
||||
|
||||
client = await self._get_client()
|
||||
all_vectors: list[list[float]] = []
|
||||
|
||||
for start in range(0, len(texts), self.batch_size):
|
||||
batch = texts[start : start + self.batch_size]
|
||||
response = await client.embeddings.create(
|
||||
model=self.model_name,
|
||||
input=batch,
|
||||
)
|
||||
vectors_by_index: dict[int, list[float]] = {
|
||||
int(item.index): [float(value) for value in item.embedding]
|
||||
for item in response.data
|
||||
}
|
||||
for index in range(len(batch)):
|
||||
vector = vectors_by_index.get(index)
|
||||
if vector is None:
|
||||
raise RuntimeError(
|
||||
"OpenAI embedding response is missing expected vector index."
|
||||
)
|
||||
all_vectors.append(vector)
|
||||
|
||||
if all_vectors and len(all_vectors[0]) != self.dimensions:
|
||||
raise RuntimeError(
|
||||
f"Embedding model returned {len(all_vectors[0])}-dimensional vectors "
|
||||
f"but provider was configured for {self.dimensions} dimensions."
|
||||
)
|
||||
return all_vectors
|
||||
|
||||
async def embed_query(self, text: str) -> list[float]:
|
||||
vectors = await self.embed_documents([text])
|
||||
return vectors[0] if vectors else [0.0] * self.dimensions
|
||||
@@ -1,27 +1,31 @@
|
||||
"""PostgreSQL tsvector-based search repository implementation."""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import re
|
||||
from datetime import datetime
|
||||
from typing import List, Optional
|
||||
|
||||
|
||||
from loguru import logger
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from basic_memory import db
|
||||
from basic_memory.config import BasicMemoryConfig, ConfigManager
|
||||
from basic_memory.repository.embedding_provider import EmbeddingProvider
|
||||
from basic_memory.repository.embedding_provider_factory import create_embedding_provider
|
||||
from basic_memory.repository.search_index_row import SearchIndexRow
|
||||
from basic_memory.repository.search_repository_base import SearchRepositoryBase
|
||||
from basic_memory.repository.metadata_filters import (
|
||||
parse_metadata_filters,
|
||||
build_postgres_json_path,
|
||||
)
|
||||
from basic_memory.repository.semantic_errors import SemanticDependenciesMissingError
|
||||
from basic_memory.schemas.search import SearchItemType, SearchRetrievalMode
|
||||
from basic_memory.schemas.search import SearchItemType
|
||||
|
||||
|
||||
def _strip_nul_from_row(row_data: dict) -> dict:
|
||||
"""Strip NUL bytes from all string values in a row dict.
|
||||
|
||||
Secondary defense: PostgreSQL text columns cannot store \\x00.
|
||||
Primary sanitization happens in SearchService.index_entity_markdown().
|
||||
"""
|
||||
return {k: v.replace("\x00", "") if isinstance(v, str) else v for k, v in row_data.items()}
|
||||
|
||||
|
||||
class PostgresSearchRepository(SearchRepositoryBase):
|
||||
@@ -40,27 +44,6 @@ class PostgresSearchRepository(SearchRepositoryBase):
|
||||
permalinks per project.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
session_maker,
|
||||
project_id: int,
|
||||
app_config: BasicMemoryConfig | None = None,
|
||||
embedding_provider: EmbeddingProvider | None = None,
|
||||
):
|
||||
super().__init__(session_maker, project_id)
|
||||
self._app_config = app_config or ConfigManager().config
|
||||
self._semantic_enabled = self._app_config.semantic_search_enabled
|
||||
self._semantic_vector_k = self._app_config.semantic_vector_k
|
||||
self._embedding_provider = embedding_provider
|
||||
self._vector_dimensions = 384
|
||||
self._vector_tables_initialized = False
|
||||
self._vector_tables_lock = asyncio.Lock()
|
||||
|
||||
if self._semantic_enabled and self._embedding_provider is None:
|
||||
self._embedding_provider = create_embedding_provider(self._app_config)
|
||||
if self._embedding_provider is not None:
|
||||
self._vector_dimensions = self._embedding_provider.dimensions
|
||||
|
||||
async def init_search_index(self):
|
||||
"""Create Postgres table with tsvector column and GIN indexes.
|
||||
|
||||
@@ -133,10 +116,6 @@ class PostgresSearchRepository(SearchRepositoryBase):
|
||||
logger.debug(f"indexed row {search_index_row}")
|
||||
await session.commit()
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# tsquery preparation (backend-specific)
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _prepare_search_term(self, term: str, is_prefix: bool = True) -> str:
|
||||
"""Prepare a search term for tsquery format.
|
||||
|
||||
@@ -240,342 +219,6 @@ class PostgresSearchRepository(SearchRepositoryBase):
|
||||
else:
|
||||
return cleaned_term
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# pgvector utility
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
@staticmethod
|
||||
def _format_pgvector_literal(vector: list[float]) -> str:
|
||||
if not vector:
|
||||
return "[]"
|
||||
values = ",".join(f"{float(value):.12g}" for value in vector)
|
||||
return f"[{values}]"
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Abstract hook implementations (vector/semantic, Postgres-specific)
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
async def _ensure_vector_tables(self) -> None:
|
||||
self._assert_semantic_available()
|
||||
if self._vector_tables_initialized:
|
||||
return
|
||||
|
||||
async with self._vector_tables_lock:
|
||||
if self._vector_tables_initialized:
|
||||
return
|
||||
|
||||
async with db.scoped_session(self.session_maker) as session:
|
||||
try:
|
||||
await session.execute(text("CREATE EXTENSION IF NOT EXISTS vector"))
|
||||
except Exception as exc:
|
||||
raise SemanticDependenciesMissingError(
|
||||
"pgvector extension is unavailable for this Postgres database."
|
||||
) from exc
|
||||
|
||||
# --- Chunks table (dimension-independent, may already exist via migration) ---
|
||||
await session.execute(
|
||||
text(
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS search_vector_chunks (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
entity_id INTEGER NOT NULL,
|
||||
project_id INTEGER NOT NULL,
|
||||
chunk_key TEXT NOT NULL,
|
||||
chunk_text TEXT NOT NULL,
|
||||
source_hash TEXT NOT NULL,
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
UNIQUE (project_id, entity_id, chunk_key)
|
||||
)
|
||||
"""
|
||||
)
|
||||
)
|
||||
await session.execute(
|
||||
text(
|
||||
"""
|
||||
CREATE INDEX IF NOT EXISTS idx_search_vector_chunks_project_entity
|
||||
ON search_vector_chunks (project_id, entity_id)
|
||||
"""
|
||||
)
|
||||
)
|
||||
|
||||
# --- Embeddings table (dimension-dependent, created at runtime) ---
|
||||
# Trigger: provider dimensions may differ from what was previously deployed.
|
||||
# Why: the column type `vector(N)` is fixed at table creation; switching
|
||||
# from FastEmbed (384) to OpenAI (1536) requires recreation.
|
||||
# Outcome: mismatched table is dropped and recreated with correct dims.
|
||||
# Embeddings are derived data — re-indexing will repopulate them.
|
||||
existing_dims = await self._get_existing_embedding_dims(session)
|
||||
if existing_dims is not None and existing_dims != self._vector_dimensions:
|
||||
logger.warning(
|
||||
f"Embedding dimension mismatch: table has {existing_dims}, "
|
||||
f"provider expects {self._vector_dimensions}. "
|
||||
"Dropping and recreating search_vector_embeddings."
|
||||
)
|
||||
await session.execute(text("DROP TABLE IF EXISTS search_vector_embeddings"))
|
||||
|
||||
await session.execute(
|
||||
text(
|
||||
f"""
|
||||
CREATE TABLE IF NOT EXISTS search_vector_embeddings (
|
||||
chunk_id BIGINT PRIMARY KEY
|
||||
REFERENCES search_vector_chunks(id) ON DELETE CASCADE,
|
||||
project_id INTEGER NOT NULL,
|
||||
embedding vector({self._vector_dimensions}) NOT NULL,
|
||||
embedding_dims INTEGER NOT NULL,
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
)
|
||||
"""
|
||||
)
|
||||
)
|
||||
await session.execute(
|
||||
text(
|
||||
"""
|
||||
CREATE INDEX IF NOT EXISTS idx_search_vector_embeddings_project_dims
|
||||
ON search_vector_embeddings (project_id, embedding_dims)
|
||||
"""
|
||||
)
|
||||
)
|
||||
# HNSW index for approximate nearest-neighbour search.
|
||||
# Without this every vector query is a sequential scan.
|
||||
await session.execute(
|
||||
text(
|
||||
"""
|
||||
CREATE INDEX IF NOT EXISTS idx_search_vector_embeddings_hnsw
|
||||
ON search_vector_embeddings
|
||||
USING hnsw (embedding vector_cosine_ops)
|
||||
WITH (m = 16, ef_construction = 64)
|
||||
"""
|
||||
)
|
||||
)
|
||||
await session.commit()
|
||||
|
||||
self._vector_tables_initialized = True
|
||||
|
||||
async def _get_existing_embedding_dims(self, session: AsyncSession) -> int | None:
|
||||
"""Query the vector column dimension from an existing search_vector_embeddings table.
|
||||
|
||||
Returns None when the table does not exist.
|
||||
Uses information_schema to avoid regclass cast errors on missing tables,
|
||||
then reads atttypmod from pg_attribute for the actual dimension value.
|
||||
"""
|
||||
# Check table existence via information_schema (no exception on missing)
|
||||
exists_result = await session.execute(
|
||||
text(
|
||||
"""
|
||||
SELECT 1 FROM information_schema.tables
|
||||
WHERE table_name = 'search_vector_embeddings'
|
||||
"""
|
||||
)
|
||||
)
|
||||
if exists_result.fetchone() is None:
|
||||
return None
|
||||
|
||||
result = await session.execute(
|
||||
text(
|
||||
"""
|
||||
SELECT atttypmod
|
||||
FROM pg_attribute
|
||||
WHERE attrelid = 'search_vector_embeddings'::regclass
|
||||
AND attname = 'embedding'
|
||||
"""
|
||||
)
|
||||
)
|
||||
row = result.fetchone()
|
||||
if row is None:
|
||||
return None
|
||||
# pgvector stores dimensions in atttypmod directly
|
||||
return int(row[0])
|
||||
|
||||
async def _run_vector_query(
|
||||
self,
|
||||
session: AsyncSession,
|
||||
query_embedding: list[float],
|
||||
candidate_limit: int,
|
||||
) -> list[dict]:
|
||||
if not query_embedding:
|
||||
return []
|
||||
|
||||
embedding_dims = len(query_embedding)
|
||||
query_embedding_literal = self._format_pgvector_literal(query_embedding)
|
||||
|
||||
vector_result = await session.execute(
|
||||
text(
|
||||
"""
|
||||
WITH vector_matches AS (
|
||||
SELECT
|
||||
e.chunk_id,
|
||||
(e.embedding <=> CAST(:query_embedding AS vector)) AS distance
|
||||
FROM search_vector_embeddings e
|
||||
WHERE e.project_id = :project_id
|
||||
AND e.embedding_dims = :embedding_dims
|
||||
ORDER BY e.embedding <=> CAST(:query_embedding AS vector)
|
||||
LIMIT :vector_k
|
||||
)
|
||||
SELECT c.entity_id, c.chunk_key, vector_matches.distance AS best_distance
|
||||
FROM vector_matches
|
||||
JOIN search_vector_chunks c ON c.id = vector_matches.chunk_id
|
||||
WHERE c.project_id = :project_id
|
||||
ORDER BY best_distance ASC
|
||||
LIMIT :vector_k
|
||||
"""
|
||||
),
|
||||
{
|
||||
"query_embedding": query_embedding_literal,
|
||||
"project_id": self.project_id,
|
||||
"embedding_dims": embedding_dims,
|
||||
"vector_k": candidate_limit,
|
||||
},
|
||||
)
|
||||
return [dict(row) for row in vector_result.mappings().all()]
|
||||
|
||||
async def _write_embeddings(
|
||||
self,
|
||||
session: AsyncSession,
|
||||
jobs: list[tuple[int, str]],
|
||||
embeddings: list[list[float]],
|
||||
) -> None:
|
||||
for (row_id, _), vector in zip(jobs, embeddings, strict=True):
|
||||
vector_literal = self._format_pgvector_literal(vector)
|
||||
await session.execute(
|
||||
text(
|
||||
"INSERT INTO search_vector_embeddings ("
|
||||
"chunk_id, project_id, embedding, embedding_dims, updated_at"
|
||||
") VALUES ("
|
||||
":chunk_id, :project_id, CAST(:embedding AS vector), :embedding_dims, NOW()"
|
||||
") "
|
||||
"ON CONFLICT (chunk_id) DO UPDATE SET "
|
||||
"project_id = EXCLUDED.project_id, "
|
||||
"embedding = EXCLUDED.embedding, "
|
||||
"embedding_dims = EXCLUDED.embedding_dims, "
|
||||
"updated_at = NOW()"
|
||||
),
|
||||
{
|
||||
"chunk_id": row_id,
|
||||
"project_id": self.project_id,
|
||||
"embedding": vector_literal,
|
||||
"embedding_dims": len(vector),
|
||||
},
|
||||
)
|
||||
|
||||
async def _delete_entity_chunks(
|
||||
self,
|
||||
session: AsyncSession,
|
||||
entity_id: int,
|
||||
) -> None:
|
||||
# Postgres has ON DELETE CASCADE from embeddings → chunks
|
||||
await session.execute(
|
||||
text(
|
||||
"DELETE FROM search_vector_chunks "
|
||||
"WHERE project_id = :project_id AND entity_id = :entity_id"
|
||||
),
|
||||
{"project_id": self.project_id, "entity_id": entity_id},
|
||||
)
|
||||
|
||||
async def _delete_stale_chunks(
|
||||
self,
|
||||
session: AsyncSession,
|
||||
stale_ids: list[int],
|
||||
entity_id: int,
|
||||
) -> None:
|
||||
stale_placeholders = ", ".join(f":stale_id_{idx}" for idx in range(len(stale_ids)))
|
||||
stale_params = {
|
||||
"project_id": self.project_id,
|
||||
"entity_id": entity_id,
|
||||
**{f"stale_id_{idx}": row_id for idx, row_id in enumerate(stale_ids)},
|
||||
}
|
||||
# CASCADE handles embedding deletion
|
||||
await session.execute(
|
||||
text(
|
||||
"DELETE FROM search_vector_chunks "
|
||||
f"WHERE id IN ({stale_placeholders}) "
|
||||
"AND project_id = :project_id AND entity_id = :entity_id"
|
||||
),
|
||||
stale_params,
|
||||
)
|
||||
|
||||
async def _update_timestamp_sql(self) -> str:
|
||||
return "NOW()" # pragma: no cover
|
||||
|
||||
def _timestamp_now_expr(self) -> str:
|
||||
return "NOW()"
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Index / bulk index overrides (Postgres UPSERT)
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
async def bulk_index_items(self, search_index_rows: List[SearchIndexRow]) -> None:
|
||||
"""Index multiple items in a single batch operation using UPSERT.
|
||||
|
||||
Uses INSERT ... ON CONFLICT to handle race conditions during parallel
|
||||
entity indexing. The partial unique index uix_search_index_permalink_project
|
||||
on (permalink, project_id) WHERE permalink IS NOT NULL prevents duplicate
|
||||
permalinks.
|
||||
|
||||
For rows with non-null permalinks (entities), conflicts are resolved by
|
||||
updating the existing row. For rows with null permalinks (observations,
|
||||
relations), the partial index doesn't apply and they are inserted directly.
|
||||
|
||||
Args:
|
||||
search_index_rows: List of SearchIndexRow objects to index
|
||||
"""
|
||||
|
||||
if not search_index_rows:
|
||||
return
|
||||
|
||||
async with db.scoped_session(self.session_maker) as session:
|
||||
# When using text() raw SQL, always serialize JSON to string
|
||||
# Both SQLite (TEXT) and Postgres (JSONB) accept JSON strings in raw SQL
|
||||
# The database driver/column type will handle conversion
|
||||
insert_data_list = []
|
||||
for row in search_index_rows:
|
||||
insert_data = row.to_insert(serialize_json=True)
|
||||
insert_data["project_id"] = self.project_id
|
||||
insert_data_list.append(insert_data)
|
||||
|
||||
# Use upsert to handle race conditions during parallel indexing
|
||||
# ON CONFLICT (permalink, project_id) matches the partial unique index
|
||||
# uix_search_index_permalink_project WHERE permalink IS NOT NULL
|
||||
# For rows with NULL permalinks (observations, relations), no conflict occurs
|
||||
await session.execute(
|
||||
text("""
|
||||
INSERT INTO search_index (
|
||||
id, title, content_stems, content_snippet, permalink, file_path, type, metadata,
|
||||
from_id, to_id, relation_type,
|
||||
entity_id, category,
|
||||
created_at, updated_at,
|
||||
project_id
|
||||
) VALUES (
|
||||
:id, :title, :content_stems, :content_snippet, :permalink, :file_path, :type, :metadata,
|
||||
:from_id, :to_id, :relation_type,
|
||||
:entity_id, :category,
|
||||
:created_at, :updated_at,
|
||||
:project_id
|
||||
)
|
||||
ON CONFLICT (permalink, project_id) WHERE permalink IS NOT NULL DO UPDATE SET
|
||||
id = EXCLUDED.id,
|
||||
title = EXCLUDED.title,
|
||||
content_stems = EXCLUDED.content_stems,
|
||||
content_snippet = EXCLUDED.content_snippet,
|
||||
file_path = EXCLUDED.file_path,
|
||||
type = EXCLUDED.type,
|
||||
metadata = EXCLUDED.metadata,
|
||||
from_id = EXCLUDED.from_id,
|
||||
to_id = EXCLUDED.to_id,
|
||||
relation_type = EXCLUDED.relation_type,
|
||||
entity_id = EXCLUDED.entity_id,
|
||||
category = EXCLUDED.category,
|
||||
created_at = EXCLUDED.created_at,
|
||||
updated_at = EXCLUDED.updated_at
|
||||
"""),
|
||||
insert_data_list,
|
||||
)
|
||||
logger.debug(f"Bulk indexed {len(search_index_rows)} rows")
|
||||
await session.commit()
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# FTS search (Postgres-specific)
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
async def search(
|
||||
self,
|
||||
search_text: Optional[str] = None,
|
||||
@@ -586,29 +229,10 @@ class PostgresSearchRepository(SearchRepositoryBase):
|
||||
after_date: Optional[datetime] = None,
|
||||
search_item_types: Optional[List[SearchItemType]] = None,
|
||||
metadata_filters: Optional[dict] = None,
|
||||
retrieval_mode: SearchRetrievalMode = SearchRetrievalMode.FTS,
|
||||
limit: int = 10,
|
||||
offset: int = 0,
|
||||
) -> List[SearchIndexRow]:
|
||||
"""Search across all indexed content using PostgreSQL tsvector."""
|
||||
# --- Dispatch vector / hybrid modes (shared logic) ---
|
||||
dispatched = await self._dispatch_retrieval_mode(
|
||||
search_text=search_text,
|
||||
permalink=permalink,
|
||||
permalink_match=permalink_match,
|
||||
title=title,
|
||||
types=types,
|
||||
after_date=after_date,
|
||||
search_item_types=search_item_types,
|
||||
metadata_filters=metadata_filters,
|
||||
retrieval_mode=retrieval_mode,
|
||||
limit=limit,
|
||||
offset=offset,
|
||||
)
|
||||
if dispatched is not None:
|
||||
return dispatched
|
||||
|
||||
# --- FTS mode (Postgres-specific) ---
|
||||
conditions = []
|
||||
params = {}
|
||||
order_by_clause = ""
|
||||
@@ -846,3 +470,72 @@ class PostgresSearchRepository(SearchRepositoryBase):
|
||||
)
|
||||
|
||||
return results
|
||||
|
||||
async def bulk_index_items(self, search_index_rows: List[SearchIndexRow]) -> None:
|
||||
"""Index multiple items in a single batch operation using UPSERT.
|
||||
|
||||
Uses INSERT ... ON CONFLICT to handle race conditions during parallel
|
||||
entity indexing. The partial unique index uix_search_index_permalink_project
|
||||
on (permalink, project_id) WHERE permalink IS NOT NULL prevents duplicate
|
||||
permalinks.
|
||||
|
||||
For rows with non-null permalinks (entities), conflicts are resolved by
|
||||
updating the existing row. For rows with null permalinks (observations,
|
||||
relations), the partial index doesn't apply and they are inserted directly.
|
||||
|
||||
Args:
|
||||
search_index_rows: List of SearchIndexRow objects to index
|
||||
"""
|
||||
|
||||
if not search_index_rows:
|
||||
return
|
||||
|
||||
async with db.scoped_session(self.session_maker) as session:
|
||||
# When using text() raw SQL, always serialize JSON to string
|
||||
# Both SQLite (TEXT) and Postgres (JSONB) accept JSON strings in raw SQL
|
||||
# The database driver/column type will handle conversion
|
||||
insert_data_list = []
|
||||
for row in search_index_rows:
|
||||
insert_data = row.to_insert(serialize_json=True)
|
||||
insert_data["project_id"] = self.project_id
|
||||
insert_data_list.append(_strip_nul_from_row(insert_data))
|
||||
|
||||
# Use upsert to handle race conditions during parallel indexing
|
||||
# ON CONFLICT (permalink, project_id) matches the partial unique index
|
||||
# uix_search_index_permalink_project WHERE permalink IS NOT NULL
|
||||
# For rows with NULL permalinks (observations, relations), no conflict occurs
|
||||
await session.execute(
|
||||
text("""
|
||||
INSERT INTO search_index (
|
||||
id, title, content_stems, content_snippet, permalink, file_path, type, metadata,
|
||||
from_id, to_id, relation_type,
|
||||
entity_id, category,
|
||||
created_at, updated_at,
|
||||
project_id
|
||||
) VALUES (
|
||||
:id, :title, :content_stems, :content_snippet, :permalink, :file_path, :type, :metadata,
|
||||
:from_id, :to_id, :relation_type,
|
||||
:entity_id, :category,
|
||||
:created_at, :updated_at,
|
||||
:project_id
|
||||
)
|
||||
ON CONFLICT (permalink, project_id) WHERE permalink IS NOT NULL DO UPDATE SET
|
||||
id = EXCLUDED.id,
|
||||
title = EXCLUDED.title,
|
||||
content_stems = EXCLUDED.content_stems,
|
||||
content_snippet = EXCLUDED.content_snippet,
|
||||
file_path = EXCLUDED.file_path,
|
||||
type = EXCLUDED.type,
|
||||
metadata = EXCLUDED.metadata,
|
||||
from_id = EXCLUDED.from_id,
|
||||
to_id = EXCLUDED.to_id,
|
||||
relation_type = EXCLUDED.relation_type,
|
||||
entity_id = EXCLUDED.entity_id,
|
||||
category = EXCLUDED.category,
|
||||
created_at = EXCLUDED.created_at,
|
||||
updated_at = EXCLUDED.updated_at
|
||||
"""),
|
||||
insert_data_list,
|
||||
)
|
||||
logger.debug(f"Bulk indexed {len(search_index_rows)} rows")
|
||||
await session.commit()
|
||||
|
||||
@@ -38,13 +38,8 @@ class SearchIndexRow:
|
||||
to_id: Optional[int] = None # relations
|
||||
relation_type: Optional[str] = None # relations
|
||||
|
||||
CONTENT_DISPLAY_LIMIT = 250
|
||||
|
||||
@property
|
||||
def content(self):
|
||||
"""Return truncated content for display. Full content in content_snippet."""
|
||||
if self.content_snippet and len(self.content_snippet) > self.CONTENT_DISPLAY_LIMIT:
|
||||
return self.content_snippet[: self.CONTENT_DISPLAY_LIMIT]
|
||||
return self.content_snippet
|
||||
|
||||
@property
|
||||
|
||||
@@ -12,11 +12,11 @@ from typing import List, Optional, Protocol
|
||||
from sqlalchemy import Result
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from basic_memory.config import BasicMemoryConfig, ConfigManager, DatabaseBackend
|
||||
from basic_memory.config import ConfigManager, DatabaseBackend
|
||||
from basic_memory.repository.postgres_search_repository import PostgresSearchRepository
|
||||
from basic_memory.repository.search_index_row import SearchIndexRow
|
||||
from basic_memory.repository.sqlite_search_repository import SQLiteSearchRepository
|
||||
from basic_memory.schemas.search import SearchItemType, SearchRetrievalMode
|
||||
from basic_memory.schemas.search import SearchItemType
|
||||
|
||||
|
||||
class SearchRepository(Protocol):
|
||||
@@ -41,7 +41,6 @@ class SearchRepository(Protocol):
|
||||
after_date: Optional[datetime] = None,
|
||||
search_item_types: Optional[List[SearchItemType]] = None,
|
||||
metadata_filters: Optional[dict] = None,
|
||||
retrieval_mode: SearchRetrievalMode = SearchRetrievalMode.FTS,
|
||||
limit: int = 10,
|
||||
offset: int = 0,
|
||||
) -> List[SearchIndexRow]:
|
||||
@@ -64,10 +63,6 @@ class SearchRepository(Protocol):
|
||||
"""Delete items by entity ID."""
|
||||
...
|
||||
|
||||
async def sync_entity_vectors(self, entity_id: int) -> None:
|
||||
"""Sync semantic vector chunks for an entity."""
|
||||
...
|
||||
|
||||
async def execute_query(self, query, params: dict) -> Result:
|
||||
"""Execute a raw SQL query."""
|
||||
...
|
||||
@@ -76,7 +71,6 @@ class SearchRepository(Protocol):
|
||||
def create_search_repository(
|
||||
session_maker: async_sessionmaker[AsyncSession],
|
||||
project_id: int,
|
||||
app_config: Optional[BasicMemoryConfig] = None,
|
||||
database_backend: Optional[DatabaseBackend] = None,
|
||||
) -> SearchRepository:
|
||||
"""Factory function to create the appropriate search repository based on database backend.
|
||||
@@ -92,17 +86,13 @@ def create_search_repository(
|
||||
"""
|
||||
# Prefer explicit parameter; fall back to ConfigManager for backwards compatibility
|
||||
if database_backend is None:
|
||||
config = app_config or ConfigManager().config
|
||||
config = ConfigManager().config
|
||||
database_backend = config.database_backend
|
||||
|
||||
if database_backend == DatabaseBackend.POSTGRES: # pragma: no cover
|
||||
return PostgresSearchRepository( # pragma: no cover
|
||||
session_maker,
|
||||
project_id=project_id,
|
||||
app_config=app_config,
|
||||
)
|
||||
return PostgresSearchRepository(session_maker, project_id=project_id) # pragma: no cover
|
||||
else:
|
||||
return SQLiteSearchRepository(session_maker, project_id=project_id, app_config=app_config)
|
||||
return SQLiteSearchRepository(session_maker, project_id=project_id)
|
||||
|
||||
|
||||
__all__ = [
|
||||
|
||||
@@ -1,35 +1,17 @@
|
||||
"""Abstract base class for search repository implementations."""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import time
|
||||
from abc import ABC, abstractmethod
|
||||
from dataclasses import replace
|
||||
from datetime import datetime
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
|
||||
from loguru import logger
|
||||
from sqlalchemy import Executable, Result, text
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from basic_memory import db
|
||||
from basic_memory.repository.embedding_provider import EmbeddingProvider
|
||||
from basic_memory.schemas.search import SearchItemType
|
||||
from basic_memory.repository.search_index_row import SearchIndexRow
|
||||
from basic_memory.repository.semantic_errors import (
|
||||
SemanticDependenciesMissingError,
|
||||
SemanticSearchDisabledError,
|
||||
)
|
||||
from basic_memory.schemas.search import SearchItemType, SearchRetrievalMode
|
||||
|
||||
# --- Semantic search constants ---
|
||||
|
||||
VECTOR_FILTER_SCAN_LIMIT = 50000
|
||||
RRF_K = 60
|
||||
MAX_VECTOR_CHUNK_CHARS = 900
|
||||
VECTOR_CHUNK_OVERLAP_CHARS = 120
|
||||
HEADER_LINE_PATTERN = re.compile(r"^\s*#{1,6}\s+")
|
||||
BULLET_PATTERN = re.compile(r"^[\-\*]\s+")
|
||||
|
||||
|
||||
class SearchRepositoryBase(ABC):
|
||||
@@ -38,21 +20,11 @@ class SearchRepositoryBase(ABC):
|
||||
This class defines the common interface that all search repositories must implement,
|
||||
regardless of whether they use SQLite FTS5 or Postgres tsvector for full-text search.
|
||||
|
||||
Shared semantic search logic (chunking, embedding orchestration, hybrid RRF fusion)
|
||||
lives here. Backend-specific operations are delegated to abstract hooks.
|
||||
|
||||
Concrete implementations:
|
||||
- SQLiteSearchRepository: Uses FTS5 virtual tables with MATCH queries
|
||||
- PostgresSearchRepository: Uses tsvector/tsquery with GIN indexes
|
||||
"""
|
||||
|
||||
# --- Subclass-populated attributes ---
|
||||
_semantic_enabled: bool
|
||||
_semantic_vector_k: int
|
||||
_embedding_provider: Optional[EmbeddingProvider]
|
||||
_vector_dimensions: int
|
||||
_vector_tables_initialized: bool
|
||||
|
||||
def __init__(self, session_maker: async_sessionmaker[AsyncSession], project_id: int):
|
||||
"""Initialize with session maker and project_id filter.
|
||||
|
||||
@@ -69,10 +41,6 @@ class SearchRepositoryBase(ABC):
|
||||
self.session_maker = session_maker
|
||||
self.project_id = project_id
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Abstract methods — FTS and schema (backend-specific)
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
@abstractmethod
|
||||
async def init_search_index(self) -> None:
|
||||
"""Create or recreate the search index.
|
||||
@@ -111,7 +79,6 @@ class SearchRepositoryBase(ABC):
|
||||
after_date: Optional[datetime] = None,
|
||||
search_item_types: Optional[List[SearchItemType]] = None,
|
||||
metadata_filters: Optional[Dict[str, Any]] = None,
|
||||
retrieval_mode: SearchRetrievalMode = SearchRetrievalMode.FTS,
|
||||
limit: int = 10,
|
||||
offset: int = 0,
|
||||
) -> List[SearchIndexRow]:
|
||||
@@ -138,74 +105,6 @@ class SearchRepositoryBase(ABC):
|
||||
"""
|
||||
pass
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Abstract methods — semantic search (backend-specific DB operations)
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
@abstractmethod
|
||||
async def _ensure_vector_tables(self) -> None:
|
||||
"""Create backend-specific vector chunk and embedding tables."""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
async def _run_vector_query(
|
||||
self,
|
||||
session: AsyncSession,
|
||||
query_embedding: list[float],
|
||||
candidate_limit: int,
|
||||
) -> list[dict]:
|
||||
"""Execute backend-specific nearest-neighbour vector query.
|
||||
|
||||
Returns list of mappings with keys ``entity_id`` and ``best_distance``.
|
||||
"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
async def _write_embeddings(
|
||||
self,
|
||||
session: AsyncSession,
|
||||
jobs: list[tuple[int, str]],
|
||||
embeddings: list[list[float]],
|
||||
) -> None:
|
||||
"""Write embedding vectors for the given chunk row IDs.
|
||||
|
||||
``jobs`` is a list of ``(chunk_row_id, chunk_text)`` pairs.
|
||||
``embeddings`` is the corresponding list of vectors.
|
||||
"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
async def _delete_entity_chunks(
|
||||
self,
|
||||
session: AsyncSession,
|
||||
entity_id: int,
|
||||
) -> None:
|
||||
"""Delete all chunk + embedding rows for an entity.
|
||||
|
||||
SQLite must explicitly delete embeddings first (no CASCADE).
|
||||
Postgres relies on ON DELETE CASCADE from the FK.
|
||||
"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
async def _delete_stale_chunks(
|
||||
self,
|
||||
session: AsyncSession,
|
||||
stale_ids: list[int],
|
||||
entity_id: int,
|
||||
) -> None:
|
||||
"""Delete stale chunk rows (and their embeddings) by ID."""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
async def _update_timestamp_sql(self) -> str:
|
||||
"""Return the SQL expression for current timestamp in the backend."""
|
||||
pass # pragma: no cover
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Shared index / delete operations
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
async def index_item(self, search_index_row: SearchIndexRow) -> None:
|
||||
"""Index or update a single item.
|
||||
|
||||
@@ -333,6 +232,8 @@ class SearchRepositoryBase(ABC):
|
||||
|
||||
This implementation is shared across backends for utility query execution.
|
||||
"""
|
||||
import time
|
||||
|
||||
async with db.scoped_session(self.session_maker) as session:
|
||||
start_time = time.perf_counter()
|
||||
result = await session.execute(query, params)
|
||||
@@ -340,751 +241,3 @@ class SearchRepositoryBase(ABC):
|
||||
elapsed_time = end_time - start_time
|
||||
logger.debug(f"Query executed successfully in {elapsed_time:.2f}s.")
|
||||
return result
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Shared semantic search: guard, text processing, chunking
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _assert_semantic_available(self) -> None:
|
||||
if not self._semantic_enabled:
|
||||
raise SemanticSearchDisabledError(
|
||||
"Semantic search is disabled. Set BASIC_MEMORY_SEMANTIC_SEARCH_ENABLED=true."
|
||||
)
|
||||
if self._embedding_provider is None:
|
||||
raise SemanticDependenciesMissingError(
|
||||
"No embedding provider configured. "
|
||||
"Install semantic extras: pip install 'basic-memory[semantic]' "
|
||||
"and set semantic_search_enabled=true."
|
||||
)
|
||||
|
||||
def _compose_row_source_text(self, row) -> str:
|
||||
"""Build the text blob that will be chunked and embedded for one search_index row.
|
||||
|
||||
For entity rows we use title, permalink, and content_snippet (the actual
|
||||
human-readable content). content_stems is an FTS-optimised variant that
|
||||
includes word-boundary expansions and would dilute embedding quality.
|
||||
"""
|
||||
if row.type == SearchItemType.ENTITY.value:
|
||||
row_parts = [
|
||||
row.title or "",
|
||||
row.permalink or "",
|
||||
row.content_snippet or "",
|
||||
]
|
||||
return "\n\n".join(part for part in row_parts if part)
|
||||
|
||||
if row.type == SearchItemType.OBSERVATION.value:
|
||||
row_parts = [
|
||||
row.title or "",
|
||||
row.permalink or "",
|
||||
row.category or "",
|
||||
row.content_snippet or "",
|
||||
]
|
||||
return "\n\n".join(part for part in row_parts if part)
|
||||
|
||||
row_parts = [
|
||||
row.title or "",
|
||||
row.permalink or "",
|
||||
row.relation_type or "",
|
||||
row.content_snippet or "",
|
||||
]
|
||||
return "\n\n".join(part for part in row_parts if part)
|
||||
|
||||
def _build_chunk_records(self, rows) -> list[dict[str, str]]:
|
||||
records: list[dict[str, str]] = []
|
||||
for row in rows:
|
||||
source_text = self._compose_row_source_text(row)
|
||||
chunks = self._split_text_into_chunks(source_text)
|
||||
for chunk_index, chunk_text in enumerate(chunks):
|
||||
chunk_key = f"{row.type}:{row.id}:{chunk_index}"
|
||||
source_hash = hashlib.sha256(chunk_text.encode("utf-8")).hexdigest()
|
||||
records.append(
|
||||
{
|
||||
"chunk_key": chunk_key,
|
||||
"chunk_text": chunk_text,
|
||||
"source_hash": source_hash,
|
||||
}
|
||||
)
|
||||
return records
|
||||
|
||||
# --- Text splitting ---
|
||||
|
||||
def _split_text_into_chunks(self, text_value: str) -> list[str]:
|
||||
normalized = (text_value or "").strip()
|
||||
if not normalized:
|
||||
return []
|
||||
|
||||
# Split on markdown headers AND bullet boundaries to ensure each
|
||||
# discrete fact gets its own embedding vector for granular retrieval.
|
||||
lines = normalized.splitlines()
|
||||
sections: list[str] = []
|
||||
current_section: list[str] = []
|
||||
for line in lines:
|
||||
if HEADER_LINE_PATTERN.match(line) and current_section:
|
||||
sections.append("\n".join(current_section).strip())
|
||||
current_section = [line]
|
||||
elif BULLET_PATTERN.match(line) and current_section:
|
||||
sections.append("\n".join(current_section).strip())
|
||||
current_section = [line]
|
||||
else:
|
||||
current_section.append(line)
|
||||
if current_section:
|
||||
sections.append("\n".join(current_section).strip())
|
||||
|
||||
chunked_sections: list[str] = []
|
||||
current_chunk = ""
|
||||
|
||||
for section in sections:
|
||||
is_bullet = bool(BULLET_PATTERN.match(section))
|
||||
|
||||
if len(section) > MAX_VECTOR_CHUNK_CHARS:
|
||||
if current_chunk:
|
||||
chunked_sections.append(current_chunk)
|
||||
current_chunk = ""
|
||||
long_chunks = self._split_long_section(section)
|
||||
if long_chunks:
|
||||
chunked_sections.extend(long_chunks[:-1])
|
||||
current_chunk = long_chunks[-1]
|
||||
continue
|
||||
|
||||
# Keep bullets as individual chunks for granular fact retrieval.
|
||||
# Non-bullet sections (headers, prose) merge up to MAX_VECTOR_CHUNK_CHARS.
|
||||
if is_bullet:
|
||||
if current_chunk:
|
||||
chunked_sections.append(current_chunk)
|
||||
current_chunk = ""
|
||||
chunked_sections.append(section)
|
||||
continue
|
||||
|
||||
candidate = section if not current_chunk else f"{current_chunk}\n\n{section}"
|
||||
if len(candidate) <= MAX_VECTOR_CHUNK_CHARS:
|
||||
current_chunk = candidate
|
||||
continue
|
||||
|
||||
chunked_sections.append(current_chunk)
|
||||
current_chunk = section
|
||||
|
||||
if current_chunk:
|
||||
chunked_sections.append(current_chunk)
|
||||
|
||||
return [chunk for chunk in chunked_sections if chunk.strip()]
|
||||
|
||||
@staticmethod
|
||||
def _split_into_paragraphs(section_text: str) -> list[str]:
|
||||
"""Split section into paragraphs, treating bullet lists as separate items.
|
||||
|
||||
Double newlines always split. Within a single-newline block, bullet
|
||||
boundaries (lines starting with - or *) also create splits so that
|
||||
individual facts in a list become separate embeddable chunks.
|
||||
"""
|
||||
raw_paragraphs = [p.strip() for p in section_text.split("\n\n") if p.strip()]
|
||||
result: list[str] = []
|
||||
for para in raw_paragraphs:
|
||||
lines = para.split("\n")
|
||||
# Check if this paragraph contains bullet items
|
||||
has_bullets = any(BULLET_PATTERN.match(line) for line in lines)
|
||||
if not has_bullets:
|
||||
result.append(para)
|
||||
continue
|
||||
# Split on bullet boundaries: group consecutive non-bullet lines
|
||||
# with their preceding bullet
|
||||
current_item: list[str] = []
|
||||
for line in lines:
|
||||
if BULLET_PATTERN.match(line) and current_item:
|
||||
result.append("\n".join(current_item).strip())
|
||||
current_item = [line]
|
||||
else:
|
||||
current_item.append(line)
|
||||
if current_item:
|
||||
result.append("\n".join(current_item).strip())
|
||||
return [p for p in result if p]
|
||||
|
||||
def _split_long_section(self, section_text: str) -> list[str]:
|
||||
paragraphs = self._split_into_paragraphs(section_text)
|
||||
if not paragraphs:
|
||||
return []
|
||||
|
||||
chunks: list[str] = []
|
||||
current = ""
|
||||
for paragraph in paragraphs:
|
||||
if len(paragraph) > MAX_VECTOR_CHUNK_CHARS:
|
||||
if current:
|
||||
chunks.append(current)
|
||||
current = ""
|
||||
chunks.extend(self._split_by_char_window(paragraph))
|
||||
continue
|
||||
|
||||
candidate = paragraph if not current else f"{current}\n\n{paragraph}"
|
||||
if len(candidate) <= MAX_VECTOR_CHUNK_CHARS:
|
||||
current = candidate
|
||||
else:
|
||||
if current:
|
||||
chunks.append(current)
|
||||
current = paragraph
|
||||
|
||||
if current:
|
||||
chunks.append(current)
|
||||
return chunks
|
||||
|
||||
def _split_by_char_window(self, paragraph: str) -> list[str]:
|
||||
text_value = paragraph.strip()
|
||||
if not text_value:
|
||||
return []
|
||||
|
||||
chunks: list[str] = []
|
||||
start = 0
|
||||
while start < len(text_value):
|
||||
end = min(len(text_value), start + MAX_VECTOR_CHUNK_CHARS)
|
||||
chunk = text_value[start:end].strip()
|
||||
if chunk:
|
||||
chunks.append(chunk)
|
||||
if end >= len(text_value):
|
||||
break
|
||||
start = max(0, end - VECTOR_CHUNK_OVERLAP_CHARS)
|
||||
return chunks
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Shared semantic search: sync_entity_vectors orchestration
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
async def sync_entity_vectors(self, entity_id: int) -> None:
|
||||
"""Sync semantic chunk rows + embeddings for a single entity.
|
||||
|
||||
This is the shared orchestration logic. Backend-specific SQL operations
|
||||
are delegated to abstract hooks (_delete_entity_chunks, _write_embeddings, etc.).
|
||||
"""
|
||||
self._assert_semantic_available()
|
||||
await self._ensure_vector_tables()
|
||||
assert self._embedding_provider is not None
|
||||
|
||||
async with db.scoped_session(self.session_maker) as session:
|
||||
await self._prepare_vector_session(session)
|
||||
|
||||
row_result = await session.execute(
|
||||
text(
|
||||
"SELECT id, type, title, permalink, content_stems, content_snippet, "
|
||||
"category, relation_type "
|
||||
"FROM search_index "
|
||||
"WHERE entity_id = :entity_id AND project_id = :project_id "
|
||||
"ORDER BY "
|
||||
"CASE type "
|
||||
"WHEN :entity_type THEN 0 "
|
||||
"WHEN :observation_type THEN 1 "
|
||||
"WHEN :relation_type_type THEN 2 "
|
||||
"ELSE 3 END, id ASC"
|
||||
),
|
||||
{
|
||||
"entity_id": entity_id,
|
||||
"project_id": self.project_id,
|
||||
"entity_type": SearchItemType.ENTITY.value,
|
||||
"observation_type": SearchItemType.OBSERVATION.value,
|
||||
"relation_type_type": SearchItemType.RELATION.value,
|
||||
},
|
||||
)
|
||||
rows = row_result.fetchall()
|
||||
|
||||
# No search_index rows → delete all chunk/embedding data for this entity.
|
||||
if not rows:
|
||||
await self._delete_entity_chunks(session, entity_id)
|
||||
await session.commit()
|
||||
return
|
||||
|
||||
chunk_records = self._build_chunk_records(rows)
|
||||
if not chunk_records:
|
||||
await self._delete_entity_chunks(session, entity_id)
|
||||
await session.commit()
|
||||
return
|
||||
|
||||
# --- Diff existing chunks against incoming ---
|
||||
existing_rows_result = await session.execute(
|
||||
text(
|
||||
"SELECT id, chunk_key, source_hash "
|
||||
"FROM search_vector_chunks "
|
||||
"WHERE project_id = :project_id AND entity_id = :entity_id"
|
||||
),
|
||||
{"project_id": self.project_id, "entity_id": entity_id},
|
||||
)
|
||||
existing_by_key = {row.chunk_key: row for row in existing_rows_result.fetchall()}
|
||||
incoming_hashes = {
|
||||
record["chunk_key"]: record["source_hash"] for record in chunk_records
|
||||
}
|
||||
stale_ids = [
|
||||
int(row.id)
|
||||
for chunk_key, row in existing_by_key.items()
|
||||
if chunk_key not in incoming_hashes
|
||||
]
|
||||
|
||||
if stale_ids:
|
||||
await self._delete_stale_chunks(session, stale_ids, entity_id)
|
||||
|
||||
# --- Orphan cleanup: chunks without corresponding embeddings ---
|
||||
# Trigger: a previous sync crashed between chunk insert and embedding write.
|
||||
# Why: self-healing on next sync prevents permanent data skew.
|
||||
# Outcome: orphaned chunks are re-embedded instead of silently dropped.
|
||||
orphan_result = await session.execute(
|
||||
text(self._orphan_detection_sql()),
|
||||
{"project_id": self.project_id, "entity_id": entity_id},
|
||||
)
|
||||
orphan_rows = orphan_result.fetchall()
|
||||
|
||||
# --- Upsert changed / new chunks, collect embedding jobs ---
|
||||
timestamp_expr = self._timestamp_now_expr()
|
||||
embedding_jobs: list[tuple[int, str]] = []
|
||||
for record in chunk_records:
|
||||
current = existing_by_key.get(record["chunk_key"])
|
||||
|
||||
# Trigger: chunk exists and hash matches (no content change)
|
||||
# but chunk has no embedding (orphan from crash).
|
||||
# Outcome: schedule re-embedding without touching chunk metadata.
|
||||
is_orphan = current and any(o.id == current.id for o in orphan_rows)
|
||||
if current and current.source_hash == record["source_hash"] and not is_orphan:
|
||||
continue
|
||||
|
||||
if current:
|
||||
row_id = int(current.id)
|
||||
if current.source_hash != record["source_hash"]:
|
||||
await session.execute(
|
||||
text(
|
||||
"UPDATE search_vector_chunks "
|
||||
"SET chunk_text = :chunk_text, source_hash = :source_hash, "
|
||||
f"updated_at = {timestamp_expr} "
|
||||
"WHERE id = :id"
|
||||
),
|
||||
{
|
||||
"id": row_id,
|
||||
"chunk_text": record["chunk_text"],
|
||||
"source_hash": record["source_hash"],
|
||||
},
|
||||
)
|
||||
embedding_jobs.append((row_id, record["chunk_text"]))
|
||||
continue
|
||||
|
||||
inserted = await session.execute(
|
||||
text(
|
||||
"INSERT INTO search_vector_chunks ("
|
||||
"entity_id, project_id, chunk_key, chunk_text, source_hash, updated_at"
|
||||
") VALUES ("
|
||||
f":entity_id, :project_id, :chunk_key, :chunk_text, :source_hash, "
|
||||
f"{timestamp_expr}"
|
||||
") RETURNING id"
|
||||
),
|
||||
{
|
||||
"entity_id": entity_id,
|
||||
"project_id": self.project_id,
|
||||
"chunk_key": record["chunk_key"],
|
||||
"chunk_text": record["chunk_text"],
|
||||
"source_hash": record["source_hash"],
|
||||
},
|
||||
)
|
||||
row_id = int(inserted.scalar_one())
|
||||
embedding_jobs.append((row_id, record["chunk_text"]))
|
||||
|
||||
await session.commit()
|
||||
|
||||
if not embedding_jobs:
|
||||
return
|
||||
|
||||
texts = [t for _, t in embedding_jobs]
|
||||
embeddings = await self._embedding_provider.embed_documents(texts)
|
||||
if len(embeddings) != len(embedding_jobs):
|
||||
raise RuntimeError("Embedding provider returned an unexpected number of vectors.")
|
||||
|
||||
async with db.scoped_session(self.session_maker) as session:
|
||||
await self._prepare_vector_session(session)
|
||||
await self._write_embeddings(session, embedding_jobs, embeddings)
|
||||
await session.commit()
|
||||
|
||||
async def _prepare_vector_session(self, session: AsyncSession) -> None:
|
||||
"""Hook for per-session setup (e.g. loading sqlite-vec extension).
|
||||
|
||||
Default implementation is a no-op. SQLite overrides this.
|
||||
"""
|
||||
pass
|
||||
|
||||
def _timestamp_now_expr(self) -> str:
|
||||
"""SQL expression for 'now' in the backend.
|
||||
|
||||
SQLite uses CURRENT_TIMESTAMP, Postgres uses NOW().
|
||||
"""
|
||||
return "CURRENT_TIMESTAMP"
|
||||
|
||||
def _orphan_detection_sql(self) -> str:
|
||||
"""SQL to find chunk rows without corresponding embeddings.
|
||||
|
||||
Default implementation works for both backends; SQLite overrides
|
||||
to reference the rowid-based embedding table layout.
|
||||
"""
|
||||
return (
|
||||
"SELECT c.id FROM search_vector_chunks c "
|
||||
"LEFT JOIN search_vector_embeddings e ON e.chunk_id = c.id "
|
||||
"WHERE c.project_id = :project_id AND c.entity_id = :entity_id "
|
||||
"AND e.chunk_id IS NULL"
|
||||
)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Shared semantic search: retrieval mode dispatch
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _check_vector_eligible(
|
||||
self,
|
||||
search_text: Optional[str],
|
||||
permalink: Optional[str],
|
||||
permalink_match: Optional[str],
|
||||
title: Optional[str],
|
||||
) -> bool:
|
||||
"""Check whether search_text allows vector / hybrid retrieval."""
|
||||
return (
|
||||
bool(search_text)
|
||||
and bool(search_text.strip())
|
||||
and search_text.strip() != "*"
|
||||
and not permalink
|
||||
and not permalink_match
|
||||
and not title
|
||||
)
|
||||
|
||||
async def _dispatch_retrieval_mode(
|
||||
self,
|
||||
*,
|
||||
search_text: Optional[str],
|
||||
permalink: Optional[str],
|
||||
permalink_match: Optional[str],
|
||||
title: Optional[str],
|
||||
types: Optional[List[str]],
|
||||
after_date: Optional[datetime],
|
||||
search_item_types: Optional[List[SearchItemType]],
|
||||
metadata_filters: Optional[dict],
|
||||
retrieval_mode: SearchRetrievalMode,
|
||||
limit: int,
|
||||
offset: int,
|
||||
) -> Optional[List[SearchIndexRow]]:
|
||||
"""Dispatch vector or hybrid retrieval if requested.
|
||||
|
||||
Returns None when the mode is FTS so the caller should continue
|
||||
with its backend-specific FTS query.
|
||||
"""
|
||||
mode = (
|
||||
retrieval_mode.value
|
||||
if isinstance(retrieval_mode, SearchRetrievalMode)
|
||||
else str(retrieval_mode)
|
||||
)
|
||||
can_use_vector = self._check_vector_eligible(search_text, permalink, permalink_match, title)
|
||||
search_text_value = search_text or ""
|
||||
|
||||
if mode == SearchRetrievalMode.VECTOR.value:
|
||||
if not can_use_vector:
|
||||
raise ValueError(
|
||||
"Vector retrieval requires a non-empty text query and does not support "
|
||||
"title/permalink-only searches."
|
||||
)
|
||||
return await self._search_vector_only(
|
||||
search_text=search_text_value,
|
||||
permalink=permalink,
|
||||
permalink_match=permalink_match,
|
||||
title=title,
|
||||
types=types,
|
||||
after_date=after_date,
|
||||
search_item_types=search_item_types,
|
||||
metadata_filters=metadata_filters,
|
||||
limit=limit,
|
||||
offset=offset,
|
||||
)
|
||||
if mode == SearchRetrievalMode.HYBRID.value:
|
||||
if not can_use_vector:
|
||||
raise ValueError(
|
||||
"Hybrid retrieval requires a non-empty text query and does not support "
|
||||
"title/permalink-only searches."
|
||||
)
|
||||
return await self._search_hybrid(
|
||||
search_text=search_text_value,
|
||||
permalink=permalink,
|
||||
permalink_match=permalink_match,
|
||||
title=title,
|
||||
types=types,
|
||||
after_date=after_date,
|
||||
search_item_types=search_item_types,
|
||||
metadata_filters=metadata_filters,
|
||||
limit=limit,
|
||||
offset=offset,
|
||||
)
|
||||
|
||||
# FTS mode: return None to let the subclass handle it
|
||||
return None
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Shared semantic search: vector-only retrieval
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
@staticmethod
|
||||
def _parse_chunk_key(chunk_key: str) -> tuple[str, int]:
|
||||
"""Parse a chunk_key like 'observation:5:0' into (type, search_index_id)."""
|
||||
parts = chunk_key.split(":")
|
||||
return parts[0], int(parts[1])
|
||||
|
||||
async def _search_vector_only(
|
||||
self,
|
||||
*,
|
||||
search_text: str,
|
||||
permalink: Optional[str],
|
||||
permalink_match: Optional[str],
|
||||
title: Optional[str],
|
||||
types: Optional[List[str]],
|
||||
after_date: Optional[datetime],
|
||||
search_item_types: Optional[List[SearchItemType]],
|
||||
metadata_filters: Optional[dict],
|
||||
limit: int,
|
||||
offset: int,
|
||||
) -> List[SearchIndexRow]:
|
||||
"""Run vector-only search returning chunk-level results.
|
||||
|
||||
Returns individual search_index rows (entities, observations, relations)
|
||||
ranked by vector similarity. Each observation or relation is a first-class
|
||||
result, not collapsed into its parent entity.
|
||||
"""
|
||||
self._assert_semantic_available()
|
||||
await self._ensure_vector_tables()
|
||||
assert self._embedding_provider is not None
|
||||
query_embedding = await self._embedding_provider.embed_query(search_text.strip())
|
||||
candidate_limit = max(self._semantic_vector_k, (limit + offset) * 5)
|
||||
|
||||
async with db.scoped_session(self.session_maker) as session:
|
||||
await self._prepare_vector_session(session)
|
||||
vector_rows = await self._run_vector_query(session, query_embedding, candidate_limit)
|
||||
|
||||
if not vector_rows:
|
||||
return []
|
||||
|
||||
# Build per-search_index_row similarity scores from chunk-level results.
|
||||
# Each chunk_key encodes the search_index row type and id.
|
||||
# Keep the best similarity per search_index row id.
|
||||
similarity_by_si_id: dict[int, float] = {}
|
||||
for row in vector_rows:
|
||||
chunk_key = row.get("chunk_key", "")
|
||||
distance = float(row["best_distance"])
|
||||
similarity = 1.0 / (1.0 + max(distance, 0.0))
|
||||
try:
|
||||
_, si_id = self._parse_chunk_key(chunk_key)
|
||||
except (ValueError, IndexError):
|
||||
# Fallback: group by entity_id for chunks without parseable keys
|
||||
continue
|
||||
current = similarity_by_si_id.get(si_id)
|
||||
if current is None or similarity > current:
|
||||
similarity_by_si_id[si_id] = similarity
|
||||
|
||||
if not similarity_by_si_id:
|
||||
return []
|
||||
|
||||
# Fetch the actual search_index rows
|
||||
si_ids = list(similarity_by_si_id.keys())
|
||||
search_index_rows = await self._fetch_search_index_rows_by_ids(si_ids)
|
||||
|
||||
# Apply optional filters if requested
|
||||
filter_requested = any(
|
||||
[
|
||||
permalink,
|
||||
permalink_match,
|
||||
title,
|
||||
types,
|
||||
after_date,
|
||||
search_item_types,
|
||||
metadata_filters,
|
||||
]
|
||||
)
|
||||
|
||||
if filter_requested:
|
||||
filtered_rows = await self.search(
|
||||
search_text=None,
|
||||
permalink=permalink,
|
||||
permalink_match=permalink_match,
|
||||
title=title,
|
||||
types=types,
|
||||
after_date=after_date,
|
||||
search_item_types=search_item_types,
|
||||
metadata_filters=metadata_filters,
|
||||
retrieval_mode=SearchRetrievalMode.FTS,
|
||||
limit=VECTOR_FILTER_SCAN_LIMIT,
|
||||
offset=0,
|
||||
)
|
||||
# Use (id, type) tuples to avoid collisions between different
|
||||
# search_index row types that share the same auto-increment id.
|
||||
allowed_keys = {(row.id, row.type) for row in filtered_rows if row.id is not None}
|
||||
search_index_rows = {
|
||||
k: v for k, v in search_index_rows.items() if (v.id, v.type) in allowed_keys
|
||||
}
|
||||
|
||||
ranked_rows: list[SearchIndexRow] = []
|
||||
for si_id, similarity in similarity_by_si_id.items():
|
||||
row = search_index_rows.get(si_id)
|
||||
if row is None:
|
||||
continue
|
||||
ranked_rows.append(replace(row, score=similarity))
|
||||
|
||||
ranked_rows.sort(key=lambda item: item.score or 0.0, reverse=True)
|
||||
return ranked_rows[offset : offset + limit]
|
||||
|
||||
async def _fetch_entity_rows_by_ids(self, entity_ids: list[int]) -> dict[int, SearchIndexRow]:
|
||||
"""Fetch entity-type search_index rows by their entity_id values."""
|
||||
placeholders = ",".join(f":id_{idx}" for idx in range(len(entity_ids)))
|
||||
params: dict[str, Any] = {
|
||||
**{f"id_{idx}": eid for idx, eid in enumerate(entity_ids)},
|
||||
"project_id": self.project_id,
|
||||
"item_type": SearchItemType.ENTITY.value,
|
||||
}
|
||||
sql = f"""
|
||||
SELECT
|
||||
project_id, id, title, permalink, file_path, type, metadata,
|
||||
from_id, to_id, relation_type, entity_id, content_snippet,
|
||||
category, created_at, updated_at, 0 as score
|
||||
FROM search_index
|
||||
WHERE project_id = :project_id
|
||||
AND type = :item_type
|
||||
AND entity_id IN ({placeholders})
|
||||
"""
|
||||
result: dict[int, SearchIndexRow] = {}
|
||||
async with db.scoped_session(self.session_maker) as session:
|
||||
row_result = await session.execute(text(sql), params)
|
||||
for row in row_result.fetchall():
|
||||
result[row.entity_id] = SearchIndexRow(
|
||||
project_id=self.project_id,
|
||||
id=row.id,
|
||||
title=row.title,
|
||||
permalink=row.permalink,
|
||||
file_path=row.file_path,
|
||||
type=row.type,
|
||||
score=0.0,
|
||||
metadata=(
|
||||
row.metadata
|
||||
if isinstance(row.metadata, dict)
|
||||
else (json.loads(row.metadata) if row.metadata else {})
|
||||
),
|
||||
from_id=row.from_id,
|
||||
to_id=row.to_id,
|
||||
relation_type=row.relation_type,
|
||||
entity_id=row.entity_id,
|
||||
content_snippet=row.content_snippet,
|
||||
category=row.category,
|
||||
created_at=row.created_at,
|
||||
updated_at=row.updated_at,
|
||||
)
|
||||
return result
|
||||
|
||||
async def _fetch_search_index_rows_by_ids(
|
||||
self, row_ids: list[int]
|
||||
) -> dict[int, SearchIndexRow]:
|
||||
"""Fetch search_index rows by their primary key (id), any type."""
|
||||
if not row_ids:
|
||||
return {}
|
||||
placeholders = ",".join(f":id_{idx}" for idx in range(len(row_ids)))
|
||||
params: dict[str, Any] = {
|
||||
**{f"id_{idx}": rid for idx, rid in enumerate(row_ids)},
|
||||
"project_id": self.project_id,
|
||||
}
|
||||
sql = f"""
|
||||
SELECT
|
||||
project_id, id, title, permalink, file_path, type, metadata,
|
||||
from_id, to_id, relation_type, entity_id, content_snippet,
|
||||
category, created_at, updated_at, 0 as score
|
||||
FROM search_index
|
||||
WHERE project_id = :project_id
|
||||
AND id IN ({placeholders})
|
||||
"""
|
||||
result: dict[int, SearchIndexRow] = {}
|
||||
async with db.scoped_session(self.session_maker) as session:
|
||||
row_result = await session.execute(text(sql), params)
|
||||
for row in row_result.fetchall():
|
||||
result[row.id] = SearchIndexRow(
|
||||
project_id=self.project_id,
|
||||
id=row.id,
|
||||
title=row.title,
|
||||
permalink=row.permalink,
|
||||
file_path=row.file_path,
|
||||
type=row.type,
|
||||
score=0.0,
|
||||
metadata=(
|
||||
row.metadata
|
||||
if isinstance(row.metadata, dict)
|
||||
else (json.loads(row.metadata) if row.metadata else {})
|
||||
),
|
||||
from_id=row.from_id,
|
||||
to_id=row.to_id,
|
||||
relation_type=row.relation_type,
|
||||
entity_id=row.entity_id,
|
||||
content_snippet=row.content_snippet,
|
||||
category=row.category,
|
||||
created_at=row.created_at,
|
||||
updated_at=row.updated_at,
|
||||
)
|
||||
return result
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Shared semantic search: hybrid RRF fusion
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
async def _search_hybrid(
|
||||
self,
|
||||
*,
|
||||
search_text: str,
|
||||
permalink: Optional[str],
|
||||
permalink_match: Optional[str],
|
||||
title: Optional[str],
|
||||
types: Optional[List[str]],
|
||||
after_date: Optional[datetime],
|
||||
search_item_types: Optional[List[SearchItemType]],
|
||||
metadata_filters: Optional[dict],
|
||||
limit: int,
|
||||
offset: int,
|
||||
) -> List[SearchIndexRow]:
|
||||
"""Fuse FTS and vector rankings using reciprocal rank fusion (RRF).
|
||||
|
||||
Uses entity_id as the fusion key (not permalink) to correctly handle
|
||||
entities with NULL permalinks.
|
||||
"""
|
||||
self._assert_semantic_available()
|
||||
candidate_limit = max(self._semantic_vector_k, (limit + offset) * 10)
|
||||
fts_results = await self.search(
|
||||
search_text=search_text,
|
||||
permalink=permalink,
|
||||
permalink_match=permalink_match,
|
||||
title=title,
|
||||
types=types,
|
||||
after_date=after_date,
|
||||
search_item_types=search_item_types,
|
||||
metadata_filters=metadata_filters,
|
||||
retrieval_mode=SearchRetrievalMode.FTS,
|
||||
limit=candidate_limit,
|
||||
offset=0,
|
||||
)
|
||||
vector_results = await self._search_vector_only(
|
||||
search_text=search_text,
|
||||
permalink=permalink,
|
||||
permalink_match=permalink_match,
|
||||
title=title,
|
||||
types=types,
|
||||
after_date=after_date,
|
||||
search_item_types=search_item_types,
|
||||
metadata_filters=metadata_filters,
|
||||
limit=candidate_limit,
|
||||
offset=0,
|
||||
)
|
||||
|
||||
# RRF fusion keyed on search_index row id for granular results.
|
||||
# This allows observations and relations to surface as individual results,
|
||||
# not collapsed into their parent entity.
|
||||
fused_scores: dict[int, float] = {}
|
||||
rows_by_id: dict[int, SearchIndexRow] = {}
|
||||
|
||||
for rank, row in enumerate(fts_results, start=1):
|
||||
if row.id is None:
|
||||
continue
|
||||
fused_scores[row.id] = fused_scores.get(row.id, 0.0) + (1.0 / (RRF_K + rank))
|
||||
rows_by_id[row.id] = row
|
||||
|
||||
for rank, row in enumerate(vector_results, start=1):
|
||||
if row.id is None:
|
||||
continue
|
||||
fused_scores[row.id] = fused_scores.get(row.id, 0.0) + (1.0 / (RRF_K + rank))
|
||||
rows_by_id[row.id] = row
|
||||
|
||||
ranked = sorted(fused_scores.items(), key=lambda item: item[1], reverse=True)
|
||||
output: list[SearchIndexRow] = []
|
||||
for row_id, fused_score in ranked[offset : offset + limit]:
|
||||
output.append(replace(rows_by_id[row_id], score=fused_score))
|
||||
return output
|
||||
|
||||
@@ -1,9 +0,0 @@
|
||||
"""Typed errors for semantic search configuration and dependency failures."""
|
||||
|
||||
|
||||
class SemanticSearchDisabledError(RuntimeError):
|
||||
"""Raised when vector or hybrid retrieval is requested but semantic search is disabled."""
|
||||
|
||||
|
||||
class SemanticDependenciesMissingError(RuntimeError):
|
||||
"""Raised when a semantic search dependency is unavailable or misconfigured."""
|
||||
@@ -5,28 +5,16 @@ import re
|
||||
from datetime import datetime
|
||||
from typing import List, Optional
|
||||
|
||||
import asyncio
|
||||
|
||||
from loguru import logger
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.exc import OperationalError as SAOperationalError
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from basic_memory import db
|
||||
from basic_memory.config import BasicMemoryConfig, ConfigManager
|
||||
from basic_memory.models.search import (
|
||||
CREATE_SEARCH_INDEX,
|
||||
CREATE_SQLITE_SEARCH_VECTOR_CHUNKS,
|
||||
CREATE_SQLITE_SEARCH_VECTOR_CHUNKS_PROJECT_ENTITY,
|
||||
CREATE_SQLITE_SEARCH_VECTOR_CHUNKS_UNIQUE,
|
||||
create_sqlite_search_vector_embeddings,
|
||||
)
|
||||
from basic_memory.repository.embedding_provider import EmbeddingProvider
|
||||
from basic_memory.repository.embedding_provider_factory import create_embedding_provider
|
||||
from basic_memory.models.search import CREATE_SEARCH_INDEX
|
||||
from basic_memory.repository.search_index_row import SearchIndexRow
|
||||
from basic_memory.repository.search_repository_base import SearchRepositoryBase
|
||||
from basic_memory.repository.metadata_filters import parse_metadata_filters, build_sqlite_json_path
|
||||
from basic_memory.repository.semantic_errors import SemanticDependenciesMissingError
|
||||
from basic_memory.schemas.search import SearchItemType, SearchRetrievalMode
|
||||
from basic_memory.schemas.search import SearchItemType
|
||||
|
||||
|
||||
class SQLiteSearchRepository(SearchRepositoryBase):
|
||||
@@ -39,27 +27,9 @@ class SQLiteSearchRepository(SearchRepositoryBase):
|
||||
- Prefix wildcard matching with *
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
session_maker,
|
||||
project_id: int,
|
||||
app_config: BasicMemoryConfig | None = None,
|
||||
embedding_provider: EmbeddingProvider | None = None,
|
||||
):
|
||||
def __init__(self, session_maker, project_id: int):
|
||||
super().__init__(session_maker, project_id)
|
||||
self._entity_columns: set[str] | None = None
|
||||
self._app_config = app_config or ConfigManager().config
|
||||
self._semantic_enabled = self._app_config.semantic_search_enabled
|
||||
self._semantic_vector_k = self._app_config.semantic_vector_k
|
||||
self._embedding_provider = embedding_provider
|
||||
self._sqlite_vec_lock = asyncio.Lock()
|
||||
self._vector_tables_initialized = False
|
||||
self._vector_dimensions = 384
|
||||
|
||||
if self._semantic_enabled and self._embedding_provider is None:
|
||||
self._embedding_provider = create_embedding_provider(self._app_config)
|
||||
if self._embedding_provider is not None:
|
||||
self._vector_dimensions = self._embedding_provider.dimensions
|
||||
|
||||
async def _get_entity_columns(self) -> set[str]:
|
||||
if self._entity_columns is None:
|
||||
@@ -84,10 +54,6 @@ class SQLiteSearchRepository(SearchRepositoryBase):
|
||||
logger.error(f"Error initializing search index: {e}")
|
||||
raise e
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# FTS5 query preparation (backend-specific)
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _prepare_boolean_query(self, query: str) -> str:
|
||||
"""Prepare a Boolean query by quoting individual terms while preserving operators.
|
||||
|
||||
@@ -324,237 +290,6 @@ class SQLiteSearchRepository(SearchRepositoryBase):
|
||||
# For non-Boolean queries, use the single term preparation logic
|
||||
return self._prepare_single_term(term, is_prefix)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# sqlite-vec extension loading (SQLite-specific)
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
async def _ensure_sqlite_vec_loaded(self, session) -> None:
|
||||
try:
|
||||
await session.execute(text("SELECT vec_version()"))
|
||||
return
|
||||
except SAOperationalError:
|
||||
pass
|
||||
|
||||
try:
|
||||
import sqlite_vec
|
||||
except ImportError as exc:
|
||||
raise SemanticDependenciesMissingError(
|
||||
"sqlite-vec package is missing. "
|
||||
"Install semantic extras: pip install 'basic-memory[semantic]'"
|
||||
) from exc
|
||||
|
||||
async with self._sqlite_vec_lock:
|
||||
try:
|
||||
await session.execute(text("SELECT vec_version()"))
|
||||
return
|
||||
except SAOperationalError:
|
||||
pass
|
||||
|
||||
async_connection = await session.connection()
|
||||
raw_connection = await async_connection.get_raw_connection()
|
||||
driver_connection = raw_connection.driver_connection
|
||||
await driver_connection.enable_load_extension(True)
|
||||
await driver_connection.load_extension(sqlite_vec.loadable_path())
|
||||
await driver_connection.enable_load_extension(False)
|
||||
await session.execute(text("SELECT vec_version()"))
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Abstract hook implementations (vector/semantic, SQLite-specific)
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
async def _ensure_vector_tables(self) -> None:
|
||||
self._assert_semantic_available()
|
||||
if self._vector_tables_initialized:
|
||||
return
|
||||
|
||||
async with db.scoped_session(self.session_maker) as session:
|
||||
await self._ensure_sqlite_vec_loaded(session)
|
||||
|
||||
chunks_columns_result = await session.execute(
|
||||
text("PRAGMA table_info(search_vector_chunks)")
|
||||
)
|
||||
chunks_columns = [row[1] for row in chunks_columns_result.fetchall()]
|
||||
|
||||
expected_columns = {
|
||||
"id",
|
||||
"entity_id",
|
||||
"project_id",
|
||||
"chunk_key",
|
||||
"chunk_text",
|
||||
"source_hash",
|
||||
"updated_at",
|
||||
}
|
||||
schema_mismatch = bool(chunks_columns) and set(chunks_columns) != expected_columns
|
||||
if schema_mismatch:
|
||||
await session.execute(text("DROP TABLE IF EXISTS search_vector_embeddings"))
|
||||
await session.execute(text("DROP TABLE IF EXISTS search_vector_chunks"))
|
||||
|
||||
await session.execute(CREATE_SQLITE_SEARCH_VECTOR_CHUNKS)
|
||||
await session.execute(CREATE_SQLITE_SEARCH_VECTOR_CHUNKS_PROJECT_ENTITY)
|
||||
await session.execute(CREATE_SQLITE_SEARCH_VECTOR_CHUNKS_UNIQUE)
|
||||
|
||||
# Trigger: legacy table from previous semantic implementation exists.
|
||||
# Why: old schema stores JSON vectors in a normal table and conflicts with sqlite-vec.
|
||||
# Outcome: remove disposable derived data so chunk/vector schema is deterministic.
|
||||
await session.execute(text("DROP TABLE IF EXISTS search_vector_index"))
|
||||
|
||||
vector_sql_result = await session.execute(
|
||||
text(
|
||||
"SELECT sql FROM sqlite_master "
|
||||
"WHERE type = 'table' AND name = 'search_vector_embeddings'"
|
||||
)
|
||||
)
|
||||
vector_sql = vector_sql_result.scalar()
|
||||
expected_dimension_sql = f"float[{self._vector_dimensions}]"
|
||||
|
||||
if vector_sql and expected_dimension_sql not in vector_sql:
|
||||
await session.execute(text("DROP TABLE IF EXISTS search_vector_embeddings"))
|
||||
|
||||
await session.execute(create_sqlite_search_vector_embeddings(self._vector_dimensions))
|
||||
await session.commit()
|
||||
|
||||
self._vector_tables_initialized = True
|
||||
|
||||
async def _prepare_vector_session(self, session: AsyncSession) -> None:
|
||||
"""Load sqlite-vec extension for the session."""
|
||||
await self._ensure_sqlite_vec_loaded(session)
|
||||
|
||||
async def _run_vector_query(
|
||||
self,
|
||||
session: AsyncSession,
|
||||
query_embedding: list[float],
|
||||
candidate_limit: int,
|
||||
) -> list[dict]:
|
||||
query_embedding_json = json.dumps(query_embedding)
|
||||
vector_result = await session.execute(
|
||||
text(
|
||||
"WITH vector_matches AS ("
|
||||
" SELECT rowid, distance "
|
||||
" FROM search_vector_embeddings "
|
||||
" WHERE embedding MATCH :query_embedding "
|
||||
" AND k = :vector_k"
|
||||
") "
|
||||
"SELECT c.entity_id, c.chunk_key, vector_matches.distance AS best_distance "
|
||||
"FROM vector_matches "
|
||||
"JOIN search_vector_chunks c ON c.id = vector_matches.rowid "
|
||||
"WHERE c.project_id = :project_id "
|
||||
"ORDER BY best_distance ASC "
|
||||
"LIMIT :vector_k"
|
||||
),
|
||||
{
|
||||
"query_embedding": query_embedding_json,
|
||||
"project_id": self.project_id,
|
||||
"vector_k": candidate_limit,
|
||||
},
|
||||
)
|
||||
return [dict(row) for row in vector_result.mappings().all()]
|
||||
|
||||
async def _write_embeddings(
|
||||
self,
|
||||
session: AsyncSession,
|
||||
jobs: list[tuple[int, str]],
|
||||
embeddings: list[list[float]],
|
||||
) -> None:
|
||||
rowids = [row_id for row_id, _ in jobs]
|
||||
delete_params = {f"rowid_{idx}": rowid for idx, rowid in enumerate(rowids)}
|
||||
delete_placeholders = ", ".join(f":rowid_{idx}" for idx in range(len(rowids)))
|
||||
await session.execute(
|
||||
text(f"DELETE FROM search_vector_embeddings WHERE rowid IN ({delete_placeholders})"),
|
||||
delete_params,
|
||||
)
|
||||
|
||||
insert_rows = [
|
||||
{"rowid": row_id, "embedding": json.dumps(embedding)}
|
||||
for (row_id, _), embedding in zip(jobs, embeddings, strict=True)
|
||||
]
|
||||
await session.execute(
|
||||
text(
|
||||
"INSERT INTO search_vector_embeddings (rowid, embedding) "
|
||||
"VALUES (:rowid, :embedding)"
|
||||
),
|
||||
insert_rows,
|
||||
)
|
||||
|
||||
async def _delete_entity_chunks(
|
||||
self,
|
||||
session: AsyncSession,
|
||||
entity_id: int,
|
||||
) -> None:
|
||||
# sqlite-vec has no CASCADE — must delete embeddings before chunks
|
||||
await session.execute(
|
||||
text(
|
||||
"DELETE FROM search_vector_embeddings "
|
||||
"WHERE rowid IN ("
|
||||
"SELECT id FROM search_vector_chunks "
|
||||
"WHERE project_id = :project_id AND entity_id = :entity_id"
|
||||
")"
|
||||
),
|
||||
{"project_id": self.project_id, "entity_id": entity_id},
|
||||
)
|
||||
await session.execute(
|
||||
text(
|
||||
"DELETE FROM search_vector_chunks "
|
||||
"WHERE project_id = :project_id AND entity_id = :entity_id"
|
||||
),
|
||||
{"project_id": self.project_id, "entity_id": entity_id},
|
||||
)
|
||||
|
||||
async def _delete_stale_chunks(
|
||||
self,
|
||||
session: AsyncSession,
|
||||
stale_ids: list[int],
|
||||
entity_id: int,
|
||||
) -> None:
|
||||
stale_params = {
|
||||
"project_id": self.project_id,
|
||||
"entity_id": entity_id,
|
||||
**{f"row_{idx}": row_id for idx, row_id in enumerate(stale_ids)},
|
||||
}
|
||||
stale_placeholders = ", ".join(f":row_{idx}" for idx in range(len(stale_ids)))
|
||||
await session.execute(
|
||||
text(f"DELETE FROM search_vector_embeddings WHERE rowid IN ({stale_placeholders})"),
|
||||
stale_params,
|
||||
)
|
||||
await session.execute(
|
||||
text(
|
||||
"DELETE FROM search_vector_chunks "
|
||||
f"WHERE id IN ({stale_placeholders}) "
|
||||
"AND project_id = :project_id AND entity_id = :entity_id"
|
||||
),
|
||||
stale_params,
|
||||
)
|
||||
|
||||
async def _update_timestamp_sql(self) -> str:
|
||||
return "CURRENT_TIMESTAMP" # pragma: no cover
|
||||
|
||||
def _orphan_detection_sql(self) -> str:
|
||||
"""SQLite sqlite-vec uses rowid-based embedding table."""
|
||||
return (
|
||||
"SELECT c.id FROM search_vector_chunks c "
|
||||
"LEFT JOIN search_vector_embeddings e ON e.rowid = c.id "
|
||||
"WHERE c.project_id = :project_id AND c.entity_id = :entity_id "
|
||||
"AND e.rowid IS NULL"
|
||||
)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Index / bulk index overrides (FTS-only, no vector side-effects)
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
async def index_item(self, search_index_row: SearchIndexRow) -> None:
|
||||
"""Index a single row in FTS only.
|
||||
|
||||
Vector chunks are derived asynchronously via sync_entity_vectors().
|
||||
"""
|
||||
await super().index_item(search_index_row)
|
||||
|
||||
async def bulk_index_items(self, search_index_rows: List[SearchIndexRow]) -> None:
|
||||
"""Index multiple rows in FTS only."""
|
||||
await super().bulk_index_items(search_index_rows)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# FTS search (backend-specific)
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
async def search(
|
||||
self,
|
||||
search_text: Optional[str] = None,
|
||||
@@ -565,29 +300,10 @@ class SQLiteSearchRepository(SearchRepositoryBase):
|
||||
after_date: Optional[datetime] = None,
|
||||
search_item_types: Optional[List[SearchItemType]] = None,
|
||||
metadata_filters: Optional[dict] = None,
|
||||
retrieval_mode: SearchRetrievalMode = SearchRetrievalMode.FTS,
|
||||
limit: int = 10,
|
||||
offset: int = 0,
|
||||
) -> List[SearchIndexRow]:
|
||||
"""Search across all indexed content using SQLite FTS5."""
|
||||
# --- Dispatch vector / hybrid modes (shared logic) ---
|
||||
dispatched = await self._dispatch_retrieval_mode(
|
||||
search_text=search_text,
|
||||
permalink=permalink,
|
||||
permalink_match=permalink_match,
|
||||
title=title,
|
||||
types=types,
|
||||
after_date=after_date,
|
||||
search_item_types=search_item_types,
|
||||
metadata_filters=metadata_filters,
|
||||
retrieval_mode=retrieval_mode,
|
||||
limit=limit,
|
||||
offset=offset,
|
||||
)
|
||||
if dispatched is not None:
|
||||
return dispatched
|
||||
|
||||
# --- FTS mode (SQLite-specific) ---
|
||||
conditions = []
|
||||
params = {}
|
||||
order_by_clause = ""
|
||||
|
||||
@@ -1,58 +0,0 @@
|
||||
"""Schema system for Basic Memory.
|
||||
|
||||
Provides Picoschema-based validation for notes using observation/relation mapping.
|
||||
Schemas are just notes with type: schema — no new data model, no migration.
|
||||
"""
|
||||
|
||||
from basic_memory.schema.parser import (
|
||||
SchemaField,
|
||||
SchemaDefinition,
|
||||
parse_picoschema,
|
||||
parse_schema_note,
|
||||
)
|
||||
from basic_memory.schema.resolver import resolve_schema
|
||||
from basic_memory.schema.validator import (
|
||||
FieldResult,
|
||||
ValidationResult,
|
||||
validate_note,
|
||||
)
|
||||
from basic_memory.schema.inference import (
|
||||
FieldFrequency,
|
||||
InferenceResult,
|
||||
ObservationData,
|
||||
RelationData,
|
||||
NoteData,
|
||||
infer_schema,
|
||||
analyze_observations,
|
||||
analyze_relations,
|
||||
)
|
||||
from basic_memory.schema.diff import (
|
||||
SchemaDrift,
|
||||
diff_schema,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
# Parser
|
||||
"SchemaField",
|
||||
"SchemaDefinition",
|
||||
"parse_picoschema",
|
||||
"parse_schema_note",
|
||||
# Resolver
|
||||
"resolve_schema",
|
||||
# Validator
|
||||
"FieldResult",
|
||||
"ValidationResult",
|
||||
"validate_note",
|
||||
# Inference
|
||||
"FieldFrequency",
|
||||
"InferenceResult",
|
||||
"ObservationData",
|
||||
"RelationData",
|
||||
"NoteData",
|
||||
"infer_schema",
|
||||
"analyze_observations",
|
||||
"analyze_relations",
|
||||
# Diff
|
||||
"SchemaDrift",
|
||||
"diff_schema",
|
||||
]
|
||||
@@ -1,111 +0,0 @@
|
||||
"""Schema diff for Basic Memory.
|
||||
|
||||
Compares a schema definition against actual note usage to detect drift.
|
||||
Drift happens naturally as notes evolve -- new observation categories appear,
|
||||
old ones fall out of use, single-value fields become multi-value.
|
||||
|
||||
The diff engine reuses inference analysis internally, comparing inferred
|
||||
frequencies against the declared schema fields to surface:
|
||||
- New fields: common in notes but not declared in schema
|
||||
- Dropped fields: declared in schema but rare in actual notes
|
||||
- Cardinality changes: field changed from single to array or vice versa
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from basic_memory.schema.inference import (
|
||||
FieldFrequency,
|
||||
NoteData,
|
||||
analyze_observations,
|
||||
analyze_relations,
|
||||
)
|
||||
from basic_memory.schema.parser import SchemaDefinition
|
||||
|
||||
|
||||
@dataclass
|
||||
class SchemaDrift:
|
||||
"""Result of comparing a schema against actual note usage."""
|
||||
|
||||
new_fields: list[FieldFrequency] = field(default_factory=list)
|
||||
dropped_fields: list[FieldFrequency] = field(default_factory=list)
|
||||
cardinality_changes: list[str] = field(default_factory=list)
|
||||
|
||||
|
||||
def diff_schema(
|
||||
schema: SchemaDefinition,
|
||||
notes: list[NoteData],
|
||||
new_field_threshold: float = 0.25,
|
||||
dropped_field_threshold: float = 0.10,
|
||||
) -> SchemaDrift:
|
||||
"""Compare a schema against actual note usage to detect drift.
|
||||
|
||||
Args:
|
||||
schema: The current schema definition.
|
||||
notes: List of NoteData objects representing actual notes.
|
||||
new_field_threshold: Frequency above which an undeclared field is considered
|
||||
"new" and worth adding to the schema.
|
||||
dropped_field_threshold: Frequency below which a declared field is considered
|
||||
"dropped" and worth removing from the schema.
|
||||
|
||||
Returns:
|
||||
A SchemaDrift describing the differences between schema and reality.
|
||||
"""
|
||||
total = len(notes)
|
||||
if total == 0:
|
||||
return SchemaDrift()
|
||||
|
||||
# --- Analyze actual usage ---
|
||||
obs_frequencies = analyze_observations(notes, total, max_sample_values=3)
|
||||
rel_frequencies = analyze_relations(notes, total, max_sample_values=3)
|
||||
|
||||
# Build lookup from schema fields
|
||||
schema_field_names = {f.name for f in schema.fields}
|
||||
|
||||
# Build lookup from actual frequencies
|
||||
obs_freq_by_name = {f.name: f for f in obs_frequencies}
|
||||
rel_freq_by_name = {f.name: f for f in rel_frequencies}
|
||||
all_freq_by_name = {**obs_freq_by_name, **rel_freq_by_name}
|
||||
|
||||
result = SchemaDrift()
|
||||
|
||||
# --- Detect new fields ---
|
||||
# Fields that appear frequently in notes but aren't declared in the schema
|
||||
for freq in obs_frequencies + rel_frequencies:
|
||||
if freq.name not in schema_field_names and freq.percentage >= new_field_threshold:
|
||||
result.new_fields.append(freq)
|
||||
|
||||
# --- Detect dropped fields ---
|
||||
# Fields declared in the schema but rarely appearing in actual notes
|
||||
for schema_field in schema.fields:
|
||||
freq = all_freq_by_name.get(schema_field.name)
|
||||
if freq is None:
|
||||
# Field doesn't appear at all in any note
|
||||
result.dropped_fields.append(
|
||||
FieldFrequency(
|
||||
name=schema_field.name,
|
||||
source="relation" if schema_field.is_entity_ref else "observation",
|
||||
count=0,
|
||||
total=total,
|
||||
percentage=0.0,
|
||||
)
|
||||
)
|
||||
elif freq.percentage < dropped_field_threshold:
|
||||
result.dropped_fields.append(freq)
|
||||
|
||||
# --- Detect cardinality changes ---
|
||||
# Fields where the schema says single but usage shows array, or vice versa
|
||||
for schema_field in schema.fields:
|
||||
freq = all_freq_by_name.get(schema_field.name)
|
||||
if freq is None:
|
||||
continue
|
||||
|
||||
if schema_field.is_array and not freq.is_array:
|
||||
result.cardinality_changes.append(
|
||||
f"{schema_field.name}: schema declares array but usage is typically single-value"
|
||||
)
|
||||
elif not schema_field.is_array and freq.is_array:
|
||||
result.cardinality_changes.append(
|
||||
f"{schema_field.name}: schema declares single-value but usage is typically array"
|
||||
)
|
||||
|
||||
return result
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user