admin/basicmachines-co-basic-memory
1
0
Fork 0
mirror of https://github.com/basicmachines-co/basic-memory synced 2026-06-21 13:47:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: Schema system for Basic Memory (#549)

Browse Source 
Signed-off-by: phernandez <paul@basicmachines.co>
Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Paul Hernandez
2026-02-11 15:04:42 -06:00
committed by GitHub
parent 00537272c6
commit c97733d785
108 changed files with 7123 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/basic_memory/api/app.py
+2
View File
@@ -18,6 +18,7 @@ from basic_memory.api.v2.routers import (
directory_router as v2_directory,
prompt_router as v2_prompt,
importer_router as v2_importer,
schema_router as v2_schema,
)
from basic_memory.api.v2.routers.project_router import (
add_project,
@@ -84,6 +85,7 @@ app.include_router(v2_resource, prefix="/v2/projects/{project_id}")
app.include_router(v2_directory, prefix="/v2/projects/{project_id}")
app.include_router(v2_prompt, prefix="/v2/projects/{project_id}")
app.include_router(v2_importer, prefix="/v2/projects/{project_id}")
app.include_router(v2_schema, prefix="/v2/projects/{project_id}")
app.include_router(v2_project, prefix="/v2")
# Legacy web app proxy paths (compat with /proxy/projects/projects)
src/basic_memory/api/v2/routers/__init__.py
+2
View File
@@ -8,6 +8,7 @@ from basic_memory.api.v2.routers.resource_router import router as resource_route
from basic_memory.api.v2.routers.directory_router import router as directory_router
from basic_memory.api.v2.routers.prompt_router import router as prompt_router
from basic_memory.api.v2.routers.importer_router import router as importer_router
from basic_memory.api.v2.routers.schema_router import router as schema_router
__all__ = [
"knowledge_router",
@@ -18,4 +19,5 @@ __all__ = [
"directory_router",
"prompt_router",
"importer_router",
"schema_router",
]
src/basic_memory/api/v2/routers/schema_router.py
+305
View File
@@ -0,0 +1,305 @@
"""V2 router for schema operations.
Provides endpoints for schema validation, inference, and drift detection.
The schema system validates notes against Picoschema definitions without
introducing any new data model -- it works entirely with existing
observations and relations.
Flow: Entity loaded with eager observations/relations -> convert to tuples -> core functions.
"""
from fastapi import APIRouter, Path, Query
from basic_memory.deps import (
SearchServiceV2ExternalDep,
EntityRepositoryV2ExternalDep,
)
from basic_memory.models.knowledge import Entity
from basic_memory.schemas.schema import (
ValidationReport,
InferenceReport,
DriftReport,
NoteValidationResponse,
FieldResultResponse,
FieldFrequencyResponse,
DriftFieldResponse,
)
from basic_memory.schemas.search import SearchQuery
from basic_memory.schema.resolver import resolve_schema
from basic_memory.schema.validator import validate_note
from basic_memory.schema.inference import infer_schema, NoteData, ObservationData, RelationData
from basic_memory.schema.diff import diff_schema
# Note: No prefix here -- it's added during registration as /v2/{project_id}/schema
router = APIRouter(tags=["schema"])
# --- ORM to core data conversion ---
def _entity_observations(entity: Entity) -> list[ObservationData]:
"""Extract ObservationData from an entity's observations."""
return [ObservationData(obs.category, obs.content) for obs in entity.observations]
def _entity_relations(entity: Entity) -> list[RelationData]:
"""Extract RelationData from an entity's outgoing relations.
Carries the target entity's type on each relation so the inference engine
can suggest correct types (e.g. works_at -> Organization, not the source type).
"""
return [
RelationData(
relation_type=rel.relation_type,
target_name=rel.to_name,
target_entity_type=rel.to_entity.entity_type if rel.to_entity else None,
)
for rel in entity.outgoing_relations
]
def _entity_to_note_data(entity: Entity) -> NoteData:
"""Convert an ORM Entity to a NoteData for inference/diff analysis."""
return NoteData(
identifier=entity.permalink or entity.file_path,
observations=_entity_observations(entity),
relations=_entity_relations(entity),
)
def _entity_frontmatter(entity: Entity) -> dict:
"""Build a frontmatter dict from an entity for schema resolution."""
frontmatter = dict(entity.entity_metadata) if entity.entity_metadata else {}
if entity.entity_type:
frontmatter.setdefault("type", entity.entity_type)
return frontmatter
# --- Validation ---
@router.post("/schema/validate", response_model=ValidationReport)
async def validate_schema(
entity_repository: EntityRepositoryV2ExternalDep,
search_service: SearchServiceV2ExternalDep,
project_id: str = Path(..., description="Project external UUID"),
entity_type: str | None = Query(None, description="Entity type to validate"),
identifier: str | None = Query(None, description="Specific note identifier"),
):
"""Validate notes against their resolved schemas.
Validates a specific note (by identifier) or all notes of a given type.
Returns warnings/errors based on the schema's validation mode.
"""
results: list[NoteValidationResponse] = []
async def search_fn(query: str) -> list:
# Search for schema notes, then load full entity_metadata from the entity table.
# The search index only stores minimal metadata (e.g., {"entity_type": "schema"}),
# but parse_schema_note needs the full frontmatter with entity/schema/version keys.
results = await search_service.search(SearchQuery(text=query, types=["schema"]), limit=5)
frontmatters = []
for row in results:
if row.permalink:
entity = await entity_repository.get_by_permalink(row.permalink)
if entity:
frontmatters.append(_entity_frontmatter(entity))
return frontmatters
# --- Single note validation ---
if identifier:
entity = await entity_repository.get_by_permalink(identifier)
if not entity:
return ValidationReport(entity_type=entity_type, total_notes=0, results=[])
schema_def = await resolve_schema(_entity_frontmatter(entity), search_fn)
if schema_def:
result = validate_note(
entity.permalink or identifier,
schema_def,
_entity_observations(entity),
_entity_relations(entity),
)
results.append(_to_note_validation_response(result))
return ValidationReport(
entity_type=entity_type or entity.entity_type,
total_notes=1,
valid_count=1 if (results and results[0].passed) else 0,
warning_count=sum(len(r.warnings) for r in results),
error_count=sum(len(r.errors) for r in results),
results=results,
)
# --- Batch validation by entity type ---
entities = await _find_by_entity_type(entity_repository, entity_type) if entity_type else []
for entity in entities:
schema_def = await resolve_schema(_entity_frontmatter(entity), search_fn)
if schema_def:
result = validate_note(
entity.permalink or entity.file_path,
schema_def,
_entity_observations(entity),
_entity_relations(entity),
)
results.append(_to_note_validation_response(result))
valid = sum(1 for r in results if r.passed)
return ValidationReport(
entity_type=entity_type,
total_notes=len(results),
valid_count=valid,
warning_count=sum(len(r.warnings) for r in results),
error_count=sum(len(r.errors) for r in results),
results=results,
)
# --- Inference ---
@router.post("/schema/infer", response_model=InferenceReport)
async def infer_schema_endpoint(
entity_repository: EntityRepositoryV2ExternalDep,
project_id: str = Path(..., description="Project external UUID"),
entity_type: str = Query(..., description="Entity type to analyze"),
threshold: float = Query(0.25, description="Minimum frequency for optional fields"),
):
"""Infer a schema from existing notes of a given type.
Examines observation categories and relation types across all notes
of the given type. Returns frequency analysis and suggested Picoschema.
"""
entities = await _find_by_entity_type(entity_repository, entity_type)
notes_data = [_entity_to_note_data(entity) for entity in entities]
result = infer_schema(entity_type, notes_data, optional_threshold=threshold)
return InferenceReport(
entity_type=result.entity_type,
notes_analyzed=result.notes_analyzed,
field_frequencies=[
FieldFrequencyResponse(
name=f.name,
source=f.source,
count=f.count,
total=f.total,
percentage=f.percentage,
sample_values=f.sample_values,
is_array=f.is_array,
target_type=f.target_type,
)
for f in result.field_frequencies
],
suggested_schema=result.suggested_schema,
suggested_required=result.suggested_required,
suggested_optional=result.suggested_optional,
excluded=result.excluded,
)
# --- Drift Detection ---
@router.get("/schema/diff/{entity_type}", response_model=DriftReport)
async def diff_schema_endpoint(
entity_repository: EntityRepositoryV2ExternalDep,
search_service: SearchServiceV2ExternalDep,
entity_type: str = Path(..., description="Entity type to check for drift"),
project_id: str = Path(..., description="Project external UUID"),
):
"""Show drift between a schema definition and actual note usage.
Compares the existing schema for an entity type against how notes
of that type are actually structured. Identifies new fields, dropped
fields, and cardinality changes.
"""
async def search_fn(query: str) -> list:
# Search for schema notes, then load full entity_metadata from the entity table.
# The search index only stores minimal metadata (e.g., {"entity_type": "schema"}),
# but parse_schema_note needs the full frontmatter with entity/schema/version keys.
results = await search_service.search(SearchQuery(text=query, types=["schema"]), limit=5)
frontmatters = []
for row in results:
if row.permalink:
entity = await entity_repository.get_by_permalink(row.permalink)
if entity:
frontmatters.append(_entity_frontmatter(entity))
return frontmatters
# Resolve schema by entity type
schema_frontmatter = {"type": entity_type}
schema_def = await resolve_schema(schema_frontmatter, search_fn)
if not schema_def:
return DriftReport(entity_type=entity_type)
# Collect all notes of this type
entities = await _find_by_entity_type(entity_repository, entity_type)
notes_data = [_entity_to_note_data(entity) for entity in entities]
result = diff_schema(schema_def, notes_data)
return DriftReport(
entity_type=entity_type,
new_fields=[
DriftFieldResponse(
name=f.name,
source=f.source,
count=f.count,
total=f.total,
percentage=f.percentage,
)
for f in result.new_fields
],
dropped_fields=[
DriftFieldResponse(
name=f.name,
source=f.source,
count=f.count,
total=f.total,
percentage=f.percentage,
)
for f in result.dropped_fields
],
cardinality_changes=result.cardinality_changes,
)
# --- Helpers ---
async def _find_by_entity_type(
entity_repository: EntityRepositoryV2ExternalDep,
entity_type: str,
) -> list[Entity]:
"""Find all entities of a given type using the repository's select pattern."""
query = entity_repository.select().where(Entity.entity_type == entity_type)
result = await entity_repository.execute_query(query)
return list(result.scalars().all())
def _to_note_validation_response(result) -> NoteValidationResponse:
"""Convert a core ValidationResult to a Pydantic response model."""
return NoteValidationResponse(
note_identifier=result.note_identifier,
schema_entity=result.schema_entity,
passed=result.passed,
field_results=[
FieldResultResponse(
field_name=fr.field.name,
field_type=fr.field.type,
required=fr.field.required,
status=fr.status,
values=fr.values,
message=fr.message,
)
for fr in result.field_results
],
unmatched_observations=result.unmatched_observations,
unmatched_relations=result.unmatched_relations,
warnings=result.warnings,
errors=result.errors,
)
src/basic_memory/cli/commands/__init__.py
+2 -1
View File
@@ -1,7 +1,7 @@
"""CLI commands for basic-memory."""
from . import status, db, doctor, import_memory_json, mcp, import_claude_conversations
from . import import_claude_projects, import_chatgpt, tool, project, format
from . import import_claude_projects, import_chatgpt, tool, project, format, schema
__all__ = [
"status",
@@ -15,4 +15,5 @@ __all__ = [
"tool",
"project",
"format",
"schema",
]
src/basic_memory/cli/commands/schema.py
+336
View File
@@ -0,0 +1,336 @@
"""Schema management CLI commands for Basic Memory.
Provides CLI access to schema validation, inference, and drift detection.
Registered as a subcommand group: `bm schema validate`, `bm schema infer`, `bm schema diff`.
"""
import json
from typing import Annotated, Optional
import typer
from loguru import logger
from rich.console import Console
from rich.panel import Panel
from rich.table import Table
from basic_memory.cli.app import app
from basic_memory.cli.commands.command_utils import run_with_cleanup
from basic_memory.cli.commands.routing import force_routing, validate_routing_flags
from basic_memory.config import ConfigManager
from basic_memory.mcp.async_client import get_client
from basic_memory.mcp.project_context import get_active_project
console = Console()
schema_app = typer.Typer(help="Schema management commands")
app.add_typer(schema_app, name="schema")
def _resolve_project_name(project: Optional[str]) -> Optional[str]:
"""Resolve project name from CLI argument or config default."""
config_manager = ConfigManager()
if project is not None:
project_name, _ = config_manager.get_project(project)
if not project_name:
typer.echo(f"No project found named: {project}", err=True)
raise typer.Exit(1)
return project_name
return config_manager.default_project
# --- Validate ---
async def _run_validate(
target: Optional[str] = None,
project: Optional[str] = None,
strict: bool = False,
):
"""Run schema validation via the API."""
from basic_memory.mcp.clients.schema import SchemaClient
async with get_client() as client:
active_project = await get_active_project(client, project, None)
schema_client = SchemaClient(client, active_project.external_id)
# Determine if target is a note identifier or entity type
# Heuristic: if target contains / or ., treat as identifier
entity_type = None
identifier = None
if target:
if "/" in target or "." in target:
identifier = target
else:
entity_type = target
report = await schema_client.validate(
entity_type=entity_type,
identifier=identifier,
)
# --- Display results ---
if report.total_notes == 0:
console.print("[yellow]No notes matched for validation.[/yellow]")
return
table = Table(title=f"Schema Validation: {entity_type or identifier or 'all'}")
table.add_column("Note", style="cyan")
table.add_column("Status", justify="center")
table.add_column("Warnings", justify="right")
table.add_column("Errors", justify="right")
for result in report.results:
if result.passed and not result.warnings:
status = "[green]pass[/green]"
elif result.passed:
status = "[yellow]warn[/yellow]"
else:
status = "[red]fail[/red]"
table.add_row(
result.note_identifier,
status,
str(len(result.warnings)),
str(len(result.errors)),
)
console.print(table)
console.print(
f"\nSummary: {report.valid_count}/{report.total_notes} valid, "
f"{report.warning_count} warnings, {report.error_count} errors"
)
# Exit with error code in strict mode if there are failures
if strict and report.error_count > 0:
raise typer.Exit(1)
@schema_app.command()
def validate(
target: Annotated[
Optional[str],
typer.Argument(help="Note path or entity type to validate"),
] = None,
project: Annotated[
Optional[str],
typer.Option(help="The project name."),
] = None,
strict: bool = typer.Option(False, "--strict", help="Exit with error on validation failures"),
local: bool = typer.Option(
False, "--local", help="Force local API routing (ignore cloud mode)"
),
cloud: bool = typer.Option(False, "--cloud", help="Force cloud API routing"),
):
"""Validate notes against their schemas.
TARGET can be a note path (e.g., people/ada-lovelace.md) or an entity type
(e.g., Person). If omitted, validates all notes that have schemas.
Use --strict to exit with error code 1 if any validation errors are found.
Use --local to force local routing when cloud mode is enabled.
Use --cloud to force cloud routing when cloud mode is disabled.
"""
try:
validate_routing_flags(local, cloud)
project_name = _resolve_project_name(project)
with force_routing(local=local, cloud=cloud):
run_with_cleanup(_run_validate(target, project_name, strict))
except ValueError as e:
console.print(f"[red]Error: {e}[/red]")
raise typer.Exit(1)
except Exception as e:
if not isinstance(e, typer.Exit):
logger.error(f"Error during schema validate: {e}")
typer.echo(f"Error during schema validate: {e}", err=True)
raise typer.Exit(1)
raise
# --- Infer ---
async def _run_infer(
entity_type: str,
project: Optional[str] = None,
threshold: float = 0.25,
save: bool = False,
):
"""Run schema inference via the API."""
from basic_memory.mcp.clients.schema import SchemaClient
async with get_client() as client:
active_project = await get_active_project(client, project, None)
schema_client = SchemaClient(client, active_project.external_id)
report = await schema_client.infer(entity_type, threshold=threshold)
if report.notes_analyzed == 0:
console.print(f"[yellow]No notes found with type: {entity_type}[/yellow]")
return
# --- Display frequency analysis ---
console.print(
f"\n[bold]Analyzing {report.notes_analyzed} notes with type: {entity_type}...[/bold]\n"
)
table = Table(title="Field Frequencies")
table.add_column("Field", style="cyan")
table.add_column("Source")
table.add_column("Count", justify="right")
table.add_column("Percentage", justify="right")
table.add_column("Suggested")
for freq in report.field_frequencies:
pct = f"{freq.percentage:.0%}"
if freq.name in report.suggested_required:
suggested = "[green]required[/green]"
elif freq.name in report.suggested_optional:
suggested = "[yellow]optional[/yellow]"
else:
suggested = "[dim]excluded[/dim]"
table.add_row(
freq.name,
freq.source,
str(freq.count),
pct,
suggested,
)
console.print(table)
# --- Display suggested schema ---
console.print("\n[bold]Suggested schema:[/bold]")
console.print(Panel(json.dumps(report.suggested_schema, indent=2), title="Picoschema"))
if save:
console.print(
f"\n[yellow]--save not yet implemented. "
f"Copy the schema above into schema/{entity_type}.md[/yellow]"
)
@schema_app.command()
def infer(
entity_type: Annotated[
str,
typer.Argument(help="Entity type to analyze (e.g., Person, meeting)"),
],
project: Annotated[
Optional[str],
typer.Option(help="The project name."),
] = None,
threshold: float = typer.Option(
0.25, "--threshold", help="Minimum frequency for optional fields (0-1)"
),
save: bool = typer.Option(False, "--save", help="Save inferred schema to schema/ directory"),
local: bool = typer.Option(
False, "--local", help="Force local API routing (ignore cloud mode)"
),
cloud: bool = typer.Option(False, "--cloud", help="Force cloud API routing"),
):
"""Infer schema from existing notes of a type.
Analyzes all notes with the given entity type and suggests a Picoschema
definition based on observation and relation frequency.
Fields present in 95%+ of notes become required. Fields above the
threshold (default 25%) become optional. Fields below threshold are excluded.
Use --local to force local routing when cloud mode is enabled.
Use --cloud to force cloud routing when cloud mode is disabled.
"""
try:
validate_routing_flags(local, cloud)
project_name = _resolve_project_name(project)
with force_routing(local=local, cloud=cloud):
run_with_cleanup(_run_infer(entity_type, project_name, threshold, save))
except ValueError as e:
console.print(f"[red]Error: {e}[/red]")
raise typer.Exit(1)
except Exception as e:
if not isinstance(e, typer.Exit):
logger.error(f"Error during schema infer: {e}")
typer.echo(f"Error during schema infer: {e}", err=True)
raise typer.Exit(1)
raise
# --- Diff ---
async def _run_diff(
entity_type: str,
project: Optional[str] = None,
):
"""Run schema drift detection via the API."""
from basic_memory.mcp.clients.schema import SchemaClient
async with get_client() as client:
active_project = await get_active_project(client, project, None)
schema_client = SchemaClient(client, active_project.external_id)
report = await schema_client.diff(entity_type)
has_drift = report.new_fields or report.dropped_fields or report.cardinality_changes
if not has_drift:
console.print(f"[green]No drift detected for {entity_type} schema.[/green]")
return
console.print(f"\n[bold]Schema drift detected for {entity_type}:[/bold]\n")
if report.new_fields:
console.print("[green]+ New fields (common in notes, not in schema):[/green]")
for f in report.new_fields:
console.print(f" + {f.name}: {f.percentage:.0%} of notes ({f.source})")
if report.dropped_fields:
console.print("[red]- Dropped fields (in schema, rare in notes):[/red]")
for f in report.dropped_fields:
console.print(f" - {f.name}: {f.percentage:.0%} of notes ({f.source})")
if report.cardinality_changes:
console.print("[yellow]~ Cardinality changes:[/yellow]")
for change in report.cardinality_changes:
console.print(f" ~ {change}")
@schema_app.command()
def diff(
entity_type: Annotated[
str,
typer.Argument(help="Entity type to check for drift"),
],
project: Annotated[
Optional[str],
typer.Option(help="The project name."),
] = None,
local: bool = typer.Option(
False, "--local", help="Force local API routing (ignore cloud mode)"
),
cloud: bool = typer.Option(False, "--cloud", help="Force cloud API routing"),
):
"""Show drift between schema and actual usage.
Compares the existing schema definition for an entity type against
how notes of that type are actually structured. Identifies new fields,
dropped fields, and cardinality changes.
Use --local to force local routing when cloud mode is enabled.
Use --cloud to force cloud routing when cloud mode is disabled.
"""
try:
validate_routing_flags(local, cloud)
project_name = _resolve_project_name(project)
with force_routing(local=local, cloud=cloud):
run_with_cleanup(_run_diff(entity_type, project_name))
except ValueError as e:
console.print(f"[red]Error: {e}[/red]")
raise typer.Exit(1)
except Exception as e:
if not isinstance(e, typer.Exit):
logger.error(f"Error during schema diff: {e}")
typer.echo(f"Error during schema diff: {e}", err=True)
raise typer.Exit(1)
raise
src/basic_memory/cli/main.py
+1
View File
@@ -13,6 +13,7 @@ from basic_memory.cli.commands import ( # noqa: F401 # pragma: no cover
import_memory_json,
mcp,
project,
schema,
status,
tool,
)
src/basic_memory/mcp/clients/__init__.py
+2
View File
@@ -17,6 +17,7 @@ from basic_memory.mcp.clients.memory import MemoryClient
from basic_memory.mcp.clients.directory import DirectoryClient
from basic_memory.mcp.clients.resource import ResourceClient
from basic_memory.mcp.clients.project import ProjectClient
from basic_memory.mcp.clients.schema import SchemaClient
__all__ = [
"KnowledgeClient",
@@ -25,4 +26,5 @@ __all__ = [
"DirectoryClient",
"ResourceClient",
"ProjectClient",
"SchemaClient",
]
src/basic_memory/mcp/clients/schema.py
+113
View File
@@ -0,0 +1,113 @@
"""Typed client for schema API operations.
Encapsulates all /v2/projects/{project_id}/schema/* endpoints.
"""
from httpx import AsyncClient
from basic_memory.mcp.tools.utils import call_post, call_get
from basic_memory.schemas.schema import (
ValidationReport,
InferenceReport,
DriftReport,
)
class SchemaClient:
"""Typed client for schema operations.
Centralizes:
- API path construction for /v2/projects/{project_id}/schema/*
- Response validation via Pydantic models
- Consistent error handling through call_* utilities
Usage:
async with get_client() as http_client:
client = SchemaClient(http_client, project_id)
report = await client.validate(entity_type="Person")
"""
def __init__(self, http_client: AsyncClient, project_id: str):
"""Initialize the schema client.
Args:
http_client: HTTPX AsyncClient for making requests
project_id: Project external_id (UUID) for API calls
"""
self.http_client = http_client
self.project_id = project_id
self._base_path = f"/v2/projects/{project_id}/schema"
async def validate(
self,
*,
entity_type: str | None = None,
identifier: str | None = None,
) -> ValidationReport:
"""Validate notes against their resolved schemas.
Args:
entity_type: Optional entity type to batch-validate
identifier: Optional specific note to validate
Returns:
ValidationReport with per-note results
Raises:
ToolError: If the request fails
"""
params: dict[str, str] = {}
if entity_type:
params["entity_type"] = entity_type
if identifier:
params["identifier"] = identifier
response = await call_post(
self.http_client,
f"{self._base_path}/validate",
params=params,
)
return ValidationReport.model_validate(response.json())
async def infer(
self,
entity_type: str,
*,
threshold: float = 0.25,
) -> InferenceReport:
"""Infer a schema from existing notes of a given type.
Args:
entity_type: The entity type to analyze
threshold: Minimum frequency for optional fields (0-1)
Returns:
InferenceReport with frequency data and suggested schema
Raises:
ToolError: If the request fails
"""
response = await call_post(
self.http_client,
f"{self._base_path}/infer",
params={"entity_type": entity_type, "threshold": threshold},
)
return InferenceReport.model_validate(response.json())
async def diff(self, entity_type: str) -> DriftReport:
"""Show drift between schema definition and actual usage.
Args:
entity_type: The entity type to check for drift
Returns:
DriftReport with detected differences
Raises:
ToolError: If the request fails
"""
response = await call_get(
self.http_client,
f"{self._base_path}/diff/{entity_type}",
)
return DriftReport.model_validate(response.json())
src/basic_memory/mcp/tools/__init__.py
+6
View File
@@ -27,6 +27,9 @@ from basic_memory.mcp.tools.project_management import (
# ChatGPT-compatible tools
from basic_memory.mcp.tools.chatgpt_tools import search, fetch
# Schema tools
from basic_memory.mcp.tools.schema import schema_validate, schema_infer, schema_diff
__all__ = [
"build_context",
"canvas",
@@ -41,6 +44,9 @@ __all__ = [
"read_content",
"read_note",
"recent_activity",
"schema_diff",
"schema_infer",
"schema_validate",
"search",
"search_by_metadata",
"search_notes",
src/basic_memory/mcp/tools/schema.py
+244
View File
@@ -0,0 +1,244 @@
"""Schema tools for Basic Memory MCP server.
Provides tools for schema validation, inference, and drift detection through the MCP protocol.
These tools call the schema API endpoints via the typed SchemaClient.
"""
from typing import Optional
from loguru import logger
from fastmcp import Context
from basic_memory.mcp.async_client import get_client
from basic_memory.mcp.project_context import get_active_project
from basic_memory.mcp.server import mcp
from basic_memory.schemas.schema import ValidationReport, InferenceReport, DriftReport
@mcp.tool(
description="Validate notes against their Picoschema definitions.",
)
async def schema_validate(
entity_type: Optional[str] = None,
identifier: Optional[str] = None,
project: Optional[str] = None,
context: Context | None = None,
) -> ValidationReport | str:
"""Validate notes against their resolved schema.
Validates a specific note (by identifier) or all notes of a given type.
Returns warnings/errors based on the schema's validation mode.
Schemas are resolved in priority order:
1. Inline schema (dict in frontmatter)
2. Explicit reference (string in frontmatter)
3. Implicit by type (type field matches schema note entity field)
4. No schema (no validation)
Project Resolution:
Server resolves projects in this order: Single Project Mode -> project parameter -> default.
If project unknown, use list_memory_projects() first.
Args:
entity_type: Entity type to batch-validate (e.g., "Person").
If provided, validates all notes of this type.
identifier: Specific note to validate (permalink, title, or path).
If provided, validates only this note.
project: Project name. Optional -- server will resolve.
context: Optional FastMCP context for performance caching.
Returns:
ValidationReport with per-note results, or error guidance string
Examples:
# Validate all Person notes
schema_validate(entity_type="Person")
# Validate a specific note
schema_validate(identifier="people/paul-graham")
# Validate in a specific project
schema_validate(entity_type="Person", project="my-research")
"""
async with get_client() as client:
active_project = await get_active_project(client, project, context)
logger.info(
f"MCP tool call tool=schema_validate project={active_project.name} "
f"entity_type={entity_type} identifier={identifier}"
)
try:
from basic_memory.mcp.clients.schema import SchemaClient
schema_client = SchemaClient(client, active_project.external_id)
result = await schema_client.validate(
entity_type=entity_type,
identifier=identifier,
)
logger.info(
f"MCP tool response: tool=schema_validate project={active_project.name} "
f"total={result.total_notes} valid={result.valid_count} "
f"warnings={result.warning_count} errors={result.error_count}"
)
return result
except Exception as e:
logger.error(f"Schema validation failed: {e}, project: {active_project.name}")
return (
f"# Schema Validation Failed\n\n"
f"Error validating schemas: {e}\n\n"
f"## Troubleshooting\n"
f"1. Ensure schema notes exist (type: schema) for the target entity type\n"
f"2. Check that notes have the correct type in frontmatter\n"
f"3. Verify the project has been synced: `basic-memory status`\n"
)
@mcp.tool(
description="Analyze existing notes and suggest a Picoschema definition.",
)
async def schema_infer(
entity_type: str,
threshold: float = 0.25,
project: Optional[str] = None,
context: Context | None = None,
) -> InferenceReport | str:
"""Analyze existing notes and suggest a schema definition.
Examines observation categories and relation types across all notes
of the given type. Returns frequency analysis and suggested Picoschema
YAML that can be saved as a schema note.
Frequency thresholds:
- 95%+ present -> required field
- threshold+ present -> optional field
- Below threshold -> excluded (but noted)
Project Resolution:
Server resolves projects in this order: Single Project Mode -> project parameter -> default.
If project unknown, use list_memory_projects() first.
Args:
entity_type: The entity type to analyze (e.g., "Person", "meeting").
threshold: Minimum frequency (0-1) for a field to be suggested as optional.
Default 0.25 (25%). Fields above 95% become required.
project: Project name. Optional -- server will resolve.
context: Optional FastMCP context for performance caching.
Returns:
InferenceReport with frequency data and suggested schema, or error string
Examples:
# Infer schema for Person notes
schema_infer("Person")
# Use a higher threshold (50% minimum)
schema_infer("meeting", threshold=0.5)
# Infer in a specific project
schema_infer("Person", project="my-research")
"""
async with get_client() as client:
active_project = await get_active_project(client, project, context)
logger.info(
f"MCP tool call tool=schema_infer project={active_project.name} "
f"entity_type={entity_type} threshold={threshold}"
)
try:
from basic_memory.mcp.clients.schema import SchemaClient
schema_client = SchemaClient(client, active_project.external_id)
result = await schema_client.infer(entity_type, threshold=threshold)
logger.info(
f"MCP tool response: tool=schema_infer project={active_project.name} "
f"entity_type={entity_type} notes_analyzed={result.notes_analyzed} "
f"required={len(result.suggested_required)} "
f"optional={len(result.suggested_optional)}"
)
return result
except Exception as e:
logger.error(f"Schema inference failed: {e}, project: {active_project.name}")
return (
f"# Schema Inference Failed\n\n"
f"Error inferring schema for '{entity_type}': {e}\n\n"
f"## Troubleshooting\n"
f"1. Ensure notes of type '{entity_type}' exist in the project\n"
f'2. Try searching: `search_notes("{entity_type}", types=["{entity_type}"])`\n'
f"3. Verify the project has been synced: `basic-memory status`\n"
)
@mcp.tool(
description="Detect drift between a schema definition and actual note usage.",
)
async def schema_diff(
entity_type: str,
project: Optional[str] = None,
context: Context | None = None,
) -> DriftReport | str:
"""Detect drift between a schema definition and actual note usage.
Compares the existing schema for an entity type against how notes of
that type are actually structured. Identifies new fields that have
appeared, declared fields that are rarely used, and cardinality changes
(single-value vs array).
Useful for evolving schemas as your knowledge base grows -- run
periodically to see if your schema still matches reality.
Project Resolution:
Server resolves projects in this order: Single Project Mode -> project parameter -> default.
If project unknown, use list_memory_projects() first.
Args:
entity_type: The entity type to check for drift (e.g., "Person").
project: Project name. Optional -- server will resolve.
context: Optional FastMCP context for performance caching.
Returns:
DriftReport with new fields, dropped fields, and cardinality changes,
or error guidance string
Examples:
# Check drift for Person schema
schema_diff("Person")
# Check drift in a specific project
schema_diff("Person", project="my-research")
"""
async with get_client() as client:
active_project = await get_active_project(client, project, context)
logger.info(
f"MCP tool call tool=schema_diff project={active_project.name} "
f"entity_type={entity_type}"
)
try:
from basic_memory.mcp.clients.schema import SchemaClient
schema_client = SchemaClient(client, active_project.external_id)
result = await schema_client.diff(entity_type)
logger.info(
f"MCP tool response: tool=schema_diff project={active_project.name} "
f"entity_type={entity_type} "
f"new_fields={len(result.new_fields)} "
f"dropped_fields={len(result.dropped_fields)} "
f"cardinality_changes={len(result.cardinality_changes)}"
)
return result
except Exception as e:
logger.error(f"Schema diff failed: {e}, project: {active_project.name}")
return (
f"# Schema Diff Failed\n\n"
f"Error detecting drift for '{entity_type}': {e}\n\n"
f"## Troubleshooting\n"
f"1. Ensure a schema note exists for entity type '{entity_type}'\n"
f"2. Ensure notes of type '{entity_type}' exist in the project\n"
f"3. Verify the project has been synced: `basic-memory status`\n"
)
src/basic_memory/schema/__init__.py
+58
View File
@@ -0,0 +1,58 @@
"""Schema system for Basic Memory.
Provides Picoschema-based validation for notes using observation/relation mapping.
Schemas are just notes with type: schema — no new data model, no migration.
"""
from basic_memory.schema.parser import (
SchemaField,
SchemaDefinition,
parse_picoschema,
parse_schema_note,
)
from basic_memory.schema.resolver import resolve_schema
from basic_memory.schema.validator import (
FieldResult,
ValidationResult,
validate_note,
)
from basic_memory.schema.inference import (
FieldFrequency,
InferenceResult,
ObservationData,
RelationData,
NoteData,
infer_schema,
analyze_observations,
analyze_relations,
)
from basic_memory.schema.diff import (
SchemaDrift,
diff_schema,
)
__all__ = [
# Parser
"SchemaField",
"SchemaDefinition",
"parse_picoschema",
"parse_schema_note",
# Resolver
"resolve_schema",
# Validator
"FieldResult",
"ValidationResult",
"validate_note",
# Inference
"FieldFrequency",
"InferenceResult",
"ObservationData",
"RelationData",
"NoteData",
"infer_schema",
"analyze_observations",
"analyze_relations",
# Diff
"SchemaDrift",
"diff_schema",
]
src/basic_memory/schema/diff.py
+111
View File
@@ -0,0 +1,111 @@
"""Schema diff for Basic Memory.
Compares a schema definition against actual note usage to detect drift.
Drift happens naturally as notes evolve -- new observation categories appear,
old ones fall out of use, single-value fields become multi-value.
The diff engine reuses inference analysis internally, comparing inferred
frequencies against the declared schema fields to surface:
- New fields: common in notes but not declared in schema
- Dropped fields: declared in schema but rare in actual notes
- Cardinality changes: field changed from single to array or vice versa
"""
from dataclasses import dataclass, field
from basic_memory.schema.inference import (
FieldFrequency,
NoteData,
analyze_observations,
analyze_relations,
)
from basic_memory.schema.parser import SchemaDefinition
@dataclass
class SchemaDrift:
"""Result of comparing a schema against actual note usage."""
new_fields: list[FieldFrequency] = field(default_factory=list)
dropped_fields: list[FieldFrequency] = field(default_factory=list)
cardinality_changes: list[str] = field(default_factory=list)
def diff_schema(
schema: SchemaDefinition,
notes: list[NoteData],
new_field_threshold: float = 0.25,
dropped_field_threshold: float = 0.10,
) -> SchemaDrift:
"""Compare a schema against actual note usage to detect drift.
Args:
schema: The current schema definition.
notes: List of NoteData objects representing actual notes.
new_field_threshold: Frequency above which an undeclared field is considered
"new" and worth adding to the schema.
dropped_field_threshold: Frequency below which a declared field is considered
"dropped" and worth removing from the schema.
Returns:
A SchemaDrift describing the differences between schema and reality.
"""
total = len(notes)
if total == 0:
return SchemaDrift()
# --- Analyze actual usage ---
obs_frequencies = analyze_observations(notes, total, max_sample_values=3)
rel_frequencies = analyze_relations(notes, total, max_sample_values=3)
# Build lookup from schema fields
schema_field_names = {f.name for f in schema.fields}
# Build lookup from actual frequencies
obs_freq_by_name = {f.name: f for f in obs_frequencies}
rel_freq_by_name = {f.name: f for f in rel_frequencies}
all_freq_by_name = {**obs_freq_by_name, **rel_freq_by_name}
result = SchemaDrift()
# --- Detect new fields ---
# Fields that appear frequently in notes but aren't declared in the schema
for freq in obs_frequencies + rel_frequencies:
if freq.name not in schema_field_names and freq.percentage >= new_field_threshold:
result.new_fields.append(freq)
# --- Detect dropped fields ---
# Fields declared in the schema but rarely appearing in actual notes
for schema_field in schema.fields:
freq = all_freq_by_name.get(schema_field.name)
if freq is None:
# Field doesn't appear at all in any note
result.dropped_fields.append(
FieldFrequency(
name=schema_field.name,
source="relation" if schema_field.is_entity_ref else "observation",
count=0,
total=total,
percentage=0.0,
)
)
elif freq.percentage < dropped_field_threshold:
result.dropped_fields.append(freq)
# --- Detect cardinality changes ---
# Fields where the schema says single but usage shows array, or vice versa
for schema_field in schema.fields:
freq = all_freq_by_name.get(schema_field.name)
if freq is None:
continue
if schema_field.is_array and not freq.is_array:
result.cardinality_changes.append(
f"{schema_field.name}: schema declares array but usage is typically single-value"
)
elif not schema_field.is_array and freq.is_array:
result.cardinality_changes.append(
f"{schema_field.name}: schema declares single-value but usage is typically array"
)
return result
src/basic_memory/schema/inference.py
+323
View File
@@ -0,0 +1,323 @@
"""Schema inference engine for Basic Memory.
Analyzes notes of a given type and suggests a schema based on observation
and relation frequency. Instead of requiring users to define schemas upfront,
schemas emerge from actual usage patterns:
Write notes freely -> Patterns emerge -> Crystallize into schema
Frequency thresholds:
- 95%+ present -> required field
- 25%+ present -> optional field
- Below 25% -> excluded from suggestion (but noted)
"""
from collections import Counter
from dataclasses import dataclass, field
# --- Result Data Model ---
@dataclass
class FieldFrequency:
"""Frequency analysis for a single field across notes of a type."""
name: str
source: str # "observation" | "relation"
count: int # notes containing this field
total: int # total notes analyzed
percentage: float
sample_values: list[str] = field(default_factory=list)
is_array: bool = False # True if typically appears multiple times per note
target_type: str | None = None # For relations, the most common target entity type
@dataclass
class InferenceResult:
"""Complete inference result with frequency analysis and suggested schema."""
entity_type: str
notes_analyzed: int
field_frequencies: list[FieldFrequency]
suggested_schema: dict # Ready-to-use Picoschema YAML dict
suggested_required: list[str]
suggested_optional: list[str]
excluded: list[str] # Below threshold
# --- Note Data Abstraction ---
# Instead of depending on the ORM Entity model, we accept simple data structures.
# This keeps the inference engine decoupled from the data access layer.
@dataclass
class ObservationData:
"""Lightweight observation for schema analysis. Decoupled from ORM."""
category: str
content: str
@dataclass
class RelationData:
"""Lightweight relation for schema analysis. Decoupled from ORM."""
relation_type: str
target_name: str
target_entity_type: str | None = None
@dataclass
class NoteData:
"""Minimal note representation for inference analysis.
Decoupled from ORM models so the inference engine can work with
any data source (database, files, API responses).
"""
identifier: str
observations: list[ObservationData]
relations: list[RelationData]
# --- Inference Logic ---
def infer_schema(
entity_type: str,
notes: list[NoteData],
required_threshold: float = 0.95,
optional_threshold: float = 0.25,
max_sample_values: int = 5,
) -> InferenceResult:
"""Analyze notes and suggest a Picoschema definition.
Examines observation categories and relation types across all provided notes.
Fields that appear in a high percentage of notes become required; those that
appear less frequently become optional.
Args:
entity_type: The entity type being analyzed (e.g., "Person").
notes: List of NoteData objects to analyze.
required_threshold: Frequency at or above which a field is required (default 0.95).
optional_threshold: Frequency at or above which a field is optional (default 0.25).
max_sample_values: Maximum number of sample values to include per field.
Returns:
An InferenceResult with frequency analysis and suggested Picoschema dict.
"""
total = len(notes)
if total == 0:
return InferenceResult(
entity_type=entity_type,
notes_analyzed=0,
field_frequencies=[],
suggested_schema={},
suggested_required=[],
suggested_optional=[],
excluded=[],
)
# --- Analyze observation frequencies ---
obs_frequencies = analyze_observations(notes, total, max_sample_values)
# --- Analyze relation frequencies ---
rel_frequencies = analyze_relations(notes, total, max_sample_values)
# --- Classify fields by threshold ---
all_frequencies = obs_frequencies + rel_frequencies
suggested_required: list[str] = []
suggested_optional: list[str] = []
excluded: list[str] = []
for freq in all_frequencies:
if freq.percentage >= required_threshold:
suggested_required.append(freq.name)
elif freq.percentage >= optional_threshold:
suggested_optional.append(freq.name)
else:
excluded.append(freq.name)
# --- Build suggested Picoschema dict ---
suggested_schema = _build_picoschema_dict(
all_frequencies, required_threshold, optional_threshold
)
return InferenceResult(
entity_type=entity_type,
notes_analyzed=total,
field_frequencies=all_frequencies,
suggested_schema=suggested_schema,
suggested_required=suggested_required,
suggested_optional=suggested_optional,
excluded=excluded,
)
# --- Observation Analysis ---
def analyze_observations(
notes: list[NoteData],
total: int,
max_sample_values: int,
) -> list[FieldFrequency]:
"""Count observation category frequencies across notes.
A category is counted once per note (presence), not per occurrence.
Array detection: if a category appears multiple times in a single note
in more than half the notes where it appears, it's flagged as an array.
"""
# Count how many notes contain each category (presence per note)
category_note_count: Counter[str] = Counter()
# Count how many notes have multiple occurrences (for array detection)
category_multi_count: Counter[str] = Counter()
# Collect sample values
category_samples: dict[str, list[str]] = {}
for note in notes:
# Group observations by category within this note
note_categories: dict[str, list[str]] = {}
for obs in note.observations:
note_categories.setdefault(obs.category, []).append(obs.content)
for category, values in note_categories.items():
category_note_count[category] += 1
if len(values) > 1:
category_multi_count[category] += 1
# Collect sample values (deduplicated)
samples = category_samples.setdefault(category, [])
for v in values:
if v not in samples and len(samples) < max_sample_values:
samples.append(v)
# Build FieldFrequency objects
frequencies: list[FieldFrequency] = []
for category, count in category_note_count.most_common():
# Array detection: if more than half of notes with this category have
# multiple occurrences, treat it as an array field
multi_count = category_multi_count.get(category, 0)
is_array = multi_count > (count / 2)
frequencies.append(
FieldFrequency(
name=category,
source="observation",
count=count,
total=total,
percentage=count / total,
sample_values=category_samples.get(category, []),
is_array=is_array,
)
)
return frequencies
# --- Relation Analysis ---
def analyze_relations(
notes: list[NoteData],
total: int,
max_sample_values: int,
) -> list[FieldFrequency]:
"""Count relation type frequencies across notes.
Similar to observations, a relation type is counted once per note.
Array detection follows the same logic.
"""
rel_note_count: Counter[str] = Counter()
rel_multi_count: Counter[str] = Counter()
rel_samples: dict[str, list[str]] = {}
# Track target entity types to suggest the type in the schema
rel_target_types: dict[str, Counter[str]] = {}
for note in notes:
note_rels: dict[str, list[str]] = {}
note_rel_objects: dict[str, list[RelationData]] = {}
for rel in note.relations:
note_rels.setdefault(rel.relation_type, []).append(rel.target_name)
note_rel_objects.setdefault(rel.relation_type, []).append(rel)
for rel_type, targets in note_rels.items():
rel_note_count[rel_type] += 1
if len(targets) > 1:
rel_multi_count[rel_type] += 1
samples = rel_samples.setdefault(rel_type, [])
for t in targets:
if t not in samples and len(samples) < max_sample_values:
samples.append(t)
# Track target entity types from individual relations (not the source note)
target_counter = rel_target_types.setdefault(rel_type, Counter())
for rel in note_rel_objects[rel_type]:
if rel.target_entity_type:
target_counter[rel.target_entity_type] += 1
frequencies: list[FieldFrequency] = []
for rel_type, count in rel_note_count.most_common():
multi_count = rel_multi_count.get(rel_type, 0)
is_array = multi_count > (count / 2)
# Determine most common target type
target_counter = rel_target_types.get(rel_type, Counter())
most_common_target = target_counter.most_common(1)[0][0] if target_counter else None
frequencies.append(
FieldFrequency(
name=rel_type,
source="relation",
count=count,
total=total,
percentage=count / total,
sample_values=rel_samples.get(rel_type, []),
is_array=is_array,
target_type=most_common_target,
)
)
return frequencies
# --- Schema Generation ---
def _build_picoschema_dict(
frequencies: list[FieldFrequency],
required_threshold: float,
optional_threshold: float,
) -> dict:
"""Build a Picoschema YAML dict from field frequencies.
Only includes fields at or above the optional threshold.
"""
schema: dict = {}
for freq in frequencies:
if freq.percentage < optional_threshold:
continue
is_required = freq.percentage >= required_threshold
# --- Build the field key ---
key = freq.name
if not is_required:
key += "?"
if freq.is_array:
key += "(array)"
# --- Build the field value ---
if freq.source == "relation":
# Relations become entity reference fields
target = freq.target_type or "string"
# Capitalize first letter for entity ref convention
target = target[0].upper() + target[1:] if target != "string" else "string"
schema[key] = target
else:
schema[key] = "string"
return schema
src/basic_memory/schema/parser.py
+236
View File
@@ -0,0 +1,236 @@
"""Picoschema parser for Basic Memory.
Parses Picoschema YAML dicts (from note frontmatter) into typed dataclass
representations. Picoschema is a compact schema notation from Google's Dotprompt
that fits naturally in YAML frontmatter.
Syntax reference:
field: type, description # required field
field?: type, description # optional field
field(array): type # array of values
field?(enum): [val1, val2] # enumeration
field?(object): # nested object
sub_field: type
EntityName as type (capitalized) # entity reference
"""
from dataclasses import dataclass, field
# --- Data Model ---
@dataclass
class SchemaField:
"""A single field in a Picoschema definition.
Maps to either an observation category or a relation type in Basic Memory notes.
"""
name: str
type: str # string, integer, number, boolean, any, or EntityName
required: bool # True unless field name ends with ?
is_array: bool = False # True if (array) notation
is_enum: bool = False # True if (enum) notation
enum_values: list[str] = field(default_factory=list)
description: str | None = None # Text after comma
is_entity_ref: bool = False # True if type is capitalized (entity reference)
children: list["SchemaField"] = field(default_factory=list) # For (object) types
@dataclass
class SchemaDefinition:
"""A complete schema definition parsed from a schema note's frontmatter.
Combines the parsed fields with metadata about the schema itself.
"""
entity: str # The entity type this schema describes
version: int # Schema version
fields: list[SchemaField] # Parsed fields
validation_mode: str # "warn" | "strict" | "off"
# --- Built-in scalar types ---
# Types that are NOT entity references. Anything not in this set and starting
# with an uppercase letter is treated as an entity reference.
SCALAR_TYPES = frozenset({"string", "integer", "number", "boolean", "any"})
# --- Field Name Parsing ---
def _parse_field_key(key: str) -> tuple[str, bool, bool, bool, bool]:
"""Parse a Picoschema field key into its components.
Returns (name, required, is_array, is_enum, is_object).
The key format is: name[?][(array|enum|object)]
Examples:
"name" -> ("name", True, False, False)
"role?" -> ("role", False, False, False)
"tags?(array)" -> ("tags", False, True, False)
"status?(enum)" -> ("status", False, False, True)
"metadata?(object)" -> ("metadata", False, False, False) + children
"""
required = True
is_array = False
is_enum = False
is_object = False
# Check for modifier suffix: (array), (enum), (object)
if key.endswith("(array)"):
is_array = True
key = key[: -len("(array)")]
elif key.endswith("(enum)"):
is_enum = True
key = key[: -len("(enum)")]
elif key.endswith("(object)"):
is_object = True
key = key[: -len("(object)")]
# Check for optional marker
if key.endswith("?"):
required = False
key = key[:-1]
return key, required, is_array, is_enum, is_object
def _parse_type_and_description(value: str) -> tuple[str, str | None]:
"""Parse a type string that may include a comma-separated description.
Examples:
"string" -> ("string", None)
"string, full name" -> ("string", "full name")
"Organization, employer" -> ("Organization", "employer")
"""
if "," in value:
type_str, desc = value.split(",", 1)
return type_str.strip(), desc.strip()
return value.strip(), None
def _is_entity_ref_type(type_str: str) -> bool:
"""Determine if a type string represents an entity reference.
Entity references are capitalized type names that are not built-in scalar types.
"""
if type_str in SCALAR_TYPES:
return False
# Capitalized first letter = entity reference
return len(type_str) > 0 and type_str[0].isupper()
# --- Main Parser ---
def parse_picoschema(yaml_dict: dict) -> list[SchemaField]:
"""Parse a Picoschema YAML dict into a list of SchemaField objects.
This is the core parser that converts YAML frontmatter schema definitions
into structured SchemaField dataclasses.
Args:
yaml_dict: The schema dict from YAML frontmatter. Keys are field
declarations (e.g., "name", "role?", "tags?(array)"), values are
type declarations (e.g., "string", "string, description").
Returns:
List of SchemaField objects representing the schema.
"""
fields: list[SchemaField] = []
for key, value in yaml_dict.items():
name, required, is_array, is_enum, is_object = _parse_field_key(key)
# --- Enum fields ---
# Trigger: value is a list (e.g., [active, inactive])
# Why: enums declare allowed values directly as a YAML list
# Outcome: SchemaField with is_enum=True and enum_values populated
if is_enum:
enum_values = value if isinstance(value, list) else [str(value)]
fields.append(
SchemaField(
name=name,
type="enum",
required=required,
is_enum=True,
enum_values=[str(v) for v in enum_values],
)
)
continue
# --- Object fields ---
# Trigger: value is a dict (nested sub-fields)
# Why: objects contain child fields parsed recursively
# Outcome: SchemaField with children populated via recursive parse
if is_object or (isinstance(value, dict) and not is_enum):
children = parse_picoschema(value) if isinstance(value, dict) else []
fields.append(
SchemaField(
name=name,
type="object",
required=required,
children=children,
)
)
continue
# --- Scalar and entity ref fields ---
type_str, description = _parse_type_and_description(str(value))
is_entity_ref = _is_entity_ref_type(type_str)
fields.append(
SchemaField(
name=name,
type=type_str,
required=required,
is_array=is_array,
description=description,
is_entity_ref=is_entity_ref,
)
)
return fields
def parse_schema_note(frontmatter: dict) -> SchemaDefinition:
"""Parse a full schema note's frontmatter into a SchemaDefinition.
A schema note has type: schema and contains:
- entity: the entity type this schema describes
- version: schema version number
- schema: the Picoschema dict
- settings.validation: validation mode (warn/strict/off)
Args:
frontmatter: The complete YAML frontmatter dict from a schema note.
Returns:
A SchemaDefinition with parsed fields and metadata.
Raises:
ValueError: If required fields (entity, schema) are missing.
"""
entity = frontmatter.get("entity")
if not entity:
raise ValueError("Schema note missing required 'entity' field in frontmatter")
schema_dict = frontmatter.get("schema")
if not schema_dict or not isinstance(schema_dict, dict):
raise ValueError("Schema note missing required 'schema' dict in frontmatter")
version = frontmatter.get("version", 1)
settings = frontmatter.get("settings", {})
validation_mode = settings.get("validation", "warn") if isinstance(settings, dict) else "warn"
fields = parse_picoschema(schema_dict)
return SchemaDefinition(
entity=entity,
version=version,
fields=fields,
validation_mode=validation_mode,
)
src/basic_memory/schema/resolver.py
+118
View File
@@ -0,0 +1,118 @@
"""Schema resolver for Basic Memory.
Finds the applicable schema for a note using a priority-based resolution order:
1. Inline schema -> frontmatter['schema'] is a dict
2. Explicit ref -> frontmatter['schema'] is a string (entity name or permalink)
3. Implicit by type -> frontmatter['type'] matches a schema note's entity field
4. No schema -> returns None (perfectly fine)
The resolver takes a search function as a dependency instead of importing
repository code directly, keeping the schema package decoupled from the
data access layer.
"""
from collections.abc import Callable, Awaitable
from basic_memory.schema.parser import SchemaDefinition, parse_picoschema, parse_schema_note
# Type alias for the search function dependency.
# Given a query string, returns a list of frontmatter dicts from matching schema notes.
type SchemaSearchFn = Callable[[str], Awaitable[list[dict]]]
async def resolve_schema(
note_frontmatter: dict,
search_fn: SchemaSearchFn,
) -> SchemaDefinition | None:
"""Resolve the schema for a note based on its frontmatter.
Resolution order:
1. Inline schema (frontmatter['schema'] is a dict) - parsed directly
2. Explicit reference (frontmatter['schema'] is a string) - looked up via search_fn
3. Implicit by type (frontmatter['type']) - searches for schema note with matching entity
4. No schema - returns None
Args:
note_frontmatter: The YAML frontmatter dict from the note being validated.
search_fn: An async callable that takes a query string and returns a list
of frontmatter dicts from matching schema notes. This keeps the resolver
decoupled from the repository/service layer.
Returns:
A SchemaDefinition if a schema is found, None otherwise.
"""
schema_value = note_frontmatter.get("schema")
# --- 1. Inline schema ---
# Trigger: schema field is a dict (the Picoschema definition lives in this note)
# Why: inline schemas are self-contained, no lookup needed
# Outcome: parse and return immediately
if isinstance(schema_value, dict):
return _schema_from_inline(schema_value, note_frontmatter)
# --- 2. Explicit reference ---
# Trigger: schema field is a string (entity name or permalink)
# Why: the note points to a specific schema note by name
# Outcome: search for the referenced schema note and parse it
if isinstance(schema_value, str):
result = await _schema_from_reference(schema_value, search_fn)
if result is not None:
return result
# --- 3. Implicit by type ---
# Trigger: no schema field, but the note has a type field
# Why: convention — a note with type: Person looks for a schema with entity: Person
# Outcome: search for a schema note whose entity matches the note's type
note_type = note_frontmatter.get("type")
if note_type:
result = await _schema_from_type(note_type, search_fn)
if result is not None:
return result
# --- 4. No schema ---
return None
def _schema_from_inline(schema_dict: dict, frontmatter: dict) -> SchemaDefinition:
"""Build a SchemaDefinition from an inline schema dict.
For inline schemas, we derive metadata from the note's own frontmatter
since there's no separate schema note.
"""
fields = parse_picoschema(schema_dict)
entity = frontmatter.get("type", "unknown")
settings = frontmatter.get("settings", {})
validation_mode = settings.get("validation", "warn") if isinstance(settings, dict) else "warn"
return SchemaDefinition(
entity=entity,
version=1,
fields=fields,
validation_mode=validation_mode,
)
async def _schema_from_reference(
ref: str,
search_fn: SchemaSearchFn,
) -> SchemaDefinition | None:
"""Look up a schema by entity name or permalink reference.
The search function is expected to find schema notes matching the reference.
"""
results = await search_fn(ref)
if not results:
return None
return parse_schema_note(results[0])
async def _schema_from_type(
note_type: str,
search_fn: SchemaSearchFn,
) -> SchemaDefinition | None:
"""Look up a schema implicitly by matching the note's type to a schema's entity field."""
results = await search_fn(note_type)
if not results:
return None
return parse_schema_note(results[0])
src/basic_memory/schema/validator.py
+259
View File
@@ -0,0 +1,259 @@
"""Schema validator for Basic Memory.
Validates a note's observations and relations against a resolved schema definition.
The mapping rules ground schema fields in the existing Basic Memory note format:
Schema Declaration -> Grounded In
-----------------------------------------------
field: string -> observation [field] value
field?(array): string -> multiple [field] observations
field?: EntityType -> relation 'field [[Target]]'
field?(array): EntityType -> multiple 'field' relations
field?(enum): [values] -> observation [field] value where value is in set
Validation is soft by default (warn mode). Unmatched observations and relations
are informational, not errors -- schemas are a subset, not a straitjacket.
"""
from dataclasses import dataclass, field as dataclass_field
from basic_memory.schema.inference import ObservationData, RelationData
from basic_memory.schema.parser import SchemaDefinition, SchemaField
# --- Result Data Model ---
@dataclass
class FieldResult:
"""Validation result for a single schema field."""
field: SchemaField
status: str # "present" | "missing" | "enum_mismatch"
values: list[str] = dataclass_field(default_factory=list) # Matched values
message: str | None = None
@dataclass
class ValidationResult:
"""Complete validation result for a note against a schema."""
note_identifier: str
schema_entity: str
passed: bool # True if no errors (warnings are OK)
field_results: list[FieldResult] = dataclass_field(default_factory=list)
unmatched_observations: dict[str, int] = dataclass_field(default_factory=dict) # cat -> count
unmatched_relations: list[str] = dataclass_field(default_factory=list) # types not in schema
warnings: list[str] = dataclass_field(default_factory=list)
errors: list[str] = dataclass_field(default_factory=list)
# --- Validation Logic ---
def validate_note(
note_identifier: str,
schema: SchemaDefinition,
observations: list[ObservationData],
relations: list[RelationData],
) -> ValidationResult:
"""Validate a note against a schema definition.
Args:
note_identifier: The note's title, permalink, or file path for reporting.
schema: The resolved SchemaDefinition to validate against.
observations: List of ObservationData from the note's observations.
relations: List of RelationData from the note's relations.
Returns:
A ValidationResult with per-field results, unmatched items, and warnings/errors.
"""
result = ValidationResult(
note_identifier=note_identifier,
schema_entity=schema.entity,
passed=True,
)
# Build lookup structures from the note's actual content
obs_by_category = _group_observations(observations)
rel_by_type = _group_relations(relations)
# Track which observation categories and relation types are matched by schema fields
matched_categories: set[str] = set()
matched_relation_types: set[str] = set()
# --- Validate each schema field ---
for schema_field in schema.fields:
field_result = _validate_field(schema_field, obs_by_category, rel_by_type)
result.field_results.append(field_result)
# Track which categories/relation types this field consumed
if schema_field.is_entity_ref:
matched_relation_types.add(schema_field.name)
else:
matched_categories.add(schema_field.name)
# --- Generate warnings or errors based on validation mode ---
# Trigger: field declared in schema but not found in note
# Why: required missing = warning (or error in strict); optional missing = silent
# Outcome: only required missing fields produce diagnostics
if field_result.status == "missing" and schema_field.required:
msg = _missing_field_message(schema_field)
if schema.validation_mode == "strict":
result.errors.append(msg)
result.passed = False
else:
result.warnings.append(msg)
elif field_result.status == "enum_mismatch":
msg = field_result.message or f"Field '{schema_field.name}' has invalid enum value"
if schema.validation_mode == "strict":
result.errors.append(msg)
result.passed = False
else:
result.warnings.append(msg)
# --- Collect unmatched observations ---
for category, values in obs_by_category.items():
if category not in matched_categories:
result.unmatched_observations[category] = len(values)
# --- Collect unmatched relations ---
for rel_type in rel_by_type:
if rel_type not in matched_relation_types:
result.unmatched_relations.append(rel_type)
return result
# --- Field Validation ---
def _validate_field(
schema_field: SchemaField,
obs_by_category: dict[str, list[str]],
rel_by_type: dict[str, list[str]],
) -> FieldResult:
"""Validate a single schema field against the note's data.
Entity ref fields map to relations; all other fields map to observations.
"""
# --- Entity reference fields map to relations ---
if schema_field.is_entity_ref:
return _validate_entity_ref_field(schema_field, rel_by_type)
# --- Enum fields require value membership check ---
if schema_field.is_enum:
return _validate_enum_field(schema_field, obs_by_category)
# --- Scalar and array fields map to observations ---
return _validate_observation_field(schema_field, obs_by_category)
def _validate_observation_field(
schema_field: SchemaField,
obs_by_category: dict[str, list[str]],
) -> FieldResult:
"""Validate a field that maps to observation categories."""
values = obs_by_category.get(schema_field.name, [])
if not values:
return FieldResult(
field=schema_field,
status="missing",
message=_missing_field_message(schema_field),
)
return FieldResult(
field=schema_field,
status="present",
values=values,
)
def _validate_entity_ref_field(
schema_field: SchemaField,
rel_by_type: dict[str, list[str]],
) -> FieldResult:
"""Validate a field that maps to relations (entity references)."""
targets = rel_by_type.get(schema_field.name, [])
if not targets:
return FieldResult(
field=schema_field,
status="missing",
message=f"Missing relation: {schema_field.name} (no '{schema_field.name} [[...]]' "
f"relation found)",
)
return FieldResult(
field=schema_field,
status="present",
values=targets,
)
def _validate_enum_field(
schema_field: SchemaField,
obs_by_category: dict[str, list[str]],
) -> FieldResult:
"""Validate an enum field -- value must be in the allowed set."""
values = obs_by_category.get(schema_field.name, [])
if not values:
return FieldResult(
field=schema_field,
status="missing",
message=_missing_field_message(schema_field),
)
# Check each value against the allowed enum values
invalid_values = [v for v in values if v not in schema_field.enum_values]
if invalid_values:
allowed = ", ".join(schema_field.enum_values)
invalid = ", ".join(invalid_values)
return FieldResult(
field=schema_field,
status="enum_mismatch",
values=values,
message=f"Field '{schema_field.name}' has invalid value(s): {invalid} "
f"(allowed: {allowed})",
)
return FieldResult(
field=schema_field,
status="present",
values=values,
)
# --- Helper Functions ---
def _group_observations(observations: list[ObservationData]) -> dict[str, list[str]]:
"""Group observations by category."""
result: dict[str, list[str]] = {}
for obs in observations:
result.setdefault(obs.category, []).append(obs.content)
return result
def _group_relations(relations: list[RelationData]) -> dict[str, list[str]]:
"""Group relations by relation type."""
result: dict[str, list[str]] = {}
for rel in relations:
result.setdefault(rel.relation_type, []).append(rel.target_name)
return result
def _missing_field_message(schema_field: SchemaField) -> str:
"""Generate a human-readable message for a missing field."""
kind = "required" if schema_field.required else "optional"
if schema_field.is_entity_ref:
return (
f"Missing {kind} field: {schema_field.name} "
f"(no '{schema_field.name} [[...]]' relation found)"
)
return f"Missing {kind} field: {schema_field.name} (expected [{schema_field.name}] observation)"
src/basic_memory/schemas/schema.py
+121
View File
@@ -0,0 +1,121 @@
"""Pydantic response models for the schema system.
These models define the API response format for schema validation,
inference, and drift detection operations. They mirror the dataclass
structures in basic_memory.schema but are Pydantic models suitable
for API serialization.
"""
from pydantic import BaseModel, Field
# --- Validation Response Models ---
class FieldResultResponse(BaseModel):
"""Result of validating a single schema field against a note."""
field_name: str
field_type: str
required: bool
status: str = Field(description="One of: present, missing, type_mismatch")
values: list[str] = Field(default_factory=list, description="Matched values from the note")
message: str | None = None
class NoteValidationResponse(BaseModel):
"""Validation result for a single note against a schema."""
note_identifier: str
schema_entity: str
passed: bool = Field(description="True if no errors (warnings are OK)")
field_results: list[FieldResultResponse] = Field(default_factory=list)
unmatched_observations: dict[str, int] = Field(
default_factory=dict,
description="Observation categories not covered by schema, with counts",
)
unmatched_relations: list[str] = Field(
default_factory=list,
description="Relation types not covered by schema",
)
warnings: list[str] = Field(default_factory=list)
errors: list[str] = Field(default_factory=list)
class ValidationReport(BaseModel):
"""Full validation report for one or more notes."""
entity_type: str | None = None
total_notes: int = 0
valid_count: int = 0
warning_count: int = 0
error_count: int = 0
results: list[NoteValidationResponse] = Field(default_factory=list)
# --- Inference Response Models ---
class FieldFrequencyResponse(BaseModel):
"""Frequency analysis for a single field across notes."""
name: str
source: str = Field(description="One of: observation, relation")
count: int = Field(description="Number of notes containing this field")
total: int = Field(description="Total notes analyzed")
percentage: float
sample_values: list[str] = Field(default_factory=list)
is_array: bool = Field(
default=False,
description="True if field typically appears multiple times per note",
)
target_type: str | None = Field(
default=None,
description="For relations, the most common target entity type",
)
class InferenceReport(BaseModel):
"""Inference result with suggested schema definition."""
entity_type: str
notes_analyzed: int
field_frequencies: list[FieldFrequencyResponse] = Field(default_factory=list)
suggested_schema: dict = Field(
default_factory=dict,
description="Ready-to-use Picoschema YAML dict",
)
suggested_required: list[str] = Field(default_factory=list)
suggested_optional: list[str] = Field(default_factory=list)
excluded: list[str] = Field(
default_factory=list,
description="Fields below the inclusion threshold",
)
# --- Drift Response Models ---
class DriftFieldResponse(BaseModel):
"""A field involved in schema drift."""
name: str
source: str
count: int
total: int
percentage: float
class DriftReport(BaseModel):
"""Schema drift analysis comparing schema definition to actual usage."""
entity_type: str
new_fields: list[DriftFieldResponse] = Field(
default_factory=list,
description="Fields common in notes but not in schema",
)
dropped_fields: list[DriftFieldResponse] = Field(
default_factory=list,
description="Fields in schema but rare in notes",
)
cardinality_changes: list[str] = Field(default_factory=list)