mirror of
https://github.com/basicmachines-co/basic-memory
synced 2026-06-21 13:47:35 +00:00
feat: Schema system for Basic Memory (#549)
Signed-off-by: phernandez <paul@basicmachines.co> Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Paul Hernandez
@@ -0,0 +1,58 @@
|
||||
"""Schema system for Basic Memory.
|
||||
|
||||
Provides Picoschema-based validation for notes using observation/relation mapping.
|
||||
Schemas are just notes with type: schema — no new data model, no migration.
|
||||
"""
|
||||
|
||||
from basic_memory.schema.parser import (
|
||||
SchemaField,
|
||||
SchemaDefinition,
|
||||
parse_picoschema,
|
||||
parse_schema_note,
|
||||
)
|
||||
from basic_memory.schema.resolver import resolve_schema
|
||||
from basic_memory.schema.validator import (
|
||||
FieldResult,
|
||||
ValidationResult,
|
||||
validate_note,
|
||||
)
|
||||
from basic_memory.schema.inference import (
|
||||
FieldFrequency,
|
||||
InferenceResult,
|
||||
ObservationData,
|
||||
RelationData,
|
||||
NoteData,
|
||||
infer_schema,
|
||||
analyze_observations,
|
||||
analyze_relations,
|
||||
)
|
||||
from basic_memory.schema.diff import (
|
||||
SchemaDrift,
|
||||
diff_schema,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
# Parser
|
||||
"SchemaField",
|
||||
"SchemaDefinition",
|
||||
"parse_picoschema",
|
||||
"parse_schema_note",
|
||||
# Resolver
|
||||
"resolve_schema",
|
||||
# Validator
|
||||
"FieldResult",
|
||||
"ValidationResult",
|
||||
"validate_note",
|
||||
# Inference
|
||||
"FieldFrequency",
|
||||
"InferenceResult",
|
||||
"ObservationData",
|
||||
"RelationData",
|
||||
"NoteData",
|
||||
"infer_schema",
|
||||
"analyze_observations",
|
||||
"analyze_relations",
|
||||
# Diff
|
||||
"SchemaDrift",
|
||||
"diff_schema",
|
||||
]
|
||||
@@ -0,0 +1,111 @@
|
||||
"""Schema diff for Basic Memory.
|
||||
|
||||
Compares a schema definition against actual note usage to detect drift.
|
||||
Drift happens naturally as notes evolve -- new observation categories appear,
|
||||
old ones fall out of use, single-value fields become multi-value.
|
||||
|
||||
The diff engine reuses inference analysis internally, comparing inferred
|
||||
frequencies against the declared schema fields to surface:
|
||||
- New fields: common in notes but not declared in schema
|
||||
- Dropped fields: declared in schema but rare in actual notes
|
||||
- Cardinality changes: field changed from single to array or vice versa
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from basic_memory.schema.inference import (
|
||||
FieldFrequency,
|
||||
NoteData,
|
||||
analyze_observations,
|
||||
analyze_relations,
|
||||
)
|
||||
from basic_memory.schema.parser import SchemaDefinition
|
||||
|
||||
|
||||
@dataclass
|
||||
class SchemaDrift:
|
||||
"""Result of comparing a schema against actual note usage."""
|
||||
|
||||
new_fields: list[FieldFrequency] = field(default_factory=list)
|
||||
dropped_fields: list[FieldFrequency] = field(default_factory=list)
|
||||
cardinality_changes: list[str] = field(default_factory=list)
|
||||
|
||||
|
||||
def diff_schema(
|
||||
schema: SchemaDefinition,
|
||||
notes: list[NoteData],
|
||||
new_field_threshold: float = 0.25,
|
||||
dropped_field_threshold: float = 0.10,
|
||||
) -> SchemaDrift:
|
||||
"""Compare a schema against actual note usage to detect drift.
|
||||
|
||||
Args:
|
||||
schema: The current schema definition.
|
||||
notes: List of NoteData objects representing actual notes.
|
||||
new_field_threshold: Frequency above which an undeclared field is considered
|
||||
"new" and worth adding to the schema.
|
||||
dropped_field_threshold: Frequency below which a declared field is considered
|
||||
"dropped" and worth removing from the schema.
|
||||
|
||||
Returns:
|
||||
A SchemaDrift describing the differences between schema and reality.
|
||||
"""
|
||||
total = len(notes)
|
||||
if total == 0:
|
||||
return SchemaDrift()
|
||||
|
||||
# --- Analyze actual usage ---
|
||||
obs_frequencies = analyze_observations(notes, total, max_sample_values=3)
|
||||
rel_frequencies = analyze_relations(notes, total, max_sample_values=3)
|
||||
|
||||
# Build lookup from schema fields
|
||||
schema_field_names = {f.name for f in schema.fields}
|
||||
|
||||
# Build lookup from actual frequencies
|
||||
obs_freq_by_name = {f.name: f for f in obs_frequencies}
|
||||
rel_freq_by_name = {f.name: f for f in rel_frequencies}
|
||||
all_freq_by_name = {**obs_freq_by_name, **rel_freq_by_name}
|
||||
|
||||
result = SchemaDrift()
|
||||
|
||||
# --- Detect new fields ---
|
||||
# Fields that appear frequently in notes but aren't declared in the schema
|
||||
for freq in obs_frequencies + rel_frequencies:
|
||||
if freq.name not in schema_field_names and freq.percentage >= new_field_threshold:
|
||||
result.new_fields.append(freq)
|
||||
|
||||
# --- Detect dropped fields ---
|
||||
# Fields declared in the schema but rarely appearing in actual notes
|
||||
for schema_field in schema.fields:
|
||||
freq = all_freq_by_name.get(schema_field.name)
|
||||
if freq is None:
|
||||
# Field doesn't appear at all in any note
|
||||
result.dropped_fields.append(
|
||||
FieldFrequency(
|
||||
name=schema_field.name,
|
||||
source="relation" if schema_field.is_entity_ref else "observation",
|
||||
count=0,
|
||||
total=total,
|
||||
percentage=0.0,
|
||||
)
|
||||
)
|
||||
elif freq.percentage < dropped_field_threshold:
|
||||
result.dropped_fields.append(freq)
|
||||
|
||||
# --- Detect cardinality changes ---
|
||||
# Fields where the schema says single but usage shows array, or vice versa
|
||||
for schema_field in schema.fields:
|
||||
freq = all_freq_by_name.get(schema_field.name)
|
||||
if freq is None:
|
||||
continue
|
||||
|
||||
if schema_field.is_array and not freq.is_array:
|
||||
result.cardinality_changes.append(
|
||||
f"{schema_field.name}: schema declares array but usage is typically single-value"
|
||||
)
|
||||
elif not schema_field.is_array and freq.is_array:
|
||||
result.cardinality_changes.append(
|
||||
f"{schema_field.name}: schema declares single-value but usage is typically array"
|
||||
)
|
||||
|
||||
return result
|
||||
@@ -0,0 +1,323 @@
|
||||
"""Schema inference engine for Basic Memory.
|
||||
|
||||
Analyzes notes of a given type and suggests a schema based on observation
|
||||
and relation frequency. Instead of requiring users to define schemas upfront,
|
||||
schemas emerge from actual usage patterns:
|
||||
|
||||
Write notes freely -> Patterns emerge -> Crystallize into schema
|
||||
|
||||
Frequency thresholds:
|
||||
- 95%+ present -> required field
|
||||
- 25%+ present -> optional field
|
||||
- Below 25% -> excluded from suggestion (but noted)
|
||||
"""
|
||||
|
||||
from collections import Counter
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
|
||||
# --- Result Data Model ---
|
||||
|
||||
|
||||
@dataclass
|
||||
class FieldFrequency:
|
||||
"""Frequency analysis for a single field across notes of a type."""
|
||||
|
||||
name: str
|
||||
source: str # "observation" | "relation"
|
||||
count: int # notes containing this field
|
||||
total: int # total notes analyzed
|
||||
percentage: float
|
||||
sample_values: list[str] = field(default_factory=list)
|
||||
is_array: bool = False # True if typically appears multiple times per note
|
||||
target_type: str | None = None # For relations, the most common target entity type
|
||||
|
||||
|
||||
@dataclass
|
||||
class InferenceResult:
|
||||
"""Complete inference result with frequency analysis and suggested schema."""
|
||||
|
||||
entity_type: str
|
||||
notes_analyzed: int
|
||||
field_frequencies: list[FieldFrequency]
|
||||
suggested_schema: dict # Ready-to-use Picoschema YAML dict
|
||||
suggested_required: list[str]
|
||||
suggested_optional: list[str]
|
||||
excluded: list[str] # Below threshold
|
||||
|
||||
|
||||
# --- Note Data Abstraction ---
|
||||
# Instead of depending on the ORM Entity model, we accept simple data structures.
|
||||
# This keeps the inference engine decoupled from the data access layer.
|
||||
|
||||
|
||||
@dataclass
|
||||
class ObservationData:
|
||||
"""Lightweight observation for schema analysis. Decoupled from ORM."""
|
||||
|
||||
category: str
|
||||
content: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class RelationData:
|
||||
"""Lightweight relation for schema analysis. Decoupled from ORM."""
|
||||
|
||||
relation_type: str
|
||||
target_name: str
|
||||
target_entity_type: str | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class NoteData:
|
||||
"""Minimal note representation for inference analysis.
|
||||
|
||||
Decoupled from ORM models so the inference engine can work with
|
||||
any data source (database, files, API responses).
|
||||
"""
|
||||
|
||||
identifier: str
|
||||
observations: list[ObservationData]
|
||||
relations: list[RelationData]
|
||||
|
||||
|
||||
# --- Inference Logic ---
|
||||
|
||||
|
||||
def infer_schema(
|
||||
entity_type: str,
|
||||
notes: list[NoteData],
|
||||
required_threshold: float = 0.95,
|
||||
optional_threshold: float = 0.25,
|
||||
max_sample_values: int = 5,
|
||||
) -> InferenceResult:
|
||||
"""Analyze notes and suggest a Picoschema definition.
|
||||
|
||||
Examines observation categories and relation types across all provided notes.
|
||||
Fields that appear in a high percentage of notes become required; those that
|
||||
appear less frequently become optional.
|
||||
|
||||
Args:
|
||||
entity_type: The entity type being analyzed (e.g., "Person").
|
||||
notes: List of NoteData objects to analyze.
|
||||
required_threshold: Frequency at or above which a field is required (default 0.95).
|
||||
optional_threshold: Frequency at or above which a field is optional (default 0.25).
|
||||
max_sample_values: Maximum number of sample values to include per field.
|
||||
|
||||
Returns:
|
||||
An InferenceResult with frequency analysis and suggested Picoschema dict.
|
||||
"""
|
||||
total = len(notes)
|
||||
if total == 0:
|
||||
return InferenceResult(
|
||||
entity_type=entity_type,
|
||||
notes_analyzed=0,
|
||||
field_frequencies=[],
|
||||
suggested_schema={},
|
||||
suggested_required=[],
|
||||
suggested_optional=[],
|
||||
excluded=[],
|
||||
)
|
||||
|
||||
# --- Analyze observation frequencies ---
|
||||
obs_frequencies = analyze_observations(notes, total, max_sample_values)
|
||||
|
||||
# --- Analyze relation frequencies ---
|
||||
rel_frequencies = analyze_relations(notes, total, max_sample_values)
|
||||
|
||||
# --- Classify fields by threshold ---
|
||||
all_frequencies = obs_frequencies + rel_frequencies
|
||||
suggested_required: list[str] = []
|
||||
suggested_optional: list[str] = []
|
||||
excluded: list[str] = []
|
||||
|
||||
for freq in all_frequencies:
|
||||
if freq.percentage >= required_threshold:
|
||||
suggested_required.append(freq.name)
|
||||
elif freq.percentage >= optional_threshold:
|
||||
suggested_optional.append(freq.name)
|
||||
else:
|
||||
excluded.append(freq.name)
|
||||
|
||||
# --- Build suggested Picoschema dict ---
|
||||
suggested_schema = _build_picoschema_dict(
|
||||
all_frequencies, required_threshold, optional_threshold
|
||||
)
|
||||
|
||||
return InferenceResult(
|
||||
entity_type=entity_type,
|
||||
notes_analyzed=total,
|
||||
field_frequencies=all_frequencies,
|
||||
suggested_schema=suggested_schema,
|
||||
suggested_required=suggested_required,
|
||||
suggested_optional=suggested_optional,
|
||||
excluded=excluded,
|
||||
)
|
||||
|
||||
|
||||
# --- Observation Analysis ---
|
||||
|
||||
|
||||
def analyze_observations(
|
||||
notes: list[NoteData],
|
||||
total: int,
|
||||
max_sample_values: int,
|
||||
) -> list[FieldFrequency]:
|
||||
"""Count observation category frequencies across notes.
|
||||
|
||||
A category is counted once per note (presence), not per occurrence.
|
||||
Array detection: if a category appears multiple times in a single note
|
||||
in more than half the notes where it appears, it's flagged as an array.
|
||||
"""
|
||||
# Count how many notes contain each category (presence per note)
|
||||
category_note_count: Counter[str] = Counter()
|
||||
# Count how many notes have multiple occurrences (for array detection)
|
||||
category_multi_count: Counter[str] = Counter()
|
||||
# Collect sample values
|
||||
category_samples: dict[str, list[str]] = {}
|
||||
|
||||
for note in notes:
|
||||
# Group observations by category within this note
|
||||
note_categories: dict[str, list[str]] = {}
|
||||
for obs in note.observations:
|
||||
note_categories.setdefault(obs.category, []).append(obs.content)
|
||||
|
||||
for category, values in note_categories.items():
|
||||
category_note_count[category] += 1
|
||||
if len(values) > 1:
|
||||
category_multi_count[category] += 1
|
||||
|
||||
# Collect sample values (deduplicated)
|
||||
samples = category_samples.setdefault(category, [])
|
||||
for v in values:
|
||||
if v not in samples and len(samples) < max_sample_values:
|
||||
samples.append(v)
|
||||
|
||||
# Build FieldFrequency objects
|
||||
frequencies: list[FieldFrequency] = []
|
||||
for category, count in category_note_count.most_common():
|
||||
# Array detection: if more than half of notes with this category have
|
||||
# multiple occurrences, treat it as an array field
|
||||
multi_count = category_multi_count.get(category, 0)
|
||||
is_array = multi_count > (count / 2)
|
||||
|
||||
frequencies.append(
|
||||
FieldFrequency(
|
||||
name=category,
|
||||
source="observation",
|
||||
count=count,
|
||||
total=total,
|
||||
percentage=count / total,
|
||||
sample_values=category_samples.get(category, []),
|
||||
is_array=is_array,
|
||||
)
|
||||
)
|
||||
|
||||
return frequencies
|
||||
|
||||
|
||||
# --- Relation Analysis ---
|
||||
|
||||
|
||||
def analyze_relations(
|
||||
notes: list[NoteData],
|
||||
total: int,
|
||||
max_sample_values: int,
|
||||
) -> list[FieldFrequency]:
|
||||
"""Count relation type frequencies across notes.
|
||||
|
||||
Similar to observations, a relation type is counted once per note.
|
||||
Array detection follows the same logic.
|
||||
"""
|
||||
rel_note_count: Counter[str] = Counter()
|
||||
rel_multi_count: Counter[str] = Counter()
|
||||
rel_samples: dict[str, list[str]] = {}
|
||||
# Track target entity types to suggest the type in the schema
|
||||
rel_target_types: dict[str, Counter[str]] = {}
|
||||
|
||||
for note in notes:
|
||||
note_rels: dict[str, list[str]] = {}
|
||||
note_rel_objects: dict[str, list[RelationData]] = {}
|
||||
for rel in note.relations:
|
||||
note_rels.setdefault(rel.relation_type, []).append(rel.target_name)
|
||||
note_rel_objects.setdefault(rel.relation_type, []).append(rel)
|
||||
|
||||
for rel_type, targets in note_rels.items():
|
||||
rel_note_count[rel_type] += 1
|
||||
if len(targets) > 1:
|
||||
rel_multi_count[rel_type] += 1
|
||||
|
||||
samples = rel_samples.setdefault(rel_type, [])
|
||||
for t in targets:
|
||||
if t not in samples and len(samples) < max_sample_values:
|
||||
samples.append(t)
|
||||
|
||||
# Track target entity types from individual relations (not the source note)
|
||||
target_counter = rel_target_types.setdefault(rel_type, Counter())
|
||||
for rel in note_rel_objects[rel_type]:
|
||||
if rel.target_entity_type:
|
||||
target_counter[rel.target_entity_type] += 1
|
||||
|
||||
frequencies: list[FieldFrequency] = []
|
||||
for rel_type, count in rel_note_count.most_common():
|
||||
multi_count = rel_multi_count.get(rel_type, 0)
|
||||
is_array = multi_count > (count / 2)
|
||||
|
||||
# Determine most common target type
|
||||
target_counter = rel_target_types.get(rel_type, Counter())
|
||||
most_common_target = target_counter.most_common(1)[0][0] if target_counter else None
|
||||
|
||||
frequencies.append(
|
||||
FieldFrequency(
|
||||
name=rel_type,
|
||||
source="relation",
|
||||
count=count,
|
||||
total=total,
|
||||
percentage=count / total,
|
||||
sample_values=rel_samples.get(rel_type, []),
|
||||
is_array=is_array,
|
||||
target_type=most_common_target,
|
||||
)
|
||||
)
|
||||
|
||||
return frequencies
|
||||
|
||||
|
||||
# --- Schema Generation ---
|
||||
|
||||
|
||||
def _build_picoschema_dict(
|
||||
frequencies: list[FieldFrequency],
|
||||
required_threshold: float,
|
||||
optional_threshold: float,
|
||||
) -> dict:
|
||||
"""Build a Picoschema YAML dict from field frequencies.
|
||||
|
||||
Only includes fields at or above the optional threshold.
|
||||
"""
|
||||
schema: dict = {}
|
||||
|
||||
for freq in frequencies:
|
||||
if freq.percentage < optional_threshold:
|
||||
continue
|
||||
|
||||
is_required = freq.percentage >= required_threshold
|
||||
|
||||
# --- Build the field key ---
|
||||
key = freq.name
|
||||
if not is_required:
|
||||
key += "?"
|
||||
if freq.is_array:
|
||||
key += "(array)"
|
||||
|
||||
# --- Build the field value ---
|
||||
if freq.source == "relation":
|
||||
# Relations become entity reference fields
|
||||
target = freq.target_type or "string"
|
||||
# Capitalize first letter for entity ref convention
|
||||
target = target[0].upper() + target[1:] if target != "string" else "string"
|
||||
schema[key] = target
|
||||
else:
|
||||
schema[key] = "string"
|
||||
|
||||
return schema
|
||||
@@ -0,0 +1,236 @@
|
||||
"""Picoschema parser for Basic Memory.
|
||||
|
||||
Parses Picoschema YAML dicts (from note frontmatter) into typed dataclass
|
||||
representations. Picoschema is a compact schema notation from Google's Dotprompt
|
||||
that fits naturally in YAML frontmatter.
|
||||
|
||||
Syntax reference:
|
||||
field: type, description # required field
|
||||
field?: type, description # optional field
|
||||
field(array): type # array of values
|
||||
field?(enum): [val1, val2] # enumeration
|
||||
field?(object): # nested object
|
||||
sub_field: type
|
||||
EntityName as type (capitalized) # entity reference
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
|
||||
# --- Data Model ---
|
||||
|
||||
|
||||
@dataclass
|
||||
class SchemaField:
|
||||
"""A single field in a Picoschema definition.
|
||||
|
||||
Maps to either an observation category or a relation type in Basic Memory notes.
|
||||
"""
|
||||
|
||||
name: str
|
||||
type: str # string, integer, number, boolean, any, or EntityName
|
||||
required: bool # True unless field name ends with ?
|
||||
is_array: bool = False # True if (array) notation
|
||||
is_enum: bool = False # True if (enum) notation
|
||||
enum_values: list[str] = field(default_factory=list)
|
||||
description: str | None = None # Text after comma
|
||||
is_entity_ref: bool = False # True if type is capitalized (entity reference)
|
||||
children: list["SchemaField"] = field(default_factory=list) # For (object) types
|
||||
|
||||
|
||||
@dataclass
|
||||
class SchemaDefinition:
|
||||
"""A complete schema definition parsed from a schema note's frontmatter.
|
||||
|
||||
Combines the parsed fields with metadata about the schema itself.
|
||||
"""
|
||||
|
||||
entity: str # The entity type this schema describes
|
||||
version: int # Schema version
|
||||
fields: list[SchemaField] # Parsed fields
|
||||
validation_mode: str # "warn" | "strict" | "off"
|
||||
|
||||
|
||||
# --- Built-in scalar types ---
|
||||
# Types that are NOT entity references. Anything not in this set and starting
|
||||
# with an uppercase letter is treated as an entity reference.
|
||||
|
||||
SCALAR_TYPES = frozenset({"string", "integer", "number", "boolean", "any"})
|
||||
|
||||
|
||||
# --- Field Name Parsing ---
|
||||
|
||||
|
||||
def _parse_field_key(key: str) -> tuple[str, bool, bool, bool, bool]:
|
||||
"""Parse a Picoschema field key into its components.
|
||||
|
||||
Returns (name, required, is_array, is_enum, is_object).
|
||||
The key format is: name[?][(array|enum|object)]
|
||||
|
||||
Examples:
|
||||
"name" -> ("name", True, False, False)
|
||||
"role?" -> ("role", False, False, False)
|
||||
"tags?(array)" -> ("tags", False, True, False)
|
||||
"status?(enum)" -> ("status", False, False, True)
|
||||
"metadata?(object)" -> ("metadata", False, False, False) + children
|
||||
"""
|
||||
required = True
|
||||
is_array = False
|
||||
is_enum = False
|
||||
is_object = False
|
||||
|
||||
# Check for modifier suffix: (array), (enum), (object)
|
||||
if key.endswith("(array)"):
|
||||
is_array = True
|
||||
key = key[: -len("(array)")]
|
||||
elif key.endswith("(enum)"):
|
||||
is_enum = True
|
||||
key = key[: -len("(enum)")]
|
||||
elif key.endswith("(object)"):
|
||||
is_object = True
|
||||
key = key[: -len("(object)")]
|
||||
|
||||
# Check for optional marker
|
||||
if key.endswith("?"):
|
||||
required = False
|
||||
key = key[:-1]
|
||||
|
||||
return key, required, is_array, is_enum, is_object
|
||||
|
||||
|
||||
def _parse_type_and_description(value: str) -> tuple[str, str | None]:
|
||||
"""Parse a type string that may include a comma-separated description.
|
||||
|
||||
Examples:
|
||||
"string" -> ("string", None)
|
||||
"string, full name" -> ("string", "full name")
|
||||
"Organization, employer" -> ("Organization", "employer")
|
||||
"""
|
||||
if "," in value:
|
||||
type_str, desc = value.split(",", 1)
|
||||
return type_str.strip(), desc.strip()
|
||||
return value.strip(), None
|
||||
|
||||
|
||||
def _is_entity_ref_type(type_str: str) -> bool:
|
||||
"""Determine if a type string represents an entity reference.
|
||||
|
||||
Entity references are capitalized type names that are not built-in scalar types.
|
||||
"""
|
||||
if type_str in SCALAR_TYPES:
|
||||
return False
|
||||
# Capitalized first letter = entity reference
|
||||
return len(type_str) > 0 and type_str[0].isupper()
|
||||
|
||||
|
||||
# --- Main Parser ---
|
||||
|
||||
|
||||
def parse_picoschema(yaml_dict: dict) -> list[SchemaField]:
|
||||
"""Parse a Picoschema YAML dict into a list of SchemaField objects.
|
||||
|
||||
This is the core parser that converts YAML frontmatter schema definitions
|
||||
into structured SchemaField dataclasses.
|
||||
|
||||
Args:
|
||||
yaml_dict: The schema dict from YAML frontmatter. Keys are field
|
||||
declarations (e.g., "name", "role?", "tags?(array)"), values are
|
||||
type declarations (e.g., "string", "string, description").
|
||||
|
||||
Returns:
|
||||
List of SchemaField objects representing the schema.
|
||||
"""
|
||||
fields: list[SchemaField] = []
|
||||
|
||||
for key, value in yaml_dict.items():
|
||||
name, required, is_array, is_enum, is_object = _parse_field_key(key)
|
||||
|
||||
# --- Enum fields ---
|
||||
# Trigger: value is a list (e.g., [active, inactive])
|
||||
# Why: enums declare allowed values directly as a YAML list
|
||||
# Outcome: SchemaField with is_enum=True and enum_values populated
|
||||
if is_enum:
|
||||
enum_values = value if isinstance(value, list) else [str(value)]
|
||||
fields.append(
|
||||
SchemaField(
|
||||
name=name,
|
||||
type="enum",
|
||||
required=required,
|
||||
is_enum=True,
|
||||
enum_values=[str(v) for v in enum_values],
|
||||
)
|
||||
)
|
||||
continue
|
||||
|
||||
# --- Object fields ---
|
||||
# Trigger: value is a dict (nested sub-fields)
|
||||
# Why: objects contain child fields parsed recursively
|
||||
# Outcome: SchemaField with children populated via recursive parse
|
||||
if is_object or (isinstance(value, dict) and not is_enum):
|
||||
children = parse_picoschema(value) if isinstance(value, dict) else []
|
||||
fields.append(
|
||||
SchemaField(
|
||||
name=name,
|
||||
type="object",
|
||||
required=required,
|
||||
children=children,
|
||||
)
|
||||
)
|
||||
continue
|
||||
|
||||
# --- Scalar and entity ref fields ---
|
||||
type_str, description = _parse_type_and_description(str(value))
|
||||
is_entity_ref = _is_entity_ref_type(type_str)
|
||||
|
||||
fields.append(
|
||||
SchemaField(
|
||||
name=name,
|
||||
type=type_str,
|
||||
required=required,
|
||||
is_array=is_array,
|
||||
description=description,
|
||||
is_entity_ref=is_entity_ref,
|
||||
)
|
||||
)
|
||||
|
||||
return fields
|
||||
|
||||
|
||||
def parse_schema_note(frontmatter: dict) -> SchemaDefinition:
|
||||
"""Parse a full schema note's frontmatter into a SchemaDefinition.
|
||||
|
||||
A schema note has type: schema and contains:
|
||||
- entity: the entity type this schema describes
|
||||
- version: schema version number
|
||||
- schema: the Picoschema dict
|
||||
- settings.validation: validation mode (warn/strict/off)
|
||||
|
||||
Args:
|
||||
frontmatter: The complete YAML frontmatter dict from a schema note.
|
||||
|
||||
Returns:
|
||||
A SchemaDefinition with parsed fields and metadata.
|
||||
|
||||
Raises:
|
||||
ValueError: If required fields (entity, schema) are missing.
|
||||
"""
|
||||
entity = frontmatter.get("entity")
|
||||
if not entity:
|
||||
raise ValueError("Schema note missing required 'entity' field in frontmatter")
|
||||
|
||||
schema_dict = frontmatter.get("schema")
|
||||
if not schema_dict or not isinstance(schema_dict, dict):
|
||||
raise ValueError("Schema note missing required 'schema' dict in frontmatter")
|
||||
|
||||
version = frontmatter.get("version", 1)
|
||||
settings = frontmatter.get("settings", {})
|
||||
validation_mode = settings.get("validation", "warn") if isinstance(settings, dict) else "warn"
|
||||
|
||||
fields = parse_picoschema(schema_dict)
|
||||
|
||||
return SchemaDefinition(
|
||||
entity=entity,
|
||||
version=version,
|
||||
fields=fields,
|
||||
validation_mode=validation_mode,
|
||||
)
|
||||
@@ -0,0 +1,118 @@
|
||||
"""Schema resolver for Basic Memory.
|
||||
|
||||
Finds the applicable schema for a note using a priority-based resolution order:
|
||||
1. Inline schema -> frontmatter['schema'] is a dict
|
||||
2. Explicit ref -> frontmatter['schema'] is a string (entity name or permalink)
|
||||
3. Implicit by type -> frontmatter['type'] matches a schema note's entity field
|
||||
4. No schema -> returns None (perfectly fine)
|
||||
|
||||
The resolver takes a search function as a dependency instead of importing
|
||||
repository code directly, keeping the schema package decoupled from the
|
||||
data access layer.
|
||||
"""
|
||||
|
||||
from collections.abc import Callable, Awaitable
|
||||
|
||||
from basic_memory.schema.parser import SchemaDefinition, parse_picoschema, parse_schema_note
|
||||
|
||||
|
||||
# Type alias for the search function dependency.
|
||||
# Given a query string, returns a list of frontmatter dicts from matching schema notes.
|
||||
type SchemaSearchFn = Callable[[str], Awaitable[list[dict]]]
|
||||
|
||||
|
||||
async def resolve_schema(
|
||||
note_frontmatter: dict,
|
||||
search_fn: SchemaSearchFn,
|
||||
) -> SchemaDefinition | None:
|
||||
"""Resolve the schema for a note based on its frontmatter.
|
||||
|
||||
Resolution order:
|
||||
1. Inline schema (frontmatter['schema'] is a dict) - parsed directly
|
||||
2. Explicit reference (frontmatter['schema'] is a string) - looked up via search_fn
|
||||
3. Implicit by type (frontmatter['type']) - searches for schema note with matching entity
|
||||
4. No schema - returns None
|
||||
|
||||
Args:
|
||||
note_frontmatter: The YAML frontmatter dict from the note being validated.
|
||||
search_fn: An async callable that takes a query string and returns a list
|
||||
of frontmatter dicts from matching schema notes. This keeps the resolver
|
||||
decoupled from the repository/service layer.
|
||||
|
||||
Returns:
|
||||
A SchemaDefinition if a schema is found, None otherwise.
|
||||
"""
|
||||
schema_value = note_frontmatter.get("schema")
|
||||
|
||||
# --- 1. Inline schema ---
|
||||
# Trigger: schema field is a dict (the Picoschema definition lives in this note)
|
||||
# Why: inline schemas are self-contained, no lookup needed
|
||||
# Outcome: parse and return immediately
|
||||
if isinstance(schema_value, dict):
|
||||
return _schema_from_inline(schema_value, note_frontmatter)
|
||||
|
||||
# --- 2. Explicit reference ---
|
||||
# Trigger: schema field is a string (entity name or permalink)
|
||||
# Why: the note points to a specific schema note by name
|
||||
# Outcome: search for the referenced schema note and parse it
|
||||
if isinstance(schema_value, str):
|
||||
result = await _schema_from_reference(schema_value, search_fn)
|
||||
if result is not None:
|
||||
return result
|
||||
|
||||
# --- 3. Implicit by type ---
|
||||
# Trigger: no schema field, but the note has a type field
|
||||
# Why: convention — a note with type: Person looks for a schema with entity: Person
|
||||
# Outcome: search for a schema note whose entity matches the note's type
|
||||
note_type = note_frontmatter.get("type")
|
||||
if note_type:
|
||||
result = await _schema_from_type(note_type, search_fn)
|
||||
if result is not None:
|
||||
return result
|
||||
|
||||
# --- 4. No schema ---
|
||||
return None
|
||||
|
||||
|
||||
def _schema_from_inline(schema_dict: dict, frontmatter: dict) -> SchemaDefinition:
|
||||
"""Build a SchemaDefinition from an inline schema dict.
|
||||
|
||||
For inline schemas, we derive metadata from the note's own frontmatter
|
||||
since there's no separate schema note.
|
||||
"""
|
||||
fields = parse_picoschema(schema_dict)
|
||||
entity = frontmatter.get("type", "unknown")
|
||||
settings = frontmatter.get("settings", {})
|
||||
validation_mode = settings.get("validation", "warn") if isinstance(settings, dict) else "warn"
|
||||
|
||||
return SchemaDefinition(
|
||||
entity=entity,
|
||||
version=1,
|
||||
fields=fields,
|
||||
validation_mode=validation_mode,
|
||||
)
|
||||
|
||||
|
||||
async def _schema_from_reference(
|
||||
ref: str,
|
||||
search_fn: SchemaSearchFn,
|
||||
) -> SchemaDefinition | None:
|
||||
"""Look up a schema by entity name or permalink reference.
|
||||
|
||||
The search function is expected to find schema notes matching the reference.
|
||||
"""
|
||||
results = await search_fn(ref)
|
||||
if not results:
|
||||
return None
|
||||
return parse_schema_note(results[0])
|
||||
|
||||
|
||||
async def _schema_from_type(
|
||||
note_type: str,
|
||||
search_fn: SchemaSearchFn,
|
||||
) -> SchemaDefinition | None:
|
||||
"""Look up a schema implicitly by matching the note's type to a schema's entity field."""
|
||||
results = await search_fn(note_type)
|
||||
if not results:
|
||||
return None
|
||||
return parse_schema_note(results[0])
|
||||
@@ -0,0 +1,259 @@
|
||||
"""Schema validator for Basic Memory.
|
||||
|
||||
Validates a note's observations and relations against a resolved schema definition.
|
||||
The mapping rules ground schema fields in the existing Basic Memory note format:
|
||||
|
||||
Schema Declaration -> Grounded In
|
||||
-----------------------------------------------
|
||||
field: string -> observation [field] value
|
||||
field?(array): string -> multiple [field] observations
|
||||
field?: EntityType -> relation 'field [[Target]]'
|
||||
field?(array): EntityType -> multiple 'field' relations
|
||||
field?(enum): [values] -> observation [field] value where value is in set
|
||||
|
||||
Validation is soft by default (warn mode). Unmatched observations and relations
|
||||
are informational, not errors -- schemas are a subset, not a straitjacket.
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass, field as dataclass_field
|
||||
|
||||
from basic_memory.schema.inference import ObservationData, RelationData
|
||||
from basic_memory.schema.parser import SchemaDefinition, SchemaField
|
||||
|
||||
|
||||
# --- Result Data Model ---
|
||||
|
||||
|
||||
@dataclass
|
||||
class FieldResult:
|
||||
"""Validation result for a single schema field."""
|
||||
|
||||
field: SchemaField
|
||||
status: str # "present" | "missing" | "enum_mismatch"
|
||||
values: list[str] = dataclass_field(default_factory=list) # Matched values
|
||||
message: str | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class ValidationResult:
|
||||
"""Complete validation result for a note against a schema."""
|
||||
|
||||
note_identifier: str
|
||||
schema_entity: str
|
||||
passed: bool # True if no errors (warnings are OK)
|
||||
field_results: list[FieldResult] = dataclass_field(default_factory=list)
|
||||
unmatched_observations: dict[str, int] = dataclass_field(default_factory=dict) # cat -> count
|
||||
unmatched_relations: list[str] = dataclass_field(default_factory=list) # types not in schema
|
||||
warnings: list[str] = dataclass_field(default_factory=list)
|
||||
errors: list[str] = dataclass_field(default_factory=list)
|
||||
|
||||
|
||||
# --- Validation Logic ---
|
||||
|
||||
|
||||
def validate_note(
|
||||
note_identifier: str,
|
||||
schema: SchemaDefinition,
|
||||
observations: list[ObservationData],
|
||||
relations: list[RelationData],
|
||||
) -> ValidationResult:
|
||||
"""Validate a note against a schema definition.
|
||||
|
||||
Args:
|
||||
note_identifier: The note's title, permalink, or file path for reporting.
|
||||
schema: The resolved SchemaDefinition to validate against.
|
||||
observations: List of ObservationData from the note's observations.
|
||||
relations: List of RelationData from the note's relations.
|
||||
|
||||
Returns:
|
||||
A ValidationResult with per-field results, unmatched items, and warnings/errors.
|
||||
"""
|
||||
result = ValidationResult(
|
||||
note_identifier=note_identifier,
|
||||
schema_entity=schema.entity,
|
||||
passed=True,
|
||||
)
|
||||
|
||||
# Build lookup structures from the note's actual content
|
||||
obs_by_category = _group_observations(observations)
|
||||
rel_by_type = _group_relations(relations)
|
||||
|
||||
# Track which observation categories and relation types are matched by schema fields
|
||||
matched_categories: set[str] = set()
|
||||
matched_relation_types: set[str] = set()
|
||||
|
||||
# --- Validate each schema field ---
|
||||
for schema_field in schema.fields:
|
||||
field_result = _validate_field(schema_field, obs_by_category, rel_by_type)
|
||||
result.field_results.append(field_result)
|
||||
|
||||
# Track which categories/relation types this field consumed
|
||||
if schema_field.is_entity_ref:
|
||||
matched_relation_types.add(schema_field.name)
|
||||
else:
|
||||
matched_categories.add(schema_field.name)
|
||||
|
||||
# --- Generate warnings or errors based on validation mode ---
|
||||
# Trigger: field declared in schema but not found in note
|
||||
# Why: required missing = warning (or error in strict); optional missing = silent
|
||||
# Outcome: only required missing fields produce diagnostics
|
||||
if field_result.status == "missing" and schema_field.required:
|
||||
msg = _missing_field_message(schema_field)
|
||||
if schema.validation_mode == "strict":
|
||||
result.errors.append(msg)
|
||||
result.passed = False
|
||||
else:
|
||||
result.warnings.append(msg)
|
||||
|
||||
elif field_result.status == "enum_mismatch":
|
||||
msg = field_result.message or f"Field '{schema_field.name}' has invalid enum value"
|
||||
if schema.validation_mode == "strict":
|
||||
result.errors.append(msg)
|
||||
result.passed = False
|
||||
else:
|
||||
result.warnings.append(msg)
|
||||
|
||||
# --- Collect unmatched observations ---
|
||||
for category, values in obs_by_category.items():
|
||||
if category not in matched_categories:
|
||||
result.unmatched_observations[category] = len(values)
|
||||
|
||||
# --- Collect unmatched relations ---
|
||||
for rel_type in rel_by_type:
|
||||
if rel_type not in matched_relation_types:
|
||||
result.unmatched_relations.append(rel_type)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
# --- Field Validation ---
|
||||
|
||||
|
||||
def _validate_field(
|
||||
schema_field: SchemaField,
|
||||
obs_by_category: dict[str, list[str]],
|
||||
rel_by_type: dict[str, list[str]],
|
||||
) -> FieldResult:
|
||||
"""Validate a single schema field against the note's data.
|
||||
|
||||
Entity ref fields map to relations; all other fields map to observations.
|
||||
"""
|
||||
# --- Entity reference fields map to relations ---
|
||||
if schema_field.is_entity_ref:
|
||||
return _validate_entity_ref_field(schema_field, rel_by_type)
|
||||
|
||||
# --- Enum fields require value membership check ---
|
||||
if schema_field.is_enum:
|
||||
return _validate_enum_field(schema_field, obs_by_category)
|
||||
|
||||
# --- Scalar and array fields map to observations ---
|
||||
return _validate_observation_field(schema_field, obs_by_category)
|
||||
|
||||
|
||||
def _validate_observation_field(
|
||||
schema_field: SchemaField,
|
||||
obs_by_category: dict[str, list[str]],
|
||||
) -> FieldResult:
|
||||
"""Validate a field that maps to observation categories."""
|
||||
values = obs_by_category.get(schema_field.name, [])
|
||||
|
||||
if not values:
|
||||
return FieldResult(
|
||||
field=schema_field,
|
||||
status="missing",
|
||||
message=_missing_field_message(schema_field),
|
||||
)
|
||||
|
||||
return FieldResult(
|
||||
field=schema_field,
|
||||
status="present",
|
||||
values=values,
|
||||
)
|
||||
|
||||
|
||||
def _validate_entity_ref_field(
|
||||
schema_field: SchemaField,
|
||||
rel_by_type: dict[str, list[str]],
|
||||
) -> FieldResult:
|
||||
"""Validate a field that maps to relations (entity references)."""
|
||||
targets = rel_by_type.get(schema_field.name, [])
|
||||
|
||||
if not targets:
|
||||
return FieldResult(
|
||||
field=schema_field,
|
||||
status="missing",
|
||||
message=f"Missing relation: {schema_field.name} (no '{schema_field.name} [[...]]' "
|
||||
f"relation found)",
|
||||
)
|
||||
|
||||
return FieldResult(
|
||||
field=schema_field,
|
||||
status="present",
|
||||
values=targets,
|
||||
)
|
||||
|
||||
|
||||
def _validate_enum_field(
|
||||
schema_field: SchemaField,
|
||||
obs_by_category: dict[str, list[str]],
|
||||
) -> FieldResult:
|
||||
"""Validate an enum field -- value must be in the allowed set."""
|
||||
values = obs_by_category.get(schema_field.name, [])
|
||||
|
||||
if not values:
|
||||
return FieldResult(
|
||||
field=schema_field,
|
||||
status="missing",
|
||||
message=_missing_field_message(schema_field),
|
||||
)
|
||||
|
||||
# Check each value against the allowed enum values
|
||||
invalid_values = [v for v in values if v not in schema_field.enum_values]
|
||||
if invalid_values:
|
||||
allowed = ", ".join(schema_field.enum_values)
|
||||
invalid = ", ".join(invalid_values)
|
||||
return FieldResult(
|
||||
field=schema_field,
|
||||
status="enum_mismatch",
|
||||
values=values,
|
||||
message=f"Field '{schema_field.name}' has invalid value(s): {invalid} "
|
||||
f"(allowed: {allowed})",
|
||||
)
|
||||
|
||||
return FieldResult(
|
||||
field=schema_field,
|
||||
status="present",
|
||||
values=values,
|
||||
)
|
||||
|
||||
|
||||
# --- Helper Functions ---
|
||||
|
||||
|
||||
def _group_observations(observations: list[ObservationData]) -> dict[str, list[str]]:
|
||||
"""Group observations by category."""
|
||||
result: dict[str, list[str]] = {}
|
||||
for obs in observations:
|
||||
result.setdefault(obs.category, []).append(obs.content)
|
||||
return result
|
||||
|
||||
|
||||
def _group_relations(relations: list[RelationData]) -> dict[str, list[str]]:
|
||||
"""Group relations by relation type."""
|
||||
result: dict[str, list[str]] = {}
|
||||
for rel in relations:
|
||||
result.setdefault(rel.relation_type, []).append(rel.target_name)
|
||||
return result
|
||||
|
||||
|
||||
def _missing_field_message(schema_field: SchemaField) -> str:
|
||||
"""Generate a human-readable message for a missing field."""
|
||||
kind = "required" if schema_field.required else "optional"
|
||||
|
||||
if schema_field.is_entity_ref:
|
||||
return (
|
||||
f"Missing {kind} field: {schema_field.name} "
|
||||
f"(no '{schema_field.name} [[...]]' relation found)"
|
||||
)
|
||||
|
||||
return f"Missing {kind} field: {schema_field.name} (expected [{schema_field.name}] observation)"
|
||||
Reference in New Issue
Block a user