admin/basicmachines-co-basic-memory
1
0
Fork 0
mirror of https://github.com/basicmachines-co/basic-memory synced 2026-06-21 13:47:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: add Logfire phased instrumentation (#692)

Browse Source 
Signed-off-by: phernandez <paul@basicmachines.co>
Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Paul Hernandez
2026-03-24 20:39:42 -05:00
committed by GitHub
parent 36848410a1
commit 4791e19685
34 changed files with 3838 additions and 1329 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/basic_memory/api/app.py
+26 -16
View File
@@ -25,6 +25,7 @@ from basic_memory.api.v2.routers.project_router import (
list_projects,
synchronize_projects,
)
from basic_memory import telemetry
from basic_memory.config import init_api_logging
from basic_memory.services.exceptions import EntityAlreadyExistsError
from basic_memory.services.initialization import initialize_app
@@ -43,30 +44,39 @@ async def lifespan(app: FastAPI): # pragma: no cover
set_container(container)
app.state.container = container
logger.info(f"Starting Basic Memory API (mode={container.mode.name})")
with telemetry.operation(
"api.lifecycle.startup",
entrypoint="api",
mode=container.mode.name.lower(),
):
logger.info(f"Starting Basic Memory API (mode={container.mode.name})")
await initialize_app(container.config)
await initialize_app(container.config)
# Cache database connections in app state for performance
logger.info("Initializing database and caching connections...")
engine, session_maker = await container.init_database()
app.state.engine = engine
app.state.session_maker = session_maker
logger.info("Database connections cached in app state")
# Cache database connections in app state for performance
logger.info("Initializing database and caching connections...")
engine, session_maker = await container.init_database()
app.state.engine = engine
app.state.session_maker = session_maker
logger.info("Database connections cached in app state")
# Create and start sync coordinator (lifecycle centralized in coordinator)
sync_coordinator = container.create_sync_coordinator()
await sync_coordinator.start()
app.state.sync_coordinator = sync_coordinator
# Create and start sync coordinator (lifecycle centralized in coordinator)
sync_coordinator = container.create_sync_coordinator()
await sync_coordinator.start()
app.state.sync_coordinator = sync_coordinator
# Proceed with startup
yield
# Shutdown - coordinator handles clean task cancellation
logger.info("Shutting down Basic Memory API")
await sync_coordinator.stop()
await container.shutdown_database()
with telemetry.operation(
"api.lifecycle.shutdown",
entrypoint="api",
mode=container.mode.name.lower(),
):
logger.info("Shutting down Basic Memory API")
await sync_coordinator.stop()
await container.shutdown_database()
# Initialize FastAPI app
src/basic_memory/api/v2/routers/search_router.py
+33 -22
View File
@@ -6,6 +6,7 @@ V1 uses string-based project names which are less efficient and less stable.
from fastapi import APIRouter, HTTPException, Path
from basic_memory import telemetry
from basic_memory.api.v2.utils import to_search_results
from basic_memory.repository.semantic_errors import (
SemanticDependenciesMissingError,
@@ -47,29 +48,39 @@ async def search(
Returns:
SearchResponse with paginated search results
"""
offset = (page - 1) * page_size
# Fetch one extra item to detect whether more pages exist (N+1 trick)
fetch_limit = page_size + 1
try:
results = await search_service.search(query, limit=fetch_limit, offset=offset)
except SemanticSearchDisabledError as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc
except SemanticDependenciesMissingError as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc
except ValueError as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc
has_more = len(results) > page_size
if has_more:
results = results[:page_size]
search_results = await to_search_results(entity_service, results)
return SearchResponse(
results=search_results,
current_page=page,
with telemetry.operation(
"api.request.search",
entrypoint="api",
page=page,
page_size=page_size,
has_more=has_more,
)
retrieval_mode=query.retrieval_mode.value,
has_text_query=bool(query.text and query.text.strip()),
has_title_query=bool(query.title),
has_permalink_query=bool(query.permalink or query.permalink_match),
):
offset = (page - 1) * page_size
# Fetch one extra item to detect whether more pages exist (N+1 trick)
fetch_limit = page_size + 1
try:
results = await search_service.search(query, limit=fetch_limit, offset=offset)
except SemanticSearchDisabledError as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc
except SemanticDependenciesMissingError as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc
except ValueError as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc
has_more = len(results) > page_size
if has_more:
results = results[:page_size]
search_results = await to_search_results(entity_service, results)
return SearchResponse(
results=search_results,
current_page=page,
page_size=page_size,
has_more=has_more,
)
@router.post("/search/reindex")
src/basic_memory/cli/app.py
+9
View File
@@ -12,6 +12,7 @@ from basic_memory.cli.auto_update import maybe_run_periodic_auto_update # noqa:
from basic_memory.cli.container import CliContainer, set_container # noqa: E402
from basic_memory.cli.promo import maybe_show_cloud_promo, maybe_show_init_line # noqa: E402
from basic_memory.config import init_cli_logging # noqa: E402
from basic_memory import telemetry # noqa: E402
def version_callback(value: bool) -> None:
@@ -42,6 +43,14 @@ def app_callback(
# Initialize logging for CLI (file only, no stdout)
init_cli_logging()
command_name = ctx.invoked_subcommand or "root"
ctx.with_resource(
telemetry.operation(
f"cli.command.{command_name}",
entrypoint="cli",
command_name=command_name,
)
)
# --- Composition Root ---
# Create container and read config (single point of config access)
src/basic_memory/config.py
+41 -4
View File
@@ -6,14 +6,16 @@ import os
import shutil
from dataclasses import dataclass
from datetime import datetime
from enum import Enum
from pathlib import Path
from typing import Any, Dict, Literal, Optional, List, Tuple
from enum import Enum
from loguru import logger
from pydantic import AliasChoices, BaseModel, Field, model_validator
from pydantic_settings import BaseSettings, SettingsConfigDict
from basic_memory import __version__
from basic_memory.telemetry import configure_telemetry
from basic_memory.utils import setup_logging, generate_permalink
@@ -140,6 +142,24 @@ class BasicMemoryConfig(BaseSettings):
# overridden by ~/.basic-memory/config.json
log_level: str = "INFO"
# Optional Logfire telemetry (disabled by default)
logfire_enabled: bool = Field(
default=False,
description="Enable Logfire instrumentation for local development or managed deployments.",
)
logfire_send_to_logfire: bool = Field(
default=False,
description="When true, allow Logfire to export telemetry to the configured backend.",
)
logfire_service_name: str = Field(
default="basic-memory",
description="Base service name used when constructing entrypoint-specific Logfire service names.",
)
logfire_environment: str | None = Field(
default=None,
description="Optional override for Logfire environment. Defaults to env when unset.",
)
# Database configuration
database_backend: DatabaseBackend = Field(
default=DatabaseBackend.SQLITE,
@@ -955,33 +975,50 @@ def save_basic_memory_config(file_path: Path, config: BasicMemoryConfig) -> None
# Logging initialization functions for different entry points
def init_cli_logging() -> None: # pragma: no cover
def _configure_logfire_for_entrypoint(entrypoint: str) -> None:
"""Configure optional Logfire telemetry for a specific entrypoint."""
config = ConfigManager().config
service_name = f"{config.logfire_service_name}-{entrypoint}"
environment = config.logfire_environment or config.env
configure_telemetry(
service_name=service_name,
environment=environment,
service_version=__version__,
enable_logfire=config.logfire_enabled,
send_to_logfire=config.logfire_send_to_logfire,
)
def init_cli_logging() -> None:
"""Initialize logging for CLI commands - file only.
CLI commands should not log to stdout to avoid interfering with
command output and shell integration.
"""
log_level = os.getenv("BASIC_MEMORY_LOG_LEVEL", "INFO")
_configure_logfire_for_entrypoint("cli")
setup_logging(log_level=log_level, log_to_file=True)
def init_mcp_logging() -> None: # pragma: no cover
def init_mcp_logging() -> None:
"""Initialize logging for MCP server - file only.
MCP server must not log to stdout as it would corrupt the
JSON-RPC protocol communication.
"""
log_level = os.getenv("BASIC_MEMORY_LOG_LEVEL", "INFO")
_configure_logfire_for_entrypoint("mcp")
setup_logging(log_level=log_level, log_to_file=True)
def init_api_logging() -> None: # pragma: no cover
def init_api_logging() -> None:
"""Initialize logging for API server.
Cloud mode (BASIC_MEMORY_CLOUD_MODE=1): stdout with structured context
Local mode: file only
"""
log_level = os.getenv("BASIC_MEMORY_LOG_LEVEL", "INFO")
_configure_logfire_for_entrypoint("api")
cloud_mode = os.getenv("BASIC_MEMORY_CLOUD_MODE", "").lower() in ("1", "true")
if cloud_mode:
setup_logging(log_level=log_level, log_to_stdout=True, structured_context=True)
src/basic_memory/mcp/async_client.py
+18 -12
View File
@@ -5,6 +5,7 @@ from typing import AsyncIterator, Callable, Optional
from httpx import ASGITransport, AsyncClient, Timeout
from loguru import logger
from basic_memory import telemetry
from basic_memory.api.app import app as fastapi_app
from basic_memory.config import ConfigManager, ProjectMode
@@ -43,21 +44,26 @@ def _asgi_client(timeout: Timeout) -> AsyncClient:
async def _resolve_cloud_token(config) -> str:
"""Resolve cloud token with API key preferred, OAuth fallback."""
token = config.cloud_api_key
if token:
return token
with telemetry.span(
"routing.resolve_cloud_credentials",
has_api_key=bool(config.cloud_api_key),
):
token = config.cloud_api_key
if token:
return token
from basic_memory.cli.auth import CLIAuth
from basic_memory.cli.auth import CLIAuth
auth = CLIAuth(client_id=config.cloud_client_id, authkit_domain=config.cloud_domain)
token = await auth.get_valid_token()
if token:
return token
auth = CLIAuth(client_id=config.cloud_client_id, authkit_domain=config.cloud_domain)
token = await auth.get_valid_token()
if token:
return token
raise RuntimeError(
"Cloud routing requested but no credentials found. "
"Run 'bm cloud api-key save <key>' or 'bm cloud login' first."
)
logger.error("Cloud routing requested but no credentials were available")
raise RuntimeError(
"Cloud routing requested but no credentials found. "
"Run 'bm cloud api-key save <key>' or 'bm cloud login' first."
)
@asynccontextmanager
src/basic_memory/mcp/project_context.py
+260 -168
View File
@@ -19,6 +19,7 @@ from loguru import logger
from fastmcp import Context
from mcp.server.fastmcp.exceptions import ToolError
from basic_memory import telemetry
from basic_memory.config import BasicMemoryConfig, ConfigManager, ProjectMode
from basic_memory.project_resolver import ProjectResolver
from basic_memory.schemas.cloud import WorkspaceInfo, WorkspaceListResponse
@@ -63,6 +64,28 @@ async def _resolve_default_project_from_api() -> Optional[str]:
return None
def _canonicalize_project_name(
project_name: Optional[str],
config: BasicMemoryConfig,
) -> Optional[str]:
"""Return the configured project name when the identifier matches by permalink.
Project routing happens before API validation, so we normalize explicit inputs
here to keep local/cloud routing aligned with the database's case-insensitive
project resolver.
"""
if project_name is None:
return None
requested_permalink = generate_permalink(project_name)
for configured_name in config.projects:
if generate_permalink(configured_name) == requested_permalink:
return configured_name
return project_name
async def resolve_project_parameter(
project: Optional[str] = None,
allow_discovery: bool = False,
@@ -89,22 +112,28 @@ async def resolve_project_parameter(
Returns:
Resolved project name or None if no resolution possible
"""
# Load config for any values not explicitly provided.
# ConfigManager reads from the local config file, which doesn't exist in cloud mode.
# When it returns None, fall back to querying the projects API for the is_default flag.
if default_project is None:
with telemetry.span(
"routing.resolve_project",
requested_project=project,
allow_discovery=allow_discovery,
):
config = ConfigManager().config
default_project = config.default_project
if default_project is None:
default_project = await _resolve_default_project_from_api()
# Load config for any values not explicitly provided.
# ConfigManager reads from the local config file, which doesn't exist in cloud mode.
# When it returns None, fall back to querying the projects API for the is_default flag.
if default_project is None:
default_project = config.default_project
# Create resolver with configuration and resolve
resolver = ProjectResolver.from_env(
default_project=default_project,
)
result = resolver.resolve(project=project, allow_discovery=allow_discovery)
return result.project
if default_project is None:
default_project = await _resolve_default_project_from_api()
# Create resolver with configuration and resolve
resolver = ProjectResolver.from_env(
default_project=default_project,
)
result = resolver.resolve(project=project, allow_discovery=allow_discovery)
return _canonicalize_project_name(result.project, config)
async def get_project_names(client: AsyncClient, headers: HeaderTypes | None = None) -> List[str]:
@@ -177,51 +206,60 @@ async def resolve_workspace_parameter(
context: Optional[Context] = None,
) -> WorkspaceInfo:
"""Resolve workspace using explicit input, session cache, and cloud discovery."""
if context:
cached_raw = await context.get_state("active_workspace")
if isinstance(cached_raw, dict):
cached_workspace = WorkspaceInfo.model_validate(cached_raw)
if workspace is None or _workspace_matches_identifier(cached_workspace, workspace):
logger.debug(f"Using cached workspace from context: {cached_workspace.tenant_id}")
return cached_workspace
with telemetry.scope(
"routing.resolve_workspace",
workspace_requested=workspace is not None,
has_context=context is not None,
):
if context:
cached_raw = await context.get_state("active_workspace")
if isinstance(cached_raw, dict):
cached_workspace = WorkspaceInfo.model_validate(cached_raw)
if workspace is None or _workspace_matches_identifier(cached_workspace, workspace):
logger.debug(
f"Using cached workspace from context: {cached_workspace.tenant_id}"
)
return cached_workspace
workspaces = await get_available_workspaces(context=context)
if not workspaces:
raise ValueError(
"No accessible workspaces found for this account. "
"Ensure you have an active subscription and tenant access."
)
selected_workspace: WorkspaceInfo | None = None
if workspace:
matches = [item for item in workspaces if _workspace_matches_identifier(item, workspace)]
if not matches:
workspaces = await get_available_workspaces(context=context)
if not workspaces:
raise ValueError(
f"Workspace '{workspace}' was not found.\n"
"No accessible workspaces found for this account. "
"Ensure you have an active subscription and tenant access."
)
selected_workspace: WorkspaceInfo | None = None
if workspace:
matches = [
item for item in workspaces if _workspace_matches_identifier(item, workspace)
]
if not matches:
raise ValueError(
f"Workspace '{workspace}' was not found.\n"
f"Available workspaces:\n{_workspace_choices(workspaces)}"
)
if len(matches) > 1:
raise ValueError(
f"Workspace name '{workspace}' matches multiple workspaces. "
"Use tenant_id instead.\n"
f"Available workspaces:\n{_workspace_choices(workspaces)}"
)
selected_workspace = matches[0]
elif len(workspaces) == 1:
selected_workspace = workspaces[0]
else:
raise ValueError(
"Multiple workspaces are available. Ask the user which workspace to use, then retry "
"with the 'workspace' argument set to the tenant_id or unique name.\n"
f"Available workspaces:\n{_workspace_choices(workspaces)}"
)
if len(matches) > 1:
raise ValueError(
f"Workspace name '{workspace}' matches multiple workspaces. "
"Use tenant_id instead.\n"
f"Available workspaces:\n{_workspace_choices(workspaces)}"
)
selected_workspace = matches[0]
elif len(workspaces) == 1:
selected_workspace = workspaces[0]
else:
raise ValueError(
"Multiple workspaces are available. Ask the user which workspace to use, then retry "
"with the 'workspace' argument set to the tenant_id or unique name.\n"
f"Available workspaces:\n{_workspace_choices(workspaces)}"
)
if context:
await context.set_state("active_workspace", selected_workspace.model_dump())
logger.debug(f"Cached workspace in context: {selected_workspace.tenant_id}")
if context:
await context.set_state("active_workspace", selected_workspace.model_dump())
logger.debug(f"Cached workspace in context: {selected_workspace.tenant_id}")
return selected_workspace
return selected_workspace
async def get_active_project(
@@ -244,53 +282,58 @@ async def get_active_project(
ValueError: If no project can be resolved
HTTPError: If project doesn't exist or is inaccessible
"""
# Deferred import to avoid circular dependency with tools
from basic_memory.mcp.tools.utils import call_post
with telemetry.scope(
"routing.validate_project",
requested_project=project,
has_context=context is not None,
):
# Deferred import to avoid circular dependency with tools
from basic_memory.mcp.tools.utils import call_post
resolved_project = await resolve_project_parameter(project)
if not resolved_project:
project_names = await get_project_names(client, headers)
raise ValueError(
"No project specified. "
"Either set 'default_project' in config, or use 'project' argument.\n"
f"Available projects: {project_names}"
resolved_project = await resolve_project_parameter(project)
if not resolved_project:
project_names = await get_project_names(client, headers)
raise ValueError(
"No project specified. "
"Either set 'default_project' in config, or use 'project' argument.\n"
f"Available projects: {project_names}"
)
project = resolved_project
# Check if already cached in context
if context:
cached_raw = await context.get_state("active_project")
if isinstance(cached_raw, dict):
cached_project = ProjectItem.model_validate(cached_raw)
if cached_project.name == project:
logger.debug(f"Using cached project from context: {project}")
return cached_project
# Validate project exists by calling API
logger.debug(f"Validating project: {project}")
response = await call_post(
client,
"/v2/projects/resolve",
json={"identifier": project},
headers=headers,
)
resolved = ProjectResolveResponse.model_validate(response.json())
active_project = ProjectItem(
id=resolved.project_id,
external_id=resolved.external_id,
name=resolved.name,
path=resolved.path,
is_default=resolved.is_default,
)
project = resolved_project
# Cache in context if available
if context:
await context.set_state("active_project", active_project.model_dump())
logger.debug(f"Cached project in context: {project}")
# Check if already cached in context
if context:
cached_raw = await context.get_state("active_project")
if isinstance(cached_raw, dict):
cached_project = ProjectItem.model_validate(cached_raw)
if cached_project.name == project:
logger.debug(f"Using cached project from context: {project}")
return cached_project
# Validate project exists by calling API
logger.debug(f"Validating project: {project}")
response = await call_post(
client,
"/v2/projects/resolve",
json={"identifier": project},
headers=headers,
)
resolved = ProjectResolveResponse.model_validate(response.json())
active_project = ProjectItem(
id=resolved.project_id,
external_id=resolved.external_id,
name=resolved.name,
path=resolved.path,
is_default=resolved.is_default,
)
# Cache in context if available
if context:
await context.set_state("active_project", active_project.model_dump())
logger.debug(f"Cached project in context: {project}")
logger.debug(f"Validated project: {active_project.name}")
return active_project
logger.debug(f"Validated project: {active_project.name}")
return active_project
def _split_project_prefix(path: str) -> tuple[Optional[str], str]:
@@ -321,66 +364,77 @@ async def resolve_project_and_path(
Tuple of (active_project, normalized_path, is_memory_url)
"""
is_memory_url = identifier.strip().startswith("memory://")
if not is_memory_url:
active_project = await get_active_project(client, project, context, headers)
return active_project, identifier, False
config = ConfigManager().config
include_project = config.permalinks_include_project if is_memory_url else None
with telemetry.scope(
"routing.resolve_memory_url",
is_memory_url=is_memory_url,
requested_project=project,
include_project_prefix=include_project,
):
if not is_memory_url:
active_project = await get_active_project(client, project, context, headers)
return active_project, identifier, False
normalized_path = normalize_project_reference(memory_url_path(identifier))
project_prefix, remainder = _split_project_prefix(normalized_path)
include_project = ConfigManager().config.permalinks_include_project
normalized_path = normalize_project_reference(memory_url_path(identifier))
project_prefix, remainder = _split_project_prefix(normalized_path)
include_project = config.permalinks_include_project
# Trigger: memory URL begins with a potential project segment
# Why: allow project-scoped memory URLs without requiring a separate project parameter
# Outcome: attempt to resolve the prefix as a project and route to it
if project_prefix:
try:
from basic_memory.mcp.tools.utils import call_post
# Trigger: memory URL begins with a potential project segment
# Why: allow project-scoped memory URLs without requiring a separate project parameter
# Outcome: attempt to resolve the prefix as a project and route to it
if project_prefix:
try:
from basic_memory.mcp.tools.utils import call_post
response = await call_post(
client,
"/v2/projects/resolve",
json={"identifier": project_prefix},
headers=headers,
)
resolved = ProjectResolveResponse.model_validate(response.json())
except ToolError as exc:
if "project not found" not in str(exc).lower():
raise
else:
resolved_project = await resolve_project_parameter(project_prefix)
if resolved_project and generate_permalink(resolved_project) != generate_permalink(
project_prefix
):
raise ValueError(
f"Project is constrained to '{resolved_project}', cannot use '{project_prefix}'."
response = await call_post(
client,
"/v2/projects/resolve",
json={"identifier": project_prefix},
headers=headers,
)
resolved = ProjectResolveResponse.model_validate(response.json())
except ToolError as exc:
if "project not found" not in str(exc).lower():
raise
else:
resolved_project = await resolve_project_parameter(project_prefix)
if resolved_project and generate_permalink(resolved_project) != generate_permalink(
project_prefix
):
raise ValueError(
f"Project is constrained to '{resolved_project}', cannot use '{project_prefix}'."
)
active_project = ProjectItem(
id=resolved.project_id,
external_id=resolved.external_id,
name=resolved.name,
path=resolved.path,
is_default=resolved.is_default,
)
if context:
await context.set_state("active_project", active_project.model_dump())
active_project = ProjectItem(
id=resolved.project_id,
external_id=resolved.external_id,
name=resolved.name,
path=resolved.path,
is_default=resolved.is_default,
)
if context:
await context.set_state("active_project", active_project.model_dump())
resolved_path = f"{resolved.permalink}/{remainder}" if include_project else remainder
return active_project, resolved_path, True
resolved_path = (
f"{resolved.permalink}/{remainder}" if include_project else remainder
)
return active_project, resolved_path, True
# Trigger: no resolvable project prefix in the memory URL
# Why: preserve existing memory URL behavior within the active project
# Outcome: use the active project and normalize the path for lookup
active_project = await get_active_project(client, project, context, headers)
resolved_path = normalized_path
if include_project:
# Trigger: project-prefixed permalinks are enabled and the path lacks a prefix
# Why: ensure memory URL lookups align with canonical permalinks
# Outcome: prefix the path with the active project's permalink
project_prefix = active_project.permalink
if resolved_path != project_prefix and not resolved_path.startswith(f"{project_prefix}/"):
resolved_path = f"{project_prefix}/{resolved_path}"
return active_project, resolved_path, True
# Trigger: no resolvable project prefix in the memory URL
# Why: preserve existing memory URL behavior within the active project
# Outcome: use the active project and normalize the path for lookup
active_project = await get_active_project(client, project, context, headers)
resolved_path = normalized_path
if include_project:
# Trigger: project-prefixed permalinks are enabled and the path lacks a prefix
# Why: ensure memory URL lookups align with canonical permalinks
# Outcome: prefix the path with the active project's permalink
project_prefix = active_project.permalink
if resolved_path != project_prefix and not resolved_path.startswith(
f"{project_prefix}/"
):
resolved_path = f"{project_prefix}/{resolved_path}"
return active_project, resolved_path, True
def add_project_metadata(result: str, project_name: str) -> str:
@@ -494,9 +548,17 @@ async def get_project_client(
# control-plane API with no valid credentials and fail with 401
# Outcome: use the factory client directly, skip workspace resolution
if is_factory_mode():
async with get_client() as client:
active_project = await get_active_project(client, resolved_project, context)
yield client, active_project
route_mode = "factory"
with telemetry.scope(
"routing.client_session",
project_name=resolved_project,
route_mode=route_mode,
workspace_id=workspace,
):
logger.debug("Using injected client factory for project routing")
async with get_client() as client:
active_project = await get_active_project(client, resolved_project, context)
yield client, active_project
return
# Step 2: Check explicit routing BEFORE workspace resolution
@@ -504,9 +566,16 @@ async def get_project_client(
# Why: explicit flags must be deterministic — skip workspace entirely for --local
# Outcome: route strictly based on explicit flag, no workspace network calls
if _explicit_routing() and _force_local_mode():
async with get_client(project_name=resolved_project) as client:
active_project = await get_active_project(client, resolved_project, context)
yield client, active_project
route_mode = "explicit_local"
with telemetry.scope(
"routing.client_session",
project_name=resolved_project,
route_mode=route_mode,
):
logger.debug("Explicit local routing selected for project client")
async with get_client(project_name=resolved_project) as client:
active_project = await get_active_project(client, resolved_project, context)
yield client, active_project
return
# Step 3: Determine if cloud routing is needed
@@ -535,28 +604,51 @@ async def get_project_client(
if effective_workspace is None and config.default_workspace:
effective_workspace = config.default_workspace
route_mode = "cloud_proxy"
# Priorities 4-6: if still unresolved, fall back to resolve_workspace_parameter
# which checks context cache, auto-selects single workspace, or errors
if effective_workspace is not None:
# Config-resolved workspace — pass directly to get_client, skip network lookup
async with get_client(
with telemetry.scope(
"routing.client_session",
project_name=resolved_project,
workspace=effective_workspace,
) as client:
active_project = await get_active_project(client, resolved_project, context)
yield client, active_project
route_mode=route_mode,
workspace_id=effective_workspace,
):
logger.debug("Using configured workspace for cloud project routing")
async with get_client(
project_name=resolved_project,
workspace=effective_workspace,
) as client:
active_project = await get_active_project(client, resolved_project, context)
yield client, active_project
else:
# No config-based workspace — use resolve_workspace_parameter for discovery
active_ws = await resolve_workspace_parameter(workspace=None, context=context)
async with get_client(
with telemetry.scope(
"routing.client_session",
project_name=resolved_project,
workspace=active_ws.tenant_id,
) as client:
active_project = await get_active_project(client, resolved_project, context)
yield client, active_project
route_mode=route_mode,
workspace_id=active_ws.tenant_id,
):
logger.debug("Resolved workspace dynamically for cloud project routing")
async with get_client(
project_name=resolved_project,
workspace=active_ws.tenant_id,
) as client:
active_project = await get_active_project(client, resolved_project, context)
yield client, active_project
return
# Step 4: Local routing (default)
async with get_client(project_name=resolved_project) as client:
active_project = await get_active_project(client, resolved_project, context)
yield client, active_project
route_mode = "local_asgi"
with telemetry.scope(
"routing.client_session",
project_name=resolved_project,
route_mode=route_mode,
):
logger.debug("Using default local ASGI routing for project client")
async with get_client(project_name=resolved_project) as client:
active_project = await get_active_project(client, resolved_project, context)
yield client, active_project
src/basic_memory/mcp/server.py
+74 -60
View File
@@ -21,6 +21,7 @@ from basic_memory.db import (
)
from basic_memory.mcp.container import McpContainer, set_container
from basic_memory.services.initialization import initialize_app
from basic_memory import telemetry
async def _log_embedding_status(session_maker: async_sessionmaker[AsyncSession]) -> None:
@@ -82,83 +83,96 @@ async def lifespan(app: FastMCP):
set_container(container)
config = container.config
logger.info(f"Starting Basic Memory MCP server (mode={container.mode.name})")
logger.info(
f"Config: database_backend={config.database_backend.value}, "
f"semantic_search_enabled={config.semantic_search_enabled}, "
f"default_project={config.default_project}"
)
if config.semantic_search_enabled:
with telemetry.operation(
"mcp.lifecycle.startup",
entrypoint="mcp",
mode=container.mode.name.lower(),
default_project=config.default_project,
):
logger.info(f"Starting Basic Memory MCP server (mode={container.mode.name})")
logger.info(
f"Semantic search: provider={config.semantic_embedding_provider}, "
f"model={config.semantic_embedding_model}, "
f"dimensions={config.semantic_embedding_dimensions or 'auto'}, "
f"batch_size={config.semantic_embedding_batch_size}"
f"Config: database_backend={config.database_backend.value}, "
f"semantic_search_enabled={config.semantic_search_enabled}, "
f"default_project={config.default_project}"
)
if config.semantic_search_enabled:
logger.info(
f"Semantic search: provider={config.semantic_embedding_provider}, "
f"model={config.semantic_embedding_model}, "
f"dimensions={config.semantic_embedding_dimensions or 'auto'}, "
f"batch_size={config.semantic_embedding_batch_size}"
)
# Log configured projects with their routing mode
for name, entry in config.projects.items():
default = " (default)" if name == config.default_project else ""
logger.info(f"Project: {name} -> {entry.path} [mode={entry.mode.value}]{default}")
# Log configured projects with their routing mode
for name, entry in config.projects.items():
default = " (default)" if name == config.default_project else ""
logger.info(f"Project: {name} -> {entry.path} [mode={entry.mode.value}]{default}")
# Check cloud auth status (local file check, no network call)
auth = CLIAuth(client_id=config.cloud_client_id, authkit_domain=config.cloud_domain)
tokens = auth.load_tokens()
if tokens is not None:
if not auth.is_token_valid(tokens):
expires_at = tokens.get("expires_at", 0)
expired_ago = int(time.time() - expires_at)
logger.warning(f"Cloud token expired {expired_ago}s ago - may need 'bm cloud login'")
else:
logger.info("Cloud: authenticated (OAuth token valid)")
# Check cloud auth status (local file check, no network call)
auth = CLIAuth(client_id=config.cloud_client_id, authkit_domain=config.cloud_domain)
tokens = auth.load_tokens()
if tokens is not None:
if not auth.is_token_valid(tokens):
expires_at = tokens.get("expires_at", 0)
expired_ago = int(time.time() - expires_at)
logger.warning(
f"Cloud token expired {expired_ago}s ago - may need 'bm cloud login'"
)
else:
logger.info("Cloud: authenticated (OAuth token valid)")
if config.cloud_api_key:
logger.info("Cloud: API key configured")
if config.cloud_api_key:
logger.info("Cloud: API key configured")
# Track if we created the engine (vs test fixtures providing it)
# This prevents disposing an engine provided by test fixtures when
# multiple Client connections are made in the same test
engine_was_none = db._engine is None
# Track if we created the engine (vs test fixtures providing it)
# This prevents disposing an engine provided by test fixtures when
# multiple Client connections are made in the same test
engine_was_none = db._engine is None
# Initialize app (runs migrations, reconciles projects)
await initialize_app(container.config)
# Initialize app (runs migrations, reconciles projects)
await initialize_app(container.config)
# Log embedding status so it's easy to spot in the logs
backfill_task: asyncio.Task | None = None # type: ignore[type-arg]
if config.semantic_search_enabled and db._session_maker is not None:
await _log_embedding_status(db._session_maker)
# Launch backfill in background so MCP server is ready immediately
backfill_task = asyncio.create_task(
_background_embedding_backfill(config, db._session_maker),
name="embedding-backfill",
)
# Log embedding status so it's easy to spot in the logs
backfill_task: asyncio.Task | None = None # type: ignore[type-arg]
if config.semantic_search_enabled and db._session_maker is not None:
await _log_embedding_status(db._session_maker)
# Launch backfill in background so MCP server is ready immediately
backfill_task = asyncio.create_task(
_background_embedding_backfill(config, db._session_maker),
name="embedding-backfill",
)
# Create and start sync coordinator (lifecycle centralized in coordinator)
sync_coordinator = container.create_sync_coordinator()
await sync_coordinator.start()
# Create and start sync coordinator (lifecycle centralized in coordinator)
sync_coordinator = container.create_sync_coordinator()
await sync_coordinator.start()
try:
yield
finally:
# Shutdown - coordinator handles clean task cancellation
logger.debug("Shutting down Basic Memory MCP server")
with telemetry.operation(
"mcp.lifecycle.shutdown",
entrypoint="mcp",
mode=container.mode.name.lower(),
):
logger.debug("Shutting down Basic Memory MCP server")
# Cancel embedding backfill if still running
if backfill_task is not None and not backfill_task.done():
backfill_task.cancel()
try:
await backfill_task
except asyncio.CancelledError:
logger.info("Background embedding backfill cancelled during shutdown")
# Cancel embedding backfill if still running
if backfill_task is not None and not backfill_task.done():
backfill_task.cancel()
try:
await backfill_task
except asyncio.CancelledError:
logger.info("Background embedding backfill cancelled during shutdown")
await sync_coordinator.stop()
await sync_coordinator.stop()
# Only shutdown DB if we created it (not if test fixture provided it)
if engine_was_none:
await db.shutdown_db()
logger.debug("Database connections closed")
else: # pragma: no cover
logger.debug("Skipping DB shutdown - engine provided externally")
# Only shutdown DB if we created it (not if test fixture provided it)
if engine_was_none:
await db.shutdown_db()
logger.debug("Database connections closed")
else: # pragma: no cover
logger.debug("Skipping DB shutdown - engine provided externally")
mcp = FastMCP(
src/basic_memory/mcp/tools/build_context.py
+56 -20
View File
@@ -6,6 +6,7 @@ from loguru import logger
from fastmcp import Context
from basic_memory.config import ConfigManager
from basic_memory import telemetry
from basic_memory.mcp.project_context import (
detect_project_from_url_prefix,
get_project_client,
@@ -190,8 +191,6 @@ async def build_context(
if detected:
project = detected
logger.info(f"Building context from {url} in project {project}")
# Convert string depth to integer if needed
if isinstance(depth, str):
try:
@@ -203,25 +202,62 @@ async def build_context(
# URL is already validated and normalized by MemoryUrl type annotation
async with get_project_client(project, workspace, context) as (client, active_project):
# Resolve memory:// identifier with project-prefix awareness
_, resolved_path, _ = await resolve_project_and_path(client, url, project, context)
with telemetry.operation(
"mcp.tool.build_context",
entrypoint="mcp",
tool_name="build_context",
requested_project=project,
workspace_id=workspace,
depth=depth or 1,
timeframe=timeframe,
page=page,
page_size=page_size,
max_related=max_related,
output_format=output_format,
is_memory_url=str(url).startswith("memory://"),
):
async with get_project_client(project, workspace, context) as (client, active_project):
with telemetry.contextualize(
project_name=active_project.name,
workspace_id=workspace,
tool_name="build_context",
):
logger.info(
f"MCP tool call tool=build_context project={active_project.name} "
f"url={url} depth={depth} timeframe={timeframe} output_format={output_format}"
)
# Import here to avoid circular import
from basic_memory.mcp.clients import MemoryClient
# Resolve memory:// identifier with project-prefix awareness
_, resolved_path, _ = await resolve_project_and_path(
client,
url,
active_project.name,
context,
)
# Use typed MemoryClient for API calls
memory_client = MemoryClient(client, active_project.external_id)
graph = await memory_client.build_context(
resolved_path,
depth=depth or 1,
timeframe=timeframe,
page=page,
page_size=page_size,
max_related=max_related,
)
# Import here to avoid circular import
from basic_memory.mcp.clients import MemoryClient
if output_format == "text":
return _format_context_markdown(graph, active_project.name)
# Use typed MemoryClient for API calls
memory_client = MemoryClient(client, active_project.external_id)
graph = await memory_client.build_context(
resolved_path,
depth=depth or 1,
timeframe=timeframe,
page=page,
page_size=page_size,
max_related=max_related,
)
return graph.model_dump()
logger.info(
f"MCP tool response: tool=build_context project={active_project.name} "
f"uri={graph.metadata.uri or resolved_path} "
f"primary_count={graph.metadata.primary_count or 0} "
f"related_count={graph.metadata.related_count or 0} "
f"output_format={output_format}"
)
if output_format == "text":
return _format_context_markdown(graph, active_project.name)
return graph.model_dump()
src/basic_memory/mcp/tools/edit_note.py
+233 -207
View File
@@ -6,6 +6,7 @@ from loguru import logger
from fastmcp import Context
from basic_memory.config import ConfigManager
from basic_memory import telemetry
from basic_memory.mcp.project_context import (
detect_project_from_url_prefix,
get_project_client,
@@ -270,218 +271,243 @@ async def edit_note(
if detected:
project = detected
async with get_project_client(project, workspace, context) as (client, active_project):
logger.info("MCP tool call", tool="edit_note", identifier=identifier, operation=operation)
with telemetry.operation(
"mcp.tool.edit_note",
entrypoint="mcp",
tool_name="edit_note",
requested_project=project,
workspace_id=workspace,
edit_operation=operation,
output_format=output_format,
has_section=bool(section),
has_find_text=bool(find_text),
expected_replacements=effective_replacements,
):
async with get_project_client(project, workspace, context) as (client, active_project):
with telemetry.contextualize(
project_name=active_project.name,
workspace_id=workspace,
tool_name="edit_note",
):
logger.info(
f"MCP tool call tool=edit_note project={active_project.name} "
f"identifier={identifier} operation={operation} output_format={output_format}"
)
# Validate operation
valid_operations = [
"append",
"prepend",
"find_replace",
"replace_section",
"insert_before_section",
"insert_after_section",
]
if operation not in valid_operations:
raise ValueError(
f"Invalid operation '{operation}'. Must be one of: {', '.join(valid_operations)}"
)
# Validate required parameters for specific operations
if operation == "find_replace" and not find_text:
raise ValueError("find_text parameter is required for find_replace operation")
section_ops = ("replace_section", "insert_before_section", "insert_after_section")
if operation in section_ops and not section:
raise ValueError("section parameter is required for section-based operations")
# Use the PATCH endpoint to edit the entity
try:
# Import here to avoid circular import
from basic_memory.mcp.clients import KnowledgeClient
# Use typed KnowledgeClient for API calls
knowledge_client = KnowledgeClient(client, active_project.external_id)
file_created = False
entity_id = ""
result: EntityResponse | None = None
# Try to resolve the entity; for append/prepend, create it if not found
try:
entity_id = await knowledge_client.resolve_entity(identifier, strict=True)
except Exception as resolve_error:
# Trigger: entity does not exist yet
# Why: append/prepend can meaningfully create a new note from the content,
# while find_replace/replace_section require existing content to modify
# Outcome: note is created via the same path as write_note
error_msg = str(resolve_error).lower()
is_not_found = "entity not found" in error_msg or "not found" in error_msg
if is_not_found and operation in ("append", "prepend"):
title, directory = _parse_identifier_to_title_and_directory(identifier)
# Validate directory path (same security check as write_note)
project_path = active_project.home
if directory and not validate_project_path(directory, project_path):
logger.warning(
"Attempted path traversal attack blocked",
directory=directory,
project=active_project.name,
)
if output_format == "json":
return {
"title": title,
"permalink": None,
"file_path": None,
"checksum": None,
"operation": operation,
"fileCreated": False,
"error": "SECURITY_VALIDATION_ERROR",
}
return f"# Error\n\nDirectory path '{directory}' is not allowed - paths must stay within project boundaries"
entity = Entity(
title=title,
directory=directory,
content_type="text/markdown",
content=content,
# Validate operation
valid_operations = [
"append",
"prepend",
"find_replace",
"replace_section",
"insert_before_section",
"insert_after_section",
]
if operation not in valid_operations:
raise ValueError(
f"Invalid operation '{operation}'. Must be one of: {', '.join(valid_operations)}"
)
# Validate required parameters for specific operations
if operation == "find_replace" and not find_text:
raise ValueError("find_text parameter is required for find_replace operation")
section_ops = ("replace_section", "insert_before_section", "insert_after_section")
if operation in section_ops and not section:
raise ValueError("section parameter is required for section-based operations")
# Use the PATCH endpoint to edit the entity
try:
# Import here to avoid circular import
from basic_memory.mcp.clients import KnowledgeClient
# Use typed KnowledgeClient for API calls
knowledge_client = KnowledgeClient(client, active_project.external_id)
file_created = False
entity_id = ""
result: EntityResponse | None = None
# Try to resolve the entity; for append/prepend, create it if not found
try:
entity_id = await knowledge_client.resolve_entity(identifier, strict=True)
except Exception as resolve_error:
# Trigger: entity does not exist yet
# Why: append/prepend can meaningfully create a new note from the content,
# while find_replace/replace_section require existing content to modify
# Outcome: note is created via the same path as write_note
error_msg = str(resolve_error).lower()
is_not_found = "entity not found" in error_msg or "not found" in error_msg
if is_not_found and operation in ("append", "prepend"):
title, directory = _parse_identifier_to_title_and_directory(identifier)
# Validate directory path (same security check as write_note)
project_path = active_project.home
if directory and not validate_project_path(directory, project_path):
logger.warning(
"Attempted path traversal attack blocked",
directory=directory,
project=active_project.name,
)
if output_format == "json":
return {
"title": title,
"permalink": None,
"file_path": None,
"checksum": None,
"operation": operation,
"fileCreated": False,
"error": "SECURITY_VALIDATION_ERROR",
}
return f"# Error\n\nDirectory path '{directory}' is not allowed - paths must stay within project boundaries"
entity = Entity(
title=title,
directory=directory,
content_type="text/markdown",
content=content,
)
logger.info(
"Creating note via edit_note auto-create",
title=title,
directory=directory,
operation=operation,
)
result = await knowledge_client.create_entity(
entity.model_dump(), fast=False
)
file_created = True
else:
# find_replace/replace_section require existing content — re-raise
raise resolve_error
# --- Standard edit path (entity already existed) ---
if not file_created:
# Prepare the edit request data
edit_data = {
"operation": operation,
"content": content,
}
# Add optional parameters
if section:
edit_data["section"] = section
if find_text:
edit_data["find_text"] = find_text
if effective_replacements != 1: # Only send if different from default
edit_data["expected_replacements"] = str(effective_replacements)
# Call the PATCH endpoint
result = await knowledge_client.patch_entity(
entity_id, edit_data, fast=False
)
# --- Format response ---
# result is always set: either by create_entity (auto-create) or patch_entity (edit)
assert result is not None
if file_created:
summary = [
f"# Created note ({operation})",
f"project: {active_project.name}",
f"file_path: {result.file_path}",
f"permalink: {result.permalink}",
f"checksum: {result.checksum[:8] if result.checksum else 'unknown'}",
"fileCreated: true",
]
lines_added = len(content.split("\n"))
summary.append(f"operation: Created note with {lines_added} lines")
else:
summary = [
f"# Edited note ({operation})",
f"project: {active_project.name}",
f"file_path: {result.file_path}",
f"permalink: {result.permalink}",
f"checksum: {result.checksum[:8] if result.checksum else 'unknown'}",
]
# Add operation-specific details
if operation == "append":
lines_added = len(content.split("\n"))
summary.append(f"operation: Added {lines_added} lines to end of note")
elif operation == "prepend":
lines_added = len(content.split("\n"))
summary.append(
f"operation: Added {lines_added} lines to beginning of note"
)
elif operation == "find_replace":
# For find_replace, we can't easily count replacements from here
# since we don't have the original content, but the server handled it
summary.append("operation: Find and replace operation completed")
elif operation == "replace_section":
summary.append(f"operation: Replaced content under section '{section}'")
elif operation == "insert_before_section":
summary.append(
f"operation: Inserted content before section '{section}'"
)
elif operation == "insert_after_section":
summary.append(f"operation: Inserted content after section '{section}'")
# Count observations by category (reuse logic from write_note)
categories = {}
if result.observations:
for obs in result.observations:
categories[obs.category] = categories.get(obs.category, 0) + 1
summary.append("\n## Observations")
for category, count in sorted(categories.items()):
summary.append(f"- {category}: {count}")
# Count resolved/unresolved relations
unresolved = 0
resolved = 0
if result.relations:
unresolved = sum(1 for r in result.relations if not r.to_id)
resolved = len(result.relations) - unresolved
summary.append("\n## Relations")
summary.append(f"- Resolved: {resolved}")
if unresolved:
summary.append(f"- Unresolved: {unresolved}")
logger.info(
"Creating note via edit_note auto-create",
title=title,
directory=directory,
operation=operation,
f"MCP tool response: tool=edit_note project={active_project.name} "
f"operation={operation} permalink={result.permalink} "
f"observations_count={len(result.observations)} "
f"relations_count={len(result.relations)} "
f"file_created={str(file_created).lower()}"
)
result = await knowledge_client.create_entity(entity.model_dump(), fast=False)
file_created = True
else:
# find_replace/replace_section require existing content — re-raise
raise resolve_error
# --- Standard edit path (entity already existed) ---
if not file_created:
# Prepare the edit request data
edit_data = {
"operation": operation,
"content": content,
}
if output_format == "json":
return {
"title": result.title,
"permalink": result.permalink,
"file_path": result.file_path,
"checksum": result.checksum,
"operation": operation,
"fileCreated": file_created,
}
# Add optional parameters
if section:
edit_data["section"] = section
if find_text:
edit_data["find_text"] = find_text
if effective_replacements != 1: # Only send if different from default
edit_data["expected_replacements"] = str(effective_replacements)
summary_result = "\n".join(summary)
return add_project_metadata(summary_result, active_project.name)
# Call the PATCH endpoint
result = await knowledge_client.patch_entity(entity_id, edit_data, fast=False)
# --- Format response ---
# result is always set: either by create_entity (auto-create) or patch_entity (edit)
assert result is not None
if file_created:
summary = [
f"# Created note ({operation})",
f"project: {active_project.name}",
f"file_path: {result.file_path}",
f"permalink: {result.permalink}",
f"checksum: {result.checksum[:8] if result.checksum else 'unknown'}",
"fileCreated: true",
]
lines_added = len(content.split("\n"))
summary.append(f"operation: Created note with {lines_added} lines")
else:
summary = [
f"# Edited note ({operation})",
f"project: {active_project.name}",
f"file_path: {result.file_path}",
f"permalink: {result.permalink}",
f"checksum: {result.checksum[:8] if result.checksum else 'unknown'}",
]
# Add operation-specific details
if operation == "append":
lines_added = len(content.split("\n"))
summary.append(f"operation: Added {lines_added} lines to end of note")
elif operation == "prepend":
lines_added = len(content.split("\n"))
summary.append(f"operation: Added {lines_added} lines to beginning of note")
elif operation == "find_replace":
# For find_replace, we can't easily count replacements from here
# since we don't have the original content, but the server handled it
summary.append("operation: Find and replace operation completed")
elif operation == "replace_section":
summary.append(f"operation: Replaced content under section '{section}'")
elif operation == "insert_before_section":
summary.append(f"operation: Inserted content before section '{section}'")
elif operation == "insert_after_section":
summary.append(f"operation: Inserted content after section '{section}'")
# Count observations by category (reuse logic from write_note)
categories = {}
if result.observations:
for obs in result.observations:
categories[obs.category] = categories.get(obs.category, 0) + 1
summary.append("\n## Observations")
for category, count in sorted(categories.items()):
summary.append(f"- {category}: {count}")
# Count resolved/unresolved relations
unresolved = 0
resolved = 0
if result.relations:
unresolved = sum(1 for r in result.relations if not r.to_id)
resolved = len(result.relations) - unresolved
summary.append("\n## Relations")
summary.append(f"- Resolved: {resolved}")
if unresolved:
summary.append(f"- Unresolved: {unresolved}")
logger.info(
"MCP tool response",
tool="edit_note",
operation=operation,
project=active_project.name,
permalink=result.permalink,
observations_count=len(result.observations),
relations_count=len(result.relations),
file_created=file_created,
)
if output_format == "json":
return {
"title": result.title,
"permalink": result.permalink,
"file_path": result.file_path,
"checksum": result.checksum,
"operation": operation,
"fileCreated": file_created,
}
summary_result = "\n".join(summary)
return add_project_metadata(summary_result, active_project.name)
except Exception as e:
logger.error(f"Error editing note: {e}")
if output_format == "json":
return {
"title": None,
"permalink": None,
"file_path": None,
"checksum": None,
"operation": operation,
"fileCreated": False,
"error": str(e),
}
return _format_error_response(
str(e),
operation,
identifier,
find_text,
effective_replacements,
active_project.name,
)
except Exception as e:
logger.error(f"Error editing note: {e}")
if output_format == "json":
return {
"title": None,
"permalink": None,
"file_path": None,
"checksum": None,
"operation": operation,
"fileCreated": False,
"error": str(e),
}
return _format_error_response(
str(e),
operation,
identifier,
find_text,
effective_replacements,
active_project.name,
)
src/basic_memory/mcp/tools/move_note.py
+7 -6
View File
@@ -477,8 +477,11 @@ async def move_note(
}
return f"# Move Failed - Invalid Parameters\n\n{error_msg}"
async with get_project_client(project, workspace, context) as (client, active_project):
logger.debug(
f"Moving {'directory' if is_directory else 'note'}: {identifier} to {destination_path} in project: {active_project.name}"
destination_target = destination_folder or destination_path
logger.info(
f"MCP tool call tool=move_note project={active_project.name} "
f"identifier={identifier} destination={destination_target} "
f"is_directory={str(is_directory).lower()}"
)
# Validate destination path to prevent path traversal attacks
@@ -834,10 +837,8 @@ move_note("{identifier}", destination_folder="notes")
# Log the operation
logger.info(
"Move note completed",
identifier=identifier,
destination_path=destination_path,
project=active_project.name,
f"MCP tool response: tool=move_note project={active_project.name} "
f"source={identifier} destination={result.file_path} permalink={result.permalink}"
)
return "\n".join(result_lines)
src/basic_memory/mcp/tools/read_content.py
+13 -1
View File
@@ -216,7 +216,7 @@ async def read_content(
if detected:
project = detected
logger.info("Reading file", path=path, project=project)
logger.info(f"MCP tool call tool=read_content project={project} path={path}")
async with get_project_client(project, workspace, context) as (client, active_project):
# Resolve path with project-prefix awareness for memory:// URLs
@@ -260,6 +260,10 @@ async def read_content(
# Handle text or json
if content_type.startswith("text/") or content_type == "application/json":
logger.debug("Processing text resource")
logger.info(
f"MCP tool response: tool=read_content project={active_project.name} "
f"path={url} type=text content_type={content_type}"
)
return {
"type": "text",
"text": response.text,
@@ -272,6 +276,10 @@ async def read_content(
logger.debug("Processing image")
img = PILImage.open(io.BytesIO(response.content))
img_bytes = optimize_image(img, content_length)
logger.info(
f"MCP tool response: tool=read_content project={active_project.name} "
f"path={url} type=image content_type=image/jpeg"
)
return {
"type": "image",
@@ -291,6 +299,10 @@ async def read_content(
"type": "error",
"error": f"Document size {content_length} bytes exceeds maximum allowed size",
}
logger.info(
f"MCP tool response: tool=read_content project={active_project.name} "
f"path={url} type=document content_type={content_type}"
)
return {
"type": "document",
"source": {
src/basic_memory/mcp/tools/read_note.py
+191 -164
View File
@@ -8,6 +8,7 @@ import yaml
from loguru import logger
from fastmcp import Context
from basic_memory import telemetry
from basic_memory.config import ConfigManager
from basic_memory.mcp.project_context import (
detect_project_from_url_prefix,
@@ -139,186 +140,212 @@ async def read_note(
if detected:
project = detected
async with get_project_client(project, workspace, context) as (client, active_project):
# Resolve identifier with project-prefix awareness for memory:// URLs
_, entity_path, _ = await resolve_project_and_path(client, identifier, project, context)
with telemetry.operation(
"mcp.tool.read_note",
entrypoint="mcp",
tool_name="read_note",
requested_project=project,
workspace_id=workspace,
output_format=output_format,
page=page,
page_size=page_size,
include_frontmatter=include_frontmatter,
):
async with get_project_client(project, workspace, context) as (client, active_project):
with telemetry.contextualize(
project_name=active_project.name,
workspace_id=workspace,
tool_name="read_note",
):
# Resolve identifier with project-prefix awareness for memory:// URLs
_, entity_path, _ = await resolve_project_and_path(
client, identifier, project, context
)
# Validate identifier to prevent path traversal attacks
# For memory:// URLs, validate the extracted path (not the raw URL which
# has a scheme prefix that confuses path validation)
raw_path = memory_url_path(identifier) if identifier.startswith("memory://") else identifier
processed_path = entity_path
project_path = active_project.home
# Validate identifier to prevent path traversal attacks
# For memory:// URLs, validate the extracted path (not the raw URL which
# has a scheme prefix that confuses path validation)
raw_path = (
memory_url_path(identifier)
if identifier.startswith("memory://")
else identifier
)
processed_path = entity_path
project_path = active_project.home
if not validate_project_path(raw_path, project_path) or not validate_project_path(
processed_path, project_path
):
logger.warning(
"Attempted path traversal attack blocked",
identifier=identifier,
processed_path=processed_path,
project=active_project.name,
)
if output_format == "json":
return {
"title": None,
"permalink": None,
"file_path": None,
"content": None,
"frontmatter": None,
"error": "SECURITY_VALIDATION_ERROR",
}
return f"# Error\n\nIdentifier '{identifier}' is not allowed - paths must stay within project boundaries"
# Get the file via REST API - first try direct identifier resolution
logger.info(
f"Attempting to read note from Project: {active_project.name} identifier: {entity_path}"
)
# Import here to avoid circular import
from basic_memory.mcp.clients import KnowledgeClient, ResourceClient
# Use typed clients for API calls
knowledge_client = KnowledgeClient(client, active_project.external_id)
resource_client = ResourceClient(client, active_project.external_id)
async def _read_json_payload(entity_id: str) -> dict:
entity = await knowledge_client.get_entity(entity_id)
response = await resource_client.read(entity_id, page=page, page_size=page_size)
content_text = response.text
body_content, parsed_frontmatter = _parse_opening_frontmatter(content_text)
return {
"title": entity.title,
"permalink": entity.permalink,
"file_path": entity.file_path,
"content": content_text if include_frontmatter else body_content,
"frontmatter": parsed_frontmatter,
}
def _empty_json_payload() -> dict:
return {
"title": None,
"permalink": None,
"file_path": None,
"content": None,
"frontmatter": None,
}
def _search_results(payload: object) -> list[dict]:
if not isinstance(payload, dict):
return []
results = payload.get("results")
return results if isinstance(results, list) else []
def _result_title(item: dict) -> str:
return str(item.get("title") or "")
def _result_permalink(item: dict) -> Optional[str]:
value = item.get("permalink")
return str(value) if value else None
def _result_file_path(item: dict) -> Optional[str]:
value = item.get("file_path")
return str(value) if value else None
try:
# Try to resolve identifier to entity ID
entity_id = await knowledge_client.resolve_entity(entity_path, strict=True)
# Fetch content using entity ID
response = await resource_client.read(entity_id, page=page, page_size=page_size)
# If successful, return the content
if response.status_code == 200:
logger.info("Returning read_note result from resource: {path}", path=entity_path)
if output_format == "json":
return await _read_json_payload(entity_id)
return response.text
except Exception as e: # pragma: no cover
logger.info(f"Direct lookup failed for '{entity_path}': {e}")
# Continue to fallback methods
# Fallback 1: Try title search via API
logger.info(f"Search title for: {identifier}")
title_results = await search_notes(
query=identifier,
search_type="title",
project=active_project.name,
workspace=workspace,
output_format="json",
context=context,
)
title_candidates = _search_results(title_results)
if title_candidates:
# Trigger: direct resolution failed and title search returned candidates.
# Why: avoid returning unrelated notes when search yields only fuzzy matches.
# Outcome: fetch content only when a true exact title match exists.
result = next(
(
candidate
for candidate in title_candidates
if _is_exact_title_match(identifier, _result_title(candidate))
),
None,
)
if not result:
logger.info(f"No exact title match found for: {identifier}")
elif _result_permalink(result):
try:
# Resolve the permalink to entity ID
entity_id = await knowledge_client.resolve_entity(
_result_permalink(result) or "", strict=True
if not validate_project_path(raw_path, project_path) or not validate_project_path(
processed_path, project_path
):
logger.warning(
"Attempted path traversal attack blocked",
identifier=identifier,
processed_path=processed_path,
project=active_project.name,
)
if output_format == "json":
return {
"title": None,
"permalink": None,
"file_path": None,
"content": None,
"frontmatter": None,
"error": "SECURITY_VALIDATION_ERROR",
}
return f"# Error\n\nIdentifier '{identifier}' is not allowed - paths must stay within project boundaries"
# Fetch content using the entity ID
# Get the file via REST API - first try direct identifier resolution
logger.info(
f"Attempting to read note from Project: {active_project.name} identifier: {entity_path}"
)
# Import here to avoid circular import
from basic_memory.mcp.clients import KnowledgeClient, ResourceClient
# Use typed clients for API calls
knowledge_client = KnowledgeClient(client, active_project.external_id)
resource_client = ResourceClient(client, active_project.external_id)
async def _read_json_payload(entity_id: str) -> dict:
entity = await knowledge_client.get_entity(entity_id)
response = await resource_client.read(entity_id, page=page, page_size=page_size)
content_text = response.text
body_content, parsed_frontmatter = _parse_opening_frontmatter(content_text)
return {
"title": entity.title,
"permalink": entity.permalink,
"file_path": entity.file_path,
"content": content_text if include_frontmatter else body_content,
"frontmatter": parsed_frontmatter,
}
def _empty_json_payload() -> dict:
return {
"title": None,
"permalink": None,
"file_path": None,
"content": None,
"frontmatter": None,
}
def _search_results(payload: object) -> list[dict]:
if not isinstance(payload, dict):
return []
results = payload.get("results")
return results if isinstance(results, list) else []
def _result_title(item: dict) -> str:
return str(item.get("title") or "")
def _result_permalink(item: dict) -> Optional[str]:
value = item.get("permalink")
return str(value) if value else None
def _result_file_path(item: dict) -> Optional[str]:
value = item.get("file_path")
return str(value) if value else None
try:
# Try to resolve identifier to entity ID
entity_id = await knowledge_client.resolve_entity(entity_path, strict=True)
# Fetch content using entity ID
response = await resource_client.read(entity_id, page=page, page_size=page_size)
# If successful, return the content
if response.status_code == 200:
logger.info(
f"Found note by exact title search: {_result_permalink(result)}"
"Returning read_note result from resource: {path}", path=entity_path
)
if output_format == "json":
return await _read_json_payload(entity_id)
return response.text
except Exception as e: # pragma: no cover
logger.info(
f"Failed to fetch content for found title match {_result_permalink(result)}: {e}"
logger.info(f"Direct lookup failed for '{entity_path}': {e}")
# Continue to fallback methods
# Fallback 1: Try title search via API
logger.info(f"Search title for: {identifier}")
title_results = await search_notes(
query=identifier,
search_type="title",
project=active_project.name,
workspace=workspace,
output_format="json",
context=context,
)
title_candidates = _search_results(title_results)
if title_candidates:
# Trigger: direct resolution failed and title search returned candidates.
# Why: avoid returning unrelated notes when search yields only fuzzy matches.
# Outcome: fetch content only when a true exact title match exists.
result = next(
(
candidate
for candidate in title_candidates
if _is_exact_title_match(identifier, _result_title(candidate))
),
None,
)
else:
logger.info(
f"No results in title search for: {identifier} in project {active_project.name}"
)
if not result:
logger.info(f"No exact title match found for: {identifier}")
elif _result_permalink(result):
try:
# Resolve the permalink to entity ID
entity_id = await knowledge_client.resolve_entity(
_result_permalink(result) or "", strict=True
)
# Fallback 2: Text search as a last resort
logger.info(f"Title search failed, trying text search for: {identifier}")
text_results = await search_notes(
query=identifier,
search_type="text",
project=active_project.name,
workspace=workspace,
output_format="json",
context=context,
)
# Fetch content using the entity ID
response = await resource_client.read(
entity_id, page=page, page_size=page_size
)
# We didn't find a direct match, construct a helpful error message
text_candidates = _search_results(text_results)
if not text_candidates:
if output_format == "json":
return _empty_json_payload()
return format_not_found_message(active_project.name, identifier)
if output_format == "json":
payload = _empty_json_payload()
payload["related_results"] = [
{
"title": _result_title(result),
"permalink": _result_permalink(result),
"file_path": _result_file_path(result),
}
for result in text_candidates[:5]
]
return payload
return format_related_results(active_project.name, identifier, text_candidates[:5])
if response.status_code == 200:
logger.info(
f"Found note by exact title search: {_result_permalink(result)}"
)
if output_format == "json":
return await _read_json_payload(entity_id)
return response.text
except Exception as e: # pragma: no cover
logger.info(
f"Failed to fetch content for found title match {_result_permalink(result)}: {e}"
)
else:
logger.info(
f"No results in title search for: {identifier} in project {active_project.name}"
)
# Fallback 2: Text search as a last resort
logger.info(f"Title search failed, trying text search for: {identifier}")
text_results = await search_notes(
query=identifier,
search_type="text",
project=active_project.name,
workspace=workspace,
output_format="json",
context=context,
)
# We didn't find a direct match, construct a helpful error message
text_candidates = _search_results(text_results)
if not text_candidates:
if output_format == "json":
return _empty_json_payload()
return format_not_found_message(active_project.name, identifier)
if output_format == "json":
payload = _empty_json_payload()
payload["related_results"] = [
{
"title": _result_title(result),
"permalink": _result_permalink(result),
"file_path": _result_file_path(result),
}
for result in text_candidates[:5]
]
return payload
return format_related_results(active_project.name, identifier, text_candidates[:5])
def format_not_found_message(project: str | None, identifier: str) -> str:
src/basic_memory/mcp/tools/search.py
+148 -114
View File
@@ -8,6 +8,7 @@ from loguru import logger
from fastmcp import Context
from pydantic import BeforeValidator
from basic_memory import telemetry
from basic_memory.config import ConfigManager
from basic_memory.utils import coerce_dict, coerce_list
from basic_memory.mcp.container import get_container
@@ -523,124 +524,157 @@ async def search_notes(
if detected:
project = detected
async with get_project_client(project, workspace, context) as (client, active_project):
# Handle memory:// URLs by resolving to permalink search
is_memory_url = False
if query is not None:
_, resolved_query, is_memory_url = await resolve_project_and_path(
client, query, project, context
)
if is_memory_url:
query = resolved_query
effective_search_type = search_type or _default_search_type()
if is_memory_url:
effective_search_type = "permalink"
with telemetry.operation(
"mcp.tool.search_notes",
entrypoint="mcp",
tool_name="search_notes",
requested_project=project,
workspace_id=workspace,
search_type=search_type or "default",
output_format=output_format,
page=page,
page_size=page_size,
has_query=bool(query and query.strip()),
note_type_filter_count=len(note_types),
entity_type_filter_count=len(entity_types),
has_metadata_filters=bool(metadata_filters),
has_tags_filter=bool(tags),
has_status_filter=bool(status),
):
async with get_project_client(project, workspace, context) as (client, active_project):
with telemetry.contextualize(
project_name=active_project.name,
workspace_id=workspace,
tool_name="search_notes",
):
# Handle memory:// URLs by resolving to permalink search
is_memory_url = False
if query is not None:
_, resolved_query, is_memory_url = await resolve_project_and_path(
client, query, project, context
)
if is_memory_url:
query = resolved_query
effective_search_type = search_type or _default_search_type()
if is_memory_url:
effective_search_type = "permalink"
try:
# Create a SearchQuery object based on the parameters
search_query = SearchQuery()
try:
# Create a SearchQuery object based on the parameters
search_query = SearchQuery()
# Only map search_type to query fields when there is an actual query string.
# When query is None/empty, skip the search mode block — filters-only path.
effective_query = (query or "").strip()
if effective_query:
valid_search_types = {
"text",
"title",
"permalink",
"vector",
"semantic",
"hybrid",
}
if effective_search_type == "text":
search_query.text = effective_query
search_query.retrieval_mode = SearchRetrievalMode.FTS
elif effective_search_type in ("vector", "semantic"):
search_query.text = effective_query
search_query.retrieval_mode = SearchRetrievalMode.VECTOR
elif effective_search_type == "hybrid":
search_query.text = effective_query
search_query.retrieval_mode = SearchRetrievalMode.HYBRID
elif effective_search_type == "title":
search_query.title = effective_query
elif effective_search_type == "permalink" and "*" in effective_query:
search_query.permalink_match = effective_query
elif effective_search_type == "permalink":
search_query.permalink = effective_query
else:
raise ValueError(
f"Invalid search_type '{effective_search_type}'. "
f"Valid options: {', '.join(sorted(valid_search_types))}"
# Only map search_type to query fields when there is an actual query string.
# When query is None/empty, skip the search mode block — filters-only path.
effective_query = (query or "").strip()
if effective_query:
valid_search_types = {
"text",
"title",
"permalink",
"vector",
"semantic",
"hybrid",
}
if effective_search_type == "text":
search_query.text = effective_query
search_query.retrieval_mode = SearchRetrievalMode.FTS
elif effective_search_type in ("vector", "semantic"):
search_query.text = effective_query
search_query.retrieval_mode = SearchRetrievalMode.VECTOR
elif effective_search_type == "hybrid":
search_query.text = effective_query
search_query.retrieval_mode = SearchRetrievalMode.HYBRID
elif effective_search_type == "title":
search_query.title = effective_query
elif effective_search_type == "permalink" and "*" in effective_query:
search_query.permalink_match = effective_query
elif effective_search_type == "permalink":
search_query.permalink = effective_query
else:
raise ValueError(
f"Invalid search_type '{effective_search_type}'. "
f"Valid options: {', '.join(sorted(valid_search_types))}"
)
# Add optional filters if provided (empty lists are treated as no filter)
if entity_types:
search_query.entity_types = [SearchItemType(t) for t in entity_types]
if note_types:
search_query.note_types = note_types
if after_date:
search_query.after_date = after_date
if metadata_filters:
# Alias common column/model names to their frontmatter key equivalents.
# Users often pass "note_type" (the entity model column) when the
# frontmatter field is actually "type".
_METADATA_KEY_ALIASES = {"note_type": "type"}
metadata_filters = {
_METADATA_KEY_ALIASES.get(k, k): v for k, v in metadata_filters.items()
}
search_query.metadata_filters = metadata_filters
if tags:
search_query.tags = tags
if status:
search_query.status = status
if min_similarity is not None:
search_query.min_similarity = min_similarity
# Reject searches with no criteria at all
if search_query.no_criteria():
return (
"# No Search Criteria\n\n"
"Please provide at least one of: `query`, `metadata_filters`, "
"`tags`, `status`, `note_types`, `entity_types`, or `after_date`."
)
# Default to entity-level results to avoid returning individual
# observations/relations as separate search results (see issue #31).
# Applied after no_criteria() so that the implicit default doesn't
# mask a truly empty search request.
if not search_query.entity_types:
search_query.entity_types = [SearchItemType("entity")]
logger.debug(
f"Search request: project={active_project.name} "
f"search_type={effective_search_type} "
f"query={effective_query or '<filters-only>'} "
f"note_types={len(note_types)} entity_types={len(search_query.entity_types or [])} "
f"page={page} page_size={page_size}"
)
# Import here to avoid circular import (tools → clients → utils → tools)
from basic_memory.mcp.clients import SearchClient
# Use typed SearchClient for API calls
search_client = SearchClient(client, active_project.external_id)
result = await search_client.search(
search_query.model_dump(),
page=page,
page_size=page_size,
)
logger.debug(
f"Search response: project={active_project.name} "
f"results={len(result.results)} has_more={str(result.has_more).lower()} "
f"page={result.current_page} page_size={result.page_size}"
)
# Add optional filters if provided (empty lists are treated as no filter)
if entity_types:
search_query.entity_types = [SearchItemType(t) for t in entity_types]
if note_types:
search_query.note_types = note_types
if after_date:
search_query.after_date = after_date
if metadata_filters:
# Alias common column/model names to their frontmatter key equivalents.
# Users often pass "note_type" (the entity model column) when the
# frontmatter field is actually "type".
_METADATA_KEY_ALIASES = {"note_type": "type"}
metadata_filters = {
_METADATA_KEY_ALIASES.get(k, k): v for k, v in metadata_filters.items()
}
search_query.metadata_filters = metadata_filters
if tags:
search_query.tags = tags
if status:
search_query.status = status
if min_similarity is not None:
search_query.min_similarity = min_similarity
# Check if we got no results and provide helpful guidance
if not result.results:
logger.debug(
f"Search returned no results for query: {query} in project {active_project.name}"
)
# Don't treat this as an error, but the user might want guidance
# We return the empty result as normal - the user can decide if they need help
# Reject searches with no criteria at all
if search_query.no_criteria():
return (
"# No Search Criteria\n\n"
"Please provide at least one of: `query`, `metadata_filters`, "
"`tags`, `status`, `note_types`, `entity_types`, or `after_date`."
)
if output_format == "json":
return result.model_dump(mode="json", exclude_none=True)
# Default to entity-level results to avoid returning individual
# observations/relations as separate search results (see issue #31).
# Applied after no_criteria() so that the implicit default doesn't
# mask a truly empty search request.
if not search_query.entity_types:
search_query.entity_types = [SearchItemType("entity")]
return _format_search_markdown(result, active_project.name, query)
logger.debug(f"Searching for {search_query} in project {active_project.name}")
# Import here to avoid circular import (tools → clients → utils → tools)
from basic_memory.mcp.clients import SearchClient
# Use typed SearchClient for API calls
search_client = SearchClient(client, active_project.external_id)
result = await search_client.search(
search_query.model_dump(),
page=page,
page_size=page_size,
)
# Check if we got no results and provide helpful guidance
if not result.results:
logger.debug(
f"Search returned no results for query: {query} in project {active_project.name}"
)
# Don't treat this as an error, but the user might want guidance
# We return the empty result as normal - the user can decide if they need help
if output_format == "json":
return result.model_dump(mode="json", exclude_none=True)
return _format_search_markdown(result, active_project.name, query)
except Exception as e:
logger.error(
f"Search failed for query '{query or ''}': {e}, project: {active_project.name}"
)
# Return formatted error message as string for better user experience
return _format_search_error_response(
active_project.name, str(e), query or "", effective_search_type
)
except Exception as e:
logger.error(
f"Search failed for query '{query or ''}': {e}, project: {active_project.name}"
)
# Return formatted error message as string for better user experience
return _format_search_error_response(
active_project.name, str(e), query or "", effective_search_type
)
src/basic_memory/mcp/tools/write_note.py
+158 -138
View File
@@ -6,6 +6,7 @@ from typing import Annotated, List, Union, Optional, Literal
from loguru import logger
from pydantic import BeforeValidator
from basic_memory import telemetry
from basic_memory.config import ConfigManager
from basic_memory.mcp.project_context import get_project_client, add_project_metadata
from basic_memory.mcp.server import mcp
@@ -148,161 +149,180 @@ async def write_note(
overwrite if overwrite is not None else ConfigManager().config.write_note_overwrite_default
)
async with get_project_client(project, workspace, context) as (client, active_project):
logger.info(
f"MCP tool call tool=write_note project={active_project.name} directory={directory}, title={title}, tags={tags}"
)
# Normalize "/" to empty string for root directory (must happen before validation)
if directory == "/":
directory = ""
# Validate directory path to prevent path traversal attacks
project_path = active_project.home
if directory and not validate_project_path(directory, project_path):
logger.warning(
"Attempted path traversal attack blocked",
directory=directory,
project=active_project.name,
)
if output_format == "json":
return {
"title": title,
"permalink": None,
"file_path": None,
"checksum": None,
"action": "created",
"error": "SECURITY_VALIDATION_ERROR",
}
return f"# Error\n\nDirectory path '{directory}' is not allowed - paths must stay within project boundaries"
# Process tags using the helper function
tag_list = parse_tags(tags)
# Build entity_metadata from optional metadata, then explicit tags on top
# Order matters: explicit tags parameter takes precedence over metadata["tags"]
entity_metadata = {}
if metadata:
entity_metadata.update(metadata)
if tag_list:
entity_metadata["tags"] = tag_list
entity = Entity(
title=title,
directory=directory,
note_type=note_type,
content_type="text/markdown",
content=content,
entity_metadata=entity_metadata or None,
)
# Import here to avoid circular import
from basic_memory.mcp.clients import KnowledgeClient
# Use typed KnowledgeClient for API calls
knowledge_client = KnowledgeClient(client, active_project.external_id)
# Try to create the entity first (optimistic create)
logger.debug(f"Attempting to create entity permalink={entity.permalink}")
action = "Created" # Default to created
try:
result = await knowledge_client.create_entity(entity.model_dump(), fast=False)
action = "Created"
except Exception as e:
# If creation failed due to conflict (already exists), try to update
if (
"409" in str(e)
or "conflict" in str(e).lower()
or "already exists" in str(e).lower()
with telemetry.operation(
"mcp.tool.write_note",
entrypoint="mcp",
tool_name="write_note",
requested_project=project,
workspace_id=workspace,
note_type=note_type,
overwrite=effective_overwrite,
output_format=output_format,
):
async with get_project_client(project, workspace, context) as (client, active_project):
with telemetry.contextualize(
project_name=active_project.name,
workspace_id=workspace,
tool_name="write_note",
):
# Guard: block overwrite unless explicitly enabled
if not effective_overwrite:
logger.info(
f"MCP tool call tool=write_note project={active_project.name} directory={directory}, title={title}, tags={tags}"
)
# Normalize "/" to empty string for root directory (must happen before validation)
if directory == "/":
directory = ""
# Validate directory path to prevent path traversal attacks
project_path = active_project.home
if directory and not validate_project_path(directory, project_path):
logger.warning(
f"write_note blocked: note already exists (overwrite not enabled) "
f"permalink={entity.permalink}"
"Attempted path traversal attack blocked",
directory=directory,
project=active_project.name,
)
if output_format == "json":
return {
"title": title,
"permalink": entity.permalink,
"permalink": None,
"file_path": None,
"checksum": None,
"action": "conflict",
"error": "NOTE_ALREADY_EXISTS",
"action": "created",
"error": "SECURITY_VALIDATION_ERROR",
}
return _format_overwrite_error(title, entity.permalink, active_project.name)
return f"# Error\n\nDirectory path '{directory}' is not allowed - paths must stay within project boundaries"
logger.debug(f"Entity exists, updating instead permalink={entity.permalink}")
# Process tags using the helper function
tag_list = parse_tags(tags)
# Build entity_metadata from optional metadata, then explicit tags on top
# Order matters: explicit tags parameter takes precedence over metadata["tags"]
entity_metadata = {}
if metadata:
entity_metadata.update(metadata)
if tag_list:
entity_metadata["tags"] = tag_list
entity = Entity(
title=title,
directory=directory,
note_type=note_type,
content_type="text/markdown",
content=content,
entity_metadata=entity_metadata or None,
)
# Import here to avoid circular import
from basic_memory.mcp.clients import KnowledgeClient
# Use typed KnowledgeClient for API calls
knowledge_client = KnowledgeClient(client, active_project.external_id)
# Try to create the entity first (optimistic create)
logger.debug(f"Attempting to create entity permalink={entity.permalink}")
action = "Created" # Default to created
try:
if not entity.permalink:
raise ValueError(
"Entity permalink is required for updates"
) # pragma: no cover
entity_id = await knowledge_client.resolve_entity(entity.permalink)
result = await knowledge_client.update_entity(
entity_id, entity.model_dump(), fast=False
)
action = "Updated"
except Exception as update_error: # pragma: no cover
# Re-raise the original error if update also fails
raise e from update_error # pragma: no cover
else:
# Re-raise if it's not a conflict error
raise # pragma: no cover
summary = [
f"# {action} note",
f"project: {active_project.name}",
f"file_path: {result.file_path}",
f"permalink: {result.permalink}",
f"checksum: {result.checksum[:8] if result.checksum else 'unknown'}",
]
result = await knowledge_client.create_entity(entity.model_dump(), fast=False)
action = "Created"
except Exception as e:
# If creation failed due to conflict (already exists), try to update
if (
"409" in str(e)
or "conflict" in str(e).lower()
or "already exists" in str(e).lower()
):
# Guard: block overwrite unless explicitly enabled
if not effective_overwrite:
logger.warning(
f"write_note blocked: note already exists (overwrite not enabled) "
f"permalink={entity.permalink}"
)
if output_format == "json":
return {
"title": title,
"permalink": entity.permalink,
"file_path": None,
"checksum": None,
"action": "conflict",
"error": "NOTE_ALREADY_EXISTS",
}
return _format_overwrite_error(
title, entity.permalink, active_project.name
)
# Count observations by category
categories = {}
if result.observations:
for obs in result.observations:
categories[obs.category] = categories.get(obs.category, 0) + 1
logger.debug(
f"Entity exists, updating instead permalink={entity.permalink}"
)
try:
if not entity.permalink:
raise ValueError(
"Entity permalink is required for updates"
) # pragma: no cover
entity_id = await knowledge_client.resolve_entity(entity.permalink)
result = await knowledge_client.update_entity(
entity_id, entity.model_dump(), fast=False
)
action = "Updated"
except Exception as update_error: # pragma: no cover
# Re-raise the original error if update also fails
raise e from update_error # pragma: no cover
else:
# Re-raise if it's not a conflict error
raise # pragma: no cover
summary = [
f"# {action} note",
f"project: {active_project.name}",
f"file_path: {result.file_path}",
f"permalink: {result.permalink}",
f"checksum: {result.checksum[:8] if result.checksum else 'unknown'}",
]
summary.append("\n## Observations")
for category, count in sorted(categories.items()):
summary.append(f"- {category}: {count}")
# Count observations by category
categories = {}
if result.observations:
for obs in result.observations:
categories[obs.category] = categories.get(obs.category, 0) + 1
# Count resolved/unresolved relations
unresolved = 0
resolved = 0
if result.relations:
unresolved = sum(1 for r in result.relations if not r.to_id)
resolved = len(result.relations) - unresolved
summary.append("\n## Observations")
for category, count in sorted(categories.items()):
summary.append(f"- {category}: {count}")
summary.append("\n## Relations")
summary.append(f"- Resolved: {resolved}")
if unresolved:
summary.append(f"- Unresolved: {unresolved}")
summary.append(
"\nNote: Unresolved relations point to entities that don't exist yet."
)
summary.append(
"They will be automatically resolved when target entities are created or during sync operations."
# Count resolved/unresolved relations
unresolved = 0
resolved = 0
if result.relations:
unresolved = sum(1 for r in result.relations if not r.to_id)
resolved = len(result.relations) - unresolved
summary.append("\n## Relations")
summary.append(f"- Resolved: {resolved}")
if unresolved:
summary.append(f"- Unresolved: {unresolved}")
summary.append(
"\nNote: Unresolved relations point to entities that don't exist yet."
)
summary.append(
"They will be automatically resolved when target entities are created or during sync operations."
)
if tag_list:
summary.append(f"\n## Tags\n- {', '.join(tag_list)}")
# Log the response with structured data
logger.info(
f"MCP tool response: tool=write_note project={active_project.name} action={action} permalink={result.permalink} observations_count={len(result.observations)} relations_count={len(result.relations)} resolved_relations={resolved} unresolved_relations={unresolved}"
)
if output_format == "json":
return {
"title": result.title,
"permalink": result.permalink,
"file_path": result.file_path,
"checksum": result.checksum,
"action": action.lower(),
}
if tag_list:
summary.append(f"\n## Tags\n- {', '.join(tag_list)}")
# Log the response with structured data
logger.info(
f"MCP tool response: tool=write_note project={active_project.name} action={action} permalink={result.permalink} observations_count={len(result.observations)} relations_count={len(result.relations)} resolved_relations={resolved} unresolved_relations={unresolved}"
)
if output_format == "json":
return {
"title": result.title,
"permalink": result.permalink,
"file_path": result.file_path,
"checksum": result.checksum,
"action": action.lower(),
}
summary_result = "\n".join(summary)
return add_project_metadata(summary_result, active_project.name)
summary_result = "\n".join(summary)
return add_project_metadata(summary_result, active_project.name)
def _format_overwrite_error(title: str, permalink: str | None, project_name: str) -> str:
src/basic_memory/services/search_service.py
+44 -28
View File
@@ -5,12 +5,12 @@ import re
from datetime import datetime
from typing import List, Optional, Set, Dict, Any
from dateparser import parse
from fastapi import BackgroundTasks
from loguru import logger
from sqlalchemy import text
from basic_memory import telemetry
from basic_memory.models import Entity
from basic_memory.repository import EntityRepository
from basic_memory.repository.search_repository import (
@@ -152,8 +152,6 @@ class SearchService:
logger.debug("no criteria passed to query")
return []
logger.trace(f"Searching with query: {query}")
after_date = (
(
query.after_date
@@ -176,21 +174,32 @@ class SearchService:
retrieval_mode = query.retrieval_mode or SearchRetrievalMode.FTS
strict_search_text = query.text
# First pass: preserve existing strict search behavior.
results = await self.repository.search(
search_text=strict_search_text,
permalink=query.permalink,
permalink_match=query.permalink_match,
title=query.title,
note_types=query.note_types,
search_item_types=query.entity_types,
after_date=after_date,
metadata_filters=metadata_filters,
retrieval_mode=retrieval_mode,
min_similarity=query.min_similarity,
with telemetry.scope(
"search.execute",
retrieval_mode=retrieval_mode.value,
has_text_query=bool(strict_search_text),
has_title_query=bool(query.title),
has_permalink_query=bool(query.permalink or query.permalink_match),
has_metadata_filters=bool(metadata_filters),
limit=limit,
offset=offset,
)
):
logger.trace(f"Searching with query: {query}")
# First pass: preserve existing strict search behavior.
results = await self.repository.search(
search_text=strict_search_text,
permalink=query.permalink,
permalink_match=query.permalink_match,
title=query.title,
note_types=query.note_types,
search_item_types=query.entity_types,
after_date=after_date,
metadata_filters=metadata_filters,
retrieval_mode=retrieval_mode,
min_similarity=query.min_similarity,
limit=limit,
offset=offset,
)
# Trigger: strict FTS with plain multi-term text returned no results.
# Why: natural-language queries often include stopwords that over-constrain implicit AND.
@@ -209,20 +218,27 @@ class SearchService:
"Strict FTS returned 0 results; retrying relaxed FTS query "
f"strict='{strict_search_text}' relaxed='{relaxed_search_text}'"
)
return await self.repository.search(
search_text=relaxed_search_text,
permalink=query.permalink,
permalink_match=query.permalink_match,
title=query.title,
note_types=query.note_types,
search_item_types=query.entity_types,
after_date=after_date,
metadata_filters=metadata_filters,
retrieval_mode=retrieval_mode,
min_similarity=query.min_similarity,
with telemetry.scope(
"search.relaxed_fts_retry",
retrieval_mode=retrieval_mode.value,
token_count=len(self._tokenize_fts_text(strict_search_text)),
limit=limit,
offset=offset,
)
):
return await self.repository.search(
search_text=relaxed_search_text,
permalink=query.permalink,
permalink_match=query.permalink_match,
title=query.title,
note_types=query.note_types,
search_item_types=query.entity_types,
after_date=after_date,
metadata_filters=metadata_filters,
retrieval_mode=retrieval_mode,
min_similarity=query.min_similarity,
limit=limit,
offset=offset,
)
@staticmethod
def _tokenize_fts_text(search_text: str) -> list[str]:
src/basic_memory/sync/sync_service.py
+384 -306
View File
@@ -15,6 +15,7 @@ import aiofiles.os
from loguru import logger
from sqlalchemy.exc import IntegrityError
from basic_memory import telemetry
from basic_memory import db
from basic_memory.config import BasicMemoryConfig, ConfigManager
from basic_memory.file_utils import has_frontmatter
@@ -36,6 +37,7 @@ from basic_memory.services.search_service import SearchService
# Circuit breaker configuration
MAX_CONSECUTIVE_FAILURES = 3
SLOW_FILE_SYNC_WARNING_MS = 500
@dataclass
@@ -265,138 +267,161 @@ class SyncService:
start_time = time.time()
sync_start_timestamp = time.time() # Capture at start for watermark
logger.info(f"Sync operation started for directory: {directory} (force_full={force_full})")
# initial paths from db to sync
# path -> checksum
report = await self.scan(directory, force_full=force_full)
# order of sync matters to resolve relations effectively
logger.info(
f"Sync changes detected: new_files={len(report.new)}, modified_files={len(report.modified)}, "
+ f"deleted_files={len(report.deleted)}, moved_files={len(report.moves)}"
)
# sync moves first
for old_path, new_path in report.moves.items():
# in the case where a file has been deleted and replaced by another file
# it will show up in the move and modified lists, so handle it in modified
if new_path in report.modified:
report.modified.remove(new_path)
logger.debug(
f"File marked as moved and modified: old_path={old_path}, new_path={new_path}"
)
else:
await self.handle_move(old_path, new_path)
# deleted next
for path in report.deleted:
await self.handle_delete(path)
# then new and modified — collect entity IDs for batch vector embedding
synced_entity_ids: list[int] = []
for path in report.new:
entity, _ = await self.sync_file(path, new=True)
if entity is not None:
synced_entity_ids.append(entity.id)
# Track if file was skipped
elif await self._should_skip_file(path):
failure_info = self._file_failures[path]
report.skipped_files.append(
SkippedFile(
path=path,
reason=failure_info.last_error,
failure_count=failure_info.count,
first_failed=failure_info.first_failure,
)
)
for path in report.modified:
entity, _ = await self.sync_file(path, new=False)
if entity is not None:
synced_entity_ids.append(entity.id)
# Track if file was skipped
elif await self._should_skip_file(path):
failure_info = self._file_failures[path]
report.skipped_files.append(
SkippedFile(
path=path,
reason=failure_info.last_error,
failure_count=failure_info.count,
first_failed=failure_info.first_failure,
)
)
# Only resolve relations if there were actual changes
# If no files changed, no new unresolved relations could have been created
if report.total > 0:
await self.resolve_relations()
else:
logger.info("Skipping relation resolution - no file changes detected")
# Batch-generate vector embeddings for all synced entities
if synced_entity_ids and self.app_config.semantic_search_enabled:
try:
logger.info(
f"Generating semantic embeddings for {len(synced_entity_ids)} entities..."
)
batch_result = await self.search_service.sync_entity_vectors_batch(
synced_entity_ids
)
logger.info(
f"Semantic embeddings complete: "
f"synced={batch_result.entities_synced}, "
f"failed={batch_result.entities_failed}"
)
except SemanticDependenciesMissingError:
logger.warning(
"Semantic search dependencies missing — vector embeddings skipped. "
"Run 'bm reindex --embeddings' after resolving the dependency issue."
)
# Update scan watermark after successful sync
# Use the timestamp from sync start (not end) to ensure we catch files
# created during the sync on the next iteration
current_file_count = await self._quick_count_files(directory)
if self.entity_repository.project_id is not None:
project = await self.project_repository.find_by_id(self.entity_repository.project_id)
if project:
await self.project_repository.update(
project.id,
{
"last_scan_timestamp": sync_start_timestamp,
"last_file_count": current_file_count,
},
)
logger.debug(
f"Updated scan watermark: timestamp={sync_start_timestamp}, "
f"file_count={current_file_count}"
)
duration_ms = int((time.time() - start_time) * 1000)
# Log summary with skipped files if any
if report.skipped_files:
logger.warning(
f"Sync completed with {len(report.skipped_files)} skipped files: "
f"directory={directory}, total_changes={report.total}, "
f"skipped={len(report.skipped_files)}, duration_ms={duration_ms}"
)
for skipped in report.skipped_files:
logger.warning(
f"Skipped file: path={skipped.path}, "
f"failures={skipped.failure_count}, reason={skipped.reason}"
)
else:
with telemetry.operation(
"sync.project.run",
project_name=project_name,
force_full=force_full,
):
logger.info(
f"Sync operation completed: directory={directory}, "
f"total_changes={report.total}, duration_ms={duration_ms}"
f"Sync operation started for directory: {directory} (force_full={force_full})"
)
return report
# initial paths from db to sync
# path -> checksum
with telemetry.scope("sync.project.scan", force_full=force_full):
report = await self.scan(directory, force_full=force_full)
# order of sync matters to resolve relations effectively
logger.info(
f"Sync changes detected: new_files={len(report.new)}, modified_files={len(report.modified)}, "
+ f"deleted_files={len(report.deleted)}, moved_files={len(report.moves)}"
)
with telemetry.scope(
"sync.project.apply_changes",
new_count=len(report.new),
modified_count=len(report.modified),
deleted_count=len(report.deleted),
move_count=len(report.moves),
):
# sync moves first
for old_path, new_path in report.moves.items():
# in the case where a file has been deleted and replaced by another file
# it will show up in the move and modified lists, so handle it in modified
if new_path in report.modified:
report.modified.remove(new_path)
logger.debug(
f"File marked as moved and modified: old_path={old_path}, new_path={new_path}"
)
else:
await self.handle_move(old_path, new_path)
# deleted next
for path in report.deleted:
await self.handle_delete(path)
# then new and modified — collect entity IDs for batch vector embedding
synced_entity_ids: list[int] = []
for path in report.new:
entity, _ = await self.sync_file(path, new=True)
if entity is not None:
synced_entity_ids.append(entity.id)
# Track if file was skipped
elif await self._should_skip_file(path):
failure_info = self._file_failures[path]
report.skipped_files.append(
SkippedFile(
path=path,
reason=failure_info.last_error,
failure_count=failure_info.count,
first_failed=failure_info.first_failure,
)
)
for path in report.modified:
entity, _ = await self.sync_file(path, new=False)
if entity is not None:
synced_entity_ids.append(entity.id)
# Track if file was skipped
elif await self._should_skip_file(path):
failure_info = self._file_failures[path]
report.skipped_files.append(
SkippedFile(
path=path,
reason=failure_info.last_error,
failure_count=failure_info.count,
first_failed=failure_info.first_failure,
)
)
# Only resolve relations if there were actual changes
# If no files changed, no new unresolved relations could have been created
if report.total > 0:
with telemetry.scope("sync.project.resolve_relations", relation_scope="all_pending"):
await self.resolve_relations()
else:
logger.info("Skipping relation resolution - no file changes detected")
# Batch-generate vector embeddings for all synced entities
if synced_entity_ids and self.app_config.semantic_search_enabled:
try:
with telemetry.scope(
"sync.project.sync_embeddings",
entity_count=len(synced_entity_ids),
):
logger.info(
f"Generating semantic embeddings for {len(synced_entity_ids)} entities..."
)
batch_result = await self.search_service.sync_entity_vectors_batch(
synced_entity_ids
)
logger.info(
f"Semantic embeddings complete: "
f"synced={batch_result.entities_synced}, "
f"failed={batch_result.entities_failed}"
)
except SemanticDependenciesMissingError:
logger.warning(
"Semantic search dependencies missing — vector embeddings skipped. "
"Run 'bm reindex --embeddings' after resolving the dependency issue."
)
# Update scan watermark after successful sync
# Use the timestamp from sync start (not end) to ensure we catch files
# created during the sync on the next iteration
with telemetry.scope("sync.project.update_watermark"):
current_file_count = await self._quick_count_files(directory)
if self.entity_repository.project_id is not None:
project = await self.project_repository.find_by_id(
self.entity_repository.project_id
)
if project:
await self.project_repository.update(
project.id,
{
"last_scan_timestamp": sync_start_timestamp,
"last_file_count": current_file_count,
},
)
logger.debug(
f"Updated scan watermark: timestamp={sync_start_timestamp}, "
f"file_count={current_file_count}"
)
duration_ms = int((time.time() - start_time) * 1000)
# Log summary with skipped files if any
if report.skipped_files:
logger.warning(
f"Sync completed with {len(report.skipped_files)} skipped files: "
f"directory={directory}, total_changes={report.total}, "
f"skipped={len(report.skipped_files)}, duration_ms={duration_ms}"
)
for skipped in report.skipped_files:
logger.warning(
f"Skipped file: path={skipped.path}, "
f"failures={skipped.failure_count}, reason={skipped.reason}"
)
else:
logger.info(
f"Sync operation completed: directory={directory}, "
f"total_changes={report.total}, duration_ms={duration_ms}"
)
return report
async def scan(self, directory, force_full: bool = False):
"""Smart scan using watermark and file count for large project optimization.
@@ -433,171 +458,180 @@ class SyncService:
if project is None:
raise ValueError(f"Project not found: {self.entity_repository.project_id}")
# Step 1: Quick file count
logger.debug("Counting files in directory")
current_count = await self._quick_count_files(directory)
logger.debug(f"Found {current_count} files in directory")
with telemetry.scope("sync.project.select_scan_strategy", force_full=force_full):
# Step 1: Quick file count
logger.debug("Counting files in directory")
current_count = await self._quick_count_files(directory)
logger.debug(f"Found {current_count} files in directory")
# Step 2: Determine scan strategy based on watermark and file count
if force_full:
# User explicitly requested full scan → bypass watermark optimization
scan_type = "full_forced"
logger.info("Force full scan requested, bypassing watermark optimization")
file_paths_to_scan = await self._scan_directory_full(directory)
# Step 2: Determine scan strategy based on watermark and file count
if force_full:
# User explicitly requested full scan → bypass watermark optimization
scan_type = "full_forced"
logger.info("Force full scan requested, bypassing watermark optimization")
scan_coro = self._scan_directory_full(directory)
elif project.last_file_count is None:
# First sync ever → full scan
scan_type = "full_initial"
logger.info("First sync for this project, performing full scan")
file_paths_to_scan = await self._scan_directory_full(directory)
elif project.last_file_count is None:
# First sync ever → full scan
scan_type = "full_initial"
logger.info("First sync for this project, performing full scan")
scan_coro = self._scan_directory_full(directory)
elif current_count < project.last_file_count:
# Files deleted → need full scan to detect which ones
scan_type = "full_deletions"
logger.info(
f"File count decreased ({project.last_file_count} → {current_count}), "
f"running full scan to detect deletions"
)
file_paths_to_scan = await self._scan_directory_full(directory)
elif current_count < project.last_file_count:
# Files deleted → need full scan to detect which ones
scan_type = "full_deletions"
logger.info(
f"File count decreased ({project.last_file_count} → {current_count}), "
f"running full scan to detect deletions"
)
scan_coro = self._scan_directory_full(directory)
elif project.last_scan_timestamp is not None:
# Incremental scan: only files modified since last scan
scan_type = "incremental"
logger.debug(
f"Running incremental scan for files modified since {project.last_scan_timestamp}"
)
file_paths_to_scan = await self._scan_directory_modified_since(
directory, project.last_scan_timestamp
)
logger.debug(
f"Incremental scan found {len(file_paths_to_scan)} potentially changed files"
)
elif project.last_scan_timestamp is not None:
# Incremental scan: only files modified since last scan
scan_type = "incremental"
logger.debug(
f"Running incremental scan for files modified since {project.last_scan_timestamp}"
)
scan_coro = self._scan_directory_modified_since(
directory, project.last_scan_timestamp
)
else:
# Fallback to full scan (no watermark available)
scan_type = "full_fallback"
logger.warning("No scan watermark available, falling back to full scan")
file_paths_to_scan = await self._scan_directory_full(directory)
# Step 3: Process each file with mtime-based comparison
scanned_paths: Set[str] = set()
changed_checksums: Dict[str, str] = {}
logger.debug(f"Processing {len(file_paths_to_scan)} files with mtime-based comparison")
for rel_path in file_paths_to_scan:
scanned_paths.add(rel_path)
# Get file stats
abs_path = directory / rel_path
if not abs_path.exists():
# File was deleted between scan and now (race condition)
continue
stat_info = abs_path.stat()
# Indexed lookup - single file query (not full table scan)
db_entity = await self.entity_repository.get_by_file_path(rel_path)
if db_entity is None:
# New file - need checksum for move detection
checksum = await self.file_service.compute_checksum(rel_path)
report.new.add(rel_path)
changed_checksums[rel_path] = checksum
logger.trace(f"New file detected: {rel_path}")
continue
# File exists in DB - check if mtime/size changed
db_mtime = db_entity.mtime
db_size = db_entity.size
fs_mtime = stat_info.st_mtime
fs_size = stat_info.st_size
# Compare mtime and size (like rsync/rclone)
# Allow small epsilon for float comparison (0.01s = 10ms)
mtime_changed = db_mtime is None or abs(fs_mtime - db_mtime) > 0.01
size_changed = db_size is None or fs_size != db_size
if mtime_changed or size_changed:
# File modified - compute checksum
checksum = await self.file_service.compute_checksum(rel_path)
db_checksum = db_entity.checksum
# Only mark as modified if checksum actually differs
# (handles cases where mtime changed but content didn't, e.g., git operations)
if checksum != db_checksum:
report.modified.add(rel_path)
changed_checksums[rel_path] = checksum
logger.trace(
f"Modified file detected: {rel_path}, "
f"mtime_changed={mtime_changed}, size_changed={size_changed}"
)
else:
# File unchanged - no checksum needed
logger.trace(f"File unchanged (mtime/size match): {rel_path}")
# Fallback to full scan (no watermark available)
scan_type = "full_fallback"
logger.warning("No scan watermark available, falling back to full scan")
scan_coro = self._scan_directory_full(directory)
# Step 4: Detect moves (for both full and incremental scans)
# Check if any "new" files are actually moves by matching checksums
for new_path in list(report.new): # Use list() to allow modification during iteration
new_checksum = changed_checksums.get(new_path)
if not new_checksum:
continue
with telemetry.scope("sync.project.filesystem_scan", scan_type=scan_type):
file_paths_to_scan = await scan_coro
if scan_type == "incremental":
logger.debug(
f"Incremental scan found {len(file_paths_to_scan)} potentially changed files"
)
# Look for existing entity with same checksum but different path
# This could be a move or a copy
existing_entities = await self.entity_repository.find_by_checksum(new_checksum)
# Step 3: Process each file with mtime-based comparison
scanned_paths: Set[str] = set()
changed_checksums: Dict[str, str] = {}
for candidate in existing_entities:
if candidate.file_path == new_path:
# Same path, skip (shouldn't happen for "new" files but be safe)
logger.debug(f"Processing {len(file_paths_to_scan)} files with mtime-based comparison")
for rel_path in file_paths_to_scan:
scanned_paths.add(rel_path)
# Get file stats
abs_path = directory / rel_path
if not abs_path.exists():
# File was deleted between scan and now (race condition)
continue
# Check if the old path still exists on disk
old_path_abs = directory / candidate.file_path
if old_path_abs.exists():
# Original still exists → this is a copy, not a move
logger.trace(
f"File copy detected (not move): {candidate.file_path} copied to {new_path}"
)
stat_info = abs_path.stat()
# Indexed lookup - single file query (not full table scan)
db_entity = await self.entity_repository.get_by_file_path(rel_path)
if db_entity is None:
# New file - need checksum for move detection
checksum = await self.file_service.compute_checksum(rel_path)
report.new.add(rel_path)
changed_checksums[rel_path] = checksum
logger.trace(f"New file detected: {rel_path}")
continue
# Original doesn't exist → this is a move!
report.moves[candidate.file_path] = new_path
report.new.remove(new_path)
logger.trace(f"Move detected: {candidate.file_path} -> {new_path}")
break # Only match first candidate
# File exists in DB - check if mtime/size changed
db_mtime = db_entity.mtime
db_size = db_entity.size
fs_mtime = stat_info.st_mtime
fs_size = stat_info.st_size
# Step 5: Detect deletions (only for full scans)
# Incremental scans can't reliably detect deletions since they only see modified files
if scan_type in ("full_initial", "full_deletions", "full_fallback", "full_forced"):
# Use optimized query for just file paths (not full entities)
db_file_paths = await self.entity_repository.get_all_file_paths()
logger.debug(f"Found {len(db_file_paths)} db paths for deletion detection")
# Compare mtime and size (like rsync/rclone)
# Allow small epsilon for float comparison (0.01s = 10ms)
mtime_changed = db_mtime is None or abs(fs_mtime - db_mtime) > 0.01
size_changed = db_size is None or fs_size != db_size
for db_path in db_file_paths:
if db_path not in scanned_paths:
# File in DB but not on filesystem
# Check if it was already detected as a move
if db_path in report.moves:
# Already handled as a move, skip
if mtime_changed or size_changed:
# File modified - compute checksum
checksum = await self.file_service.compute_checksum(rel_path)
db_checksum = db_entity.checksum
# Only mark as modified if checksum actually differs
# (handles cases where mtime changed but content didn't, e.g., git operations)
if checksum != db_checksum:
report.modified.add(rel_path)
changed_checksums[rel_path] = checksum
logger.trace(
f"Modified file detected: {rel_path}, "
f"mtime_changed={mtime_changed}, size_changed={size_changed}"
)
else:
# File unchanged - no checksum needed
logger.trace(f"File unchanged (mtime/size match): {rel_path}")
# Step 4: Detect moves (for both full and incremental scans)
# Check if any "new" files are actually moves by matching checksums
with telemetry.scope("sync.project.detect_moves", new_count=len(report.new)):
for new_path in list(
report.new
): # Use list() to allow modification during iteration
new_checksum = changed_checksums.get(new_path)
if not new_checksum:
continue
# File was deleted
report.deleted.add(db_path)
logger.trace(f"Deleted file detected: {db_path}")
# Look for existing entity with same checksum but different path
# This could be a move or a copy
existing_entities = await self.entity_repository.find_by_checksum(new_checksum)
# Store checksums for files that need syncing
report.checksums = changed_checksums
for candidate in existing_entities:
if candidate.file_path == new_path:
# Same path, skip (shouldn't happen for "new" files but be safe)
continue
scan_duration_ms = int((time.time() - scan_start_time) * 1000)
# Check if the old path still exists on disk
old_path_abs = directory / candidate.file_path
if old_path_abs.exists():
# Original still exists → this is a copy, not a move
logger.trace(
f"File copy detected (not move): {candidate.file_path} copied to {new_path}"
)
continue
logger.info(
f"Completed {scan_type} scan for directory {directory} in {scan_duration_ms}ms, "
f"found {report.total} changes (new={len(report.new)}, "
f"modified={len(report.modified)}, deleted={len(report.deleted)}, "
f"moves={len(report.moves)})"
)
return report
# Original doesn't exist → this is a move!
report.moves[candidate.file_path] = new_path
report.new.remove(new_path)
logger.trace(f"Move detected: {candidate.file_path} -> {new_path}")
break # Only match first candidate
# Step 5: Detect deletions (only for full scans)
# Incremental scans can't reliably detect deletions since they only see modified files
if scan_type in ("full_initial", "full_deletions", "full_fallback", "full_forced"):
with telemetry.scope("sync.project.detect_deletions", scan_type=scan_type):
# Use optimized query for just file paths (not full entities)
db_file_paths = await self.entity_repository.get_all_file_paths()
logger.debug(f"Found {len(db_file_paths)} db paths for deletion detection")
for db_path in db_file_paths:
if db_path not in scanned_paths:
# File in DB but not on filesystem
# Check if it was already detected as a move
if db_path in report.moves:
# Already handled as a move, skip
continue
# File was deleted
report.deleted.add(db_path)
logger.trace(f"Deleted file detected: {db_path}")
# Store checksums for files that need syncing
report.checksums = changed_checksums
scan_duration_ms = int((time.time() - scan_start_time) * 1000)
logger.info(
f"Completed {scan_type} scan for directory {directory} in {scan_duration_ms}ms, "
f"found {report.total} changes (new={len(report.new)}, "
f"modified={len(report.modified)}, deleted={len(report.deleted)}, "
f"moves={len(report.moves)})"
)
return report
async def sync_file(
self, path: str, new: bool = True
@@ -616,12 +650,14 @@ class SyncService:
logger.warning(f"Skipping file due to repeated failures: {path}")
return None, None
try:
logger.debug(
f"Syncing file path={path} is_new={new} is_markdown={self.file_service.is_markdown(path)}"
)
start_time = time.time()
is_markdown = self.file_service.is_markdown(path)
file_kind = "markdown" if is_markdown else "regular"
if self.file_service.is_markdown(path):
try:
logger.debug(f"Syncing file path={path} is_new={new} is_markdown={is_markdown}")
if is_markdown:
entity, checksum = await self.sync_markdown_file(path, new)
else:
entity, checksum = await self.sync_regular_file(path, new)
@@ -646,33 +682,63 @@ class SyncService:
logger.debug(
f"File sync completed, path={path}, entity_id={entity.id}, checksum={checksum[:8]}"
)
duration_ms = int((time.time() - start_time) * 1000)
if duration_ms >= SLOW_FILE_SYNC_WARNING_MS:
logger.warning(
f"Slow file sync detected: path={path}, file_kind={file_kind}, duration_ms={duration_ms}"
)
return entity, checksum
except FileNotFoundError:
# File exists in database but not on filesystem
# This indicates a database/filesystem inconsistency - treat as deletion
logger.warning(
f"File not found during sync, treating as deletion: path={path}. "
"This may indicate a race condition or manual file deletion."
)
await self.handle_delete(path)
with telemetry.scope(
"sync.file.failure",
failure_type="file_not_found",
path=path,
file_kind=file_kind,
is_new=new,
is_fatal=False,
):
logger.warning(
f"File not found during sync, treating as deletion: path={path}. "
"This may indicate a race condition or manual file deletion."
)
await self.handle_delete(path)
return None, None
except Exception as e:
failure_type = type(e).__name__
# Check if this is a fatal error (or caused by one)
# Fatal errors like project deletion should terminate sync immediately
if isinstance(e, SyncFatalError) or isinstance(
e.__cause__, SyncFatalError
): # pragma: no cover
logger.error(f"Fatal sync error encountered, terminating sync: path={path}")
with telemetry.scope(
"sync.file.failure",
failure_type=failure_type,
path=path,
file_kind=file_kind,
is_new=new,
is_fatal=True,
):
logger.error(f"Fatal sync error encountered, terminating sync: path={path}")
raise
# Otherwise treat as recoverable file-level error
error_msg = str(e)
logger.error(f"Failed to sync file: path={path}, error={error_msg}")
with telemetry.scope(
"sync.file.failure",
failure_type=failure_type,
path=path,
file_kind=file_kind,
is_new=new,
is_fatal=False,
):
logger.error(f"Failed to sync file: path={path}, error={error_msg}")
# Record failure for circuit breaker
await self._record_failure(path, error_msg)
# Record failure for circuit breaker
await self._record_failure(path, error_msg)
return None, None
@@ -1066,24 +1132,36 @@ class SyncService:
# update search index only on successful resolution
await self.search_service.index_entity(resolved_entity)
except IntegrityError:
# IntegrityError means a relation with this (from_id, to_id, relation_type)
# already exists. The UPDATE was rolled back, so our unresolved relation
# (to_id=NULL) still exists in the database. We delete it because:
# 1. It's redundant - a resolved relation already captures this relationship
# 2. If we don't delete it, future syncs will try to resolve it again
# and get the same IntegrityError
logger.debug(
"Deleting duplicate unresolved relation "
f"relation_id={relation.id} "
f"from_id={relation.from_id} "
f"to_name={relation.to_name} "
f"resolved_to_id={resolved_entity.id}"
)
try:
await self.relation_repository.delete(relation.id)
except Exception as e:
# Log but don't fail - the relation may have been deleted already
logger.debug(f"Could not delete duplicate relation {relation.id}: {e}")
with telemetry.scope(
"sync.relation.resolve_conflict",
relation_id=relation.id,
relation_type=relation.relation_type,
):
# IntegrityError means a relation with this (from_id, to_id, relation_type)
# already exists. The UPDATE was rolled back, so our unresolved relation
# (to_id=NULL) still exists in the database. We delete it because:
# 1. It's redundant - a resolved version already captures this relationship
# 2. If we don't delete it, future syncs will try to resolve it again
# and get the same IntegrityError
logger.debug(
"Deleting duplicate unresolved relation "
f"relation_id={relation.id} "
f"from_id={relation.from_id} "
f"to_name={relation.to_name} "
f"resolved_to_id={resolved_entity.id}"
)
try:
await self.relation_repository.delete(relation.id)
except Exception as e:
with telemetry.scope(
"sync.relation.cleanup_failure",
relation_id=relation.id,
relation_type=relation.relation_type,
):
# Log but don't fail - the relation may have been deleted already
logger.debug(
f"Could not delete duplicate relation {relation.id}: {e}"
)
async def _quick_count_files(self, directory: Path) -> int:
"""Fast file count using find command.
src/basic_memory/telemetry.py
+209
View File
@@ -0,0 +1,209 @@
"""Optional Logfire telemetry helpers for Basic Memory.
Telemetry is disabled by default. When enabled, this module configures Logfire,
exposes a `loguru` handler for trace-aware logging, and provides lightweight
helpers for manual spans and logger context binding.
"""
from __future__ import annotations
from contextlib import contextmanager
from contextvars import ContextVar
from dataclasses import dataclass, field
from typing import Any, Iterator
from loguru import logger
REPOSITORY_URL = "https://github.com/basicmachines-co/basic-memory"
ROOT_PATH = "src/basic_memory"
def _load_logfire() -> Any | None:
"""Load the optional logfire dependency lazily."""
try:
import logfire
except ImportError:
return None
return logfire
@dataclass
class TelemetryState:
"""Process-local Logfire configuration state."""
enabled: bool = False
configured: bool = False
service_name: str | None = None
environment: str | None = None
send_to_logfire: bool = False
warnings: list[str] = field(default_factory=list)
_STATE = TelemetryState()
_LOGFIRE_HANDLER: dict[str, Any] | None = None
_ACTIVE_LOG_CONTEXT: ContextVar[dict[str, Any]] = ContextVar("basic_memory_log_context", default={})
def reset_telemetry_state() -> None:
"""Reset process-local telemetry state.
Primarily used by tests.
"""
global _LOGFIRE_HANDLER
_STATE.enabled = False
_STATE.configured = False
_STATE.service_name = None
_STATE.environment = None
_STATE.send_to_logfire = False
_STATE.warnings.clear()
_LOGFIRE_HANDLER = None
_ACTIVE_LOG_CONTEXT.set({})
def _filter_attributes(attrs: dict[str, Any]) -> dict[str, Any]:
"""Drop null attributes so span and log payloads stay compact."""
return {key: value for key, value in attrs.items() if value is not None}
def _current_log_context() -> dict[str, Any]:
"""Return the currently active telemetry context for this execution flow."""
return dict(_ACTIVE_LOG_CONTEXT.get())
def configure_telemetry(
service_name: str,
*,
environment: str,
service_version: str | None = None,
enable_logfire: bool = False,
send_to_logfire: bool = False,
log_level: str = "INFO",
) -> bool:
"""Configure optional Logfire instrumentation for the current process."""
global _LOGFIRE_HANDLER
reset_telemetry_state()
_STATE.service_name = service_name
_STATE.environment = environment
_STATE.send_to_logfire = send_to_logfire
_STATE.enabled = enable_logfire
if not enable_logfire:
return False
logfire = _load_logfire()
if logfire is None:
_STATE.enabled = False
_STATE.warnings.append(
"Logfire telemetry was enabled but the 'logfire' package is not installed. "
"Telemetry remains disabled."
)
return False
configure_kwargs = {
"service_name": service_name,
"environment": environment,
"code_source": logfire.CodeSource(
repository=REPOSITORY_URL,
revision=service_version or "",
root_path=ROOT_PATH,
),
"min_level": log_level.lower(),
"send_to_logfire": send_to_logfire,
}
try:
logfire.configure(**configure_kwargs)
except TypeError:
configure_kwargs.pop("send_to_logfire", None)
logfire.configure(**configure_kwargs)
except Exception as exc: # pragma: no cover
_STATE.enabled = False # pragma: no cover
_STATE.warnings.append(f"Failed to configure Logfire telemetry: {exc}") # pragma: no cover
return False # pragma: no cover
_LOGFIRE_HANDLER = logfire.loguru_handler()
_STATE.configured = True
return True
def telemetry_enabled() -> bool:
"""Return True when telemetry is both enabled and configured."""
return _STATE.enabled and _STATE.configured
def get_logfire_handler() -> dict[str, Any] | None:
"""Return the active Logfire `loguru` handler, if any."""
return _LOGFIRE_HANDLER
def pop_telemetry_warnings() -> list[str]:
"""Return and clear pending telemetry warnings."""
warnings = list(_STATE.warnings)
_STATE.warnings.clear()
return warnings
def bind_telemetry_context(**attrs: Any):
"""Bind stable telemetry attributes onto the shared Loguru logger."""
merged_attrs = _current_log_context()
merged_attrs.update(_filter_attributes(attrs))
return logger.bind(**merged_attrs)
@contextmanager
def contextualize(**attrs: Any) -> Iterator[None]:
"""Apply stable telemetry attributes to all Loguru calls in this scope."""
filtered_attrs = _filter_attributes(attrs)
merged_attrs = _current_log_context()
merged_attrs.update(filtered_attrs)
context_token = _ACTIVE_LOG_CONTEXT.set(merged_attrs)
try:
with logger.contextualize(**filtered_attrs):
yield
finally:
_ACTIVE_LOG_CONTEXT.reset(context_token)
@contextmanager
def scope(name: str, **attrs: Any) -> Iterator[None]:
"""Create a span and bind the same stable attributes into Loguru context."""
with contextualize(**attrs):
with span(name, **attrs):
yield
# Alias: `operation` signals a root-level boundary (entrypoint, tool invocation),
# while `scope` signals a nested phase. The distinction is convention only.
operation = scope
@contextmanager
def span(name: str, **attrs: Any) -> Iterator[None]:
"""Create a manual Logfire span when telemetry is enabled."""
if not telemetry_enabled():
yield
return
logfire = _load_logfire()
if logfire is None: # pragma: no cover
yield # pragma: no cover
return # pragma: no cover
with logfire.span(name, **_filter_attributes(attrs)):
yield
__all__ = [
"bind_telemetry_context",
"contextualize",
"configure_telemetry",
"get_logfire_handler",
"operation",
"pop_telemetry_warnings",
"reset_telemetry_state",
"scope",
"span",
"telemetry_enabled",
]
src/basic_memory/utils.py
+10
View File
@@ -13,6 +13,8 @@ from typing import Any, Protocol, Union, runtime_checkable, List, Optional
from loguru import logger
from unidecode import unidecode
from basic_memory import telemetry
def normalize_project_path(path: str) -> str:
"""Normalize project path by stripping mount point prefix.
@@ -300,6 +302,11 @@ def setup_logging(
if log_to_stdout:
logger.add(sys.stderr, level=log_level, backtrace=True, diagnose=True, colorize=True)
# Add Logfire sink when telemetry bootstrap enabled it for this process.
logfire_handler = telemetry.get_logfire_handler()
if logfire_handler is not None:
logger.add(**logfire_handler)
# Bind structured context for cloud observability
if structured_context:
logger.configure(
@@ -315,6 +322,9 @@ def setup_logging(
logging.getLogger("httpx").setLevel(logging.WARNING)
logging.getLogger("watchfiles.main").setLevel(logging.WARNING)
for warning_message in telemetry.pop_telemetry_warnings():
logger.warning(warning_message)
def _cleanup_windows_log_files(log_dir: Path, current_log_name: str) -> None:
"""Trim stale per-process Windows log files so the directory stays bounded."""