fix(mcp): recover edit_note when file exists on disk but is not indexed (#934)

Closes #581

Signed-off-by: phernandez <paul@basicmemory.com>

src/basic_memory/api/v2/routers/knowledge_router.py

@@ -10,10 +10,18 @@ Key improvements:
 - Simplified caching strategies
 """
 
+import os
+import pathlib
+
 from fastapi import APIRouter, HTTPException, Response, Path
 from loguru import logger
 
 import logfire
+from basic_memory.ignore_utils import (
+    IGNORED_PATH_REJECTION_DETAIL,
+    load_gitignore_patterns,
+    should_ignore_path,
+)
 from basic_memory.deps import (
     EntityServiceV2ExternalDep,
     SearchServiceV2ExternalDep,
@@ -24,6 +32,7 @@ from basic_memory.deps import (
     EntityRepositoryV2ExternalDep,
     RelationRepositoryV2ExternalDep,
     ProjectExternalIdPathDep,
+    SyncServiceV2ExternalDep,
     TaskSchedulerDep,
 )
 from basic_memory.schemas import DeleteEntitiesResponse
@@ -40,8 +49,10 @@ from basic_memory.schemas.v2 import (
     MoveDirectoryRequestV2,
     DeleteDirectoryRequestV2,
     OrphanEntitiesResponse,
+    SyncFileRequest,
 )
 from basic_memory.schemas.response import DirectoryMoveResult, DirectoryDeleteResult
+from basic_memory.utils import validate_project_path
 
 router = APIRouter(prefix="/knowledge", tags=["knowledge-v2"])
 
@@ -236,6 +247,187 @@ async def resolve_identifier(
         return result
 
 
+## Single-file sync endpoint
+
+
+def _canonical_file_path(home: pathlib.Path, segments: list[str]) -> str | None:
+    """Resolve the actual on-disk casing of a file path under the project home.
+
+    Trigger: case-insensitive filesystems (macOS/Windows) pass existence checks for
+        wrong-cased paths like 'notes/Disk-Note.md' when the file is 'notes/disk-note.md'.
+    Why: indexing the caller-supplied casing misses the existing DB row keyed by the
+        on-disk path and inserts a duplicate entity under the wrong-cased path.
+    Outcome: each segment is matched against real directory entries — exact name first
+        (so distinct case-variant files on case-sensitive filesystems stay distinct),
+        then a unique case-insensitive match. Returns None when any segment cannot be
+        matched to exactly one entry, including missing files. Traversal stops at the
+        project boundary: a directory whose resolved path escapes the project home is
+        never scanned.
+    """
+    resolved_home = home.resolve()
+    current = home
+    canonical_segments: list[str] = []
+    for segment in segments:
+        # Trigger: a previously matched segment may be a symlink whose target lies
+        #     outside the project root (e.g. wrong-cased 'LINK' matched the on-disk
+        #     'link' -> /tmp/outside on a case-sensitive filesystem).
+        # Why: os.scandir follows symlinked directories, so continuing would read
+        #     directory contents outside the project boundary even though the
+        #     post-canonicalization containment check rejects the request later.
+        # Outcome: bail before scanning the moment resolution escapes the home.
+        if not current.resolve().is_relative_to(resolved_home):
+            return None
+        try:
+            with os.scandir(current) as entries_iter:
+                entries = [entry.name for entry in entries_iter]
+        except OSError:
+            # A parent segment resolved to a non-directory (or vanished): no canonical
+            # path exists for the remaining segments.
+            return None
+        if segment in entries:
+            matched = segment
+        else:
+            matches = [entry for entry in entries if entry.lower() == segment.lower()]
+            if len(matches) != 1:
+                return None
+            matched = matches[0]
+        canonical_segments.append(matched)
+        current = current / matched
+    return "/".join(canonical_segments)
+
+
+@router.post("/sync-file", response_model=EntityResponseV2)
+async def sync_file(
+    data: SyncFileRequest,
+    project_id: ProjectExternalIdPathDep,
+    sync_service: SyncServiceV2ExternalDep,
+    project_config: ProjectConfigV2ExternalDep,
+    search_service: SearchServiceV2ExternalDep,
+    app_config: AppConfigDep,
+) -> EntityResponseV2:
+    """Index a single markdown file that exists on disk but is not indexed yet.
+
+    Recovery path for files written directly to disk before the watcher indexed
+    them (#581): callers such as edit_note can index the exact file and retry
+    identifier resolution without running a full project sync.
+
+    Args:
+        data: Request containing the markdown file path relative to project root
+
+    Returns:
+        The indexed entity
+
+    Raises:
+        HTTPException: 400 if the path escapes the project root, contains
+            non-normalized segments, matches the project ignore rules, or is
+            not markdown, 404 if the file does not exist on disk
+    """
+    with logfire.span(
+        "api.request.knowledge.sync_file",
+        entrypoint="api",
+        domain="knowledge",
+        action="sync_file",
+    ):
+        logger.info(f"API v2 request: sync_file file_path='{data.file_path}'")
+
+        if not validate_project_path(data.file_path, project_config.home):
+            raise HTTPException(
+                status_code=400,
+                detail=f"File path '{data.file_path}' is not allowed - "
+                "paths must stay within project boundaries",
+            )
+
+        # Trigger: segments like './' or '//' survive the traversal check above
+        # Why: a non-normalized path would index under a non-canonical DB key
+        # Outcome: reject fail-fast instead of guessing the canonical form
+        segments = data.file_path.replace("\\", "/").split("/")
+        if any(segment in ("", ".") for segment in segments):
+            raise HTTPException(
+                status_code=400,
+                detail=f"File path '{data.file_path}' is not normalized - "
+                "segments like './' or '//' are not allowed",
+            )
+
+        # Canonicalize to the actual on-disk casing so the DB lookup below hits the
+        # row keyed by the real path instead of inserting a wrong-cased duplicate.
+        file_path = _canonical_file_path(project_config.home, segments)
+        if file_path is None:
+            raise HTTPException(
+                status_code=404, detail=f"File not found on disk: '{data.file_path}'"
+            )
+        # Trigger: canonicalization rewrote a segment to its on-disk form, and that
+        #     segment may be a symlink. The pre-check above validated the ORIGINAL
+        #     request path — on a case-sensitive filesystem 'LINK/secret.md' does not
+        #     exist, so resolve() cannot follow the real 'link' symlink and the check
+        #     passes even when 'link' points outside the project root.
+        # Why: indexing through an escaping symlink would read and index content
+        #     outside the project boundary — and even an is_file() existence probe on
+        #     the joined path would follow the symlink and stat its target, so
+        #     containment must hold BEFORE any filesystem probe that follows symlinks.
+        #     Path.resolve() only walks symlink names (readlink); it never opens or
+        #     stats the final target, so it is safe to run pre-containment.
+        # Outcome: the canonical path is re-validated and the fully-resolved absolute
+        #     target must stay inside the resolved project home; escapes get a 400
+        #     before the file-existence probe below ever touches the target.
+        resolved_target = (project_config.home / file_path).resolve()
+        if not validate_project_path(file_path, project_config.home) or not (
+            resolved_target.is_relative_to(project_config.home.resolve())
+        ):
+            raise HTTPException(
+                status_code=400,
+                detail=f"File path '{data.file_path}' is not allowed - "
+                "paths must stay within project boundaries",
+            )
+        # Containment holds, so probing the resolved target cannot leave the project.
+        if not resolved_target.is_file():
+            raise HTTPException(
+                status_code=404, detail=f"File not found on disk: '{data.file_path}'"
+            )
+        # Trigger: the canonical path matches the .bmignore / project .gitignore rules
+        # Why: scan and watch flows filter ignored files before they ever reach the
+        #      indexer; indexing one here would bypass the ignored-file contract and
+        #      make hidden or gitignored content searchable
+        # Outcome: the same should_ignore_path() rules apply to single-file sync
+        ignore_patterns = load_gitignore_patterns(project_config.home)
+        if should_ignore_path(
+            project_config.home / file_path, project_config.home, ignore_patterns
+        ):
+            raise HTTPException(
+                status_code=400,
+                detail=f"File path '{data.file_path}' {IGNORED_PATH_REJECTION_DETAIL} "
+                "and cannot be indexed",
+            )
+        if not sync_service.file_service.is_markdown(file_path):
+            raise HTTPException(
+                status_code=400,
+                detail=f"Only markdown files can be indexed: '{data.file_path}'",
+            )
+
+        # Trigger: the file may already have a DB row (e.g. modified on disk after indexing)
+        # Why: the indexer needs to know whether to insert or update the entity
+        # Outcome: new is computed from the database instead of assumed by the caller
+        existing = await sync_service.entity_repository.get_by_file_path(file_path)
+        synced = await sync_service.sync_one_markdown_file(
+            file_path, new=existing is None, index_search=True
+        )
+
+        # Trigger: semantic search is enabled and the entity index was just refreshed
+        # Why: the project sync flow awaits sync_entity_vectors_batch() inline after
+        #      indexing changed files (SyncService.sync); without the single-entity
+        #      equivalent, a note recovered via sync-file stays missing or stale in
+        #      semantic search until a later edit or full project sync
+        # Outcome: vectors refresh synchronously before the response returns,
+        #          mirroring the sync flow instead of the out-of-band scheduler
+        if app_config.semantic_search_enabled:
+            await search_service.sync_entity_vectors_batch([synced.entity.id])
+
+        result = EntityResponseV2.model_validate(synced.entity)
+        logger.info(
+            f"API v2 response: sync_file file_path='{file_path}' external_id={result.external_id}"
+        )
+        return result
+
+
 ## Read endpoints
 
 

src/basic_memory/ignore_utils.py

@@ -7,6 +7,15 @@ from typing import Set
 from basic_memory.config import resolve_data_dir
 
 
+# Marker shared by the API ignored-path rejection detail and MCP-side error handling.
+# The sync-file endpoint embeds it in its 400 detail and edit_note's disk recovery
+# matches on it, so "exists but ignored" stays distinguishable from generic rejections
+# without duplicating message text across layers.
+IGNORED_PATH_REJECTION_DETAIL = (
+    "matches Basic Memory ignore rules (.bmignore or project .gitignore)"
+)
+
+
 # Common directories and patterns to ignore by default
 # These are used as fallback if .bmignore doesn't exist
 DEFAULT_IGNORE_PATTERNS = {

src/basic_memory/mcp/clients/knowledge.py

@@ -276,6 +276,35 @@ class KnowledgeClient:
             )
         return DirectoryDeleteResult.model_validate(response.json())
 
+    # --- Single-file sync ---
+
+    async def sync_file(self, file_path: str) -> EntityResponse:
+        """Index a markdown file that exists on disk but is not indexed yet.
+
+        Args:
+            file_path: Markdown file path relative to the project root
+
+        Returns:
+            EntityResponse for the indexed entity
+
+        Raises:
+            ToolError: If the file does not exist on disk or indexing fails
+        """
+        with logfire.span(
+            "mcp.client.knowledge.sync_file",
+            client_name="knowledge",
+            operation="sync_file",
+        ):
+            response = await call_post(
+                self.http_client,
+                f"{self._base_path}/sync-file",
+                json={"file_path": file_path},
+                client_name="knowledge",
+                operation="sync_file",
+                path_template="/v2/projects/{project_id}/knowledge/sync-file",
+            )
+        return EntityResponse.model_validate(response.json())
+
     # --- Orphan detection ---
 
     async def get_orphans(self) -> list[GraphNode]:

src/basic_memory/mcp/tools/edit_note.py

@@ -1,13 +1,19 @@
 """Edit note tool for Basic Memory MCP server."""
 
-from typing import Annotated, Optional, Literal
+from typing import TYPE_CHECKING, Annotated, Optional, Literal
 
 import logfire
+from httpx import HTTPStatusError
 from loguru import logger
 from fastmcp import Context
+from mcp.server.fastmcp.exceptions import ToolError
 from pydantic import AliasChoices, Field
 
+if TYPE_CHECKING:  # pragma: no cover
+    from basic_memory.mcp.clients import KnowledgeClient
+
 from basic_memory.config import ConfigManager
+from basic_memory.ignore_utils import IGNORED_PATH_REJECTION_DETAIL
 from basic_memory.mcp.project_context import (
     _workspace_identifier_discovery_available,
     detect_project_from_memory_url_prefix,
@@ -16,6 +22,7 @@ from basic_memory.mcp.project_context import (
     resolve_project_and_path,
 )
 from basic_memory.mcp.server import mcp
+from basic_memory.mcp.tools.utils import _extract_response_data, _response_detail_text
 from basic_memory.schemas.base import Entity
 from basic_memory.schemas.response import EntityResponse
 from basic_memory.services.link_resolver import (
@@ -52,6 +59,79 @@ def _parse_identifier_to_title_and_directory(identifier: str) -> tuple[str, str]
     return title, directory
 
 
+# Suffixes mimetypes maps to text/markdown (extension matching is case-insensitive),
+# mirroring FileService.is_markdown which gates the sync-file endpoint server-side.
+_MARKDOWN_SUFFIXES = (".md", ".markdown")
+
+
+async def _resolve_after_disk_recovery(
+    knowledge_client: "KnowledgeClient",
+    identifier: str,
+) -> Optional[str]:
+    """Recover from a resolution miss when the note exists on disk but is not indexed.
+
+    Trigger: identifier resolution failed with "not found", but the identifier may map
+        to a markdown file written directly to disk before the watcher indexed it (#581).
+    Why: editing an on-disk note should not require a manual full sync or watcher restart.
+    Outcome: the single file is indexed server-side and resolution is retried exactly
+        once. Returns None when the identifier does not map to an indexable file on
+        disk, so the caller keeps its existing not-found handling.
+    """
+    # Try the identifier as-is first so existing .markdown/.MD files are found; only
+    # fall back to appending markdown suffixes (".md" first, then ".markdown") when
+    # the identifier does not already carry one, so 'notes/foo.markdown' never becomes
+    # 'notes/foo.markdown.md' and a stem identifier still reaches 'notes/foo.markdown'.
+    candidates = [identifier]
+    if not identifier.lower().endswith(_MARKDOWN_SUFFIXES):
+        candidates.extend(f"{identifier}{suffix}" for suffix in _MARKDOWN_SUFFIXES)
+
+    for candidate in candidates:
+        try:
+            synced = await knowledge_client.sync_file(candidate)
+        except ToolError as sync_error:
+            # Trigger: the sync-file request failed
+            # Why: 400/404 are the expected "nothing to recover" rejections (missing
+            #      file, traversal, non-markdown) — except the ignored-path 400, which
+            #      means the file exists on disk but the ignore rules forbid indexing
+            #      it, so falling through to auto-create would silently shadow the
+            #      file. Anything else — auth, server, transport-level failures — is a
+            #      real error that must not be masked as a not-found miss.
+            # Outcome: ignored-path rejections raise a clear ToolError; other expected
+            #      rejections try the next candidate or fall through to the caller's
+            #      existing not-found behavior; unexpected failures propagate.
+            cause = sync_error.__cause__
+            candidate_rejected = isinstance(
+                cause, HTTPStatusError
+            ) and cause.response.status_code in (400, 404)
+            if not candidate_rejected:
+                raise
+            detail = _response_detail_text(_extract_response_data(cause.response)) or ""
+            if IGNORED_PATH_REJECTION_DETAIL in detail:
+                raise ToolError(
+                    f"Note file '{candidate}' exists on disk but {IGNORED_PATH_REJECTION_DETAIL} "
+                    "and will not be edited"
+                ) from sync_error
+            logger.debug(f"edit_note disk recovery skipped for '{candidate}': {sync_error}")
+            continue
+
+        # Trigger: sync-file succeeded and returned the indexed entity.
+        # Why: the server may have canonicalized the path casing (notes/Disk-Note ->
+        #      notes/disk-note.md), so strictly re-resolving the raw identifier can
+        #      still miss the entity we just indexed.
+        # Outcome: use the entity identity from the sync-file response directly; only
+        #      fall back to a strict re-resolve when an older server omits external_id,
+        #      and let that re-resolve fail loudly instead of guessing.
+        if synced.external_id:
+            logger.info(
+                f"edit_note indexed unindexed file '{candidate}' as entity {synced.external_id}"
+            )
+            return synced.external_id
+        logger.info(f"edit_note indexed unindexed file '{candidate}'; retrying resolution")
+        return await knowledge_client.resolve_entity(identifier, strict=True)
+
+    return None
+
+
 def _compose_workspace_project_route(
     *,
     workspace: Optional[str],
@@ -126,7 +206,8 @@ The note with identifier '{identifier}' could not be found. The `find_replace` a
 ## Suggestions to try:
 1. **Use append/prepend instead**: These operations will create the note automatically if it doesn't exist
 2. **Search for the note first**: Use `search_notes("{project or "project-name"}", "{identifier.split("/")[-1]}")` to find similar notes with exact identifiers
-3. **Try different exact identifier formats**:
+3. **File exists on disk but is not indexed yet?**: edit_note indexes the file automatically when the identifier matches its path (e.g. 'folder/note' for 'folder/note.md'). If your identifier is a title or differs from the file path, run a sync (`basic-memory sync`) or wait for the file watcher, then retry
+4. **Try different exact identifier formats**:
    - If you used a permalink like "folder/note-title", try the exact title: "{identifier.split("/")[-1].replace("-", " ").title()}"
    - If you used a title, try the exact permalink format: "{identifier.lower().replace(" ", "-")}"
    - Use `read_note("{project or "project-name"}", "{identifier}")` first to verify the note exists and get the exact identifier
@@ -343,7 +424,9 @@ async def edit_note(
 
     Note:
         Edit operations require exact identifier matches. If unsure, use read_note() or
-        search_notes() first to find the correct identifier. The tool provides detailed
+        search_notes() first to find the correct identifier. When the identifier looks
+        like a file path and the file exists on disk but is not indexed yet, edit_note
+        indexes that file automatically and retries the edit. The tool provides detailed
         error messages with suggestions if operations fail.
     """
     # Resolve effective default: allow MCP clients to send null for optional int field
@@ -465,14 +548,27 @@ async def edit_note(
                         strict=True,
                     )
                 except Exception as resolve_error:
-                    # Trigger: entity does not exist yet
-                    # Why: append/prepend can meaningfully create a new note from the content,
-                    #      while find_replace/replace_section require existing content to modify
-                    # Outcome: note is created via the same path as write_note
                     error_msg = str(resolve_error).lower()
                     is_not_found = "entity not found" in error_msg or "not found" in error_msg
 
-                    if is_not_found and operation in ("append", "prepend"):
+                    # Trigger: resolution missed but the file may already exist on disk
+                    # Why: files written directly to disk are invisible to identifier
+                    #      resolution until indexed; editing them should just work (#581)
+                    # Outcome: the single file is indexed and resolution retried once
+                    recovered_entity_id: str | None = None
+                    if is_not_found:
+                        recovered_entity_id = await _resolve_after_disk_recovery(
+                            knowledge_client, entity_identifier
+                        )
+
+                    if recovered_entity_id is not None:
+                        entity_id = recovered_entity_id
+                    elif is_not_found and operation in ("append", "prepend"):
+                        # Trigger: entity does not exist yet (on disk or in the index)
+                        # Why: append/prepend can meaningfully create a new note from the
+                        #      content, while find_replace/replace_section require existing
+                        #      content to modify
+                        # Outcome: note is created via the same path as write_note
                         title, directory = _parse_identifier_to_title_and_directory(identifier)
 
                         # Validate directory path (same security check as write_note)

src/basic_memory/schemas/v2/__init__.py

@@ -9,6 +9,7 @@ from basic_memory.schemas.v2.entity import (
     DeleteDirectoryRequestV2,
     ProjectResolveRequest,
     ProjectResolveResponse,
+    SyncFileRequest,
 )
 from basic_memory.schemas.v2.graph import (
     GraphEdge,
@@ -31,6 +32,7 @@ __all__ = [
     "DeleteDirectoryRequestV2",
     "ProjectResolveRequest",
     "ProjectResolveResponse",
+    "SyncFileRequest",
     "GraphEdge",
     "GraphNode",
     "GraphResponse",

src/basic_memory/schemas/v2/entity.py

@@ -54,6 +54,21 @@ class EntityResolveResponse(BaseModel):
     )
 
 
+class SyncFileRequest(BaseModel):
+    """Request to index a single markdown file that exists on disk.
+
+    Used as a recovery path when an identifier fails resolution but maps to a
+    file written directly to disk that the watcher has not indexed yet (#581).
+    """
+
+    file_path: str = Field(
+        ...,
+        description="Markdown file path to index (relative to project root)",
+        min_length=1,
+        max_length=500,
+    )
+
+
 class MoveEntityRequestV2(BaseModel):
     """V2 request schema for moving an entity to a new file location.