perf(sync): batch file indexing in core (#726)

Signed-off-by: phernandez <paul@basicmachines.co>

src/basic_memory/services/file_service.py

@@ -3,6 +3,7 @@
 import asyncio
 import hashlib
 import mimetypes
+from dataclasses import dataclass
 from datetime import datetime
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple, Union
@@ -25,6 +26,14 @@ from basic_memory.utils import FilePath
 from loguru import logger
 
 
+@dataclass(slots=True)
+class FrontmatterUpdateResult:
+    """Final content emitted by a frontmatter rewrite without a follow-up reread."""
+
+    checksum: str
+    content: str
+
+
 class FileService:
     """Service for handling file operations with concurrency control.
 
@@ -301,7 +310,7 @@ class FileService:
 
         except Exception as e:
             logger.exception("File read error", path=str(full_path), error=str(e))
-            raise FileOperationError(f"Failed to read file: {e}")
+            raise FileOperationError(f"Failed to read file: {e}") from e
 
     async def read_file(self, path: FilePath) -> Tuple[str, str]:
         """Read file and compute checksum using true async I/O.
@@ -401,12 +410,14 @@ class FileService:
             )
             raise FileOperationError(f"Failed to move file {source} -> {destination}: {e}")
 
-    async def update_frontmatter(self, path: FilePath, updates: Dict[str, Any]) -> str:
-        """Update frontmatter fields in a file while preserving all content.
+    async def update_frontmatter_with_result(
+        self, path: FilePath, updates: Dict[str, Any]
+    ) -> FrontmatterUpdateResult:
+        """Update frontmatter and return the exact final written markdown content.
 
         Only modifies the frontmatter section, leaving all content untouched.
         Creates frontmatter section if none exists.
-        Returns checksum of updated file.
+        Returns both checksum and final content so callers do not need a reread.
 
         Uses aiofiles for true async I/O (non-blocking).
 
@@ -415,7 +426,7 @@ class FileService:
             updates: Dict of frontmatter fields to update
 
         Returns:
-            Checksum of updated file
+            Typed result containing checksum and final content
 
         Raises:
             FileOperationError: If file operations fail
@@ -467,7 +478,10 @@ class FileService:
                 if formatted_content is not None:
                     content_for_checksum = formatted_content  # pragma: no cover
 
-            return await file_utils.compute_checksum(content_for_checksum)
+            return FrontmatterUpdateResult(
+                checksum=await file_utils.compute_checksum(content_for_checksum),
+                content=content_for_checksum,
+            )
 
         except Exception as e:  # pragma: no cover
             # Only log real errors (not YAML parsing, which is handled above)
@@ -479,6 +493,11 @@ class FileService:
                 )
             raise FileOperationError(f"Failed to update frontmatter: {e}")
 
+    async def update_frontmatter(self, path: FilePath, updates: Dict[str, Any]) -> str:
+        """Update frontmatter fields in a file while preserving all content."""
+        result = await self.update_frontmatter_with_result(path, updates)
+        return result.checksum
+
     async def compute_checksum(self, path: FilePath) -> str:
         """Compute checksum for a file using true async I/O.
 

src/basic_memory/services/search_service.py

@@ -1,5 +1,6 @@
 """Service for search operations."""
 
+import asyncio
 import ast
 import re
 from datetime import datetime
@@ -427,6 +428,15 @@ class SearchService:
 
     async def sync_entity_vectors(self, entity_id: int) -> None:
         """Refresh vector chunks for one entity in repositories that support semantic indexing."""
+        entity = await self.entity_repository.find_by_id(entity_id)
+        if entity is None:
+            await self._clear_entity_vectors(entity_id)
+            return
+
+        if not self._entity_embeddings_enabled(entity):
+            await self._clear_entity_vectors(entity_id)
+            return
+
         await self.repository.sync_entity_vectors(entity_id)
 
     async def sync_entity_vectors_batch(
@@ -435,10 +445,90 @@ class SearchService:
         progress_callback=None,
     ) -> VectorSyncBatchResult:
         """Refresh vector chunks for a batch of entities."""
-        return await self.repository.sync_entity_vectors_batch(
-            entity_ids,
-            progress_callback=progress_callback,
+        if not entity_ids:
+            return VectorSyncBatchResult(
+                entities_total=0,
+                entities_synced=0,
+                entities_failed=0,
+            )
+
+        entities_by_id = {
+            entity.id: entity for entity in await self.entity_repository.find_by_ids(entity_ids)
+        }
+        unknown_ids = [entity_id for entity_id in entity_ids if entity_id not in entities_by_id]
+        opted_out_ids = [
+            entity_id
+            for entity_id in entity_ids
+            if (
+                (entity := entities_by_id.get(entity_id)) is not None
+                and not self._entity_embeddings_enabled(entity)
+            )
+        ]
+        if opted_out_ids:
+            await asyncio.gather(
+                *(self._clear_entity_vectors(entity_id) for entity_id in opted_out_ids)
+            )
+
+        eligible_entity_ids = [
+            entity_id
+            for entity_id in entity_ids
+            if entity_id in entities_by_id and entity_id not in opted_out_ids
+        ]
+
+        cleanup_task = (
+            self.repository.sync_entity_vectors_batch(unknown_ids) if unknown_ids else None
         )
+        eligible_task = (
+            self.repository.sync_entity_vectors_batch(
+                eligible_entity_ids,
+                progress_callback=progress_callback,
+            )
+            if eligible_entity_ids
+            else None
+        )
+        repository_results = [
+            result
+            for result in await asyncio.gather(
+                cleanup_task if cleanup_task is not None else asyncio.sleep(0, result=None),
+                eligible_task if eligible_task is not None else asyncio.sleep(0, result=None),
+            )
+            if result is not None
+        ]
+
+        if not repository_results:
+            return VectorSyncBatchResult(
+                entities_total=len(entity_ids),
+                entities_synced=0,
+                entities_failed=0,
+                entities_skipped=len(opted_out_ids),
+            )
+
+        batch_result = VectorSyncBatchResult(
+            entities_total=len(entity_ids),
+            entities_synced=sum(result.entities_synced for result in repository_results),
+            entities_failed=sum(result.entities_failed for result in repository_results),
+            entities_deferred=sum(result.entities_deferred for result in repository_results),
+            entities_skipped=(
+                len(opted_out_ids)
+                + sum(result.entities_skipped for result in repository_results)
+                - len(unknown_ids)
+            ),
+            failed_entity_ids=[
+                failed_entity_id
+                for result in repository_results
+                for failed_entity_id in result.failed_entity_ids
+            ],
+            chunks_total=sum(result.chunks_total for result in repository_results),
+            chunks_skipped=sum(result.chunks_skipped for result in repository_results),
+            embedding_jobs_total=sum(result.embedding_jobs_total for result in repository_results),
+            prepare_seconds_total=sum(result.prepare_seconds_total for result in repository_results),
+            queue_wait_seconds_total=sum(
+                result.queue_wait_seconds_total for result in repository_results
+            ),
+            embed_seconds_total=sum(result.embed_seconds_total for result in repository_results),
+            write_seconds_total=sum(result.write_seconds_total for result in repository_results),
+        )
+        return batch_result
 
     async def reindex_vectors(self, progress_callback=None) -> dict:
         """Rebuild vector embeddings for all entities.
@@ -456,14 +546,14 @@ class SearchService:
         # that reference entity_ids no longer in the entity table
         await self._purge_stale_search_rows()
 
-        batch_result = await self.repository.sync_entity_vectors_batch(
+        batch_result = await self.sync_entity_vectors_batch(
             entity_ids,
             progress_callback=progress_callback,
         )
         stats = {
             "total_entities": batch_result.entities_total,
             "embedded": batch_result.entities_synced,
-            "skipped": 0,
+            "skipped": batch_result.entities_skipped,
             "errors": batch_result.entities_failed,
         }
 
@@ -518,6 +608,60 @@ class SearchService:
 
         logger.info("Purged stale search rows for deleted entities", project_id=project_id)
 
+    @staticmethod
+    def _entity_embeddings_enabled(entity: Entity) -> bool:
+        """Return whether semantic embeddings should be generated for this entity."""
+        if not entity.entity_metadata:
+            return True
+
+        embed_value = entity.entity_metadata.get("embed")
+        if embed_value is None:
+            return True
+        if isinstance(embed_value, bool):
+            return embed_value
+        if isinstance(embed_value, str):
+            normalized = embed_value.strip().lower()
+            if normalized in {"false", "0", "no", "off"}:
+                return False
+            if normalized in {"true", "1", "yes", "on"}:
+                return True
+        if isinstance(embed_value, (int, float)):
+            return bool(embed_value)
+
+        # Default unknown values to enabled so malformed metadata does not silently
+        # remove notes from semantic search.
+        return True
+
+    async def _clear_entity_vectors(self, entity_id: int) -> None:
+        """Delete derived vector rows for one entity."""
+        from basic_memory.repository.search_repository_base import SearchRepositoryBase
+        from basic_memory.repository.sqlite_search_repository import SQLiteSearchRepository
+
+        # Trigger: semantic indexing is disabled for this repository instance.
+        # Why: repositories only create vector tables when semantic search is enabled.
+        # Outcome: skip cleanup because there are no active derived vector rows to maintain.
+        if isinstance(self.repository, SearchRepositoryBase) and not self.repository._semantic_enabled:
+            return
+
+        params = {"project_id": self.repository.project_id, "entity_id": entity_id}
+        if isinstance(self.repository, SQLiteSearchRepository):
+            await self.repository.execute_query(
+                text(
+                    "DELETE FROM search_vector_embeddings WHERE rowid IN ("
+                    "SELECT id FROM search_vector_chunks "
+                    "WHERE project_id = :project_id AND entity_id = :entity_id)"
+                ),
+                params,
+            )
+
+        await self.repository.execute_query(
+            text(
+                "DELETE FROM search_vector_chunks "
+                "WHERE project_id = :project_id AND entity_id = :entity_id"
+            ),
+            params,
+        )
+
     async def index_entity_file(
         self,
         entity: Entity,