perf(sync): batch file indexing in core (#726)

Signed-off-by: phernandez <paul@basicmachines.co>

src/basic_memory/indexing/batching.py

@@ -0,0 +1,63 @@
+"""Deterministic helpers for planning bounded indexing batches."""
+
+from __future__ import annotations
+
+from collections.abc import Mapping, Sequence
+
+from basic_memory.indexing.models import IndexBatch, IndexFileMetadata
+
+
+def build_index_batches(
+    paths: Sequence[str],
+    metadata_by_path: Mapping[str, IndexFileMetadata],
+    *,
+    max_files: int,
+    max_bytes: int,
+) -> list[IndexBatch]:
+    """Build deterministic batches bounded by file count and total bytes."""
+    if max_files <= 0:
+        raise ValueError("max_files must be greater than zero")
+    if max_bytes <= 0:
+        raise ValueError("max_bytes must be greater than zero")
+
+    ordered_paths = sorted(paths)
+    batches: list[IndexBatch] = []
+    current_paths: list[str] = []
+    current_bytes = 0
+
+    for path in ordered_paths:
+        metadata = metadata_by_path.get(path)
+        if metadata is None:
+            raise KeyError(f"Missing metadata for path: {path}")
+
+        file_bytes = max(metadata.size, 0)
+
+        # Trigger: the next file would overflow the active batch.
+        # Why: keep batches memory-bounded and predictable for both local and remote callers.
+        # Outcome: flush the current batch before placing the next file.
+        if current_paths and (
+            len(current_paths) >= max_files or current_bytes + file_bytes > max_bytes
+        ):
+            batches.append(IndexBatch(paths=current_paths, total_bytes=current_bytes))
+            current_paths = []
+            current_bytes = 0
+
+        # Trigger: one file is larger than the configured byte budget.
+        # Why: we still need to index it, but splitting a single file is out of scope.
+        # Outcome: emit a dedicated single-file batch that may exceed max_bytes.
+        if file_bytes > max_bytes:
+            batches.append(IndexBatch(paths=[path], total_bytes=file_bytes))
+            continue
+
+        current_paths.append(path)
+        current_bytes += file_bytes
+
+        if len(current_paths) >= max_files or current_bytes == max_bytes:
+            batches.append(IndexBatch(paths=current_paths, total_bytes=current_bytes))
+            current_paths = []
+            current_bytes = 0
+
+    if current_paths:
+        batches.append(IndexBatch(paths=current_paths, total_bytes=current_bytes))
+
+    return batches