mirror of
https://github.com/basicmachines-co/basic-memory
synced 2026-06-21 13:47:35 +00:00
fix(core): self-heal corrupt FastEmbed model cache (#900)
Signed-off-by: phernandez <paul@basicmachines.co> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -83,6 +83,47 @@ def _format_search_error_response(
|
||||
`search_notes("{project}", "{query}", search_type="{search_type}")`
|
||||
""").strip()
|
||||
|
||||
# Corrupt/missing FastEmbed model cache (interrupted download leaves a partial
|
||||
# snapshot missing model_optimized.onnx; the ONNX runtime then raises NO_SUCHFILE).
|
||||
# Basic Memory self-heals by re-downloading on the next load, but if the user still
|
||||
# hits this, point them at the cache dir to clear manually and offer a text fallback.
|
||||
error_lower = error_message.lower()
|
||||
# "load model from" is the exact ONNX phrasing ("Load model from <path>.onnx failed").
|
||||
# The looser "load model" matched unrelated errors, so we keep only the specific phrase
|
||||
# alongside the onnxruntime / no_suchfile / model_optimized.onnx fingerprints.
|
||||
if (
|
||||
"onnxruntime" in error_lower
|
||||
or "no_suchfile" in error_lower
|
||||
or "model_optimized.onnx" in error_lower
|
||||
or "load model from" in error_lower
|
||||
):
|
||||
# Deferred import: keeps the repository layer out of the tool's import graph
|
||||
# (matches the SearchClient deferral below) and is only needed on this error path.
|
||||
from basic_memory.repository.embedding_provider_factory import _resolve_cache_dir
|
||||
|
||||
try:
|
||||
cache_dir = _resolve_cache_dir(get_container().config)
|
||||
except RuntimeError:
|
||||
cache_dir = _resolve_cache_dir(ConfigManager().config)
|
||||
return dedent(f"""
|
||||
# Search Failed - Embedding Model Missing or Corrupt
|
||||
|
||||
The local FastEmbed model could not be loaded for query '{query}': {error_message}
|
||||
|
||||
This usually means an earlier model download was interrupted and left an
|
||||
incomplete file in the model cache.
|
||||
|
||||
## How to fix
|
||||
1. Delete the FastEmbed model cache so it re-downloads on the next search:
|
||||
`{cache_dir}`
|
||||
2. Run your search again (the model downloads automatically on first use):
|
||||
`search_notes("{project}", "{query}", search_type="{search_type}")`
|
||||
|
||||
## Workaround right now
|
||||
- Use full-text search, which needs no embedding model:
|
||||
`search_notes("{project}", "{query}", search_type="text")`
|
||||
""").strip()
|
||||
|
||||
# FTS5 syntax errors
|
||||
if "syntax error" in error_message.lower() or "fts5" in error_message.lower():
|
||||
clean_query = (
|
||||
|
||||
@@ -4,6 +4,8 @@ from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import math
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from loguru import logger
|
||||
@@ -15,6 +17,24 @@ if TYPE_CHECKING:
|
||||
from fastembed import TextEmbedding # pragma: no cover
|
||||
|
||||
|
||||
# Substrings that identify the ONNX "model artifact file is missing" load failure (as
|
||||
# opposed to a config error, a download/network error, or a genuinely offline machine).
|
||||
# An interrupted FastEmbed download can leave the HuggingFace snapshot dir present but
|
||||
# missing ``model_optimized.onnx``; the ONNX runtime then raises ``NO_SUCHFILE`` and every
|
||||
# subsequent load repeats it until the cache is cleared. Matched case-insensitively.
|
||||
#
|
||||
# IMPORTANT: this text match is necessary but NOT sufficient to trigger a purge. The error
|
||||
# text alone cannot distinguish a corrupt cache from a normal cold load (model not yet
|
||||
# downloaded). Purging is gated on a positive filesystem confirmation that the snapshot dir
|
||||
# exists on disk but the model artifact file is missing — see ``_corrupt_model_subdirs``.
|
||||
_MISSING_ARTIFACT_ERROR_MARKERS = (
|
||||
"no_suchfile",
|
||||
"model_optimized.onnx",
|
||||
"file doesn't exist",
|
||||
"no such file",
|
||||
)
|
||||
|
||||
|
||||
class FastEmbedEmbeddingProvider(EmbeddingProvider):
|
||||
"""Local ONNX embedding provider backed by FastEmbed."""
|
||||
|
||||
@@ -53,6 +73,156 @@ class FastEmbedEmbeddingProvider(EmbeddingProvider):
|
||||
self._model: TextEmbedding | None = None
|
||||
self._model_lock = asyncio.Lock()
|
||||
|
||||
def _resolved_model_name(self) -> str:
|
||||
"""Return the FastEmbed model name after applying our local aliases."""
|
||||
return self._MODEL_ALIASES.get(self.model_name, self.model_name)
|
||||
|
||||
def _create_model(self) -> "TextEmbedding":
|
||||
try:
|
||||
from fastembed import TextEmbedding
|
||||
except ImportError as exc: # pragma: no cover - exercised via tests with monkeypatch
|
||||
raise SemanticDependenciesMissingError(
|
||||
"fastembed package is missing. "
|
||||
"Install/update basic-memory to include semantic dependencies: "
|
||||
"pip install -U basic-memory"
|
||||
) from exc
|
||||
resolved_model_name = self._resolved_model_name()
|
||||
if self.cache_dir is not None and self.threads is not None:
|
||||
return TextEmbedding(
|
||||
model_name=resolved_model_name,
|
||||
cache_dir=self.cache_dir,
|
||||
threads=self.threads,
|
||||
)
|
||||
if self.cache_dir is not None:
|
||||
return TextEmbedding(model_name=resolved_model_name, cache_dir=self.cache_dir)
|
||||
if self.threads is not None:
|
||||
return TextEmbedding(model_name=resolved_model_name, threads=self.threads)
|
||||
return TextEmbedding(model_name=resolved_model_name)
|
||||
|
||||
def _model_cache_candidates(self) -> list[tuple[Path, str]]:
|
||||
"""Resolve ``(snapshot_dir, model_file)`` pairs for this model under ``cache_dir``.
|
||||
|
||||
FastEmbed stores each model under ``<cache_dir>/models--<org>--<repo>`` where the
|
||||
repo is the model's HuggingFace source (e.g. ``BAAI/bge-small-en-v1.5`` resolves to
|
||||
``models--qdrant--bge-small-en-v1.5-onnx-q``). We resolve the source and the expected
|
||||
model artifact filename from FastEmbed's own model description so corruption detection
|
||||
and deletion are scoped to exactly this model's tree — never the whole cache or
|
||||
unrelated models.
|
||||
|
||||
Note: ``TextEmbedding._list_supported_models()`` is an intentional use of an
|
||||
undocumented FastEmbed API. The broad ``except`` below is a known defensive fallback:
|
||||
if the lookup ever changes shape we degrade to "no candidates" (so we never purge)
|
||||
rather than crashing the load path.
|
||||
"""
|
||||
if self.cache_dir is None:
|
||||
return []
|
||||
|
||||
# FastEmbed matches model names case-insensitively (model_management.py:
|
||||
# ``model_name.lower() == model.model.lower()``). Mirror that here so a config like
|
||||
# model="baai/bge-small-en-v1.5" still resolves to the same HF source/cache subdir.
|
||||
resolved_model_name = self._resolved_model_name().lower()
|
||||
candidates: list[tuple[Path, str]] = []
|
||||
seen: set[Path] = set()
|
||||
cache_root = Path(self.cache_dir)
|
||||
try:
|
||||
from fastembed import TextEmbedding
|
||||
|
||||
for description in TextEmbedding._list_supported_models():
|
||||
if description.model.lower() != resolved_model_name:
|
||||
continue
|
||||
hf_source = description.sources.hf
|
||||
model_file = description.model_file
|
||||
if not hf_source or not model_file:
|
||||
continue
|
||||
# HuggingFace hub names cache dirs ``models--<repo with '/' -> '--'>``.
|
||||
snapshot_dir = cache_root / f"models--{hf_source.replace('/', '--')}"
|
||||
if snapshot_dir not in seen:
|
||||
seen.add(snapshot_dir)
|
||||
candidates.append((snapshot_dir, model_file))
|
||||
except Exception as exc: # pragma: no cover - defensive: never block load on lookup
|
||||
logger.warning(
|
||||
"Could not resolve FastEmbed model source for cache cleanup: "
|
||||
"model_name={model_name} error={error}",
|
||||
model_name=resolved_model_name,
|
||||
error=exc,
|
||||
)
|
||||
|
||||
return candidates
|
||||
|
||||
def _corrupt_model_subdirs(self) -> list[Path]:
|
||||
"""Return cache subdirs that are POSITIVELY confirmed corrupt by filesystem state.
|
||||
|
||||
A model is corrupt when its HuggingFace cache dir exists on disk but at least one
|
||||
materialized snapshot revision is missing the expected model artifact file (e.g.
|
||||
``model_optimized.onnx``) — the exact fingerprint of an interrupted download. A normal
|
||||
cold load (no cache dir yet) is NOT corruption and yields no entries here, so it can
|
||||
never trigger a purge.
|
||||
|
||||
Inspection is PER-REVISION on purpose: HuggingFace keeps multiple revisions under one
|
||||
``models--<repo>`` tree, so a corrupt current snapshot can coexist with an older
|
||||
complete one. Checking ``rglob(model_file)`` across the whole tree would let the old
|
||||
artifact mask the broken current revision and leave it self-perpetuating, so we
|
||||
require every revision to carry the artifact.
|
||||
"""
|
||||
corrupt: list[Path] = []
|
||||
for model_dir, model_file in self._model_cache_candidates():
|
||||
# Trigger: the model's cache dir does not exist at all.
|
||||
# Why: this is a normal cold/first load — the model simply hasn't been
|
||||
# downloaded yet. Purging here would be wrong and pointless.
|
||||
# Outcome: skip; not corrupt.
|
||||
if not model_dir.exists():
|
||||
continue
|
||||
snapshots_root = model_dir / "snapshots"
|
||||
revision_dirs = (
|
||||
[d for d in snapshots_root.iterdir() if d.is_dir()]
|
||||
if snapshots_root.is_dir()
|
||||
else []
|
||||
)
|
||||
# Trigger: the cache dir exists but no snapshot revision has materialized.
|
||||
# Why/Outcome: an interrupted download that never wrote a revision — corrupt.
|
||||
if not revision_dirs:
|
||||
corrupt.append(model_dir)
|
||||
continue
|
||||
# Trigger: any individual revision is missing the artifact (rglob covers the
|
||||
# artifact at any depth within that revision, e.g. snapshots/<rev>/onnx/...).
|
||||
# Why: a complete OLD revision must not mask a corrupt CURRENT one.
|
||||
# Outcome: flag the model dir so the whole tree re-downloads cleanly.
|
||||
if any(not any(rev.rglob(model_file)) for rev in revision_dirs):
|
||||
corrupt.append(model_dir)
|
||||
return corrupt
|
||||
|
||||
def _purge_model_subdirs(self, subdirs: list[Path]) -> bool:
|
||||
"""Delete confirmed-corrupt cache subtrees so the next load re-downloads them.
|
||||
|
||||
Returns True when at least one targeted subdir is actually gone afterwards. On
|
||||
Windows a locked file can make ``shutil.rmtree(ignore_errors=True)`` silently no-op;
|
||||
reporting success in that case would let the caller retry against the same broken
|
||||
cache, so each subdir only counts as removed once it has actually disappeared.
|
||||
"""
|
||||
removed_any = False
|
||||
for subdir in subdirs:
|
||||
logger.warning(
|
||||
"Removing corrupt FastEmbed model cache to force re-download: {path}",
|
||||
path=str(subdir),
|
||||
)
|
||||
shutil.rmtree(subdir, ignore_errors=True)
|
||||
# Set removed only when the subdir is truly gone — a silent rmtree no-op
|
||||
# (e.g. a locked file on Windows) must not be reported as a successful purge.
|
||||
if not subdir.exists():
|
||||
removed_any = True
|
||||
return removed_any
|
||||
|
||||
@staticmethod
|
||||
def _is_missing_artifact_error(exc: Exception) -> bool:
|
||||
"""Return True when the load failure text matches the ONNX missing-artifact signature.
|
||||
|
||||
This is only the text-level gate; it is necessary but NOT sufficient to purge. The
|
||||
purge additionally requires filesystem-confirmed corruption (``_corrupt_model_subdirs``)
|
||||
so a transient/offline/"from any source" load error never deletes a valid cache.
|
||||
"""
|
||||
message = str(exc).lower()
|
||||
return any(marker in message for marker in _MISSING_ARTIFACT_ERROR_MARKERS)
|
||||
|
||||
async def _load_model(self) -> "TextEmbedding":
|
||||
if self._model is not None:
|
||||
return self._model
|
||||
@@ -61,36 +231,42 @@ class FastEmbedEmbeddingProvider(EmbeddingProvider):
|
||||
if self._model is not None:
|
||||
return self._model
|
||||
|
||||
def _create_model() -> "TextEmbedding":
|
||||
try:
|
||||
from fastembed import TextEmbedding
|
||||
except (
|
||||
ImportError
|
||||
) as exc: # pragma: no cover - exercised via tests with monkeypatch
|
||||
raise SemanticDependenciesMissingError(
|
||||
"fastembed package is missing. "
|
||||
"Install/update basic-memory to include semantic dependencies: "
|
||||
"pip install -U basic-memory"
|
||||
) from exc
|
||||
resolved_model_name = self._MODEL_ALIASES.get(self.model_name, self.model_name)
|
||||
if self.cache_dir is not None and self.threads is not None:
|
||||
return TextEmbedding(
|
||||
model_name=resolved_model_name,
|
||||
cache_dir=self.cache_dir,
|
||||
threads=self.threads,
|
||||
)
|
||||
if self.cache_dir is not None:
|
||||
return TextEmbedding(model_name=resolved_model_name, cache_dir=self.cache_dir)
|
||||
if self.threads is not None:
|
||||
return TextEmbedding(model_name=resolved_model_name, threads=self.threads)
|
||||
return TextEmbedding(model_name=resolved_model_name)
|
||||
try:
|
||||
self._model = await asyncio.to_thread(self._create_model)
|
||||
except Exception as exc:
|
||||
# Trigger: model construction raised the ONNX missing-artifact error AND a
|
||||
# filesystem check positively confirms a corrupt cache subdir (the
|
||||
# snapshot dir exists but the model artifact file is missing — the
|
||||
# fingerprint of an interrupted download).
|
||||
# Why: the raw ONNXRuntimeError is self-perpetuating — every retry hits the
|
||||
# same broken snapshot until the cache is cleared. We must NOT misread a
|
||||
# normal cold load (no snapshot dir, model simply not downloaded yet) or a
|
||||
# transient/offline "from any source" error as corruption, because purging
|
||||
# then breaks the happy path. Both the error-text gate and the positive
|
||||
# filesystem confirmation are required before we delete anything.
|
||||
# Outcome: confirmed corruption → purge exactly this model's subdir and retry
|
||||
# once so a fresh download can land. Every other failure (including a
|
||||
# retry that still fails) re-raises the ORIGINAL exception so the
|
||||
# message stays actionable and we never loop.
|
||||
if not self._is_missing_artifact_error(exc):
|
||||
raise
|
||||
corrupt_subdirs = self._corrupt_model_subdirs()
|
||||
if not corrupt_subdirs:
|
||||
raise
|
||||
if not self._purge_model_subdirs(corrupt_subdirs):
|
||||
raise
|
||||
logger.info(
|
||||
"Retrying FastEmbed model load after clearing corrupt cache: "
|
||||
"model_name={model_name}",
|
||||
model_name=self._resolved_model_name(),
|
||||
)
|
||||
self._model = await asyncio.to_thread(self._create_model)
|
||||
|
||||
self._model = await asyncio.to_thread(_create_model)
|
||||
logger.info(
|
||||
"FastEmbed model loaded: model_name={model_name} batch_size={batch_size} "
|
||||
"threads={threads} configured_parallel={configured_parallel} "
|
||||
"effective_parallel={effective_parallel}",
|
||||
model_name=self._MODEL_ALIASES.get(self.model_name, self.model_name),
|
||||
model_name=self._resolved_model_name(),
|
||||
batch_size=self.batch_size,
|
||||
threads=self.threads,
|
||||
configured_parallel=self.parallel,
|
||||
|
||||
@@ -479,6 +479,42 @@ class TestSearchErrorFormatting:
|
||||
assert "# Search Failed - Semantic Dependencies Missing" in result
|
||||
assert "pip install -U basic-memory" in result
|
||||
|
||||
def test_format_search_error_corrupt_embedding_model(self):
|
||||
"""Test formatting for a corrupt/missing FastEmbed model (ONNX NO_SUCHFILE)."""
|
||||
from basic_memory.config import ConfigManager
|
||||
from basic_memory.repository.embedding_provider_factory import _resolve_cache_dir
|
||||
|
||||
result = _format_search_error_response(
|
||||
"test-project",
|
||||
"[ONNXRuntimeError] : 3 : NO_SUCHFILE : Load model from "
|
||||
"/home/u/.basic-memory/fastembed_cache/models--qdrant--bge-small-en-v1.5-onnx-q/"
|
||||
"snapshots/abc/model_optimized.onnx failed. File doesn't exist",
|
||||
"semantic query",
|
||||
"hybrid",
|
||||
)
|
||||
|
||||
expected_cache_dir = _resolve_cache_dir(ConfigManager().config)
|
||||
assert "# Search Failed - Embedding Model Missing or Corrupt" in result
|
||||
# Names the actual resolved cache dir so the user knows what to delete.
|
||||
assert expected_cache_dir in result
|
||||
# Offers full-text search as an immediate workaround.
|
||||
assert 'search_type="text"' in result
|
||||
|
||||
def test_format_search_error_load_model_phrase_does_not_overmatch(self):
|
||||
"""A generic error mentioning 'load model' (no 'from') must not hit the embedding branch.
|
||||
|
||||
The marker was tightened from the broad 'load model' to the exact ONNX phrasing
|
||||
'load model from' so unrelated failures fall through to the generic handler.
|
||||
"""
|
||||
result = _format_search_error_response(
|
||||
"test-project",
|
||||
"Failed to load model configuration for this project",
|
||||
"test query",
|
||||
)
|
||||
|
||||
assert "# Search Failed - Embedding Model Missing or Corrupt" not in result
|
||||
assert "# Search Failed" in result
|
||||
|
||||
def test_format_search_error_generic(self):
|
||||
"""Test formatting for generic errors."""
|
||||
result = _format_search_error_response("test-project", "unknown error", "test query")
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
import builtins
|
||||
import math
|
||||
import sys
|
||||
from dataclasses import dataclass
|
||||
|
||||
import pytest
|
||||
|
||||
@@ -211,3 +212,346 @@ async def test_fastembed_provider_zero_vector_does_not_raise(monkeypatch):
|
||||
result = await provider.embed_documents(["zero vector"])
|
||||
|
||||
assert result == [[0.0, 0.0, 0.0, 0.0]]
|
||||
|
||||
|
||||
# --- Self-heal of corrupt/partial model cache (#895) ---
|
||||
#
|
||||
# A real interrupted FastEmbed download is non-deterministic and offline-unfriendly, so we
|
||||
# stub TextEmbedding to (a) advertise an HF source + model_file via _list_supported_models so
|
||||
# the provider can compute the exact models--<org>--<repo> cache subdir and the artifact name,
|
||||
# and (b) raise a NO_SUCHFILE-style ONNX error on the first construction. This is the justified
|
||||
# mock case called out in the task. The purge is gated on a filesystem confirmation that the
|
||||
# snapshot dir exists but the artifact is missing, so each test stages the cache accordingly.
|
||||
|
||||
|
||||
@dataclass
|
||||
class _StubModelSource:
|
||||
hf: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class _StubModelDescription:
|
||||
model: str
|
||||
sources: _StubModelSource
|
||||
model_file: str = "model_optimized.onnx"
|
||||
|
||||
|
||||
class _SelfHealStubTextEmbedding:
|
||||
"""Raises a NO_SUCHFILE-style ONNX error on the first N constructions, then succeeds."""
|
||||
|
||||
fail_first_n = 1
|
||||
construct_count = 0
|
||||
HF_SOURCE = "stub-org/stub-model-onnx-q"
|
||||
RESOLVED_MODEL = "stub-model"
|
||||
MODEL_FILE = "model_optimized.onnx"
|
||||
|
||||
def __init__(self, model_name: str, cache_dir: str | None = None, threads: int | None = None):
|
||||
type(self).construct_count += 1
|
||||
if type(self).construct_count <= type(self).fail_first_n:
|
||||
raise RuntimeError(
|
||||
"[ONNXRuntimeError] : 3 : NO_SUCHFILE : Load model from "
|
||||
f"{cache_dir}/models--stub-org--stub-model-onnx-q/snapshots/abc123/"
|
||||
"model_optimized.onnx failed. File doesn't exist"
|
||||
)
|
||||
self.model_name = model_name
|
||||
|
||||
def embed(self, texts: list[str], batch_size: int = 64, **kwargs):
|
||||
for _ in texts:
|
||||
yield _StubVector([1.0, 0.0, 0.0, 0.0])
|
||||
|
||||
@classmethod
|
||||
def _list_supported_models(cls):
|
||||
# Include decoys so the resolver's skip branches are exercised: a model with a
|
||||
# different name (name-mismatch skip) and one with an empty HF source (no-source skip).
|
||||
return [
|
||||
_StubModelDescription(
|
||||
model="some-other-model",
|
||||
sources=_StubModelSource(hf="other-org/other-model"),
|
||||
model_file=cls.MODEL_FILE,
|
||||
),
|
||||
_StubModelDescription(
|
||||
model=cls.RESOLVED_MODEL,
|
||||
sources=_StubModelSource(hf=""),
|
||||
model_file=cls.MODEL_FILE,
|
||||
),
|
||||
_StubModelDescription(
|
||||
model=cls.RESOLVED_MODEL,
|
||||
sources=_StubModelSource(hf=cls.HF_SOURCE),
|
||||
model_file=cls.MODEL_FILE,
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def _install_self_heal_stub(monkeypatch):
|
||||
module = type(sys)("fastembed")
|
||||
setattr(module, "TextEmbedding", _SelfHealStubTextEmbedding)
|
||||
monkeypatch.setitem(sys.modules, "fastembed", module)
|
||||
_SelfHealStubTextEmbedding.construct_count = 0
|
||||
_SelfHealStubTextEmbedding.fail_first_n = 1
|
||||
_SelfHealStubTextEmbedding.RESOLVED_MODEL = "stub-model"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fastembed_provider_self_heals_corrupt_model_cache(monkeypatch, tmp_path):
|
||||
"""A NO_SUCHFILE load failure should purge the model cache subdir and retry once."""
|
||||
_install_self_heal_stub(monkeypatch)
|
||||
|
||||
# Simulate the partial-download artifact: the model's HF cache subdir exists on disk
|
||||
# but is incomplete. The provider must remove exactly this subdir, not the whole cache.
|
||||
cache_dir = tmp_path / "fastembed_cache"
|
||||
model_subdir = cache_dir / "models--stub-org--stub-model-onnx-q"
|
||||
model_subdir.mkdir(parents=True)
|
||||
(model_subdir / "stale.bin").write_text("partial download")
|
||||
unrelated_subdir = cache_dir / "models--other--keep-me"
|
||||
unrelated_subdir.mkdir(parents=True)
|
||||
(unrelated_subdir / "data.bin").write_text("do not delete")
|
||||
|
||||
provider = FastEmbedEmbeddingProvider(
|
||||
model_name="stub-model", dimensions=4, cache_dir=str(cache_dir)
|
||||
)
|
||||
|
||||
vectors = await provider.embed_documents(["recover after corrupt cache"])
|
||||
|
||||
# Construction was attempted exactly twice: the failing load, then the post-purge retry.
|
||||
assert _SelfHealStubTextEmbedding.construct_count == 2
|
||||
# The corrupt model subdir was removed; the unrelated model cache was untouched.
|
||||
assert not model_subdir.exists()
|
||||
assert unrelated_subdir.exists()
|
||||
assert (unrelated_subdir / "data.bin").read_text() == "do not delete"
|
||||
# The retry produced real vectors.
|
||||
assert len(vectors) == 1
|
||||
assert len(vectors[0]) == 4
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fastembed_provider_fails_fast_on_persistent_corrupt_cache(monkeypatch, tmp_path):
|
||||
"""A second consecutive NO_SUCHFILE failure must fail fast (no infinite retry loop)."""
|
||||
_install_self_heal_stub(monkeypatch)
|
||||
# Both constructions fail — the retry does not loop.
|
||||
_SelfHealStubTextEmbedding.fail_first_n = 2
|
||||
|
||||
cache_dir = tmp_path / "fastembed_cache"
|
||||
model_subdir = cache_dir / "models--stub-org--stub-model-onnx-q"
|
||||
model_subdir.mkdir(parents=True)
|
||||
|
||||
provider = FastEmbedEmbeddingProvider(
|
||||
model_name="stub-model", dimensions=4, cache_dir=str(cache_dir)
|
||||
)
|
||||
|
||||
with pytest.raises(RuntimeError, match="NO_SUCHFILE"):
|
||||
await provider.embed_documents(["still broken"])
|
||||
|
||||
# Exactly one retry: two total construction attempts, then fail fast.
|
||||
assert _SelfHealStubTextEmbedding.construct_count == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fastembed_provider_fails_fast_when_purge_silently_noops(monkeypatch, tmp_path):
|
||||
"""If rmtree silently fails (e.g. Windows locked files), do not claim success or retry.
|
||||
|
||||
shutil.rmtree(ignore_errors=True) can no-op when a file is locked. Treating that as a
|
||||
successful purge would retry against the same broken cache; instead the load must fail
|
||||
fast with the original error. We inject a no-op rmtree to simulate the locked-file case.
|
||||
"""
|
||||
import basic_memory.repository.fastembed_provider as fastembed_provider
|
||||
|
||||
_install_self_heal_stub(monkeypatch)
|
||||
|
||||
cache_dir = tmp_path / "fastembed_cache"
|
||||
model_subdir = cache_dir / "models--stub-org--stub-model-onnx-q"
|
||||
model_subdir.mkdir(parents=True)
|
||||
(model_subdir / "stale.bin").write_text("partial download")
|
||||
|
||||
# Simulate a deletion that silently fails to remove the directory.
|
||||
monkeypatch.setattr(
|
||||
fastembed_provider.shutil, "rmtree", lambda *args, **kwargs: None, raising=True
|
||||
)
|
||||
|
||||
provider = FastEmbedEmbeddingProvider(
|
||||
model_name="stub-model", dimensions=4, cache_dir=str(cache_dir)
|
||||
)
|
||||
|
||||
with pytest.raises(RuntimeError, match="NO_SUCHFILE"):
|
||||
await provider.embed_documents(["locked cache"])
|
||||
|
||||
# rmtree no-oped, so the subdir survives and no retry was attempted.
|
||||
assert model_subdir.exists()
|
||||
assert _SelfHealStubTextEmbedding.construct_count == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fastembed_provider_does_not_purge_on_unrelated_error(monkeypatch, tmp_path):
|
||||
"""A non-cache load error must propagate without deleting any cache subdir."""
|
||||
|
||||
class _ConfigErrorTextEmbedding:
|
||||
construct_count = 0
|
||||
|
||||
def __init__(self, model_name: str, cache_dir: str | None = None, **_kwargs):
|
||||
type(self).construct_count += 1
|
||||
raise ValueError("invalid model configuration")
|
||||
|
||||
@classmethod
|
||||
def _list_supported_models(cls):
|
||||
return [
|
||||
_StubModelDescription(
|
||||
model="stub-model",
|
||||
sources=_StubModelSource(hf="stub-org/stub-model-onnx-q"),
|
||||
)
|
||||
]
|
||||
|
||||
module = type(sys)("fastembed")
|
||||
setattr(module, "TextEmbedding", _ConfigErrorTextEmbedding)
|
||||
monkeypatch.setitem(sys.modules, "fastembed", module)
|
||||
|
||||
cache_dir = tmp_path / "fastembed_cache"
|
||||
model_subdir = cache_dir / "models--stub-org--stub-model-onnx-q"
|
||||
model_subdir.mkdir(parents=True)
|
||||
(model_subdir / "keep.bin").write_text("keep")
|
||||
|
||||
provider = FastEmbedEmbeddingProvider(
|
||||
model_name="stub-model", dimensions=4, cache_dir=str(cache_dir)
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="invalid model configuration"):
|
||||
await provider.embed_documents(["bad config"])
|
||||
|
||||
# No retry and no deletion for errors that are not missing-artifact failures.
|
||||
assert _ConfigErrorTextEmbedding.construct_count == 1
|
||||
assert model_subdir.exists()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fastembed_provider_cold_load_does_not_purge_or_retry(monkeypatch, tmp_path):
|
||||
"""A cold load (snapshot dir absent) must NOT be misread as corruption.
|
||||
|
||||
This is the CI happy-path regression: on a cold model cache the first load can fail
|
||||
before the model is downloaded, but with no snapshot dir there is nothing corrupt to
|
||||
purge. The original error must propagate unchanged with no retry, so a normal
|
||||
not-yet-downloaded model is never deleted.
|
||||
"""
|
||||
_install_self_heal_stub(monkeypatch)
|
||||
|
||||
cache_dir = tmp_path / "fastembed_cache"
|
||||
cache_dir.mkdir(parents=True)
|
||||
# Intentionally do NOT create the model subdir: this is a normal cold load.
|
||||
|
||||
provider = FastEmbedEmbeddingProvider(
|
||||
model_name="stub-model", dimensions=4, cache_dir=str(cache_dir)
|
||||
)
|
||||
|
||||
with pytest.raises(RuntimeError, match="NO_SUCHFILE"):
|
||||
await provider.embed_documents(["nothing to purge"])
|
||||
|
||||
# Only the initial attempt ran — no snapshot dir means no confirmed corruption, no retry.
|
||||
assert _SelfHealStubTextEmbedding.construct_count == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fastembed_provider_does_not_purge_when_artifact_present(monkeypatch, tmp_path):
|
||||
"""A NO_SUCHFILE-shaped error must NOT purge when the artifact is actually on disk.
|
||||
|
||||
The error-text gate alone is not enough: if filesystem inspection finds the model
|
||||
artifact present in the snapshot, the cache is not corrupt and must be left intact.
|
||||
Re-raise the original error rather than deleting a healthy cache.
|
||||
"""
|
||||
_install_self_heal_stub(monkeypatch)
|
||||
# Construction keeps failing with the NO_SUCHFILE text regardless of cache state.
|
||||
_SelfHealStubTextEmbedding.fail_first_n = 99
|
||||
|
||||
cache_dir = tmp_path / "fastembed_cache"
|
||||
snapshot_dir = cache_dir / "models--stub-org--stub-model-onnx-q" / "snapshots" / "rev1"
|
||||
snapshot_dir.mkdir(parents=True)
|
||||
artifact = snapshot_dir / "model_optimized.onnx"
|
||||
artifact.write_text("valid model artifact")
|
||||
|
||||
provider = FastEmbedEmbeddingProvider(
|
||||
model_name="stub-model", dimensions=4, cache_dir=str(cache_dir)
|
||||
)
|
||||
|
||||
with pytest.raises(RuntimeError, match="NO_SUCHFILE"):
|
||||
await provider.embed_documents(["artifact is fine"])
|
||||
|
||||
# No purge and no retry: the artifact is present, so the cache is not corrupt.
|
||||
assert _SelfHealStubTextEmbedding.construct_count == 1
|
||||
assert artifact.exists()
|
||||
assert artifact.read_text() == "valid model artifact"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fastembed_provider_self_heals_when_current_revision_corrupt(monkeypatch, tmp_path):
|
||||
"""A corrupt current revision must be detected even when an older revision is complete.
|
||||
|
||||
HuggingFace keeps multiple revisions under one models--<repo> tree. Per-revision
|
||||
inspection is required: a whole-tree rglob would find the OLD revision's artifact and
|
||||
wrongly conclude the cache is healthy, leaving the broken current snapshot
|
||||
self-perpetuating (PR #900 review).
|
||||
"""
|
||||
_install_self_heal_stub(monkeypatch)
|
||||
|
||||
cache_dir = tmp_path / "fastembed_cache"
|
||||
snapshots = cache_dir / "models--stub-org--stub-model-onnx-q" / "snapshots"
|
||||
# Old revision: complete (has the artifact).
|
||||
good_rev = snapshots / "rev_old"
|
||||
good_rev.mkdir(parents=True)
|
||||
(good_rev / "model_optimized.onnx").write_text("complete old artifact")
|
||||
# Current revision: interrupted download — directory present, artifact missing.
|
||||
bad_rev = snapshots / "rev_current"
|
||||
bad_rev.mkdir(parents=True)
|
||||
(bad_rev / "stale.partial").write_text("partial download")
|
||||
|
||||
provider = FastEmbedEmbeddingProvider(
|
||||
model_name="stub-model", dimensions=4, cache_dir=str(cache_dir)
|
||||
)
|
||||
|
||||
vectors = await provider.embed_documents(["recover from mixed-revision cache"])
|
||||
|
||||
# The corrupt-current-revision cache was detected (not masked by the old revision),
|
||||
# purged, and the retry succeeded.
|
||||
assert _SelfHealStubTextEmbedding.construct_count == 2
|
||||
assert not (cache_dir / "models--stub-org--stub-model-onnx-q").exists()
|
||||
assert len(vectors) == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fastembed_provider_self_heals_with_case_insensitive_model_name(
|
||||
monkeypatch, tmp_path
|
||||
):
|
||||
"""A lower-cased model name must still resolve the HF cache subdir for the purge.
|
||||
|
||||
FastEmbed matches model names case-insensitively, so a config like
|
||||
model="baai/bge-small-en-v1.5" is valid. The purge resolver must mirror that, otherwise
|
||||
the corrupt subdir resolves to nothing and self-heal silently does nothing.
|
||||
"""
|
||||
_install_self_heal_stub(monkeypatch)
|
||||
# Advertise the model under its canonical mixed-case name.
|
||||
_SelfHealStubTextEmbedding.RESOLVED_MODEL = "Stub-Model"
|
||||
|
||||
cache_dir = tmp_path / "fastembed_cache"
|
||||
model_subdir = cache_dir / "models--stub-org--stub-model-onnx-q"
|
||||
model_subdir.mkdir(parents=True)
|
||||
(model_subdir / "stale.bin").write_text("partial download")
|
||||
|
||||
# Configure the provider with the lower-cased spelling.
|
||||
provider = FastEmbedEmbeddingProvider(
|
||||
model_name="stub-model", dimensions=4, cache_dir=str(cache_dir)
|
||||
)
|
||||
|
||||
vectors = await provider.embed_documents(["recover with case-insensitive name"])
|
||||
|
||||
assert _SelfHealStubTextEmbedding.construct_count == 2
|
||||
assert not model_subdir.exists()
|
||||
assert len(vectors) == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fastembed_provider_fails_fast_without_cache_dir(monkeypatch):
|
||||
"""Without a configured cache_dir there is nothing to purge, so fail fast."""
|
||||
_install_self_heal_stub(monkeypatch)
|
||||
|
||||
# cache_dir defaults to None — _model_cache_candidates() returns no candidates.
|
||||
provider = FastEmbedEmbeddingProvider(model_name="stub-model", dimensions=4)
|
||||
|
||||
with pytest.raises(RuntimeError, match="NO_SUCHFILE"):
|
||||
await provider.embed_documents(["no cache dir"])
|
||||
|
||||
assert _SelfHealStubTextEmbedding.construct_count == 1
|
||||
|
||||
Reference in New Issue
Block a user