fix(core): L2-normalize FastEmbed vectors (#843)

L2-normalizes FastEmbed output vectors at the provider boundary so SQLite vector scoring keeps its unit-vector contract for custom FastEmbed models such as multilingual MiniLM variants.

Zero vectors are preserved as-is to avoid division errors, and the provider tests cover both non-unit vectors and zero-vector behavior.

Verification:
- uv run pytest tests/repository/test_fastembed_provider.py -q
- uv run ruff check src/basic_memory/repository/fastembed_provider.py tests/repository/test_fastembed_provider.py
- uv run ruff format --check src/basic_memory/repository/fastembed_provider.py tests/repository/test_fastembed_provider.py

Co-authored-by: Claude Sonnet 4.6 <noreply@anthropic.com>
Signed-off-by: tk-pkm111 <133480534+tk-pkm111@users.noreply.github.com>
Signed-off-by: phernandez <paul@basicmachines.co>
This commit is contained in:
tk
2026-06-07 07:15:19 +09:00
committed by GitHub
parent b6e8c636ce
commit f6565b9d23
2 changed files with 73 additions and 2 deletions
@@ -3,6 +3,7 @@
from __future__ import annotations
import asyncio
import math
from typing import TYPE_CHECKING
from loguru import logger
@@ -119,10 +120,17 @@ class FastEmbedEmbeddingProvider(EmbeddingProvider):
if effective_parallel is not None:
embed_kwargs["parallel"] = effective_parallel
vectors = list(model.embed(texts, **embed_kwargs))
# sqlite_search_repository.py uses a distance-to-similarity formula that assumes
# unit-normalized vectors (see the comment on line 65-67 of that file).
# Some models (e.g. multilingual ones) return vectors with norm > 1, so we
# L2-normalize here to satisfy that contract regardless of the chosen model.
normalized: list[list[float]] = []
for vector in vectors:
values = vector.tolist() if hasattr(vector, "tolist") else vector
normalized.append([float(value) for value in values])
values = vector.tolist() if hasattr(vector, "tolist") else list(vector)
norm = math.sqrt(sum(x * x for x in values))
if norm > 0:
values = [x / norm for x in values]
normalized.append([float(v) for v in values])
return normalized
vectors = await asyncio.to_thread(_embed_batch)