mirror of
https://github.com/basicmachines-co/basic-memory
synced 2026-06-21 13:47:35 +00:00
Aarish Alam
b6e8c636ce
Adds LiteLLM as a semantic embedding provider, including provider configuration, vector normalization, live-provider evaluation tooling, and documentation for OpenAI, Cohere, Azure Foundry, Azure OpenAI, and NVIDIA NIM-style cases.
Maintainer follow-up on this PR added provider hardening, asymmetric document/query embedding support, dimension-forwarding controls, SQLite/Postgres vector invalidation coverage, and the repeatable live LiteLLM harness.
Verification:
- Full base-repo Tests workflow passed for 3d4e092ceb: https://github.com/basicmachines-co/basic-memory/actions/runs/27072071785
- Live LiteLLM harness passed locally for OpenAI, Cohere, and Azure Foundry.
Co-authored-by: Aarish Alam <arishalam121@gmail.com>
Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
Signed-off-by: RheagalFire <arishalam121@gmail.com>
Signed-off-by: phernandez <paul@basicmachines.co>
56 lines
1.5 KiB
Python
56 lines
1.5 KiB
Python
"""Opt-in live LiteLLM provider checks against real embedding APIs."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
from typing import Any
|
|
|
|
import pytest
|
|
|
|
from semantic.litellm_live_harness import LiteLLMLiveCase, configured_cases, evaluate_case
|
|
|
|
|
|
pytestmark = [
|
|
pytest.mark.semantic,
|
|
pytest.mark.slow,
|
|
pytest.mark.live,
|
|
pytest.mark.skipif(
|
|
os.getenv("BASIC_MEMORY_RUN_LITELLM_INTEGRATION") != "1",
|
|
reason="Set BASIC_MEMORY_RUN_LITELLM_INTEGRATION=1 to run live LiteLLM tests",
|
|
),
|
|
]
|
|
|
|
|
|
def _live_cases() -> list[LiteLLMLiveCase | Any]:
|
|
"""Return built-in and user-supplied live cases whose credentials are available."""
|
|
cases = configured_cases(os.environ)
|
|
if cases:
|
|
return cases
|
|
|
|
return [
|
|
pytest.param(
|
|
None,
|
|
marks=pytest.mark.skip(
|
|
reason=(
|
|
"No LiteLLM live cases configured. Set OPENAI_API_KEY, "
|
|
"COHERE_API_KEY, or BASIC_MEMORY_TEST_LITELLM_CASES."
|
|
)
|
|
),
|
|
)
|
|
]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.parametrize(
|
|
"case",
|
|
_live_cases(),
|
|
ids=lambda case: case.name if isinstance(case, LiteLLMLiveCase) else "no-live-cases",
|
|
)
|
|
async def test_litellm_live_model_embeds_documents_and_queries(
|
|
case: LiteLLMLiveCase,
|
|
) -> None:
|
|
"""A live LiteLLM model should embed documents and rank a related query higher."""
|
|
result = await evaluate_case(case)
|
|
|
|
assert result.related_score > result.distractor_score
|