diff --git a/common/src/buttercup/common/llm.py b/common/src/buttercup/common/llm.py index db9ae0c7..d2a68e35 100644 --- a/common/src/buttercup/common/llm.py +++ b/common/src/buttercup/common/llm.py @@ -133,9 +133,17 @@ def create_default_llm_with_temperature(**kwargs: Any) -> Runnable: def create_llm(**kwargs: Any) -> BaseChatModel: - """Create an LLM object with the given configuration.""" + """Create an LLM object with the given configuration. + + Prefer ``LITELLM_API_KEY`` (the per-deployment budgeted virtual key created + by litellm-user-keys-setup / the k8s key Job) over ``BUTTERCUP_LITELLM_KEY``. + ``BUTTERCUP_LITELLM_KEY`` is the litellm *master* key, which bypasses all + budget/rate enforcement, so using it directly makes ``LITELLM_MAX_BUDGET`` + ineffective. Fall back to it only when no budgeted key is provided. + """ + api_key = os.environ.get("LITELLM_API_KEY") or os.environ["BUTTERCUP_LITELLM_KEY"] return ChatOpenAI( openai_api_base=os.environ["BUTTERCUP_LITELLM_HOSTNAME"], - openai_api_key=SecretStr(os.environ["BUTTERCUP_LITELLM_KEY"]), + openai_api_key=SecretStr(api_key), **kwargs, ) diff --git a/dev/docker-compose/env.template b/dev/docker-compose/env.template index 221b9892..91e7ef7c 100644 --- a/dev/docker-compose/env.template +++ b/dev/docker-compose/env.template @@ -2,6 +2,11 @@ # LiteLLM variables BUTTERCUP_LITELLM_KEY=sk-1234 +# Max spend (USD) enforced on the generated llm-user LiteLLM key. Once the +# cumulative spend on that key exceeds this value, LiteLLM rejects further +# requests. Picked up by compose interpolation in compose.yaml; defaults to 100 +# if unset. Kept low here to cap spend during local development. +LITELLM_MAX_BUDGET=15 # You only need to change variables for APIs you will use, but you need all these variables set ANTHROPIC_API_KEY= AZURE_API_BASE=