From ec7031c5ef9e178fcf70232753a89f0b56b03639 Mon Sep 17 00:00:00 2001 From: Riccardo Schirone <562321+ret2libc@users.noreply.github.com> Date: Mon, 18 May 2026 15:52:14 +0200 Subject: [PATCH] fix(common): use budgeted LITELLM_API_KEY so LITELLM_MAX_BUDGET is enforced (#550) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(common): use budgeted LITELLM_API_KEY for the LLM client create_llm() authenticated with BUTTERCUP_LITELLM_KEY, which litellm_config.yaml defines as general_settings.master_key. LiteLLM master keys bypass all budget/rate enforcement, so LITELLM_MAX_BUDGET had no effect — an e2e run with --budget 1 spent ~$4.77 unthrottled with every /chat/completions returning 200. The budgeted virtual key (llm-user, created with max_budget= $LITELLM_MAX_BUDGET by litellm-user-keys-setup) is already plumbed: seed-gen/patcher entrypoints export LITELLM_API_KEY from the mounted key file. create_llm() just never read it. Prefer LITELLM_API_KEY when set; fall back to BUTTERCUP_LITELLM_KEY so environments without a provisioned budgeted key are unaffected. Co-Authored-By: Claude Opus 4.7 (1M context) * chore(compose): document LITELLM_MAX_BUDGET in env.template (complements #550) Co-Authored-By: Claude Opus 4.7 (1M context) --------- Co-authored-by: Claude Opus 4.7 (1M context) --- common/src/buttercup/common/llm.py | 12 ++++++++++-- dev/docker-compose/env.template | 5 +++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/common/src/buttercup/common/llm.py b/common/src/buttercup/common/llm.py index db9ae0c7..d2a68e35 100644 --- a/common/src/buttercup/common/llm.py +++ b/common/src/buttercup/common/llm.py @@ -133,9 +133,17 @@ def create_default_llm_with_temperature(**kwargs: Any) -> Runnable: def create_llm(**kwargs: Any) -> BaseChatModel: - """Create an LLM object with the given configuration.""" + """Create an LLM object with the given configuration. + + Prefer ``LITELLM_API_KEY`` (the per-deployment budgeted virtual key created + by litellm-user-keys-setup / the k8s key Job) over ``BUTTERCUP_LITELLM_KEY``. + ``BUTTERCUP_LITELLM_KEY`` is the litellm *master* key, which bypasses all + budget/rate enforcement, so using it directly makes ``LITELLM_MAX_BUDGET`` + ineffective. Fall back to it only when no budgeted key is provided. + """ + api_key = os.environ.get("LITELLM_API_KEY") or os.environ["BUTTERCUP_LITELLM_KEY"] return ChatOpenAI( openai_api_base=os.environ["BUTTERCUP_LITELLM_HOSTNAME"], - openai_api_key=SecretStr(os.environ["BUTTERCUP_LITELLM_KEY"]), + openai_api_key=SecretStr(api_key), **kwargs, ) diff --git a/dev/docker-compose/env.template b/dev/docker-compose/env.template index 221b9892..91e7ef7c 100644 --- a/dev/docker-compose/env.template +++ b/dev/docker-compose/env.template @@ -2,6 +2,11 @@ # LiteLLM variables BUTTERCUP_LITELLM_KEY=sk-1234 +# Max spend (USD) enforced on the generated llm-user LiteLLM key. Once the +# cumulative spend on that key exceeds this value, LiteLLM rejects further +# requests. Picked up by compose interpolation in compose.yaml; defaults to 100 +# if unset. Kept low here to cap spend during local development. +LITELLM_MAX_BUDGET=15 # You only need to change variables for APIs you will use, but you need all these variables set ANTHROPIC_API_KEY= AZURE_API_BASE=