From c1856c4ae38f5e5b1445d904745d79db99337f3a Mon Sep 17 00:00:00 2001 From: Riccardo Schirone Date: Tue, 19 May 2026 11:10:48 +0000 Subject: [PATCH] fix(scripts): e2e.sh approval wait-loop + viable budget/duration defaults Three defects found while verifying the pipeline end-to-end: 1. Approval one-shot race: capture_line 'competition_patch_id=' ran once right after the patch-generated milestone, but the scheduler logs that id only minutes later (after it builds+verifies+submits the patch). The capture always lost the race, so approval was always skipped and the local stack never reached Patch passed / bundle. Replace with a wait_capture() poll loop (mirrors wait_for) so approval actually fires. 2. Default --task-duration 1800 is self-defeating: build->POV->seed-gen-> patch exceeds 30 min on normal hardware, so the task expires mid-patch ("task expired/cancelled? Will discard") and never reaches patch/bundle. Default to 7200 so the task outlives the pipeline. 3. Default --budget 3 cannot reach patch/bundle: a full run through patch generation costs ~$10; $3 is exhausted around POV. Default to 10. e2e.md updated to match (defaults, the cheap --budget 3 caveat, and the poll-then-approve description). Co-Authored-By: Claude Opus 4.7 (1M context) --- .claude/commands/e2e.md | 13 ++++++------ scripts/e2e.sh | 47 +++++++++++++++++++++++++++++++++++++---- 2 files changed, 50 insertions(+), 10 deletions(-) diff --git a/.claude/commands/e2e.md b/.claude/commands/e2e.md index 7c81d7e3..a3da6ea2 100644 --- a/.claude/commands/e2e.md +++ b/.claude/commands/e2e.md @@ -6,7 +6,7 @@ allowed-tools: Bash(./scripts/e2e.sh:*), Bash(make e2e*), Bash(docker compose:*) # /e2e — Docker-only end-to-end Buttercup run (example-libpng) -This command exercises the full Buttercup pipeline on the [example-libpng](https://github.com/tob-challenges/example-libpng) challenge **using Docker only — no Kubernetes/minikube**. It uses the `dev/docker-compose/` stack with the **`compose.prebuilt.yaml` overlay** — every component runs from its prebuilt GHCR image (`ghcr.io/trailofbits/buttercup/*`, tag `main` by default), so **nothing is built locally**. A low LiteLLM budget (default **$3**) keeps an accidental run cheap. +This command exercises the full Buttercup pipeline on the [example-libpng](https://github.com/tob-challenges/example-libpng) challenge **using Docker only — no Kubernetes/minikube**. It uses the `dev/docker-compose/` stack with the **`compose.prebuilt.yaml` overlay** — every component runs from its prebuilt GHCR image (`ghcr.io/trailofbits/buttercup/*`, tag `main` by default), so **nothing is built locally**. A LiteLLM budget cap (default **$10**) bounds the spend — a full run through patch generation costs roughly that; a lower cap stops the pipeline before patch/bundle, so `--budget 3` only exercises up to seed-gen. > **Image tag:** defaults to `main`. Override with `--image-tag ` or `BUTTERCUP_IMAGE_TAG=...` to test a specific build. Private images require `docker login ghcr.io` first. > @@ -17,7 +17,7 @@ Mirrors the milestones in `.github/workflows/system-integration.yml`, but tails ## What it does 1. Checks for `docker`, `docker compose`, `curl`, and at least one LLM provider key (`ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or `GEMINI_API_KEY`) in your env (or already saved in `dev/docker-compose/.env`). -2. Writes `dev/docker-compose/.env` with the provider keys and `LITELLM_MAX_BUDGET=$BUDGET` (default `3`). +2. Writes `dev/docker-compose/.env` with the provider keys and `LITELLM_MAX_BUDGET=$BUDGET` (default `10`). The submitted task's `duration` defaults to `7200`s (2h) — the CRS discards a task's work once its deadline passes, and the full pipeline can exceed 30 min, so a short duration would expire mid-patch. 3. Pulls the prebuilt component images (`docker compose -f compose.yaml -f compose.prebuilt.yaml pull`, skippable with `--no-pull`) and starts every service (redis, dind, litellm, task-server, task-downloader, scheduler, program-model, build-bot, fuzzer-bot, coverage-bot, tracer-bot, seed-gen, patcher, buttercup-ui). No local image build. 4. POSTs the canned libpng `trigger_task` payload to `http://localhost:31323/webhook/trigger_task`. 5. Waits, in order, for these scheduler/seed-gen log markers: @@ -26,7 +26,7 @@ Mirrors the milestones in `.github/workflows/system-integration.yml`, but tails - `Updated POV status. New status PASSED` — POV accepted by competition API - `Copied N files to corpus` — seed-gen produced seeds - `Appending patch for task` — patch generated - - approves the patch via `POST /v1/task//patch//approve` + - polls for the `competition_patch_id=` summary line (logged only after the scheduler builds, verifies and submits the patch — minutes after the patch is generated), then approves via `POST /v1/task//patch//approve` - `Patch passed` — patch accepted - `bundle_id=` — bundle submitted 6. Prints a colored summary and tears the stack down with `docker compose down -v`. @@ -36,15 +36,16 @@ Mirrors the milestones in `.github/workflows/system-integration.yml`, but tails The driver is `scripts/e2e.sh`. The `Makefile` exposes `make e2e`. ```bash -# Plain run with the $3 budget default +# Plain run with the $10 budget / 7200s task-duration defaults make e2e # Pass flags through the Makefile -make e2e E2E_ARGS="--budget 5 --no-pull" +make e2e E2E_ARGS="--budget 15 --no-pull" # Or call the script directly -./scripts/e2e.sh --budget 3 --task-duration 1800 +./scripts/e2e.sh --budget 10 --task-duration 7200 ./scripts/e2e.sh --image-tag my-branch --no-pull # run already-present images +./scripts/e2e.sh --budget 3 # cheap: only reaches ~seed-gen ``` The script writes/overwrites `dev/docker-compose/.env` on each run. diff --git a/scripts/e2e.sh b/scripts/e2e.sh index 5b08e428..84f93799 100755 --- a/scripts/e2e.sh +++ b/scripts/e2e.sh @@ -23,8 +23,17 @@ COMPOSE_DIR="${REPO_ROOT}/dev/docker-compose" ENV_FILE="${COMPOSE_DIR}/.env" # Defaults — overridable via flags or environment. -BUDGET="${LITELLM_MAX_BUDGET:-3}" -TASK_DURATION="${E2E_TASK_DURATION:-1800}" +# +# BUDGET: a full run through patch generation costs ~$10 of LLM spend; $3 is +# exhausted during/just after POV, so anything past seed-gen would always time +# out. Default to 10 so the whole pipeline (incl. patch+bundle) is reachable. +# +# TASK_DURATION: the CRS discards a task's work once its deadline passes. On +# normal hardware build->POV->seed-gen->patch exceeds 30 min, so an 1800s task +# expires mid-patch ("task expired/cancelled? Will discard") and never reaches +# patch/bundle. Default to 7200 (2h) so the task outlives the pipeline. +BUDGET="${LITELLM_MAX_BUDGET:-10}" +TASK_DURATION="${E2E_TASK_DURATION:-7200}" # Prebuilt GHCR images instead of local builds (compose.prebuilt.yaml overlay). IMAGE_TAG="${BUTTERCUP_IMAGE_TAG:-main}" @@ -324,6 +333,33 @@ capture_line() { | grep -E "$pattern" | head -n1 || true } +# wait_capture SERVICE PATTERN TIMEOUT_SEC LABEL +# +# Like capture_line, but polls until the pattern appears or TIMEOUT_SEC +# elapses, echoing the first matching line on stdout (empty on timeout). +# Progress goes to stderr so stdout stays just the captured line. +# +# Needed because `competition_patch_id=` is logged by the scheduler only +# *after* it builds, verifies and submits the patch — minutes after the +# "Appending patch for task" milestone. A one-shot capture right after that +# milestone always races and loses, so approval would always be skipped. +wait_capture() { + local service="$1" pattern="$2" timeout="$3" label="$4" + local deadline=$(( $(date +%s) + timeout )) + log "Waiting to capture: ${label} ${C_DIM}(service=${service}, timeout=${timeout}s)${C_RST}" >&2 + while [[ $(date +%s) -lt $deadline ]]; do + local match + match="$(dc logs --no-color --no-log-prefix --tail=all "$service" 2>/dev/null \ + | grep -m1 -E "$pattern" || true)" + if [[ -n "$match" ]]; then + printf '%s\n' "$match" + return 0 + fi + sleep 15 + done + return 1 +} + ############################################################################### # Walk through the pipeline ############################################################################### @@ -375,8 +411,11 @@ else fi # Approve the patch (the local UI requires explicit approval, unlike scored -# rounds where it is automatic). -PATCH_LINE="$(capture_line scheduler 'competition_patch_id=')" +# rounds where it is automatic). competition_patch_id= only appears once the +# scheduler has built+verified+submitted the patch, well after the patch was +# generated, so poll for it rather than capturing once (which always races). +PATCH_LINE="$(wait_capture scheduler 'competition_patch_id=[0-9a-fA-F-]' \ + "$MILESTONE_TIMEOUT" "competition_patch_id (for approval)" || true)" if [[ -n "$PATCH_LINE" ]]; then PATCH_ID=$(printf '%s' "$PATCH_LINE" | sed -n 's/.*competition_patch_id=\([^ ]*\).*/\1/p') # Task id is inside the first [...] block, after the last ':'.