mirror of
https://github.com/trailofbits/buttercup
synced 2026-06-21 14:11:39 +00:00
Henrik Brodin
a3d805cb35
* ci: shell-quote env values for system integration tests Two bugs in the env file step prevented the weekly System Integration Tests from ever passing since #427 (2026-01-26): 1. The deletion regex `/^KEY=/` did not match the template's `export KEY=replace-me` lines, leaving stale `replace-me` placeholders in the file alongside the appended real values. 2. Appended values were not shell-quoted, so `source ./env` in deployment/crs-architecture.sh treated whitespace and metacharacters as shell syntax. The `OTEL_TOKEN` secret (a `Bearer <token>` header) tripped this with `command not found` on the token component. The same gap was a code-execution sink: a secret containing `$(...)` or backticks would execute on the runner at source time. Fix: strip with `(export[[:space:]]+)?` so template lines are actually removed, and write with `printf 'export %s=%q\n'` so every value is shell-escaped — defending against both whitespace and injection. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com> * ci: drop placeholder block from ci-env.template The placeholder lines (`export KEY=replace-me`) only existed because the pre-rewrite step substituted into them with `sed -i "s|KEY=.*|...|"`. The new `printf %q` write path doesn't need them, so removing them at the source eliminates the entire delete-then-rewrite dance. Drops `strip_var` and the for-loop. The workflow step is now: copy the static template, then append shell-quoted assignments for every secret the env: block injects. GHCR_AUTH is no longer defined at all, so the `[ -n "$GHCR_AUTH" ]` warning branch in crs-architecture.sh:63 fires as intended. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
280 lines
12 KiB
YAML
280 lines
12 KiB
YAML
name: System Integration Tests
|
|
|
|
on:
|
|
# Daily schedule - 3 AM UTC
|
|
schedule:
|
|
- cron: "0 3 * * 0"
|
|
# Pull requests with 'full-integration' label
|
|
pull_request:
|
|
types: [labeled, synchronize]
|
|
# Manual trigger
|
|
workflow_dispatch:
|
|
inputs:
|
|
trigger_task_data:
|
|
description: "competition-api webhook/trigger_task data (JSON)"
|
|
required: true
|
|
default: '{"challenge_repo_url": "git@github.com:tob-challenges/example-libpng.git", "challenge_repo_base_ref": "0cc367aaeaac3f888f255cee5d394968996f736e", "challenge_repo_head_ref": "fdacd5a1dcff42175117d674b0fda9f8a005ae88", "fuzz_tooling_url": "git@github.com:tob-challenges/oss-fuzz-aixcc.git", "fuzz_tooling_ref": "d5fbd68fca66e6fa4f05899170d24e572b01853d", "fuzz_tooling_project_name": "libpng", "duration": 7200}'
|
|
build_timeout:
|
|
description: "Timeout for fuzzer build in minutes"
|
|
required: false
|
|
default: ""
|
|
vuln_timeout:
|
|
description: "Timeout for vuln submission in minutes"
|
|
required: false
|
|
default: ""
|
|
patch_timeout:
|
|
description: "Timeout for patch submission in minutes"
|
|
required: false
|
|
default: ""
|
|
seed_gen_timeout:
|
|
description: "Timeout for seed-gen submission in minutes"
|
|
required: false
|
|
default: ""
|
|
bundle_timeout:
|
|
description: "Timeout for bundle submission in minutes"
|
|
required: false
|
|
default: ""
|
|
sarif_timeout:
|
|
description: "Timeout for SARIF submission in minutes"
|
|
required: false
|
|
default: ""
|
|
merger_timeout:
|
|
description: "Timeout for merger bot in minutes"
|
|
required: false
|
|
default: ""
|
|
|
|
concurrency:
|
|
group: ${{ github.workflow }}-${{ github.ref }}
|
|
cancel-in-progress: true
|
|
|
|
env:
|
|
FUZZER_BUILD_TIMEOUT: ${{ inputs.build_timeout || 25 }}
|
|
VULN_TIMEOUT: ${{ inputs.vuln_timeout || 25 }}
|
|
PATCH_TIMEOUT: ${{ inputs.patch_timeout || 25 }}
|
|
BUNDLE_TIMEOUT: ${{ inputs.bundle_timeout || 5 }}
|
|
SARIF_TIMEOUT: ${{ inputs.sarif_timeout || 5 }}
|
|
SEED_GEN_TIMEOUT: ${{ inputs.seed_gen_timeout || 25 }}
|
|
MERGER_TIMEOUT: ${{ inputs.merger_timeout || 10 }}
|
|
|
|
permissions:
|
|
contents: read
|
|
|
|
jobs:
|
|
integration:
|
|
# Only run on PRs if they have the 'full-integration' label
|
|
if: |
|
|
github.event_name == 'schedule' ||
|
|
github.event_name != 'pull_request' ||
|
|
contains(github.event.pull_request.labels.*.name, 'full-integration')
|
|
runs-on: gha-ubuntu-8
|
|
steps:
|
|
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
|
with:
|
|
persist-credentials: false
|
|
submodules: true
|
|
|
|
- name: Set BUTTERCUP_NAMESPACE for PRs
|
|
if: github.event_name == 'pull_request'
|
|
run: |
|
|
pull_number=$(jq --raw-output .pull_request.number "$GITHUB_EVENT_PATH")
|
|
export BUTTERCUP_NAMESPACE="pr-${pull_number}-${{ github.run_number }}"
|
|
echo "BUTTERCUP_NAMESPACE=${BUTTERCUP_NAMESPACE}" >> "$GITHUB_ENV"
|
|
|
|
- name: Set BUTTERCUP_NAMESPACE for branch
|
|
if: github.event_name == 'push' || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule'
|
|
run: |
|
|
export BUTTERCUP_NAMESPACE="${GITHUB_REF_NAME/\//-}-${{ github.run_number }}"
|
|
echo "BUTTERCUP_NAMESPACE=${BUTTERCUP_NAMESPACE}" >> "$GITHUB_ENV"
|
|
|
|
- name: Configure env file for minikube
|
|
env:
|
|
BUTTERCUP_NS: ${{ env.BUTTERCUP_NAMESPACE }}
|
|
OPENAI_KEY: ${{ secrets.OPENAI_API_KEY }}
|
|
ANTHROPIC_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
|
|
GEMINI_KEY: ${{ secrets.GEMINI_API_KEY }}
|
|
LF_HOST: ${{ secrets.LANGFUSE_HOST }}
|
|
LF_PUBLIC_KEY: ${{ secrets.LANGFUSE_PUBLIC_KEY }}
|
|
LF_SECRET_KEY: ${{ secrets.LANGFUSE_SECRET_KEY }}
|
|
OTEL_EP: ${{ secrets.OTEL_ENDPOINT }}
|
|
OTEL_TK: ${{ secrets.OTEL_TOKEN }}
|
|
run: |
|
|
# The env file is sourced by deployment/crs-architecture.sh, so every
|
|
# value must be shell-quoted. printf %q produces a form bash can
|
|
# re-parse as the literal string, defending against shell injection
|
|
# from secret contents (whitespace, $(), backticks, quotes, etc.).
|
|
write_var() {
|
|
printf 'export %s=%q\n' "$1" "$2" >> ./env
|
|
}
|
|
|
|
cp ../.github/ci-env.template ./env
|
|
write_var BUTTERCUP_NAMESPACE "$BUTTERCUP_NS"
|
|
write_var OPENAI_API_KEY "$OPENAI_KEY"
|
|
write_var ANTHROPIC_API_KEY "$ANTHROPIC_KEY"
|
|
write_var GEMINI_API_KEY "$GEMINI_KEY"
|
|
write_var LANGFUSE_HOST "$LF_HOST"
|
|
write_var LANGFUSE_PUBLIC_KEY "$LF_PUBLIC_KEY"
|
|
write_var LANGFUSE_SECRET_KEY "$LF_SECRET_KEY"
|
|
write_var OTEL_ENDPOINT "$OTEL_EP"
|
|
write_var OTEL_TOKEN "$OTEL_TK"
|
|
working-directory: deployment
|
|
|
|
- name: Run CRS
|
|
run: FORCE=true make deploy
|
|
|
|
- name: Submit custom task to the CRS
|
|
if: github.event_name == 'workflow_dispatch'
|
|
run: |
|
|
kubectl port-forward -n "$BUTTERCUP_NAMESPACE" service/buttercup-ui 31323:1323 &
|
|
sleep 5
|
|
./orchestrator/scripts/custom_task_crs.sh "$DATA"
|
|
sleep 5
|
|
env:
|
|
DATA: ${{ inputs.trigger_task_data }}
|
|
|
|
- name: Submit example-libpng task to the CRS
|
|
if: github.event_name != 'workflow_dispatch'
|
|
run: |
|
|
kubectl port-forward -n "$BUTTERCUP_NAMESPACE" service/buttercup-ui 31323:1323 &
|
|
sleep 5
|
|
./orchestrator/scripts/task_crs.sh
|
|
sleep 5
|
|
|
|
- name: Wait for fuzzer build processing
|
|
timeout-minutes: ${{ fromJSON(env.FUZZER_BUILD_TIMEOUT) }}
|
|
run: |
|
|
while ! kubectl logs -n "$BUTTERCUP_NAMESPACE" -l app=scheduler --tail=-1 | grep "buttercup.orchestrator.scheduler.scheduler - INFO - .* Processing build output for type FUZZER"; do
|
|
sleep 60
|
|
done
|
|
|
|
- name: Wait for vuln to be found
|
|
timeout-minutes: ${{ fromJSON(env.VULN_TIMEOUT) }}
|
|
run: |
|
|
while ! kubectl logs -n "$BUTTERCUP_NAMESPACE" -l app=scheduler --tail=-1 | grep "] POV submission response: pov_id=" ; do
|
|
sleep 60
|
|
done
|
|
- name: Wait for vuln to enter the passed state in competition api
|
|
timeout-minutes: ${{ fromJSON(env.VULN_TIMEOUT) }}
|
|
run: |
|
|
while ! kubectl logs -n "$BUTTERCUP_NAMESPACE" -l app=scheduler --tail=-1 | grep "Updated POV status. New status PASSED" ; do
|
|
sleep 60
|
|
done
|
|
|
|
- name: Wait for seed-gen to submit at least 1 seed
|
|
timeout-minutes: ${{ fromJSON(env.SEED_GEN_TIMEOUT) }}
|
|
run: |
|
|
while ! kubectl logs -n "$BUTTERCUP_NAMESPACE" -l app=seed-gen -c seed-gen --tail=-1 | grep "Copied [1-9][0-9]* files to corpus"; do
|
|
sleep 60
|
|
done
|
|
|
|
- name: Wait for merger-bot to submit files to remote corpus and prune local corpus
|
|
timeout-minutes: ${{ fromJSON(env.MERGER_TIMEOUT) }}
|
|
run: |
|
|
while ! kubectl logs -n "$BUTTERCUP_NAMESPACE" -l app=merger-bot -c merger-bot --tail=-1 | grep "Synced [1-9][0-9]* files that add coverage to remote corpus"; do
|
|
sleep 60
|
|
done
|
|
while ! kubectl logs -n "$BUTTERCUP_NAMESPACE" -l app=merger-bot -c merger-bot --tail=-1 | grep "Removed [1-9][0-9]* files from local corpus .* that don't add coverage"; do
|
|
sleep 60
|
|
done
|
|
|
|
- name: Wait for patch to be recorded
|
|
timeout-minutes: ${{ fromJSON(env.PATCH_TIMEOUT) }}
|
|
run: |
|
|
while ! kubectl logs -n "$BUTTERCUP_NAMESPACE" -l app=scheduler --tail=-1 | grep "Appending patch for task"; do
|
|
sleep 60
|
|
done
|
|
- name: Approve the patch in the buttercup-ui
|
|
run: |
|
|
# Wait until a log line with competition_patch_id= is available, then extract PATCH_ID and TASK_ID from that line
|
|
while true; do
|
|
PATCH_LINE=$(kubectl logs -n "$BUTTERCUP_NAMESPACE" -l app=scheduler --tail=-1 | grep "competition_patch_id=" | head -n 1)
|
|
if [ -n "$PATCH_LINE" ]; then
|
|
break
|
|
fi
|
|
echo "Waiting for PATCH_ID to be available..."
|
|
sleep 10
|
|
done
|
|
echo "Patch log line: $PATCH_LINE"
|
|
# Extract PATCH_ID from the log line (after 'competition_patch_id=' and before any space or end of line)
|
|
PATCH_ID=$(echo "$PATCH_LINE" | sed -n 's/.*competition_patch_id=\([^ ]*\).*/\1/p')
|
|
# Extract TASK_ID from the log line (between the first '[' and the first ']'), then after the first ':' (if present)
|
|
TASK_ID=$(echo "$PATCH_LINE" | sed -n 's/.*\[\([^]]*\)\].*/\1/p' | sed 's/^[^:]*://')
|
|
echo "Patch ID: $PATCH_ID"
|
|
echo "Task ID: $TASK_ID"
|
|
kubectl port-forward -n "$BUTTERCUP_NAMESPACE" service/buttercup-ui 31323:1323 &
|
|
sleep 5
|
|
curl -X POST "http://localhost:31323/v1/task/${TASK_ID}/patch/${PATCH_ID}/approve"
|
|
sleep 5
|
|
- name: Wait for patch to enter the passed state in competition api
|
|
timeout-minutes: ${{ fromJSON(env.PATCH_TIMEOUT) }}
|
|
run: |
|
|
while ! kubectl logs -n "$BUTTERCUP_NAMESPACE" -l app=scheduler --tail=-1 | grep "Patch passed"; do
|
|
sleep 60
|
|
done
|
|
|
|
- name: Wait for bundle to be submitted
|
|
timeout-minutes: ${{ fromJSON(env.BUNDLE_TIMEOUT) }}
|
|
run: |
|
|
while ! kubectl logs -n "$BUTTERCUP_NAMESPACE" -l app=scheduler --tail=-1 | grep "Bundle submission response: bundle_id="; do
|
|
sleep 60
|
|
done
|
|
|
|
- name: Send a SARIF broadcast
|
|
timeout-minutes: ${{ fromJSON(env.SARIF_TIMEOUT) }}
|
|
run: |
|
|
TASK_ID=$(kubectl logs -n "$BUTTERCUP_NAMESPACE" -l app=scheduler --tail=-1 | grep "Submitting bundle for harness" | grep -o "\[[^]]*\]" | grep -o "[^[]*$" | cut -d: -f3 | tr -d ']')
|
|
echo "Task ID: $TASK_ID"
|
|
while true; do
|
|
# Kill any existing port-forward processes
|
|
pkill -f "kubectl port-forward.*buttercup-competition-api" || true
|
|
|
|
# Start port forwarding in background
|
|
kubectl port-forward -n "$BUTTERCUP_NAMESPACE" service/buttercup-ui 31323:1323 &
|
|
|
|
# Give port-forward time to establish
|
|
sleep 5
|
|
|
|
# Try to send SARIF report
|
|
if ./orchestrator/scripts/send_sarif.sh "$TASK_ID"; then
|
|
# Success - exit loop
|
|
break
|
|
fi
|
|
|
|
# Failed - wait a bit before retrying
|
|
sleep 10
|
|
done
|
|
- name: Wait for Bundle to be patched to include the SARIF
|
|
timeout-minutes: ${{ fromJSON(env.SARIF_TIMEOUT) }}
|
|
run: |
|
|
while ! kubectl logs -n "$BUTTERCUP_NAMESPACE" -l app=scheduler --tail=-1 | grep "Bundle patch submission response: broadcast_sarif_id="; do
|
|
sleep 60
|
|
done
|
|
- name: Wait for SARIF to be submitted as correct
|
|
timeout-minutes: ${{ fromJSON(env.SARIF_TIMEOUT) }}
|
|
run: |
|
|
while ! kubectl logs -n "$BUTTERCUP_NAMESPACE" -l app=scheduler --tail=-1 | grep "Matching SARIF submission response"; do
|
|
sleep 60
|
|
done
|
|
|
|
# Disable them for now because they can contain sensitive information
|
|
# - name: Collect logs
|
|
# run: |
|
|
# ./collect-logs.sh
|
|
# mkdir -p docker_logs
|
|
# cp -rv crs_pod_logs_* docker_logs
|
|
# if: always()
|
|
# working-directory: deployment
|
|
|
|
- name: Turn off CRS
|
|
run: |
|
|
make undeploy
|
|
make clean-local
|
|
if: always()
|
|
|
|
# - name: Upload Docker logs
|
|
# uses: actions/upload-artifact@v4
|
|
# with:
|
|
# name: docker-logs
|
|
# path: deployment/docker_logs/
|
|
# retention-days: 4
|
|
# if: always()
|