trailofbits-buttercup/.github/workflows/integration.yml

name: System Integration Tests

on:
  # Daily schedule - 3 AM UTC
  schedule:
    - cron: "0 3 * * *"
  # Pull requests with 'full-integration' label
  pull_request:
    types: [labeled, synchronize]
  # Manual trigger
  workflow_dispatch:
    inputs:
      trigger_task_data:
        description: "competition-api webhook/trigger_task data (JSON)"
        required: true
        default: '{"challenge_repo_url": "git@github.com:tob-challenges/example-libpng.git", "challenge_repo_base_ref": "0cc367aaeaac3f888f255cee5d394968996f736e", "challenge_repo_head_ref": "fdacd5a1dcff42175117d674b0fda9f8a005ae88", "fuzz_tooling_url": "git@github.com:tob-challenges/oss-fuzz-aixcc.git", "fuzz_tooling_ref": "d5fbd68fca66e6fa4f05899170d24e572b01853d", "fuzz_tooling_project_name": "libpng", "duration": 7200}'
      build_timeout:
        description: "Timeout for fuzzer build in minutes"
        required: false
        default: ""
      vuln_timeout:
        description: "Timeout for vuln submission in minutes"
        required: false
        default: ""
      patch_timeout:
        description: "Timeout for patch submission in minutes"
        required: false
        default: ""
      seed_gen_timeout:
        description: "Timeout for seed-gen submission in minutes"
        required: false
        default: ""
      bundle_timeout:
        description: "Timeout for bundle submission in minutes"
        required: false
        default: ""
      sarif_timeout:
        description: "Timeout for SARIF submission in minutes"
        required: false
        default: ""
      merger_timeout:
        description: "Timeout for merger bot in minutes"
        required: false
        default: ""

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

env:
  FUZZER_BUILD_TIMEOUT: ${{ inputs.build_timeout || 25 }}
  VULN_TIMEOUT: ${{ inputs.vuln_timeout || 25 }}
  PATCH_TIMEOUT: ${{ inputs.patch_timeout || 25 }}
  BUNDLE_TIMEOUT: ${{ inputs.bundle_timeout || 5 }}
  SARIF_TIMEOUT: ${{ inputs.sarif_timeout || 5 }}
  SEED_GEN_TIMEOUT: ${{ inputs.seed_gen_timeout || 25 }}
  MERGER_TIMEOUT: ${{ inputs.merger_timeout || 10 }}

permissions:
  contents: read
  packages: write

jobs:
  integration:
    # Only run on PRs if they have the 'full-integration' label
    if: |
      github.event_name == 'schedule' ||
      github.event_name != 'pull_request' ||
      contains(github.event.pull_request.labels.*.name, 'full-integration')
    runs-on: gha-ubuntu-8
    steps:
      - uses: actions/checkout@v5
        with:
          persist-credentials: false
          submodules: true

      - name: Set BUTTERCUP_NAMESPACE for PRs
        if: github.event_name == 'pull_request'
        run: |
          pull_number=$(jq --raw-output .pull_request.number "$GITHUB_EVENT_PATH")
          export BUTTERCUP_NAMESPACE=pr-${pull_number}-${{ github.run_number }}
          echo "BUTTERCUP_NAMESPACE=${BUTTERCUP_NAMESPACE}" >> $GITHUB_ENV

      - name: Set BUTTERCUP_NAMESPACE for branch
        if: github.event_name == 'push' || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule'
        run: |
          export BUTTERCUP_NAMESPACE=${GITHUB_REF_NAME/\//-}-${{ github.run_number }}
          echo "BUTTERCUP_NAMESPACE=${BUTTERCUP_NAMESPACE}" >> $GITHUB_ENV

      - name: Configure env file for minikube
        run: |
          cp ../.github/ci-env.template env

          sed -i "s|BUTTERCUP_NAMESPACE=.*|BUTTERCUP_NAMESPACE=${{ env.BUTTERCUP_NAMESPACE }}|" env
          sed -i "s|OPENAI_API_KEY=.*|OPENAI_API_KEY=${{ secrets.OPENAI_API_KEY }}|" env
          sed -i "s|ANTHROPIC_API_KEY=.*|ANTHROPIC_API_KEY=${{ secrets.ANTHROPIC_API_KEY }}|" env
          sed -i "s|GEMINI_API_KEY=.*|GEMINI_API_KEY=${{ secrets.GEMINI_API_KEY }}|" env
          sed -i "s|LANGFUSE_HOST=.*|LANGFUSE_HOST=${{ secrets.LANGFUSE_HOST }}|" env
          sed -i "s|LANGFUSE_PUBLIC_KEY=.*|LANGFUSE_PUBLIC_KEY=${{ secrets.LANGFUSE_PUBLIC_KEY }}|" env
          sed -i "s|LANGFUSE_SECRET_KEY=.*|LANGFUSE_SECRET_KEY=${{ secrets.LANGFUSE_SECRET_KEY }}|" env
          # Base64 encoding is required for Docker registry authentication (not for security, but for format)
          sed -i "s|GHCR_AUTH=.*|GHCR_AUTH=$(echo "USERNAME:${{ secrets.GH_TOKEN }}" | base64 --wrap=0)|" env
          sed -i "s|OTEL_ENDPOINT=.*|OTEL_ENDPOINT=${{ secrets.OTEL_ENDPOINT }}|" env

        working-directory: deployment

      - name: Run CRS
        run: make deploy-local

      - name: Submit custom task to the CRS
        if: github.event_name == 'workflow_dispatch'
        run: |
          kubectl port-forward -n $BUTTERCUP_NAMESPACE service/buttercup-ui 31323:1323 &
          sleep 5
          ./orchestrator/scripts/custom_task_crs.sh "$DATA"
          sleep 5
        env:
          DATA: ${{ inputs.trigger_task_data }}

      - name: Submit example-libpng task to the CRS
        if: github.event_name != 'workflow_dispatch'
        run: |
          kubectl port-forward -n $BUTTERCUP_NAMESPACE service/buttercup-ui 31323:1323 &
          sleep 5
          ./orchestrator/scripts/task_crs.sh
          sleep 5

      - name: Wait for fuzzer build processing
        timeout-minutes: ${{ fromJSON(env.FUZZER_BUILD_TIMEOUT) }}
        run: |
          while ! kubectl logs -n $BUTTERCUP_NAMESPACE -l app=scheduler --tail=-1 | grep "buttercup.orchestrator.scheduler.scheduler - INFO - .* Processing build output for type FUZZER"; do
            sleep 60
          done

      - name: Wait for vuln to be found
        timeout-minutes: ${{ fromJSON(env.VULN_TIMEOUT) }}
        run: |
          while ! kubectl logs -n $BUTTERCUP_NAMESPACE -l app=scheduler --tail=-1 | grep "] POV submission response: pov_id=" ; do
            sleep 60
          done
      - name: Wait for vuln to enter the passed state in competition api
        timeout-minutes: ${{ fromJSON(env.VULN_TIMEOUT) }}
        run: |
          while ! kubectl logs -n $BUTTERCUP_NAMESPACE -l app=scheduler --tail=-1 | grep "Updated POV status. New status PASSED" ; do
            sleep 60
          done

      - name: Wait for seed-gen to submit at least 1 seed
        timeout-minutes: ${{ fromJSON(env.SEED_GEN_TIMEOUT) }}
        run: |
          while ! kubectl logs -n $BUTTERCUP_NAMESPACE -l app=seed-gen -c seed-gen --tail=-1 | grep "Copied [1-9][0-9]* files to corpus"; do
            sleep 60
          done

      - name: Wait for merger-bot to submit files to remote corpus and prune local corpus
        timeout-minutes: ${{ fromJSON(env.MERGER_TIMEOUT) }}
        run: |
          while ! kubectl logs -n $BUTTERCUP_NAMESPACE -l app=merger-bot -c merger-bot --tail=-1  | grep "Synced [1-9][0-9]* files that add coverage to remote corpus"; do
            sleep 60
          done
          while ! kubectl logs -n $BUTTERCUP_NAMESPACE -l app=merger-bot -c merger-bot --tail=-1  | grep "Removed [1-9][0-9]* files from local corpus .* that don't add coverage"; do
            sleep 60
          done

      - name: Wait for patch to be recorded
        timeout-minutes: ${{ fromJSON(env.PATCH_TIMEOUT) }}
        run: |
          while ! kubectl logs -n $BUTTERCUP_NAMESPACE -l app=scheduler --tail=-1 | grep "Appending patch for task"; do
            sleep 60
          done
      - name: Wait for patch to enter the passed state in competition api
        timeout-minutes: ${{ fromJSON(env.PATCH_TIMEOUT) }}
        run: |
          while ! kubectl logs -n $BUTTERCUP_NAMESPACE -l app=scheduler --tail=-1 | grep "Patch passed"; do
            sleep 60
          done

      - name: Wait for bundle to be submitted
        timeout-minutes: ${{ fromJSON(env.BUNDLE_TIMEOUT) }}
        run: |
          while ! kubectl logs -n $BUTTERCUP_NAMESPACE -l app=scheduler --tail=-1 | grep "Bundle submission response: bundle_id="; do
            sleep 60
          done

      - name: Send a SARIF broadcast
        timeout-minutes: ${{ fromJSON(env.SARIF_TIMEOUT) }}
        run: |
          TASK_ID=$(kubectl logs -n $BUTTERCUP_NAMESPACE -l app=scheduler --tail=-1 | grep "Submitting bundle for harness" | grep -o "\[[^]]*\]" | grep -o "[^[]*$" | cut -d: -f3 | tr -d ']')
          echo "Task ID: $TASK_ID"
          while true; do
            # Kill any existing port-forward processes
            pkill -f "kubectl port-forward.*buttercup-competition-api" || true

            # Start port forwarding in background
            kubectl port-forward -n $BUTTERCUP_NAMESPACE service/buttercup-ui 31323:1323 &

            # Give port-forward time to establish
            sleep 5

            # Try to send SARIF report
            if ./orchestrator/scripts/send_sarif.sh $TASK_ID; then
              # Success - exit loop
              break
            fi

            # Failed - wait a bit before retrying
            sleep 10
          done
      - name: Wait for Bundle to be patched to include the SARIF
        timeout-minutes: ${{ fromJSON(env.SARIF_TIMEOUT) }}
        run: |
          while ! kubectl logs -n $BUTTERCUP_NAMESPACE -l app=scheduler --tail=-1 | grep "Bundle patch submission response: broadcast_sarif_id="; do
            sleep 60
          done
      - name: Wait for SARIF to be submitted as correct
        timeout-minutes: ${{ fromJSON(env.SARIF_TIMEOUT) }}
        run: |
          while ! kubectl logs -n $BUTTERCUP_NAMESPACE -l app=scheduler --tail=-1 | grep "Matching SARIF submission response"; do
            sleep 60
          done

      # Disable them for now because they can contain sensitive information
      # - name: Collect logs
      #   run: |
      #     ./collect-logs.sh
      #     mkdir -p docker_logs
      #     cp -rv crs_pod_logs_* docker_logs
      #   if: always()
      #   working-directory: deployment

      - name: Turn off CRS
        run: |
          make undeploy
          make clean-local
        if: always()

      # - name: Upload Docker logs
      #   uses: actions/upload-artifact@v4
      #   with:
      #     name: docker-logs
      #     path: deployment/docker_logs/
      #     retention-days: 4
      #   if: always()