From caabb07dd59a256af5e5c2345f686d38046ed3f3 Mon Sep 17 00:00:00 2001 From: Riccardo Schirone <562321+ret2libc@users.noreply.github.com> Date: Tue, 22 Apr 2025 13:51:28 +0200 Subject: [PATCH] ci: allow to customize workflow dispatch (#427) --- .github/ci-env.template | 1 + .github/workflows/integration.yml | 49 +++++++++++++++++++++--- deployment/crs-architecture.sh | 1 + deployment/env.template | 4 ++ deployment/k8s/templates/common-env.yaml | 9 +++++ deployment/k8s/values-aks.template | 1 + deployment/k8s/values-ci.template | 1 + deployment/k8s/values-minikube.template | 1 + deployment/k8s/values-prod.template | 1 + deployment/k8s/values.yaml | 1 + orchestrator/scripts/custom_task_crs.sh | 2 + 11 files changed, 66 insertions(+), 5 deletions(-) create mode 100755 orchestrator/scripts/custom_task_crs.sh diff --git a/.github/ci-env.template b/.github/ci-env.template index 66de8dea..8b71ee05 100644 --- a/.github/ci-env.template +++ b/.github/ci-env.template @@ -23,3 +23,4 @@ export LANGFUSE_PUBLIC_KEY=replace-me export LANGFUSE_SECRET_KEY=replace-me export DOCKER_USERNAME=replace-me export DOCKER_PAT=replace-me +export FUZZ_TOOLING_CONTAINER_ORG=replace-me diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index fd96a9b5..8e3fd30e 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -9,6 +9,33 @@ on: branches: - "*" workflow_dispatch: + inputs: + trigger_task_data: + description: "competition-api webhook/trigger_task data (JSON)" + required: true + default: "{\"challenge_repo_url\": \"git@github.com:aixcc-finals/example-libpng.git\", \"challenge_repo_base_ref\": \"0cc367aaeaac3f888f255cee5d394968996f736e\", \"challenge_repo_head_ref\": \"fdacd5a1dcff42175117d674b0fda9f8a005ae88\", \"fuzz_tooling_url\": \"git@github.com:aixcc-finals/oss-fuzz-aixcc.git\", \"fuzz_tooling_ref\": \"d5fbd68fca66e6fa4f05899170d24e572b01853d\", \"fuzz_tooling_project_name\": \"libpng\", \"duration\": 7200}" + fuzz_tooling_container_org: + description: "Org-part of the container image for the fuzz-tooling repo (default to aixcc-afc). Alternative is gcr.io/oss-fuzz" + required: false + default: "" + vuln_timeout: + description: "Timeout for vuln submission in minutes" + required: false + default: "" + patch_timeout: + description: "Timeout for patch submission in minutes" + required: false + default: "" + bundle_timeout: + description: "Timeout for bundle submission in minutes" + required: false + default: "" + +env: + VULN_TIMEOUT: ${{ inputs.vuln_timeout || 25 }} + PATCH_TIMEOUT: ${{ inputs.patch_timeout || 15 }} + BUNDLE_TIMEOUT: ${{ inputs.bundle_timeout || 5 }} + FUZZ_TOOLING_CONTAINER_ORG: ${{ inputs.fuzz_tooling_container_org || 'aixcc-afc' }} permissions: contents: read @@ -37,7 +64,7 @@ jobs: echo "BUTTERCUP_NAMESPACE=${BUTTERCUP_NAMESPACE}" >> $GITHUB_ENV - name: Set BUTTERCUP_NAMESPACE for branch - if: github.event_name == 'push' + if: github.event_name == 'push' || github.event_name == 'workflow_dispatch' run: | export BUTTERCUP_NAMESPACE=${GITHUB_REF_NAME/\//-}-${{ github.run_number }} echo "BUTTERCUP_NAMESPACE=${BUTTERCUP_NAMESPACE}" >> $GITHUB_ENV @@ -56,6 +83,7 @@ jobs: sed -i "s|DOCKER_USERNAME=.*|DOCKER_USERNAME=${{ secrets.DOCKER_USERNAME }}|" env sed -i "s|DOCKER_PAT=.*|DOCKER_PAT=${{ secrets.DOCKER_PAT }}|" env sed -i "s|SCANTRON_GITHUB_PAT=.*|SCANTRON_GITHUB_PAT=${{ secrets.SCANTRON_GITHUB_PAT }}|" env + sed -i "s|FUZZ_TOOLING_CONTAINER_ORG=.*|FUZZ_TOOLING_CONTAINER_ORG=${{ env.FUZZ_TOOLING_CONTAINER_ORG }}|" env working-directory: deployment @@ -80,7 +108,18 @@ jobs: sleep 3 done - - name: Submit a task to the CRS + - name: Submit a custom task to the CRS + if: github.event_name == 'workflow_dispatch' + run: | + kubectl port-forward -n $BUTTERCUP_NAMESPACE service/buttercup-competition-api 31323:1323 & + sleep 5 + ./orchestrator/scripts/custom_task_crs.sh "$DATA" + sleep 5 + env: + DATA: ${{ inputs.trigger_task_data }} + + - name: Submit example-libpng task to the CRS + if: github.event_name != 'workflow_dispatch' run: | kubectl port-forward -n $BUTTERCUP_NAMESPACE service/buttercup-competition-api 31323:1323 & sleep 5 @@ -88,19 +127,19 @@ jobs: sleep 5 - name: Wait for vuln submission to happen - timeout-minutes: 25 + timeout-minutes: ${{ fromJSON(env.VULN_TIMEOUT) }} run: | while ! kubectl logs -n $BUTTERCUP_NAMESPACE -l app=scheduler | grep "Updated PoV .* status to TypesSubmissionStatus.PASSED" ; do sleep 10 done - name: Wait for patch to be submitted - timeout-minutes: 15 + timeout-minutes: ${{ fromJSON(env.PATCH_TIMEOUT) }} run: | while ! kubectl logs -n $BUTTERCUP_NAMESPACE -l app=scheduler | grep "Updated patch .* status to TypesSubmissionStatus.PASSED"; do sleep 10 done - name: Wait for bundle to be submitted - timeout-minutes: 5 + timeout-minutes: ${{ fromJSON(env.BUNDLE_TIMEOUT) }} run: | while ! kubectl logs -n $BUTTERCUP_NAMESPACE -l app=scheduler | grep "Bundle accepted: task=.* vuln=.* patch=.* bundle_id=bundle_id='.*' status="; do sleep 10 diff --git a/deployment/crs-architecture.sh b/deployment/crs-architecture.sh index aeb5c402..c9b9ed13 100755 --- a/deployment/crs-architecture.sh +++ b/deployment/crs-architecture.sh @@ -32,6 +32,7 @@ fi BUTTERCUP_NAMESPACE=${BUTTERCUP_NAMESPACE:-crs} DEPLOY_CLUSTER=${DEPLOY_CLUSTER:-true} CLUSTER_TYPE=${CLUSTER_TYPE:-minikube} +FUZZ_TOOLING_CONTAINER_ORG=${FUZZ_TOOLING_CONTAINER_ORG:-aixcc-afc} if [ "$DEPLOY_CLUSTER" = "true" ] && [ "$CLUSTER_TYPE" = "aks" ]; then echo -e "${GRN}Current azure account status:${NC}" diff --git a/deployment/env.template b/deployment/env.template index 869c1baa..154dd639 100644 --- a/deployment/env.template +++ b/deployment/env.template @@ -72,3 +72,7 @@ export ANTHROPIC_API_KEY="" # Images from DockerHub. export DOCKER_USERNAME="" export DOCKER_PAT="" + +# Optionally modify the container image org part used for accessing challenges +# containers. Use aixcc-afc by default, but modify this to use upstream oss-fuzz. +# export FUZZ_TOOLING_CONTAINER_ORG="gcr.io/oss-fuzz" diff --git a/deployment/k8s/templates/common-env.yaml b/deployment/k8s/templates/common-env.yaml index 3029dd79..ac0c9ed1 100644 --- a/deployment/k8s/templates/common-env.yaml +++ b/deployment/k8s/templates/common-env.yaml @@ -86,6 +86,14 @@ Define common variables by categories value: "{{ int .Values.global.queueTimeouts.tracedVulnerabilitiesTaskTimeoutMs | default 120000 }}" {{- end }} +{{/* +Define OSS-Fuzz container organization environment variable +*/}} +{{- define "buttercup.env.ossFuzzContainerOrg" }} +- name: OSS_FUZZ_CONTAINER_ORG + value: "{{ .Values.global.ossFuzzContainerOrg | default "aixcc-afc" }}" +{{- end }} + {{/* Define common environment for all services */}} @@ -94,6 +102,7 @@ Define common environment for all services {{- include "buttercup.env.timeouts" . | nindent 0 }} {{- include "buttercup.env.dirs" . | nindent 0 }} {{- include "buttercup.env.nodeData" . | nindent 0 }} +{{- include "buttercup.env.ossFuzzContainerOrg" . | nindent 0 }} {{- end }} {{/* diff --git a/deployment/k8s/values-aks.template b/deployment/k8s/values-aks.template index 1a570574..6860133b 100644 --- a/deployment/k8s/values-aks.template +++ b/deployment/k8s/values-aks.template @@ -1,5 +1,6 @@ global: environment: "aks" + ossFuzzContainerOrg: "${FUZZ_TOOLING_CONTAINER_ORG}" langfuse: enabled: true host: "${LANGFUSE_HOST}" diff --git a/deployment/k8s/values-ci.template b/deployment/k8s/values-ci.template index 53ad4b2c..b39516bc 100644 --- a/deployment/k8s/values-ci.template +++ b/deployment/k8s/values-ci.template @@ -1,5 +1,6 @@ global: environment: "aks" + ossFuzzContainerOrg: "${FUZZ_TOOLING_CONTAINER_ORG}" orchestratorImage: tag: "${BUTTERCUP_NAMESPACE}" fuzzerImage: diff --git a/deployment/k8s/values-minikube.template b/deployment/k8s/values-minikube.template index 64de29fa..24f24925 100644 --- a/deployment/k8s/values-minikube.template +++ b/deployment/k8s/values-minikube.template @@ -1,5 +1,6 @@ global: environment: "minikube" + ossFuzzContainerOrg: "${FUZZ_TOOLING_CONTAINER_ORG}" orchestratorImage: repository: localhost/orchestrator tag: "latest" diff --git a/deployment/k8s/values-prod.template b/deployment/k8s/values-prod.template index 06b3eb2b..6ce41855 100644 --- a/deployment/k8s/values-prod.template +++ b/deployment/k8s/values-prod.template @@ -1,6 +1,7 @@ global: environment: "aks" + ossFuzzContainerOrg: "${FUZZ_TOOLING_CONTAINER_ORG}" orchestratorImage: tag: "v0.1.3" fuzzerImage: diff --git a/deployment/k8s/values.yaml b/deployment/k8s/values.yaml index 5350ac53..b1d90d04 100644 --- a/deployment/k8s/values.yaml +++ b/deployment/k8s/values.yaml @@ -2,6 +2,7 @@ global: # Set environment: "minikube" or "aks" environment: "minikube" + ossFuzzContainerOrg: "aixcc-afc" orchestratorImage: repository: ghcr.io/aixcc-finals/afc-crs-trail-of-bits/buttercup-orchestrator tag: "main" diff --git a/orchestrator/scripts/custom_task_crs.sh b/orchestrator/scripts/custom_task_crs.sh new file mode 100755 index 00000000..03e869ef --- /dev/null +++ b/orchestrator/scripts/custom_task_crs.sh @@ -0,0 +1,2 @@ +#!/bin/bash +curl -X 'POST' 'http://127.0.0.1:31323/webhook/trigger_task' -H 'Content-Type: application/json' -d "$1"