From e2e5e45ceede4946bfb41cafbe3416c494cec23a Mon Sep 17 00:00:00 2001 From: Ronald Eytchison <58823072+reytchison@users.noreply.github.com> Date: Thu, 10 Jul 2025 15:49:09 -0400 Subject: [PATCH] Build cscope image locally instead of pulling (#148) * Add cscope as submodule * Use cscope submodule instead of docker image * Pass token to checkout private submodule in CI --- .github/workflows/claude.yml | 3 +++ .github/workflows/docker.yml.disabled | 4 ++++ .github/workflows/integration.yml.disabled | 2 ++ .github/workflows/tests.yml | 4 ++-- .gitmodules | 3 +++ CLAUDE.md | 3 --- README.md | 2 ++ deployment/crs-architecture.sh | 10 ---------- deployment/env.template | 1 - external/aixcc-cscope | 1 + justfile | 9 +-------- patcher/Dockerfile | 5 +---- program-model/Dockerfile | 5 +---- program-model/README.md | 13 ------------- seed-gen/Dockerfile | 5 +---- 15 files changed, 21 insertions(+), 49 deletions(-) create mode 160000 external/aixcc-cscope diff --git a/.github/workflows/claude.yml b/.github/workflows/claude.yml index 15b4c5b2..44faa47b 100644 --- a/.github/workflows/claude.yml +++ b/.github/workflows/claude.yml @@ -28,6 +28,9 @@ jobs: uses: actions/checkout@v4 with: fetch-depth: 1 + persist-credentials: false + token: ${{ secrets.GH_CLONE_REPO_PAT }} + submodules: true - name: Run Claude Code id: claude diff --git a/.github/workflows/docker.yml.disabled b/.github/workflows/docker.yml.disabled index 94386a87..1d731ba2 100644 --- a/.github/workflows/docker.yml.disabled +++ b/.github/workflows/docker.yml.disabled @@ -45,6 +45,10 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v4 + with: + persist-credentials: false + token: ${{ secrets.GH_CLONE_REPO_PAT }} + submodules: true - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 diff --git a/.github/workflows/integration.yml.disabled b/.github/workflows/integration.yml.disabled index f97b08c7..cdaf831f 100644 --- a/.github/workflows/integration.yml.disabled +++ b/.github/workflows/integration.yml.disabled @@ -67,6 +67,8 @@ jobs: - uses: actions/checkout@v4 with: persist-credentials: false + token: ${{ secrets.GH_CLONE_REPO_PAT }} + submodules: true - name: Set BUTTERCUP_NAMESPACE for PRs if: github.event_name == 'pull_request' diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 3118bb8b..b649d8b2 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -36,6 +36,8 @@ jobs: - uses: actions/checkout@v4 with: persist-credentials: false + token: ${{ secrets.GH_CLONE_REPO_PAT }} + submodules: true - name: Install uv uses: astral-sh/setup-uv@v6 @@ -49,8 +51,6 @@ jobs: run: | sudo apt-get update sudo apt-get install -y codequery just ripgrep git-lfs - echo ${{ secrets.GITHUB_TOKEN }} | docker login ghcr.io -u ${{ github.actor }} --password-stdin - just download-cscope just install-cscope - name: Prepare environment diff --git a/.gitmodules b/.gitmodules index e69de29b..3a5c5b29 100644 --- a/.gitmodules +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "external/aixcc-cscope"] + path = external/aixcc-cscope + url = https://github.com/trailofbits/aixcc-cscope.git diff --git a/CLAUDE.md b/CLAUDE.md index 14946fd1..e819400a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -17,9 +17,6 @@ just lint-python-all # Format, lint, and type-check all components # Example: lint the patcher component just lint-python patcher - -# Download dependencies -just download-cscope # Download cscope static analysis tools ``` ### Testing diff --git a/README.md b/README.md index adce8298..9ccca2fe 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,8 @@ ## Dependencies +Clone the repo with `--recurse-submodules` as some dependencies are submodules. + Follow the install instructions for the required dependencies: * [Docker install guide](https://docs.docker.com/engine/install/ubuntu/) diff --git a/deployment/crs-architecture.sh b/deployment/crs-architecture.sh index 546365bc..bbd9a205 100755 --- a/deployment/crs-architecture.sh +++ b/deployment/crs-architecture.sh @@ -115,16 +115,6 @@ up() { FUZZER_BUILD_ARGS="" fi - if [ -n "$CSCOPE_IMAGE" ]; then - PATCHER_BUILD_ARGS="--build-arg CSCOPE_IMAGE=$CSCOPE_IMAGE" - SEED_GEN_BUILD_ARGS="--build-arg CSCOPE_IMAGE=$CSCOPE_IMAGE" - PROGRAM_MODEL_BUILD_ARGS="--build-arg CSCOPE_IMAGE=$CSCOPE_IMAGE" - else - PATCHER_BUILD_ARGS="" - SEED_GEN_BUILD_ARGS="" - PROGRAM_MODEL_BUILD_ARGS="" - fi - docker build $ORCHESTRATOR_BUILD_ARGS -f "$SCRIPT_DIR"/../orchestrator/Dockerfile -t localhost/orchestrator:latest "$SCRIPT_DIR"/.. docker build $FUZZER_BUILD_ARGS -f "$SCRIPT_DIR"/../fuzzer/dockerfiles/runner_image.Dockerfile -t localhost/fuzzer:latest "$SCRIPT_DIR"/.. docker build $SEED_GEN_BUILD_ARGS -f "$SCRIPT_DIR"/../seed-gen/Dockerfile -t localhost/seed-gen:latest "$SCRIPT_DIR"/.. diff --git a/deployment/env.template b/deployment/env.template index ed9b79b8..aa373f52 100644 --- a/deployment/env.template +++ b/deployment/env.template @@ -86,4 +86,3 @@ export DOCKER_PAT="" # Docker build arguments, useful for local deployment # By default these points to the aixcc-finals images # export FUZZER_BASE_IMAGE="gcr.io/oss-fuzz-base/base-runner" -# export CSCOPE_IMAGE="ghcr.io/trailofbits/buttercup-cscope:main" diff --git a/external/aixcc-cscope b/external/aixcc-cscope new file mode 160000 index 00000000..d814aabe --- /dev/null +++ b/external/aixcc-cscope @@ -0,0 +1 @@ +Subproject commit d814aabe8558d2014445ae3dcbec7bac0dba0eaf diff --git a/justfile b/justfile index 786b78ec..ba0a3318 100644 --- a/justfile +++ b/justfile @@ -1,12 +1,5 @@ -download-cscope: - mkdir -p cscope/ - docker pull ghcr.io/trailofbits/buttercup-cscope:main - docker create --name temp-cscope ghcr.io/trailofbits/buttercup-cscope:main - docker cp temp-cscope:/cscope cscope/ - docker rm temp-cscope - install-cscope: - cd cscope/cscope/ && autoreconf -i -s && ./configure && make && sudo make install + cd external/aixcc-cscope/ && autoreconf -i -s && ./configure && make && sudo make install lint-python COMPONENT: cd {{ COMPONENT }} && uv sync --all-extras && uv run ruff format && uv run ruff check --fix && uv run mypy diff --git a/patcher/Dockerfile b/patcher/Dockerfile index caa68c43..2e887497 100644 --- a/patcher/Dockerfile +++ b/patcher/Dockerfile @@ -1,7 +1,4 @@ ARG BASE_IMAGE=ubuntu:24.04 -ARG CSCOPE_IMAGE=ghcr.io/aixcc-finals/buttercup-cscope:main - -FROM $CSCOPE_IMAGE AS cscope-image FROM $BASE_IMAGE AS python_base ARG PYTHON_VERSION=3.12 @@ -51,8 +48,8 @@ RUN --mount=type=cache,target=/root/.cache/uv \ FROM runtime-base AS cscope-builder -COPY --from=cscope-image /cscope /cscope RUN DEBIAN_FRONTEND=noninteractive apt-get update && apt-get install -y autoconf gcc make bison flex libncurses-dev +COPY external/aixcc-cscope /cscope RUN cd /cscope && autoreconf -i -s && ./configure && make && make install FROM runtime-base AS runtime diff --git a/program-model/Dockerfile b/program-model/Dockerfile index 582beef9..ddeadc51 100644 --- a/program-model/Dockerfile +++ b/program-model/Dockerfile @@ -1,7 +1,4 @@ ARG BASE_IMAGE=ubuntu:24.04@sha256:f8b860e4f9036f2694571770da292642eebcc4c2ea0c70a1a9244c2a1d436cd9 -ARG CSCOPE_IMAGE=ghcr.io/aixcc-finals/buttercup-cscope:main - -FROM $CSCOPE_IMAGE AS cscope-image FROM $BASE_IMAGE AS base RUN apt-get update && apt-get install -y software-properties-common @@ -35,8 +32,8 @@ RUN --mount=type=cache,target=/root/.cache/uv \ uv sync --frozen --no-editable FROM base AS cscope-builder -COPY --from=cscope-image /cscope /cscope RUN DEBIAN_FRONTEND=noninteractive apt-get update && apt-get install -y autoconf gcc make bison flex libncurses-dev +COPY external/aixcc-cscope /cscope RUN cd /cscope && autoreconf -i -s && ./configure && make && make install FROM base AS runtime diff --git a/program-model/README.md b/program-model/README.md index 825aa4b6..b892a196 100644 --- a/program-model/README.md +++ b/program-model/README.md @@ -4,19 +4,6 @@ Indexes a program's source code to be queried by `seed-gen` and `patcher`. ## Setup -Build the `cscope` docker image and push to `trailofbits`. - -```shell -git clone git@github.com:trailofbits/aixcc-cscope.git -cd aixcc-cscope/ -git checkout buttercup - -docker build -t aixcc-cscope -f aixcc.Dockerfile . -docker tag aixcc-cscope ghcr.io/trailofbits/buttercup-cscope:main -docker push ghcr.io/trailofbits/buttercup-cscope:main -``` - -You should see the `buttercup-cscope` [package](https://github.com/orgs/trailofbits/packages) ## Development diff --git a/seed-gen/Dockerfile b/seed-gen/Dockerfile index 75a5d16d..bd5e28c8 100644 --- a/seed-gen/Dockerfile +++ b/seed-gen/Dockerfile @@ -1,7 +1,4 @@ ARG BASE_IMAGE=ubuntu:24.04 -ARG CSCOPE_IMAGE=ghcr.io/aixcc-finals/buttercup-cscope:main - -FROM $CSCOPE_IMAGE AS cscope-image FROM $BASE_IMAGE AS python_base ARG PYTHON_VERSION=3.12 @@ -45,8 +42,8 @@ RUN --mount=type=cache,target=/root/.cache/uv \ cd seed-gen && uv sync --frozen --no-editable FROM runtime-base AS cscope-builder -COPY --from=cscope-image /cscope /cscope RUN DEBIAN_FRONTEND=noninteractive apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y autoconf gcc make bison flex libncurses-dev +COPY external/aixcc-cscope /cscope RUN cd /cscope && autoreconf -i -s && ./configure && make && make install FROM runtime-base AS runtime