admin/trailofbits-buttercup
1
0
Fork 0
mirror of https://github.com/trailofbits/buttercup synced 2026-06-21 14:11:39 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into feat/ai-optimized-lint-scripts

Browse Source
This commit is contained in:
Dan Guido
2025-08-30 13:49:24 -04:00
committed by GitHub
parent fe5af109db 54de06efa3
commit 8211bd9f65
37 changed files with 488 additions and 254 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/ci-env.template
+1
View File
@@ -21,6 +21,7 @@ export BUTTERCUP_NAMESPACE=replace-me
export GHCR_AUTH=replace-me
export OPENAI_API_KEY=replace-me
export ANTHROPIC_API_KEY=replace-me
export GEMINI_API_KEY=replace-me
export LANGFUSE_HOST=replace-me
export LANGFUSE_PUBLIC_KEY=replace-me
export LANGFUSE_SECRET_KEY=replace-me
.github/workflows/comp-integration.yml
+200
View File
@@ -0,0 +1,200 @@
name: Component Integration Tests
on:
push:
branches:
- main
# Always run full test suite on main branch
pull_request:
schedule:
# Run integration tests daily at 2 AM UTC
- cron: '0 2 * * *'
workflow_dispatch:
inputs:
components:
description: 'Components to test (comma-separated: common,patcher,program-model,seed-gen)'
required: false
default: 'common,patcher,program-model,seed-gen'
type: string
run_integration:
description: 'Run integration tests'
required: false
default: false
type: boolean
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
jobs:
# Integration tests - run on schedule, manual trigger, or labeled PRs
test-integration:
# Run if:
# - Scheduled run
# - Manual dispatch with run_integration=true
# - PR with 'integration-tests' label
if: |
github.event_name == 'push' ||
github.event_name == 'schedule' ||
(github.event_name == 'workflow_dispatch' && github.event.inputs.run_integration == 'true') ||
contains(github.event.pull_request.labels.*.name, 'integration-tests')
strategy:
fail-fast: false
matrix:
include:
- component: common
coverage_module: buttercup.common
python: "3.12"
- component: patcher
coverage_module: buttercup.patcher
python: "3.12"
- component: program-model
coverage_module: buttercup.program_model
python: "3.12"
- component: seed-gen
coverage_module: buttercup.seed_gen
python: "3.12"
runs-on: ubuntu-latest
services:
redis:
image: redis
options: >-
--health-cmd "redis-cli ping"
--health-interval 10s
--health-timeout 5s
--health-retries 5
ports:
- 6379:6379
steps:
- uses: actions/checkout@v5
with:
persist-credentials: false
submodules: true
- name: Check if component should be tested
id: should_test
if: github.event_name == 'workflow_dispatch'
run: |
components="${{ github.event.inputs.components }}"
if [[ -z "$components" ]] || [[ "$components" == *"${{ matrix.component }}"* ]]; then
echo "test=true" >> $GITHUB_OUTPUT
else
echo "test=false" >> $GITHUB_OUTPUT
fi
- name: Install uv
if: steps.should_test.outputs.test != 'false'
uses: astral-sh/setup-uv@v6
- name: Setup uv cache
if: steps.should_test.outputs.test != 'false'
uses: actions/cache@v4
with:
path: |
~/.cache/uv
~/.local/share/uv
key: ${{ runner.os }}-uv-integration-${{ matrix.component }}-${{ hashFiles(format('{0}/uv.lock', matrix.component)) }}
restore-keys: |
${{ runner.os }}-uv-integration-${{ matrix.component }}-
${{ runner.os }}-uv-
- name: Download Wasm runtime
if: matrix.component == 'seed-gen' && steps.should_test.outputs.test != 'false'
run: wget https://github.com/vmware-labs/webassembly-language-runtimes/releases/download/python%2F3.12.0%2B20231211-040d5a6/python-3.12.0.wasm
working-directory: seed-gen
- name: Install integration test dependencies
if: steps.should_test.outputs.test != 'false'
run: |
sudo apt-get update
sudo apt-get install -y codequery ripgrep
make install-cscope
- name: Prepare environment
if: steps.should_test.outputs.test != 'false'
run: |
export DEBIAN_FRONTEND=noninteractive
sudo apt-get update
sudo mkdir -p /crs_scratch
sudo chmod -R 777 /crs_scratch
- name: Setup ${{ matrix.component }} component
if: steps.should_test.outputs.test != 'false'
run: |
uv sync --all-extras --frozen
uv pip install --isolated pytest-html>=4.1.1 pytest-cov>=6.0.0
working-directory: ${{ matrix.component }}
- name: Run program-model libpng integration test
if: matrix.component == 'program-model' && steps.should_test.outputs.test != 'false'
run: |
uv run --frozen pytest -svv --runintegration tests/c/test_libpng.py \
--junit-xml=integration-test-results.xml \
--html=integration-test-report.html \
--self-contained-html \
--cov=${{ matrix.coverage_module }} \
--cov-report=xml:integration-coverage.xml \
--cov-report=html:integration-htmlcov \
--cov-report=term
env:
PYTHON_WASM_BUILD_PATH: "python-3.12.0.wasm"
working-directory: ${{ matrix.component }}
timeout-minutes: 30
- name: Run integration tests on ${{ matrix.component }}
if: matrix.component != 'program-model' && steps.should_test.outputs.test != 'false'
run: |
uv run --frozen pytest -svv --runintegration \
--junit-xml=integration-test-results.xml \
--html=integration-test-report.html \
--self-contained-html \
--cov=${{ matrix.coverage_module }} \
--cov-report=xml:integration-coverage.xml \
--cov-report=html:integration-htmlcov \
--cov-report=term
env:
PYTHON_WASM_BUILD_PATH: "python-3.12.0.wasm"
working-directory: ${{ matrix.component }}
timeout-minutes: 30
- name: Generate integration test summary
if: always() && steps.should_test.outputs.test != 'false'
run: |
echo "### Integration Test Results: ${{ matrix.component }}" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
if [ -f ${{ matrix.component }}/integration-test-results.xml ]; then
python -c "
import xml.etree.ElementTree as ET
tree = ET.parse('${{ matrix.component }}/integration-test-results.xml')
root = tree.getroot()
tests = root.get('tests', '0')
failures = root.get('failures', '0')
errors = root.get('errors', '0')
skipped = root.get('skipped', '0')
time = root.get('time', '0')
print(f'- **Total Tests**: {tests}')
print(f'- **Passed**: {int(tests) - int(failures) - int(errors) - int(skipped)}')
print(f'- **Failed**: {failures}')
print(f'- **Errors**: {errors}')
print(f'- **Skipped**: {skipped}')
print(f'- **Duration**: {float(time):.2f}s')
" >> $GITHUB_STEP_SUMMARY
else
echo "No integration test results found" >> $GITHUB_STEP_SUMMARY
fi
- name: Upload integration test results
if: always() && steps.should_test.outputs.test != 'false'
uses: actions/upload-artifact@v4
with:
name: integration-test-results-${{ matrix.component }}-py${{ matrix.python }}
path: |
${{ matrix.component }}/integration-test-results.xml
${{ matrix.component }}/integration-test-report.html
${{ matrix.component }}/integration-coverage.xml
${{ matrix.component }}/integration-htmlcov/
retention-days: 30
.github/workflows/integration.yml
+3 -1
View File
@@ -1,4 +1,4 @@
name: Minikube Integration Tests
name: System Integration Tests
on:
# Daily schedule - 3 AM UTC
@@ -64,6 +64,7 @@ jobs:
integration:
# Only run on PRs if they have the 'full-integration' label
if: |
github.event_name == 'schedule' ||
github.event_name != 'pull_request' ||
contains(github.event.pull_request.labels.*.name, 'full-integration')
runs-on: gha-ubuntu-8
@@ -93,6 +94,7 @@ jobs:
sed -i "s|BUTTERCUP_NAMESPACE=.*|BUTTERCUP_NAMESPACE=${{ env.BUTTERCUP_NAMESPACE }}|" env
sed -i "s|OPENAI_API_KEY=.*|OPENAI_API_KEY=${{ secrets.OPENAI_API_KEY }}|" env
sed -i "s|ANTHROPIC_API_KEY=.*|ANTHROPIC_API_KEY=${{ secrets.ANTHROPIC_API_KEY }}|" env
sed -i "s|GEMINI_API_KEY=.*|GEMINI_API_KEY=${{ secrets.GEMINI_API_KEY }}|" env
sed -i "s|LANGFUSE_HOST=.*|LANGFUSE_HOST=${{ secrets.LANGFUSE_HOST }}|" env
sed -i "s|LANGFUSE_PUBLIC_KEY=.*|LANGFUSE_PUBLIC_KEY=${{ secrets.LANGFUSE_PUBLIC_KEY }}|" env
sed -i "s|LANGFUSE_SECRET_KEY=.*|LANGFUSE_SECRET_KEY=${{ secrets.LANGFUSE_SECRET_KEY }}|" env
.github/workflows/tests.yml
-192
View File
@@ -6,23 +6,6 @@ on:
- main
# Always run full test suite on main branch
pull_request:
# TEMPORARILY REMOVED PATH FILTERING TO DEBUG CI
# Will re-enable after confirming tests pass
schedule:
# Run integration tests daily at 2 AM UTC
- cron: '0 2 * * *'
workflow_dispatch:
inputs:
components:
description: 'Components to test (comma-separated: common,patcher,program-model,seed-gen)'
required: false
default: 'common,patcher,program-model,seed-gen'
type: string
run_integration:
description: 'Run integration tests'
required: false
default: false
type: boolean
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
@@ -52,12 +35,6 @@ jobs:
- component: fuzzer
coverage_module: buttercup.fuzzer
python: "3.12"
# TODO: Add integration tests back in
# include:
# - component: common
# pytest_args: "--runintegration"
# - component: program-model
# pytest_args: "--runintegration"
runs-on: ubuntu-latest
# Removed if: matrix.should_run since we're not using path filtering right now
@@ -206,172 +183,3 @@ jobs:
fail_ci_if_error: false
verbose: true
# Integration tests - run on schedule, manual trigger, or labeled PRs
test-integration:
# Run if:
# - Scheduled run
# - Manual dispatch with run_integration=true
# - PR with 'integration-tests' label
if: |
github.event_name == 'schedule' ||
(github.event_name == 'workflow_dispatch' && github.event.inputs.run_integration == 'true') ||
contains(github.event.pull_request.labels.*.name, 'integration-tests')
strategy:
fail-fast: false
matrix:
include:
- component: common
coverage_module: buttercup.common
python: "3.12"
- component: patcher
coverage_module: buttercup.patcher
python: "3.12"
- component: program-model
coverage_module: buttercup.program_model
python: "3.12"
- component: seed-gen
coverage_module: buttercup.seed_gen
python: "3.12"
runs-on: ubuntu-latest
services:
redis:
image: redis
options: >-
--health-cmd "redis-cli ping"
--health-interval 10s
--health-timeout 5s
--health-retries 5
ports:
- 6379:6379
steps:
- uses: actions/checkout@v5
with:
persist-credentials: false
submodules: true
- name: Check if component should be tested
id: should_test
if: github.event_name == 'workflow_dispatch'
run: |
components="${{ github.event.inputs.components }}"
if [[ -z "$components" ]] || [[ "$components" == *"${{ matrix.component }}"* ]]; then
echo "test=true" >> $GITHUB_OUTPUT
else
echo "test=false" >> $GITHUB_OUTPUT
fi
- name: Install uv
if: steps.should_test.outputs.test != 'false'
uses: astral-sh/setup-uv@v6
- name: Setup uv cache
if: steps.should_test.outputs.test != 'false'
uses: actions/cache@v4
with:
path: |
~/.cache/uv
~/.local/share/uv
key: ${{ runner.os }}-uv-integration-${{ matrix.component }}-${{ hashFiles(format('{0}/uv.lock', matrix.component)) }}
restore-keys: |
${{ runner.os }}-uv-integration-${{ matrix.component }}-
${{ runner.os }}-uv-
- name: Download Wasm runtime
if: matrix.component == 'seed-gen' && steps.should_test.outputs.test != 'false'
run: wget https://github.com/vmware-labs/webassembly-language-runtimes/releases/download/python%2F3.12.0%2B20231211-040d5a6/python-3.12.0.wasm
working-directory: seed-gen
- name: Install integration test dependencies
if: steps.should_test.outputs.test != 'false'
run: |
sudo apt-get update
sudo apt-get install -y codequery ripgrep
make install-cscope
- name: Prepare environment
if: steps.should_test.outputs.test != 'false'
run: |
export DEBIAN_FRONTEND=noninteractive
sudo apt-get update
sudo mkdir -p /crs_scratch
sudo chmod -R 777 /crs_scratch
- name: Setup ${{ matrix.component }} component
if: steps.should_test.outputs.test != 'false'
run: |
uv sync --all-extras --frozen
uv pip install --isolated pytest-html>=4.1.1 pytest-cov>=6.0.0
working-directory: ${{ matrix.component }}
- name: Run program-model libpng integration test
if: matrix.component == 'program-model' && steps.should_test.outputs.test != 'false'
run: |
uv run --frozen pytest -svv --runintegration tests/c/test_libpng.py \
--junit-xml=integration-test-results.xml \
--html=integration-test-report.html \
--self-contained-html \
--cov=${{ matrix.coverage_module }} \
--cov-report=xml:integration-coverage.xml \
--cov-report=html:integration-htmlcov \
--cov-report=term
env:
PYTHON_WASM_BUILD_PATH: "python-3.12.0.wasm"
working-directory: ${{ matrix.component }}
timeout-minutes: 30
- name: Run integration tests on ${{ matrix.component }}
if: matrix.component != 'program-model' && steps.should_test.outputs.test != 'false'
run: |
uv run --frozen pytest -svv --runintegration \
--junit-xml=integration-test-results.xml \
--html=integration-test-report.html \
--self-contained-html \
--cov=${{ matrix.coverage_module }} \
--cov-report=xml:integration-coverage.xml \
--cov-report=html:integration-htmlcov \
--cov-report=term
env:
PYTHON_WASM_BUILD_PATH: "python-3.12.0.wasm"
working-directory: ${{ matrix.component }}
timeout-minutes: 30
- name: Generate integration test summary
if: always() && steps.should_test.outputs.test != 'false'
run: |
echo "### Integration Test Results: ${{ matrix.component }}" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
if [ -f ${{ matrix.component }}/integration-test-results.xml ]; then
python -c "
import xml.etree.ElementTree as ET
tree = ET.parse('${{ matrix.component }}/integration-test-results.xml')
root = tree.getroot()
tests = root.get('tests', '0')
failures = root.get('failures', '0')
errors = root.get('errors', '0')
skipped = root.get('skipped', '0')
time = root.get('time', '0')
print(f'- **Total Tests**: {tests}')
print(f'- **Passed**: {int(tests) - int(failures) - int(errors) - int(skipped)}')
print(f'- **Failed**: {failures}')
print(f'- **Errors**: {errors}')
print(f'- **Skipped**: {skipped}')
print(f'- **Duration**: {float(time):.2f}s')
" >> $GITHUB_STEP_SUMMARY
else
echo "No integration test results found" >> $GITHUB_STEP_SUMMARY
fi
- name: Upload integration test results
if: always() && steps.should_test.outputs.test != 'false'
uses: actions/upload-artifact@v4
with:
name: integration-test-results-${{ matrix.component }}-py${{ matrix.python }}
path: |
${{ matrix.component }}/integration-test-results.xml
${{ matrix.component }}/integration-test-report.html
${{ matrix.component }}/integration-coverage.xml
${{ matrix.component }}/integration-htmlcov/
retention-days: 30
CONTRIBUTING.md
+3 -3
View File
@@ -111,10 +111,10 @@ kubectl port-forward -n crs service/buttercup-competition-api 31323:1323
## Resources
- [Quick Reference](QUICK_REFERENCE.md) - Common commands and troubleshooting
- [Quick Reference](guides/QUICK_REFERENCE.md) - Common commands and troubleshooting
- [Deployment Guide](deployment/README.md) - Detailed deployment information
- [Custom Challenges](CUSTOM_CHALLENGES.md) - Adding new test cases
- [Custom Challenges](guides/CUSTOM_CHALLENGES.md) - Adding new test cases
## Questions?
Open an issue or reach out to the development team.
Open an issue or reach out to the development team.
README.md
+18 -15
View File
@@ -1,8 +1,8 @@
# Buttercup Cyber Reasoning System (CRS)
[![Tests](https://github.com/trailofbits/buttercup/actions/workflows/tests.yml/badge.svg)](https://github.com/trailofbits/buttercup/actions/workflows/tests.yml)
[![Tests (Nightly)](https://github.com/trailofbits/buttercup/actions/workflows/tests.yml/badge.svg?event=schedule)](https://github.com/trailofbits/buttercup/actions/workflows/tests.yml)
[![Integration](https://github.com/trailofbits/buttercup/actions/workflows/integration.yml/badge.svg)](https://github.com/trailofbits/buttercup/actions/workflows/integration.yml)
[![Component Integration Tests](https://github.com/trailofbits/buttercup/actions/workflows/comp-integration.yml/badge.svg)](https://github.com/trailofbits/buttercup/actions/workflows/comp-integration.yml)
[![System Integration](https://github.com/trailofbits/buttercup/actions/workflows/integration.yml/badge.svg)](https://github.com/trailofbits/buttercup/actions/workflows/integration.yml)
**Buttercup** is a Cyber Reasoning System (CRS) developed by **Trail of Bits** for the **DARPA AIxCC (AI Cyber Challenge)**. Buttercup finds and patches software vulnerabilities in open-source code repositories like [example-libpng](https://github.com/tob-challenges/example-libpng). It starts by running an AI/ML-assisted fuzzing campaign (built on oss-fuzz) for the program. When vulnerabilities are found, Buttercup analyzes them and uses a multi-agent AI-driven patcher to repair the vulnerability. **Buttercup** system consists of several components:
@@ -65,7 +65,7 @@ git clone --recurse-submodules https://github.com/trailofbits/buttercup.git
cd buttercup
```
2. Run automated setup (Recommended)
1. Run automated setup (Recommended)
```bash
make setup-local
@@ -73,15 +73,15 @@ make setup-local
This script will install all dependencies, configure the environment, and guide you through the setup process.
**Note:** If you prefer manual setup, see the [Manual Setup Guide](MANUAL_SETUP.md).
**Note:** If you prefer manual setup, see the [Manual Setup Guide](guides/MANUAL_SETUP.md).
3. Start Buttercup locally
1. Start Buttercup locally
```bash
make deploy-local
```
4. Verify local deployment:
1. Verify local deployment:
```bash
make status
@@ -89,7 +89,7 @@ make status
When a deployment is successful, you should see all pods in "Running" or "Completed" status.
5. Send Buttercup a simple task
1. Send Buttercup a simple task
**Note:** When tasked, Buttercup will start consuming third-party AI resources.
@@ -99,7 +99,7 @@ This command will make Buttercup pull down an example repo [example-libpng](http
make send-libpng-task
```
6. Access Buttercup's web-based GUI
1. Access Buttercup's web-based GUI
Run:
@@ -111,7 +111,7 @@ Then navigate to `http://localhost:31323` in your web browser.
In the GUI you can monitor active tasks and see when Buttercup finds bugs and generates patches for them.
7. Stop Buttercup
1. Stop Buttercup
**Note:** This is an important step to ensure Buttercup shuts down and stops consuming third-party AI resources.
@@ -128,20 +128,23 @@ make signoz-ui
```
Then navigate to `http://localhost:33301` in your web browser to view:
- Distributed traces
- Application metrics
- Application metrics
- Error monitoring
- Performance insights
If you configured LangFuse during setup, you can also monitor LLM usage and costs there.
For additional log access methods, see the [Quick Reference Guide](QUICK_REFERENCE.md).
For additional log access methods, see the [Quick Reference Guide](guides/QUICK_REFERENCE.md).
## Additional Resources
- [Quick Reference Guide](QUICK_REFERENCE.md) - Common commands and troubleshooting
- [Manual Setup Guide](MANUAL_SETUP.md) - Detailed manual installation steps
- [AKS Deployment Guide](AKS_DEPLOYMENT.md) - Production deployment on Azure
- [Quick Reference Guide](guides/QUICK_REFERENCE.md) - Common commands and troubleshooting
- [Manual Setup Guide](guides/MANUAL_SETUP.md) - Detailed manual installation steps
- [AKS Deployment Guide](guides/AKS_DEPLOYMENT.md) - Production deployment on Azure
- [Contributing Guidelines](CONTRIBUTING.md) - Development workflow and standards
- [Deployment Documentation](deployment/README.md) - Advanced deployment configuration
- [Writing Custom Challenges](CUSTOM_CHALLENGES.md) - Custom project configuration and setup
- [Writing Custom Challenges](guides/CUSTOM_CHALLENGES.md) - Custom project configuration and setup
- [Unscored rounds](guides/UNSCORED.md) - Running unscored round challenges
- [Scored round](guides/SCORED.md) - Parsing post-final round results
common/src/buttercup/common/llm.py
+29 -16
View File
@@ -7,7 +7,7 @@ from typing import Any
import requests
from langchain.callbacks.base import BaseCallbackHandler
from langchain_core.language_models import BaseChatModel
from langchain_core.runnables import ConfigurableField
from langchain_core.runnables import ConfigurableField, Runnable
from langchain_openai.chat_models import ChatOpenAI
from langfuse.callback import CallbackHandler
@@ -32,6 +32,9 @@ class ButtercupLLM(Enum):
CLAUDE_3_5_SONNET = "claude-3.5-sonnet"
CLAUDE_3_7_SONNET = "claude-3.7-sonnet"
CLAUDE_4_SONNET = "claude-4-sonnet"
GEMINI_PRO = "gemini-pro"
GEMINI_2_5_FLASH = "gemini-2.5-flash"
GEMINI_2_5_FLASH_EXP = "gemini-2.5-flash-exp"
@functools.cache
@@ -94,31 +97,41 @@ def get_langfuse_callbacks() -> list[BaseCallbackHandler]:
return []
def create_default_llm(**kwargs: Any) -> BaseChatModel:
def create_default_llm(**kwargs: Any) -> Runnable:
"""Create an LLM object with the default configuration."""
fallback_models = kwargs.pop("fallback_models", [])
fallback_models = [create_default_llm(**{**kwargs, "model_name": m.value}) for m in fallback_models]
return create_llm(
model_name=kwargs.pop("model_name", ButtercupLLM.OPENAI_GPT_4_1.value),
temperature=kwargs.pop("temperature", 0.1),
timeout=420.0,
max_retries=3,
**kwargs,
)
).with_fallbacks(fallback_models)
def create_default_llm_with_temperature(**kwargs: Any) -> BaseChatModel:
def create_default_llm_with_temperature(**kwargs: Any) -> Runnable:
"""Create an LLM object with the default configuration and temperature."""
return create_llm( # type: ignore
model_name=kwargs.pop("model_name", ButtercupLLM.OPENAI_GPT_4_1.value),
temperature=kwargs.pop("temperature", 0.1),
timeout=420.0,
max_retries=3,
**kwargs,
).configurable_fields(
temperature=ConfigurableField(
id="llm_temperature",
name="LLM temperature",
description="The temperature for the LLM model",
),
fallback_models = kwargs.pop("fallback_models", [])
fallback_models = [
create_default_llm_with_temperature(**{**kwargs, "model_name": m.value}) for m in fallback_models
]
return (
create_llm(
model_name=kwargs.pop("model_name", ButtercupLLM.OPENAI_GPT_4_1.value),
temperature=kwargs.pop("temperature", 0.1),
timeout=420.0,
max_retries=3,
**kwargs,
)
.configurable_fields(
temperature=ConfigurableField(
id="llm_temperature",
name="LLM temperature",
description="The temperature for the LLM model",
),
)
.with_fallbacks(fallback_models)
)
common/src/buttercup/common/util_cli.py
+5
View File
@@ -96,6 +96,7 @@ class ReadBuildsSettings(BaseModel):
class ReadSubmissionsSettings(BaseModel):
task_id: str = Field(default="", description="Task ID")
verbose: bool = Field(False, description="Show full stacktraces instead of truncated versions")
filter_stop: bool = Field(False, description="Filter out submissions that are stopped")
@@ -228,6 +229,10 @@ def handle_subcommand(redis: Redis, command: BaseModel | None) -> None:
logger.error(f"Task {task_id} not found in registry")
continue
if command.task_id and task_id != command.task_id:
logger.info(f"Skipping submission {i} for task {task_id} (not {command.task_id})")
continue
if task_id not in result:
result[task_id] = TaskResult(
task_id=task_id,
deployment/k8s/README.md
+5 -1
View File
@@ -8,7 +8,7 @@ This guide explains how to set up and run the Buttercup system on Kubernetes.
- kubectl configured to communicate with your cluster
- Helm v3
- Access to container registries (ghcr.io)
- Access to required API keys (OpenAI, Azure, Anthropic)
- Access to required API keys (OpenAI, Azure, Anthropic, Gemini)
## Environment Configuration
@@ -43,6 +43,8 @@ This is controlled via the `global.environment` setting in the values.yaml or va
apiKey: "your-openai-api-key"
anthropic:
apiKey: "your-anthropic-api-key"
gemini:
apiKey: "your-google-gemini-key"
crs:
api_key_id: 515cc8a0-3019-4c9f-8c1c-72d0b54ae561
@@ -77,6 +79,7 @@ This secret contains API keys for LLM providers:
- **AZURE_API_KEY**: Azure OpenAI API key
- **OPENAI_API_KEY**: OpenAI API key
- **ANTHROPIC_API_KEY**: Anthropic API key
- **GEMINI_API_KEY**: Google Gemini API key
These are populated from the `values-override.yaml` file.
@@ -197,6 +200,7 @@ The system uses LiteLLM to proxy all LLM requests. The configuration includes mo
- Azure OpenAI
- OpenAI
- Anthropic
- Google Gemini
To modify the available models, edit the `litellm-helm.proxy_config.model_list` section in values.yaml.
deployment/k8s/templates/litellm-secret.yaml
+1
View File
@@ -13,5 +13,6 @@ data:
AZURE_API_KEY: {{ .Values.litellm.azure.apiKey | b64enc | quote }}
OPENAI_API_KEY: {{ .Values.litellm.openai.apiKey | b64enc | quote }}
ANTHROPIC_API_KEY: {{ .Values.litellm.anthropic.apiKey | b64enc | quote }}
GEMINI_API_KEY: {{ .Values.litellm.gemini.apiKey | b64enc | quote }}
BUTTERCUP_LITELLM_KEY: {{ .Values.litellm.masterKey | b64enc | quote }}
{{- end }}
deployment/k8s/values-aks.template
+2
View File
@@ -34,6 +34,8 @@ litellm:
apiKey: "${OPENAI_API_KEY}"
anthropic:
apiKey: "${ANTHROPIC_API_KEY}"
gemini:
apiKey: "${GEMINI_API_KEY}"
competition-api:
enabled: true
deployment/k8s/values-ci.template
+2
View File
@@ -53,6 +53,8 @@ litellm:
apiKey: "${OPENAI_API_KEY}"
anthropic:
apiKey: "${ANTHROPIC_API_KEY}"
gemini:
apiKey: "${GEMINI_API_KEY}"
competition-api:
enabled: ${COMPETITION_API_ENABLED}
deployment/k8s/values-minikube.template
+2
View File
@@ -57,6 +57,8 @@ litellm:
apiKey: "${OPENAI_API_KEY}"
anthropic:
apiKey: "${ANTHROPIC_API_KEY}"
gemini:
apiKey: "${GEMINI_API_KEY}"
competition-api:
enabled: ${COMPETITION_API_ENABLED}
deployment/k8s/values-prod.template
+2
View File
@@ -228,6 +228,8 @@ litellm:
apiKey: "${OPENAI_API_KEY}"
anthropic:
apiKey: "${ANTHROPIC_API_KEY}"
gemini:
apiKey: "${GEMINI_API_KEY}"
litellm-helm:
resources:
deployment/k8s/values-upstream-minikube.template
+2
View File
@@ -63,6 +63,8 @@ litellm:
apiKey: "${OPENAI_API_KEY}"
anthropic:
apiKey: "${ANTHROPIC_API_KEY}"
gemini:
apiKey: "${GEMINI_API_KEY}"
ui:
enabled: true
deployment/k8s/values.yaml
+22
View File
@@ -375,6 +375,28 @@ litellm-helm:
api_key: os.environ/ANTHROPIC_API_KEY
tpm: 1775000
- model_name: gemini-pro
litellm_params:
model: gemini/gemini-pro
api_key: os.environ/GEMINI_API_KEY
tpm: 2000000
rpm: 150
- model_name: gemini-2.5-flash-exp
litellm_params:
model: gemini/gemini-2.5-flash-exp
api_key: os.environ/GEMINI_API_KEY
tpm: 4000000
rpm: 2000
- model_name: gemini-2.5-flash
litellm_params:
model: gemini/gemini-2.5-flash
api_key: os.environ/GEMINI_API_KEY
tpm: 4000000
rpm: 2000
general_settings:
master_key: os.environ/BUTTERCUP_LITELLM_KEY
database_url: "postgresql://litellm_user:litellm_password11@buttercup-postgresql:5432/litellm"
AKS_DEPLOYMENT.md → guides/AKS_DEPLOYMENT.md
+5 -5
View File
@@ -1,8 +1,8 @@
# Production AKS Deployment Guide
> **⚠️ Notice:**
> The following production deployment instructions have **not been fully tested**.
> Please proceed with caution and verify each step in your environment.
> **⚠️ Notice:**
> The following production deployment instructions have **not been fully tested**.
> Please proceed with caution and verify each step in your environment.
> If you encounter issues, consult the script comments and configuration files for troubleshooting.
Full production deployment of the **Buttercup CRS** on Azure Kubernetes Service with proper networking, monitoring, and scaling for the DARPA AIxCC competition.
@@ -81,7 +81,7 @@ cd deployment && make up
## Additional Resources
For detailed deployment instructions and advanced configuration options, see the [deployment README](deployment/README.md).
For detailed deployment instructions and advanced configuration options, see the [deployment README](../deployment/README.md).
## Troubleshooting
@@ -102,4 +102,4 @@ az aks get-credentials --name <cluster-name> --resource-group <resource-group>
az aks show --name <cluster-name> --resource-group <resource-group>
```
For more troubleshooting information, see the main [Quick Reference Guide](QUICK_REFERENCE.md).
For more troubleshooting information, see the main [Quick Reference Guide](QUICK_REFERENCE.md).
CUSTOM_CHALLENGES.md → guides/CUSTOM_CHALLENGES.md
+4 -3
View File
@@ -350,11 +350,13 @@ Buttercup determines which mode to use based on your challenge configuration:
### Choosing the Right Mode
**Choose Full Mode when you want to:**
- Perform an initial assessment of a new project
- Search for vulnerabilities in all code reachable from harnesses
- Analyze a project without specific change targets
**Choose Delta Mode when you want to:**
- Test specific code changes
- Reproduce known vulnerabilities
@@ -369,8 +371,7 @@ Now that you understand how to create custom challenges, you can:
3. **Monitor progress**: Use the debugging techniques to track your challenge's execution
4. **Iterate**: Refine your harnesses and build scripts based on the results
For additional help, consult the main [Buttercup documentation](README.md) or check the [troubleshooting guide](CLAUDE.md#common-debugging-commands).
For additional help, consult the main [Buttercup documentation](../README.md) or check the [troubleshooting guide](../CLAUDE.md#common-debugging-commands).
[1]: https://aicyberchallenge.com/
[2]: https://google.github.io/oss-fuzz/getting-started/new-project-guide/
[2]: https://google.github.io/oss-fuzz/getting-started/new-project-guide/
MANUAL_SETUP.md → guides/MANUAL_SETUP.md
+1 -1
View File
@@ -133,4 +133,4 @@ helm repo update
helm dependency update deployment/k8s/
```
For additional troubleshooting, see the [Quick Reference Guide](QUICK_REFERENCE.md).
For additional troubleshooting, see the [Quick Reference Guide](QUICK_REFERENCE.md).
QUICK_REFERENCE.md → guides/QUICK_REFERENCE.md
+19 -1
View File
@@ -3,6 +3,7 @@
## Setup Commands
### Local Development
```bash
# Automated setup
./scripts/setup-local.sh
@@ -14,6 +15,7 @@ cd deployment && make up
```
### Production AKS
```bash
# Automated setup
./scripts/setup-azure.sh
@@ -27,6 +29,7 @@ cd deployment && make up
## Common Commands
### Kubernetes Management
```bash
# View all resources
kubectl get pods -A
@@ -51,6 +54,7 @@ kubectl exec -it -n crs <pod-name> -- /bin/bash
```
### Testing
```bash
# Send test task
./orchestrator/scripts/task_crs.sh
@@ -63,6 +67,7 @@ kubectl exec -it -n crs <pod-name> -- /bin/bash
```
### Development
```bash
# Lint Python code
make lint
@@ -74,6 +79,7 @@ docker-compose --profile fuzzer-test up
```
### Minikube
```bash
# Start/stop
minikube start --driver=docker
@@ -86,6 +92,7 @@ minikube dashboard
```
### Azure AKS
```bash
# Get credentials
az aks get-credentials --name <cluster-name> --resource-group <resource-group>
@@ -105,6 +112,7 @@ az aks show --name <cluster-name> --resource-group <resource-group>
### Common Issues
#### Minikube Issues
```bash
# Reset minikube
minikube delete
@@ -116,6 +124,7 @@ kubectl cluster-info
```
#### Docker Issues
```bash
# Permission issues
sudo usermod -aG docker $USER
@@ -127,6 +136,7 @@ sudo systemctl start docker
```
#### Helm Issues
```bash
# Update repositories
helm repo update
@@ -137,6 +147,7 @@ helm lint deployment/k8s/
```
#### Azure Issues
```bash
# Authentication
az login --tenant <your-tenant-here>
@@ -147,6 +158,7 @@ az ad sp list --display-name "ButtercupCRS*"
```
#### Kubernetes Issues
```bash
# Check cluster connectivity
kubectl cluster-info
@@ -163,16 +175,19 @@ kubectl get events -n crs --sort-by='.lastTimestamp'
### Log Analysis
#### Check Patch Submission
```bash
kubectl logs -n crs -l app=scheduler --tail=-1 --prefix | grep "WAIT_PATCH_PASS -> SUBMIT_BUNDLE"
```
#### Check Competition API
```bash
kubectl logs -n crs -l app=competition-api --tail=-1 --prefix
```
#### Check Fuzzer
```bash
kubectl logs -n crs -l app=fuzzer --tail=-1 --prefix
```
@@ -180,16 +195,19 @@ kubectl logs -n crs -l app=fuzzer --tail=-1 --prefix
## File Locations
### Configuration
- `deployment/env` - Main configuration file
- `deployment/env.template` - Configuration template
- `deployment/k8s/values-*.template` - Kubernetes value templates
### Scripts
- `scripts/setup-local.sh` - Local development setup
- `scripts/setup-azure.sh` - Production AKS setup
- `orchestrator/scripts/` - Testing and task scripts
### Documentation
- `README.md` - Main documentation
- `deployment/README.md` - Detailed deployment guide
@@ -198,4 +216,4 @@ kubectl logs -n crs -l app=fuzzer --tail=-1 --prefix
- Check logs: `kubectl logs -n crs <pod-name>`
- View events: `kubectl get events -n crs`
- Debug pods: `kubectl exec -it -n crs <pod-name> -- /bin/bash`
- Monitor resources: `kubectl top pods -A`
- Monitor resources: `kubectl top pods -A`
guides/SCORED.md
+69
View File
@@ -0,0 +1,69 @@
# Scored
This document describes how to parse logs from the final round of the AIxCC competition.
**NOTE**: PoVs were not released to us immediately because DARPA is still involved in responsible disclosure: <https://aicyberchallenge.com/Finals-winners-announcement/>.
## Data
Download the following files from the competition to `buttercup/data/`
- `litellm_backup.sql`
- `redis-backup.rdb`
- `task-storage.tar`
## Get Task IDs for Specific Projects
For example, find the Task IDs for `libpng`.
```shell
cd data/
../scripts/results_unpack.sh
../scripts/results_find.sh libpng
```
## Parse
Query the CRS database for submissions. Documentation can be found at <https://redis.io/learn/explore/import#restore-an-rdb-file>.
Let's assume a task ID `0197b76a-b429-78fa-8d0e-ad84994c8f44` was outputted from above.
1. Start `redis-server` in `data/` directory
```shell
cd data/
redis-server
```
1. Print directory to confirm
```shell
redis-cli config get dir
1) "dir"
2) "<path/to/data>"
```
1. Stop `redis-server` and check if `dump.rdb` exists in `data/` directory.
1. Copy `.rdb` file
```shell
cp redis-backup.rdb dump.rdb
```
1. Start `redis-server` and query to make sure it has data.
```shell
redis-server
redis-cli keys \*
```
1. Parse database for task id
```shell
cd common/
uv sync --all-extras
uv run buttercup-util read_submissions --task_id 0197b76a-b429-78fa-8d0e-ad84994c8f44 --verbose True > ../data/submissions.txt
```
1. Read through `submissions.txt` for information like crash harnesses, stacktraces, and patches.
UNSCORED.md → guides/UNSCORED.md
+1 -1
View File
@@ -9,7 +9,7 @@ A list of vulnerabilities to find and patch is in this [spreadsheet](https://doc
## Setup
* Set up minikube locally on your machine (see [README.md](README.md)).
* Set up minikube locally on your machine (see [README.md](../README.md)).
* `cd deployment && make up`
litellm/litellm_config.yaml
+21
View File
@@ -109,5 +109,26 @@ model_list:
api_key: os.environ/ANTHROPIC_API_KEY
tpm: 400000
- model_name: gemini-pro
litellm_params:
model: gemini/gemini-pro
api_key: os.environ/GEMINI_API_KEY
tpm: 2000000
rpm: 150
- model_name: gemini-2.5-flash-exp
litellm_params:
model: gemini/gemini-2.5-flash-exp
api_key: os.environ/GEMINI_API_KEY
tpm: 4000000
rpm: 2000
- model_name: gemini-2.5-flash
litellm_params:
model: gemini/gemini-2.5-flash
api_key: os.environ/GEMINI_API_KEY
tpm: 4000000
rpm: 2000
general_settings:
master_key: os.environ/BUTTERCUP_LITELLM_KEY
orchestrator/test/test_crs_client.py
+2 -2
View File
@@ -61,7 +61,7 @@ class TestCRSClient:
# Verify request was made correctly
mock_post.assert_called_once_with(
"http://test-crs:8080/v1/task/",
json=sample_task.dict(),
json=sample_task.model_dump(),
auth=("test_user", "test_pass"),
headers={"Content-Type": "application/json"},
timeout=30,
@@ -82,7 +82,7 @@ class TestCRSClient:
# Verify request was made without auth
mock_post.assert_called_once_with(
"http://test-crs:8080/v1/task/",
json=sample_task.dict(),
json=sample_task.model_dump(),
auth=None,
headers={"Content-Type": "application/json"},
timeout=30,
patcher/src/buttercup/patcher/agents/common.py
+4 -1
View File
@@ -434,7 +434,10 @@ UNDERSTAND_CODE_SNIPPET_PROMPT = ChatPromptTemplate.from_messages(
def _create_understand_code_snippet_chain() -> Runnable:
return ( # type: ignore[no-any-return]
UNDERSTAND_CODE_SNIPPET_PROMPT
| create_default_llm_with_temperature(model_name=ButtercupLLM.OPENAI_GPT_4_1.value)
| create_default_llm_with_temperature(
model_name=ButtercupLLM.OPENAI_GPT_4_1.value,
fallback_models=[ButtercupLLM.CLAUDE_4_SONNET, ButtercupLLM.GEMINI_PRO],
)
| StrOutputParser()
)
patcher/src/buttercup/patcher/agents/context_retriever.py
+5 -1
View File
@@ -585,7 +585,10 @@ Remember to call the `test_instructions` tool to log and validate any test comma
def _are_test_instructions_valid(instructions: str, output: bytes, error: bytes) -> bool:
"""Validate a set of test instructions by executing them inside the project environment."""
llm = create_default_llm(model_name=ButtercupLLM.OPENAI_GPT_4_1.value)
llm = create_default_llm(
model_name=ButtercupLLM.OPENAI_GPT_4_1.value,
fallback_models=[ButtercupLLM.CLAUDE_4_SONNET, ButtercupLLM.GEMINI_PRO],
)
chain = ARE_VALID_TEST_INSTRUCTIONS_PROMPT | llm | StrOutputParser()
res = chain.invoke(
{
@@ -708,6 +711,7 @@ class ContextRetrieverAgent(PatcherAgentBase):
self.cheap_llm = create_default_llm(model_name=ButtercupLLM.OPENAI_GPT_4_1_MINI.value)
self.cheap_fallback_llms = [
create_default_llm(model_name=ButtercupLLM.CLAUDE_3_5_SONNET.value),
create_default_llm(model_name=ButtercupLLM.GEMINI_PRO.value),
]
self.tools = [
patcher/src/buttercup/patcher/agents/qe.py
+1 -1
View File
@@ -157,7 +157,7 @@ class QEAgent(PatcherAgentBase):
tools=tools,
prompt=self._check_harness_changes_prompt,
)
for llm in [ButtercupLLM.CLAUDE_3_7_SONNET]
for llm in [ButtercupLLM.CLAUDE_3_7_SONNET, ButtercupLLM.GEMINI_PRO]
]
self.check_harness_changes_chain = default_agent.with_fallbacks(fallback_agents)
patcher/src/buttercup/patcher/agents/reflection.py
+1 -3
View File
@@ -407,9 +407,7 @@ class ReflectionAgent(PatcherAgentBase):
"""Initialize a few fields"""
default_llm = create_default_llm(model_name=ButtercupLLM.OPENAI_GPT_4_1.value)
fallback_llms: list[Runnable] = []
for fb_model in [
ButtercupLLM.CLAUDE_3_7_SONNET,
]:
for fb_model in [ButtercupLLM.CLAUDE_3_7_SONNET, ButtercupLLM.GEMINI_PRO]:
fallback_llms.append(create_default_llm(model_name=fb_model.value))
self.llm = default_llm.with_fallbacks(fallback_llms)
patcher/src/buttercup/patcher/agents/rootcause.py
+3 -2
View File
@@ -135,9 +135,10 @@ class RootCauseAgent(PatcherAgentBase):
)
fallback_llms = [
create_default_llm_with_temperature(
model_name=ButtercupLLM.CLAUDE_3_7_SONNET.value,
model_name=m.value,
**kwargs,
),
)
for m in [ButtercupLLM.CLAUDE_3_7_SONNET, ButtercupLLM.GEMINI_PRO]
]
self.llm = default_llm.with_fallbacks(fallback_llms)
patcher/src/buttercup/patcher/agents/swe.py
+1
View File
@@ -352,6 +352,7 @@ class SWEAgent(PatcherAgentBase):
fallback_llms: list[Runnable] = []
for fb_model in [
ButtercupLLM.CLAUDE_3_7_SONNET,
ButtercupLLM.GEMINI_PRO,
]:
fallback_llms.append(create_default_llm_with_temperature(model_name=fb_model.value, **kwargs))
self.llm = self.default_llm.with_fallbacks(fallback_llms)
patcher/tests/test_context_retriever.py
+5 -3
View File
@@ -192,7 +192,7 @@ def example_libpng_oss_fuzz_task(tmp_path: Path) -> ChallengeTask:
"-C",
str(oss_fuzz_dir),
"clone",
"git@github.com:tob-challenges/oss-fuzz-aixcc.git",
"https://github.com/tob-challenges/oss-fuzz-aixcc.git",
],
check=True,
)
@@ -209,8 +209,8 @@ def example_libpng_oss_fuzz_task(tmp_path: Path) -> ChallengeTask:
check=True,
)
# Download selinux source code
url = "git@github.com:tob-challenges/example-libpng.git"
# Download libpng source code
url = "https://github.com/tob-challenges/example-libpng.git"
subprocess.run(["git", "-C", str(source_dir), "clone", url], check=True)
subprocess.run(
[
@@ -474,6 +474,7 @@ def mock_runnable_config(tmp_path: Path) -> dict:
}
@pytest.mark.skip(reason="Temporarily disabled due to unknown issue")
@pytest.mark.integration
def test_retrieve_context_basic(
selinux_agent: ContextRetrieverAgent,
@@ -2281,6 +2282,7 @@ def test_find_tests_agent_uses_existing_test_sh(mock_agent: ContextRetrieverAgen
assert result.goto == PatcherAgentName.ROOT_CAUSE_ANALYSIS.value
@pytest.mark.skip(reason="Required commit not available in mirror of oss-fuzz-aixcc")
@pytest.mark.integration
def test_grep_libpng(libpng_agent: ContextRetrieverAgent, mock_agent_llm: MagicMock, mock_runnable_config) -> None:
"""Test the grep command on Makefile in libpng."""
patcher/tests/test_utils.py
+1
View File
@@ -131,6 +131,7 @@ def test_find_file_in_source_dir_absolute(mock_challenge_task: ChallengeTask):
assert res == Path("test.txt")
@pytest.mark.skip(reason="Target not currently publicly available")
@pytest.mark.integration
def test_tika_find_file_in_source_dir(tika_challenge_task: ChallengeTask):
res = find_file_in_source_dir(
scripts/common.sh
+16 -2
View File
@@ -497,6 +497,13 @@ configure_local_api_keys() {
print_status "Generate your API key at: https://console.anthropic.com/settings/keys"
configure_service "ANTHROPIC_API_KEY" "Anthropic API key" "$ANTHROPIC_API_KEY" "<your-anthropic-api-key>" false
# Anthropic API Key (Optional)
print_linebreak
print_status "Google Gemini API Key (Optional): Fallback model."
print_status "Use this model as a fallback if other models are not configured or not available."
print_status "Generate your API key at: https://aistudio.google.com/apikey"
configure_service "GEMINI_API_KEY" "Gemini API key" "$GEMINI_API_KEY" "<your-gemini-api-key>" false
# GitHub Personal Access Token (Optional)
print_linebreak
print_status "GitHub Personal Access Token (Optional): Access to private GitHub resources."
@@ -525,9 +532,15 @@ configure_local_api_keys() {
else
anthropic_configured=true
fi
if [ -z "$GEMINI_API_KEY" ] || [ "$GEMINI_API_KEY" = "<your-gemini-api-key>" ]; then
gemini_configured=false
else
gemini_configured=true
fi
if [ "$openai_configured" = false ] && [ "$anthropic_configured" = false ]; then
print_error "At least one LLM API key (OpenAI or Anthropic) must be configured."
if [ "$openai_configured" = false ] && [ "$anthropic_configured" = false ] && [ "$gemini_configured" = false ]; then
print_error "At least one LLM API key (OpenAI, Anthropic, or Gemini) must be configured."
print_error "Rerun the setup and set at least one LLM API key."
return 1
fi
@@ -632,6 +645,7 @@ check_aks_config() {
local api_vars=(
"OPENAI_API_KEY"
"ANTHROPIC_API_KEY"
"GEMINI_API_KEY"
"GHCR_AUTH"
"CRS_KEY_ID"
"CRS_KEY_TOKEN"
scripts/results_find.sh
Executable
+8
View File
@@ -0,0 +1,8 @@
#!/bin/bash
if [ "$#" -ne 1 ]; then
echo "usage: $0 <keyword>"
exit 1
fi
find tasks_storage -name "task_meta.json" -exec grep -rIil "$1" {} \; | cut -d '/' -f 1-2 | sort -u
scripts/results_unpack.sh
Executable
+20
View File
@@ -0,0 +1,20 @@
#!/bin/bash
if [ "$#" -ne 0 ]; then
echo "usage: $0"
exit 1
fi
# Clean up
rm -rf tasks_storage
# Unpack
tar xvf tasks_storage.tar
# Decompress
for fn in `find tasks_storage -name *.tgz`; do
dst="${fn%.tgz}"
mkdir $dst
tar xvzf $fn -C $dst
rm $fn
done
seed-gen/src/buttercup/seed_gen/task.py
+1
View File
@@ -96,6 +96,7 @@ class Task:
fallbacks = [
ButtercupLLM.CLAUDE_3_7_SONNET,
ButtercupLLM.CLAUDE_3_5_SONNET,
ButtercupLLM.GEMINI_PRO,
]
self.llm = Task.get_llm(ButtercupLLM.CLAUDE_4_SONNET, fallbacks)
self.tools = [
seed-gen/test/test_find_harness.py
+3
View File
@@ -615,6 +615,7 @@ def curl_oss_fuzz_cq(curl_oss_fuzz_ct: ChallengeTask) -> Iterator[CodeQuery]:
return CodeQuery(challenge=curl_oss_fuzz_ct)
@pytest.mark.skip(reason="Target not currently publicly available")
@pytest.mark.integration
def test_find_harness_in_curl(curl_oss_fuzz_cq: CodeQuery):
harnesses = find_libfuzzer_harnesses(curl_oss_fuzz_cq)
@@ -631,6 +632,7 @@ def test_find_harness_in_curl(curl_oss_fuzz_cq: CodeQuery):
assert "driver.c" in {h.name for h in harnesses}
@pytest.mark.skip(reason="Target not currently publicly available")
@pytest.mark.integration
@pytest.mark.parametrize(
"harness_name,expected_first_match",
@@ -665,6 +667,7 @@ def test_get_harness_source_candidates_curl(
assert expected_first_match == harnesses[0].name
@pytest.mark.skip(reason="Target not currently publicly available")
@pytest.mark.integration
@pytest.mark.parametrize(
"harness_name,expected_harness_source_path,coverage_map_function_paths",