From 2be6f9154512908118064d5600a6d3ff068192f6 Mon Sep 17 00:00:00 2001 From: Evan Downing <2077950+evandowning@users.noreply.github.com> Date: Wed, 27 Aug 2025 15:26:38 -0400 Subject: [PATCH] Parse result logs (#326) * Add doc on parsing logs and scripts to unpack dataset * Reorganize scripts * Reorg markdown files. --- CONTRIBUTING.md | 6 +- README.md | 29 ++++---- common/src/buttercup/common/util_cli.py | 5 ++ AKS_DEPLOYMENT.md => guides/AKS_DEPLOYMENT.md | 10 +-- .../CUSTOM_CHALLENGES.md | 7 +- MANUAL_SETUP.md => guides/MANUAL_SETUP.md | 2 +- .../QUICK_REFERENCE.md | 20 +++++- guides/SCORED.md | 69 +++++++++++++++++++ UNSCORED.md => guides/UNSCORED.md | 2 +- scripts/results_find.sh | 8 +++ scripts/results_unpack.sh | 20 ++++++ 11 files changed, 151 insertions(+), 27 deletions(-) rename AKS_DEPLOYMENT.md => guides/AKS_DEPLOYMENT.md (94%) rename CUSTOM_CHALLENGES.md => guides/CUSTOM_CHALLENGES.md (99%) rename MANUAL_SETUP.md => guides/MANUAL_SETUP.md (99%) rename QUICK_REFERENCE.md => guides/QUICK_REFERENCE.md (98%) create mode 100644 guides/SCORED.md rename UNSCORED.md => guides/UNSCORED.md (98%) create mode 100755 scripts/results_find.sh create mode 100755 scripts/results_unpack.sh diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0149102d..9f235ae5 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -111,10 +111,10 @@ kubectl port-forward -n crs service/buttercup-competition-api 31323:1323 ## Resources -- [Quick Reference](QUICK_REFERENCE.md) - Common commands and troubleshooting +- [Quick Reference](guides/QUICK_REFERENCE.md) - Common commands and troubleshooting - [Deployment Guide](deployment/README.md) - Detailed deployment information -- [Custom Challenges](CUSTOM_CHALLENGES.md) - Adding new test cases +- [Custom Challenges](guides/CUSTOM_CHALLENGES.md) - Adding new test cases ## Questions? -Open an issue or reach out to the development team. \ No newline at end of file +Open an issue or reach out to the development team. diff --git a/README.md b/README.md index 82a257e4..83ae4119 100644 --- a/README.md +++ b/README.md @@ -65,7 +65,7 @@ git clone --recurse-submodules https://github.com/trailofbits/buttercup.git cd buttercup ``` -2. Run automated setup (Recommended) +1. Run automated setup (Recommended) ```bash make setup-local @@ -73,15 +73,15 @@ make setup-local This script will install all dependencies, configure the environment, and guide you through the setup process. -**Note:** If you prefer manual setup, see the [Manual Setup Guide](MANUAL_SETUP.md). +**Note:** If you prefer manual setup, see the [Manual Setup Guide](guides/MANUAL_SETUP.md). -3. Start Buttercup locally +1. Start Buttercup locally ```bash make deploy-local ``` -4. Verify local deployment: +1. Verify local deployment: ```bash make status @@ -89,7 +89,7 @@ make status When a deployment is successful, you should see all pods in "Running" or "Completed" status. -5. Send Buttercup a simple task +1. Send Buttercup a simple task **Note:** When tasked, Buttercup will start consuming third-party AI resources. @@ -99,7 +99,7 @@ This command will make Buttercup pull down an example repo [example-libpng](http make send-libpng-task ``` -6. Access Buttercup's web-based GUI +1. Access Buttercup's web-based GUI Run: @@ -111,7 +111,7 @@ Then navigate to `http://localhost:31323` in your web browser. In the GUI you can monitor active tasks and see when Buttercup finds bugs and generates patches for them. -7. Stop Buttercup +1. Stop Buttercup **Note:** This is an important step to ensure Buttercup shuts down and stops consuming third-party AI resources. @@ -128,20 +128,23 @@ make signoz-ui ``` Then navigate to `http://localhost:33301` in your web browser to view: + - Distributed traces -- Application metrics +- Application metrics - Error monitoring - Performance insights If you configured LangFuse during setup, you can also monitor LLM usage and costs there. -For additional log access methods, see the [Quick Reference Guide](QUICK_REFERENCE.md). +For additional log access methods, see the [Quick Reference Guide](guides/QUICK_REFERENCE.md). ## Additional Resources -- [Quick Reference Guide](QUICK_REFERENCE.md) - Common commands and troubleshooting -- [Manual Setup Guide](MANUAL_SETUP.md) - Detailed manual installation steps -- [AKS Deployment Guide](AKS_DEPLOYMENT.md) - Production deployment on Azure +- [Quick Reference Guide](guides/QUICK_REFERENCE.md) - Common commands and troubleshooting +- [Manual Setup Guide](guides/MANUAL_SETUP.md) - Detailed manual installation steps +- [AKS Deployment Guide](guides/AKS_DEPLOYMENT.md) - Production deployment on Azure - [Contributing Guidelines](CONTRIBUTING.md) - Development workflow and standards - [Deployment Documentation](deployment/README.md) - Advanced deployment configuration -- [Writing Custom Challenges](CUSTOM_CHALLENGES.md) - Custom project configuration and setup +- [Writing Custom Challenges](guides/CUSTOM_CHALLENGES.md) - Custom project configuration and setup +- [Unscored rounds](guides/UNSCORED.md) - Running unscored round challenges +- [Scored round](guides/SCORED.md) - Parsing post-final round results diff --git a/common/src/buttercup/common/util_cli.py b/common/src/buttercup/common/util_cli.py index beb92712..7179e792 100644 --- a/common/src/buttercup/common/util_cli.py +++ b/common/src/buttercup/common/util_cli.py @@ -96,6 +96,7 @@ class ReadBuildsSettings(BaseModel): class ReadSubmissionsSettings(BaseModel): + task_id: str = Field(default="", description="Task ID") verbose: bool = Field(False, description="Show full stacktraces instead of truncated versions") filter_stop: bool = Field(False, description="Filter out submissions that are stopped") @@ -228,6 +229,10 @@ def handle_subcommand(redis: Redis, command: BaseModel | None) -> None: logger.error(f"Task {task_id} not found in registry") continue + if command.task_id and task_id != command.task_id: + logger.info(f"Skipping submission {i} for task {task_id} (not {command.task_id})") + continue + if task_id not in result: result[task_id] = TaskResult( task_id=task_id, diff --git a/AKS_DEPLOYMENT.md b/guides/AKS_DEPLOYMENT.md similarity index 94% rename from AKS_DEPLOYMENT.md rename to guides/AKS_DEPLOYMENT.md index 8d111f96..15219f8e 100644 --- a/AKS_DEPLOYMENT.md +++ b/guides/AKS_DEPLOYMENT.md @@ -1,8 +1,8 @@ # Production AKS Deployment Guide -> **⚠️ Notice:** -> The following production deployment instructions have **not been fully tested**. -> Please proceed with caution and verify each step in your environment. +> **⚠️ Notice:** +> The following production deployment instructions have **not been fully tested**. +> Please proceed with caution and verify each step in your environment. > If you encounter issues, consult the script comments and configuration files for troubleshooting. Full production deployment of the **Buttercup CRS** on Azure Kubernetes Service with proper networking, monitoring, and scaling for the DARPA AIxCC competition. @@ -81,7 +81,7 @@ cd deployment && make up ## Additional Resources -For detailed deployment instructions and advanced configuration options, see the [deployment README](deployment/README.md). +For detailed deployment instructions and advanced configuration options, see the [deployment README](../deployment/README.md). ## Troubleshooting @@ -102,4 +102,4 @@ az aks get-credentials --name --resource-group az aks show --name --resource-group ``` -For more troubleshooting information, see the main [Quick Reference Guide](QUICK_REFERENCE.md). \ No newline at end of file +For more troubleshooting information, see the main [Quick Reference Guide](QUICK_REFERENCE.md). diff --git a/CUSTOM_CHALLENGES.md b/guides/CUSTOM_CHALLENGES.md similarity index 99% rename from CUSTOM_CHALLENGES.md rename to guides/CUSTOM_CHALLENGES.md index 865a3cd0..c3cd7f11 100644 --- a/CUSTOM_CHALLENGES.md +++ b/guides/CUSTOM_CHALLENGES.md @@ -350,11 +350,13 @@ Buttercup determines which mode to use based on your challenge configuration: ### Choosing the Right Mode **Choose Full Mode when you want to:** + - Perform an initial assessment of a new project - Search for vulnerabilities in all code reachable from harnesses - Analyze a project without specific change targets **Choose Delta Mode when you want to:** + - Test specific code changes - Reproduce known vulnerabilities @@ -369,8 +371,7 @@ Now that you understand how to create custom challenges, you can: 3. **Monitor progress**: Use the debugging techniques to track your challenge's execution 4. **Iterate**: Refine your harnesses and build scripts based on the results -For additional help, consult the main [Buttercup documentation](README.md) or check the [troubleshooting guide](CLAUDE.md#common-debugging-commands). - +For additional help, consult the main [Buttercup documentation](../README.md) or check the [troubleshooting guide](../CLAUDE.md#common-debugging-commands). [1]: https://aicyberchallenge.com/ -[2]: https://google.github.io/oss-fuzz/getting-started/new-project-guide/ \ No newline at end of file +[2]: https://google.github.io/oss-fuzz/getting-started/new-project-guide/ diff --git a/MANUAL_SETUP.md b/guides/MANUAL_SETUP.md similarity index 99% rename from MANUAL_SETUP.md rename to guides/MANUAL_SETUP.md index 6118816f..053e1ed7 100644 --- a/MANUAL_SETUP.md +++ b/guides/MANUAL_SETUP.md @@ -133,4 +133,4 @@ helm repo update helm dependency update deployment/k8s/ ``` -For additional troubleshooting, see the [Quick Reference Guide](QUICK_REFERENCE.md). \ No newline at end of file +For additional troubleshooting, see the [Quick Reference Guide](QUICK_REFERENCE.md). diff --git a/QUICK_REFERENCE.md b/guides/QUICK_REFERENCE.md similarity index 98% rename from QUICK_REFERENCE.md rename to guides/QUICK_REFERENCE.md index bfb42a26..872400e7 100644 --- a/QUICK_REFERENCE.md +++ b/guides/QUICK_REFERENCE.md @@ -3,6 +3,7 @@ ## Setup Commands ### Local Development + ```bash # Automated setup ./scripts/setup-local.sh @@ -14,6 +15,7 @@ cd deployment && make up ``` ### Production AKS + ```bash # Automated setup ./scripts/setup-azure.sh @@ -27,6 +29,7 @@ cd deployment && make up ## Common Commands ### Kubernetes Management + ```bash # View all resources kubectl get pods -A @@ -51,6 +54,7 @@ kubectl exec -it -n crs -- /bin/bash ``` ### Testing + ```bash # Send test task ./orchestrator/scripts/task_crs.sh @@ -63,6 +67,7 @@ kubectl exec -it -n crs -- /bin/bash ``` ### Development + ```bash # Lint Python code make lint @@ -74,6 +79,7 @@ docker-compose --profile fuzzer-test up ``` ### Minikube + ```bash # Start/stop minikube start --driver=docker @@ -86,6 +92,7 @@ minikube dashboard ``` ### Azure AKS + ```bash # Get credentials az aks get-credentials --name --resource-group @@ -105,6 +112,7 @@ az aks show --name --resource-group ### Common Issues #### Minikube Issues + ```bash # Reset minikube minikube delete @@ -116,6 +124,7 @@ kubectl cluster-info ``` #### Docker Issues + ```bash # Permission issues sudo usermod -aG docker $USER @@ -127,6 +136,7 @@ sudo systemctl start docker ``` #### Helm Issues + ```bash # Update repositories helm repo update @@ -137,6 +147,7 @@ helm lint deployment/k8s/ ``` #### Azure Issues + ```bash # Authentication az login --tenant @@ -147,6 +158,7 @@ az ad sp list --display-name "ButtercupCRS*" ``` #### Kubernetes Issues + ```bash # Check cluster connectivity kubectl cluster-info @@ -163,16 +175,19 @@ kubectl get events -n crs --sort-by='.lastTimestamp' ### Log Analysis #### Check Patch Submission + ```bash kubectl logs -n crs -l app=scheduler --tail=-1 --prefix | grep "WAIT_PATCH_PASS -> SUBMIT_BUNDLE" ``` #### Check Competition API + ```bash kubectl logs -n crs -l app=competition-api --tail=-1 --prefix ``` #### Check Fuzzer + ```bash kubectl logs -n crs -l app=fuzzer --tail=-1 --prefix ``` @@ -180,16 +195,19 @@ kubectl logs -n crs -l app=fuzzer --tail=-1 --prefix ## File Locations ### Configuration + - `deployment/env` - Main configuration file - `deployment/env.template` - Configuration template - `deployment/k8s/values-*.template` - Kubernetes value templates ### Scripts + - `scripts/setup-local.sh` - Local development setup - `scripts/setup-azure.sh` - Production AKS setup - `orchestrator/scripts/` - Testing and task scripts ### Documentation + - `README.md` - Main documentation - `deployment/README.md` - Detailed deployment guide @@ -198,4 +216,4 @@ kubectl logs -n crs -l app=fuzzer --tail=-1 --prefix - Check logs: `kubectl logs -n crs ` - View events: `kubectl get events -n crs` - Debug pods: `kubectl exec -it -n crs -- /bin/bash` -- Monitor resources: `kubectl top pods -A` +- Monitor resources: `kubectl top pods -A` diff --git a/guides/SCORED.md b/guides/SCORED.md new file mode 100644 index 00000000..9e18c767 --- /dev/null +++ b/guides/SCORED.md @@ -0,0 +1,69 @@ +# Scored + +This document describes how to parse logs from the final round of the AIxCC competition. + +**NOTE**: PoVs were not released to us immediately because DARPA is still involved in responsible disclosure: . + +## Data + +Download the following files from the competition to `buttercup/data/` + +- `litellm_backup.sql` +- `redis-backup.rdb` +- `task-storage.tar` + +## Get Task IDs for Specific Projects + +For example, find the Task IDs for `libpng`. + +```shell +cd data/ +../scripts/results_unpack.sh +../scripts/results_find.sh libpng +``` + +## Parse + +Query the CRS database for submissions. Documentation can be found at . + +Let's assume a task ID `0197b76a-b429-78fa-8d0e-ad84994c8f44` was outputted from above. + +1. Start `redis-server` in `data/` directory + + ```shell + cd data/ + redis-server + ``` + +1. Print directory to confirm + + ```shell + redis-cli config get dir + 1) "dir" + 2) "" + ``` + +1. Stop `redis-server` and check if `dump.rdb` exists in `data/` directory. + +1. Copy `.rdb` file + + ```shell + cp redis-backup.rdb dump.rdb + ``` + +1. Start `redis-server` and query to make sure it has data. + + ```shell + redis-server + redis-cli keys \* + ``` + +1. Parse database for task id + + ```shell + cd common/ + uv sync --all-extras + uv run buttercup-util read_submissions --task_id 0197b76a-b429-78fa-8d0e-ad84994c8f44 --verbose True > ../data/submissions.txt + ``` + +1. Read through `submissions.txt` for information like crash harnesses, stacktraces, and patches. diff --git a/UNSCORED.md b/guides/UNSCORED.md similarity index 98% rename from UNSCORED.md rename to guides/UNSCORED.md index a81f3050..b7e552a7 100644 --- a/UNSCORED.md +++ b/guides/UNSCORED.md @@ -9,7 +9,7 @@ A list of vulnerabilities to find and patch is in this [spreadsheet](https://doc ## Setup -* Set up minikube locally on your machine (see [README.md](README.md)). +* Set up minikube locally on your machine (see [README.md](../README.md)). * `cd deployment && make up` diff --git a/scripts/results_find.sh b/scripts/results_find.sh new file mode 100755 index 00000000..c9c16407 --- /dev/null +++ b/scripts/results_find.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +if [ "$#" -ne 1 ]; then + echo "usage: $0 " + exit 1 +fi + +find tasks_storage -name "task_meta.json" -exec grep -rIil "$1" {} \; | cut -d '/' -f 1-2 | sort -u \ No newline at end of file diff --git a/scripts/results_unpack.sh b/scripts/results_unpack.sh new file mode 100755 index 00000000..863ea39d --- /dev/null +++ b/scripts/results_unpack.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +if [ "$#" -ne 0 ]; then + echo "usage: $0" + exit 1 +fi + +# Clean up +rm -rf tasks_storage + +# Unpack +tar xvf tasks_storage.tar + +# Decompress +for fn in `find tasks_storage -name *.tgz`; do + dst="${fn%.tgz}" + mkdir $dst + tar xvzf $fn -C $dst + rm $fn +done \ No newline at end of file