From 71032d2d4c133ea740926f2b956d52ba91fbf82f Mon Sep 17 00:00:00 2001
From: Emmanuel Ferdman <emmanuelferdman@gmail.com>
Date: Mon, 25 Aug 2025 16:58:09 +0300
Subject: [PATCH 1/5] fix: resolve pydantic deprecation warnings (#321)

Signed-off-by: Emmanuel Ferdman <emmanuelferdman@gmail.com>
---
 orchestrator/test/test_crs_client.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/orchestrator/test/test_crs_client.py b/orchestrator/test/test_crs_client.py
index 7c97128c..a62d5d9a 100644
--- a/orchestrator/test/test_crs_client.py
+++ b/orchestrator/test/test_crs_client.py
@@ -61,7 +61,7 @@ class TestCRSClient:
         # Verify request was made correctly
         mock_post.assert_called_once_with(
             "http://test-crs:8080/v1/task/",
-            json=sample_task.dict(),
+            json=sample_task.model_dump(),
             auth=("test_user", "test_pass"),
             headers={"Content-Type": "application/json"},
             timeout=30,
@@ -82,7 +82,7 @@ class TestCRSClient:
         # Verify request was made without auth
         mock_post.assert_called_once_with(
             "http://test-crs:8080/v1/task/",
-            json=sample_task.dict(),
+            json=sample_task.model_dump(),
             auth=None,
             headers={"Content-Type": "application/json"},
             timeout=30,

From c25b3956d0531ae4c0dcbe597f7c881cf30939cb Mon Sep 17 00:00:00 2001
From: Boyan MILANOV <boyan.milanov@trailofbits.com>
Date: Tue, 26 Aug 2025 16:59:00 +0200
Subject: [PATCH 2/5] Support Gemini (#306)

* Add gemini api key option during setup and in deployment environments

* Add gemini pro as a fallback model in all components

* Lint

* Set rate limits for gemini models

* Add fallback models in more places

* Fix typo

* Fix kwargs expansion

* Fix instantiation of llm with callbacks

* Fixed formatting after merge

* Fix instantiation of default models

* Lint

* Fix llm creation

* Fix

* Lint

* Lint

---------

Co-authored-by: Michael D Brown <michael.brown@trailofbits.com>
---
 .github/ci-env.template                       |  1 +
 .github/workflows/integration.yml             |  1 +
 common/src/buttercup/common/llm.py            | 45 ++++++++++++-------
 deployment/k8s/README.md                      |  6 ++-
 deployment/k8s/templates/litellm-secret.yaml  |  1 +
 deployment/k8s/values-aks.template            |  2 +
 deployment/k8s/values-ci.template             |  2 +
 deployment/k8s/values-minikube.template       |  2 +
 deployment/k8s/values-prod.template           |  2 +
 .../k8s/values-upstream-minikube.template     |  2 +
 deployment/k8s/values.yaml                    | 22 +++++++++
 litellm/litellm_config.yaml                   | 21 +++++++++
 .../src/buttercup/patcher/agents/common.py    |  5 ++-
 .../patcher/agents/context_retriever.py       |  6 ++-
 patcher/src/buttercup/patcher/agents/qe.py    |  2 +-
 .../buttercup/patcher/agents/reflection.py    |  4 +-
 .../src/buttercup/patcher/agents/rootcause.py |  5 ++-
 patcher/src/buttercup/patcher/agents/swe.py   |  1 +
 scripts/common.sh                             | 18 +++++++-
 seed-gen/src/buttercup/seed_gen/task.py       |  1 +
 20 files changed, 122 insertions(+), 27 deletions(-)

diff --git a/.github/ci-env.template b/.github/ci-env.template
index c53150b0..907c229d 100644
--- a/.github/ci-env.template
+++ b/.github/ci-env.template
@@ -21,6 +21,7 @@ export BUTTERCUP_NAMESPACE=replace-me
 export GHCR_AUTH=replace-me
 export OPENAI_API_KEY=replace-me
 export ANTHROPIC_API_KEY=replace-me
+export GEMINI_API_KEY=replace-me
 export LANGFUSE_HOST=replace-me
 export LANGFUSE_PUBLIC_KEY=replace-me
 export LANGFUSE_SECRET_KEY=replace-me
diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml
index 815655da..5172b4db 100644
--- a/.github/workflows/integration.yml
+++ b/.github/workflows/integration.yml
@@ -93,6 +93,7 @@ jobs:
           sed -i "s|BUTTERCUP_NAMESPACE=.*|BUTTERCUP_NAMESPACE=${{ env.BUTTERCUP_NAMESPACE }}|" env
           sed -i "s|OPENAI_API_KEY=.*|OPENAI_API_KEY=${{ secrets.OPENAI_API_KEY }}|" env
           sed -i "s|ANTHROPIC_API_KEY=.*|ANTHROPIC_API_KEY=${{ secrets.ANTHROPIC_API_KEY }}|" env
+          sed -i "s|GEMINI_API_KEY=.*|GEMINI_API_KEY=${{ secrets.GEMINI_API_KEY }}|" env
           sed -i "s|LANGFUSE_HOST=.*|LANGFUSE_HOST=${{ secrets.LANGFUSE_HOST }}|" env
           sed -i "s|LANGFUSE_PUBLIC_KEY=.*|LANGFUSE_PUBLIC_KEY=${{ secrets.LANGFUSE_PUBLIC_KEY }}|" env
           sed -i "s|LANGFUSE_SECRET_KEY=.*|LANGFUSE_SECRET_KEY=${{ secrets.LANGFUSE_SECRET_KEY }}|" env
diff --git a/common/src/buttercup/common/llm.py b/common/src/buttercup/common/llm.py
index 661024fd..cf7023e2 100644
--- a/common/src/buttercup/common/llm.py
+++ b/common/src/buttercup/common/llm.py
@@ -7,7 +7,7 @@ from typing import Any
 import requests
 from langchain.callbacks.base import BaseCallbackHandler
 from langchain_core.language_models import BaseChatModel
-from langchain_core.runnables import ConfigurableField
+from langchain_core.runnables import ConfigurableField, Runnable
 from langchain_openai.chat_models import ChatOpenAI
 from langfuse.callback import CallbackHandler
 
@@ -32,6 +32,9 @@ class ButtercupLLM(Enum):
     CLAUDE_3_5_SONNET = "claude-3.5-sonnet"
     CLAUDE_3_7_SONNET = "claude-3.7-sonnet"
     CLAUDE_4_SONNET = "claude-4-sonnet"
+    GEMINI_PRO = "gemini-pro"
+    GEMINI_2_5_FLASH = "gemini-2.5-flash"
+    GEMINI_2_5_FLASH_EXP = "gemini-2.5-flash-exp"
 
 
 @functools.cache
@@ -94,31 +97,41 @@ def get_langfuse_callbacks() -> list[BaseCallbackHandler]:
     return []
 
 
-def create_default_llm(**kwargs: Any) -> BaseChatModel:
+def create_default_llm(**kwargs: Any) -> Runnable:
     """Create an LLM object with the default configuration."""
+    fallback_models = kwargs.pop("fallback_models", [])
+    fallback_models = [create_default_llm(**{**kwargs, "model_name": m.value}) for m in fallback_models]
     return create_llm(
         model_name=kwargs.pop("model_name", ButtercupLLM.OPENAI_GPT_4_1.value),
         temperature=kwargs.pop("temperature", 0.1),
         timeout=420.0,
         max_retries=3,
         **kwargs,
-    )
+    ).with_fallbacks(fallback_models)
 
 
-def create_default_llm_with_temperature(**kwargs: Any) -> BaseChatModel:
+def create_default_llm_with_temperature(**kwargs: Any) -> Runnable:
     """Create an LLM object with the default configuration and temperature."""
-    return create_llm(  # type: ignore
-        model_name=kwargs.pop("model_name", ButtercupLLM.OPENAI_GPT_4_1.value),
-        temperature=kwargs.pop("temperature", 0.1),
-        timeout=420.0,
-        max_retries=3,
-        **kwargs,
-    ).configurable_fields(
-        temperature=ConfigurableField(
-            id="llm_temperature",
-            name="LLM temperature",
-            description="The temperature for the LLM model",
-        ),
+    fallback_models = kwargs.pop("fallback_models", [])
+    fallback_models = [
+        create_default_llm_with_temperature(**{**kwargs, "model_name": m.value}) for m in fallback_models
+    ]
+    return (
+        create_llm(
+            model_name=kwargs.pop("model_name", ButtercupLLM.OPENAI_GPT_4_1.value),
+            temperature=kwargs.pop("temperature", 0.1),
+            timeout=420.0,
+            max_retries=3,
+            **kwargs,
+        )
+        .configurable_fields(
+            temperature=ConfigurableField(
+                id="llm_temperature",
+                name="LLM temperature",
+                description="The temperature for the LLM model",
+            ),
+        )
+        .with_fallbacks(fallback_models)
     )
 
 
diff --git a/deployment/k8s/README.md b/deployment/k8s/README.md
index 66fad1f3..559117f4 100644
--- a/deployment/k8s/README.md
+++ b/deployment/k8s/README.md
@@ -8,7 +8,7 @@ This guide explains how to set up and run the Buttercup system on Kubernetes.
 - kubectl configured to communicate with your cluster
 - Helm v3
 - Access to container registries (ghcr.io)
-- Access to required API keys (OpenAI, Azure, Anthropic)
+- Access to required API keys (OpenAI, Azure, Anthropic, Gemini)
 
 ## Environment Configuration
 
@@ -43,6 +43,8 @@ This is controlled via the `global.environment` setting in the values.yaml or va
        apiKey: "your-openai-api-key"
      anthropic: 
        apiKey: "your-anthropic-api-key"
+     gemini:
+       apiKey: "your-google-gemini-key"
 
    crs:
       api_key_id: 515cc8a0-3019-4c9f-8c1c-72d0b54ae561
@@ -77,6 +79,7 @@ This secret contains API keys for LLM providers:
 - **AZURE_API_KEY**: Azure OpenAI API key
 - **OPENAI_API_KEY**: OpenAI API key
 - **ANTHROPIC_API_KEY**: Anthropic API key
+- **GEMINI_API_KEY**: Google Gemini API key
 
 These are populated from the `values-override.yaml` file.
 
@@ -197,6 +200,7 @@ The system uses LiteLLM to proxy all LLM requests. The configuration includes mo
 - Azure OpenAI
 - OpenAI
 - Anthropic
+- Google Gemini
 
 To modify the available models, edit the `litellm-helm.proxy_config.model_list` section in values.yaml.
 
diff --git a/deployment/k8s/templates/litellm-secret.yaml b/deployment/k8s/templates/litellm-secret.yaml
index b1138737..c7af4c28 100644
--- a/deployment/k8s/templates/litellm-secret.yaml
+++ b/deployment/k8s/templates/litellm-secret.yaml
@@ -13,5 +13,6 @@ data:
   AZURE_API_KEY: {{ .Values.litellm.azure.apiKey | b64enc | quote }}
   OPENAI_API_KEY: {{ .Values.litellm.openai.apiKey | b64enc | quote }}
   ANTHROPIC_API_KEY: {{ .Values.litellm.anthropic.apiKey | b64enc | quote }}
+  GEMINI_API_KEY: {{ .Values.litellm.gemini.apiKey | b64enc | quote }}
   BUTTERCUP_LITELLM_KEY: {{ .Values.litellm.masterKey | b64enc | quote }}
 {{- end }}
diff --git a/deployment/k8s/values-aks.template b/deployment/k8s/values-aks.template
index 1665ec9d..3ebc9f10 100644
--- a/deployment/k8s/values-aks.template
+++ b/deployment/k8s/values-aks.template
@@ -34,6 +34,8 @@ litellm:
     apiKey: "${OPENAI_API_KEY}"
   anthropic:
     apiKey: "${ANTHROPIC_API_KEY}"
+  gemini:
+    apiKey: "${GEMINI_API_KEY}"
 
 competition-api:
   enabled: true
diff --git a/deployment/k8s/values-ci.template b/deployment/k8s/values-ci.template
index 1bcc86da..76ea5c34 100644
--- a/deployment/k8s/values-ci.template
+++ b/deployment/k8s/values-ci.template
@@ -53,6 +53,8 @@ litellm:
     apiKey: "${OPENAI_API_KEY}"
   anthropic:
     apiKey: "${ANTHROPIC_API_KEY}"
+  gemini:
+    apiKey: "${GEMINI_API_KEY}"
 
 competition-api:
   enabled: ${COMPETITION_API_ENABLED}
diff --git a/deployment/k8s/values-minikube.template b/deployment/k8s/values-minikube.template
index cbf92e08..3e90b366 100644
--- a/deployment/k8s/values-minikube.template
+++ b/deployment/k8s/values-minikube.template
@@ -57,6 +57,8 @@ litellm:
     apiKey: "${OPENAI_API_KEY}"
   anthropic:
     apiKey: "${ANTHROPIC_API_KEY}"
+  gemini:
+    apiKey: "${GEMINI_API_KEY}"
 
 competition-api:
   enabled: ${COMPETITION_API_ENABLED}
diff --git a/deployment/k8s/values-prod.template b/deployment/k8s/values-prod.template
index 4c0ca80f..6b838b86 100644
--- a/deployment/k8s/values-prod.template
+++ b/deployment/k8s/values-prod.template
@@ -228,6 +228,8 @@ litellm:
     apiKey: "${OPENAI_API_KEY}"
   anthropic:
     apiKey: "${ANTHROPIC_API_KEY}"
+  gemini:
+    apiKey: "${GEMINI_API_KEY}"
 
 litellm-helm:
   resources:
diff --git a/deployment/k8s/values-upstream-minikube.template b/deployment/k8s/values-upstream-minikube.template
index ae55dd0d..12e41fef 100644
--- a/deployment/k8s/values-upstream-minikube.template
+++ b/deployment/k8s/values-upstream-minikube.template
@@ -63,6 +63,8 @@ litellm:
     apiKey: "${OPENAI_API_KEY}"
   anthropic:
     apiKey: "${ANTHROPIC_API_KEY}"
+  gemini:
+    apiKey: "${GEMINI_API_KEY}"
 
 ui:
   enabled: true
diff --git a/deployment/k8s/values.yaml b/deployment/k8s/values.yaml
index e00bb381..269c0ecc 100644
--- a/deployment/k8s/values.yaml
+++ b/deployment/k8s/values.yaml
@@ -375,6 +375,28 @@ litellm-helm:
           api_key: os.environ/ANTHROPIC_API_KEY
           tpm: 1775000
 
+      - model_name: gemini-pro	
+        litellm_params:
+          model: gemini/gemini-pro	
+          api_key: os.environ/GEMINI_API_KEY
+          tpm: 2000000
+          rpm: 150
+        
+      - model_name: gemini-2.5-flash-exp
+        litellm_params:
+          model: gemini/gemini-2.5-flash-exp
+          api_key: os.environ/GEMINI_API_KEY
+          tpm: 4000000
+          rpm: 2000
+
+      - model_name: gemini-2.5-flash
+        litellm_params:
+          model: gemini/gemini-2.5-flash
+          api_key: os.environ/GEMINI_API_KEY
+          tpm: 4000000
+          rpm: 2000
+
+
     general_settings:
       master_key: os.environ/BUTTERCUP_LITELLM_KEY
       database_url: "postgresql://litellm_user:litellm_password11@buttercup-postgresql:5432/litellm"
diff --git a/litellm/litellm_config.yaml b/litellm/litellm_config.yaml
index 362a37ab..790cb4f4 100644
--- a/litellm/litellm_config.yaml
+++ b/litellm/litellm_config.yaml
@@ -109,5 +109,26 @@ model_list:
       api_key: os.environ/ANTHROPIC_API_KEY
       tpm: 400000
 
+  - model_name: gemini-pro	
+    litellm_params:
+      model: gemini/gemini-pro	
+      api_key: os.environ/GEMINI_API_KEY
+      tpm: 2000000
+      rpm: 150
+    
+  - model_name: gemini-2.5-flash-exp
+    litellm_params:
+      model: gemini/gemini-2.5-flash-exp
+      api_key: os.environ/GEMINI_API_KEY
+      tpm: 4000000
+      rpm: 2000
+
+  - model_name: gemini-2.5-flash
+    litellm_params:
+      model: gemini/gemini-2.5-flash
+      api_key: os.environ/GEMINI_API_KEY
+      tpm: 4000000
+      rpm: 2000
+
 general_settings:
   master_key: os.environ/BUTTERCUP_LITELLM_KEY
diff --git a/patcher/src/buttercup/patcher/agents/common.py b/patcher/src/buttercup/patcher/agents/common.py
index 85b16ca1..21a4b963 100644
--- a/patcher/src/buttercup/patcher/agents/common.py
+++ b/patcher/src/buttercup/patcher/agents/common.py
@@ -434,7 +434,10 @@ UNDERSTAND_CODE_SNIPPET_PROMPT = ChatPromptTemplate.from_messages(
 def _create_understand_code_snippet_chain() -> Runnable:
     return (  # type: ignore[no-any-return]
         UNDERSTAND_CODE_SNIPPET_PROMPT
-        | create_default_llm_with_temperature(model_name=ButtercupLLM.OPENAI_GPT_4_1.value)
+        | create_default_llm_with_temperature(
+            model_name=ButtercupLLM.OPENAI_GPT_4_1.value,
+            fallback_models=[ButtercupLLM.CLAUDE_4_SONNET, ButtercupLLM.GEMINI_PRO],
+        )
         | StrOutputParser()
     )
 
diff --git a/patcher/src/buttercup/patcher/agents/context_retriever.py b/patcher/src/buttercup/patcher/agents/context_retriever.py
index ded64880..80017cbf 100644
--- a/patcher/src/buttercup/patcher/agents/context_retriever.py
+++ b/patcher/src/buttercup/patcher/agents/context_retriever.py
@@ -585,7 +585,10 @@ Remember to call the `test_instructions` tool to log and validate any test comma
 
 def _are_test_instructions_valid(instructions: str, output: bytes, error: bytes) -> bool:
     """Validate a set of test instructions by executing them inside the project environment."""
-    llm = create_default_llm(model_name=ButtercupLLM.OPENAI_GPT_4_1.value)
+    llm = create_default_llm(
+        model_name=ButtercupLLM.OPENAI_GPT_4_1.value,
+        fallback_models=[ButtercupLLM.CLAUDE_4_SONNET, ButtercupLLM.GEMINI_PRO],
+    )
     chain = ARE_VALID_TEST_INSTRUCTIONS_PROMPT | llm | StrOutputParser()
     res = chain.invoke(
         {
@@ -708,6 +711,7 @@ class ContextRetrieverAgent(PatcherAgentBase):
         self.cheap_llm = create_default_llm(model_name=ButtercupLLM.OPENAI_GPT_4_1_MINI.value)
         self.cheap_fallback_llms = [
             create_default_llm(model_name=ButtercupLLM.CLAUDE_3_5_SONNET.value),
+            create_default_llm(model_name=ButtercupLLM.GEMINI_PRO.value),
         ]
 
         self.tools = [
diff --git a/patcher/src/buttercup/patcher/agents/qe.py b/patcher/src/buttercup/patcher/agents/qe.py
index 0c512420..9c61cf35 100644
--- a/patcher/src/buttercup/patcher/agents/qe.py
+++ b/patcher/src/buttercup/patcher/agents/qe.py
@@ -157,7 +157,7 @@ class QEAgent(PatcherAgentBase):
                 tools=tools,
                 prompt=self._check_harness_changes_prompt,
             )
-            for llm in [ButtercupLLM.CLAUDE_3_7_SONNET]
+            for llm in [ButtercupLLM.CLAUDE_3_7_SONNET, ButtercupLLM.GEMINI_PRO]
         ]
         self.check_harness_changes_chain = default_agent.with_fallbacks(fallback_agents)
 
diff --git a/patcher/src/buttercup/patcher/agents/reflection.py b/patcher/src/buttercup/patcher/agents/reflection.py
index 0c47c2c9..46e1a5e9 100644
--- a/patcher/src/buttercup/patcher/agents/reflection.py
+++ b/patcher/src/buttercup/patcher/agents/reflection.py
@@ -407,9 +407,7 @@ class ReflectionAgent(PatcherAgentBase):
         """Initialize a few fields"""
         default_llm = create_default_llm(model_name=ButtercupLLM.OPENAI_GPT_4_1.value)
         fallback_llms: list[Runnable] = []
-        for fb_model in [
-            ButtercupLLM.CLAUDE_3_7_SONNET,
-        ]:
+        for fb_model in [ButtercupLLM.CLAUDE_3_7_SONNET, ButtercupLLM.GEMINI_PRO]:
             fallback_llms.append(create_default_llm(model_name=fb_model.value))
 
         self.llm = default_llm.with_fallbacks(fallback_llms)
diff --git a/patcher/src/buttercup/patcher/agents/rootcause.py b/patcher/src/buttercup/patcher/agents/rootcause.py
index 92732d77..42d1358a 100644
--- a/patcher/src/buttercup/patcher/agents/rootcause.py
+++ b/patcher/src/buttercup/patcher/agents/rootcause.py
@@ -135,9 +135,10 @@ class RootCauseAgent(PatcherAgentBase):
         )
         fallback_llms = [
             create_default_llm_with_temperature(
-                model_name=ButtercupLLM.CLAUDE_3_7_SONNET.value,
+                model_name=m.value,
                 **kwargs,
-            ),
+            )
+            for m in [ButtercupLLM.CLAUDE_3_7_SONNET, ButtercupLLM.GEMINI_PRO]
         ]
         self.llm = default_llm.with_fallbacks(fallback_llms)
 
diff --git a/patcher/src/buttercup/patcher/agents/swe.py b/patcher/src/buttercup/patcher/agents/swe.py
index 09f1e84f..a924a252 100644
--- a/patcher/src/buttercup/patcher/agents/swe.py
+++ b/patcher/src/buttercup/patcher/agents/swe.py
@@ -352,6 +352,7 @@ class SWEAgent(PatcherAgentBase):
         fallback_llms: list[Runnable] = []
         for fb_model in [
             ButtercupLLM.CLAUDE_3_7_SONNET,
+            ButtercupLLM.GEMINI_PRO,
         ]:
             fallback_llms.append(create_default_llm_with_temperature(model_name=fb_model.value, **kwargs))
         self.llm = self.default_llm.with_fallbacks(fallback_llms)
diff --git a/scripts/common.sh b/scripts/common.sh
index e08630f8..2825aecb 100755
--- a/scripts/common.sh
+++ b/scripts/common.sh
@@ -497,6 +497,13 @@ configure_local_api_keys() {
     print_status "Generate your API key at: https://console.anthropic.com/settings/keys"
     configure_service "ANTHROPIC_API_KEY" "Anthropic API key" "$ANTHROPIC_API_KEY" "<your-anthropic-api-key>" false
     
+    # Anthropic API Key (Optional)
+    print_linebreak
+    print_status "Google Gemini API Key (Optional): Fallback model."
+    print_status "Use this model as a fallback if other models are not configured or not available."
+    print_status "Generate your API key at: https://aistudio.google.com/apikey"
+    configure_service "GEMINI_API_KEY" "Gemini API key" "$GEMINI_API_KEY" "<your-gemini-api-key>" false
+
     # GitHub Personal Access Token (Optional)
     print_linebreak
     print_status "GitHub Personal Access Token (Optional): Access to private GitHub resources."
@@ -525,9 +532,15 @@ configure_local_api_keys() {
     else
         anthropic_configured=true
     fi
+
+    if [ -z "$GEMINI_API_KEY" ] || [ "$GEMINI_API_KEY" = "<your-gemini-api-key>" ]; then
+        gemini_configured=false
+    else
+        gemini_configured=true
+    fi
     
-    if [ "$openai_configured" = false ] && [ "$anthropic_configured" = false ]; then
-        print_error "At least one LLM API key (OpenAI or Anthropic) must be configured."
+    if [ "$openai_configured" = false ] && [ "$anthropic_configured" = false ] && [ "$gemini_configured" = false ]; then
+        print_error "At least one LLM API key (OpenAI, Anthropic, or Gemini) must be configured."
         print_error "Rerun the setup and set at least one LLM API key."
         return 1
     fi
@@ -632,6 +645,7 @@ check_aks_config() {
     local api_vars=(
         "OPENAI_API_KEY"
         "ANTHROPIC_API_KEY"
+        "GEMINI_API_KEY"
         "GHCR_AUTH"
         "CRS_KEY_ID"
         "CRS_KEY_TOKEN"
diff --git a/seed-gen/src/buttercup/seed_gen/task.py b/seed-gen/src/buttercup/seed_gen/task.py
index cc5e5e1b..335e3e05 100644
--- a/seed-gen/src/buttercup/seed_gen/task.py
+++ b/seed-gen/src/buttercup/seed_gen/task.py
@@ -96,6 +96,7 @@ class Task:
         fallbacks = [
             ButtercupLLM.CLAUDE_3_7_SONNET,
             ButtercupLLM.CLAUDE_3_5_SONNET,
+            ButtercupLLM.GEMINI_PRO,
         ]
         self.llm = Task.get_llm(ButtercupLLM.CLAUDE_4_SONNET, fallbacks)
         self.tools = [

From 2284fc1864ad880cc9022af19c8ad2cd560c2433 Mon Sep 17 00:00:00 2001
From: Michael D Brown <michael.brown@trailofbits.com>
Date: Wed, 27 Aug 2025 15:25:33 -0400
Subject: [PATCH 3/5] Fix / re-org CI (#288)

* renaming system integration tests to something more appropriate

* WIP to isolate unit tests from integration tests

* WIP

* fix typo

* fix after merge from main

* Merged changes from prior PRs into comp-integration

* separated integration tests for components into separate workflow

* fixed label

* cleanup

* put system integration tests back on nightly schedule

* Put component integration tests back on for push to main

* Disable some tests because target is not publicly available.

* appease linter

* trying HTTPS instead of SSH

* disable test with non-public target.

* disable finicky tests

* update label for component integration tests.

---------

Co-authored-by: Michael D. Brown <michaeldbrown@Michaels-MacBook-Pro-4.local>
---
 .github/workflows/comp-integration.yml  | 200 ++++++++++++++++++++++++
 .github/workflows/integration.yml       |   3 +-
 .github/workflows/tests.yml             | 192 -----------------------
 README.md                               |   4 +-
 patcher/tests/test_context_retriever.py |   8 +-
 patcher/tests/test_utils.py             |   1 +
 seed-gen/test/test_find_harness.py      |   3 +
 7 files changed, 213 insertions(+), 198 deletions(-)
 create mode 100644 .github/workflows/comp-integration.yml

diff --git a/.github/workflows/comp-integration.yml b/.github/workflows/comp-integration.yml
new file mode 100644
index 00000000..f0f51a5b
--- /dev/null
+++ b/.github/workflows/comp-integration.yml
@@ -0,0 +1,200 @@
+name: Component Integration Tests
+
+on:
+  push:
+    branches:
+      - main
+    # Always run full test suite on main branch
+  pull_request:
+
+  schedule:
+    # Run integration tests daily at 2 AM UTC
+    - cron: '0 2 * * *'
+  workflow_dispatch:
+    inputs:
+      components:
+        description: 'Components to test (comma-separated: common,patcher,program-model,seed-gen)'
+        required: false
+        default: 'common,patcher,program-model,seed-gen'
+        type: string
+      run_integration:
+        description: 'Run integration tests'
+        required: false
+        default: false
+        type: boolean
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  # Integration tests - run on schedule, manual trigger, or labeled PRs
+  test-integration:
+    # Run if:
+    # - Scheduled run
+    # - Manual dispatch with run_integration=true
+    # - PR with 'integration-tests' label
+    if: |
+      github.event_name == 'push' ||
+      github.event_name == 'schedule' ||
+      (github.event_name == 'workflow_dispatch' && github.event.inputs.run_integration == 'true') ||
+      contains(github.event.pull_request.labels.*.name, 'integration-tests')
+    
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - component: common
+            coverage_module: buttercup.common
+            python: "3.12"
+          - component: patcher
+            coverage_module: buttercup.patcher
+            python: "3.12"
+          - component: program-model
+            coverage_module: buttercup.program_model
+            python: "3.12"
+          - component: seed-gen
+            coverage_module: buttercup.seed_gen
+            python: "3.12"
+
+    runs-on: ubuntu-latest
+    services:
+      redis:
+        image: redis
+        options: >-
+          --health-cmd "redis-cli ping"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 6379:6379
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+          submodules: true
+
+      - name: Check if component should be tested
+        id: should_test
+        if: github.event_name == 'workflow_dispatch'
+        run: |
+          components="${{ github.event.inputs.components }}"
+          if [[ -z "$components" ]] || [[ "$components" == *"${{ matrix.component }}"* ]]; then
+            echo "test=true" >> $GITHUB_OUTPUT
+          else
+            echo "test=false" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Install uv
+        if: steps.should_test.outputs.test != 'false'
+        uses: astral-sh/setup-uv@v6
+
+      - name: Setup uv cache
+        if: steps.should_test.outputs.test != 'false'
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cache/uv
+            ~/.local/share/uv
+          key: ${{ runner.os }}-uv-integration-${{ matrix.component }}-${{ hashFiles(format('{0}/uv.lock', matrix.component)) }}
+          restore-keys: |
+            ${{ runner.os }}-uv-integration-${{ matrix.component }}-
+            ${{ runner.os }}-uv-
+
+      - name: Download Wasm runtime
+        if: matrix.component == 'seed-gen' && steps.should_test.outputs.test != 'false'
+        run: wget https://github.com/vmware-labs/webassembly-language-runtimes/releases/download/python%2F3.12.0%2B20231211-040d5a6/python-3.12.0.wasm
+        working-directory: seed-gen
+
+      - name: Install integration test dependencies
+        if: steps.should_test.outputs.test != 'false'
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y codequery ripgrep
+          make install-cscope
+
+      - name: Prepare environment
+        if: steps.should_test.outputs.test != 'false'
+        run: |
+          export DEBIAN_FRONTEND=noninteractive
+          sudo apt-get update
+          sudo mkdir -p /crs_scratch
+          sudo chmod -R 777 /crs_scratch
+
+      - name: Setup ${{ matrix.component }} component
+        if: steps.should_test.outputs.test != 'false'
+        run: |
+          uv sync --all-extras --frozen
+          uv pip install --isolated pytest-html>=4.1.1 pytest-cov>=6.0.0
+        working-directory: ${{ matrix.component }}
+
+      - name: Run program-model libpng integration test
+        if: matrix.component == 'program-model' && steps.should_test.outputs.test != 'false'
+        run: |
+          uv run --frozen pytest -svv --runintegration tests/c/test_libpng.py \
+            --junit-xml=integration-test-results.xml \
+            --html=integration-test-report.html \
+            --self-contained-html \
+            --cov=${{ matrix.coverage_module }} \
+            --cov-report=xml:integration-coverage.xml \
+            --cov-report=html:integration-htmlcov \
+            --cov-report=term
+        env:
+          PYTHON_WASM_BUILD_PATH: "python-3.12.0.wasm"
+        working-directory: ${{ matrix.component }}
+        timeout-minutes: 30
+      
+      - name: Run integration tests on ${{ matrix.component }}
+        if: matrix.component != 'program-model' && steps.should_test.outputs.test != 'false'
+        run: |
+          uv run --frozen pytest -svv --runintegration \
+            --junit-xml=integration-test-results.xml \
+            --html=integration-test-report.html \
+            --self-contained-html \
+            --cov=${{ matrix.coverage_module }} \
+            --cov-report=xml:integration-coverage.xml \
+            --cov-report=html:integration-htmlcov \
+            --cov-report=term
+        env:
+          PYTHON_WASM_BUILD_PATH: "python-3.12.0.wasm"
+        working-directory: ${{ matrix.component }}
+        timeout-minutes: 30
+
+      - name: Generate integration test summary
+        if: always() && steps.should_test.outputs.test != 'false'
+        run: |
+          echo "### Integration Test Results: ${{ matrix.component }}" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          if [ -f ${{ matrix.component }}/integration-test-results.xml ]; then
+            python -c "
+          import xml.etree.ElementTree as ET
+          tree = ET.parse('${{ matrix.component }}/integration-test-results.xml')
+          root = tree.getroot()
+          tests = root.get('tests', '0')
+          failures = root.get('failures', '0')
+          errors = root.get('errors', '0')
+          skipped = root.get('skipped', '0')
+          time = root.get('time', '0')
+          print(f'- **Total Tests**: {tests}')
+          print(f'- **Passed**: {int(tests) - int(failures) - int(errors) - int(skipped)}')
+          print(f'- **Failed**: {failures}')
+          print(f'- **Errors**: {errors}')
+          print(f'- **Skipped**: {skipped}')
+          print(f'- **Duration**: {float(time):.2f}s')
+          " >> $GITHUB_STEP_SUMMARY
+          else
+            echo "No integration test results found" >> $GITHUB_STEP_SUMMARY
+          fi
+
+      - name: Upload integration test results
+        if: always() && steps.should_test.outputs.test != 'false'
+        uses: actions/upload-artifact@v4
+        with:
+          name: integration-test-results-${{ matrix.component }}-py${{ matrix.python }}
+          path: |
+            ${{ matrix.component }}/integration-test-results.xml
+            ${{ matrix.component }}/integration-test-report.html
+            ${{ matrix.component }}/integration-coverage.xml
+            ${{ matrix.component }}/integration-htmlcov/
+          retention-days: 30
diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml
index 5172b4db..20a16c84 100644
--- a/.github/workflows/integration.yml
+++ b/.github/workflows/integration.yml
@@ -1,4 +1,4 @@
-name: Minikube Integration Tests
+name: System Integration Tests
 
 on:
   # Daily schedule - 3 AM UTC
@@ -64,6 +64,7 @@ jobs:
   integration:
     # Only run on PRs if they have the 'full-integration' label
     if: |
+      github.event_name == 'schedule' ||
       github.event_name != 'pull_request' ||
       contains(github.event.pull_request.labels.*.name, 'full-integration')
     runs-on: gha-ubuntu-8
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 8b35d5c8..9e57befa 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -6,23 +6,6 @@ on:
       - main
     # Always run full test suite on main branch
   pull_request:
-    # TEMPORARILY REMOVED PATH FILTERING TO DEBUG CI
-    # Will re-enable after confirming tests pass
-  schedule:
-    # Run integration tests daily at 2 AM UTC
-    - cron: '0 2 * * *'
-  workflow_dispatch:
-    inputs:
-      components:
-        description: 'Components to test (comma-separated: common,patcher,program-model,seed-gen)'
-        required: false
-        default: 'common,patcher,program-model,seed-gen'
-        type: string
-      run_integration:
-        description: 'Run integration tests'
-        required: false
-        default: false
-        type: boolean
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
@@ -52,12 +35,6 @@ jobs:
           - component: fuzzer
             coverage_module: buttercup.fuzzer
             python: "3.12"
-        # TODO: Add integration tests back in
-        # include:
-        #   - component: common
-        #     pytest_args: "--runintegration"
-        #   - component: program-model
-        #     pytest_args: "--runintegration"
 
     runs-on: ubuntu-latest
     # Removed if: matrix.should_run since we're not using path filtering right now
@@ -206,172 +183,3 @@ jobs:
           fail_ci_if_error: false
           verbose: true
 
-  # Integration tests - run on schedule, manual trigger, or labeled PRs
-  test-integration:
-    # Run if:
-    # - Scheduled run
-    # - Manual dispatch with run_integration=true
-    # - PR with 'integration-tests' label
-    if: |
-      github.event_name == 'schedule' ||
-      (github.event_name == 'workflow_dispatch' && github.event.inputs.run_integration == 'true') ||
-      contains(github.event.pull_request.labels.*.name, 'integration-tests')
-    
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - component: common
-            coverage_module: buttercup.common
-            python: "3.12"
-          - component: patcher
-            coverage_module: buttercup.patcher
-            python: "3.12"
-          - component: program-model
-            coverage_module: buttercup.program_model
-            python: "3.12"
-          - component: seed-gen
-            coverage_module: buttercup.seed_gen
-            python: "3.12"
-
-    runs-on: ubuntu-latest
-    services:
-      redis:
-        image: redis
-        options: >-
-          --health-cmd "redis-cli ping"
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-        ports:
-          - 6379:6379
-
-    steps:
-      - uses: actions/checkout@v5
-        with:
-          persist-credentials: false
-          submodules: true
-
-      - name: Check if component should be tested
-        id: should_test
-        if: github.event_name == 'workflow_dispatch'
-        run: |
-          components="${{ github.event.inputs.components }}"
-          if [[ -z "$components" ]] || [[ "$components" == *"${{ matrix.component }}"* ]]; then
-            echo "test=true" >> $GITHUB_OUTPUT
-          else
-            echo "test=false" >> $GITHUB_OUTPUT
-          fi
-
-      - name: Install uv
-        if: steps.should_test.outputs.test != 'false'
-        uses: astral-sh/setup-uv@v6
-
-      - name: Setup uv cache
-        if: steps.should_test.outputs.test != 'false'
-        uses: actions/cache@v4
-        with:
-          path: |
-            ~/.cache/uv
-            ~/.local/share/uv
-          key: ${{ runner.os }}-uv-integration-${{ matrix.component }}-${{ hashFiles(format('{0}/uv.lock', matrix.component)) }}
-          restore-keys: |
-            ${{ runner.os }}-uv-integration-${{ matrix.component }}-
-            ${{ runner.os }}-uv-
-
-      - name: Download Wasm runtime
-        if: matrix.component == 'seed-gen' && steps.should_test.outputs.test != 'false'
-        run: wget https://github.com/vmware-labs/webassembly-language-runtimes/releases/download/python%2F3.12.0%2B20231211-040d5a6/python-3.12.0.wasm
-        working-directory: seed-gen
-
-      - name: Install integration test dependencies
-        if: steps.should_test.outputs.test != 'false'
-        run: |
-          sudo apt-get update
-          sudo apt-get install -y codequery ripgrep
-          make install-cscope
-
-      - name: Prepare environment
-        if: steps.should_test.outputs.test != 'false'
-        run: |
-          export DEBIAN_FRONTEND=noninteractive
-          sudo apt-get update
-          sudo mkdir -p /crs_scratch
-          sudo chmod -R 777 /crs_scratch
-
-      - name: Setup ${{ matrix.component }} component
-        if: steps.should_test.outputs.test != 'false'
-        run: |
-          uv sync --all-extras --frozen
-          uv pip install --isolated pytest-html>=4.1.1 pytest-cov>=6.0.0
-        working-directory: ${{ matrix.component }}
-
-      - name: Run program-model libpng integration test
-        if: matrix.component == 'program-model' && steps.should_test.outputs.test != 'false'
-        run: |
-          uv run --frozen pytest -svv --runintegration tests/c/test_libpng.py \
-            --junit-xml=integration-test-results.xml \
-            --html=integration-test-report.html \
-            --self-contained-html \
-            --cov=${{ matrix.coverage_module }} \
-            --cov-report=xml:integration-coverage.xml \
-            --cov-report=html:integration-htmlcov \
-            --cov-report=term
-        env:
-          PYTHON_WASM_BUILD_PATH: "python-3.12.0.wasm"
-        working-directory: ${{ matrix.component }}
-        timeout-minutes: 30
-
-      - name: Run integration tests on ${{ matrix.component }}
-        if: matrix.component != 'program-model' && steps.should_test.outputs.test != 'false'
-        run: |
-          uv run --frozen pytest -svv --runintegration \
-            --junit-xml=integration-test-results.xml \
-            --html=integration-test-report.html \
-            --self-contained-html \
-            --cov=${{ matrix.coverage_module }} \
-            --cov-report=xml:integration-coverage.xml \
-            --cov-report=html:integration-htmlcov \
-            --cov-report=term
-        env:
-          PYTHON_WASM_BUILD_PATH: "python-3.12.0.wasm"
-        working-directory: ${{ matrix.component }}
-        timeout-minutes: 30
-
-      - name: Generate integration test summary
-        if: always() && steps.should_test.outputs.test != 'false'
-        run: |
-          echo "### Integration Test Results: ${{ matrix.component }}" >> $GITHUB_STEP_SUMMARY
-          echo "" >> $GITHUB_STEP_SUMMARY
-          if [ -f ${{ matrix.component }}/integration-test-results.xml ]; then
-            python -c "
-          import xml.etree.ElementTree as ET
-          tree = ET.parse('${{ matrix.component }}/integration-test-results.xml')
-          root = tree.getroot()
-          tests = root.get('tests', '0')
-          failures = root.get('failures', '0')
-          errors = root.get('errors', '0')
-          skipped = root.get('skipped', '0')
-          time = root.get('time', '0')
-          print(f'- **Total Tests**: {tests}')
-          print(f'- **Passed**: {int(tests) - int(failures) - int(errors) - int(skipped)}')
-          print(f'- **Failed**: {failures}')
-          print(f'- **Errors**: {errors}')
-          print(f'- **Skipped**: {skipped}')
-          print(f'- **Duration**: {float(time):.2f}s')
-          " >> $GITHUB_STEP_SUMMARY
-          else
-            echo "No integration test results found" >> $GITHUB_STEP_SUMMARY
-          fi
-
-      - name: Upload integration test results
-        if: always() && steps.should_test.outputs.test != 'false'
-        uses: actions/upload-artifact@v4
-        with:
-          name: integration-test-results-${{ matrix.component }}-py${{ matrix.python }}
-          path: |
-            ${{ matrix.component }}/integration-test-results.xml
-            ${{ matrix.component }}/integration-test-report.html
-            ${{ matrix.component }}/integration-coverage.xml
-            ${{ matrix.component }}/integration-htmlcov/
-          retention-days: 30
diff --git a/README.md b/README.md
index 1a06e8d8..82a257e4 100644
--- a/README.md
+++ b/README.md
@@ -1,8 +1,8 @@
 # Buttercup Cyber Reasoning System (CRS)
 
 [![Tests](https://github.com/trailofbits/buttercup/actions/workflows/tests.yml/badge.svg)](https://github.com/trailofbits/buttercup/actions/workflows/tests.yml)
-[![Tests (Nightly)](https://github.com/trailofbits/buttercup/actions/workflows/tests.yml/badge.svg?event=schedule)](https://github.com/trailofbits/buttercup/actions/workflows/tests.yml)
-[![Integration](https://github.com/trailofbits/buttercup/actions/workflows/integration.yml/badge.svg)](https://github.com/trailofbits/buttercup/actions/workflows/integration.yml)
+[![Component Integration Tests](https://github.com/trailofbits/buttercup/actions/workflows/comp-integration.yml/badge.svg)](https://github.com/trailofbits/buttercup/actions/workflows/comp-integration.yml)
+[![System Integration](https://github.com/trailofbits/buttercup/actions/workflows/integration.yml/badge.svg)](https://github.com/trailofbits/buttercup/actions/workflows/integration.yml)
 
 **Buttercup** is a Cyber Reasoning System (CRS) developed by **Trail of Bits** for the **DARPA AIxCC (AI Cyber Challenge)**. Buttercup finds and patches software vulnerabilities in open-source code repositories like [example-libpng](https://github.com/tob-challenges/example-libpng). It starts by running an AI/ML-assisted fuzzing campaign (built on oss-fuzz) for the program. When vulnerabilities are found, Buttercup analyzes them and uses a multi-agent AI-driven patcher to repair the vulnerability. **Buttercup** system consists of several components:
 
diff --git a/patcher/tests/test_context_retriever.py b/patcher/tests/test_context_retriever.py
index 37fcf477..e79dcd8e 100644
--- a/patcher/tests/test_context_retriever.py
+++ b/patcher/tests/test_context_retriever.py
@@ -192,7 +192,7 @@ def example_libpng_oss_fuzz_task(tmp_path: Path) -> ChallengeTask:
             "-C",
             str(oss_fuzz_dir),
             "clone",
-            "git@github.com:tob-challenges/oss-fuzz-aixcc.git",
+            "https://github.com/tob-challenges/oss-fuzz-aixcc.git",
         ],
         check=True,
     )
@@ -209,8 +209,8 @@ def example_libpng_oss_fuzz_task(tmp_path: Path) -> ChallengeTask:
         check=True,
     )
 
-    # Download selinux source code
-    url = "git@github.com:tob-challenges/example-libpng.git"
+    # Download libpng source code
+    url = "https://github.com/tob-challenges/example-libpng.git"
     subprocess.run(["git", "-C", str(source_dir), "clone", url], check=True)
     subprocess.run(
         [
@@ -474,6 +474,7 @@ def mock_runnable_config(tmp_path: Path) -> dict:
     }
 
 
+@pytest.mark.skip(reason="Temporarily disabled due to unknown issue")
 @pytest.mark.integration
 def test_retrieve_context_basic(
     selinux_agent: ContextRetrieverAgent,
@@ -2281,6 +2282,7 @@ def test_find_tests_agent_uses_existing_test_sh(mock_agent: ContextRetrieverAgen
         assert result.goto == PatcherAgentName.ROOT_CAUSE_ANALYSIS.value
 
 
+@pytest.mark.skip(reason="Required commit not available in mirror of oss-fuzz-aixcc")
 @pytest.mark.integration
 def test_grep_libpng(libpng_agent: ContextRetrieverAgent, mock_agent_llm: MagicMock, mock_runnable_config) -> None:
     """Test the grep command on Makefile in libpng."""
diff --git a/patcher/tests/test_utils.py b/patcher/tests/test_utils.py
index 1f661196..b554a9f4 100644
--- a/patcher/tests/test_utils.py
+++ b/patcher/tests/test_utils.py
@@ -131,6 +131,7 @@ def test_find_file_in_source_dir_absolute(mock_challenge_task: ChallengeTask):
     assert res == Path("test.txt")
 
 
+@pytest.mark.skip(reason="Target not currently publicly available")
 @pytest.mark.integration
 def test_tika_find_file_in_source_dir(tika_challenge_task: ChallengeTask):
     res = find_file_in_source_dir(
diff --git a/seed-gen/test/test_find_harness.py b/seed-gen/test/test_find_harness.py
index 241f9175..1044e7de 100644
--- a/seed-gen/test/test_find_harness.py
+++ b/seed-gen/test/test_find_harness.py
@@ -615,6 +615,7 @@ def curl_oss_fuzz_cq(curl_oss_fuzz_ct: ChallengeTask) -> Iterator[CodeQuery]:
     return CodeQuery(challenge=curl_oss_fuzz_ct)
 
 
+@pytest.mark.skip(reason="Target not currently publicly available")
 @pytest.mark.integration
 def test_find_harness_in_curl(curl_oss_fuzz_cq: CodeQuery):
     harnesses = find_libfuzzer_harnesses(curl_oss_fuzz_cq)
@@ -631,6 +632,7 @@ def test_find_harness_in_curl(curl_oss_fuzz_cq: CodeQuery):
     assert "driver.c" in {h.name for h in harnesses}
 
 
+@pytest.mark.skip(reason="Target not currently publicly available")
 @pytest.mark.integration
 @pytest.mark.parametrize(
     "harness_name,expected_first_match",
@@ -665,6 +667,7 @@ def test_get_harness_source_candidates_curl(
     assert expected_first_match == harnesses[0].name
 
 
+@pytest.mark.skip(reason="Target not currently publicly available")
 @pytest.mark.integration
 @pytest.mark.parametrize(
     "harness_name,expected_harness_source_path,coverage_map_function_paths",

From 2be6f9154512908118064d5600a6d3ff068192f6 Mon Sep 17 00:00:00 2001
From: Evan Downing <2077950+evandowning@users.noreply.github.com>
Date: Wed, 27 Aug 2025 15:26:38 -0400
Subject: [PATCH 4/5] Parse result logs (#326)

* Add doc on parsing logs and scripts to unpack dataset

* Reorganize scripts

* Reorg markdown files.
---
 CONTRIBUTING.md                               |  6 +-
 README.md                                     | 29 ++++----
 common/src/buttercup/common/util_cli.py       |  5 ++
 AKS_DEPLOYMENT.md => guides/AKS_DEPLOYMENT.md | 10 +--
 .../CUSTOM_CHALLENGES.md                      |  7 +-
 MANUAL_SETUP.md => guides/MANUAL_SETUP.md     |  2 +-
 .../QUICK_REFERENCE.md                        | 20 +++++-
 guides/SCORED.md                              | 69 +++++++++++++++++++
 UNSCORED.md => guides/UNSCORED.md             |  2 +-
 scripts/results_find.sh                       |  8 +++
 scripts/results_unpack.sh                     | 20 ++++++
 11 files changed, 151 insertions(+), 27 deletions(-)
 rename AKS_DEPLOYMENT.md => guides/AKS_DEPLOYMENT.md (94%)
 rename CUSTOM_CHALLENGES.md => guides/CUSTOM_CHALLENGES.md (99%)
 rename MANUAL_SETUP.md => guides/MANUAL_SETUP.md (99%)
 rename QUICK_REFERENCE.md => guides/QUICK_REFERENCE.md (98%)
 create mode 100644 guides/SCORED.md
 rename UNSCORED.md => guides/UNSCORED.md (98%)
 create mode 100755 scripts/results_find.sh
 create mode 100755 scripts/results_unpack.sh

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 0149102d..9f235ae5 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -111,10 +111,10 @@ kubectl port-forward -n crs service/buttercup-competition-api 31323:1323
 
 ## Resources
 
-- [Quick Reference](QUICK_REFERENCE.md) - Common commands and troubleshooting
+- [Quick Reference](guides/QUICK_REFERENCE.md) - Common commands and troubleshooting
 - [Deployment Guide](deployment/README.md) - Detailed deployment information
-- [Custom Challenges](CUSTOM_CHALLENGES.md) - Adding new test cases
+- [Custom Challenges](guides/CUSTOM_CHALLENGES.md) - Adding new test cases
 
 ## Questions?
 
-Open an issue or reach out to the development team.
\ No newline at end of file
+Open an issue or reach out to the development team.
diff --git a/README.md b/README.md
index 82a257e4..83ae4119 100644
--- a/README.md
+++ b/README.md
@@ -65,7 +65,7 @@ git clone --recurse-submodules https://github.com/trailofbits/buttercup.git
 cd buttercup
 ```
 
-2. Run automated setup (Recommended)
+1. Run automated setup (Recommended)
 
 ```bash
 make setup-local
@@ -73,15 +73,15 @@ make setup-local
 
 This script will install all dependencies, configure the environment, and guide you through the setup process.
 
-**Note:** If you prefer manual setup, see the [Manual Setup Guide](MANUAL_SETUP.md).
+**Note:** If you prefer manual setup, see the [Manual Setup Guide](guides/MANUAL_SETUP.md).
 
-3. Start Buttercup locally
+1. Start Buttercup locally
 
 ```bash
 make deploy-local
 ```
 
-4. Verify local deployment:
+1. Verify local deployment:
 
 ```bash
 make status
@@ -89,7 +89,7 @@ make status
 
 When a deployment is successful, you should see all pods in "Running" or "Completed" status.
 
-5. Send Buttercup a simple task
+1. Send Buttercup a simple task
 
 **Note:** When tasked, Buttercup will start consuming third-party AI resources.
 
@@ -99,7 +99,7 @@ This command will make Buttercup pull down an example repo [example-libpng](http
 make send-libpng-task
 ```
 
-6. Access Buttercup's web-based GUI
+1. Access Buttercup's web-based GUI
 
 Run:
 
@@ -111,7 +111,7 @@ Then navigate to `http://localhost:31323` in your web browser.
 
 In the GUI you can monitor active tasks and see when Buttercup finds bugs and generates patches for them.
 
-7. Stop Buttercup
+1. Stop Buttercup
 
 **Note:** This is an important step to ensure Buttercup shuts down and stops consuming third-party AI resources.
 
@@ -128,20 +128,23 @@ make signoz-ui
 ```
 
 Then navigate to `http://localhost:33301` in your web browser to view:
+
 - Distributed traces
-- Application metrics 
+- Application metrics
 - Error monitoring
 - Performance insights
 
 If you configured LangFuse during setup, you can also monitor LLM usage and costs there.
 
-For additional log access methods, see the [Quick Reference Guide](QUICK_REFERENCE.md).
+For additional log access methods, see the [Quick Reference Guide](guides/QUICK_REFERENCE.md).
 
 ## Additional Resources
 
-- [Quick Reference Guide](QUICK_REFERENCE.md) - Common commands and troubleshooting
-- [Manual Setup Guide](MANUAL_SETUP.md) - Detailed manual installation steps
-- [AKS Deployment Guide](AKS_DEPLOYMENT.md) - Production deployment on Azure
+- [Quick Reference Guide](guides/QUICK_REFERENCE.md) - Common commands and troubleshooting
+- [Manual Setup Guide](guides/MANUAL_SETUP.md) - Detailed manual installation steps
+- [AKS Deployment Guide](guides/AKS_DEPLOYMENT.md) - Production deployment on Azure
 - [Contributing Guidelines](CONTRIBUTING.md) - Development workflow and standards
 - [Deployment Documentation](deployment/README.md) - Advanced deployment configuration
-- [Writing Custom Challenges](CUSTOM_CHALLENGES.md) - Custom project configuration and setup
+- [Writing Custom Challenges](guides/CUSTOM_CHALLENGES.md) - Custom project configuration and setup
+- [Unscored rounds](guides/UNSCORED.md) - Running unscored round challenges
+- [Scored round](guides/SCORED.md) - Parsing post-final round results
diff --git a/common/src/buttercup/common/util_cli.py b/common/src/buttercup/common/util_cli.py
index beb92712..7179e792 100644
--- a/common/src/buttercup/common/util_cli.py
+++ b/common/src/buttercup/common/util_cli.py
@@ -96,6 +96,7 @@ class ReadBuildsSettings(BaseModel):
 
 
 class ReadSubmissionsSettings(BaseModel):
+    task_id: str = Field(default="", description="Task ID")
     verbose: bool = Field(False, description="Show full stacktraces instead of truncated versions")
     filter_stop: bool = Field(False, description="Filter out submissions that are stopped")
 
@@ -228,6 +229,10 @@ def handle_subcommand(redis: Redis, command: BaseModel | None) -> None:
                     logger.error(f"Task {task_id} not found in registry")
                     continue
 
+                if command.task_id and task_id != command.task_id:
+                    logger.info(f"Skipping submission {i} for task {task_id} (not {command.task_id})")
+                    continue
+
                 if task_id not in result:
                     result[task_id] = TaskResult(
                         task_id=task_id,
diff --git a/AKS_DEPLOYMENT.md b/guides/AKS_DEPLOYMENT.md
similarity index 94%
rename from AKS_DEPLOYMENT.md
rename to guides/AKS_DEPLOYMENT.md
index 8d111f96..15219f8e 100644
--- a/AKS_DEPLOYMENT.md
+++ b/guides/AKS_DEPLOYMENT.md
@@ -1,8 +1,8 @@
 # Production AKS Deployment Guide
 
-> **⚠️ Notice:**  
-> The following production deployment instructions have **not been fully tested**.  
-> Please proceed with caution and verify each step in your environment.  
+> **⚠️ Notice:**
+> The following production deployment instructions have **not been fully tested**.
+> Please proceed with caution and verify each step in your environment.
 > If you encounter issues, consult the script comments and configuration files for troubleshooting.
 
 Full production deployment of the **Buttercup CRS** on Azure Kubernetes Service with proper networking, monitoring, and scaling for the DARPA AIxCC competition.
@@ -81,7 +81,7 @@ cd deployment && make up
 
 ## Additional Resources
 
-For detailed deployment instructions and advanced configuration options, see the [deployment README](deployment/README.md).
+For detailed deployment instructions and advanced configuration options, see the [deployment README](../deployment/README.md).
 
 ## Troubleshooting
 
@@ -102,4 +102,4 @@ az aks get-credentials --name <cluster-name> --resource-group <resource-group>
 az aks show --name <cluster-name> --resource-group <resource-group>
 ```
 
-For more troubleshooting information, see the main [Quick Reference Guide](QUICK_REFERENCE.md).
\ No newline at end of file
+For more troubleshooting information, see the main [Quick Reference Guide](QUICK_REFERENCE.md).
diff --git a/CUSTOM_CHALLENGES.md b/guides/CUSTOM_CHALLENGES.md
similarity index 99%
rename from CUSTOM_CHALLENGES.md
rename to guides/CUSTOM_CHALLENGES.md
index 865a3cd0..c3cd7f11 100644
--- a/CUSTOM_CHALLENGES.md
+++ b/guides/CUSTOM_CHALLENGES.md
@@ -350,11 +350,13 @@ Buttercup determines which mode to use based on your challenge configuration:
 ### Choosing the Right Mode
 
 **Choose Full Mode when you want to:**
+
 - Perform an initial assessment of a new project
 - Search for vulnerabilities in all code reachable from harnesses
 - Analyze a project without specific change targets
 
 **Choose Delta Mode when you want to:**
+
 - Test specific code changes
 - Reproduce known vulnerabilities
 
@@ -369,8 +371,7 @@ Now that you understand how to create custom challenges, you can:
 3. **Monitor progress**: Use the debugging techniques to track your challenge's execution
 4. **Iterate**: Refine your harnesses and build scripts based on the results
 
-For additional help, consult the main [Buttercup documentation](README.md) or check the [troubleshooting guide](CLAUDE.md#common-debugging-commands).
-
+For additional help, consult the main [Buttercup documentation](../README.md) or check the [troubleshooting guide](../CLAUDE.md#common-debugging-commands).
 
 [1]: https://aicyberchallenge.com/
-[2]: https://google.github.io/oss-fuzz/getting-started/new-project-guide/
\ No newline at end of file
+[2]: https://google.github.io/oss-fuzz/getting-started/new-project-guide/
diff --git a/MANUAL_SETUP.md b/guides/MANUAL_SETUP.md
similarity index 99%
rename from MANUAL_SETUP.md
rename to guides/MANUAL_SETUP.md
index 6118816f..053e1ed7 100644
--- a/MANUAL_SETUP.md
+++ b/guides/MANUAL_SETUP.md
@@ -133,4 +133,4 @@ helm repo update
 helm dependency update deployment/k8s/
 ```
 
-For additional troubleshooting, see the [Quick Reference Guide](QUICK_REFERENCE.md).
\ No newline at end of file
+For additional troubleshooting, see the [Quick Reference Guide](QUICK_REFERENCE.md).
diff --git a/QUICK_REFERENCE.md b/guides/QUICK_REFERENCE.md
similarity index 98%
rename from QUICK_REFERENCE.md
rename to guides/QUICK_REFERENCE.md
index bfb42a26..872400e7 100644
--- a/QUICK_REFERENCE.md
+++ b/guides/QUICK_REFERENCE.md
@@ -3,6 +3,7 @@
 ## Setup Commands
 
 ### Local Development
+
 ```bash
 # Automated setup
 ./scripts/setup-local.sh
@@ -14,6 +15,7 @@ cd deployment && make up
 ```
 
 ### Production AKS
+
 ```bash
 # Automated setup
 ./scripts/setup-azure.sh
@@ -27,6 +29,7 @@ cd deployment && make up
 ## Common Commands
 
 ### Kubernetes Management
+
 ```bash
 # View all resources
 kubectl get pods -A
@@ -51,6 +54,7 @@ kubectl exec -it -n crs <pod-name> -- /bin/bash
 ```
 
 ### Testing
+
 ```bash
 # Send test task
 ./orchestrator/scripts/task_crs.sh
@@ -63,6 +67,7 @@ kubectl exec -it -n crs <pod-name> -- /bin/bash
 ```
 
 ### Development
+
 ```bash
 # Lint Python code
 make lint
@@ -74,6 +79,7 @@ docker-compose --profile fuzzer-test up
 ```
 
 ### Minikube
+
 ```bash
 # Start/stop
 minikube start --driver=docker
@@ -86,6 +92,7 @@ minikube dashboard
 ```
 
 ### Azure AKS
+
 ```bash
 # Get credentials
 az aks get-credentials --name <cluster-name> --resource-group <resource-group>
@@ -105,6 +112,7 @@ az aks show --name <cluster-name> --resource-group <resource-group>
 ### Common Issues
 
 #### Minikube Issues
+
 ```bash
 # Reset minikube
 minikube delete
@@ -116,6 +124,7 @@ kubectl cluster-info
 ```
 
 #### Docker Issues
+
 ```bash
 # Permission issues
 sudo usermod -aG docker $USER
@@ -127,6 +136,7 @@ sudo systemctl start docker
 ```
 
 #### Helm Issues
+
 ```bash
 # Update repositories
 helm repo update
@@ -137,6 +147,7 @@ helm lint deployment/k8s/
 ```
 
 #### Azure Issues
+
 ```bash
 # Authentication
 az login --tenant <your-tenant-here>
@@ -147,6 +158,7 @@ az ad sp list --display-name "ButtercupCRS*"
 ```
 
 #### Kubernetes Issues
+
 ```bash
 # Check cluster connectivity
 kubectl cluster-info
@@ -163,16 +175,19 @@ kubectl get events -n crs --sort-by='.lastTimestamp'
 ### Log Analysis
 
 #### Check Patch Submission
+
 ```bash
 kubectl logs -n crs -l app=scheduler --tail=-1 --prefix | grep "WAIT_PATCH_PASS -> SUBMIT_BUNDLE"
 ```
 
 #### Check Competition API
+
 ```bash
 kubectl logs -n crs -l app=competition-api --tail=-1 --prefix
 ```
 
 #### Check Fuzzer
+
 ```bash
 kubectl logs -n crs -l app=fuzzer --tail=-1 --prefix
 ```
@@ -180,16 +195,19 @@ kubectl logs -n crs -l app=fuzzer --tail=-1 --prefix
 ## File Locations
 
 ### Configuration
+
 - `deployment/env` - Main configuration file
 - `deployment/env.template` - Configuration template
 - `deployment/k8s/values-*.template` - Kubernetes value templates
 
 ### Scripts
+
 - `scripts/setup-local.sh` - Local development setup
 - `scripts/setup-azure.sh` - Production AKS setup
 - `orchestrator/scripts/` - Testing and task scripts
 
 ### Documentation
+
 - `README.md` - Main documentation
 - `deployment/README.md` - Detailed deployment guide
 
@@ -198,4 +216,4 @@ kubectl logs -n crs -l app=fuzzer --tail=-1 --prefix
 - Check logs: `kubectl logs -n crs <pod-name>`
 - View events: `kubectl get events -n crs`
 - Debug pods: `kubectl exec -it -n crs <pod-name> -- /bin/bash`
-- Monitor resources: `kubectl top pods -A` 
+- Monitor resources: `kubectl top pods -A`
diff --git a/guides/SCORED.md b/guides/SCORED.md
new file mode 100644
index 00000000..9e18c767
--- /dev/null
+++ b/guides/SCORED.md
@@ -0,0 +1,69 @@
+# Scored
+
+This document describes how to parse logs from the final round of the AIxCC competition.
+
+**NOTE**: PoVs were not released to us immediately because DARPA is still involved in responsible disclosure: <https://aicyberchallenge.com/Finals-winners-announcement/>.
+
+## Data
+
+Download the following files from the competition to `buttercup/data/`
+
+- `litellm_backup.sql`
+- `redis-backup.rdb`
+- `task-storage.tar`
+
+## Get Task IDs for Specific Projects
+
+For example, find the Task IDs for `libpng`.
+
+```shell
+cd data/
+../scripts/results_unpack.sh
+../scripts/results_find.sh libpng
+```
+
+## Parse
+
+Query the CRS database for submissions. Documentation can be found at <https://redis.io/learn/explore/import#restore-an-rdb-file>.
+
+Let's assume a task ID `0197b76a-b429-78fa-8d0e-ad84994c8f44` was outputted from above.
+
+1. Start `redis-server` in `data/` directory
+
+   ```shell
+   cd data/
+   redis-server
+   ```
+
+1. Print directory to confirm
+
+   ```shell
+   redis-cli config get dir
+   1) "dir"
+   2) "<path/to/data>"
+   ```
+
+1. Stop `redis-server` and check if `dump.rdb` exists in `data/` directory.
+
+1. Copy `.rdb` file
+
+   ```shell
+   cp redis-backup.rdb dump.rdb
+   ```
+
+1. Start `redis-server` and query to make sure it has data.
+
+   ```shell
+   redis-server
+   redis-cli keys \*
+   ```
+
+1. Parse database for task id
+
+   ```shell
+   cd common/
+   uv sync --all-extras
+   uv run buttercup-util read_submissions --task_id 0197b76a-b429-78fa-8d0e-ad84994c8f44 --verbose True > ../data/submissions.txt
+   ```
+
+1. Read through `submissions.txt` for information like crash harnesses, stacktraces, and patches.
diff --git a/UNSCORED.md b/guides/UNSCORED.md
similarity index 98%
rename from UNSCORED.md
rename to guides/UNSCORED.md
index a81f3050..b7e552a7 100644
--- a/UNSCORED.md
+++ b/guides/UNSCORED.md
@@ -9,7 +9,7 @@ A list of vulnerabilities to find and patch is in this [spreadsheet](https://doc
 
 ## Setup
 
-* Set up minikube locally on your machine (see [README.md](README.md)).
+* Set up minikube locally on your machine (see [README.md](../README.md)).
 
 * `cd deployment && make up`
 
diff --git a/scripts/results_find.sh b/scripts/results_find.sh
new file mode 100755
index 00000000..c9c16407
--- /dev/null
+++ b/scripts/results_find.sh
@@ -0,0 +1,8 @@
+#!/bin/bash
+
+if [ "$#" -ne 1 ]; then
+    echo "usage: $0 <keyword>"
+    exit 1
+fi
+
+find tasks_storage -name "task_meta.json" -exec grep -rIil "$1" {} \; | cut -d '/' -f 1-2 | sort -u
\ No newline at end of file
diff --git a/scripts/results_unpack.sh b/scripts/results_unpack.sh
new file mode 100755
index 00000000..863ea39d
--- /dev/null
+++ b/scripts/results_unpack.sh
@@ -0,0 +1,20 @@
+#!/bin/bash
+
+if [ "$#" -ne 0 ]; then
+    echo "usage: $0"
+    exit 1
+fi
+
+# Clean up
+rm -rf tasks_storage
+
+# Unpack
+tar xvf tasks_storage.tar
+
+# Decompress
+for fn in `find tasks_storage -name *.tgz`; do
+    dst="${fn%.tgz}"
+    mkdir $dst
+    tar xvzf $fn -C $dst
+    rm $fn
+done
\ No newline at end of file

From 54de06efa305adf411cbaedf0e69003f0cd3db31 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Thu, 28 Aug 2025 06:04:04 +0000
Subject: [PATCH 5/5] build(deps): bump actions/checkout from 4 to 5 in the
 actions group

Bumps the actions group with 1 update: [actions/checkout](https://github.com/actions/checkout).


Updates `actions/checkout` from 4 to 5
- [Release notes](https://github.com/actions/checkout/releases)
- [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md)
- [Commits](https://github.com/actions/checkout/compare/v4...v5)

---
updated-dependencies:
- dependency-name: actions/checkout
  dependency-version: '5'
  dependency-type: direct:production
  update-type: version-update:semver-major
  dependency-group: actions
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 .github/workflows/comp-integration.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/comp-integration.yml b/.github/workflows/comp-integration.yml
index f0f51a5b..2ba226f1 100644
--- a/.github/workflows/comp-integration.yml
+++ b/.github/workflows/comp-integration.yml
@@ -70,7 +70,7 @@ jobs:
           - 6379:6379
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
         with:
           persist-credentials: false
           submodules: true