mirror of
https://github.com/trailofbits/buttercup
synced 2026-06-21 14:11:39 +00:00
patcher: misc fixes (#901)
* patcher: ensure tests_passed is always set * patcher: make find_tests try harder * patcher: prefer simpler patches * patcher: make sure the rootcause also consider the last failing pov * patcher: small adjustment to reflection prompt * patcher: apply the delta-mode diff while testing test instructions * patcher: other improvements * fix tests * patcher: store found test instructions to speed up future patches (#903) * patcher: store found test instructions to speed up future patches * fix lint
This commit is contained in:
Riccardo Schirone
committed by
GitHub
GitHub
parent
e0a012d751
commit
e705a64fb2
@@ -237,6 +237,10 @@ class ChallengeTask:
|
||||
|
||||
return default
|
||||
|
||||
def dockerfile_path(self) -> Path:
|
||||
"""Read the Dockerfile for the given project."""
|
||||
return self.get_oss_fuzz_path() / "projects" / self.project_name / "Dockerfile"
|
||||
|
||||
def workdir_from_dockerfile(self) -> Path:
|
||||
"""Parses WORKDIR from the Dockerfile for the given project."""
|
||||
# NOTE: This is extracted and adapted from the OSS-Fuzz repository
|
||||
|
||||
@@ -24,6 +24,7 @@ class ButtercupLLM(Enum):
|
||||
OPENAI_GPT_4O = "openai-gpt-4o"
|
||||
OPENAI_GPT_4O_MINI = "openai-gpt-4o-mini"
|
||||
OPENAI_O3_MINI = "openai-o3-mini"
|
||||
OPENAI_O3 = "openai-o3"
|
||||
OPENAI_O1 = "openai-o1"
|
||||
OPENAI_GPT_4_1_NANO = "openai-gpt-4.1-nano"
|
||||
OPENAI_GPT_4_1_MINI = "openai-gpt-4.1-mini"
|
||||
|
||||
@@ -319,6 +319,13 @@ litellm-helm:
|
||||
tpm: 30000000
|
||||
rpm: 10000
|
||||
|
||||
- model_name: openai-o3
|
||||
litellm_params:
|
||||
model: openai/o3
|
||||
api_key: os.environ/OPENAI_API_KEY
|
||||
tpm: 30000000
|
||||
rpm: 10000
|
||||
|
||||
- model_name: openai-gpt-4.1
|
||||
litellm_params:
|
||||
model: openai/gpt-4.1
|
||||
|
||||
@@ -61,6 +61,13 @@ model_list:
|
||||
tpm: 30000000
|
||||
rpm: 10000
|
||||
|
||||
- model_name: openai-o3
|
||||
litellm_params:
|
||||
model: openai/o3
|
||||
api_key: os.environ/OPENAI_API_KEY
|
||||
tpm: 30000000
|
||||
rpm: 10000
|
||||
|
||||
- model_name: openai-gpt-4.1
|
||||
litellm_params:
|
||||
model: openai/gpt-4.1
|
||||
|
||||
@@ -29,7 +29,7 @@ import re
|
||||
import uuid
|
||||
import logging
|
||||
|
||||
MAX_STACKTRACE_LENGTH = 5000
|
||||
MAX_STACKTRACE_LENGTH = 15000
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -499,11 +499,17 @@ STACKTRACE_TMPL = """<stacktrace>
|
||||
</stacktrace>"""
|
||||
|
||||
|
||||
def stacktrace_to_str(sanitizer: str, sanitizer_output: str | None) -> str:
|
||||
ansi_escape = re.compile(r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])")
|
||||
return STACKTRACE_TMPL.format(
|
||||
SANITIZER_NAME=sanitizer,
|
||||
SANITIZER_OUTPUT=truncate_output(
|
||||
ansi_escape.sub("", sanitizer_output or ""),
|
||||
MAX_STACKTRACE_LENGTH,
|
||||
TruncatePosition.START,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def get_stacktraces_from_povs(povs: list[PatchInputPoV]) -> list[str]:
|
||||
return [
|
||||
STACKTRACE_TMPL.format(
|
||||
SANITIZER_NAME=pov.sanitizer,
|
||||
SANITIZER_OUTPUT=truncate_output(pov.sanitizer_output, MAX_STACKTRACE_LENGTH, TruncatePosition.START),
|
||||
)
|
||||
for pov in povs
|
||||
]
|
||||
return [stacktrace_to_str(pov.sanitizer, pov.sanitizer_output) for pov in povs]
|
||||
|
||||
@@ -4,6 +4,7 @@ from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import tempfile
|
||||
import time
|
||||
import langgraph.errors
|
||||
import operator
|
||||
import re
|
||||
@@ -11,7 +12,7 @@ from itertools import groupby
|
||||
from operator import itemgetter
|
||||
from langgraph.graph import END
|
||||
from langchain_openai.chat_models.base import BaseChatOpenAI
|
||||
from concurrent.futures import as_completed
|
||||
from concurrent.futures import as_completed, TimeoutError
|
||||
from langchain_core.prompts import ChatPromptTemplate
|
||||
from langchain_core.output_parsers import StrOutputParser
|
||||
from buttercup.common.stack_parsing import parse_stacktrace
|
||||
@@ -30,7 +31,7 @@ from buttercup.common.challenge_task import ChallengeTask
|
||||
from buttercup.common.stack_parsing import CrashInfo
|
||||
from langchain_core.runnables import Runnable, RunnableConfig
|
||||
from langchain_core.runnables.config import get_executor_for_config
|
||||
from buttercup.patcher.utils import truncate_output, get_challenge, TruncatePosition
|
||||
from buttercup.patcher.utils import truncate_output, get_challenge, TruncatePosition, get_codequery
|
||||
from buttercup.patcher.agents.common import (
|
||||
PatcherAgentBase,
|
||||
ContextRetrieverState,
|
||||
@@ -62,8 +63,8 @@ from buttercup.patcher.agents.tools import (
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
SYSTEM_TMPL = """You are an agent - please keep going until the user’s query is completely resolved, before ending your turn and yielding back to the user. Only terminate your turn when you are sure that the problem is solved.
|
||||
If you are not sure about file content or codebase structure pertaining to the user’s request, use your tools to read files and gather the relevant information: do NOT guess or make up an answer.
|
||||
SYSTEM_TMPL = """You are an agent - please keep going until the user's query is completely resolved, before ending your turn and yielding back to the user. Only terminate your turn when you are sure that the problem is solved.
|
||||
If you are not sure about file content or codebase structure pertaining to the user's request, use your tools to read files and gather the relevant information: do NOT guess or make up an answer.
|
||||
You MUST plan extensively before each function call, and reflect extensively on the outcomes of the previous function calls. DO NOT do this entire process by making function calls only, as this can impair your ability to solve the problem and think insightfully.
|
||||
|
||||
Assist a software engineer in finding and extracting relevant code snippets from a software project. Use only the provided tools and project context. Prioritize accuracy and completeness. Avoid speculation."""
|
||||
@@ -110,7 +111,7 @@ Guidelines:
|
||||
- Your first step should always be to identify the exact function, type, or code range using tools like `get_function` or `get_type`.
|
||||
- ONLY use `track_snippet` after you have confirmed that:
|
||||
1. The code snippet is correct and complete,
|
||||
2. It answers the engineer’s request,
|
||||
2. It answers the engineer's request,
|
||||
- Do NOT call `track_snippet` speculatively or based on partial guesses.
|
||||
- Clearly explain your reasoning for calling `track_snippet`, and what the snippet represents.
|
||||
- If a tool fails or more context is needed, explain the issue and propose next steps.
|
||||
@@ -210,6 +211,8 @@ Guidelines:
|
||||
First, list the code snippets that you think are the most relevant to the vulnerability with an explanation of why you think they are relevant.
|
||||
Then rate them from 1 to 10, where 1 is the least relevant and 10 is the most relevant.
|
||||
Finally, output the <request> tags, one per line, for only the ABSOLUTELY ESSENTIAL snippets that are NOT already available.
|
||||
|
||||
If you do not have any code snippets to request, do not output any <request> tags.
|
||||
"""
|
||||
|
||||
INITIAL_CODE_SNIPPET_REQUESTS_PROMPT = ChatPromptTemplate.from_messages(
|
||||
@@ -243,11 +246,13 @@ FILTER_CODE_SNIPPETS_PROMPT = ChatPromptTemplate.from_messages(
|
||||
]
|
||||
)
|
||||
|
||||
FIND_TESTS_SYSTEM_TMPL = """You are an agent - please keep going until the user’s query is completely resolved, before ending your turn and yielding back to the user. Only terminate your turn when you are sure that the problem is solved.
|
||||
If you are not sure about file content or codebase structure pertaining to the user’s request, use your tools to read files and gather the relevant information: do NOT guess or make up an answer.
|
||||
FIND_TESTS_SYSTEM_TMPL = """You are an agent - please keep going until the user's query is completely resolved, before ending your turn and yielding back to the user. Only terminate your turn when you are sure that the problem is solved.
|
||||
If you are not sure about file content or codebase structure pertaining to the user's request, use your tools to read files and gather the relevant information: do NOT guess or make up an answer.
|
||||
You MUST plan extensively before each function call, and reflect extensively on the outcomes of the previous function calls. DO NOT do this entire process by making function calls only, as this can impair your ability to solve the problem and think insightfully."""
|
||||
|
||||
FIND_TESTS_USER_MSG = """Your task is to build the project and run its tests using information from oss-fuzz.
|
||||
FIND_TESTS_USER_MSG = """Your goal is to provide accurate and complete \
|
||||
instructions for building and testing the project. You must continue working \
|
||||
until you have valid instructions confirmed by the `test_instructions` function.
|
||||
|
||||
Project name:
|
||||
<project_name>
|
||||
@@ -259,42 +264,107 @@ Working directory:
|
||||
{CWD}
|
||||
</current_working_directory>
|
||||
|
||||
Follow these steps:
|
||||
The container used to run the tests is built with the following Dockerfile:
|
||||
<dockerfile>
|
||||
{DOCKERFILE}
|
||||
</dockerfile>
|
||||
|
||||
1. **Build the project**
|
||||
- Look for build scripts in `/src`
|
||||
- If not found, check `README`, `Makefile`, `CMakeLists.txt`, etc.
|
||||
- Remove fuzz-specific configs (e.g., sanitizers) as needed
|
||||
- Attempt to build, retry if it fails
|
||||
Follow these steps to complete your task:
|
||||
|
||||
2. **Run the test suite**
|
||||
- Search for documented test instructions or test-related files
|
||||
- Run actual tests (not just linting or setup)
|
||||
- Troubleshoot if they fail to run
|
||||
- Make sure tests actually ran and passed
|
||||
1. Build the project:
|
||||
a. Search for build scripts in the /src directory. These are usually used by oss-fuzz to build the project.
|
||||
b. If not found, check README, Makefile, CMakeLists.txt, and similar files.
|
||||
c. Attempt to build the project (no fuzz-specific configurations).
|
||||
d. If the build fails retry.
|
||||
e. Continue until you achieve a successful build.
|
||||
|
||||
At each major step, wrap your findings in:
|
||||
<project_analysis>
|
||||
(1) Relevant files
|
||||
(2) Build/test commands
|
||||
(3) Evaluation
|
||||
(4) Confirmation you're following all rules
|
||||
(5) If you have found the test instructions, ensure you have called `test_instructions` tool with the test instructions. Do not terminate your turn otherwise.
|
||||
</project_analysis>
|
||||
2. Run the test suite:
|
||||
a. Search for documented test instructions or test-related files.
|
||||
b. Run the actual tests (not just linting or setup procedures).
|
||||
c. If tests fail to run, troubleshoot and retry.
|
||||
d. Ensure that tests actually ran and passed.
|
||||
|
||||
At the end, summarize using:
|
||||
<final_commands>
|
||||
Build Commands:
|
||||
1. ...
|
||||
3. Analyze and report:
|
||||
After each major step (build and test), analyze the results and report your findings.
|
||||
|
||||
Test Commands:
|
||||
1. ...
|
||||
4. Validate instructions:
|
||||
a. Once you believe you have found the correct set of instructions to build and test the project, call the `test_instructions` function with these instructions as the argument.
|
||||
b. If the `test_instructions` function indicates that the instructions are not valid, revise your approach and try again.
|
||||
c. Do not terminate your process until you have called `test_instructions` and received confirmation that the instructions are valid.
|
||||
|
||||
Validation:
|
||||
[Output from `test_instructions` tool]
|
||||
</final_commands>
|
||||
Throughout your analysis, wrap your discovery process inside <discovery_process> tags. Include the following:
|
||||
|
||||
1. File Analysis:
|
||||
- List all potentially relevant build and test files.
|
||||
- Briefly describe the purpose of each file.
|
||||
|
||||
2. Build Process:
|
||||
- Detail each step of the build process.
|
||||
- Document any commands attempted and their outcomes.
|
||||
- If errors occur, list them and describe your troubleshooting steps.
|
||||
|
||||
3. Test Process:
|
||||
- Outline each step of the test process.
|
||||
- Document test commands and their results.
|
||||
- If tests fail, describe the errors and your attempts to resolve them.
|
||||
|
||||
4. Decision Making:
|
||||
- Explain your reasoning for choosing specific build or test methods.
|
||||
- Discuss any alternatives considered and why they were rejected.
|
||||
|
||||
This detailed breakdown will help ensure a thorough and transparent approach to solving the task. It's OK for this section to be quite long.
|
||||
|
||||
Remember:
|
||||
- Do not try to build a project solely based on your previous knowledge of the project.
|
||||
- Do not guess or make up answers about file content or codebase structure.
|
||||
- Use your available tools to read files and gather relevant information when needed.
|
||||
- Check the build/test commands used by oss-fuzz to have an idea of what to do.
|
||||
- You SHOULD NOT build the project with fuzzing-specific configurations or fuzzers enabled.
|
||||
- Plan extensively before each function call and reflect on the outcomes of previous calls.
|
||||
- Continue working until you have valid instructions confirmed by the `test_instructions` function.
|
||||
|
||||
Begin your analysis and proceed step-by-step through the process of discovering the build and test instructions for the given project.
|
||||
"""
|
||||
|
||||
ARE_VALID_TEST_INSTRUCTIONS_USER_MSG = """
|
||||
You are an expert in software testing. Your task is to evaluate the validity of a set of test instructions.
|
||||
|
||||
Here are the test instructions:
|
||||
<test_instructions>
|
||||
{TEST_INSTRUCTIONS}
|
||||
</test_instructions>
|
||||
|
||||
Here is the output of the test instructions:
|
||||
<output>
|
||||
{OUTPUT}
|
||||
</output>
|
||||
|
||||
Here is the error of the test instructions:
|
||||
<error>
|
||||
{ERROR}
|
||||
</error>
|
||||
|
||||
Evaluate whether the test instructions ran correctly.
|
||||
Test instructions are valid if:
|
||||
- The project is actually built (e.g. the build commands are correct and the project is built successfully)
|
||||
- Tests are actually running (e.g. the test commands are correct and the tests are running)
|
||||
- All tests passed (e.g. the tests are passing)
|
||||
|
||||
Wrap your reasoning in <reasoning> tags, then return the result in <are_valid> tags.
|
||||
For example, if the test instructions are valid, you should return:
|
||||
<reasoning>
|
||||
[.....]
|
||||
</reasoning>
|
||||
<are_valid>TRUE</are_valid>
|
||||
"""
|
||||
|
||||
ARE_VALID_TEST_INSTRUCTIONS_PROMPT = ChatPromptTemplate.from_messages(
|
||||
[
|
||||
("user", ARE_VALID_TEST_INSTRUCTIONS_USER_MSG),
|
||||
("ai", "<reasoning>"),
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
class CodeSnippetManagerState(BaseCtxState):
|
||||
"""State for the code snippet manager"""
|
||||
@@ -455,9 +525,7 @@ def sh(
|
||||
"""Execute a shell command within the project environment and return its output/error.
|
||||
|
||||
Use this tool only for shell commands that are not supported by the more specific tools (e.g., `cat`, `ls`, `head`, `grep`).
|
||||
|
||||
Do NOT use this tool to run test instructions — use `test_instructions` for that.
|
||||
Remember to call the `test_instructions` tool to log and validate any test commands you find and try."""
|
||||
"""
|
||||
|
||||
logger.info("Running command: %s", command)
|
||||
|
||||
@@ -508,6 +576,24 @@ Remember to call the `test_instructions` tool to log and validate any test comma
|
||||
command_file_path.unlink()
|
||||
|
||||
|
||||
def _are_test_instructions_valid(instructions: str, output: bytes, error: bytes) -> bool:
|
||||
"""Validate a set of test instructions by executing them inside the project environment."""
|
||||
llm = create_default_llm(model_name=ButtercupLLM.OPENAI_GPT_4_1.value)
|
||||
chain = ARE_VALID_TEST_INSTRUCTIONS_PROMPT | llm | StrOutputParser()
|
||||
res = chain.invoke(
|
||||
{
|
||||
"TEST_INSTRUCTIONS": instructions,
|
||||
"OUTPUT": truncate_output(output.decode("utf-8"), MAX_OUTPUT_LENGTH, TruncatePosition.START),
|
||||
"ERROR": truncate_output(error.decode("utf-8"), MAX_OUTPUT_LENGTH, TruncatePosition.START),
|
||||
}
|
||||
)
|
||||
match = re.search(r"<are_valid>(.*?)</are_valid>", res)
|
||||
if match:
|
||||
return match.group(1).strip().lower() in ["true", "yes"]
|
||||
|
||||
return res.strip().lower() in ["true", "yes"]
|
||||
|
||||
|
||||
@tool
|
||||
def test_instructions(
|
||||
instructions: list[str],
|
||||
@@ -524,9 +610,6 @@ def test_instructions(
|
||||
- If the instructions fail (non-zero exit code or no tests run), analyze the output, correct the commands, and call this tool again with the updated instructions.
|
||||
- You can call this tool multiple times during the discovery process, but only the **last successful call** will be recorded and used.
|
||||
- Instructions must be based strictly on project files (e.g. README, Makefile, CI configs); do not guess or fabricate commands.
|
||||
|
||||
Ensure that the tests are actually running and passing before considering the request satisfied.
|
||||
If the project builds but the tests do not run, it is not enough to consider the request satisfied.
|
||||
"""
|
||||
test_file_path = None
|
||||
try:
|
||||
@@ -543,6 +626,7 @@ def test_instructions(
|
||||
|
||||
clean_challenge = ChallengeTask(state.challenge_task_dir_ro)
|
||||
with clean_challenge.get_rw_copy(state.work_dir) as challenge:
|
||||
challenge.apply_patch_diff()
|
||||
sh_cmd_res = challenge.exec_docker_cmd(
|
||||
challenge.get_test_sh_script("/tmp/test.sh"),
|
||||
mount_dirs={
|
||||
@@ -565,14 +649,14 @@ def test_instructions(
|
||||
{truncate_output(sh_cmd_res.error.decode("utf-8"), MAX_OUTPUT_LENGTH, TruncatePosition.START)}
|
||||
</error>
|
||||
"""
|
||||
if sh_cmd_res.success:
|
||||
if sh_cmd_res.success and _are_test_instructions_valid(instructions_str, sh_cmd_res.output, sh_cmd_res.error):
|
||||
res_instructions = instructions_str
|
||||
msg = f"""Test instructions passed:
|
||||
<result>
|
||||
{msg}
|
||||
</result>
|
||||
|
||||
Make sure the tests are actually running and passing before considering the request satisfied."""
|
||||
The last tracked instructions are correct and valid. You can stop here."""
|
||||
else:
|
||||
res_instructions = None
|
||||
msg = f"""Failed to run test instructions:
|
||||
@@ -580,7 +664,7 @@ Make sure the tests are actually running and passing before considering the requ
|
||||
{msg}
|
||||
</result>
|
||||
|
||||
Test were NOT run correctly, please analyze the output and correct the commands.
|
||||
Test instructions were NOT run correctly, please analyze the output and correct the commands.
|
||||
You CANNOT stop here, you MUST fix the test instructions and call this tool again.
|
||||
"""
|
||||
|
||||
@@ -746,12 +830,18 @@ class ContextRetrieverAgent(PatcherAgentBase):
|
||||
else:
|
||||
ls_src = "ls src failed"
|
||||
|
||||
try:
|
||||
dockerfile = challenge.dockerfile_path().read_text()
|
||||
except Exception:
|
||||
dockerfile = "Dockerfile not found"
|
||||
|
||||
return [
|
||||
SystemMessage(content=FIND_TESTS_SYSTEM_TMPL),
|
||||
HumanMessage(
|
||||
content=FIND_TESTS_USER_MSG.format(
|
||||
PROJECT_NAME=challenge.project_name,
|
||||
CWD=challenge.workdir_from_dockerfile(),
|
||||
DOCKERFILE=dockerfile,
|
||||
)
|
||||
),
|
||||
AIMessage(
|
||||
@@ -786,7 +876,7 @@ class ContextRetrieverAgent(PatcherAgentBase):
|
||||
return [
|
||||
CodeSnippetRequest(request=match.group(1).strip())
|
||||
for match in re.finditer(r"<request>(.*?)</request>", output, re.DOTALL)
|
||||
if match.group(1).strip()
|
||||
if match.group(1) and match.group(1).strip()
|
||||
]
|
||||
|
||||
def _parse_filter_code_snippets_output(self, output: str) -> bool:
|
||||
@@ -1090,6 +1180,10 @@ class ContextRetrieverAgent(PatcherAgentBase):
|
||||
goto=PatcherAgentName.ROOT_CAUSE_ANALYSIS.value,
|
||||
)
|
||||
|
||||
def _get_custom_test_path(self, configuration: PatcherConfig) -> Path:
|
||||
codequery = get_codequery(self.challenge.task_dir, configuration.work_dir)
|
||||
return codequery.challenge.task_dir.joinpath("custom_test.sh") # type: ignore[no-any-return]
|
||||
|
||||
def find_tests_node(
|
||||
self, state: PatcherAgentState, config: RunnableConfig
|
||||
) -> Command[Literal[PatcherAgentName.ROOT_CAUSE_ANALYSIS.value, PatcherAgentName.REFLECTION.value]]: # type: ignore[name-defined]
|
||||
@@ -1110,44 +1204,111 @@ class ContextRetrieverAgent(PatcherAgentBase):
|
||||
goto=PatcherAgentName.ROOT_CAUSE_ANALYSIS.value,
|
||||
)
|
||||
|
||||
custom_test_path = self._get_custom_test_path(configuration)
|
||||
if custom_test_path and custom_test_path.exists() and custom_test_path.is_file():
|
||||
return Command(
|
||||
update={
|
||||
"tests_instructions": custom_test_path.read_text(),
|
||||
},
|
||||
goto=PatcherAgentName.ROOT_CAUSE_ANALYSIS.value,
|
||||
)
|
||||
|
||||
clean_challenge = self.challenge.get_clean_task(configuration.tasks_storage)
|
||||
with clean_challenge.get_rw_copy(configuration.work_dir) as clean_challenge_rw:
|
||||
clean_challenge_rw.apply_patch_diff()
|
||||
input_state = {
|
||||
"challenge_task_dir_ro": clean_challenge.task_dir,
|
||||
"challenge_task_dir": clean_challenge_rw.task_dir,
|
||||
"work_dir": configuration.work_dir,
|
||||
}
|
||||
input_state = FindTestsState(
|
||||
challenge_task_dir_ro=clean_challenge.task_dir,
|
||||
challenge_task_dir=clean_challenge_rw.task_dir,
|
||||
work_dir=configuration.work_dir,
|
||||
messages=[],
|
||||
)
|
||||
|
||||
configuration = configuration.clone()
|
||||
try:
|
||||
self.find_tests_agent.invoke(
|
||||
input_state,
|
||||
config=RunnableConfig(
|
||||
recursion_limit=configuration.ctx_retriever_recursion_limit,
|
||||
configurable=configuration.model_dump(),
|
||||
),
|
||||
)
|
||||
agent_state_dict = self.find_tests_agent.get_state( # type: ignore[attr-defined]
|
||||
RunnableConfig(configurable=configuration.model_dump())
|
||||
).values
|
||||
try:
|
||||
agent_state = FindTestsState.model_validate(agent_state_dict)
|
||||
except ValidationError as e:
|
||||
logger.error("Invalid state dict for finding tests: %s", e)
|
||||
return Command(
|
||||
goto=PatcherAgentName.ROOT_CAUSE_ANALYSIS.value,
|
||||
)
|
||||
|
||||
return Command(
|
||||
update={
|
||||
"tests_instructions": agent_state.tests_instructions,
|
||||
},
|
||||
goto=PatcherAgentName.ROOT_CAUSE_ANALYSIS.value,
|
||||
)
|
||||
except langgraph.errors.GraphRecursionError:
|
||||
logger.error(
|
||||
"Reached recursion limit for finding tests in Challenge Task %s/%s",
|
||||
def run_find_tests_agent() -> FindTestsState | None:
|
||||
"""Run the find tests agent with timeout protection."""
|
||||
try:
|
||||
agent_state = input_state
|
||||
for _ in range(10):
|
||||
self.find_tests_agent.invoke(
|
||||
agent_state,
|
||||
config=RunnableConfig(
|
||||
recursion_limit=configuration.ctx_retriever_recursion_limit,
|
||||
configurable=configuration.model_dump(),
|
||||
),
|
||||
)
|
||||
agent_state_dict = self.find_tests_agent.get_state( # type: ignore[attr-defined]
|
||||
RunnableConfig(configurable=configuration.model_dump())
|
||||
).values
|
||||
try:
|
||||
agent_state = FindTestsState.model_validate(agent_state_dict)
|
||||
except ValidationError as e:
|
||||
logger.error("Invalid state dict for finding tests: %s", e)
|
||||
return None
|
||||
|
||||
if agent_state.tests_instructions:
|
||||
return agent_state
|
||||
|
||||
agent_state.messages = [
|
||||
*agent_state.messages,
|
||||
HumanMessage(
|
||||
content="You did not provide any instructions to run tests. Please try again or harder.",
|
||||
),
|
||||
]
|
||||
except langgraph.errors.GraphRecursionError:
|
||||
logger.error(
|
||||
"Reached recursion limit for finding tests in Challenge Task %s/%s",
|
||||
self.challenge.task_meta.task_id,
|
||||
self.challenge.name,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.exception("Error finding tests: %s", e)
|
||||
|
||||
return None
|
||||
|
||||
# Run the find tests agent with a 30-minute timeout
|
||||
start_time = time.time()
|
||||
timeout_seconds = 30 * 60 # 30 minutes
|
||||
|
||||
try:
|
||||
with get_executor_for_config(RunnableConfig(max_concurrency=1)) as executor:
|
||||
future = executor.submit(run_find_tests_agent)
|
||||
agent_state = future.result(timeout=timeout_seconds)
|
||||
|
||||
if agent_state and agent_state.tests_instructions:
|
||||
elapsed_time = time.time() - start_time
|
||||
logger.info(
|
||||
"Successfully found test instructions in %.2f seconds for Challenge Task %s/%s",
|
||||
elapsed_time,
|
||||
self.challenge.task_meta.task_id,
|
||||
self.challenge.name,
|
||||
)
|
||||
if not custom_test_path.exists() or not custom_test_path.is_file():
|
||||
custom_test_path.write_text(agent_state.tests_instructions)
|
||||
|
||||
return Command(
|
||||
update={
|
||||
"tests_instructions": agent_state.tests_instructions,
|
||||
},
|
||||
goto=PatcherAgentName.ROOT_CAUSE_ANALYSIS.value,
|
||||
)
|
||||
else:
|
||||
elapsed_time = time.time() - start_time
|
||||
logger.warning(
|
||||
"Failed to find test instructions after %.2f seconds for Challenge Task %s/%s",
|
||||
elapsed_time,
|
||||
self.challenge.task_meta.task_id,
|
||||
self.challenge.name,
|
||||
)
|
||||
return Command(
|
||||
goto=PatcherAgentName.ROOT_CAUSE_ANALYSIS.value,
|
||||
)
|
||||
|
||||
except TimeoutError:
|
||||
elapsed_time = time.time() - start_time
|
||||
logger.warning(
|
||||
"Timeout after %.2f seconds (30 minutes) while finding tests for Challenge Task %s/%s",
|
||||
elapsed_time,
|
||||
self.challenge.task_meta.task_id,
|
||||
self.challenge.name,
|
||||
)
|
||||
@@ -1155,7 +1316,14 @@ class ContextRetrieverAgent(PatcherAgentBase):
|
||||
goto=PatcherAgentName.ROOT_CAUSE_ANALYSIS.value,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.exception("Error finding tests: %s", e)
|
||||
elapsed_time = time.time() - start_time
|
||||
logger.exception(
|
||||
"Unexpected error after %.2f seconds while finding tests for Challenge Task %s/%s: %s",
|
||||
elapsed_time,
|
||||
self.challenge.task_meta.task_id,
|
||||
self.challenge.name,
|
||||
e,
|
||||
)
|
||||
return Command(
|
||||
goto=PatcherAgentName.ROOT_CAUSE_ANALYSIS.value,
|
||||
)
|
||||
|
||||
@@ -637,7 +637,6 @@ class QEAgent(PatcherAgentBase):
|
||||
|
||||
if sh_cmd_res is not None:
|
||||
tests_passed = sh_cmd_res.success
|
||||
last_patch_attempt.tests_passed = tests_passed
|
||||
last_patch_attempt.tests_stdout = sh_cmd_res.output
|
||||
last_patch_attempt.tests_stderr = sh_cmd_res.error
|
||||
else:
|
||||
@@ -654,6 +653,7 @@ class QEAgent(PatcherAgentBase):
|
||||
)
|
||||
tests_passed = True
|
||||
|
||||
last_patch_attempt.tests_passed = tests_passed
|
||||
if tests_passed:
|
||||
logger.info(
|
||||
"[%s / %s] Tests for Challenge Task %s ran successfully",
|
||||
|
||||
@@ -186,7 +186,7 @@ After your analysis, generate a structured reflection result using the following
|
||||
<failure_category>[Choose one: incomplete_fix, wrong_approach, misunderstood_root_cause, missing_code_snippet, build_error, regression_issue]</failure_category>
|
||||
<pattern_identified>[Describe any patterns seen across multiple failures, including loop detection results, or state "No clear pattern identified" if none are apparent]</pattern_identified>
|
||||
<next_component>[Select one of the available components, ensuring it breaks any detected loops]</next_component>
|
||||
<component_guidance>[Provide detailed, specific, actionable guidance for the selected component. Focus on security requirements and concrete steps to address the vulnerability. If breaking a loop, explain why this approach is different.]</component_guidance>
|
||||
<component_guidance>[Provide detailed, specific, actionable guidance for the selected component. Focus on security requirements and concrete steps to address the vulnerability. If breaking a loop, explain why this approach is different. Also include an explanation of the patterns identified across multiple failures and how to fix them.]</component_guidance>
|
||||
<partial_success>[True if the patch shows partial success, False if it is completely broken and should be discarded. If the next component guidance says to "improve the patch" or modify the patch in some way, then the patch is partially successful.]</partial_success>
|
||||
</reflection_result>
|
||||
|
||||
|
||||
@@ -23,16 +23,17 @@ from buttercup.patcher.agents.common import (
|
||||
ContextCodeSnippet,
|
||||
CodeSnippetRequest,
|
||||
get_stacktraces_from_povs,
|
||||
stacktrace_to_str,
|
||||
)
|
||||
from buttercup.common.llm import ButtercupLLM, create_default_llm_with_temperature
|
||||
from langgraph.types import Command
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
ROOT_CAUSE_SYSTEM_MSG = (
|
||||
"You are an expert security analyst. Your role is to analyze source code for security vulnerabilities "
|
||||
"with the depth and precision needed for automated patch generation."
|
||||
)
|
||||
ROOT_CAUSE_SYSTEM_MSG = """You are PatchGen-LLM, an autonomous component in an end-to-end security-patching pipeline.
|
||||
Goal: perform a Root Cause Analysis of one (or more) security vulnerabilities.
|
||||
The Root Cause Analysis will be used by a downstream code-generation agent, so factual and structural accuracy are critical.
|
||||
"""
|
||||
|
||||
ROOT_CAUSE_USER_MSG = """You are analyzing a security vulnerability in the following project:
|
||||
|
||||
@@ -55,13 +56,16 @@ The vulnerability has triggered one or more sanitizers, with the following stack
|
||||
{STACKTRACES}
|
||||
</stacktraces>
|
||||
|
||||
If there are multiple stacktraces, consider them as part of the same vulnerability.
|
||||
If there are multiple stacktraces, consider them as being different \
|
||||
manifestations of the same vulnerability. In such cases, you should try as much \
|
||||
as possible to discover the single real root cause of the vulnerabilities and \
|
||||
not just the immediate symptoms.
|
||||
|
||||
{REFLECTION_GUIDANCE}
|
||||
|
||||
---
|
||||
|
||||
Your task is to produce a **precise, detailed root cause analysis** of the vulnerability. This analysis will be used by an automated patching system. Be rigorous and avoid speculation.
|
||||
Your task is to produce a **precise, detailed Root Cause Analysis** of the vulnerability. Be rigorous and avoid speculation.
|
||||
|
||||
Request additional code snippets if they are *critical* to understand the root cause:
|
||||
- Exact failure location
|
||||
@@ -160,10 +164,20 @@ class RootCauseAgent(PatcherAgentBase):
|
||||
def _root_cause_prompt(self, state: PatcherAgentState) -> list[BaseMessage]:
|
||||
diff_content = "\n".join(diff.read_text() for diff in self.challenge.get_diffs())
|
||||
stacktraces = [parse_stacktrace(pov.sanitizer_output) for pov in state.context.povs]
|
||||
stacktraces_strs = get_stacktraces_from_povs(state.context.povs)
|
||||
|
||||
last_patch_attempt = state.get_last_patch_attempt()
|
||||
if last_patch_attempt and not last_patch_attempt.pov_fixed:
|
||||
sanitizer_output = last_patch_attempt.pov_stdout.decode("utf-8") if last_patch_attempt.pov_stdout else ""
|
||||
sanitizer_output += last_patch_attempt.pov_stderr.decode("utf-8") if last_patch_attempt.pov_stderr else ""
|
||||
stacktraces_strs.append(stacktrace_to_str("", sanitizer_output))
|
||||
|
||||
stacktraces_str = "\n".join(stacktraces_strs)
|
||||
|
||||
return ROOT_CAUSE_PROMPT.format_messages(
|
||||
DIFF=diff_content,
|
||||
PROJECT_NAME=self.challenge.project_name,
|
||||
STACKTRACES="\n".join(get_stacktraces_from_povs(state.context.povs)),
|
||||
STACKTRACES=stacktraces_str,
|
||||
CODE_SNIPPETS="\n".join([cs.commented_code(stacktraces) for cs in state.relevant_code_snippets]),
|
||||
REFLECTION_GUIDANCE=self._get_reflection_guidance_prompt(state),
|
||||
messages=state.messages,
|
||||
|
||||
@@ -122,15 +122,11 @@ PROMPT = ChatPromptTemplate.from_messages(
|
||||
]
|
||||
)
|
||||
|
||||
PATCH_STRATEGY_SYSTEM_MSG = (
|
||||
"You are part of an autonomous LLM-based system designed to generate precise patches for security vulnerabilities. "
|
||||
"Your responsibility is to develop a targeted patch strategy based on provided root cause analysis and code context."
|
||||
)
|
||||
|
||||
PATCH_STRATEGY_USER_MSG = """You are responsible for creating a precise and minimal patch strategy to address a specific vulnerability. Your output will be used in an automated patch generation system, so accuracy is critical.
|
||||
|
||||
Here is the information available to you:
|
||||
PATCH_STRATEGY_SYSTEM_MSG = """You are PatchGen-LLM, an autonomous component in an end-to-end security-patching pipeline.
|
||||
Goal: design a precise, minimal patch strategy that eliminates ONLY the vulnerabilities described in the Root-Cause-Analysis (RCA).
|
||||
The strategy you output will be consumed by a downstream code-generation agent, so factual and structural accuracy are critical."""
|
||||
|
||||
PATCH_STRATEGY_USER_MSG = """INPUT SECTIONS:
|
||||
<project_name>
|
||||
{PROJECT_NAME}
|
||||
</project_name>
|
||||
@@ -147,32 +143,41 @@ Here is the information available to you:
|
||||
|
||||
---
|
||||
|
||||
Your objective is to fix ONLY the identified vulnerability — no general improvements or unrelated changes.
|
||||
Use the `understand_code_snippet` tool to gain a deeper understanding where necessary.
|
||||
Consider a few different approaches to fix the vulnerability, and evaluate pros and cons of each approach, then choose the best approach.
|
||||
You do not have to do any code changes, just propose a patch strategy.
|
||||
Put all your analysis and reasoning inside <patch_development_process> tags.
|
||||
Once you have chosen a patch strategy, write a detailed description under the <full_description> tag.
|
||||
TOOLS:
|
||||
• understand_code_snippet
|
||||
Use this tool whenever you need clarification about how a given snippet works.
|
||||
|
||||
### If additional code or context is needed:
|
||||
Use the format below to request it explicitly:
|
||||
OUTPUT FORMAT (MANDATORY)
|
||||
<patch_development_process>
|
||||
a. List 2-4 alternative mitigation ideas, each with pros/cons.
|
||||
b. Identify your selected approach and justify why it is the best trade-off. Prefer the most minimalistic approach, unless instructed otherwise.
|
||||
c. Reference line numbers / function names from <code_snippets> as needed.
|
||||
</patch_development_process>
|
||||
<full_description>
|
||||
A thourough, detailed and complete description of the chosen patch strategy written for another LLM that will implement it.
|
||||
</full_description>
|
||||
|
||||
REQUESTING MORE INFORMATION
|
||||
If you need additional code or context, request it ONLY in this form:
|
||||
|
||||
```xml
|
||||
<request_information>
|
||||
[Clearly describe what additional code snippet or information you need and why it's necessary.]
|
||||
[Describe exactly what you need and why.]
|
||||
</request_information>
|
||||
```
|
||||
|
||||
### Important guidelines:
|
||||
- DO propose a fix for only the exact issue described in the root cause analysis.
|
||||
- DO NOT provide code changes, only the approach you will take to fix the vulnerability.
|
||||
- DO NOT include:
|
||||
1. General security or refactoring changes
|
||||
2. Code style or formatting adjustments
|
||||
3. Tests, documentation, or performance optimizations
|
||||
- Stay laser-focused on fixing only the vulnerability.
|
||||
POLICIES (hard constraints)
|
||||
|
||||
Proceed carefully and precisely.
|
||||
1. Scope:
|
||||
- Fix the vulnerabilities in the RCA only—nothing else
|
||||
- No stylistic, performance, refactor, or documentation changes
|
||||
- Do not propose test cases or broad hardening.
|
||||
2. Content:
|
||||
- Do NOT output code diffs or concrete code; output strategy only.
|
||||
3. Structure:
|
||||
- Use the exact tags and ordering shown in “OUTPUT FORMAT”.
|
||||
|
||||
Begin when ready.
|
||||
"""
|
||||
|
||||
PATCH_STRATEGY_PROMPT = ChatPromptTemplate.from_messages(
|
||||
|
||||
@@ -2327,6 +2327,10 @@ def test_find_tests_parallel(
|
||||
with (
|
||||
patch("buttercup.common.challenge_task.ChallengeTask.exec_docker_cmd") as mock_exec,
|
||||
patch("buttercup.common.challenge_task.ChallengeTask.get_clean_task") as mock_clean_task,
|
||||
patch("buttercup.common.challenge_task.ChallengeTask.apply_patch_diff") as mock_apply_patch_diff,
|
||||
patch(
|
||||
"buttercup.patcher.agents.context_retriever._are_test_instructions_valid"
|
||||
) as mock_are_test_instructions_valid,
|
||||
):
|
||||
|
||||
@contextmanager
|
||||
@@ -2334,6 +2338,7 @@ def test_find_tests_parallel(
|
||||
yield mock_challenge
|
||||
|
||||
mock_clean_task.return_value = mock_challenge
|
||||
mock_apply_patch_diff.return_value = True
|
||||
mock_challenge.apply_patch_diff = MagicMock(return_value=True)
|
||||
mock_challenge.get_rw_copy = MagicMock(side_effect=yield_challenge)
|
||||
|
||||
@@ -2343,7 +2348,7 @@ def test_find_tests_parallel(
|
||||
mount_dirs_list = list(mount_dirs.items())
|
||||
test_file_path = mount_dirs_list[0][0]
|
||||
test_file_content = test_file_path.read_text()
|
||||
if "cd /src" in test_file_content:
|
||||
if "cd /src2" in test_file_content:
|
||||
return CommandResult(
|
||||
success=True,
|
||||
returncode=0,
|
||||
@@ -2352,15 +2357,16 @@ def test_find_tests_parallel(
|
||||
)
|
||||
else:
|
||||
return CommandResult(
|
||||
success=False,
|
||||
success=True,
|
||||
returncode=1,
|
||||
output=b"",
|
||||
output=b"Tests failed",
|
||||
error=b"",
|
||||
)
|
||||
|
||||
return CommandResult(success=True, returncode=0, output=b"", error=b"")
|
||||
|
||||
mock_exec.side_effect = test_instructions_exec
|
||||
mock_are_test_instructions_valid.return_value = True
|
||||
|
||||
result = mock_agent.find_tests_node(state, mock_runnable_config)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user