From 00a7a4be332230fe7b709189caa5553192b13cda Mon Sep 17 00:00:00 2001 From: Henrik Brodin <90325907+hbrodin@users.noreply.github.com> Date: Sun, 22 Jun 2025 12:36:37 +0200 Subject: [PATCH] Merge SubmissionEntries based on similarity and on patch mitigation (#899) * Use an internal_patch_id instead of indices This is the first step in being able to merge sets of PoVs and test patches against all PoVs within a task. * Discard redundant builds * Initial PoV-merging strategy Still not optimal in terms of SARIF-matching/bundling etc * Appears to be working version of merging including bundle and sarif handling * Make tests pass * Update integration test steps * Fixes and cleanup from review * Removed additional request for patched builds * Refactored some loops into find-style functions to simplify * Inline small function used once * Refactors for increased robustness and readability including additional testing * SARIF matching - additional tests and refactor * Add enumerate_task_submissions * Refactor and simplify tests using a builder Cleanup unused code * Fix read_submissions to use CrashWithId * Improvements based on review * Cache final states of PoV reproduce (#909) As these never change we can limit the load on redis by caching the results. * Merge SubmissionEntries based on patches (#910) * Cache final states of PoV reproduce As these never change we can limit the load on redis by caching the results. * Merge SubmissionEntries based on patches If a PoV in another entry is mitigated by the current entry's patch, merge the entries as athey should be considered the same ChallengeVulnerability. * Add positional argument * Hold of submitting a patch while evaluating Check each already submitted patch before submitting a new one for the same task. If any of the already submitted patches mitigates any PoV in the current SubmissionEntry - do not submit this. It will be merged later on. * Additional logging, truncate ids * Only request patch if no submitted patch mitigates Before we request a new patch, we check each of the already submitted patches to see if any of them already mitigates the PoVs in the current SubmissionEntry. If they do, this will be merged at a later stage. * PR feedback --- .github/workflows/integration.yml | 28 +- common/protos/msg.proto | 55 +- .../common/datastructures/msg_pb2.py | 28 +- .../common/datastructures/msg_pb2.pyi | 74 +- common/src/buttercup/common/sets.py | 31 + common/src/buttercup/common/util_cli.py | 3 +- common/tests/test_set.py | 337 ++ .../orchestrator/scheduler/scheduler.py | 2 +- .../orchestrator/scheduler/submissions.py | 1599 ++++--- orchestrator/test/test_submissions.py | 4203 +++++++++++++++-- 10 files changed, 5281 insertions(+), 1079 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 80f4f6ed..dc46acac 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -148,13 +148,13 @@ jobs: - name: Wait for vuln to be found timeout-minutes: ${{ fromJSON(env.VULN_TIMEOUT) }} run: | - while ! kubectl logs -n $BUTTERCUP_NAMESPACE -l app=scheduler --tail=-1 | grep "] SUBMIT_PATCH_REQUEST pov_id=" ; do + while ! kubectl logs -n $BUTTERCUP_NAMESPACE -l app=scheduler --tail=-1 | grep "] POV submission response: pov_id=" ; do sleep 60 done - name: Wait for vuln to enter the passed state in competition api timeout-minutes: ${{ fromJSON(env.VULN_TIMEOUT) }} run: | - while ! kubectl logs -n $BUTTERCUP_NAMESPACE -l app=scheduler --tail=-1 | grep "WAIT_POV_PASS -> SUBMIT_PATCH" ; do + while ! kubectl logs -n $BUTTERCUP_NAMESPACE -l app=scheduler --tail=-1 | grep "Updated POV status. New status PASSED" ; do sleep 60 done @@ -165,30 +165,30 @@ jobs: sleep 60 done - - name: Wait for patch to be submitted + - name: Wait for patch to be recorded timeout-minutes: ${{ fromJSON(env.PATCH_TIMEOUT) }} run: | - while ! kubectl logs -n $BUTTERCUP_NAMESPACE -l app=scheduler --tail=-1 | grep "SUBMIT_PATCH -> WAIT_PATCH_PASS"; do + while ! kubectl logs -n $BUTTERCUP_NAMESPACE -l app=scheduler --tail=-1 | grep "Appending patch for task"; do sleep 60 done - name: Wait for patch to enter the passed state in competition api timeout-minutes: ${{ fromJSON(env.PATCH_TIMEOUT) }} run: | - while ! kubectl logs -n $BUTTERCUP_NAMESPACE -l app=scheduler --tail=-1 | grep "WAIT_PATCH_PASS -> SUBMIT_BUNDLE"; do + while ! kubectl logs -n $BUTTERCUP_NAMESPACE -l app=scheduler --tail=-1 | grep "Patch passed"; do sleep 60 done - name: Wait for bundle to be submitted timeout-minutes: ${{ fromJSON(env.BUNDLE_TIMEOUT) }} run: | - while ! kubectl logs -n $BUTTERCUP_NAMESPACE -l app=scheduler --tail=-1 | grep "SUBMIT_BUNDLE -> SUBMIT_MATCHING_SARIF"; do + while ! kubectl logs -n $BUTTERCUP_NAMESPACE -l app=scheduler --tail=-1 | grep "Bundle submission response: bundle_id="; do sleep 60 done - name: Send a SARIF broadcast timeout-minutes: ${{ fromJSON(env.SARIF_TIMEOUT) }} run: | - TASK_ID=$(kubectl logs -n $BUTTERCUP_NAMESPACE -l app=scheduler --tail=-1 | grep "SUBMIT_BUNDLE -> SUBMIT_MATCHING_SARIF" | grep -o "\[[^]]*\]" | grep -o "[^[]*$" | cut -d: -f3 | tr -d ']') + TASK_ID=$(kubectl logs -n $BUTTERCUP_NAMESPACE -l app=scheduler --tail=-1 | grep "Submitting bundle for harness" | grep -o "\[[^]]*\]" | grep -o "[^[]*$" | cut -d: -f3 | tr -d ']') echo "Task ID: $TASK_ID" while true; do # Kill any existing port-forward processes @@ -209,16 +209,16 @@ jobs: # Failed - wait a bit before retrying sleep 10 done - - name: Wait for SARIF to be submitted as correct - timeout-minutes: ${{ fromJSON(env.SARIF_TIMEOUT) }} - run: | - while ! kubectl logs -n $BUTTERCUP_NAMESPACE -l app=scheduler --tail=-1 | grep "SUBMIT_MATCHING_SARIF -> SUBMIT_BUNDLE_PATCH"; do - sleep 60 - done - name: Wait for Bundle to be patched to include the SARIF timeout-minutes: ${{ fromJSON(env.SARIF_TIMEOUT) }} run: | - while ! kubectl logs -n $BUTTERCUP_NAMESPACE -l app=scheduler --tail=-1 | grep "SUBMIT_BUNDLE_PATCH -> STOP"; do + while ! kubectl logs -n $BUTTERCUP_NAMESPACE -l app=scheduler --tail=-1 | grep "Bundle patch submission response: broadcast_sarif_id="; do + sleep 60 + done + - name: Wait for SARIF to be submitted as correct + timeout-minutes: ${{ fromJSON(env.SARIF_TIMEOUT) }} + run: | + while ! kubectl logs -n $BUTTERCUP_NAMESPACE -l app=scheduler --tail=-1 | grep "Matching SARIF submission response"; do sleep 60 done diff --git a/common/protos/msg.proto b/common/protos/msg.proto index 8edcb693..76dae473 100644 --- a/common/protos/msg.proto +++ b/common/protos/msg.proto @@ -137,40 +137,53 @@ message FunctionCoverage { int32 covered_lines = 4; } +enum SubmissionResult { + NONE = 0; // Default/unset state - no submission result yet + ACCEPTED = 1; + PASSED = 2; + FAILED = 3; + DEADLINE_EXCEEDED = 4; + ERRORED = 5; + INCONCLUSIVE = 6; +} + message SubmissionEntryPatch { string patch = 1; string internal_patch_id = 2; string competition_patch_id = 3; repeated BuildOutput build_outputs = 4; + optional SubmissionResult result = 5; } -message SubmissionEntry { - enum SubmissionState { - SUBMIT_PATCH_REQUEST = 0; - WAIT_POV_PASS = 1; - SUBMIT_PATCH = 2; - WAIT_PATCH_PASS = 3; - SUBMIT_BUNDLE = 4; - SUBMIT_MATCHING_SARIF = 5; - SUBMIT_BUNDLE_PATCH = 6; - STOP = 7; - } - - SubmissionState state = 1; - repeated TracedCrash crashes = 2; - - string pov_id = 3; - string competition_patch_id = 4; +message Bundle { + string task_id = 1; + string competition_pov_id = 2; + string competition_patch_id = 3; + string competition_sarif_id = 4; string bundle_id = 5; - string sarif_id = 6; +} + +message CrashWithId { + TracedCrash crash = 1; + string competition_pov_id = 2; + optional SubmissionResult result = 3; +} + +// This is our model of a Challenge Vulnerability +message SubmissionEntry { + bool stop = 1; + + repeated CrashWithId crashes = 2; + + repeated Bundle bundles = 3; // Content of patches targeting this vulnerability - repeated SubmissionEntryPatch patches = 7; + repeated SubmissionEntryPatch patches = 4; // Current patch being considered - int32 patch_idx = 8; - int32 patch_submission_attempt = 9; + int32 patch_idx = 5; + int32 patch_submission_attempts = 6; } message POVReproduceRequest { diff --git a/common/src/buttercup/common/datastructures/msg_pb2.py b/common/src/buttercup/common/datastructures/msg_pb2.py index cc8774a1..4fdde28c 100644 --- a/common/src/buttercup/common/datastructures/msg_pb2.py +++ b/common/src/buttercup/common/datastructures/msg_pb2.py @@ -13,7 +13,7 @@ _sym_db = _symbol_database.Default() -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\tmsg.proto\x12\x06msgpb2\"\xf0\x02\n\x04Task\x12\x12\n\nmessage_id\x18\x01 \x01(\t\x12\x14\n\x0cmessage_time\x18\x02 \x01(\x03\x12\x0f\n\x07task_id\x18\x03 \x01(\t\x12(\n\ttask_type\x18\x04 \x01(\x0e\x32\x15.msgpb2.Task.TaskType\x12%\n\x07sources\x18\x05 \x03(\x0b\x32\x14.msgpb2.SourceDetail\x12\x10\n\x08\x64\x65\x61\x64line\x18\x06 \x01(\x03\x12\x11\n\tcancelled\x18\x07 \x01(\x08\x12\x14\n\x0cproject_name\x18\x08 \x01(\t\x12\r\n\x05\x66ocus\x18\t \x01(\t\x12,\n\x08metadata\x18\n \x03(\x0b\x32\x1a.msgpb2.Task.MetadataEntry\x1a/\n\rMetadataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"3\n\x08TaskType\x12\x12\n\x0eTASK_TYPE_FULL\x10\x00\x12\x13\n\x0fTASK_TYPE_DELTA\x10\x01\"\xb9\x01\n\x0cSourceDetail\x12\x0e\n\x06sha256\x18\x01 \x01(\t\x12\x34\n\x0bsource_type\x18\x02 \x01(\x0e\x32\x1f.msgpb2.SourceDetail.SourceType\x12\x0b\n\x03url\x18\x03 \x01(\t\"V\n\nSourceType\x12\x14\n\x10SOURCE_TYPE_REPO\x10\x00\x12\x1c\n\x18SOURCE_TYPE_FUZZ_TOOLING\x10\x01\x12\x14\n\x10SOURCE_TYPE_DIFF\x10\x02\"*\n\x0cTaskDownload\x12\x1a\n\x04task\x18\x01 \x01(\x0b\x32\x0c.msgpb2.Task\"\'\n\tTaskReady\x12\x1a\n\x04task\x18\x01 \x01(\x0b\x32\x0c.msgpb2.Task\"T\n\nTaskDelete\x12\x11\n\x07task_id\x18\x01 \x01(\tH\x00\x12\r\n\x03\x61ll\x18\x03 \x01(\x08H\x00\x12\x13\n\x0breceived_at\x18\x02 \x01(\x02\x42\x0f\n\rdelete_option\"\xb9\x01\n\x0c\x42uildRequest\x12\x0e\n\x06\x65ngine\x18\x01 \x01(\t\x12\x11\n\tsanitizer\x18\x02 \x01(\t\x12\x10\n\x08task_dir\x18\x03 \x01(\t\x12\x0f\n\x07task_id\x18\x04 \x01(\t\x12%\n\nbuild_type\x18\x05 \x01(\x0e\x32\x11.msgpb2.BuildType\x12\x12\n\napply_diff\x18\x06 \x01(\x08\x12\r\n\x05patch\x18\x07 \x01(\t\x12\x19\n\x11internal_patch_id\x18\x08 \x01(\t\"\xa9\x01\n\x0b\x42uildOutput\x12\x0e\n\x06\x65ngine\x18\x01 \x01(\t\x12\x11\n\tsanitizer\x18\x02 \x01(\t\x12\x10\n\x08task_dir\x18\x03 \x01(\t\x12\x0f\n\x07task_id\x18\x04 \x01(\t\x12%\n\nbuild_type\x18\x05 \x01(\x0e\x32\x11.msgpb2.BuildType\x12\x12\n\napply_diff\x18\x06 \x01(\x08\x12\x19\n\x11internal_patch_id\x18\x07 \x01(\t\"^\n\x0fWeightedHarness\x12\x0e\n\x06weight\x18\x01 \x01(\x02\x12\x14\n\x0cpackage_name\x18\x02 \x01(\t\x12\x14\n\x0charness_name\x18\x03 \x01(\t\x12\x0f\n\x07task_id\x18\x04 \x01(\t\"\x85\x01\n\x05\x43rash\x12#\n\x06target\x18\x01 \x01(\x0b\x32\x13.msgpb2.BuildOutput\x12\x14\n\x0charness_name\x18\x02 \x01(\t\x12\x18\n\x10\x63rash_input_path\x18\x03 \x01(\t\x12\x12\n\nstacktrace\x18\x04 \x01(\t\x12\x13\n\x0b\x63rash_token\x18\x05 \x01(\t\"F\n\x0bTracedCrash\x12\x1c\n\x05\x63rash\x18\x01 \x01(\x0b\x32\r.msgpb2.Crash\x12\x19\n\x11tracer_stacktrace\x18\x02 \x01(\t\"Y\n\x16\x43onfirmedVulnerability\x12$\n\x07\x63rashes\x18\x01 \x03(\x0b\x32\x13.msgpb2.TracedCrash\x12\x19\n\x11internal_patch_id\x18\x02 \x01(\t\"B\n\x05Patch\x12\x0f\n\x07task_id\x18\x01 \x01(\t\x12\x19\n\x11internal_patch_id\x18\x02 \x01(\t\x12\r\n\x05patch\x18\x03 \x01(\t\"\x81\x01\n\x0cIndexRequest\x12%\n\nbuild_type\x18\x01 \x01(\x0e\x32\x11.msgpb2.BuildType\x12\x14\n\x0cpackage_name\x18\x02 \x01(\t\x12\x11\n\tsanitizer\x18\x03 \x01(\t\x12\x10\n\x08task_dir\x18\x04 \x01(\t\x12\x0f\n\x07task_id\x18\x05 \x01(\t\"\x80\x01\n\x0bIndexOutput\x12%\n\nbuild_type\x18\x01 \x01(\x0e\x32\x11.msgpb2.BuildType\x12\x14\n\x0cpackage_name\x18\x02 \x01(\t\x12\x11\n\tsanitizer\x18\x03 \x01(\t\x12\x10\n\x08task_dir\x18\x04 \x01(\t\x12\x0f\n\x07task_id\x18\x05 \x01(\t\"m\n\x10\x46unctionCoverage\x12\x15\n\rfunction_name\x18\x01 \x01(\t\x12\x16\n\x0e\x66unction_paths\x18\x02 \x03(\t\x12\x13\n\x0btotal_lines\x18\x03 \x01(\x05\x12\x15\n\rcovered_lines\x18\x04 \x01(\x05\"\x8a\x01\n\x14SubmissionEntryPatch\x12\r\n\x05patch\x18\x01 \x01(\t\x12\x19\n\x11internal_patch_id\x18\x02 \x01(\t\x12\x1c\n\x14\x63ompetition_patch_id\x18\x03 \x01(\t\x12*\n\rbuild_outputs\x18\x04 \x03(\x0b\x32\x13.msgpb2.BuildOutput\"\xdf\x03\n\x0fSubmissionEntry\x12\x36\n\x05state\x18\x01 \x01(\x0e\x32\'.msgpb2.SubmissionEntry.SubmissionState\x12$\n\x07\x63rashes\x18\x02 \x03(\x0b\x32\x13.msgpb2.TracedCrash\x12\x0e\n\x06pov_id\x18\x03 \x01(\t\x12\x1c\n\x14\x63ompetition_patch_id\x18\x04 \x01(\t\x12\x11\n\tbundle_id\x18\x05 \x01(\t\x12\x10\n\x08sarif_id\x18\x06 \x01(\t\x12-\n\x07patches\x18\x07 \x03(\x0b\x32\x1c.msgpb2.SubmissionEntryPatch\x12\x11\n\tpatch_idx\x18\x08 \x01(\x05\x12 \n\x18patch_submission_attempt\x18\t \x01(\x05\"\xb6\x01\n\x0fSubmissionState\x12\x18\n\x14SUBMIT_PATCH_REQUEST\x10\x00\x12\x11\n\rWAIT_POV_PASS\x10\x01\x12\x10\n\x0cSUBMIT_PATCH\x10\x02\x12\x13\n\x0fWAIT_PATCH_PASS\x10\x03\x12\x11\n\rSUBMIT_BUNDLE\x10\x04\x12\x19\n\x15SUBMIT_MATCHING_SARIF\x10\x05\x12\x17\n\x13SUBMIT_BUNDLE_PATCH\x10\x06\x12\x08\n\x04STOP\x10\x07\"|\n\x13POVReproduceRequest\x12\x0f\n\x07task_id\x18\x01 \x01(\t\x12\x19\n\x11internal_patch_id\x18\x02 \x01(\t\x12\x14\n\x0charness_name\x18\x03 \x01(\t\x12\x11\n\tsanitizer\x18\x04 \x01(\t\x12\x10\n\x08pov_path\x18\x05 \x01(\t\"W\n\x14POVReproduceResponse\x12,\n\x07request\x18\x01 \x01(\x0b\x32\x1b.msgpb2.POVReproduceRequest\x12\x11\n\tdid_crash\x18\x02 \x01(\x08*D\n\tBuildType\x12\n\n\x06\x46UZZER\x10\x00\x12\x0c\n\x08\x43OVERAGE\x10\x01\x12\x12\n\x0eTRACER_NO_DIFF\x10\x02\x12\t\n\x05PATCH\x10\x03\x62\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\tmsg.proto\x12\x06msgpb2\"\xf0\x02\n\x04Task\x12\x12\n\nmessage_id\x18\x01 \x01(\t\x12\x14\n\x0cmessage_time\x18\x02 \x01(\x03\x12\x0f\n\x07task_id\x18\x03 \x01(\t\x12(\n\ttask_type\x18\x04 \x01(\x0e\x32\x15.msgpb2.Task.TaskType\x12%\n\x07sources\x18\x05 \x03(\x0b\x32\x14.msgpb2.SourceDetail\x12\x10\n\x08\x64\x65\x61\x64line\x18\x06 \x01(\x03\x12\x11\n\tcancelled\x18\x07 \x01(\x08\x12\x14\n\x0cproject_name\x18\x08 \x01(\t\x12\r\n\x05\x66ocus\x18\t \x01(\t\x12,\n\x08metadata\x18\n \x03(\x0b\x32\x1a.msgpb2.Task.MetadataEntry\x1a/\n\rMetadataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"3\n\x08TaskType\x12\x12\n\x0eTASK_TYPE_FULL\x10\x00\x12\x13\n\x0fTASK_TYPE_DELTA\x10\x01\"\xb9\x01\n\x0cSourceDetail\x12\x0e\n\x06sha256\x18\x01 \x01(\t\x12\x34\n\x0bsource_type\x18\x02 \x01(\x0e\x32\x1f.msgpb2.SourceDetail.SourceType\x12\x0b\n\x03url\x18\x03 \x01(\t\"V\n\nSourceType\x12\x14\n\x10SOURCE_TYPE_REPO\x10\x00\x12\x1c\n\x18SOURCE_TYPE_FUZZ_TOOLING\x10\x01\x12\x14\n\x10SOURCE_TYPE_DIFF\x10\x02\"*\n\x0cTaskDownload\x12\x1a\n\x04task\x18\x01 \x01(\x0b\x32\x0c.msgpb2.Task\"\'\n\tTaskReady\x12\x1a\n\x04task\x18\x01 \x01(\x0b\x32\x0c.msgpb2.Task\"T\n\nTaskDelete\x12\x11\n\x07task_id\x18\x01 \x01(\tH\x00\x12\r\n\x03\x61ll\x18\x03 \x01(\x08H\x00\x12\x13\n\x0breceived_at\x18\x02 \x01(\x02\x42\x0f\n\rdelete_option\"\xb9\x01\n\x0c\x42uildRequest\x12\x0e\n\x06\x65ngine\x18\x01 \x01(\t\x12\x11\n\tsanitizer\x18\x02 \x01(\t\x12\x10\n\x08task_dir\x18\x03 \x01(\t\x12\x0f\n\x07task_id\x18\x04 \x01(\t\x12%\n\nbuild_type\x18\x05 \x01(\x0e\x32\x11.msgpb2.BuildType\x12\x12\n\napply_diff\x18\x06 \x01(\x08\x12\r\n\x05patch\x18\x07 \x01(\t\x12\x19\n\x11internal_patch_id\x18\x08 \x01(\t\"\xa9\x01\n\x0b\x42uildOutput\x12\x0e\n\x06\x65ngine\x18\x01 \x01(\t\x12\x11\n\tsanitizer\x18\x02 \x01(\t\x12\x10\n\x08task_dir\x18\x03 \x01(\t\x12\x0f\n\x07task_id\x18\x04 \x01(\t\x12%\n\nbuild_type\x18\x05 \x01(\x0e\x32\x11.msgpb2.BuildType\x12\x12\n\napply_diff\x18\x06 \x01(\x08\x12\x19\n\x11internal_patch_id\x18\x07 \x01(\t\"^\n\x0fWeightedHarness\x12\x0e\n\x06weight\x18\x01 \x01(\x02\x12\x14\n\x0cpackage_name\x18\x02 \x01(\t\x12\x14\n\x0charness_name\x18\x03 \x01(\t\x12\x0f\n\x07task_id\x18\x04 \x01(\t\"\x85\x01\n\x05\x43rash\x12#\n\x06target\x18\x01 \x01(\x0b\x32\x13.msgpb2.BuildOutput\x12\x14\n\x0charness_name\x18\x02 \x01(\t\x12\x18\n\x10\x63rash_input_path\x18\x03 \x01(\t\x12\x12\n\nstacktrace\x18\x04 \x01(\t\x12\x13\n\x0b\x63rash_token\x18\x05 \x01(\t\"F\n\x0bTracedCrash\x12\x1c\n\x05\x63rash\x18\x01 \x01(\x0b\x32\r.msgpb2.Crash\x12\x19\n\x11tracer_stacktrace\x18\x02 \x01(\t\"Y\n\x16\x43onfirmedVulnerability\x12$\n\x07\x63rashes\x18\x01 \x03(\x0b\x32\x13.msgpb2.TracedCrash\x12\x19\n\x11internal_patch_id\x18\x02 \x01(\t\"B\n\x05Patch\x12\x0f\n\x07task_id\x18\x01 \x01(\t\x12\x19\n\x11internal_patch_id\x18\x02 \x01(\t\x12\r\n\x05patch\x18\x03 \x01(\t\"\x81\x01\n\x0cIndexRequest\x12%\n\nbuild_type\x18\x01 \x01(\x0e\x32\x11.msgpb2.BuildType\x12\x14\n\x0cpackage_name\x18\x02 \x01(\t\x12\x11\n\tsanitizer\x18\x03 \x01(\t\x12\x10\n\x08task_dir\x18\x04 \x01(\t\x12\x0f\n\x07task_id\x18\x05 \x01(\t\"\x80\x01\n\x0bIndexOutput\x12%\n\nbuild_type\x18\x01 \x01(\x0e\x32\x11.msgpb2.BuildType\x12\x14\n\x0cpackage_name\x18\x02 \x01(\t\x12\x11\n\tsanitizer\x18\x03 \x01(\t\x12\x10\n\x08task_dir\x18\x04 \x01(\t\x12\x0f\n\x07task_id\x18\x05 \x01(\t\"m\n\x10\x46unctionCoverage\x12\x15\n\rfunction_name\x18\x01 \x01(\t\x12\x16\n\x0e\x66unction_paths\x18\x02 \x03(\t\x12\x13\n\x0btotal_lines\x18\x03 \x01(\x05\x12\x15\n\rcovered_lines\x18\x04 \x01(\x05\"\xc4\x01\n\x14SubmissionEntryPatch\x12\r\n\x05patch\x18\x01 \x01(\t\x12\x19\n\x11internal_patch_id\x18\x02 \x01(\t\x12\x1c\n\x14\x63ompetition_patch_id\x18\x03 \x01(\t\x12*\n\rbuild_outputs\x18\x04 \x03(\x0b\x32\x13.msgpb2.BuildOutput\x12-\n\x06result\x18\x05 \x01(\x0e\x32\x18.msgpb2.SubmissionResultH\x00\x88\x01\x01\x42\t\n\x07_result\"\x84\x01\n\x06\x42undle\x12\x0f\n\x07task_id\x18\x01 \x01(\t\x12\x1a\n\x12\x63ompetition_pov_id\x18\x02 \x01(\t\x12\x1c\n\x14\x63ompetition_patch_id\x18\x03 \x01(\t\x12\x1c\n\x14\x63ompetition_sarif_id\x18\x04 \x01(\t\x12\x11\n\tbundle_id\x18\x05 \x01(\t\"\x87\x01\n\x0b\x43rashWithId\x12\"\n\x05\x63rash\x18\x01 \x01(\x0b\x32\x13.msgpb2.TracedCrash\x12\x1a\n\x12\x63ompetition_pov_id\x18\x02 \x01(\t\x12-\n\x06result\x18\x03 \x01(\x0e\x32\x18.msgpb2.SubmissionResultH\x00\x88\x01\x01\x42\t\n\x07_result\"\xcb\x01\n\x0fSubmissionEntry\x12\x0c\n\x04stop\x18\x01 \x01(\x08\x12$\n\x07\x63rashes\x18\x02 \x03(\x0b\x32\x13.msgpb2.CrashWithId\x12\x1f\n\x07\x62undles\x18\x03 \x03(\x0b\x32\x0e.msgpb2.Bundle\x12-\n\x07patches\x18\x04 \x03(\x0b\x32\x1c.msgpb2.SubmissionEntryPatch\x12\x11\n\tpatch_idx\x18\x05 \x01(\x05\x12!\n\x19patch_submission_attempts\x18\x06 \x01(\x05\"|\n\x13POVReproduceRequest\x12\x0f\n\x07task_id\x18\x01 \x01(\t\x12\x19\n\x11internal_patch_id\x18\x02 \x01(\t\x12\x14\n\x0charness_name\x18\x03 \x01(\t\x12\x11\n\tsanitizer\x18\x04 \x01(\t\x12\x10\n\x08pov_path\x18\x05 \x01(\t\"W\n\x14POVReproduceResponse\x12,\n\x07request\x18\x01 \x01(\x0b\x32\x1b.msgpb2.POVReproduceRequest\x12\x11\n\tdid_crash\x18\x02 \x01(\x08*D\n\tBuildType\x12\n\n\x06\x46UZZER\x10\x00\x12\x0c\n\x08\x43OVERAGE\x10\x01\x12\x12\n\x0eTRACER_NO_DIFF\x10\x02\x12\t\n\x05PATCH\x10\x03*x\n\x10SubmissionResult\x12\x08\n\x04NONE\x10\x00\x12\x0c\n\x08\x41\x43\x43\x45PTED\x10\x01\x12\n\n\x06PASSED\x10\x02\x12\n\n\x06\x46\x41ILED\x10\x03\x12\x15\n\x11\x44\x45\x41\x44LINE_EXCEEDED\x10\x04\x12\x0b\n\x07\x45RRORED\x10\x05\x12\x10\n\x0cINCONCLUSIVE\x10\x06\x62\x06proto3') _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'msg_pb2', globals()) @@ -22,8 +22,10 @@ if _descriptor._USE_C_DESCRIPTORS == False: DESCRIPTOR._options = None _TASK_METADATAENTRY._options = None _TASK_METADATAENTRY._serialized_options = b'8\001' - _BUILDTYPE._serialized_start=2786 - _BUILDTYPE._serialized_end=2854 + _BUILDTYPE._serialized_start=2841 + _BUILDTYPE._serialized_end=2909 + _SUBMISSIONRESULT._serialized_start=2911 + _SUBMISSIONRESULT._serialized_end=3031 _TASK._serialized_start=22 _TASK._serialized_end=390 _TASK_METADATAENTRY._serialized_start=290 @@ -61,13 +63,15 @@ if _descriptor._USE_C_DESCRIPTORS == False: _FUNCTIONCOVERAGE._serialized_start=1837 _FUNCTIONCOVERAGE._serialized_end=1946 _SUBMISSIONENTRYPATCH._serialized_start=1949 - _SUBMISSIONENTRYPATCH._serialized_end=2087 - _SUBMISSIONENTRY._serialized_start=2090 - _SUBMISSIONENTRY._serialized_end=2569 - _SUBMISSIONENTRY_SUBMISSIONSTATE._serialized_start=2387 - _SUBMISSIONENTRY_SUBMISSIONSTATE._serialized_end=2569 - _POVREPRODUCEREQUEST._serialized_start=2571 - _POVREPRODUCEREQUEST._serialized_end=2695 - _POVREPRODUCERESPONSE._serialized_start=2697 - _POVREPRODUCERESPONSE._serialized_end=2784 + _SUBMISSIONENTRYPATCH._serialized_end=2145 + _BUNDLE._serialized_start=2148 + _BUNDLE._serialized_end=2280 + _CRASHWITHID._serialized_start=2283 + _CRASHWITHID._serialized_end=2418 + _SUBMISSIONENTRY._serialized_start=2421 + _SUBMISSIONENTRY._serialized_end=2624 + _POVREPRODUCEREQUEST._serialized_start=2626 + _POVREPRODUCEREQUEST._serialized_end=2750 + _POVREPRODUCERESPONSE._serialized_start=2752 + _POVREPRODUCERESPONSE._serialized_end=2839 # @@protoc_insertion_point(module_scope) diff --git a/common/src/buttercup/common/datastructures/msg_pb2.pyi b/common/src/buttercup/common/datastructures/msg_pb2.pyi index 729b9b3e..61cfbfb5 100644 --- a/common/src/buttercup/common/datastructures/msg_pb2.pyi +++ b/common/src/buttercup/common/datastructures/msg_pb2.pyi @@ -4,9 +4,16 @@ from google.protobuf import descriptor as _descriptor from google.protobuf import message as _message from typing import ClassVar as _ClassVar, Iterable as _Iterable, Mapping as _Mapping, Optional as _Optional, Union as _Union +ACCEPTED: SubmissionResult COVERAGE: BuildType +DEADLINE_EXCEEDED: SubmissionResult DESCRIPTOR: _descriptor.FileDescriptor +ERRORED: SubmissionResult +FAILED: SubmissionResult FUZZER: BuildType +INCONCLUSIVE: SubmissionResult +NONE: SubmissionResult +PASSED: SubmissionResult PATCH: BuildType TRACER_NO_DIFF: BuildType @@ -48,6 +55,20 @@ class BuildRequest(_message.Message): task_id: str def __init__(self, engine: _Optional[str] = ..., sanitizer: _Optional[str] = ..., task_dir: _Optional[str] = ..., task_id: _Optional[str] = ..., build_type: _Optional[_Union[BuildType, str]] = ..., apply_diff: bool = ..., patch: _Optional[str] = ..., internal_patch_id: _Optional[str] = ...) -> None: ... +class Bundle(_message.Message): + __slots__ = ["bundle_id", "competition_patch_id", "competition_pov_id", "competition_sarif_id", "task_id"] + BUNDLE_ID_FIELD_NUMBER: _ClassVar[int] + COMPETITION_PATCH_ID_FIELD_NUMBER: _ClassVar[int] + COMPETITION_POV_ID_FIELD_NUMBER: _ClassVar[int] + COMPETITION_SARIF_ID_FIELD_NUMBER: _ClassVar[int] + TASK_ID_FIELD_NUMBER: _ClassVar[int] + bundle_id: str + competition_patch_id: str + competition_pov_id: str + competition_sarif_id: str + task_id: str + def __init__(self, task_id: _Optional[str] = ..., competition_pov_id: _Optional[str] = ..., competition_patch_id: _Optional[str] = ..., competition_sarif_id: _Optional[str] = ..., bundle_id: _Optional[str] = ...) -> None: ... + class ConfirmedVulnerability(_message.Message): __slots__ = ["crashes", "internal_patch_id"] CRASHES_FIELD_NUMBER: _ClassVar[int] @@ -70,6 +91,16 @@ class Crash(_message.Message): target: BuildOutput def __init__(self, target: _Optional[_Union[BuildOutput, _Mapping]] = ..., harness_name: _Optional[str] = ..., crash_input_path: _Optional[str] = ..., stacktrace: _Optional[str] = ..., crash_token: _Optional[str] = ...) -> None: ... +class CrashWithId(_message.Message): + __slots__ = ["competition_pov_id", "crash", "result"] + COMPETITION_POV_ID_FIELD_NUMBER: _ClassVar[int] + CRASH_FIELD_NUMBER: _ClassVar[int] + RESULT_FIELD_NUMBER: _ClassVar[int] + competition_pov_id: str + crash: TracedCrash + result: SubmissionResult + def __init__(self, crash: _Optional[_Union[TracedCrash, _Mapping]] = ..., competition_pov_id: _Optional[str] = ..., result: _Optional[_Union[SubmissionResult, str]] = ...) -> None: ... + class FunctionCoverage(_message.Message): __slots__ = ["covered_lines", "function_name", "function_paths", "total_lines"] COVERED_LINES_FIELD_NUMBER: _ClassVar[int] @@ -158,48 +189,34 @@ class SourceDetail(_message.Message): def __init__(self, sha256: _Optional[str] = ..., source_type: _Optional[_Union[SourceDetail.SourceType, str]] = ..., url: _Optional[str] = ...) -> None: ... class SubmissionEntry(_message.Message): - __slots__ = ["bundle_id", "competition_patch_id", "crashes", "patch_idx", "patch_submission_attempt", "patches", "pov_id", "sarif_id", "state"] - class SubmissionState(int, metaclass=_enum_type_wrapper.EnumTypeWrapper): - __slots__ = [] - BUNDLE_ID_FIELD_NUMBER: _ClassVar[int] - COMPETITION_PATCH_ID_FIELD_NUMBER: _ClassVar[int] + __slots__ = ["bundles", "crashes", "patch_idx", "patch_submission_attempts", "patches", "stop"] + BUNDLES_FIELD_NUMBER: _ClassVar[int] CRASHES_FIELD_NUMBER: _ClassVar[int] PATCHES_FIELD_NUMBER: _ClassVar[int] PATCH_IDX_FIELD_NUMBER: _ClassVar[int] - PATCH_SUBMISSION_ATTEMPT_FIELD_NUMBER: _ClassVar[int] - POV_ID_FIELD_NUMBER: _ClassVar[int] - SARIF_ID_FIELD_NUMBER: _ClassVar[int] - STATE_FIELD_NUMBER: _ClassVar[int] - STOP: SubmissionEntry.SubmissionState - SUBMIT_BUNDLE: SubmissionEntry.SubmissionState - SUBMIT_BUNDLE_PATCH: SubmissionEntry.SubmissionState - SUBMIT_MATCHING_SARIF: SubmissionEntry.SubmissionState - SUBMIT_PATCH: SubmissionEntry.SubmissionState - SUBMIT_PATCH_REQUEST: SubmissionEntry.SubmissionState - WAIT_PATCH_PASS: SubmissionEntry.SubmissionState - WAIT_POV_PASS: SubmissionEntry.SubmissionState - bundle_id: str - competition_patch_id: str - crashes: _containers.RepeatedCompositeFieldContainer[TracedCrash] + PATCH_SUBMISSION_ATTEMPTS_FIELD_NUMBER: _ClassVar[int] + STOP_FIELD_NUMBER: _ClassVar[int] + bundles: _containers.RepeatedCompositeFieldContainer[Bundle] + crashes: _containers.RepeatedCompositeFieldContainer[CrashWithId] patch_idx: int - patch_submission_attempt: int + patch_submission_attempts: int patches: _containers.RepeatedCompositeFieldContainer[SubmissionEntryPatch] - pov_id: str - sarif_id: str - state: SubmissionEntry.SubmissionState - def __init__(self, state: _Optional[_Union[SubmissionEntry.SubmissionState, str]] = ..., crashes: _Optional[_Iterable[_Union[TracedCrash, _Mapping]]] = ..., pov_id: _Optional[str] = ..., competition_patch_id: _Optional[str] = ..., bundle_id: _Optional[str] = ..., sarif_id: _Optional[str] = ..., patches: _Optional[_Iterable[_Union[SubmissionEntryPatch, _Mapping]]] = ..., patch_idx: _Optional[int] = ..., patch_submission_attempt: _Optional[int] = ...) -> None: ... + stop: bool + def __init__(self, stop: bool = ..., crashes: _Optional[_Iterable[_Union[CrashWithId, _Mapping]]] = ..., bundles: _Optional[_Iterable[_Union[Bundle, _Mapping]]] = ..., patches: _Optional[_Iterable[_Union[SubmissionEntryPatch, _Mapping]]] = ..., patch_idx: _Optional[int] = ..., patch_submission_attempts: _Optional[int] = ...) -> None: ... class SubmissionEntryPatch(_message.Message): - __slots__ = ["build_outputs", "competition_patch_id", "internal_patch_id", "patch"] + __slots__ = ["build_outputs", "competition_patch_id", "internal_patch_id", "patch", "result"] BUILD_OUTPUTS_FIELD_NUMBER: _ClassVar[int] COMPETITION_PATCH_ID_FIELD_NUMBER: _ClassVar[int] INTERNAL_PATCH_ID_FIELD_NUMBER: _ClassVar[int] PATCH_FIELD_NUMBER: _ClassVar[int] + RESULT_FIELD_NUMBER: _ClassVar[int] build_outputs: _containers.RepeatedCompositeFieldContainer[BuildOutput] competition_patch_id: str internal_patch_id: str patch: str - def __init__(self, patch: _Optional[str] = ..., internal_patch_id: _Optional[str] = ..., competition_patch_id: _Optional[str] = ..., build_outputs: _Optional[_Iterable[_Union[BuildOutput, _Mapping]]] = ...) -> None: ... + result: SubmissionResult + def __init__(self, patch: _Optional[str] = ..., internal_patch_id: _Optional[str] = ..., competition_patch_id: _Optional[str] = ..., build_outputs: _Optional[_Iterable[_Union[BuildOutput, _Mapping]]] = ..., result: _Optional[_Union[SubmissionResult, str]] = ...) -> None: ... class Task(_message.Message): __slots__ = ["cancelled", "deadline", "focus", "message_id", "message_time", "metadata", "project_name", "sources", "task_id", "task_type"] @@ -280,3 +297,6 @@ class WeightedHarness(_message.Message): class BuildType(int, metaclass=_enum_type_wrapper.EnumTypeWrapper): __slots__ = [] + +class SubmissionResult(int, metaclass=_enum_type_wrapper.EnumTypeWrapper): + __slots__ = [] diff --git a/common/src/buttercup/common/sets.py b/common/src/buttercup/common/sets.py index a7803e0a..e7b1f9e4 100644 --- a/common/src/buttercup/common/sets.py +++ b/common/src/buttercup/common/sets.py @@ -5,6 +5,7 @@ from bson.json_util import dumps, CANONICAL_JSON_OPTIONS from contextlib import contextmanager import random import json +from functools import lru_cache # Import POVReproduceRequest for the refactored PoVReproduceStatus from buttercup.common.datastructures.msg_pb2 import POVReproduceRequest, POVReproduceResponse @@ -105,6 +106,28 @@ class PoVReproduceStatus: def __init__(self, redis: Redis): self.redis = redis + @lru_cache(maxsize=1000) + def _did_crash(self, key: str) -> bool | None: + """Check if POV crashed (is in final states) and return crash status. + + Args: + key: The serialized key for the POV reproduction request + + Returns: + False if mitigated (didn't crash), True if non-mitigated (did crash), None if not in final states + """ + pipeline = self.redis.pipeline() + pipeline.sismember(POV_REPRODUCE_MITIGATED_SET_NAME, key) + pipeline.sismember(POV_REPRODUCE_NON_MITIGATED_SET_NAME, key) + result = pipeline.execute() + + if result[0]: + return False # Mitigated - didn't crash + elif result[1]: + return True # Non-mitigated - did crash + else: + return None # Not in final states + def _make_key(self, request: POVReproduceRequest) -> str: """Create a unique key from a POVReproduceRequest by serializing it to string.""" return dumps( @@ -122,11 +145,19 @@ class PoVReproduceStatus: None if pending, POVReproduceResponse if completed """ key = self._make_key(request) + + # First check cache for final states only + did_crash = self._did_crash(key) + if did_crash is not None: + return POVReproduceResponse(request=request, did_crash=did_crash) + + # If not in final states, do the regular logic including pending check pipeline = self.redis.pipeline() pipeline.sismember(POV_REPRODUCE_PENDING_SET_NAME, key) pipeline.sismember(POV_REPRODUCE_MITIGATED_SET_NAME, key) pipeline.sismember(POV_REPRODUCE_NON_MITIGATED_SET_NAME, key) result = pipeline.execute() + if result[0]: return None # Pending elif result[1]: diff --git a/common/src/buttercup/common/util_cli.py b/common/src/buttercup/common/util_cli.py index 44b0b9f5..d9bef27a 100644 --- a/common/src/buttercup/common/util_cli.py +++ b/common/src/buttercup/common/util_cli.py @@ -29,7 +29,8 @@ def truncate_stacktraces(submission: SubmissionEntry, max_length: int = 80) -> S text_format.Parse(submission_text, truncated_submission) # Now truncate the stacktraces and crash token - for crash in truncated_submission.crashes: + for crash_with_id in truncated_submission.crashes: + crash = crash_with_id.crash if crash.crash.stacktrace and len(crash.crash.stacktrace) > max_length: crash.crash.stacktrace = crash.crash.stacktrace[:max_length] + "... (truncated)" diff --git a/common/tests/test_set.py b/common/tests/test_set.py index 70b05066..cfec07ac 100644 --- a/common/tests/test_set.py +++ b/common/tests/test_set.py @@ -2,6 +2,7 @@ import pytest from redis import Redis from buttercup.common.sets import RedisSet, PoVReproduceStatus from buttercup.common.datastructures.msg_pb2 import POVReproduceRequest, POVReproduceResponse +from unittest.mock import MagicMock @pytest.fixture @@ -558,3 +559,339 @@ class TestPoVReproduceStatus: # Should have 2 pending items now (requests[4] and requests[5]) pending1 = pov_status.get_one_pending() assert pending1.task_id in [requests[4].task_id, requests[5].task_id] + + +class TestPoVReproduceStatusCaching: + """Test suite for PoVReproduceStatus caching functionality.""" + + def _create_mock_redis(self): + """Helper to create properly mocked Redis client with pipeline support.""" + mock_redis = MagicMock() + mock_pipeline = MagicMock() + mock_redis.pipeline.return_value = mock_pipeline + mock_pipeline.__enter__.return_value = mock_pipeline + mock_pipeline.__exit__.return_value = None + return mock_redis, mock_pipeline + + def test_cache_hit_for_mitigated_status(self): + """Test that repeated requests for mitigated status hit cache, not Redis.""" + # Create mock Redis client + mock_redis, mock_pipeline = self._create_mock_redis() + + # Mock the final states check to return mitigated + mock_pipeline.execute.return_value = [True, False] # mitigated=True, non_mitigated=False + + pov_status = PoVReproduceStatus(mock_redis) + + # Create sample request + request = POVReproduceRequest() + request.task_id = "test-task" + request.internal_patch_id = "0" + request.pov_path = "/test/path" + request.sanitizer = "asan" + request.harness_name = "test_harness" + + # First call should hit Redis + result1 = pov_status.request_status(request) + assert isinstance(result1, POVReproduceResponse) + assert result1.did_crash is False + + # Second call should hit cache, not Redis + result2 = pov_status.request_status(request) + assert isinstance(result2, POVReproduceResponse) + assert result2.did_crash is False + + # Third call should also hit cache + result3 = pov_status.request_status(request) + assert isinstance(result3, POVReproduceResponse) + assert result3.did_crash is False + + # Redis should only be called once (for the first request) + assert mock_redis.pipeline.call_count == 1 + + def test_cache_hit_for_non_mitigated_status(self): + """Test that repeated requests for non-mitigated status hit cache, not Redis.""" + # Create mock Redis client + mock_redis, mock_pipeline = self._create_mock_redis() + + # Mock the final states check to return non-mitigated + mock_pipeline.execute.return_value = [False, True] # mitigated=False, non_mitigated=True + + pov_status = PoVReproduceStatus(mock_redis) + + # Create sample request + request = POVReproduceRequest() + request.task_id = "test-task" + request.internal_patch_id = "0" + request.pov_path = "/test/path" + request.sanitizer = "asan" + request.harness_name = "test_harness" + + # First call should hit Redis + result1 = pov_status.request_status(request) + assert isinstance(result1, POVReproduceResponse) + assert result1.did_crash is True + + # Second call should hit cache, not Redis + result2 = pov_status.request_status(request) + assert isinstance(result2, POVReproduceResponse) + assert result2.did_crash is True + + # Redis should only be called once (for the first request) + assert mock_redis.pipeline.call_count == 1 + + def test_cache_miss_for_pending_status(self): + """Test that pending status doesn't get cached and always hits Redis.""" + # Create mock Redis client + mock_redis, mock_pipeline = self._create_mock_redis() + mock_redis.sadd.return_value = 1 # Mock successful add to pending set + + # Mock responses: + # First call: _did_crash returns [False, False], then full check returns [True, False, False] + # Second call: _did_crash hits cache (no Redis call), then full check returns [True, False, False] + mock_pipeline.execute.side_effect = [ + [False, False], # First request: _did_crash: not in final states + [ + True, + False, + False, + ], # First request: request_status full check: pending=True, mitigated=False, non_mitigated=False + [True, False, False], # Second request: request_status full check: still pending (cache hit for _did_crash) + ] + + pov_status = PoVReproduceStatus(mock_redis) + + # Create sample request + request = POVReproduceRequest() + request.task_id = "test-task" + request.internal_patch_id = "0" + request.pov_path = "/test/path" + request.sanitizer = "asan" + request.harness_name = "test_harness" + + # First call should return None (pending) + result1 = pov_status.request_status(request) + assert result1 is None + + # Second call should also return None (pending) and hit Redis again for full check + # but _did_crash should hit cache + result2 = pov_status.request_status(request) + assert result2 is None + + # Redis should be called 3 times total: 1 for first _did_crash + 2 for full checks + # (second _did_crash hits cache) + assert mock_redis.pipeline.call_count == 3 + + def test_cache_works_with_different_requests(self): + """Test that cache works correctly for different requests.""" + # Create mock Redis client + mock_redis, mock_pipeline = self._create_mock_redis() + + # Mock different responses for different requests + mock_pipeline.execute.side_effect = [ + [True, False], # request1: mitigated + [False, True], # request2: non-mitigated + # Subsequent calls should hit cache + ] + + pov_status = PoVReproduceStatus(mock_redis) + + # Create two different requests + request1 = POVReproduceRequest() + request1.task_id = "test-task-1" + request1.internal_patch_id = "0" + request1.pov_path = "/test/path1" + request1.sanitizer = "asan" + request1.harness_name = "test_harness" + + request2 = POVReproduceRequest() + request2.task_id = "test-task-2" + request2.internal_patch_id = "0" + request2.pov_path = "/test/path2" + request2.sanitizer = "msan" + request2.harness_name = "test_harness" + + # First calls should hit Redis + result1 = pov_status.request_status(request1) + assert isinstance(result1, POVReproduceResponse) + assert result1.did_crash is False # mitigated + + result2 = pov_status.request_status(request2) + assert isinstance(result2, POVReproduceResponse) + assert result2.did_crash is True # non-mitigated + + # Repeat calls should hit cache + result1_cached = pov_status.request_status(request1) + assert isinstance(result1_cached, POVReproduceResponse) + assert result1_cached.did_crash is False + + result2_cached = pov_status.request_status(request2) + assert isinstance(result2_cached, POVReproduceResponse) + assert result2_cached.did_crash is True + + # Redis should only be called twice (once for each unique request) + assert mock_redis.pipeline.call_count == 2 + + def test_cache_respects_maxsize(self): + """Test that cache behavior is consistent (simplified test without maxsize override).""" + # Create mock Redis client + mock_redis, mock_pipeline = self._create_mock_redis() + + # Mock all responses as mitigated + mock_pipeline.execute.return_value = [True, False] # mitigated=True, non_mitigated=False + + pov_status = PoVReproduceStatus(mock_redis) + + # Create multiple different requests + requests = [] + for i in range(3): + request = POVReproduceRequest() + request.task_id = f"test-task-{i}" + request.internal_patch_id = "0" + request.pov_path = f"/test/path{i}" + request.sanitizer = "asan" + request.harness_name = "test_harness" + requests.append(request) + + # First requests should hit Redis + result1 = pov_status.request_status(requests[0]) + result2 = pov_status.request_status(requests[1]) + result3 = pov_status.request_status(requests[2]) + assert isinstance(result1, POVReproduceResponse) + assert isinstance(result2, POVReproduceResponse) + assert isinstance(result3, POVReproduceResponse) + assert mock_redis.pipeline.call_count == 3 + + # Repeated requests should hit cache + result1_cached = pov_status.request_status(requests[0]) + result2_cached = pov_status.request_status(requests[1]) + result3_cached = pov_status.request_status(requests[2]) + assert isinstance(result1_cached, POVReproduceResponse) + assert isinstance(result2_cached, POVReproduceResponse) + assert isinstance(result3_cached, POVReproduceResponse) + # Should not have increased - all cache hits + assert mock_redis.pipeline.call_count == 3 + + def test_separate_instances_have_separate_caches(self): + """Test that different PoVReproduceStatus instances have separate caches.""" + # Create two mock Redis clients + mock_redis1 = MagicMock() + mock_pipeline1 = MagicMock() + mock_redis1.pipeline.return_value = mock_pipeline1 + mock_pipeline1.__enter__.return_value = mock_pipeline1 + mock_pipeline1.__exit__.return_value = None + mock_pipeline1.execute.return_value = [True, False] # mitigated + + mock_redis2 = MagicMock() + mock_pipeline2 = MagicMock() + mock_redis2.pipeline.return_value = mock_pipeline2 + mock_pipeline2.__enter__.return_value = mock_pipeline2 + mock_pipeline2.__exit__.return_value = None + mock_pipeline2.execute.return_value = [False, True] # non-mitigated + + # Create two instances + pov_status1 = PoVReproduceStatus(mock_redis1) + pov_status2 = PoVReproduceStatus(mock_redis2) + + # Create same request for both + request = POVReproduceRequest() + request.task_id = "test-task" + request.internal_patch_id = "0" + request.pov_path = "/test/path" + request.sanitizer = "asan" + request.harness_name = "test_harness" + + # First calls should hit respective Redis instances + result1 = pov_status1.request_status(request) + result2 = pov_status2.request_status(request) + + assert isinstance(result1, POVReproduceResponse) + assert result1.did_crash is False # mitigated + assert isinstance(result2, POVReproduceResponse) + assert result2.did_crash is True # non-mitigated + + # Each instance should have hit its Redis once + assert mock_redis1.pipeline.call_count == 1 + assert mock_redis2.pipeline.call_count == 1 + + # Repeat calls should hit respective caches + result1_cached = pov_status1.request_status(request) + result2_cached = pov_status2.request_status(request) + + assert isinstance(result1_cached, POVReproduceResponse) + assert result1_cached.did_crash is False + assert isinstance(result2_cached, POVReproduceResponse) + assert result2_cached.did_crash is True + + # Redis call counts should not increase (cache hits) + assert mock_redis1.pipeline.call_count == 1 + assert mock_redis2.pipeline.call_count == 1 + + def test_cache_integration_with_real_workflow(self, pov_status, sample_request): + """Test that caching works correctly in a real workflow with Redis.""" + # This test uses real Redis and validates the cache behavior + + # Initial request should return None (pending) + result1 = pov_status.request_status(sample_request) + assert result1 is None + + # Mark as mitigated + mark_result = pov_status.mark_mitigated(sample_request) + assert mark_result is True + + # Now test caching: multiple requests should return same result + # We can't easily test Redis call count with real Redis, but we can + # verify that the behavior is consistent + results = [] + for _ in range(5): + result = pov_status.request_status(sample_request) + results.append(result) + + # All results should be identical POVReproduceResponse objects + for result in results: + assert isinstance(result, POVReproduceResponse) + assert result.did_crash is False # mitigated + assert result.request.task_id == sample_request.task_id + assert result.request.internal_patch_id == sample_request.internal_patch_id + assert result.request.pov_path == sample_request.pov_path + assert result.request.sanitizer == sample_request.sanitizer + assert result.request.harness_name == sample_request.harness_name + + def test_cache_clears_across_different_final_states(self): + """Test that moving between final states works correctly with caching.""" + # Create mock Redis client + mock_redis, mock_pipeline = self._create_mock_redis() + + pov_status = PoVReproduceStatus(mock_redis) + + # Create sample request + request = POVReproduceRequest() + request.task_id = "test-task" + request.internal_patch_id = "0" + request.pov_path = "/test/path" + request.sanitizer = "asan" + request.harness_name = "test_harness" + + # First: mock as mitigated + mock_pipeline.execute.return_value = [True, False] + result1 = pov_status.request_status(request) + assert isinstance(result1, POVReproduceResponse) + assert result1.did_crash is False + + # Second call should hit cache + result2 = pov_status.request_status(request) + assert isinstance(result2, POVReproduceResponse) + assert result2.did_crash is False + assert mock_redis.pipeline.call_count == 1 # Only first call hit Redis + + # Now simulate the state changing in Redis (e.g., due to another process) + # This would require cache invalidation in a real system, but our current + # implementation doesn't handle this case - the cache will return stale data + # This test documents the current behavior + mock_pipeline.execute.return_value = [False, True] # Now non-mitigated + + # This call will still return cached (stale) mitigated result + result3 = pov_status.request_status(request) + assert isinstance(result3, POVReproduceResponse) + assert result3.did_crash is False # Still cached mitigated result + assert mock_redis.pipeline.call_count == 1 # No new Redis calls diff --git a/orchestrator/src/buttercup/orchestrator/scheduler/scheduler.py b/orchestrator/src/buttercup/orchestrator/scheduler/scheduler.py index 96ccb17a..f1a9da30 100644 --- a/orchestrator/src/buttercup/orchestrator/scheduler/scheduler.py +++ b/orchestrator/src/buttercup/orchestrator/scheduler/scheduler.py @@ -378,7 +378,7 @@ class Scheduler: vuln_item: RQItem[TracedCrash] | None = self.traced_vulnerabilities_queue.pop() if vuln_item is not None: crash: TracedCrash = vuln_item.deserialized - logger.info(f"Submitting vulnerability for task {crash.crash.target.task_id}") + logger.info(f"Recording vulnerability for task {crash.crash.target.task_id}") if self.submissions.submit_vulnerability(crash): self.traced_vulnerabilities_queue.ack_item(vuln_item.item_id) collected_item = True diff --git a/orchestrator/src/buttercup/orchestrator/scheduler/submissions.py b/orchestrator/src/buttercup/orchestrator/scheduler/submissions.py index 5e8ed160..c6352172 100644 --- a/orchestrator/src/buttercup/orchestrator/scheduler/submissions.py +++ b/orchestrator/src/buttercup/orchestrator/scheduler/submissions.py @@ -4,7 +4,7 @@ import logging import base64 import uuid from redis import Redis -from typing import Iterator, List, Set, Tuple +from typing import Callable, Iterator, List, Set, Tuple from opentelemetry import trace from opentelemetry.trace import Status, StatusCode import buttercup.common.node_local as node_local @@ -15,13 +15,16 @@ from buttercup.common.datastructures.msg_pb2 import ( TracedCrash, ConfirmedVulnerability, SubmissionEntry, - Patch, SubmissionEntryPatch, BuildRequest, BuildType, BuildOutput, POVReproduceRequest, POVReproduceResponse, + CrashWithId, + Bundle, + SubmissionResult, + Patch, ) from buttercup.common.sarif_store import SARIFStore from buttercup.common.task_registry import TaskRegistry @@ -47,12 +50,25 @@ from buttercup.common.clusterfuzz_parser.crash_comparer import CrashComparer logger = logging.getLogger(__name__) +def _map_submission_status_to_result(status: TypesSubmissionStatus) -> SubmissionResult: + """Map TypesSubmissionStatus to SubmissionResult enum.""" + mapping = { + TypesSubmissionStatus.SubmissionStatusAccepted: SubmissionResult.ACCEPTED, + TypesSubmissionStatus.SubmissionStatusPassed: SubmissionResult.PASSED, + TypesSubmissionStatus.SubmissionStatusFailed: SubmissionResult.FAILED, + TypesSubmissionStatus.SubmissionStatusDeadlineExceeded: SubmissionResult.DEADLINE_EXCEEDED, + TypesSubmissionStatus.SubmissionStatusErrored: SubmissionResult.ERRORED, + TypesSubmissionStatus.SubmissionStatusInconclusive: SubmissionResult.INCONCLUSIVE, + } + return mapping.get(status, SubmissionResult.ERRORED) # Default to ERRORED for unknown statuses + + def _task_id(e: SubmissionEntry | TracedCrash) -> str: """Get the task_id from the SubmissionEntry or TracedCrash.""" if isinstance(e, TracedCrash): return e.crash.target.task_id elif isinstance(e, SubmissionEntry): - return e.crashes[0].crash.target.task_id + return e.crashes[0].crash.crash.target.task_id else: raise ValueError(f"Unknown submission entry type: {type(e)}") @@ -61,32 +77,46 @@ def log_entry( e: SubmissionEntry, msg: str = "", i: int | None = None, - old_state: int | None = None, - fn: logging.Logger = logger.info, -): + fn: Callable[[str], None] = logger.info, +) -> None: """Log a structured message for easy grepping and filtering.""" - task_id = e.crashes[0].crash.target.task_id + task_id = _task_id(e) idx_msg = f"{i}:" if i is not None else "" - log_msg = f"[{idx_msg}:{task_id}]" + log_msg = f"[{idx_msg}{task_id}]" - curr_state = SubmissionEntry.SubmissionState.Name(e.state) - if old_state: - old_state_name = SubmissionEntry.SubmissionState.Name(old_state) - log_msg += f" {old_state_name} -> {curr_state}" - else: - log_msg += f" {curr_state}" + def _truncate_join(items: list[str], max_length: int = 256) -> str: + """Join list items with commas, truncating if the result exceeds max_length.""" + joined = ",".join(items) + if len(joined) <= max_length: + return joined + # Truncate and add ellipsis + return joined[: max_length - 3] + "..." + + competition_pov_ids = [c.competition_pov_id for c in e.crashes if c.competition_pov_id] + if competition_pov_ids: + log_msg += f" pov_id={_truncate_join(competition_pov_ids)}" + + if len(e.patches) > 0: + log_msg += f" patches={len(e.patches)}" - if e.pov_id: - log_msg += f" pov_id={e.pov_id}" - if e.competition_patch_id: - log_msg += f" competition_patch_id={e.competition_patch_id}" - if e.bundle_id: - log_msg += f" bundle_id={e.bundle_id}" if e.patch_idx: log_msg += f" patch_idx={e.patch_idx}" - if e.sarif_id: - log_msg += f" sarif_id={e.sarif_id}" + + if e.patch_submission_attempts: + log_msg += f" patch_submission_attempts={e.patch_submission_attempts}" + + competition_patch_ids = [p.competition_patch_id for p in e.patches if p.competition_patch_id] + if competition_patch_ids: + log_msg += f" competition_patch_id={_truncate_join(competition_patch_ids)}" + + competition_bundle_ids = [b.bundle_id for b in e.bundles if b.bundle_id] + if competition_bundle_ids: + log_msg += f" bundle_id={_truncate_join(competition_bundle_ids)}" + + sarif_ids = [b.competition_sarif_id for b in e.bundles if b.competition_sarif_id] + if sarif_ids: + log_msg += f" sarif_id={_truncate_join(sarif_ids)}" if msg: log_msg += f" {msg}" @@ -94,15 +124,97 @@ def log_entry( fn(log_msg) -def _have_more_patches(e: SubmissionEntry) -> bool: - """Check if there are more patches to try (following the current patch if any).""" - return e.patch_idx + 1 < len(e.patches) - - def _advance_patch_idx(e: SubmissionEntry) -> None: """Advance the patch index to the next patch.""" e.patch_idx += 1 - e.patch_submission_attempt = 0 + e.patch_submission_attempts = 0 + + +def _increase_submission_attempts(e: SubmissionEntry) -> None: + """Increase the submission attempts for the current patch.""" + e.patch_submission_attempts += 1 + + +def _current_patch(e: SubmissionEntry) -> SubmissionEntryPatch | None: + """Get the current patch.""" + if not e.patches: + return None + if e.patch_idx >= len(e.patches): + return None + return e.patches[e.patch_idx] + + +def _get_pending_patch_submissions(e: SubmissionEntry) -> list[SubmissionEntryPatch]: + """Get all pending patch submissions from the submission entry. + It is considered pending if it has a competition_patch_id and is in the ACCEPTED state. + """ + return [patch for patch in e.patches if patch.competition_patch_id and patch.result == SubmissionResult.ACCEPTED] + + +def _get_first_successful_pov(e: SubmissionEntry) -> CrashWithId | None: + """Get the first successful POV from the submission entry. + + Returns None if no successful POV is found. + """ + return next( + (crash for crash in e.crashes if crash.competition_pov_id and crash.result == SubmissionResult.PASSED), + None, + ) + + +def _get_pending_pov_submissions(e: SubmissionEntry) -> list[CrashWithId]: + """Get all pending POVs from the submission entry. + It is considered pending if the POV is accepted but not yet passed. + + Returns None if no pending POV is found. + """ + return [crash for crash in e.crashes if crash.competition_pov_id and crash.result == SubmissionResult.ACCEPTED] + + +def _get_first_successful_pov_id(e: SubmissionEntry) -> str | None: + """Get the first successful POV ID from the submission entry. + + Returns None if no successful POV is found. + """ + pov = _get_first_successful_pov(e) + if pov: + return pov.competition_pov_id + return None + + +def _get_eligible_povs_for_submission(e: SubmissionEntry) -> list[CrashWithId]: + """Get all POVs that are eligible for submission. + + A POV is eligible for submission if: + - It doesn't have a competition_pov_id, or + - It has a competition_pov_id but is in ERRORED state (can be retried) + + Returns: + List of CrashWithId objects that are eligible for submission + """ + return [ + crash + for crash in e.crashes + if not crash.competition_pov_id or (crash.competition_pov_id and crash.result == SubmissionResult.ERRORED) + ] + + +def _find_matching_build_output(patch: SubmissionEntryPatch, build_output: BuildOutput) -> BuildOutput | None: + """Find the matching build output in the patch.""" + # Found the patch, now locate the placeholder for the build output + return next( + ( + bo + for bo in patch.build_outputs + if ( + bo.engine == build_output.engine + and bo.sanitizer == build_output.sanitizer + and bo.build_type == build_output.build_type + and bo.apply_diff == build_output.apply_diff + ) + ), + None, + ) class CompetitionAPI: @@ -136,7 +248,7 @@ class CompetitionAPI: """ return dict(self.task_registry.get(task_id).metadata) - def submit_pov(self, crash: TracedCrash) -> Tuple[str | None, TypesSubmissionStatus]: + def submit_pov(self, crash: TracedCrash) -> Tuple[str | None, SubmissionResult]: """ Submit a vulnerability (POV) to the competition API. @@ -146,7 +258,7 @@ class CompetitionAPI: crash: TracedCrash with crash details and metadata Returns: - Tuple[str | None, TypesSubmissionStatus]: + Tuple[str | None, SubmissionResult]: - POV ID if successful, None otherwise - Submission status """ @@ -177,29 +289,32 @@ class CompetitionAPI: }, ) + logger.debug(f"[{crash.crash.target.task_id}] Submitting POV for harness: {crash.crash.harness_name}") + # Submit Pov and get response response = PovApi(api_client=self.api_client).v1_task_task_id_pov_post( task_id=crash.crash.target.task_id, payload=submission, ) logger.debug(f"[{crash.crash.target.task_id}] POV submission response: {response}") - if response.status not in [ - TypesSubmissionStatus.SubmissionStatusAccepted, - TypesSubmissionStatus.SubmissionStatusPassed, + mapped_status = _map_submission_status_to_result(response.status) + if mapped_status not in [ + SubmissionResult.ACCEPTED, + SubmissionResult.PASSED, ]: logger.error( f"[{crash.crash.target.task_id}] POV submission rejected (status: {response.status}) for harness: {crash.crash.harness_name}" ) span.set_status(Status(StatusCode.ERROR)) - return None, response.status + return None, mapped_status span.set_status(Status(StatusCode.OK)) - return response.pov_id, response.status + return response.pov_id, mapped_status except Exception as e: logger.error(f"[{crash.crash.target.task_id}] Failed to submit vulnerability: {e}") - return None, TypesSubmissionStatus.SubmissionStatusErrored + return None, SubmissionResult.ERRORED - def get_pov_status(self, task_id: str, pov_id: str) -> TypesSubmissionStatus: + def get_pov_status(self, task_id: str, pov_id: str) -> SubmissionResult: """ Get vulnerability submission status. @@ -208,14 +323,15 @@ class CompetitionAPI: pov_id: POV ID from submit_vulnerability Returns: - TypesSubmissionStatus: Current status (SubmissionStatusAccepted, SubmissionStatusPassed, SubmissionStatusFailed, SubmissionStatusErrored, SubmissionStatusDeadlineExceeded) + SubmissionResult: Current status (ACCEPTED, PASSED, FAILED, ERRORED, DEADLINE_EXCEEDED) """ assert task_id assert pov_id - return PovApi(api_client=self.api_client).v1_task_task_id_pov_pov_id_get(task_id=task_id, pov_id=pov_id).status + response = PovApi(api_client=self.api_client).v1_task_task_id_pov_pov_id_get(task_id=task_id, pov_id=pov_id) + return _map_submission_status_to_result(response.status) - def submit_patch(self, task_id: str, patch: str) -> Tuple[str | None, TypesSubmissionStatus]: + def submit_patch(self, task_id: str, patch: str) -> Tuple[str | None, SubmissionResult]: """ Submit a patch to the competition API. @@ -224,7 +340,7 @@ class CompetitionAPI: patch: Patch content string Returns: - Tuple[str | None, TypesSubmissionStatus]: + Tuple[str | None, SubmissionResult]: - Patch ID if accepted, None otherwise - Submission status """ @@ -246,22 +362,25 @@ class CompetitionAPI: task_metadata=self._get_task_metadata(task_id), ) + logger.debug(f"[{task_id}] Submitting patch") + response = PatchApi(api_client=self.api_client).v1_task_task_id_patch_post( task_id=task_id, payload=submission ) logger.debug(f"[{task_id}] Patch submission response: {response}") - if response.status not in [ - TypesSubmissionStatus.SubmissionStatusAccepted, - TypesSubmissionStatus.SubmissionStatusPassed, + mapped_status = _map_submission_status_to_result(response.status) + if mapped_status not in [ + SubmissionResult.ACCEPTED, + SubmissionResult.PASSED, ]: logger.error(f"[{task_id}] Patch submission rejected (status: {response.status}) for harness: {patch}") span.set_status(Status(StatusCode.ERROR)) - return (None, response.status) + return (None, mapped_status) span.set_status(Status(StatusCode.OK)) - return (response.patch_id, response.status) + return (response.patch_id, mapped_status) - def get_patch_status(self, task_id: str, patch_id: str) -> TypesSubmissionStatus: + def get_patch_status(self, task_id: str, patch_id: str) -> SubmissionResult: """ Get patch submission status. @@ -270,7 +389,7 @@ class CompetitionAPI: patch_id: Patch ID from submit_patch Returns: - TypesSubmissionStatus: Current status (SubmissionStatusAccepted, SubmissionStatusPassed, SubmissionStatusFailed, SubmissionStatusErrored, SubmissionStatusDeadlineExceeded) + SubmissionResult: Current status (ACCEPTED, PASSED, FAILED, ERRORED, DEADLINE_EXCEEDED) """ assert task_id assert patch_id @@ -282,9 +401,11 @@ class CompetitionAPI: logger.info( f"[{task_id}] Patch {patch_id} functionality tests passing: {response.functionality_tests_passing}" ) - return response.status + return _map_submission_status_to_result(response.status) - def submit_bundle(self, task_id: str, pov_id: str, patch_id: str) -> Tuple[str | None, TypesSubmissionStatus]: + def submit_bundle( + self, task_id: str, pov_id: str, patch_id: str, sarif_id: str + ) -> Tuple[str | None, SubmissionResult]: """ Submit a bundle (vulnerability + patch). @@ -294,18 +415,20 @@ class CompetitionAPI: patch_id: Patch ID of verified patch Returns: - Tuple[str | None, TypesSubmissionStatus]: + Tuple[str | None, SubmissionResult]: - Bundle ID if successful, None otherwise - Submission status """ assert task_id assert pov_id - assert patch_id submission = TypesBundleSubmission( pov_id=pov_id, - patch_id=patch_id, ) + if sarif_id: + submission.broadcast_sarif_id = sarif_id + if patch_id: + submission.patch_id = patch_id # Telemetry tracer = trace.get_tracer(__name__) @@ -317,26 +440,27 @@ class CompetitionAPI: task_metadata=self._get_task_metadata(task_id), ) + logger.debug(f"[{task_id}] Submitting bundle for harness: {pov_id} {patch_id} {sarif_id}") + response = BundleApi(api_client=self.api_client).v1_task_task_id_bundle_post( task_id=task_id, payload=submission ) logger.debug(f"[{task_id}] Bundle submission response: {response}") - if response.status not in [ - TypesSubmissionStatus.SubmissionStatusAccepted, - TypesSubmissionStatus.SubmissionStatusPassed, + mapped_status = _map_submission_status_to_result(response.status) + if mapped_status not in [ + SubmissionResult.ACCEPTED, + SubmissionResult.PASSED, ]: - logger.error( - f"[{task_id}] Bundle submission rejected (status: {response.status}) for harness: {pov_id} {patch_id}" - ) + logger.error(f"[{task_id}] Bundle submission rejected (status: {response.status})") span.set_status(Status(StatusCode.ERROR)) - return (None, response.status) + return (None, mapped_status) span.set_status(Status(StatusCode.OK)) - return (response.bundle_id, response.status) + return (response.bundle_id, mapped_status) def patch_bundle( self, task_id: str, bundle_id: str, pov_id: str, patch_id: str, sarif_id: str - ) -> Tuple[bool, TypesSubmissionStatus]: + ) -> Tuple[bool, SubmissionResult]: """ Submit a bundle patch with SARIF association. @@ -348,7 +472,7 @@ class CompetitionAPI: sarif_id: SARIF ID to associate with the bundle Returns: - Tuple[bool, TypesSubmissionStatus]: + Tuple[bool, SubmissionResult]: - True if successful, False otherwise - Submission status """ @@ -368,17 +492,44 @@ class CompetitionAPI: task_id=task_id, bundle_id=bundle_id, payload=submission ) logger.debug(f"[{task_id}] Bundle patch submission response: {response}") - if response.status not in [ - TypesSubmissionStatus.SubmissionStatusAccepted, - TypesSubmissionStatus.SubmissionStatusPassed, + mapped_status = _map_submission_status_to_result(response.status) + if mapped_status not in [ + SubmissionResult.ACCEPTED, + SubmissionResult.PASSED, ]: logger.error( f"[{task_id}] Bundle patch submission rejected (status: {response.status}) for harness: {pov_id} {patch_id} {sarif_id}" ) - return (False, response.status) - return (True, response.status) + return (False, mapped_status) + return (True, mapped_status) - def submit_matching_sarif(self, task_id: str, sarif_id: str) -> Tuple[bool, TypesSubmissionStatus]: + def delete_bundle(self, task_id: str, bundle_id: str) -> bool: + """ + Delete a bundle by bundle_id. + + Args: + task_id: Task ID for the submission + bundle_id: Bundle ID to delete + + Returns: + bool: True if successful, False otherwise + """ + assert task_id + assert bundle_id + + try: + logger.debug(f"[{task_id}] Deleting bundle: {bundle_id}") + + response = BundleApi(api_client=self.api_client).v1_task_task_id_bundle_bundle_id_delete( + task_id=task_id, bundle_id=bundle_id + ) + logger.debug(f"[{task_id}] Bundle deletion response: {response}") + return True + except Exception as e: + logger.error(f"[{task_id}] Bundle deletion failed for bundle_id: {bundle_id}, error: {e}") + return False + + def submit_matching_sarif(self, task_id: str, sarif_id: str) -> Tuple[bool, SubmissionResult]: """ Submit a matching assessment for a SARIF report. @@ -389,7 +540,7 @@ class CompetitionAPI: sarif_id: ID of the broadcast SARIF report to assess Returns: - Tuple[bool, TypesSubmissionStatus]: + Tuple[bool, SubmissionResult]: - True if successful, False otherwise - Submission status """ @@ -418,18 +569,19 @@ class CompetitionAPI: task_id=task_id, broadcast_sarif_id=sarif_id, payload=submission ) logger.debug(f"[{task_id}] Matching SARIF submission response: {response}") - if response.status not in [ - TypesSubmissionStatus.SubmissionStatusAccepted, - TypesSubmissionStatus.SubmissionStatusPassed, + mapped_status = _map_submission_status_to_result(response.status) + if mapped_status not in [ + SubmissionResult.ACCEPTED, + SubmissionResult.PASSED, ]: logger.error( f"[{task_id}] Matching SARIF submission rejected (status: {response.status}) for sarif_id: {sarif_id}" ) span.set_status(Status(StatusCode.ERROR)) - return (False, response.status) + return (False, mapped_status) span.set_status(Status(StatusCode.OK)) - return (True, response.status) + return (True, mapped_status) @dataclass @@ -533,7 +685,7 @@ class Submissions: build_requests_queue: ReliableQueue[BuildRequest] = field(init=False) pov_reproduce_status: PoVReproduceStatus = field(init=False) - def __post_init__(self): + def __post_init__(self) -> None: logger.info( f"Initializing Submissions, patch_submission_retry_limit={self.patch_submission_retry_limit}, patch_requests_per_vulnerability={self.patch_requests_per_vulnerability}" ) @@ -544,7 +696,7 @@ class Submissions: self.build_requests_queue = queue_factory.create(QueueNames.BUILD, block_time=None) self.pov_reproduce_status = PoVReproduceStatus(self.redis) - def _insert_matched_sarif(self, redis: Redis, sarif_id: str): + def _insert_matched_sarif(self, redis: Redis, sarif_id: str) -> None: """Insert a matched SARIF ID into Redis.""" self.matched_sarifs.add(sarif_id) redis.sadd(self.MATCHED_SARIFS, sarif_id) @@ -553,37 +705,174 @@ class Submissions: """Get all matched SARIF IDs from Redis.""" return set(redis.smembers(self.MATCHED_SARIFS)) - def _get_sarif_candidates(self, task_id: str) -> List[TracedCrash]: - """Get all SARIFs for a task that are not matched to a vulnerability.""" - return [ - sarif for sarif in self.sarif_store.get_by_task_id(task_id) if sarif.sarif_id not in self.matched_sarifs - ] - def _get_stored_submissions(self) -> List[SubmissionEntry]: """Get all stored submissions from Redis.""" return [SubmissionEntry.FromString(raw) for raw in self.redis.lrange(self.SUBMISSIONS, 0, -1)] - def _persist(self, redis: Redis, index: int, entry: SubmissionEntry): + def _persist(self, redis: Redis, index: int, entry: SubmissionEntry) -> None: """Persist the submissions to Redis.""" redis.lset(self.SUBMISSIONS, index, entry.SerializeToString()) - def _push(self, entry: SubmissionEntry): + def _push(self, entry: SubmissionEntry) -> int: """Push a submission to Redis.""" return self.redis.rpush(self.SUBMISSIONS, entry.SerializeToString()) - def _enumerate_submissions(self) -> Iterator[SubmissionEntry]: + def _enumerate_submissions(self) -> Iterator[tuple[int, SubmissionEntry]]: """Enumerate all submissions belonging to active tasks.""" for i, e in enumerate(self.entries): + if e.stop: + continue if self.task_registry.should_stop_processing(_task_id(e)): continue yield i, e - def _stop(self, i: int, e: SubmissionEntry, msg: str): - """Stop a SubmissionEntry from further processing.""" - old_state = e.state - e.state = SubmissionEntry.STOP - self._persist(self.redis, i, e) - log_entry(e, msg, i, old_state=old_state) + def _enumerate_task_submissions(self, task_id: str) -> Iterator[tuple[int, SubmissionEntry]]: + """Enumerate all submissions belonging to a specific task (if it is active).""" + for i, e in self._enumerate_submissions(): + if _task_id(e) != task_id: + continue + yield i, e + + def _find_patch(self, internal_patch_id: str) -> Tuple[int, SubmissionEntry, SubmissionEntryPatch] | None: + """Find a patch by its internal patch id.""" + for i, e in self._enumerate_submissions(): + for patch in e.patches: + if patch.internal_patch_id == internal_patch_id: + return i, e, patch + return None + + def find_similar_entries(self, crash: TracedCrash) -> list[tuple[int, SubmissionEntry]]: + """ + Find existing submissions that have crashes similar to the given crash. + + Args: + crash: The traced crash to compare against existing submissions + + Returns: + List of (index, SubmissionEntry) tuples for similar submissions + """ + crash_data = get_crash_data(crash.crash.stacktrace) + inst_key = get_inst_key(crash.crash.stacktrace) + task_id = _task_id(crash) + + similar_entries = [] + for i, e in self._enumerate_task_submissions(task_id): + for existing_crash_with_id in e.crashes: + submission_crash_data = get_crash_data(existing_crash_with_id.crash.crash.stacktrace) + submission_inst_key = get_inst_key(existing_crash_with_id.crash.crash.stacktrace) + + cf_comparator = CrashComparer(crash_data, submission_crash_data) + instkey_comparator = CrashComparer(inst_key, submission_inst_key) + + if cf_comparator.is_similar() or instkey_comparator.is_similar(): + log_entry( + e, + f"Incoming PoV crash_data: {crash_data}, inst_key: {inst_key}, existing crash_data: {submission_crash_data}, existing inst_key: {submission_inst_key} are duplicates. ", + i, + fn=logger.debug, + ) + + similar_entries.append((i, e)) + # No need to check other crashes in this submission + break + + return similar_entries + + def _add_to_similar_submission(self, crash: TracedCrash) -> bool: + """ + Check if the crash is similar to an existing submission and add it if so. + + This method performs deduplication by comparing the crash data and instruction keys + of the new crash against existing submissions for the same task. If the crash is + similar to multiple submissions, it consolidates them by merging all data into + the first similar submission and stopping the others. + + Args: + crash: The traced crash to check for similarity + + Returns: + True if the crash was added to an existing submission (duplicate found), + False if no similar submission was found + """ + similar_entries = self.find_similar_entries(crash) + + if len(similar_entries) == 0: + # Unique PoV + return False + else: + # Similar submissions found - consolidate them (handles both single and multiple cases) + return self._consolidate_similar_submissions(crash, similar_entries) + + def _consolidate_similar_submissions( + self, crash: TracedCrash | None, similar_entries: list[tuple[int, SubmissionEntry]] + ) -> bool: + """ + Consolidate multiple similar submissions into the first one. + + Args: + crash: The new crash that triggered the consolidation (if any) + similar_entries: List of (index, SubmissionEntry) tuples for similar submissions + + Returns: + True indicating the crash was handled (consolidated) + """ + # Use the first similar submission as the target + target_index, target_entry = similar_entries[0] + + if crash is not None: + # Add the new crash to the target + crash_with_id = CrashWithId() + crash_with_id.crash.CopyFrom(crash) + target_entry.crashes.append(crash_with_id) + + log_entry( + target_entry, + i=target_index, + msg=f"Consolidating {len(similar_entries)} similar submissions into this one. Adding new crash." + if crash is not None + else f"Consolidating {len(similar_entries)} similar submissions into this one.", + ) + + # Use a Redis pipeline to ensure all operations are persisted atomically + pipeline = self.redis.pipeline() + + # Merge all other similar submissions into the target + for source_index, source_entry in similar_entries[1:]: + log_entry(source_entry, i=source_index, msg=f"Merging submission into target at index {target_index}") + + # Copy all crashes from source to target + target_entry.crashes.extend(source_entry.crashes) + + # Copy non-discarded patches from source (starting from source's current patch_idx) + if source_entry.patch_idx < len(source_entry.patches): + target_entry.patches.extend(source_entry.patches[source_entry.patch_idx :]) + + # Copy bundles from source to target + target_entry.bundles.extend(source_entry.bundles) + + # Stop the source submission and add to pipeline + source_entry.stop = True + self._persist(pipeline, source_index, source_entry) + + log_entry( + source_entry, + i=source_index, + msg=f"Submission consolidated and stopped. Total crashes in target: {len(target_entry.crashes)}, total patches: {len(target_entry.patches)}", + ) + + # Add the updated target entry to pipeline + self._persist(pipeline, target_index, target_entry) + + # Execute all operations atomically + pipeline.execute() + + log_entry( + target_entry, + i=target_index, + msg=f"Consolidation complete. Final submission has {len(target_entry.crashes)} crashes and {len(target_entry.patches)} patches.", + ) + + return True def submit_vulnerability(self, crash: TracedCrash) -> bool: """ @@ -605,72 +894,14 @@ class Submissions: logger.debug(f"CrashInfo: {crash}") return True - crash_data = get_crash_data(crash.crash.stacktrace) - inst_key = get_inst_key(crash.crash.stacktrace) - task_id = _task_id(crash) - # Check if the crash is a variant of an existing submission - similar_submissions = 0 - for i, e in self._enumerate_submissions(): - # Only check submissions for the same task - if _task_id(e) != task_id: - continue - - for existing_crash in e.crashes: - submission_crash_data = get_crash_data(existing_crash.crash.stacktrace) - submission_inst_key = get_inst_key(existing_crash.crash.stacktrace) - - cf_comparator = CrashComparer(crash_data, submission_crash_data) - instkey_comparator = CrashComparer(inst_key, submission_inst_key) - - if cf_comparator.is_similar() or instkey_comparator.is_similar(): - log_entry( - e, - f"Incoming PoV crash_data: {crash_data}, inst_key: {inst_key}, existing crash_data: {submission_crash_data}, existing inst_key: {submission_inst_key} are duplicates. ", - i, - fn=logger.debug, - ) - - e.crashes.append(crash) - log_entry( - e, - f"Adding a duplicate PoV based on similarity check. (n={len(e.crashes)})", - i, - fn=logger.info, - ) - - self._persist(self.redis, i, e) - - similar_submissions += 1 - # No need to check other crashes in this submission - break - - if similar_submissions > 1: - logger.error( - f"Found {similar_submissions} similar submissions. This is a indication that our deduplication logic isn't good enough." - ) - - if similar_submissions > 0: - # No need to submit the crash, we already have a similar submission + if self._add_to_similar_submission(crash): return True - pov_id, status = self.competition_api.submit_pov(crash) - if not pov_id: - logger.error(f"[{_task_id(crash)}] Failed to submit vulnerability. Competition API returned {status}.") - logger.debug(f"CrashInfo: {crash}") - - # If the API returned ERRORED, we want to retry. - stop = status != TypesSubmissionStatus.SubmissionStatusErrored - - if stop: - logger.info(f"Competition API returned {status}, will not retry.") - logger.debug(f"CrashInfo: {crash}") - return stop - e = SubmissionEntry() - e.crashes.append(crash) - e.pov_id = pov_id - e.state = SubmissionEntry.SUBMIT_PATCH_REQUEST + crash_with_id = CrashWithId() + crash_with_id.crash.CopyFrom(crash) + e.crashes.append(crash_with_id) # Persist to Redis length = self._push(e) @@ -681,68 +912,517 @@ class Submissions: # Keep the entries list in sync self.entries.append(e) - log_entry(e, i=index) + log_entry(e, i=index, msg="Recorded unique PoV") return True - def _request_patched_builds(self, task_id: str, patch: SubmissionEntryPatch) -> None: - """Request patched builds for a patch""" - task = ChallengeTask(read_only_task_dir=self.tasks_storage_dir / task_id) + def _submit_pov_if_needed(self, i: int, e: SubmissionEntry, _redis: Redis) -> bool: + """ + If the `SubmissionEntry` have no PoV submitted, submit the first one. + Returns True if the entry needs to be persisted, False otherwise. + """ + if _get_first_successful_pov(e): + # We already have a successful POV, we don't need to submit more. + return False + if _get_pending_pov_submissions(e): + # We have pending POVs, no need to submit yet another one. + return False + + # No successful POV, and no pending POVs, we can submit a new one. + for pov in _get_eligible_povs_for_submission(e): + # A previous submission attempt errored, we want to submit again, but all other states are final. + pov_id, status = self.competition_api.submit_pov(pov.crash) + if pov_id: + pov.competition_pov_id = pov_id + pov.result = status + log_entry(e, i=i, msg="Submitted POV") + return True + else: + logger.error( + f"[{_task_id(pov.crash)}] Failed to submit vulnerability. Competition API returned {status}. Will attempt the next PoV." + ) + logger.debug(f"CrashInfo: {pov.crash}") + log_entry(e, i=i, msg="Failed to submit POV") + + return False + + def _update_pov_status(self, i: int, e: SubmissionEntry, _redis: Redis) -> bool: + """ + Update the status of a PoV in the `ACCEPTED` state. + """ + updated = False + for pov in _get_pending_pov_submissions(e): + status = self.competition_api.get_pov_status(_task_id(pov.crash), pov.competition_pov_id) + if status != SubmissionResult.ACCEPTED: + pov.result = status + log_entry(e, i=i, msg=f"Updated POV status. New status {SubmissionResult.Name(status)}") + updated = True + return updated + + def _request_patch_if_needed(self, i: int, e: SubmissionEntry, redis: Redis) -> bool: + """ + Request the first patch if needed. + """ + + # If we already have the "current patch" no need to request more. + if _current_patch(e): + return False + + # Do not request a patch until we know if the PoV is already mitigated by an already submitted patch from another submission + # If this returns False, this will later be merged. + if self._should_wait_for_patch_mitigation_merge(i, e): + return False + + log_entry(e, i=i, msg="Submitting patch request") + + patch_tracker = self._new_patch_tracker() + confirmed = ConfirmedVulnerability() + for crash_with_id in e.crashes: + confirmed.crashes.append(crash_with_id.crash) + confirmed.internal_patch_id = patch_tracker.internal_patch_id + e.patches.append(patch_tracker) + + q = QueueFactory(redis).create(QueueNames.CONFIRMED_VULNERABILITIES, block_time=None) + self._enqueue_patch_requests(confirmed_vulnerability=confirmed, q=q) + + log_entry(e, i=i, msg="Patch request submitted") + + return True + + def _request_patched_builds_if_needed(self, i: int, e: SubmissionEntry, redis: Redis) -> bool: + """ + Make sure that builds are available for the current patch, if any. + """ + + patch = _current_patch(e) + if not patch: + # No current patch, nothing to do. + return False + + if not patch.patch: + # Patch has been requested but not yet generated, nothing to do. + return False + + if patch.build_outputs: + # We already have build outputs (or we are waiting for them), nothing to do. + return False + + # Request the patched builds + task_id = _task_id(e) + task = ChallengeTask(read_only_task_dir=self.tasks_storage_dir / task_id) project_yaml = ProjectYaml(task, task.task_meta.project_name) engine = "libfuzzer" if engine not in project_yaml.fuzzing_engines: engine = project_yaml.fuzzing_engines[0] - sanitizers = project_yaml.sanitizers - + q = QueueFactory(redis).create(QueueNames.BUILD, block_time=None) for san in sanitizers: - build_req = BuildRequest( + # Create a BuildOutput placeholder for the patched build + build_output = BuildOutput( engine=engine, - task_dir=str(task.task_dir), + sanitizer=san, + task_dir="", # Use a placeholder for the task dir, it will be updated when the build request is processed task_id=task_id, build_type=BuildType.PATCH, - sanitizer=san, apply_diff=True, - patch=patch.patch, internal_patch_id=patch.internal_patch_id, ) - self.build_requests_queue.push(build_req) + build_req = BuildRequest( + engine=build_output.engine, + task_dir=str(task.task_dir), + task_id=build_output.task_id, + build_type=build_output.build_type, + sanitizer=build_output.sanitizer, + apply_diff=build_output.apply_diff, + patch=patch.patch, + internal_patch_id=build_output.internal_patch_id, + ) + q.push(build_req) + # Add the build output placeholder to the list of patch builds + patch.build_outputs.append(build_output) logger.info( f"[{task_id}] Pushed build request {BuildType.Name(build_req.build_type)} | {build_req.sanitizer} | {build_req.engine} | {build_req.apply_diff} | {build_req.internal_patch_id}" ) + return True def record_patched_build(self, build_output: BuildOutput) -> bool: - """Record a patched build""" + """Record a patched build + + This will be invoked by the scheduler when it receives a PATCH BuildOutput. It is not part of the loop that + processes the submissions.""" key = build_output.internal_patch_id - for i, e in self._enumerate_submissions(): - for patch in e.patches: - if patch.internal_patch_id == key: - # Found the patch, now record the build output - # unless there are already build outputs for this patch/sanitizer/engine/build_type - if any( - bo.sanitizer == build_output.sanitizer - and bo.engine == build_output.engine - and bo.build_type == build_output.build_type - for bo in patch.build_outputs - ): - logger.warning( - f"Build output {build_output.internal_patch_id} already recorded for patch {patch.internal_patch_id}. Will discard." - ) - # Still acknowledge the build output, but don't add it to the patch - return True - patch.build_outputs.append(build_output) + maybe_patch = self._find_patch(key) + if not maybe_patch: + # If we get here, the build output is not associated with any patch, + # it is possible that the task was cancelled or expired. + logger.error( + f"Build output {build_output.internal_patch_id} not found in any patch (task expired/cancelled?). Will discard." + ) + return True - # Persist the entry to Redis - self._persist(self.redis, i, e) - log_entry(e, i=i, msg="Patched build recorded") - return True + i, e, patch = maybe_patch - # If we get here, the build output is not associated with any patch, - # it is possible that the task was cancelled or expired. - logger.error( - f"Build output {build_output.internal_patch_id} not found in any patch (task expired/cancelled?). Will discard." + bo = _find_matching_build_output(patch, build_output) + if not bo: + # This should never happen, but just in case. + logger.error( + f"Build output {build_output.internal_patch_id} not found in patch {patch.internal_patch_id}. Will discard." + ) + return True + + # Found the placeholder, now record the build output + if bo.task_dir: + # This could happen if the build takes longer than the build request timeout. + logger.warning( + f"Build output {build_output.internal_patch_id} already recorded for patch {patch.internal_patch_id}. Will discard." + ) + return True + + if bo.task_id != build_output.task_id: + # This should never happen, but just in case. + logger.error( + f"Build output {build_output.internal_patch_id} has a different task id than the patch. Will discard." + ) + return True + + bo.task_dir = build_output.task_dir + # Persist the entry to Redis + self._persist(self.redis, i, e) + log_entry(e, i=i, msg=f"Patched build recorded for patch {patch.internal_patch_id}") + return True + + def _submit_patch_if_good(self, i: int, e: SubmissionEntry, redis: Redis) -> bool: + """Submit the current patch if it is good""" + + # Check that at least one POV has passed validation and has a competition_pov_id + if not _get_first_successful_pov_id(e): + return False + + # No patch has been requested yet, we don't need to submit anything. + patch = _current_patch(e) + if not patch: + return False + + # No patch has been received yet, we don't need to submit anything. + if not patch.patch: + return False + + # Check if all POVs have been mitigated. + status = self._check_all_povs_are_mitigated(i, e, e.patch_idx) + if status is None: + return False # Pending evaluation + if not status: + # Bad patch, move on to next (will be requested on next iteration, or if we already have more patches we will evaluate those first) + _advance_patch_idx(e) + return True + + # Check if the patch has already been submitted. + if patch.competition_patch_id: + # This patch has already been submitted, we don't need to submit it again. + # It is either that the current patch is good, or, we advanced to the next patch (which was previously merged from a different SubmissionEntry) + return False + + # Check if this submission's POVs are mitigated by patches from other submissions + if self._should_wait_for_patch_mitigation_merge(i, e): + return False + + # If we have tried submitting the patch too many times, we will move on to the next patch. + if e.patch_submission_attempts >= self.patch_submission_retry_limit: + # Hopefully, it was something wrong with our previous patch that made the submission fail and not the competition-API side. + # If it is the competition-API that is the issue, we will end up in a loop requesting new patches and submitting them. This + # is not ideal, however the alternative scenario is that we are stuck submitting the same patch over and over again. + _advance_patch_idx(e) + return True + + # At this point, we have a good patch to submit. + competition_patch_id, status = self.competition_api.submit_patch(_task_id(e), patch.patch) + patch.result = status + if competition_patch_id: + # Good submission, record the patch id + patch.competition_patch_id = competition_patch_id + log_entry(e, i=i, msg=f"Patch sucessfully submitted id={competition_patch_id}") + elif status == SubmissionResult.ERRORED: + _increase_submission_attempts(e) + log_entry(e, i=i, msg=f"Patch submission errored, will try again. Attempts={e.patch_submission_attempts}") + elif status == SubmissionResult.FAILED or SubmissionResult.INCONCLUSIVE: + # Bad patch (or undecided state), move on to next + _advance_patch_idx(e) + log_entry(e, i=i, msg=f"Patch submission errored, will try again. Attempts={e.patch_submission_attempts}") + else: + # This is a status we won't do anything about, just record it. If it is a timeout, next iteration will probably + # filter the SubmissionEntry so we won't try this patch again. + log_entry(e, i=i, msg=f"Patch submission returned unknown status {status}") + + return True + + def _update_patch_status(self, i: int, e: SubmissionEntry, redis: Redis) -> bool: + """ + Update the status of any patch in the ACCEPTED state. + """ + updated = False + for patch in _get_pending_patch_submissions(e): + status = self.competition_api.get_patch_status(_task_id(e), patch.competition_patch_id) + patch.result = status + match status: + case SubmissionResult.PASSED: + log_entry(e, i=i, msg="Patch passed") + updated = True + case SubmissionResult.FAILED | SubmissionResult.INCONCLUSIVE: + log_entry(e, i=i, msg="Patch failed. Will advance to next patch.") + _advance_patch_idx(e) + updated = True + case SubmissionResult.ERRORED: + patch.competition_patch_id = None + log_entry(e, i=i, msg="Patch errored. Will try again. Attempts={e.patch_submission_attempts}") + _increase_submission_attempts(e) + updated = True + case SubmissionResult.DEADLINE_EXCEEDED: + patch.competition_patch_id = None + log_entry(e, i=i, msg="Patch deadline exceeded. Will stop trying.") + # NOTE: Not increasing patch index as we don't want another patch to be generated + updated = True + case SubmissionResult.ACCEPTED: + # No change + pass + + return updated + + def _ensure_single_bundle(self, i: int, e: SubmissionEntry, redis: Redis) -> bool: + """ + When SubmissionEntries are merged, we might end up having multiple bundles. + We want to avoid that so delete one bundle to get us closer to single bundle. + We only do one delete each iteration to prevent loosing too much data if something goes wrong. + """ + + if len(e.bundles) <= 1: + # All good + return False + + last_bundle_id = e.bundles[-1].bundle_id + task_id = _task_id(e) + logger.debug(f"[{task_id}] Deleting bundle {last_bundle_id}") + if self.competition_api.delete_bundle(task_id, last_bundle_id): + log_entry(e, i=i, msg=f"Deleted bundle {last_bundle_id}") + e.bundles.pop() + return True + + log_entry(e, i=i, msg=f"Failed to delete bundle {last_bundle_id}. Will keep trying.") + return False + + def _ensure_bundle_contents( + self, + i: int, + e: SubmissionEntry, + redis: Redis, + competition_pov_id: str, + competition_patch_id: str | None = None, + competition_sarif_id: str | None = None, + ) -> bool: + """ "Ensures there is a single bundle with the given competition_pov_id, competition_patch_id and competition_sarif_id + NOTE: Only called when bundles < 2 + NOTE: competition_patch_id and competition_sarif_id can be None if we are not submitting a patch or sarif + when they are not set, for a new bundle we will set them to empty strings, + for an existing bundle we will not change them. + """ + + nbundles = len(e.bundles) + if nbundles > 1: + # We only process when there is at most one bundle + return False + + task_id = _task_id(e) + if nbundles == 0: + competition_bundle_id, status = self.competition_api.submit_bundle( + task_id, competition_pov_id, competition_patch_id or "", competition_sarif_id or "" + ) + if competition_bundle_id: + bundle = Bundle( + bundle_id=competition_bundle_id, + task_id=task_id, + competition_pov_id=competition_pov_id, + competition_patch_id=competition_patch_id, + competition_sarif_id=competition_sarif_id, + ) + + e.bundles.append(bundle) + log_entry( + e, + i=i, + msg=f"Submitted bundle {competition_bundle_id} for patch {competition_patch_id} and sarif {competition_sarif_id}", + ) + return True + else: + log_entry( + e, + i=i, + msg=f"Failed to submit bundle, status: {status}. Will keep trying (not much else we can do).", + ) + return False + else: + # We have a previous bundle, check if it is still valid + bundle = e.bundles[0] + bundle_needs_update = False + if competition_patch_id is not None: + # Want to make sure this is the value in the bundle, first check if it is already set + if bundle.competition_patch_id != competition_patch_id: + # If it is not set, set it + bundle.competition_patch_id = competition_patch_id + bundle_needs_update = True + if competition_sarif_id is not None: + # Want to make sure this is the value in the bundle, first check if it is already set + if bundle.competition_sarif_id != competition_sarif_id: + # If it is not set, set it + bundle.competition_sarif_id = competition_sarif_id + bundle_needs_update = True + + if not bundle_needs_update: + # All good, no need to do anything + return False + + log_entry(e, i=i, msg="Patching bundle") + # It bundles the wrong contents, patch the bundle using the correct contents. + success, status = self.competition_api.patch_bundle( + bundle.task_id, + bundle.bundle_id, + bundle.competition_pov_id, + bundle.competition_patch_id, + bundle.competition_sarif_id, + ) + if success: + log_entry( + e, + i=i, + msg=f"Patched bundle {bundle.bundle_id} with patch {competition_patch_id} and sarif {competition_sarif_id}", + ) + return True + + log_entry(e, i=i, msg=f"Failed to patch bundle {bundle.bundle_id}. Status: {status}. Will keep trying.") + return False + + def _ensure_patch_is_bundled(self, i: int, e: SubmissionEntry, redis: Redis) -> bool: + """ + Once we have a known good patch, we want to ensure it is bundled with a PoV. + + We only submit a patch when it is known to be good (mitigates all known PoVs). That might change later on. + If more PoVs come in and are not mitigated, we will submit a new patch, that will cause this method to + patch the bundle again once the new patch passes. + """ + current_patch = _current_patch(e) + if not current_patch: + return False + + if current_patch.result != SubmissionResult.PASSED: + # We only process when the current patch is passed + return False + + nbundles = len(e.bundles) + if nbundles > 1: + # We only process when there is at most one bundle + return False + + # Either get the PoV from the bundle, or from the first successful PoV + competition_pov_id = e.bundles[0].competition_pov_id if e.bundles else _get_first_successful_pov_id(e) + + # If we don't have a PoV, we can't bundle, this should never happen. + if not competition_pov_id: + logger.error(f"No competition PoV ID found for submission {e.submission_id}") + return False + + # Update the bundle with the current patch + return self._ensure_bundle_contents( + i, e, redis, competition_pov_id, competition_patch_id=current_patch.competition_patch_id ) + + def _get_available_sarifs_for_matching(self, task_id: str): + """Get SARIFs that are available for matching for the given task. + + Returns SARIFs for the task that haven't been used in any existing bundles. + + Args: + task_id: The task ID to get SARIFs for + + Returns: + List of SARIF objects that are available for matching + """ + # Get SARIFs for the task, if none there is nothing to do. + sarifs = self.sarif_store.get_by_task_id(task_id) + if not sarifs: + return [] + + # Collect already used SARIFs, we can only bundle a SARIF once. + # Only consider active submissions - stopped submissions release their SARIFs for reuse + already_submitted_sarifs = { + bundle.competition_sarif_id + for _, submission_entry in self._enumerate_task_submissions(task_id) + for bundle in submission_entry.bundles + if bundle.competition_sarif_id + } + + # Return only SARIFs that haven't been used yet + return [sarif for sarif in sarifs if sarif.sarif_id not in already_submitted_sarifs] + + def _ensure_sarif_is_bundled(self, i: int, e: SubmissionEntry, redis: Redis) -> bool: + """Try to match a SARIF to PoV and on success submit the bundle""" + # If this already has a bundle with a SARIF no need to do anything + if e.bundles and e.bundles[0].competition_sarif_id: + return False + + # We need a successful POV to bundle a SARIF. + competition_pov_id = _get_first_successful_pov_id(e) + if not competition_pov_id: + return False + + # Find a SARIF that matches a passed PoV + for sarif in self._get_available_sarifs_for_matching(_task_id(e)): + for crash in e.crashes: + match_result = match(sarif, crash.crash) + if match_result: + log_entry( + e, + i=i, + msg=f"Found matching SARIF: {sarif.sarif_id}: {match_result}. Checking if it matches on lines.", + fn=logging.debug, + ) + # We require a match on lines to be confident that the SARIF is a good match. + if not match_result.matches_lines: + continue + log_entry( + e, + i=i, + msg=f"Found matching SARIF: {sarif.sarif_id}: {match_result}. Will bundle it.", + fn=logging.info, + ) + return self._ensure_bundle_contents( + i, e, redis, competition_pov_id, competition_sarif_id=sarif.sarif_id + ) + return False + + def _confirm_matched_sarifs(self, i: int, e: SubmissionEntry, redis: Redis) -> bool: + """Ensure the SARIF is submitted to the competition API""" + + if len(e.bundles) != 1: + # Don't make any changes while we are fixing up bundles resulting from a merge + return False + + bundle = e.bundles[0] + if not bundle.competition_sarif_id: + return False + + if bundle.competition_sarif_id not in self.matched_sarifs: + # We have a bundle with a SARIF that hasn't been confirmed yet. Do that now. + success, status = self.competition_api.submit_matching_sarif(_task_id(e), bundle.competition_sarif_id) + if success: + self._insert_matched_sarif(redis, bundle.competition_sarif_id) + return True + else: + log_entry( + e, + i=i, + msg=f"Failed to confirm SARIF {bundle.competition_sarif_id}. Status: {status}. Will keep trying.", + ) + + # We have a bundle with a SARIF that has been confirmed, no need to do anything (or confirmation failed) return True def record_patch(self, patch: Patch) -> bool: @@ -765,48 +1445,41 @@ class Submissions: True if the patch was successfully recorded, False if the submission doesn't exist """ key = patch.internal_patch_id - entry_patch = None - for i, e in self._enumerate_submissions(): - for entry_patch_tracker in e.patches: - if entry_patch_tracker.internal_patch_id == key: - if entry_patch_tracker.patch: - # There is already a patch here, this is likely the result of the request timing out - # in the patcher but multiple patchers still completed the patch. - # In this case, we just generate a new patch tracker and add it. - new_patch_tracker = self._new_patch_tracker() - new_patch_tracker.patch = patch.patch - e.patches.append(new_patch_tracker) - entry_patch = e.patches[-1] - else: - # There is no patch here, this is the first time we are recording a patch for this patch tracker. - entry_patch_tracker.patch = patch.patch - entry_patch = entry_patch_tracker + maybe_patch = self._find_patch(key) + if not maybe_patch: + # The patch is not associated with any submission, it is possible that the task was cancelled or expired. + logger.error(f"Patch {key} not found in any submission (task expired/cancelled?). Will discard.") + return True - # We have a patch now, persist the entry and double check if it will ever be used - self._persist(self.redis, i, e) - task_id = _task_id(e) - self._request_patched_builds(task_id, entry_patch) + i, e, entry_patch = maybe_patch + if entry_patch.patch: + # There is already a patch here, this is likely the result of the request timing out + # in the patcher but multiple patchers still completing the patch. + # In this case, we just generate a new patch tracker and add it. + new_patch_tracker = self._new_patch_tracker() + new_patch_tracker.patch = patch.patch + e.patches.append(new_patch_tracker) + else: + # There is no patch here, this is the first time we are recording a patch for this patch tracker. + entry_patch.patch = patch.patch - log_entry(e, i=i, msg="Patch added") - return True + # We have a patch now, persist the entry and double check if it will ever be used + self._persist(self.redis, i, e) - logger.warning( - f"Internal patch id {key} wasn't found in any active task. The original task might be cancelled or has expired. Will discard." - ) - # Still acknowledge the patch, don't have much use for this message being showed repeatedly + log_entry(e, i=i, msg="Patch added") return True - def _pov_reproduce_status_request(self, e: SubmissionEntry, patch_idx: int) -> List[POVReproduceResponse | None]: - patch = e.patches[patch_idx] - task_id = _task_id(e) + def _pov_reproduce_patch_status( + self, patch: SubmissionEntryPatch, crashes: List[CrashWithId], task_id: str + ) -> List[POVReproduceResponse | None]: result = [] - for crash in e.crashes: + for crash_with_id in crashes: request = POVReproduceRequest() request.task_id = task_id request.internal_patch_id = patch.internal_patch_id - request.harness_name = crash.crash.harness_name - request.sanitizer = crash.crash.target.sanitizer - request.pov_path = crash.crash.crash_input_path + request.harness_name = crash_with_id.crash.crash.harness_name + request.sanitizer = crash_with_id.crash.crash.target.sanitizer + request.pov_path = crash_with_id.crash.crash.crash_input_path status = self.pov_reproduce_status.request_status(request) @@ -814,15 +1487,22 @@ class Submissions: return result - def _check_all_povs_are_mitigated(self, e: SubmissionEntry, patch_idx: int) -> bool | None: + def _pov_reproduce_status_request(self, e: SubmissionEntry, patch_idx: int) -> List[POVReproduceResponse | None]: + patch = e.patches[patch_idx] + task_id = _task_id(e) + return self._pov_reproduce_patch_status(patch, e.crashes, task_id) + + def _check_all_povs_are_mitigated(self, i: int, e: SubmissionEntry, patch_idx: int) -> bool | None: """ Check if all POVs for a confirmed vulnerability are mitigated. Returns None if anyone is pending, returns True if all mitigated, returns False if any are failing. """ - logger.debug(f"Checking if all POVs for patch {patch_idx} are mitigated") statuses = self._pov_reproduce_status_request(e, patch_idx) - logger.debug(f"Statuses: {statuses}") + n_pending = sum(1 for status in statuses if status is None) + n_mitigated = sum(1 for status in statuses if status is not None and not status.did_crash) + n_failed = sum(1 for status in statuses if status is not None and status.did_crash) + log_entry(e, i=i, msg=f"Remediation status: Pending: {n_pending}, Mitigated: {n_mitigated}, Failed: {n_failed}") # If any patch is failing, we need to create a new patch. any_failing = any(status is not None and status.did_crash for status in statuses) @@ -842,17 +1522,6 @@ class Submissions: """Create a new patch tracker for a submission entry and assign it a new unique id.""" return SubmissionEntryPatch(internal_patch_id=str(uuid.uuid4())) - def _generate_patch_request( - self, i: int, e: SubmissionEntry, patch_tracker: SubmissionEntryPatch - ) -> ConfirmedVulnerability: - """Create ConfirmedVulnerability from submission entry with all crashes and a new unique id.""" - confirmed = ConfirmedVulnerability() - for crash in e.crashes: - confirmed.crashes.append(crash) - confirmed.internal_patch_id = patch_tracker.internal_patch_id - - return confirmed - def _enqueue_patch_requests( self, confirmed_vulnerability: ConfirmedVulnerability, q: ReliableQueue[ConfirmedVulnerability] | None ) -> None: @@ -863,353 +1532,99 @@ class Submissions: for _ in range(self.patch_requests_per_vulnerability): q.push(confirmed_vulnerability) - def _persist_and_enqueue_patch_request_transaction(self, i, e, old_state: int | None = None): + def _should_wait_for_patch_mitigation_merge(self, i: int, e: SubmissionEntry) -> bool: """ - Request patch generation for a confirmed vulnerability. + Check if this submission's POVs are mitigated by patches from other submissions. - Pushes the vulnerability to the confirmed_vulnerabilities_queue and persists the submission entry, all in a single transaction. + If the POVs in this SubmissionEntry are mitigated by a patch in another SubmissionEntry, + we will merge the two entries later on. However, for now we need to check each of the + SubmissionEntries for this task that has submitted a patch and if the patch mitigates + any of the POVs in this SubmissionEntry. Args: - i: Index of the submission entry - e: SubmissionEntry to process + i: Index of the current submission entry + e: The current submission entry + + Returns: + True if we should wait for patch mitigation evaluation or merge, False otherwise """ - log_entry(e, i=i, msg="Submitting patch request") + for j, e2 in self._enumerate_task_submissions(_task_id(e)): + if i == j: + continue - patch_tracker = self._new_patch_tracker() + maybe_patch = _current_patch(e2) + # No patch, nothing to check. + if not maybe_patch: + continue - confirmed = self._generate_patch_request(i, e, patch_tracker) - e.patches.append(patch_tracker) + # There is a patch, but it has not been submitted yet, nothing to check. + if not maybe_patch.competition_patch_id: + continue - with self.redis.pipeline() as pipe: - q = QueueFactory(pipe).create(QueueNames.CONFIRMED_VULNERABILITIES, block_time=None) - self._enqueue_patch_requests(confirmed_vulnerability=confirmed, q=q) - self._persist(pipe, i, e) - pipe.execute() - - log_entry(e, i=i, msg="Patch request submitted", old_state=old_state) - - def _attempt_next_patch(self, i, e): - """ - Attempts to use any additional patch available. If none, requests a new one. - - This will also persist the SubmissionEntry - """ - # NOTE: Don't advance patch index before checking if we have more patches. - if _have_more_patches(e): - _advance_patch_idx(e) - self._persist(self.redis, i, e) - else: - _advance_patch_idx(e) - self._persist_and_enqueue_patch_request_transaction(i, e) - - def _submit_patch_request(self, i, e): - """ - Request patch generation for a confirmed vulnerability. - - Pushes the vulnerability to the confirmed_vulnerabilities_queue - and updates the state to WAIT_POV_PASS. - - Args: - i: Index of the submission entry - e: SubmissionEntry to process - """ - old_state = e.state - e.state = SubmissionEntry.WAIT_POV_PASS - self._persist_and_enqueue_patch_request_transaction(i, e, old_state=old_state) - - def _wait_pov_pass(self, i, e): - """ - Check the status of a POV submission and update its state accordingly. - - Possible state transitions: - - WAIT_POV_PASS → STOP: If POV failed or deadline exceeded - - WAIT_POV_PASS → SUBMIT_PATCH: If POV passed validation - - Resubmit if POV errored - - Args: - i: Index of the submission entry - e: SubmissionEntry to process - """ - status = self.competition_api.get_pov_status(_task_id(e), e.pov_id) - match status: - case ( - TypesSubmissionStatus.SubmissionStatusFailed - | TypesSubmissionStatus.SubmissionStatusDeadlineExceeded - | TypesSubmissionStatus.SubmissionStatusInconclusive - ): - self._stop(i, e, f"POV failed (status={status}), stopping") - - case TypesSubmissionStatus.SubmissionStatusPassed: - self.task_registry.mark_successful(_task_id(e)) - old_state = e.state - e.state = SubmissionEntry.SUBMIT_PATCH - self._persist(self.redis, i, e) - log_entry(e, i=i, msg="POV passed, ready to submit patch when the patch is ready", old_state=old_state) - case TypesSubmissionStatus.SubmissionStatusErrored: - self.task_registry.mark_errored(_task_id(e)) - log_entry(e, i=i, msg="POV errored, will resubmit") - - # NOTE: Currently only submitting the first crash. This might need to be changed in the future. - pov_id, status = self.competition_api.submit_pov(e.crashes[0]) - if not pov_id: - log_entry( - e, - i=i, - msg=f"Failed to submit vulnerability. Competition API returned {status}.", - fn=logger.error, - ) - - # If the API returned ERRORED, we want to retry. - if status != TypesSubmissionStatus.SubmissionStatusErrored: - self._stop(i, e, f"POV failed (status={status}), stopping") - else: - e.pov_id = pov_id - self._persist(self.redis, i, e) - log_entry(e, i=i, msg="POV resubmitted, waiting for pass") - - case TypesSubmissionStatus.SubmissionStatusAccepted: - pass - case _: - log_entry(e, i=i, msg=f"Unexpected POV status: {status}", fn=logger.error) - - def _submit_patch(self, i, e): - """ - Submit a patch to the competition API. - - Handles patch submission and updates state based on the result: - - SUBMIT_PATCH → WAIT_PATCH_PASS: If patch is accepted - - Advances to next patch if patch fails or deadline exceeded - - Retries or advances patch if errors occur based on submission attempt count - - Args: - i: Index of the submission entry - e: SubmissionEntry to process - """ - if e.patch_idx >= len(e.patches): - # There are no patches to submit, or we've already submitted all patches - return - - status = self._check_all_povs_are_mitigated(e, e.patch_idx) - if status is None: - # We haven't checked all POVs for this patch yet, so we can't submit the patch - log_entry(e, i=i, msg="Pending POV check, will not submit patch yet.", fn=logger.debug) - return - - if status is False: - # All POVs for this patch are not mitigated, so we can't submit the patch - log_entry(e, i=i, msg="Patch does not mitigate all POVs, will not submit patch") - self._attempt_next_patch(i, e) - return - - # All good, all PoVs for this patch are mitigated. - - have_more_patches = _have_more_patches(e) - patch = e.patches[e.patch_idx].patch - - log_entry(e, i=i, msg="Submitting patch") - logger.debug(f"patch: {patch[:512]}...") - - patch_id, status = self.competition_api.submit_patch(_task_id(e), patch) - - if patch_id: - old_state = e.state - e.competition_patch_id = patch_id - e.patch_submission_attempt += 1 - e.patches[e.patch_idx].competition_patch_id = patch_id - e.state = SubmissionEntry.WAIT_PATCH_PASS - self._persist(self.redis, i, e) - log_entry(e, i=i, msg="Patch accepted", old_state=old_state) - else: - match status: - case TypesSubmissionStatus.SubmissionStatusDeadlineExceeded: - self._stop(i, e, "Patch submission deadline exceeded, stopping") - case TypesSubmissionStatus.SubmissionStatusFailed | TypesSubmissionStatus.SubmissionStatusInconclusive: - # The patch failed for some reason. Try generating a new patch - self._attempt_next_patch(i, e) - log_entry(e, i=i, msg=f"Patch submission failed ({status}), will not attempt this patch again.") - case TypesSubmissionStatus.SubmissionStatusErrored: - if have_more_patches and e.patch_submission_attempt >= self.patch_submission_retry_limit: - self._attempt_next_patch(i, e) - log_entry( - e, i=i, msg=f"Patch submission errored too many times ({status}), moved on to next patch." - ) - else: - # Keep trying the same patch (if we don't have more patches or we haven't tried too many times) - e.patch_submission_attempt += 1 - self._persist(self.redis, i, e) - log_entry( - e, - i=i, - msg=f"Patch submission failed ({status}), will attempt this patch again (attempt={e.patch_submission_attempt}).", - ) - case _: - log_entry(e, i=i, msg=f"Unexpected patch status: {status}", fn=logger.error) - - def _wait_patch_pass(self, i, e): - """ - Check the status of a patch submission and update its state accordingly. - - Possible state transitions: - - WAIT_PATCH_PASS → SUBMIT_PATCH: If patch failed, errored, or deadline exceeded - - WAIT_PATCH_PASS → SUBMIT_BUNDLE: If patch passed validation - - Args: - i: Index of the submission entry - e: SubmissionEntry to process - """ - status = self.competition_api.get_patch_status(_task_id(e), e.competition_patch_id) - match status: - case TypesSubmissionStatus.SubmissionStatusAccepted: - return # No change. - case TypesSubmissionStatus.SubmissionStatusDeadlineExceeded: - self._stop(i, e, "Patch submission deadline exceeded, stopping") - case TypesSubmissionStatus.SubmissionStatusFailed | TypesSubmissionStatus.SubmissionStatusInconclusive: - old_state = e.state - e.state = SubmissionEntry.SUBMIT_PATCH - self._attempt_next_patch(i, e) + patch_mitigates_povs = self._pov_reproduce_patch_status(maybe_patch, e.crashes, _task_id(e)) + if any(status is None for status in patch_mitigates_povs): + # Wait until we have evaluated all our PoVs against the already submitted patch + # there are still pending evaluations + log_entry(e, i=i, msg="Waiting for patch mitigation evaluation") + return True + if any(status is not None and not status.did_crash for status in patch_mitigates_povs): + # The patch mitigates at least one PoV in this SubmissionEntry, we will merge + # the two entries later on. log_entry( e, i=i, - msg=f"Patch submission failed ({status}), will not attempt this patch again, moving on to next patch.", - old_state=old_state, + msg=f"Patch competition_patch_id={maybe_patch.competition_patch_id} mitigates at least one PoV, wait for merge", ) - case TypesSubmissionStatus.SubmissionStatusErrored: - self.task_registry.mark_errored(_task_id(e)) - old_state = e.state - e.state = SubmissionEntry.SUBMIT_PATCH - self._persist(self.redis, i, e) - log_entry( - e, - i=i, - msg=f"Patch submission errored ({status}), will attempt this patch again.", - old_state=old_state, - ) - case TypesSubmissionStatus.SubmissionStatusPassed: - self.task_registry.mark_successful(_task_id(e)) - old_state = e.state - e.state = SubmissionEntry.SUBMIT_BUNDLE - self._persist(self.redis, i, e) - log_entry(e, i=i, msg="Patch passed, submitting bundle", old_state=old_state) - case _: - log_entry(e, i=i, msg=f"Unexpected patch status: {status}", fn=logger.error) + return True - def _submit_bundle(self, i: int, e: SubmissionEntry) -> None: - """ - Submit a bundle combining a validated vulnerability and patch. + return False - Possible state transitions: - - SUBMIT_BUNDLE → STOP: If bundle submission failed or deadline exceeded - - SUBMIT_BUNDLE → SUBMIT_MATCHING_SARIF: If bundle submission was successful + def _merge_entries_by_patch_mitigation(self) -> list[SubmissionEntry]: + """Check each PoV in each SubmissionEntry and if they are mitigated by a patch in another SubmissionEntry, merge the two entries.""" + for i, e in self._enumerate_submissions(): + try: + task_id = _task_id(e) - Args: - i: Index of the submission entry - e: SubmissionEntry to process - """ - bundle_id, status = self.competition_api.submit_bundle(_task_id(e), e.pov_id, e.competition_patch_id) - if not bundle_id: - match status: - case ( - TypesSubmissionStatus.SubmissionStatusFailed - | TypesSubmissionStatus.SubmissionStatusDeadlineExceeded - | TypesSubmissionStatus.SubmissionStatusInconclusive - ): - self._stop(i, e, f"Bundle submission failed ({status}), not much else to do but stop.") - case TypesSubmissionStatus.SubmissionStatusErrored: - log_entry(e, i=i, msg="Bundle submission failed, will retry") - case _: - # This should never happen, but we log it and hope to recover later? Not sure what else to do. - log_entry(e, i=i, msg=f"Unknown bundle status: {status}", fn=logger.info) - else: - old_state = e.state - e.bundle_id = bundle_id - e.state = SubmissionEntry.SUBMIT_MATCHING_SARIF - self._persist(self.redis, i, e) - log_entry(e, i=i, msg="Bundle submitted successfully", old_state=old_state) + current_patch = _current_patch(e) + if current_patch is None: + # No patch to check + continue - def _submit_matching_sarif(self, i: int, e: SubmissionEntry) -> None: - """ - Look for and submit a matching SARIF assessment if one exists. + # This is actually a redundant check as we would only have build_outputs once the patch is received + if not current_patch.patch: + continue - Finds SARIF reports that match our vulnerability and submits an assessment. + if not current_patch.build_outputs: + # No builds requested + continue - Possible state transitions: - - SUBMIT_MATCHING_SARIF → SUBMIT_BUNDLE_PATCH: If matching SARIF submission succeeded - - SUBMIT_MATCHING_SARIF → STOP: If submission failed or deadline exceeded + if not all(b.task_dir for b in current_patch.build_outputs): + # Builds aren't ready yet + continue - Args: - i: Index of the submission entry - e: SubmissionEntry to process - """ - sarif_list = self._get_sarif_candidates(_task_id(e)) + # At this point we have the patched builds available, we can check if they mitigate any PoVs in other entries (for the same task) - matching_sarif_id = None - for sarif in sarif_list: - for crash in e.crashes: - match_result = match(sarif, crash) - if match_result: - logger.debug( - f"[{i}:{_task_id(e)}] Found matching SARIF: {sarif.sarif_id}: {match_result}. Will check if it's a good enough match." - ) - # We require a match on lines to be confident that the SARIF is a good match. - if not match_result.matches_lines: + to_merge = [(i, e)] + for j, e2 in self._enumerate_task_submissions(task_id): + if i == j: continue - logger.info(f"[{i}:{_task_id(e)}] Found matching SARIF: {sarif.sarif_id}: {match_result}") - matching_sarif_id = sarif.sarif_id - break - if not matching_sarif_id: - return - success, status = self.competition_api.submit_matching_sarif(_task_id(e), matching_sarif_id) - if success: - old_state = e.state - with self.redis.pipeline() as pipe: - e.sarif_id = matching_sarif_id - e.state = SubmissionEntry.SUBMIT_BUNDLE_PATCH - self._persist(pipe, i, e) - self._insert_matched_sarif(pipe, matching_sarif_id) - pipe.execute() - log_entry(e, i=i, msg="Matching SARIF submitted successfully", old_state=old_state) - else: - match status: - case ( - TypesSubmissionStatus.SubmissionStatusFailed - | TypesSubmissionStatus.SubmissionStatusDeadlineExceeded - | TypesSubmissionStatus.SubmissionStatusInconclusive - ): - self._stop(i, e, f"Matching SARIF submission failed ({status}), not much else to do but stop.") - case _: - log_entry(e, i=i, msg=f"Submitting matching SARIF failed ({status}), will retry.") + pov_reproduce_statuses = self._pov_reproduce_patch_status(current_patch, e2.crashes, task_id) + if any(status is not None and not status.did_crash for status in pov_reproduce_statuses): + # This patch mitigates at least one PoV from e2, we should merge the entries + # TODO: Does it need to mitigate all PoVs? I think not as the patch could be a partial fix. + to_merge.append((j, e2)) - def _submit_bundle_patch(self, i: int, e: SubmissionEntry) -> None: - """ - Submit a bundle patch with associated SARIF. + if len(to_merge) > 1: + merged_indices = [j for j, _ in to_merge[1:]] # Skip the first entry (i) since it's the target + logger.info( + f"[{i}:{_task_id(e)}] Merging {len(to_merge) - 1} similar submissions into this one. Merging indices: {', '.join(map(str, merged_indices))}" + ) + self._consolidate_similar_submissions(crash=None, similar_entries=to_merge) + except Exception as err: + logger.error(f"[{i}:{_task_id(e)}] Error merging entries by patch mitigation: {err}") - Final step in the submission process that combines bundle and SARIF assessment. - - Possible state transitions: - - SUBMIT_BUNDLE_PATCH → STOP: Whether submission succeeds or fails - - Args: - i: Index of the submission entry - e: SubmissionEntry to process - """ - success, status = self.competition_api.patch_bundle( - _task_id(e), e.bundle_id, e.pov_id, e.competition_patch_id, e.sarif_id - ) - if success: - self._stop(i, e, "Bundle patch submitted successfully. No more work to be done. Will stop.") - else: - match status: - case ( - TypesSubmissionStatus.SubmissionStatusFailed - | TypesSubmissionStatus.SubmissionStatusDeadlineExceeded - | TypesSubmissionStatus.SubmissionStatusInconclusive - ): - self._stop(i, e, f"Bundle patch submission failed ({status}), not much else to do but stop.") - case _: - log_entry(e, i=i, msg=f"Submitting bundle patch failed ({status}), will retry.") - - def process_cycle(self): + def process_cycle(self) -> None: """ Process all active submissions through their state machine. @@ -1219,28 +1634,50 @@ class Submissions: """ for i, e in self._enumerate_submissions(): try: - match e.state: - case SubmissionEntry.SUBMIT_PATCH_REQUEST: - self._submit_patch_request(i, e) - case SubmissionEntry.WAIT_POV_PASS: - self._wait_pov_pass(i, e) - case SubmissionEntry.SUBMIT_PATCH: - self._submit_patch(i, e) - case SubmissionEntry.WAIT_PATCH_PASS: - self._wait_patch_pass(i, e) - case SubmissionEntry.SUBMIT_BUNDLE: - self._submit_bundle(i, e) - case SubmissionEntry.SUBMIT_MATCHING_SARIF: - self._submit_matching_sarif(i, e) - case SubmissionEntry.SUBMIT_BUNDLE_PATCH: - self._submit_bundle_patch(i, e) - case SubmissionEntry.STOP: - continue - case _: - logger.error(f"[{i}: {_task_id(e)}] Unknown submission state: {e.state}") + needs_persist = False + with self.redis.pipeline() as pipe: + # SARIF handling + if self._confirm_matched_sarifs(i, e, pipe): + needs_persist = True + if self._ensure_sarif_is_bundled(i, e, pipe): + needs_persist = True + + # Patch handling + if self._ensure_patch_is_bundled(i, e, pipe): + needs_persist = True + if self._update_patch_status(i, e, pipe): + needs_persist = True + if self._request_patch_if_needed(i, e, pipe): + needs_persist = True + if self._request_patched_builds_if_needed(i, e, pipe): + needs_persist = True + if self._submit_patch_if_good(i, e, pipe): + needs_persist = True + + # POV submission + if self._update_pov_status(i, e, pipe): + needs_persist = True + if self._submit_pov_if_needed(i, e, pipe): + needs_persist = True + + # Post merge handling + if self._ensure_single_bundle(i, e, pipe): + needs_persist = True + + if needs_persist: + self._persist(pipe, i, e) + pipe.execute() + except Exception as err: - logger.error(f"[{i}: {_task_id(e)}] Error processing submission: {err}") + logger.error(f"[{i}:{_task_id(e)}] Error processing submission: {err}") # NOTE: The question is if we should raise at some point. Worst case we are stuck in a error-condition # that can only be fixed by a restart of the scheduler. However, we don't know that. If we raise, we risk # the scheduler only attempting the first vulnerability and the rest of the cycle being skipped. This could # lead to a situation where we don't attempt any submissions. For now, we will just log the error and continue. + + # As a final phase we will check if active patches fixes vulnerabilities in other SubmissionEntries and for those we will + # consolidate the SubmissionEntries. + try: + self._merge_entries_by_patch_mitigation() + except Exception as err: + logger.error(f"[{i}:{_task_id(e)}] Error merging entries by patch mitigation: {err}") diff --git a/orchestrator/test/test_submissions.py b/orchestrator/test/test_submissions.py index 83789a10..c5b551d9 100644 --- a/orchestrator/test/test_submissions.py +++ b/orchestrator/test/test_submissions.py @@ -3,6 +3,7 @@ import base64 import uuid from unittest.mock import Mock, patch, MagicMock from pathlib import Path +from typing import Optional from buttercup.orchestrator.scheduler.submissions import CompetitionAPI, Submissions from buttercup.common.datastructures.msg_pb2 import ( TracedCrash, @@ -13,6 +14,9 @@ from buttercup.common.datastructures.msg_pb2 import ( Task, SubmissionEntryPatch, BuildType, + CrashWithId, + SubmissionResult, + Bundle, ) from buttercup.common.task_registry import TaskRegistry from buttercup.orchestrator.competition_api_client.models.types_pov_submission_response import ( @@ -26,6 +30,207 @@ from buttercup.orchestrator.competition_api_client.models.types_bundle_submissio ) from buttercup.orchestrator.competition_api_client.models.types_submission_status import TypesSubmissionStatus from buttercup.common.constants import ARCHITECTURE +from buttercup.orchestrator.scheduler.submissions import ( + _get_first_successful_pov_id, + _find_matching_build_output, + _get_first_successful_pov, + _get_pending_pov_submissions, + _get_eligible_povs_for_submission, + _get_pending_patch_submissions, +) +from buttercup.common.clusterfuzz_parser.crash_comparer import CrashComparer + + +def create_crash_comparison_mocks(similar_patterns=None): + """Create mock functions for crash comparison tests. + + Args: + similar_patterns: List of patterns that should be considered similar. + If None, patterns containing "similar_pattern" are similar. + """ + if similar_patterns is None: + similar_patterns = ["similar_pattern"] + + def mock_get_crash_data(stacktrace): + for pattern in similar_patterns: + if pattern in stacktrace: + return "similar_data" + return f"unique_data_{hash(stacktrace)}" + + def mock_get_inst_key(stacktrace): + for pattern in similar_patterns: + if pattern in stacktrace: + return "similar_inst" + return f"unique_inst_{hash(stacktrace)}" + + def mock_crash_comparer_init(self, data1, data2): + self.data1 = data1 + self.data2 = data2 + + def mock_is_similar(self): + return self.data1 == "similar_data" and self.data2 == "similar_data" + + return mock_get_crash_data, mock_get_inst_key, mock_crash_comparer_init, mock_is_similar + + +class SubmissionEntryBuilder: + """Builder pattern for creating SubmissionEntry objects in tests.""" + + def __init__(self): + self.entry = SubmissionEntry() + + def crash( + self, + task_id: Optional[str] = None, + competition_pov_id: Optional[str] = None, + result: Optional[SubmissionResult] = None, + crash_input_path: Optional[str] = None, + harness_name: Optional[str] = None, + sanitizer: Optional[str] = None, + engine: Optional[str] = None, + stacktrace: Optional[str] = None, + ): + """Add a crash to the submission entry. + + All parameters are optional. If None, protobuf defaults will be used. + """ + crash = Crash() + target = BuildOutput() + + if sanitizer is not None: + target.sanitizer = sanitizer + if engine is not None: + target.engine = engine + if task_id is not None: + target.task_id = task_id + + crash.target.CopyFrom(target) + + if harness_name is not None: + crash.harness_name = harness_name + if crash_input_path is not None: + crash.crash_input_path = crash_input_path + if stacktrace is not None: + crash.stacktrace = stacktrace + + traced_crash = TracedCrash() + traced_crash.crash.CopyFrom(crash) + + crash_with_id = CrashWithId() + crash_with_id.crash.CopyFrom(traced_crash) + if competition_pov_id is not None: + crash_with_id.competition_pov_id = competition_pov_id + if result is not None: + crash_with_id.result = result + + self.entry.crashes.append(crash_with_id) + return self + + def patch( + self, + internal_patch_id: Optional[str] = None, + patch_content: Optional[str] = None, + competition_patch_id: Optional[str] = None, + result: Optional[SubmissionResult] = None, + ): + """Add a patch to the submission entry. + + All parameters are optional. If None, protobuf defaults will be used. + """ + patch_entry = SubmissionEntryPatch() + if internal_patch_id is not None: + patch_entry.internal_patch_id = internal_patch_id + if patch_content is not None: + patch_entry.patch = patch_content + if competition_patch_id is not None: + patch_entry.competition_patch_id = competition_patch_id + if result is not None: + patch_entry.result = result + + self.entry.patches.append(patch_entry) + return self + + def bundle( + self, + bundle_id: Optional[str] = None, + competition_pov_id: Optional[str] = None, + competition_patch_id: Optional[str] = None, + competition_sarif_id: Optional[str] = None, + task_id: Optional[str] = None, + ): + """Add a bundle to the submission entry. + + All parameters are optional. If None, protobuf defaults will be used. + """ + bundle = Bundle() + if bundle_id is not None: + bundle.bundle_id = bundle_id + if competition_pov_id is not None: + bundle.competition_pov_id = competition_pov_id + if competition_patch_id is not None: + bundle.competition_patch_id = competition_patch_id + if competition_sarif_id is not None: + bundle.competition_sarif_id = competition_sarif_id + if task_id is not None: + bundle.task_id = task_id + + self.entry.bundles.append(bundle) + return self + + def stopped(self, is_stopped: bool = True): + """Mark the submission as stopped.""" + self.entry.stop = is_stopped + return self + + def patch_idx(self, idx: int): + """Set the patch index.""" + self.entry.patch_idx = idx + return self + + def patch_submission_attempts(self, attempts: int): + """Set the patch submission attempts.""" + self.entry.patch_submission_attempts = attempts + return self + + def build_output( + self, + patch_internal_id: Optional[str] = None, + sanitizer: Optional[str] = None, + engine: Optional[str] = None, + build_type: Optional[BuildType] = None, + apply_diff: Optional[bool] = None, + task_dir: Optional[str] = None, + task_id: Optional[str] = None, + ): + """Add a build output to the last patch. + + All parameters are optional. If None, protobuf defaults will be used. + """ + if not self.entry.patches: + raise ValueError("Cannot add build output without a patch. Call .patch() first.") + + build_output = BuildOutput() + if patch_internal_id is not None: + build_output.internal_patch_id = patch_internal_id + if sanitizer is not None: + build_output.sanitizer = sanitizer + if engine is not None: + build_output.engine = engine + if build_type is not None: + build_output.build_type = build_type + if apply_diff is not None: + build_output.apply_diff = apply_diff + if task_dir is not None: + build_output.task_dir = task_dir + if task_id is not None: + build_output.task_id = task_id + + self.entry.patches[-1].build_outputs.append(build_output) + return self + + def build(self) -> SubmissionEntry: + """Build and return the SubmissionEntry.""" + return self.entry @pytest.fixture @@ -79,6 +284,16 @@ def mock_redis(): mock = Mock() mock.lrange.return_value = [] # Default empty list for stored submissions mock.smembers.return_value = set() # Return empty set for smembers calls + mock.rpush.return_value = 1 # Return an integer instead of Mock object + + # Mock pipeline context manager + pipeline_mock = Mock() + pipeline_mock.__enter__ = Mock(return_value=pipeline_mock) + pipeline_mock.__exit__ = Mock(return_value=None) + pipeline_mock.lset = Mock(return_value=True) # Mock lset on pipeline + pipeline_mock.execute = Mock(return_value=[True]) # Mock execute + mock.pipeline.return_value = pipeline_mock + return mock @@ -86,7 +301,7 @@ def mock_redis(): def mock_competition_api(mock_task_registry): mock = Mock(spec=CompetitionAPI) # Add the missing method that's needed in tests - mock.submit_bundle_patch = Mock(return_value=(True, TypesSubmissionStatus.SubmissionStatusAccepted)) + mock.submit_bundle_patch = Mock(return_value=(True, SubmissionResult.ACCEPTED)) return mock @@ -96,6 +311,7 @@ def submissions(mock_redis, mock_competition_api, mock_task_registry): with patch("buttercup.orchestrator.scheduler.submissions.QueueFactory") as mock_queue_factory: # Create mock queues mock_build_queue = Mock() + mock_build_queue.push = Mock() # Ensure push method is properly mocked mock_reproduce_queue = Mock() mock_reproduce_queue.pop.return_value = None # No items in queue mock_reproduce_queue.ack_item = Mock() # Mock ack_item method @@ -134,9 +350,10 @@ def submissions(mock_redis, mock_competition_api, mock_task_registry): @pytest.fixture def sample_submission_entry(sample_crash): entry = SubmissionEntry() - entry.crashes.append(sample_crash) # Use crashes (repeated field) instead of crash - entry.pov_id = "test-pov-123" - entry.state = SubmissionEntry.SUBMIT_PATCH_REQUEST + crash_with_id = CrashWithId() + crash_with_id.crash.CopyFrom(sample_crash) + crash_with_id.competition_pov_id = "test-pov-123" + entry.crashes.append(crash_with_id) return entry @@ -158,9 +375,13 @@ def sample_build_output(): build_output.task_id = "test-task-123" build_output.build_type = BuildType.PATCH build_output.apply_diff = True + build_output.task_dir = "/tmp/build/test-task-123" return build_output +# Helper functions to work with the current SubmissionEntry structure + + # Tests for the CompetitionAPI class class TestCompetitionAPI: @patch("buttercup.common.node_local.lopen") @@ -186,7 +407,7 @@ class TestCompetitionAPI: # Verify result first assert result[0] == "test-pov-123" - assert result[1] == TypesSubmissionStatus.SubmissionStatusAccepted + assert result[1] == SubmissionResult.ACCEPTED # Verify file was read correctly mock_lopen.assert_called_once_with(sample_crash.crash.crash_input_path, "rb") @@ -224,7 +445,7 @@ class TestCompetitionAPI: # Verify result is a tuple with (None, FAILED) assert result[0] is None - assert result[1] == TypesSubmissionStatus.SubmissionStatusFailed + assert result[1] == SubmissionResult.FAILED def test_get_pov_status(self, competition_api): # Setup mock @@ -240,7 +461,7 @@ class TestCompetitionAPI: # Verify call and result mock_pov_api.v1_task_task_id_pov_pov_id_get.assert_called_once_with(task_id="task-123", pov_id="pov-456") - assert status == TypesSubmissionStatus.SubmissionStatusPassed + assert status == SubmissionResult.PASSED def test_submit_patch_successful(self, competition_api): # Setup test data @@ -270,7 +491,7 @@ class TestCompetitionAPI: # Verify result assert result[0] == "patch-123" - assert result[1] == TypesSubmissionStatus.SubmissionStatusAccepted + assert result[1] == SubmissionResult.ACCEPTED def test_get_patch_status(self, competition_api): # Setup mock @@ -288,7 +509,7 @@ class TestCompetitionAPI: mock_patch_api.v1_task_task_id_patch_patch_id_get.assert_called_once_with( task_id="task-123", patch_id="patch-456" ) - assert status == TypesSubmissionStatus.SubmissionStatusPassed + assert status == SubmissionResult.PASSED def test_submit_bundle_successful(self, competition_api): # Setup test data @@ -308,7 +529,8 @@ class TestCompetitionAPI: # Patch the BundleApi constructor with patch("buttercup.orchestrator.scheduler.submissions.BundleApi", return_value=mock_bundle_api): # Call method - result = competition_api.submit_bundle(task_id, pov_id, patch_id) + sarif_id = "sarif-789" # Add required sarif_id parameter + result = competition_api.submit_bundle(task_id, pov_id, patch_id, sarif_id) # Verify API call mock_bundle_api.v1_task_task_id_bundle_post.assert_called_once() @@ -319,7 +541,7 @@ class TestCompetitionAPI: # Verify result assert result[0] == "bundle-123" - assert result[1] == TypesSubmissionStatus.SubmissionStatusAccepted + assert result[1] == SubmissionResult.ACCEPTED def test_submit_matching_sarif_successful(self, competition_api): # Setup test data @@ -345,23 +567,19 @@ class TestCompetitionAPI: # Verify result assert result[0] is True - assert result[1] == TypesSubmissionStatus.SubmissionStatusAccepted + assert result[1] == SubmissionResult.ACCEPTED # Tests for the Submissions class class TestSubmissions: def test_submit_vulnerability_successful(self, submissions, mock_competition_api, sample_crash, mock_redis): - # Setup mock return value for submit_vulnerability - mock_competition_api.submit_pov.return_value = ("test-pov-123", TypesSubmissionStatus.SubmissionStatusAccepted) + # Configure mock Redis to return proper values mock_redis.rpush.return_value = 1 # Index of the inserted entry # Call the method result = submissions.submit_vulnerability(sample_crash) - # Verify competition API call - mock_competition_api.submit_pov.assert_called_once_with(sample_crash) - - # Verify Redis interactions + # Verify Redis interactions - entry should be added mock_redis.rpush.assert_called_once() # Verify result @@ -369,43 +587,49 @@ class TestSubmissions: # Verify entries list is updated assert len(submissions.entries) == 1 - assert submissions.entries[0].pov_id == "test-pov-123" - assert submissions.entries[0].state == SubmissionEntry.SUBMIT_PATCH_REQUEST + entry = submissions.entries[0] + # Verify crashes field is populated correctly - assert len(submissions.entries[0].crashes) == 1 - assert submissions.entries[0].crashes[0] == sample_crash + assert len(entry.crashes) == 1 + assert entry.crashes[0].crash == sample_crash + + # POV should not be submitted yet (happens in process_cycle) + mock_competition_api.submit_pov.assert_not_called() + + # POV ID should not be set yet + assert not entry.crashes[0].competition_pov_id def test_submit_vulnerability_failed(self, submissions, mock_competition_api, sample_crash): - # Setup mock to return an error - mock_competition_api.submit_pov.return_value = (None, TypesSubmissionStatus.SubmissionStatusFailed) + # Configure mock Redis to return proper values + submissions.redis.rpush.return_value = 1 # Index of the inserted entry # Call the method result = submissions.submit_vulnerability(sample_crash) - # Verify competition API call - mock_competition_api.submit_pov.assert_called_once_with(sample_crash) + # Verify result - should succeed (creates entry regardless of POV submission outcome) + assert result is True - # Verify result - assert result is True # Returns True even for a failure because we stop processing + # Verify entry was created + assert len(submissions.entries) == 1 - # Verify no entries were added - assert len(submissions.entries) == 0 + # POV submission happens later in process_cycle, not immediately + mock_competition_api.submit_pov.assert_not_called() def test_submit_vulnerability_errored(self, submissions, mock_competition_api, sample_crash): - # Setup mock to return an error - mock_competition_api.submit_pov.return_value = (None, TypesSubmissionStatus.SubmissionStatusErrored) + # Configure mock Redis to return proper values + submissions.redis.rpush.return_value = 1 # Index of the inserted entry # Call the method result = submissions.submit_vulnerability(sample_crash) - # Verify competition API call - mock_competition_api.submit_pov.assert_called_once_with(sample_crash) + # Verify result - should succeed (creates entry regardless of POV submission outcome) + assert result is True - # Verify result - assert result is False # Returns False for ERRORED, indicating we should retry + # Verify entry was created + assert len(submissions.entries) == 1 - # Verify no entries were added - assert len(submissions.entries) == 0 + # POV submission happens later in process_cycle, not immediately + mock_competition_api.submit_pov.assert_not_called() def test_record_patch(self, submissions, sample_patch, sample_submission_entry): # Setup submission entry in entries list @@ -413,7 +637,7 @@ class TestSubmissions: # Set task_id in both sample_submission_entry and sample_patch to match task_id = "test-task-123" - sample_submission_entry.crashes[0].crash.target.task_id = task_id # Access first crash in crashes list + sample_submission_entry.crashes[0].crash.crash.target.task_id = task_id # Access first crash in crashes list sample_patch.task_id = task_id # Add a patch entry that matches the sample_patch's internal_patch_id @@ -438,27 +662,9 @@ class TestSubmissions: # Call the method result = submissions.record_patch(sample_patch) - # Verify build requests were pushed to queue - one for each sanitizer - expected_calls = 3 # asan, msan, ubsan - assert submissions.build_requests_queue.push.call_count == expected_calls - - # Verify each build request has correct properties - for call_args in submissions.build_requests_queue.push.call_args_list: - build_req = call_args[0][0] # First positional argument - assert build_req.engine == "libfuzzer" - assert build_req.task_dir == str(Path("/tmp/tasks_storage") / task_id) - assert build_req.task_id == task_id - assert build_req.build_type == BuildType.PATCH - assert build_req.sanitizer in ["asan", "msan", "ubsan"] - assert build_req.apply_diff is True - assert build_req.patch == sample_patch.patch - assert build_req.internal_patch_id == "0" # internal_patch_id format - - # Verify all sanitizers were used - used_sanitizers = { - call_args[0][0].sanitizer for call_args in submissions.build_requests_queue.push.call_args_list - } - assert used_sanitizers == {"asan", "msan", "ubsan"} + # Verify NO build requests were pushed to queue during record_patch + # (builds are now requested during process_cycle instead) + assert submissions.build_requests_queue.push.call_count == 0 # Verify _persist was called submissions.redis.lset.assert_called_once() @@ -476,7 +682,7 @@ class TestSubmissions: # Set task_id in both sample_submission_entry and sample_patch to match task_id = "test-task-123" - sample_submission_entry.crashes[0].crash.target.task_id = task_id # Access first crash in crashes list + sample_submission_entry.crashes[0].crash.crash.target.task_id = task_id # Access first crash in crashes list sample_patch.task_id = task_id # Add a patch entry that matches the sample_patch's internal_patch_id @@ -507,394 +713,2507 @@ class TestSubmissions: # Verify should_stop_processing was called with the task_id mock_task_registry.should_stop_processing.assert_called_once_with(task_id) + def test_consolidate_multiple_similar_submissions(self, submissions, mock_competition_api, mock_redis): + """Test consolidation when a crash is similar to multiple existing submissions.""" -# Tests for state transitions -class TestStateTransitions: - def test_submit_patch_request_transition(self, submissions, sample_submission_entry): - # Setup entry in SUBMIT_PATCH_REQUEST state - sample_submission_entry.state = SubmissionEntry.SUBMIT_PATCH_REQUEST - submissions.entries = [sample_submission_entry] + # Create three existing submissions with different crashes that will be similar to our new crash + task_id = "test-task-consolidate" - # Mock QueueFactory - queue_mock = MagicMock() - pipeline_mock = MagicMock() - submissions.redis.pipeline.return_value = pipeline_mock + # Setup submissions using builder pattern - add to entries and mock Redis + submissions.entries = [ + # Submission 1 - has POV ID and patches + ( + SubmissionEntryBuilder() + .crash( + task_id=task_id, + stacktrace="similar_crash_pattern_stack1", + harness_name="harness1", + competition_pov_id="pov-123", + result=SubmissionResult.PASSED, + ) + .patch(internal_patch_id="patch1", patch_content="patch content 1") + .patch_idx(1) + .patch_submission_attempts(2) + .build() + ), + # Submission 2 - has bundle ID + ( + SubmissionEntryBuilder() + .crash( + task_id=task_id, + stacktrace="similar_crash_pattern_stack2", + harness_name="harness2", + competition_pov_id="pov-456", + result=SubmissionResult.PASSED, + ) + .patch(internal_patch_id="patch2a", patch_content="patch content 2a") + .patch(internal_patch_id="patch2b", patch_content="patch content 2b") + .bundle( + bundle_id="bundle-456", + competition_pov_id="pov-456", + competition_patch_id="comp-patch-456", + task_id=task_id, + ) + .patch_idx(0) + .patch_submission_attempts(1) + .build() + ), + # Submission 3 - has SARIF ID (most advanced) + ( + SubmissionEntryBuilder() + .crash( + task_id=task_id, + stacktrace="similar_crash_pattern_stack3", + harness_name="harness3", + competition_pov_id="pov-789", + result=SubmissionResult.PASSED, + ) + .patch(internal_patch_id="patch3", patch_content="patch content 3") + .bundle( + bundle_id="bundle-789", + competition_pov_id="pov-789", + competition_patch_id="comp-patch-789", + competition_sarif_id="sarif-789", + task_id=task_id, + ) + .patch_idx(2) + .patch_submission_attempts(5) + .build() + ), + ] - # Simulate process_cycle - with patch( - "buttercup.orchestrator.scheduler.submissions.QueueFactory", return_value=MagicMock() - ) as queue_factory_mock: - queue_factory_mock.return_value.create.return_value = queue_mock - submissions.process_cycle() + # Mock Redis rpush to return the correct index for the new submission + mock_redis.rpush.return_value = len(submissions.entries) + 1 - # Verify state transition - assert sample_submission_entry.state == SubmissionEntry.WAIT_POV_PASS + # Create a new crash that will be similar to all three submissions + new_crash = TracedCrash() + new_crash.crash.target.task_id = task_id + new_crash.crash.stacktrace = "similar_crash_pattern_new" # Will be detected as similar + new_crash.crash.harness_name = "new_harness" - def test_wait_pov_pass_to_submit_patch( - self, submissions, sample_submission_entry, mock_competition_api, mock_task_registry - ): - # Setup entry in WAIT_POV_PASS state - sample_submission_entry.state = SubmissionEntry.WAIT_POV_PASS - submissions.entries = [sample_submission_entry] - - # Mock competition API to return PASSED - mock_competition_api.get_pov_status.return_value = TypesSubmissionStatus.SubmissionStatusPassed - - # Simulate process_cycle - submissions.process_cycle() - - # Verify state transition - assert sample_submission_entry.state == SubmissionEntry.SUBMIT_PATCH - # Verify mark_successful was called - mock_task_registry.mark_successful.assert_called_once_with( - sample_submission_entry.crashes[0].crash.target.task_id + # Mock the crash comparison to return similar for all existing crashes + mock_get_crash_data, mock_get_inst_key, mock_crash_comparer_init, mock_is_similar = ( + create_crash_comparison_mocks(["similar_crash_pattern"]) ) - def test_wait_pov_pass_to_stop_when_failed(self, submissions, sample_submission_entry, mock_competition_api): - # Setup entry in WAIT_POV_PASS state - sample_submission_entry.state = SubmissionEntry.WAIT_POV_PASS - submissions.entries = [sample_submission_entry] - - # Mock competition API to return FAILED - mock_competition_api.get_pov_status.return_value = TypesSubmissionStatus.SubmissionStatusFailed - - # Simulate process_cycle - submissions.process_cycle() - - # Verify state transition - assert sample_submission_entry.state == SubmissionEntry.STOP - - def test_wait_pov_pass_resubmit_when_errored( - self, submissions, sample_submission_entry, mock_competition_api, mock_task_registry - ): - # Setup entry in WAIT_POV_PASS state - sample_submission_entry.state = SubmissionEntry.WAIT_POV_PASS - submissions.entries = [sample_submission_entry] - - # Mock competition API to return ERRORED then successful resubmission - mock_competition_api.get_pov_status.return_value = TypesSubmissionStatus.SubmissionStatusErrored - mock_competition_api.submit_pov.return_value = ("new-pov-456", TypesSubmissionStatus.SubmissionStatusAccepted) - - # Simulate process_cycle - submissions.process_cycle() - - # Verify POV was resubmitted with new ID - mock_competition_api.submit_pov.assert_called_once_with(sample_submission_entry.crashes[0]) - assert sample_submission_entry.pov_id == "new-pov-456" - # State remains the same as we're waiting for the new submission to be processed - assert sample_submission_entry.state == SubmissionEntry.WAIT_POV_PASS - # Verify mark_errored was called - mock_task_registry.mark_errored.assert_called_once_with(sample_submission_entry.crashes[0].crash.target.task_id) - - def test_submit_patch_successful(self, submissions, sample_submission_entry, mock_competition_api): - # Setup entry in SUBMIT_PATCH state with a patch - sample_submission_entry.state = SubmissionEntry.SUBMIT_PATCH - sample_submission_entry.patches.append(SubmissionEntryPatch(patch="test patch content")) - sample_submission_entry.patch_idx = 0 - sample_submission_entry.patch_submission_attempt = 0 - submissions.entries = [sample_submission_entry] - - # Mock competition API to return successful patch submission - mock_competition_api.submit_patch.return_value = ( - "test-patch-123", - TypesSubmissionStatus.SubmissionStatusAccepted, - ) - - # Mock the _have_more_patches function to avoid AttributeError + # Apply mocks with ( - patch("buttercup.orchestrator.scheduler.submissions._have_more_patches", return_value=True), - patch.object(submissions, "_check_all_povs_are_mitigated", return_value=True), # Mock POV mitigation check + patch("buttercup.orchestrator.scheduler.submissions.get_crash_data", side_effect=mock_get_crash_data), + patch("buttercup.orchestrator.scheduler.submissions.get_inst_key", side_effect=mock_get_inst_key), + patch.object(CrashComparer, "__init__", mock_crash_comparer_init), + patch.object(CrashComparer, "is_similar", mock_is_similar), ): - # Simulate process_cycle - submissions.process_cycle() + # Call submit_vulnerability with the new crash + result = submissions.submit_vulnerability(new_crash) - # Verify state transition and patch ID - assert sample_submission_entry.state == SubmissionEntry.WAIT_PATCH_PASS - assert sample_submission_entry.competition_patch_id == "test-patch-123" - assert sample_submission_entry.patch_submission_attempt == 1 # Incremented + # Verify the result + assert result is True - def test_wait_patch_pass_to_submit_bundle( - self, submissions, sample_submission_entry, mock_competition_api, mock_task_registry - ): - # Setup entry in WAIT_PATCH_PASS state - sample_submission_entry.state = SubmissionEntry.WAIT_PATCH_PASS - sample_submission_entry.competition_patch_id = "test-patch-123" - submissions.entries = [sample_submission_entry] + # Verify consolidation occurred: + # 1. First submission (target) should have all crashes + target_submission = submissions.entries[0] + assert len(target_submission.crashes) == 4 # original + 3 from other submissions + new crash + # Original crash should still be there + assert target_submission.crashes[0].crash.crash.stacktrace == "similar_crash_pattern_stack1" + assert target_submission.crashes[1].crash == new_crash # new crash added first + # Crashes from other submissions should be merged in + assert target_submission.crashes[2].crash.crash.stacktrace == "similar_crash_pattern_stack2" # from submission2 + assert target_submission.crashes[3].crash.crash.stacktrace == "similar_crash_pattern_stack3" # from submission3 - # Mock competition API to return PASSED - mock_competition_api.get_patch_status.return_value = TypesSubmissionStatus.SubmissionStatusPassed + # 2. First submission should have original patches plus unprocessed patches from sources + assert ( + len(target_submission.patches) == 3 + ) # 1 (original) + 2 (unprocessed from submission2) + 0 (none from submission3) + patch_ids = {p.internal_patch_id for p in target_submission.patches} + assert patch_ids == { + "patch1", + "patch2a", + "patch2b", + } # patch3 not included since submission3.patch_idx=2 >= len(submission3.patches)=1 - # Simulate process_cycle - submissions.process_cycle() + # 3. First submission should have all bundles and data from other submissions + assert _get_first_successful_pov_id(target_submission) == "pov-123" # kept original since it had one - # Verify state transition - assert sample_submission_entry.state == SubmissionEntry.SUBMIT_BUNDLE - # Verify mark_successful was called - mock_task_registry.mark_successful.assert_called_once_with( - sample_submission_entry.crashes[0].crash.target.task_id + # Should have all bundles from all submissions + assert len(target_submission.bundles) == 2 # from submission2 and submission3 + + # First bundle should be from submission2 + assert target_submission.bundles[0].competition_patch_id == "comp-patch-456" + assert target_submission.bundles[0].bundle_id == "bundle-456" + assert target_submission.bundles[0].competition_sarif_id == "" # no SARIF ID + + # Second bundle should be from submission3 + assert target_submission.bundles[1].competition_patch_id == "comp-patch-789" + assert target_submission.bundles[1].bundle_id == "bundle-789" + assert target_submission.bundles[1].competition_sarif_id == "sarif-789" + assert target_submission.patch_idx == 1 # kept from target (original submission1) + assert target_submission.patch_submission_attempts == 2 # kept from target (original submission1) + + # 4. Other submissions should be STOPPED + assert submissions.entries[1].stop + assert submissions.entries[2].stop + + # 5. Verify persistence was called for all submissions + # Target + 2 source submissions being stopped = 3 calls + # Since consolidation uses a pipeline, check pipeline.execute was called + assert mock_redis.pipeline.called + + def test_find_similar_entries(self, submissions): + """Test find_similar_entries method correctly identifies similar crashes.""" + + task_id = "test-task-similarity" + + # Create existing submissions with different crashes using builder + submissions.entries = [ + SubmissionEntryBuilder() + .crash( + task_id=task_id, + stacktrace="similar_pattern_crash1", + harness_name="harness1", + ) + .build(), + SubmissionEntryBuilder() + .crash( + task_id=task_id, + stacktrace="similar_pattern_crash2", + harness_name="harness2", + ) + .build(), + # Different task (should not be found) + SubmissionEntryBuilder() + .crash( + task_id="different-task", + stacktrace="similar_pattern_crash3", + harness_name="harness3", + ) + .build(), + # Unique crash (should not be found) + SubmissionEntryBuilder() + .crash( + task_id=task_id, + stacktrace="unique_pattern_crash4", + harness_name="harness4", + ) + .build(), + ] + + # Create a new crash to search for similar entries + new_crash = TracedCrash() + new_crash.crash.target.task_id = task_id + new_crash.crash.stacktrace = "similar_pattern_new" + new_crash.crash.harness_name = "new_harness" + + # Mock the crash comparison functions + mock_get_crash_data, mock_get_inst_key, mock_crash_comparer_init, mock_is_similar = ( + create_crash_comparison_mocks() ) - def test_wait_patch_pass_to_submit_patch_when_failed( + # Apply mocks + with ( + patch("buttercup.orchestrator.scheduler.submissions.get_crash_data", side_effect=mock_get_crash_data), + patch("buttercup.orchestrator.scheduler.submissions.get_inst_key", side_effect=mock_get_inst_key), + patch.object(CrashComparer, "__init__", mock_crash_comparer_init), + patch.object(CrashComparer, "is_similar", mock_is_similar), + ): + # Call find_similar_entries + similar_entries = submissions.find_similar_entries(new_crash) + + # Verify results + assert len(similar_entries) == 2 # Should find submission1 and submission2 + + # Extract indices and entries + indices = [idx for idx, _ in similar_entries] + entries = [entry for _, entry in similar_entries] + + # Verify indices are correct (should be 0 and 1) + assert 0 in indices # submission1 + assert 1 in indices # submission2 + assert 2 not in indices # submission3 (different task) + assert 3 not in indices # submission4 (unique pattern) + + # Verify entries are correct + assert submissions.entries[0] in entries + assert submissions.entries[1] in entries + assert submissions.entries[2] not in entries # different task + assert submissions.entries[3] not in entries # unique pattern + + def test_find_similar_entries_no_matches(self, submissions): + """Test find_similar_entries returns empty list when no similar crashes exist.""" + + task_id = "test-task-no-matches" + + # Create existing submission with unique crash using builder + submissions.entries = [ + SubmissionEntryBuilder() + .crash( + task_id=task_id, + stacktrace="unique_pattern_crash1", + harness_name="harness1", + ) + .build() + ] + + # Create a new crash with different pattern + new_crash = TracedCrash() + new_crash.crash.target.task_id = task_id + new_crash.crash.stacktrace = "different_pattern_new" + new_crash.crash.harness_name = "new_harness" + + # Mock the crash comparison functions to return non-matching data + mock_get_crash_data, mock_get_inst_key, mock_crash_comparer_init, mock_is_similar = ( + create_crash_comparison_mocks( + [] # No patterns are similar + ) + ) + + # Apply mocks + with ( + patch("buttercup.orchestrator.scheduler.submissions.get_crash_data", side_effect=mock_get_crash_data), + patch("buttercup.orchestrator.scheduler.submissions.get_inst_key", side_effect=mock_get_inst_key), + patch.object(CrashComparer, "__init__", mock_crash_comparer_init), + patch.object(CrashComparer, "is_similar", mock_is_similar), + ): + # Call find_similar_entries + similar_entries = submissions.find_similar_entries(new_crash) + + # Verify no matches found + assert len(similar_entries) == 0 + + def test_find_similar_entries_empty_submissions(self, submissions): + """Test find_similar_entries returns empty list when no submissions exist.""" + # No submissions in entries + submissions.entries = [] + + # Create a new crash + new_crash = TracedCrash() + new_crash.crash.target.task_id = "test-task" + new_crash.crash.stacktrace = "some_pattern" + new_crash.crash.harness_name = "harness" + + # Call find_similar_entries + similar_entries = submissions.find_similar_entries(new_crash) + + # Verify no matches found + assert len(similar_entries) == 0 + + def test_find_patch_success(self, submissions): + """Test _find_patch successfully finds a patch by internal_patch_id.""" + # Create submissions with patches using builder + submissions.entries = [ + SubmissionEntryBuilder() + .crash(task_id="task-1") + .patch(internal_patch_id="patch-1a", patch_content="patch content 1a") + .patch(internal_patch_id="patch-1b", patch_content="patch content 1b") + .build(), + SubmissionEntryBuilder() + .crash(task_id="task-2") + .patch(internal_patch_id="patch-2a", patch_content="patch content 2a") + .build(), + ] + + # Test finding first patch in first submission + result = submissions._find_patch("patch-1a") + assert result is not None + index, entry, patch = result + assert index == 0 + assert entry is submissions.entries[0] + assert patch.internal_patch_id == "patch-1a" + assert patch.patch == "patch content 1a" + + # Test finding second patch in first submission + result = submissions._find_patch("patch-1b") + assert result is not None + index, entry, patch = result + assert index == 0 + assert entry is submissions.entries[0] + assert patch.internal_patch_id == "patch-1b" + assert patch.patch == "patch content 1b" + + # Test finding patch in second submission + result = submissions._find_patch("patch-2a") + assert result is not None + index, entry, patch = result + assert index == 1 + assert entry is submissions.entries[1] + assert patch.internal_patch_id == "patch-2a" + assert patch.patch == "patch content 2a" + + def test_find_patch_not_found(self, submissions): + """Test _find_patch returns None when patch is not found.""" + # Create submissions with patches using builder + submissions.entries = [ + SubmissionEntryBuilder() + .crash(task_id="task-1") + .patch(internal_patch_id="patch-1", patch_content="patch content 1") + .build() + ] + + # Test finding non-existent patch + result = submissions._find_patch("nonexistent-patch") + assert result is None + + def test_find_patch_empty_submissions(self, submissions): + """Test _find_patch returns None when no submissions exist.""" + submissions.entries = [] + + # Test finding patch in empty submissions + result = submissions._find_patch("any-patch") + assert result is None + + def test_find_patch_no_patches(self, submissions): + """Test _find_patch returns None when submissions have no patches.""" + # Create submissions without patches using builder + submissions.entries = [ + SubmissionEntryBuilder().crash(task_id="task-1").build(), + SubmissionEntryBuilder().crash(task_id="task-2").build(), + ] + + # Test finding patch when no patches exist + result = submissions._find_patch("any-patch") + assert result is None + + def test_find_patch_stopped_submissions(self, submissions): + """Test _find_patch skips stopped submissions.""" + # Create submissions with patches using builder + submissions.entries = [ + SubmissionEntryBuilder() + .crash(task_id="task-1") + .patch(internal_patch_id="patch-in-stopped", patch_content="patch content in stopped") + .stopped() + .build(), + SubmissionEntryBuilder() + .crash(task_id="task-2") + .patch(internal_patch_id="patch-in-active", patch_content="patch content in active") + .build(), + ] + + # Test finding patch in stopped submission - should not find it + result = submissions._find_patch("patch-in-stopped") + assert result is None + + # Test finding patch in active submission - should find it + result = submissions._find_patch("patch-in-active") + assert result is not None + index, entry, patch = result + assert index == 1 + assert entry is submissions.entries[1] + assert patch.internal_patch_id == "patch-in-active" + assert patch.patch == "patch content in active" + + def test_find_matching_build_output_success(self, submissions): + """Test _find_matching_build_output successfully finds matching build outputs.""" + + # Create a patch with multiple build outputs + patch = SubmissionEntryPatch() + patch.internal_patch_id = "test-patch" + + # Add build outputs with different configurations + build_output1 = BuildOutput() + build_output1.engine = "libfuzzer" + build_output1.sanitizer = "asan" + build_output1.build_type = BuildType.PATCH + build_output1.apply_diff = True + build_output1.task_dir = "" # Placeholder + + build_output2 = BuildOutput() + build_output2.engine = "afl" + build_output2.sanitizer = "msan" + build_output2.build_type = BuildType.FUZZER + build_output2.apply_diff = False + build_output2.task_dir = "/path/to/build" # Already filled + + build_output3 = BuildOutput() + build_output3.engine = "libfuzzer" + build_output3.sanitizer = "ubsan" + build_output3.build_type = BuildType.COVERAGE + build_output3.apply_diff = True + build_output3.task_dir = "" # Placeholder + + patch.build_outputs.extend([build_output1, build_output2, build_output3]) + + # Create search criteria that match build_output1 + search_build_output = BuildOutput() + search_build_output.engine = "libfuzzer" + search_build_output.sanitizer = "asan" + search_build_output.build_type = BuildType.PATCH + search_build_output.apply_diff = True + + # Should find build_output1 + result = _find_matching_build_output(patch, search_build_output) + assert result is not None + assert result.engine == "libfuzzer" + assert result.sanitizer == "asan" + assert result.build_type == BuildType.PATCH + assert result.apply_diff is True + assert result.task_dir == "" # Should be the placeholder one + + # Create search criteria that match build_output2 + search_build_output2 = BuildOutput() + search_build_output2.engine = "afl" + search_build_output2.sanitizer = "msan" + search_build_output2.build_type = BuildType.FUZZER + search_build_output2.apply_diff = False + + # Should find build_output2 + result2 = _find_matching_build_output(patch, search_build_output2) + assert result2 is not None + assert result2.engine == "afl" + assert result2.sanitizer == "msan" + assert result2.build_type == BuildType.FUZZER + assert result2.apply_diff is False + assert result2.task_dir == "/path/to/build" # Should be the filled one + + # Create search criteria that match build_output3 + search_build_output3 = BuildOutput() + search_build_output3.engine = "libfuzzer" + search_build_output3.sanitizer = "ubsan" + search_build_output3.build_type = BuildType.COVERAGE + search_build_output3.apply_diff = True + + # Should find build_output3 + result3 = _find_matching_build_output(patch, search_build_output3) + assert result3 is not None + assert result3.engine == "libfuzzer" + assert result3.sanitizer == "ubsan" + assert result3.build_type == BuildType.COVERAGE + assert result3.apply_diff is True + assert result3.task_dir == "" # Should be the placeholder one + + def test_find_matching_build_output_no_match(self, submissions): + """Test _find_matching_build_output returns None when no match is found.""" + + # Create a patch with build outputs + patch = SubmissionEntryPatch() + patch.internal_patch_id = "test-patch" + + build_output1 = BuildOutput() + build_output1.engine = "libfuzzer" + build_output1.sanitizer = "asan" + build_output1.build_type = BuildType.PATCH + build_output1.apply_diff = True + + build_output2 = BuildOutput() + build_output2.engine = "afl" + build_output2.sanitizer = "msan" + build_output2.build_type = BuildType.FUZZER + build_output2.apply_diff = False + + patch.build_outputs.extend([build_output1, build_output2]) + + # Create search criteria that don't match any existing build output + search_build_output = BuildOutput() + search_build_output.engine = "honggfuzz" # Different engine + search_build_output.sanitizer = "asan" + search_build_output.build_type = BuildType.PATCH + search_build_output.apply_diff = True + + # Should not find any match + result = _find_matching_build_output(patch, search_build_output) + assert result is None + + # Test with different sanitizer + search_build_output2 = BuildOutput() + search_build_output2.engine = "libfuzzer" + search_build_output2.sanitizer = "tsan" # Different sanitizer + search_build_output2.build_type = BuildType.PATCH + search_build_output2.apply_diff = True + + result2 = _find_matching_build_output(patch, search_build_output2) + assert result2 is None + + # Test with different build_type + search_build_output3 = BuildOutput() + search_build_output3.engine = "libfuzzer" + search_build_output3.sanitizer = "asan" + search_build_output3.build_type = BuildType.COVERAGE # Different build type + search_build_output3.apply_diff = True + + result3 = _find_matching_build_output(patch, search_build_output3) + assert result3 is None + + # Test with different apply_diff + search_build_output4 = BuildOutput() + search_build_output4.engine = "libfuzzer" + search_build_output4.sanitizer = "asan" + search_build_output4.build_type = BuildType.PATCH + search_build_output4.apply_diff = False # Different apply_diff + + result4 = _find_matching_build_output(patch, search_build_output4) + assert result4 is None + + def test_find_matching_build_output_empty_patch(self, submissions): + """Test _find_matching_build_output with patch that has no build outputs.""" + + # Create a patch with no build outputs + patch = SubmissionEntryPatch() + patch.internal_patch_id = "empty-patch" + # patch.build_outputs is empty by default + + # Create search criteria + search_build_output = BuildOutput() + search_build_output.engine = "libfuzzer" + search_build_output.sanitizer = "asan" + search_build_output.build_type = BuildType.PATCH + search_build_output.apply_diff = True + + # Should not find any match + result = _find_matching_build_output(patch, search_build_output) + assert result is None + + @pytest.mark.parametrize( + "engine,sanitizer,build_type,apply_diff,description", + [ + ("afl", "asan", BuildType.PATCH, True, "different engine"), + ("libfuzzer", "msan", BuildType.PATCH, True, "different sanitizer"), + ("libfuzzer", "asan", BuildType.FUZZER, True, "different build_type"), + ("libfuzzer", "asan", BuildType.PATCH, False, "different apply_diff"), + ], + ) + def test_find_matching_build_output_parametrized_no_match( + self, submissions, engine, sanitizer, build_type, apply_diff, description + ): + """Test that _find_matching_build_output requires exact match on all fields (parametrized).""" + + # Create a patch with one build output + patch = SubmissionEntryPatch() + patch.internal_patch_id = "exact-match-patch" + + build_output = BuildOutput() + build_output.engine = "libfuzzer" + build_output.sanitizer = "asan" + build_output.build_type = BuildType.PATCH + build_output.apply_diff = True + build_output.task_dir = "/original/path" + + patch.build_outputs.append(build_output) + + # Create search build output with different field + search_build_output = BuildOutput() + search_build_output.engine = engine + search_build_output.sanitizer = sanitizer + search_build_output.build_type = build_type + search_build_output.apply_diff = apply_diff + + result = _find_matching_build_output(patch, search_build_output) + assert result is None, f"Should not match with {description}" + + def test_find_matching_build_output_exact_match(self, submissions): + """Test that _find_matching_build_output succeeds with exact match.""" + + # Create a patch with one build output + patch = SubmissionEntryPatch() + patch.internal_patch_id = "exact-match-patch" + + build_output = BuildOutput() + build_output.engine = "libfuzzer" + build_output.sanitizer = "asan" + build_output.build_type = BuildType.PATCH + build_output.apply_diff = True + build_output.task_dir = "/original/path" + + patch.build_outputs.append(build_output) + + # Test exact match - should succeed + exact_match = BuildOutput() + exact_match.engine = "libfuzzer" + exact_match.sanitizer = "asan" + exact_match.build_type = BuildType.PATCH + exact_match.apply_diff = True + # Note: task_dir doesn't need to match - it's not part of the matching criteria + exact_match.task_dir = "/different/path" + + result = _find_matching_build_output(patch, exact_match) + assert result is not None + assert result.engine == "libfuzzer" + assert result.sanitizer == "asan" + assert result.build_type == BuildType.PATCH + assert result.apply_diff is True + assert result.task_dir == "/original/path" # Should get the original task_dir + + def test_find_matching_build_output_first_match_returned(self, submissions): + """Test that _find_matching_build_output returns the first matching build output.""" + + # Create a patch with multiple identical build outputs (edge case) + patch = SubmissionEntryPatch() + patch.internal_patch_id = "duplicate-patch" + + # Create two identical build outputs with different task_dirs + build_output1 = BuildOutput() + build_output1.engine = "libfuzzer" + build_output1.sanitizer = "asan" + build_output1.build_type = BuildType.PATCH + build_output1.apply_diff = True + build_output1.task_dir = "/first/path" + + build_output2 = BuildOutput() + build_output2.engine = "libfuzzer" + build_output2.sanitizer = "asan" + build_output2.build_type = BuildType.PATCH + build_output2.apply_diff = True + build_output2.task_dir = "/second/path" + + patch.build_outputs.extend([build_output1, build_output2]) + + # Create search criteria that match both + search_build_output = BuildOutput() + search_build_output.engine = "libfuzzer" + search_build_output.sanitizer = "asan" + search_build_output.build_type = BuildType.PATCH + search_build_output.apply_diff = True + + # Should return the first match + result = _find_matching_build_output(patch, search_build_output) + assert result is not None + assert result.task_dir == "/first/path" # Should be the first one + + def test_get_first_successful_pov_with_passed_pov(self, submissions): + """Test _get_first_successful_pov returns the first passed POV.""" + + # Create a submission entry with multiple POVs using builder + entry = ( + SubmissionEntryBuilder() + .crash( + task_id="test-task", + competition_pov_id="pov-failed", + result=SubmissionResult.FAILED, + ) + .crash( + task_id="test-task", + competition_pov_id="pov-passed-first", + result=SubmissionResult.PASSED, + ) + .crash( + task_id="test-task", + competition_pov_id="pov-passed-second", + result=SubmissionResult.PASSED, + ) + .crash( + task_id="test-task", + competition_pov_id="pov-accepted", + result=SubmissionResult.ACCEPTED, + ) + .build() + ) + + # Should return the first passed POV + result = _get_first_successful_pov(entry) + assert result is not None + assert result.competition_pov_id == "pov-passed-first" + assert result.result == SubmissionResult.PASSED + + def test_get_first_successful_pov_no_passed_povs(self, submissions): + """Test _get_first_successful_pov returns None when no POVs are passed.""" + + # Create a submission entry with POVs that are not passed using builder + entry = ( + SubmissionEntryBuilder() + .crash(task_id="test-task") # No competition_pov_id + .crash( + task_id="test-task", + competition_pov_id="pov-failed", + result=SubmissionResult.FAILED, + ) + .crash( + task_id="test-task", + competition_pov_id="pov-accepted", + result=SubmissionResult.ACCEPTED, + ) + .crash( + task_id="test-task", + competition_pov_id="pov-errored", + result=SubmissionResult.ERRORED, + ) + .build() + ) + + # Should return None since no POVs are passed + result = _get_first_successful_pov(entry) + assert result is None + + def test_get_first_successful_pov_empty_entry(self, submissions): + """Test _get_first_successful_pov returns None for entry with no crashes.""" + + # Create an empty submission entry + entry = SubmissionEntry() + # entry.crashes is empty by default + + # Should return None + result = _get_first_successful_pov(entry) + assert result is None + + def test_get_first_successful_pov_requires_both_id_and_passed_status(self, submissions): + """Test _get_first_successful_pov requires both competition_pov_id and PASSED status.""" + + # Create a submission entry with edge cases + entry = SubmissionEntry() + + # POV with PASSED status but no competition_pov_id + crash1 = CrashWithId() + crash1.crash.crash.target.task_id = "test-task" + # crash1.competition_pov_id is not set + crash1.result = SubmissionResult.PASSED + entry.crashes.append(crash1) + + # POV with competition_pov_id but no result set + crash2 = CrashWithId() + crash2.crash.crash.target.task_id = "test-task" + crash2.competition_pov_id = "pov-no-result" + # crash2.result is not set (default) + entry.crashes.append(crash2) + + # POV with empty competition_pov_id and PASSED status + crash3 = CrashWithId() + crash3.crash.crash.target.task_id = "test-task" + crash3.competition_pov_id = "" # Empty string + crash3.result = SubmissionResult.PASSED + entry.crashes.append(crash3) + + # Should return None since none meet both criteria + result = _get_first_successful_pov(entry) + assert result is None + + def test_get_first_successful_pov_returns_first_match(self, submissions): + """Test _get_first_successful_pov returns the first POV that meets criteria.""" + + # Create a submission entry with multiple passed POVs + entry = SubmissionEntry() + + # First passed POV - should be returned + crash1 = CrashWithId() + crash1.crash.crash.target.task_id = "test-task" + crash1.competition_pov_id = "first-passed-pov" + crash1.result = SubmissionResult.PASSED + entry.crashes.append(crash1) + + # Second passed POV - should not be returned + crash2 = CrashWithId() + crash2.crash.crash.target.task_id = "test-task" + crash2.competition_pov_id = "second-passed-pov" + crash2.result = SubmissionResult.PASSED + entry.crashes.append(crash2) + + # Should return the first one + result = _get_first_successful_pov(entry) + assert result is not None + assert result.competition_pov_id == "first-passed-pov" + assert result.result == SubmissionResult.PASSED + # Verify it's the first POV by checking it's not the second one + assert result.competition_pov_id != "second-passed-pov" + + def test_get_pending_povs_with_accepted_povs(self, submissions): + """Test _get_pending_povs returns all POVs with ACCEPTED status.""" + + # Create a submission entry with multiple POVs using builder + entry = ( + SubmissionEntryBuilder() + .crash(task_id="test-task") # No competition_pov_id - not pending + .crash( + task_id="test-task", + competition_pov_id="pov-accepted-1", + result=SubmissionResult.ACCEPTED, + ) # Should be included + .crash( + task_id="test-task", + competition_pov_id="pov-passed", + result=SubmissionResult.PASSED, + ) # Not pending + .crash( + task_id="test-task", + competition_pov_id="pov-accepted-2", + result=SubmissionResult.ACCEPTED, + ) # Should be included + .crash( + task_id="test-task", + competition_pov_id="pov-failed", + result=SubmissionResult.FAILED, + ) # Not pending + .crash( + task_id="test-task", + competition_pov_id="pov-errored", + result=SubmissionResult.ERRORED, + ) # Not pending + .build() + ) + + # Should return only the accepted POVs + result = _get_pending_pov_submissions(entry) + assert len(result) == 2 + + # Check that both accepted POVs are in the result + pov_ids = [pov.competition_pov_id for pov in result] + assert "pov-accepted-1" in pov_ids + assert "pov-accepted-2" in pov_ids + + # Verify they all have ACCEPTED status + for pov in result: + assert pov.result == SubmissionResult.ACCEPTED + assert pov.competition_pov_id # Should have a POV ID + + def test_get_pending_povs_no_pending_povs(self, submissions): + """Test _get_pending_povs returns empty list when no POVs are pending.""" + + # Create a submission entry with POVs that are not pending + entry = SubmissionEntry() + + # POV with no competition_pov_id + crash1 = CrashWithId() + crash1.crash.crash.target.task_id = "test-task" + # crash1.competition_pov_id is not set + entry.crashes.append(crash1) + + # POV that passed + crash2 = CrashWithId() + crash2.crash.crash.target.task_id = "test-task" + crash2.competition_pov_id = "pov-passed" + crash2.result = SubmissionResult.PASSED + entry.crashes.append(crash2) + + # POV that failed + crash3 = CrashWithId() + crash3.crash.crash.target.task_id = "test-task" + crash3.competition_pov_id = "pov-failed" + crash3.result = SubmissionResult.FAILED + entry.crashes.append(crash3) + + # POV that errored + crash4 = CrashWithId() + crash4.crash.crash.target.task_id = "test-task" + crash4.competition_pov_id = "pov-errored" + crash4.result = SubmissionResult.ERRORED + entry.crashes.append(crash4) + + # Should return empty list + result = _get_pending_pov_submissions(entry) + assert len(result) == 0 + assert result == [] + + def test_get_pending_povs_empty_entry(self, submissions): + """Test _get_pending_povs returns empty list for entry with no crashes.""" + + # Create an empty submission entry + entry = SubmissionEntry() + # entry.crashes is empty by default + + # Should return empty list + result = _get_pending_pov_submissions(entry) + assert len(result) == 0 + assert result == [] + + def test_get_pending_povs_requires_both_id_and_accepted_status(self, submissions): + """Test _get_pending_povs requires both competition_pov_id and ACCEPTED status.""" + + # Create a submission entry with edge cases + entry = SubmissionEntry() + + # POV with ACCEPTED status but no competition_pov_id + crash1 = CrashWithId() + crash1.crash.crash.target.task_id = "test-task" + # crash1.competition_pov_id is not set + crash1.result = SubmissionResult.ACCEPTED + entry.crashes.append(crash1) + + # POV with competition_pov_id but no result explicitly set + # Note: protobuf now defaults result to NONE (value 0), so this will NOT be considered pending + crash2 = CrashWithId() + crash2.crash.crash.target.task_id = "test-task" + crash2.competition_pov_id = "pov-no-result" + # crash2.result is not set (defaults to NONE) + entry.crashes.append(crash2) + + # POV with empty competition_pov_id and ACCEPTED status + crash3 = CrashWithId() + crash3.crash.crash.target.task_id = "test-task" + crash3.competition_pov_id = "" # Empty string + crash3.result = SubmissionResult.ACCEPTED + entry.crashes.append(crash3) + + # POV with both competition_pov_id and explicit ACCEPTED status - should be pending + crash4 = CrashWithId() + crash4.crash.crash.target.task_id = "test-task" + crash4.competition_pov_id = "pov-accepted" + crash4.result = SubmissionResult.ACCEPTED + entry.crashes.append(crash4) + + # Should return only crash4 since it has both competition_pov_id and explicit ACCEPTED status + result = _get_pending_pov_submissions(entry) + assert len(result) == 1 + assert result[0].competition_pov_id == "pov-accepted" + assert result[0].result == SubmissionResult.ACCEPTED + + def test_get_pending_povs_preserves_order(self, submissions): + """Test _get_pending_povs preserves the order of POVs in the entry.""" + + # Create a submission entry with multiple accepted POVs + entry = SubmissionEntry() + + # Add accepted POVs in specific order + crash1 = CrashWithId() + crash1.crash.crash.target.task_id = "test-task" + crash1.competition_pov_id = "first-accepted-pov" + crash1.result = SubmissionResult.ACCEPTED + entry.crashes.append(crash1) + + # Add a non-accepted POV in between + crash2 = CrashWithId() + crash2.crash.crash.target.task_id = "test-task" + crash2.competition_pov_id = "passed-pov" + crash2.result = SubmissionResult.PASSED + entry.crashes.append(crash2) + + # Add another accepted POV + crash3 = CrashWithId() + crash3.crash.crash.target.task_id = "test-task" + crash3.competition_pov_id = "second-accepted-pov" + crash3.result = SubmissionResult.ACCEPTED + entry.crashes.append(crash3) + + # Add another accepted POV + crash4 = CrashWithId() + crash4.crash.crash.target.task_id = "test-task" + crash4.competition_pov_id = "third-accepted-pov" + crash4.result = SubmissionResult.ACCEPTED + entry.crashes.append(crash4) + + # Should return accepted POVs in the same order they appear in the entry + result = _get_pending_pov_submissions(entry) + assert len(result) == 3 + assert result[0].competition_pov_id == "first-accepted-pov" + assert result[1].competition_pov_id == "second-accepted-pov" + assert result[2].competition_pov_id == "third-accepted-pov" + + def test_get_pending_povs_only_accepted_status(self, submissions): + """Test _get_pending_povs only includes ACCEPTED status, not other statuses.""" + + # Create a submission entry with POVs having various statuses + entry = SubmissionEntry() + + # Test all possible SubmissionResult values + statuses_to_test = [ + (SubmissionResult.ACCEPTED, "should-be-included"), + (SubmissionResult.PASSED, "should-not-be-included-passed"), + (SubmissionResult.FAILED, "should-not-be-included-failed"), + (SubmissionResult.ERRORED, "should-not-be-included-errored"), + (SubmissionResult.DEADLINE_EXCEEDED, "should-not-be-included-deadline"), + ] + + for status, pov_id in statuses_to_test: + crash = CrashWithId() + crash.crash.crash.target.task_id = "test-task" + crash.competition_pov_id = pov_id + crash.result = status + entry.crashes.append(crash) + + # Should return only the ACCEPTED POV + result = _get_pending_pov_submissions(entry) + assert len(result) == 1 + assert result[0].competition_pov_id == "should-be-included" + assert result[0].result == SubmissionResult.ACCEPTED + + def test_get_eligible_povs_for_submission_no_pov_id(self, submissions): + """Test _get_eligible_povs_for_submission returns POVs without competition_pov_id.""" + + # Create a submission entry with POVs that don't have competition_pov_id + entry = SubmissionEntry() + + # POV with no competition_pov_id - should be eligible + crash1 = CrashWithId() + crash1.crash.crash.target.task_id = "test-task" + crash1.crash.crash.crash_input_path = "/path/to/crash1.bin" + # crash1.competition_pov_id is not set + entry.crashes.append(crash1) + + # POV with empty competition_pov_id - should be eligible + crash2 = CrashWithId() + crash2.crash.crash.target.task_id = "test-task" + crash2.crash.crash.crash_input_path = "/path/to/crash2.bin" + crash2.competition_pov_id = "" # Empty string + entry.crashes.append(crash2) + + # POV with competition_pov_id and PASSED status - should NOT be eligible + crash3 = CrashWithId() + crash3.crash.crash.target.task_id = "test-task" + crash3.crash.crash.crash_input_path = "/path/to/crash3.bin" + crash3.competition_pov_id = "pov-passed" + crash3.result = SubmissionResult.PASSED + entry.crashes.append(crash3) + + # Should return only the POVs without competition_pov_id + result = _get_eligible_povs_for_submission(entry) + assert len(result) == 2 + + # Verify by crash input paths + returned_paths = [pov.crash.crash.crash_input_path for pov in result] + assert "/path/to/crash1.bin" in returned_paths # No competition_pov_id + assert "/path/to/crash2.bin" in returned_paths # Empty competition_pov_id + assert "/path/to/crash3.bin" not in returned_paths # PASSED status - ineligible + + def test_get_eligible_povs_for_submission_errored_povs(self, submissions): + """Test _get_eligible_povs_for_submission returns POVs with ERRORED status.""" + + # Create a submission entry with POVs in different states + entry = SubmissionEntry() + + # POV with ERRORED status - should be eligible for retry + crash1 = CrashWithId() + crash1.crash.crash.target.task_id = "test-task" + crash1.crash.crash.crash_input_path = "/path/to/errored.bin" + crash1.competition_pov_id = "pov-errored" + crash1.result = SubmissionResult.ERRORED + entry.crashes.append(crash1) + + # POV with FAILED status - should NOT be eligible + crash2 = CrashWithId() + crash2.crash.crash.target.task_id = "test-task" + crash2.crash.crash.crash_input_path = "/path/to/failed.bin" + crash2.competition_pov_id = "pov-failed" + crash2.result = SubmissionResult.FAILED + entry.crashes.append(crash2) + + # POV with PASSED status - should NOT be eligible + crash3 = CrashWithId() + crash3.crash.crash.target.task_id = "test-task" + crash3.crash.crash.crash_input_path = "/path/to/passed.bin" + crash3.competition_pov_id = "pov-passed" + crash3.result = SubmissionResult.PASSED + entry.crashes.append(crash3) + + # POV with ACCEPTED status - should NOT be eligible + crash4 = CrashWithId() + crash4.crash.crash.target.task_id = "test-task" + crash4.crash.crash.crash_input_path = "/path/to/accepted.bin" + crash4.competition_pov_id = "pov-accepted" + crash4.result = SubmissionResult.ACCEPTED + entry.crashes.append(crash4) + + # Should return only the ERRORED POV + result = _get_eligible_povs_for_submission(entry) + assert len(result) == 1 + assert result[0].crash.crash.crash_input_path == "/path/to/errored.bin" + assert result[0].competition_pov_id == "pov-errored" + assert result[0].result == SubmissionResult.ERRORED + + def test_get_eligible_povs_for_submission_mixed_cases(self, submissions): + """Test _get_eligible_povs_for_submission with mix of eligible and ineligible POVs.""" + + # Create a submission entry with mixed POV states using builder + entry = ( + SubmissionEntryBuilder() + .crash(task_id="test-task", crash_input_path="/path/to/no_pov_id.bin") # No POV ID - eligible + .crash( + task_id="test-task", + crash_input_path="/path/to/passed.bin", + competition_pov_id="pov-passed", + result=SubmissionResult.PASSED, + ) # PASSED - ineligible + .crash( + task_id="test-task", + crash_input_path="/path/to/errored.bin", + competition_pov_id="pov-errored", + result=SubmissionResult.ERRORED, + ) # ERRORED - eligible + .crash( + task_id="test-task", + crash_input_path="/path/to/failed.bin", + competition_pov_id="pov-failed", + result=SubmissionResult.FAILED, + ) # FAILED - ineligible + .crash( + task_id="test-task", crash_input_path="/path/to/empty_pov_id.bin", competition_pov_id="" + ) # Empty string - eligible + .crash( + task_id="test-task", + crash_input_path="/path/to/accepted.bin", + competition_pov_id="pov-accepted", + result=SubmissionResult.ACCEPTED, + ) # ACCEPTED - ineligible + .build() + ) + + # Should return only the eligible POVs (crash1, crash3, crash5) + result = _get_eligible_povs_for_submission(entry) + assert len(result) == 3 + + # Verify by crash input paths + returned_paths = [pov.crash.crash.crash_input_path for pov in result] + assert "/path/to/no_pov_id.bin" in returned_paths # No competition_pov_id + assert "/path/to/errored.bin" in returned_paths # ERRORED status + assert "/path/to/empty_pov_id.bin" in returned_paths # Empty competition_pov_id + # Verify ineligible ones are not included + assert "/path/to/passed.bin" not in returned_paths # PASSED status - ineligible + assert "/path/to/failed.bin" not in returned_paths # FAILED status - ineligible + assert "/path/to/accepted.bin" not in returned_paths # ACCEPTED status - ineligible + + def test_get_eligible_povs_for_submission_empty_entry(self, submissions): + """Test _get_eligible_povs_for_submission with entry that has no crashes.""" + + # Create an empty submission entry + entry = SubmissionEntry() + # entry.crashes is empty by default + + # Should return empty list + result = _get_eligible_povs_for_submission(entry) + assert len(result) == 0 + assert result == [] + + def test_get_eligible_povs_for_submission_all_ineligible(self, submissions): + """Test _get_eligible_povs_for_submission when all POVs are ineligible.""" + + # Create a submission entry with only ineligible POVs + entry = SubmissionEntry() + + # POV with PASSED status + crash1 = CrashWithId() + crash1.crash.crash.target.task_id = "test-task" + crash1.competition_pov_id = "pov-passed" + crash1.result = SubmissionResult.PASSED + entry.crashes.append(crash1) + + # POV with FAILED status + crash2 = CrashWithId() + crash2.crash.crash.target.task_id = "test-task" + crash2.competition_pov_id = "pov-failed" + crash2.result = SubmissionResult.FAILED + entry.crashes.append(crash2) + + # POV with ACCEPTED status + crash3 = CrashWithId() + crash3.crash.crash.target.task_id = "test-task" + crash3.competition_pov_id = "pov-accepted" + crash3.result = SubmissionResult.ACCEPTED + entry.crashes.append(crash3) + + # POV with DEADLINE_EXCEEDED status + crash4 = CrashWithId() + crash4.crash.crash.target.task_id = "test-task" + crash4.competition_pov_id = "pov-deadline" + crash4.result = SubmissionResult.DEADLINE_EXCEEDED + entry.crashes.append(crash4) + + # Should return empty list since all POVs are ineligible + result = _get_eligible_povs_for_submission(entry) + assert len(result) == 0 + assert result == [] + + def test_get_eligible_povs_for_submission_all_eligible(self, submissions): + """Test _get_eligible_povs_for_submission when all POVs are eligible.""" + + # Create a submission entry with only eligible POVs + entry = SubmissionEntry() + + # POV with no competition_pov_id + crash1 = CrashWithId() + crash1.crash.crash.target.task_id = "test-task" + crash1.crash.crash.crash_input_path = "/path/to/no_pov_id.bin" + # crash1.competition_pov_id is not set + entry.crashes.append(crash1) + + # POV with ERRORED status + crash2 = CrashWithId() + crash2.crash.crash.target.task_id = "test-task" + crash2.crash.crash.crash_input_path = "/path/to/errored1.bin" + crash2.competition_pov_id = "pov-errored-1" + crash2.result = SubmissionResult.ERRORED + entry.crashes.append(crash2) + + # Another POV with ERRORED status + crash3 = CrashWithId() + crash3.crash.crash.target.task_id = "test-task" + crash3.crash.crash.crash_input_path = "/path/to/errored2.bin" + crash3.competition_pov_id = "pov-errored-2" + crash3.result = SubmissionResult.ERRORED + entry.crashes.append(crash3) + + # POV with empty competition_pov_id + crash4 = CrashWithId() + crash4.crash.crash.target.task_id = "test-task" + crash4.crash.crash.crash_input_path = "/path/to/empty_pov_id.bin" + crash4.competition_pov_id = "" # Empty string + entry.crashes.append(crash4) + + # Should return all POVs since they're all eligible + result = _get_eligible_povs_for_submission(entry) + assert len(result) == 4 + + # Verify by crash input paths + returned_paths = [pov.crash.crash.crash_input_path for pov in result] + assert "/path/to/no_pov_id.bin" in returned_paths # No competition_pov_id + assert "/path/to/errored1.bin" in returned_paths # ERRORED status + assert "/path/to/errored2.bin" in returned_paths # ERRORED status + assert "/path/to/empty_pov_id.bin" in returned_paths # Empty competition_pov_id + + def test_get_eligible_povs_for_submission_preserves_order(self, submissions): + """Test _get_eligible_povs_for_submission preserves order of eligible POVs.""" + + # Create a submission entry with mixed POVs + entry = SubmissionEntry() + + # Add POVs in specific order, mixing eligible and ineligible + crash1 = CrashWithId() # Eligible: no pov_id + crash1.crash.crash.target.task_id = "test-task" + crash1.crash.crash.crash_input_path = "/path/to/first_eligible.bin" + entry.crashes.append(crash1) + + crash2 = CrashWithId() # Ineligible: PASSED + crash2.crash.crash.target.task_id = "test-task" + crash2.crash.crash.crash_input_path = "/path/to/passed.bin" + crash2.competition_pov_id = "pov-passed" + crash2.result = SubmissionResult.PASSED + entry.crashes.append(crash2) + + crash3 = CrashWithId() # Eligible: ERRORED + crash3.crash.crash.target.task_id = "test-task" + crash3.crash.crash.crash_input_path = "/path/to/errored.bin" + crash3.competition_pov_id = "pov-errored" + crash3.result = SubmissionResult.ERRORED + entry.crashes.append(crash3) + + crash4 = CrashWithId() # Ineligible: FAILED + crash4.crash.crash.target.task_id = "test-task" + crash4.crash.crash.crash_input_path = "/path/to/failed.bin" + crash4.competition_pov_id = "pov-failed" + crash4.result = SubmissionResult.FAILED + entry.crashes.append(crash4) + + crash5 = CrashWithId() # Eligible: empty pov_id + crash5.crash.crash.target.task_id = "test-task" + crash5.crash.crash.crash_input_path = "/path/to/empty_pov_id.bin" + crash5.competition_pov_id = "" + entry.crashes.append(crash5) + + # Should return eligible POVs in the same order they appear in the entry + result = _get_eligible_povs_for_submission(entry) + assert len(result) == 3 + + # Check order is preserved by verifying crash input paths + assert result[0].crash.crash.crash_input_path == "/path/to/first_eligible.bin" # First eligible + assert result[1].crash.crash.crash_input_path == "/path/to/errored.bin" # Second eligible (skipping PASSED) + assert result[2].crash.crash.crash_input_path == "/path/to/empty_pov_id.bin" # Third eligible (skipping FAILED) + + def test_find_patch_cancelled_task(self, submissions, mock_task_registry): + """Test _find_patch skips submissions for cancelled tasks.""" + # Create submission with patch using builder + submissions.entries = [ + SubmissionEntryBuilder() + .crash(task_id="cancelled-task") + .patch(internal_patch_id="patch-in-cancelled", patch_content="patch content in cancelled") + .build() + ] + + # Mock task registry to indicate task should stop processing + mock_task_registry.should_stop_processing.return_value = True + + # Test finding patch in cancelled task - should not find it + result = submissions._find_patch("patch-in-cancelled") + assert result is None + + # Verify should_stop_processing was called with correct task_id + mock_task_registry.should_stop_processing.assert_called_with("cancelled-task") + + def test_get_pending_patch_submissions_with_accepted_patches(self, submissions): + """Test _get_pending_patch_submissions returns all patches with ACCEPTED status.""" + + # Create a submission entry with multiple patches + entry = SubmissionEntry() + + # Patch with no competition_patch_id - not pending + patch1 = SubmissionEntryPatch() + patch1.internal_patch_id = "patch-no-id" + patch1.patch = "patch content 1" + # patch1.competition_patch_id is not set + entry.patches.append(patch1) + + # Patch that's accepted - should be included + patch2 = SubmissionEntryPatch() + patch2.internal_patch_id = "patch-accepted-1" + patch2.patch = "patch content 2" + patch2.competition_patch_id = "comp-patch-accepted-1" + patch2.result = SubmissionResult.ACCEPTED + entry.patches.append(patch2) + + # Patch that's passed - not pending + patch3 = SubmissionEntryPatch() + patch3.internal_patch_id = "patch-passed" + patch3.patch = "patch content 3" + patch3.competition_patch_id = "comp-patch-passed" + patch3.result = SubmissionResult.PASSED + entry.patches.append(patch3) + + # Another patch that's accepted - should be included + patch4 = SubmissionEntryPatch() + patch4.internal_patch_id = "patch-accepted-2" + patch4.patch = "patch content 4" + patch4.competition_patch_id = "comp-patch-accepted-2" + patch4.result = SubmissionResult.ACCEPTED + entry.patches.append(patch4) + + # Patch that failed - not pending + patch5 = SubmissionEntryPatch() + patch5.internal_patch_id = "patch-failed" + patch5.patch = "patch content 5" + patch5.competition_patch_id = "comp-patch-failed" + patch5.result = SubmissionResult.FAILED + entry.patches.append(patch5) + + # Patch that errored - not pending + patch6 = SubmissionEntryPatch() + patch6.internal_patch_id = "patch-errored" + patch6.patch = "patch content 6" + patch6.competition_patch_id = "comp-patch-errored" + patch6.result = SubmissionResult.ERRORED + entry.patches.append(patch6) + + # Should return only the accepted patches + result = _get_pending_patch_submissions(entry) + assert len(result) == 2 + + # Check that both accepted patches are in the result + patch_ids = [patch_entry.competition_patch_id for patch_entry in result] + assert "comp-patch-accepted-1" in patch_ids + assert "comp-patch-accepted-2" in patch_ids + + # Verify they all have ACCEPTED status + for patch_entry in result: + assert patch_entry.result == SubmissionResult.ACCEPTED + assert patch_entry.competition_patch_id # Should have a competition patch ID + + def test_get_pending_patch_submissions_no_pending_patches(self, submissions): + """Test _get_pending_patch_submissions returns empty list when no patches are pending.""" + + # Create a submission entry with patches that are not pending + entry = SubmissionEntry() + + # Patch with no competition_patch_id + patch1 = SubmissionEntryPatch() + patch1.internal_patch_id = "patch-no-id" + patch1.patch = "patch content 1" + # patch1.competition_patch_id is not set + entry.patches.append(patch1) + + # Patch that passed + patch2 = SubmissionEntryPatch() + patch2.internal_patch_id = "patch-passed" + patch2.patch = "patch content 2" + patch2.competition_patch_id = "comp-patch-passed" + patch2.result = SubmissionResult.PASSED + entry.patches.append(patch2) + + # Patch that failed + patch3 = SubmissionEntryPatch() + patch3.internal_patch_id = "patch-failed" + patch3.patch = "patch content 3" + patch3.competition_patch_id = "comp-patch-failed" + patch3.result = SubmissionResult.FAILED + entry.patches.append(patch3) + + # Patch that errored + patch4 = SubmissionEntryPatch() + patch4.internal_patch_id = "patch-errored" + patch4.patch = "patch content 4" + patch4.competition_patch_id = "comp-patch-errored" + patch4.result = SubmissionResult.ERRORED + entry.patches.append(patch4) + + # Should return empty list + result = _get_pending_patch_submissions(entry) + assert len(result) == 0 + assert result == [] + + def test_get_pending_patch_submissions_empty_entry(self, submissions): + """Test _get_pending_patch_submissions returns empty list for entry with no patches.""" + + # Create an empty submission entry + entry = SubmissionEntry() + # entry.patches is empty by default + + # Should return empty list + result = _get_pending_patch_submissions(entry) + assert len(result) == 0 + assert result == [] + + def test_get_pending_patch_submissions_requires_both_id_and_accepted_status(self, submissions): + """Test _get_pending_patch_submissions requires both competition_patch_id and ACCEPTED status.""" + + # Create a submission entry with edge cases + entry = SubmissionEntry() + + # Patch with ACCEPTED status but no competition_patch_id + patch1 = SubmissionEntryPatch() + patch1.internal_patch_id = "patch-accepted-no-id" + patch1.patch = "patch content 1" + # patch1.competition_patch_id is not set + patch1.result = SubmissionResult.ACCEPTED + entry.patches.append(patch1) + + # Patch with competition_patch_id but no result explicitly set + # Note: protobuf now defaults result to NONE (value 0), so this will NOT be considered pending + patch2 = SubmissionEntryPatch() + patch2.internal_patch_id = "patch-id-no-result" + patch2.patch = "patch content 2" + patch2.competition_patch_id = "comp-patch-no-result" + # patch2.result is not set (defaults to NONE) + entry.patches.append(patch2) + + # Patch with empty competition_patch_id and ACCEPTED status + patch3 = SubmissionEntryPatch() + patch3.internal_patch_id = "patch-empty-id" + patch3.patch = "patch content 3" + patch3.competition_patch_id = "" # Empty string + patch3.result = SubmissionResult.ACCEPTED + entry.patches.append(patch3) + + # Patch with both competition_patch_id and explicit ACCEPTED status - should be pending + patch4 = SubmissionEntryPatch() + patch4.internal_patch_id = "patch-valid" + patch4.patch = "patch content 4" + patch4.competition_patch_id = "comp-patch-accepted" + patch4.result = SubmissionResult.ACCEPTED + entry.patches.append(patch4) + + # Should return only patch4 since it has both competition_patch_id and explicit ACCEPTED status + result = _get_pending_patch_submissions(entry) + assert len(result) == 1 + assert result[0].competition_patch_id == "comp-patch-accepted" + assert result[0].result == SubmissionResult.ACCEPTED + + def test_get_pending_patch_submissions_preserves_order(self, submissions): + """Test _get_pending_patch_submissions preserves the order of patches in the entry.""" + + # Create a submission entry with multiple accepted patches + entry = SubmissionEntry() + + # Add accepted patches in specific order + patch1 = SubmissionEntryPatch() + patch1.internal_patch_id = "first-accepted-patch" + patch1.patch = "patch content 1" + patch1.competition_patch_id = "first-accepted-patch-id" + patch1.result = SubmissionResult.ACCEPTED + entry.patches.append(patch1) + + # Add a non-accepted patch in between + patch2 = SubmissionEntryPatch() + patch2.internal_patch_id = "passed-patch" + patch2.patch = "patch content 2" + patch2.competition_patch_id = "passed-patch-id" + patch2.result = SubmissionResult.PASSED + entry.patches.append(patch2) + + # Add another accepted patch + patch3 = SubmissionEntryPatch() + patch3.internal_patch_id = "second-accepted-patch" + patch3.patch = "patch content 3" + patch3.competition_patch_id = "second-accepted-patch-id" + patch3.result = SubmissionResult.ACCEPTED + entry.patches.append(patch3) + + # Add another accepted patch + patch4 = SubmissionEntryPatch() + patch4.internal_patch_id = "third-accepted-patch" + patch4.patch = "patch content 4" + patch4.competition_patch_id = "third-accepted-patch-id" + patch4.result = SubmissionResult.ACCEPTED + entry.patches.append(patch4) + + # Should return accepted patches in the same order they appear in the entry + result = _get_pending_patch_submissions(entry) + assert len(result) == 3 + assert result[0].competition_patch_id == "first-accepted-patch-id" + assert result[1].competition_patch_id == "second-accepted-patch-id" + assert result[2].competition_patch_id == "third-accepted-patch-id" + + def test_get_pending_patch_submissions_only_accepted_status(self, submissions): + """Test _get_pending_patch_submissions only includes ACCEPTED status, not other statuses.""" + + # Create a submission entry with patches having various statuses + entry = SubmissionEntry() + + # Test all possible SubmissionResult values + statuses_to_test = [ + (SubmissionResult.ACCEPTED, "should-be-included"), + (SubmissionResult.PASSED, "should-not-be-included-passed"), + (SubmissionResult.FAILED, "should-not-be-included-failed"), + (SubmissionResult.ERRORED, "should-not-be-included-errored"), + (SubmissionResult.DEADLINE_EXCEEDED, "should-not-be-included-deadline"), + ] + + for status, patch_id in statuses_to_test: + patch = SubmissionEntryPatch() + patch.internal_patch_id = f"patch-{patch_id}" + patch.patch = f"patch content for {patch_id}" + patch.competition_patch_id = patch_id + patch.result = status + entry.patches.append(patch) + + # Should return only the ACCEPTED patch + result = _get_pending_patch_submissions(entry) + assert len(result) == 1 + assert result[0].competition_patch_id == "should-be-included" + assert result[0].result == SubmissionResult.ACCEPTED + + def test_get_pending_patch_submissions_multiple_criteria_combinations(self, submissions): + """Test _get_pending_patch_submissions with various combinations of criteria.""" + + # Create a submission entry with patches covering edge cases using builder + entry = ( + SubmissionEntryBuilder() + .patch( + internal_patch_id="valid-pending", + patch_content="patch content 1", + competition_patch_id="valid-pending-id", + result=SubmissionResult.ACCEPTED, + ) + .patch( + internal_patch_id="accepted-empty-id", + patch_content="patch content 2", + competition_patch_id="", # Empty - should not be included + result=SubmissionResult.ACCEPTED, + ) + .patch( + internal_patch_id="passed-with-id", + patch_content="patch content 3", + competition_patch_id="passed-patch-id", + result=SubmissionResult.PASSED, + ) # PASSED - should not be included + .patch( + internal_patch_id="failed-with-id", + patch_content="patch content 4", + competition_patch_id="failed-patch-id", + result=SubmissionResult.FAILED, + ) # FAILED - should not be included + .patch( + internal_patch_id="another-valid-pending", + patch_content="patch content 5", + competition_patch_id="another-valid-pending-id", + result=SubmissionResult.ACCEPTED, + ) + .build() + ) + + # Should return only the two valid pending patches + result = _get_pending_patch_submissions(entry) + assert len(result) == 2 + + # Verify the correct patches are returned + returned_ids = [patch_entry.competition_patch_id for patch_entry in result] + assert "valid-pending-id" in returned_ids + assert "another-valid-pending-id" in returned_ids + + # Verify excluded patches are not returned + assert "passed-patch-id" not in returned_ids + assert "failed-patch-id" not in returned_ids + assert "" not in returned_ids # Empty string should not be in results + + def test_get_available_sarifs_for_matching_with_available_sarifs(self, submissions): + """Test _get_available_sarifs_for_matching returns SARIFs that haven't been used.""" + # Mock the sarif_store to return some SARIFs for the task + mock_sarif1 = Mock() + mock_sarif1.sarif_id = "sarif-1" + mock_sarif2 = Mock() + mock_sarif2.sarif_id = "sarif-2" + mock_sarif3 = Mock() + mock_sarif3.sarif_id = "sarif-3" + + submissions.sarif_store = Mock() + submissions.sarif_store.get_by_task_id.return_value = [mock_sarif1, mock_sarif2, mock_sarif3] + + # Create some submissions with bundles that use some SARIFs + submission1 = SubmissionEntry() + # Add crash to submission1 + crash_with_id1 = CrashWithId() + crash_with_id1.crash.crash.target.task_id = "test-task" + submission1.crashes.append(crash_with_id1) + + # Create bundle with sarif-1 (making it unavailable) + + bundle1 = Bundle() + bundle1.competition_sarif_id = "sarif-1" + submission1.bundles.append(bundle1) + + submission2 = SubmissionEntry() + # Add crash to submission2 + crash_with_id2 = CrashWithId() + crash_with_id2.crash.crash.target.task_id = "test-task" + submission2.crashes.append(crash_with_id2) + + # Create bundle with sarif-3 (making it unavailable) + bundle2 = Bundle() + bundle2.competition_sarif_id = "sarif-3" + submission2.bundles.append(bundle2) + + submissions.entries = [submission1, submission2] + + # Call the method + result = submissions._get_available_sarifs_for_matching("test-task") + + # Should return only sarif-2 (sarif-1 and sarif-3 are used) + assert len(result) == 1 + assert result[0].sarif_id == "sarif-2" + + def test_get_available_sarifs_for_matching_no_sarifs_for_task(self, submissions): + """Test _get_available_sarifs_for_matching returns empty list when no SARIFs exist for task.""" + # Mock the sarif_store to return no SARIFs for the task + submissions.sarif_store = Mock() + submissions.sarif_store.get_by_task_id.return_value = [] + + # Call the method + result = submissions._get_available_sarifs_for_matching("test-task") + + # Should return empty list + assert result == [] + + def test_get_available_sarifs_for_matching_all_sarifs_used(self, submissions): + """Test _get_available_sarifs_for_matching returns empty list when all SARIFs are used.""" + # Mock the sarif_store to return some SARIFs + mock_sarif1 = Mock() + mock_sarif1.sarif_id = "sarif-1" + mock_sarif2 = Mock() + mock_sarif2.sarif_id = "sarif-2" + + submissions.sarif_store = Mock() + submissions.sarif_store.get_by_task_id.return_value = [mock_sarif1, mock_sarif2] + + # Create submissions with bundles that use all SARIFs + submission1 = SubmissionEntry() + # Add crash to submission1 + crash_with_id1 = CrashWithId() + crash_with_id1.crash.crash.target.task_id = "test-task" + submission1.crashes.append(crash_with_id1) + + bundle1 = Bundle() + bundle1.competition_sarif_id = "sarif-1" + submission1.bundles.append(bundle1) + + submission2 = SubmissionEntry() + # Add crash to submission2 + crash_with_id2 = CrashWithId() + crash_with_id2.crash.crash.target.task_id = "test-task" + submission2.crashes.append(crash_with_id2) + + bundle2 = Bundle() + bundle2.competition_sarif_id = "sarif-2" + submission2.bundles.append(bundle2) + + submissions.entries = [submission1, submission2] + + # Call the method + result = submissions._get_available_sarifs_for_matching("test-task") + + # Should return empty list since all SARIFs are used + assert result == [] + + def test_get_available_sarifs_for_matching_no_bundles_exist(self, submissions): + """Test _get_available_sarifs_for_matching returns all SARIFs when no bundles exist.""" + # Mock the sarif_store to return some SARIFs + mock_sarif1 = Mock() + mock_sarif1.sarif_id = "sarif-1" + mock_sarif2 = Mock() + mock_sarif2.sarif_id = "sarif-2" + + submissions.sarif_store = Mock() + submissions.sarif_store.get_by_task_id.return_value = [mock_sarif1, mock_sarif2] + + # Create submissions without bundles + submission1 = SubmissionEntry() + # Add crash to submission1 + crash_with_id1 = CrashWithId() + crash_with_id1.crash.crash.target.task_id = "test-task" + submission1.crashes.append(crash_with_id1) + # No bundles + + submissions.entries = [submission1] + + # Call the method + result = submissions._get_available_sarifs_for_matching("test-task") + + # Should return all SARIFs since none are used + assert len(result) == 2 + sarif_ids = [sarif.sarif_id for sarif in result] + assert "sarif-1" in sarif_ids + assert "sarif-2" in sarif_ids + + def test_get_available_sarifs_for_matching_empty_sarif_ids_ignored(self, submissions): + """Test _get_available_sarifs_for_matching ignores bundles with empty competition_sarif_id.""" + # Mock the sarif_store to return some SARIFs + mock_sarif1 = Mock() + mock_sarif1.sarif_id = "sarif-1" + mock_sarif2 = Mock() + mock_sarif2.sarif_id = "sarif-2" + + submissions.sarif_store = Mock() + submissions.sarif_store.get_by_task_id.return_value = [mock_sarif1, mock_sarif2] + + # Create submissions with bundles that have empty/no SARIF IDs + submission1 = SubmissionEntry() + # Add crash to submission1 + crash_with_id1 = CrashWithId() + crash_with_id1.crash.crash.target.task_id = "test-task" + submission1.crashes.append(crash_with_id1) + + bundle1 = Bundle() + bundle1.competition_sarif_id = "" # Empty string + submission1.bundles.append(bundle1) + + submission2 = SubmissionEntry() + # Add crash to submission2 + crash_with_id2 = CrashWithId() + crash_with_id2.crash.crash.target.task_id = "test-task" + submission2.crashes.append(crash_with_id2) + + bundle2 = Bundle() + # competition_sarif_id is not set (defaults to empty) + submission2.bundles.append(bundle2) + + submissions.entries = [submission1, submission2] + + # Call the method + result = submissions._get_available_sarifs_for_matching("test-task") + + # Should return all SARIFs since empty SARIF IDs are ignored + assert len(result) == 2 + sarif_ids = [sarif.sarif_id for sarif in result] + assert "sarif-1" in sarif_ids + assert "sarif-2" in sarif_ids + + def test_get_available_sarifs_for_matching_stopped_submissions_release_sarifs(self, submissions): + """Test _get_available_sarifs_for_matching excludes SARIFs from stopped submissions (makes them available again).""" + # Mock the sarif_store to return some SARIFs + mock_sarif1 = Mock() + mock_sarif1.sarif_id = "sarif-1" + mock_sarif2 = Mock() + mock_sarif2.sarif_id = "sarif-2" + + submissions.sarif_store = Mock() + submissions.sarif_store.get_by_task_id.return_value = [mock_sarif1, mock_sarif2] + + # Create a stopped submission with a bundle using sarif-1 + submission1 = SubmissionEntry() + submission1.stop = True # Mark as stopped + # Add crash to submission1 + crash_with_id1 = CrashWithId() + crash_with_id1.crash.crash.target.task_id = "test-task" + submission1.crashes.append(crash_with_id1) + + bundle1 = Bundle() + bundle1.competition_sarif_id = "sarif-1" + submission1.bundles.append(bundle1) + + submissions.entries = [submission1] + + # Call the method + result = submissions._get_available_sarifs_for_matching("test-task") + + # Should return both SARIFs since stopped submission releases sarif-1 for reuse + assert len(result) == 2 + sarif_ids = [sarif.sarif_id for sarif in result] + assert "sarif-1" in sarif_ids + assert "sarif-2" in sarif_ids + + def test_get_available_sarifs_for_matching_mixed_scenarios(self, submissions): + """Test _get_available_sarifs_for_matching with a complex mix of scenarios.""" + # Mock the sarif_store to return multiple SARIFs + mock_sarifs = [] + for i in range(1, 6): # sarif-1 through sarif-5 + mock_sarif = Mock() + mock_sarif.sarif_id = f"sarif-{i}" + mock_sarifs.append(mock_sarif) + + submissions.sarif_store = Mock() + submissions.sarif_store.get_by_task_id.return_value = mock_sarifs + + # Create various submissions with different bundle states using builder + submissions.entries = [ + # Submission 1: Uses sarif-1 + ( + SubmissionEntryBuilder() + .crash(task_id="test-task") + .bundle(competition_sarif_id="sarif-1", task_id="test-task") + .build() + ), + # Submission 2: Has bundle with empty SARIF ID (doesn't use any SARIF) + ( + SubmissionEntryBuilder() + .crash(task_id="test-task") + .bundle(competition_sarif_id="", task_id="test-task") + .build() + ), + # Submission 3: Uses sarif-3, but is stopped + ( + SubmissionEntryBuilder() + .crash(task_id="test-task") + .bundle(competition_sarif_id="sarif-3", task_id="test-task") + .stopped() + .build() + ), + # Submission 4: No bundles + (SubmissionEntryBuilder().crash(task_id="test-task").build()), + # Submission 5: Uses sarif-5 + ( + SubmissionEntryBuilder() + .crash(task_id="test-task") + .bundle(competition_sarif_id="sarif-5", task_id="test-task") + .build() + ), + ] + + # Call the method + result = submissions._get_available_sarifs_for_matching("test-task") + + # Should return sarif-2, sarif-3, and sarif-4 (sarif-1 and sarif-5 are used by active submissions, sarif-3 is released by stopped submission) + assert len(result) == 3 + sarif_ids = [sarif.sarif_id for sarif in result] + assert "sarif-2" in sarif_ids + assert "sarif-3" in sarif_ids # Available because submission3 is stopped + assert "sarif-4" in sarif_ids + # Verify SARIFs from active submissions are not returned + assert "sarif-1" not in sarif_ids # Used by active submission1 + assert "sarif-5" not in sarif_ids # Used by active submission5 + + def test_get_available_sarifs_for_matching_multiple_bundles_per_submission(self, submissions): + """Test _get_available_sarifs_for_matching handles multiple bundles per submission.""" + # Mock the sarif_store to return some SARIFs + mock_sarif1 = Mock() + mock_sarif1.sarif_id = "sarif-1" + mock_sarif2 = Mock() + mock_sarif2.sarif_id = "sarif-2" + mock_sarif3 = Mock() + mock_sarif3.sarif_id = "sarif-3" + + submissions.sarif_store = Mock() + submissions.sarif_store.get_by_task_id.return_value = [mock_sarif1, mock_sarif2, mock_sarif3] + + # Create submission with multiple bundles + submission1 = SubmissionEntry() + # Add crash to submission1 + crash_with_id1 = CrashWithId() + crash_with_id1.crash.crash.target.task_id = "test-task" + submission1.crashes.append(crash_with_id1) + + # First bundle uses sarif-1 + bundle1 = Bundle() + bundle1.competition_sarif_id = "sarif-1" + submission1.bundles.append(bundle1) + + # Second bundle uses sarif-3 + bundle2 = Bundle() + bundle2.competition_sarif_id = "sarif-3" + submission1.bundles.append(bundle2) + + submissions.entries = [submission1] + + # Call the method + result = submissions._get_available_sarifs_for_matching("test-task") + + # Should return only sarif-2 (sarif-1 and sarif-3 are used) + assert len(result) == 1 + assert result[0].sarif_id == "sarif-2" + + def test_enumerate_task_submissions_with_matching_task(self, submissions): + """Test _enumerate_task_submissions returns submissions for the specified task.""" + # Create submissions for different tasks using builder + submissions.entries = [ + SubmissionEntryBuilder().crash(task_id="task-1").build(), + SubmissionEntryBuilder().crash(task_id="task-2").build(), + SubmissionEntryBuilder().crash(task_id="task-1").build(), # Same as first + ] + + # Call the method for task-1 + result = list(submissions._enumerate_task_submissions("task-1")) + + # Should return submissions at indices 0 and 2 with their indices + assert len(result) == 2 + indices = [i for i, _ in result] + entries = [e for _, e in result] + + assert 0 in indices # first submission + assert 2 in indices # third submission + assert submissions.entries[0] in entries + assert submissions.entries[2] in entries + assert submissions.entries[1] not in entries # Different task + + def test_enumerate_task_submissions_no_matching_task(self, submissions): + """Test _enumerate_task_submissions returns empty when no submissions match the task.""" + # Create submissions for different tasks using builder + submissions.entries = [ + SubmissionEntryBuilder().crash(task_id="task-1").build(), + SubmissionEntryBuilder().crash(task_id="task-2").build(), + ] + + # Call the method for non-existent task + result = list(submissions._enumerate_task_submissions("non-existent-task")) + + # Should return empty list + assert len(result) == 0 + assert result == [] + + def test_enumerate_task_submissions_empty_entries(self, submissions): + """Test _enumerate_task_submissions returns empty when no submissions exist.""" + submissions.entries = [] + + # Call the method + result = list(submissions._enumerate_task_submissions("any-task")) + + # Should return empty list + assert len(result) == 0 + assert result == [] + + def test_enumerate_task_submissions_skips_stopped_submissions(self, submissions): + """Test _enumerate_task_submissions skips stopped submissions.""" + # Create submissions for the same task using builder + submissions.entries = [ + SubmissionEntryBuilder().crash(task_id="test-task").build(), + SubmissionEntryBuilder().crash(task_id="test-task").stopped().build(), + SubmissionEntryBuilder().crash(task_id="test-task").build(), + ] + + # Call the method + result = list(submissions._enumerate_task_submissions("test-task")) + + # Should return only submission1 and submission3 (skipping stopped submission2) + assert len(result) == 2 + indices = [i for i, _ in result] + entries = [e for _, e in result] + + assert 0 in indices # first submission + assert 2 in indices # third submission + assert 1 not in indices # second submission (stopped) + assert submissions.entries[0] in entries + assert submissions.entries[2] in entries + assert submissions.entries[1] not in entries # Stopped submission + + def test_enumerate_task_submissions_skips_cancelled_tasks(self, submissions, mock_task_registry): + """Test _enumerate_task_submissions skips submissions for cancelled tasks.""" + # Create submissions for different tasks + submission1 = SubmissionEntry() + crash_with_id1 = CrashWithId() + crash_with_id1.crash.crash.target.task_id = "active-task" + submission1.crashes.append(crash_with_id1) + + submission2 = SubmissionEntry() + crash_with_id2 = CrashWithId() + crash_with_id2.crash.crash.target.task_id = "cancelled-task" + submission2.crashes.append(crash_with_id2) + + submission3 = SubmissionEntry() + crash_with_id3 = CrashWithId() + crash_with_id3.crash.crash.target.task_id = "cancelled-task" + submission3.crashes.append(crash_with_id3) + + submissions.entries = [submission1, submission2, submission3] + + # Mock task registry to indicate cancelled-task should stop processing + def should_stop_side_effect(task_id): + return task_id == "cancelled-task" + + mock_task_registry.should_stop_processing.side_effect = should_stop_side_effect + + # Call the method for cancelled-task + result = list(submissions._enumerate_task_submissions("cancelled-task")) + + # Should return empty list since task is cancelled + assert len(result) == 0 + assert result == [] + + # Verify should_stop_processing was called for the task + mock_task_registry.should_stop_processing.assert_called_with("cancelled-task") + + def test_enumerate_task_submissions_preserves_order(self, submissions): + """Test _enumerate_task_submissions preserves the order of submissions.""" + # Create multiple submissions for the same task in specific order + submission1 = SubmissionEntry() + crash_with_id1 = CrashWithId() + crash_with_id1.crash.crash.target.task_id = "test-task" + crash_with_id1.crash.crash.crash_input_path = "/path/to/crash1.bin" + submission1.crashes.append(crash_with_id1) + + submission2 = SubmissionEntry() + crash_with_id2 = CrashWithId() + crash_with_id2.crash.crash.target.task_id = "other-task" # Different task + submission2.crashes.append(crash_with_id2) + + submission3 = SubmissionEntry() + crash_with_id3 = CrashWithId() + crash_with_id3.crash.crash.target.task_id = "test-task" + crash_with_id3.crash.crash.crash_input_path = "/path/to/crash3.bin" + submission3.crashes.append(crash_with_id3) + + submission4 = SubmissionEntry() + crash_with_id4 = CrashWithId() + crash_with_id4.crash.crash.target.task_id = "test-task" + crash_with_id4.crash.crash.crash_input_path = "/path/to/crash4.bin" + submission4.crashes.append(crash_with_id4) + + submissions.entries = [submission1, submission2, submission3, submission4] + + # Call the method + result = list(submissions._enumerate_task_submissions("test-task")) + + # Should return submissions in the same order they appear in entries + assert len(result) == 3 + assert result[0][0] == 0 # submission1 at index 0 + assert result[1][0] == 2 # submission3 at index 2 + assert result[2][0] == 3 # submission4 at index 3 + + # Verify the entries are in correct order + assert result[0][1].crashes[0].crash.crash.crash_input_path == "/path/to/crash1.bin" + assert result[1][1].crashes[0].crash.crash.crash_input_path == "/path/to/crash3.bin" + assert result[2][1].crashes[0].crash.crash.crash_input_path == "/path/to/crash4.bin" + + def test_enumerate_task_submissions_mixed_conditions(self, submissions, mock_task_registry): + """Test _enumerate_task_submissions with a mix of conditions.""" + # Create submissions with various conditions using builder + submissions.entries = [ + SubmissionEntryBuilder().crash(task_id="target-task").build(), # Active, matching task + SubmissionEntryBuilder().crash(task_id="target-task").stopped().build(), # Stopped, matching task + SubmissionEntryBuilder().crash(task_id="other-task").build(), # Active, different task + SubmissionEntryBuilder().crash(task_id="target-task").build(), # Active, matching task + SubmissionEntryBuilder().crash(task_id="cancelled-task").build(), # Active, cancelled task + ] + + # Mock task registry + def should_stop_side_effect(task_id): + return task_id == "cancelled-task" + + mock_task_registry.should_stop_processing.side_effect = should_stop_side_effect + + # Call the method for target-task + result = list(submissions._enumerate_task_submissions("target-task")) + + # Should return only submission1 and submission4 (active, matching task) + assert len(result) == 2 + indices = [i for i, _ in result] + entries = [e for _, e in result] + + assert 0 in indices # first submission + assert 3 in indices # fourth submission + assert submissions.entries[0] in entries + assert submissions.entries[3] in entries + # Verify excluded submissions + assert submissions.entries[1] not in entries # Stopped + assert submissions.entries[2] not in entries # Different task + assert submissions.entries[4] not in entries # Different task + + def test_enumerate_task_submissions_returns_correct_indices(self, submissions): + """Test _enumerate_task_submissions returns correct indices from original entries list.""" + # Create submissions with gaps (some will be filtered out) using builder + submissions.entries = [ + SubmissionEntryBuilder().crash(task_id="other-task").build(), # Index 0 - different task + SubmissionEntryBuilder().crash(task_id="target-task").build(), # Index 1 - target task + SubmissionEntryBuilder().crash(task_id="target-task").stopped().build(), # Index 2 - stopped + SubmissionEntryBuilder().crash(task_id="target-task").build(), # Index 3 - target task + ] + + # Call the method + result = list(submissions._enumerate_task_submissions("target-task")) + + # Should return submissions at indices 1 and 3 + assert len(result) == 2 + assert result[0][0] == 1 # second submission + assert result[1][0] == 3 # fourth submission + assert result[0][1] is submissions.entries[1] + assert result[1][1] is submissions.entries[3] + + +# Tests for state transitions - Updated for current data-driven architecture +class TestStateTransitions: + def test_pov_submission_and_status_update(self, submissions, sample_submission_entry, mock_competition_api): + """Test POV submission and status updates work correctly""" + # Create entry with crash but no POV ID (initial state) + entry = SubmissionEntry() + crash_with_id = CrashWithId() + crash_with_id.crash.CopyFrom(sample_submission_entry.crashes[0].crash) + entry.crashes.append(crash_with_id) + submissions.entries = [entry] + + # Mock competition API to return successful POV submission + mock_competition_api.submit_pov.return_value = ("test-pov-123", SubmissionResult.ACCEPTED) + + # Process cycle - should submit POV + submissions.process_cycle() + + # Verify POV was submitted + mock_competition_api.submit_pov.assert_called_once() + assert entry.crashes[0].competition_pov_id == "test-pov-123" + assert entry.crashes[0].result == SubmissionResult.ACCEPTED + + # Mock status update to PASSED + mock_competition_api.get_pov_status.return_value = SubmissionResult.PASSED + + # Process another cycle - should update status + submissions.process_cycle() + + # Verify status was updated + mock_competition_api.get_pov_status.assert_called_once_with( + entry.crashes[0].crash.crash.target.task_id, "test-pov-123" + ) + assert entry.crashes[0].result == SubmissionResult.PASSED + + def test_patch_request_when_pov_passes(self, submissions, sample_submission_entry, mock_competition_api): + """Test that patch is requested when POV passes""" + # Create entry with passed POV + entry = SubmissionEntry() + crash_with_id = CrashWithId() + crash_with_id.crash.CopyFrom(sample_submission_entry.crashes[0].crash) + crash_with_id.competition_pov_id = "test-pov-123" + crash_with_id.result = SubmissionResult.PASSED + entry.crashes.append(crash_with_id) + submissions.entries = [entry] + + # Mock QueueFactory for patch requests + queue_mock = MagicMock() + with patch("buttercup.orchestrator.scheduler.submissions.QueueFactory") as queue_factory_mock: + queue_factory_mock.return_value.create.return_value = queue_mock + + # Process cycle - should request patch + submissions.process_cycle() + + # Verify patch request was made + queue_mock.push.assert_called() + assert len(entry.patches) == 1 + assert entry.patches[0].internal_patch_id # Should have generated UUID + + def test_patch_request_waits_for_mitigation_merge(self, submissions, sample_submission_entry, mock_competition_api): + """Test that patch request waits when _should_wait_for_patch_mitigation_merge returns True""" + # Create entry with passed POV that needs patch using the builder + entry = ( + SubmissionEntryBuilder() + .crash( + task_id="test-task-123", + competition_pov_id="test-pov-123", + result=SubmissionResult.PASSED, + harness_name="test_harness", + sanitizer="asan", + engine="libfuzzer", + ) + .build() + ) + submissions.entries = [entry] + + # Mock QueueFactory for patch requests + queue_mock = MagicMock() + with patch("buttercup.orchestrator.scheduler.submissions.QueueFactory") as queue_factory_mock: + queue_factory_mock.return_value.create.return_value = queue_mock + + # Mock _should_wait_for_patch_mitigation_merge to return True (should wait) + with patch.object(submissions, "_should_wait_for_patch_mitigation_merge", return_value=True) as mock_wait: + # Process cycle - should NOT request patch due to waiting + submissions.process_cycle() + + # Verify _should_wait_for_patch_mitigation_merge was called + mock_wait.assert_called_once_with(0, entry) + + # Verify NO patch request was made + queue_mock.push.assert_not_called() + assert len(entry.patches) == 0 # No patches should be added + + def test_patch_request_proceeds_when_no_mitigation_wait_needed( self, submissions, sample_submission_entry, mock_competition_api ): - # Setup entry in WAIT_PATCH_PASS state - sample_submission_entry.state = SubmissionEntry.WAIT_PATCH_PASS - sample_submission_entry.competition_patch_id = "test-patch-123" - sample_submission_entry.patch_idx = 0 - sample_submission_entry.patches.append(SubmissionEntryPatch(patch="test patch content")) - sample_submission_entry.patches.append( - SubmissionEntryPatch(patch="another test patch") - ) # Add a second patch for advancement - submissions.entries = [sample_submission_entry] - - # Mock competition API to return FAILED - mock_competition_api.get_patch_status.return_value = TypesSubmissionStatus.SubmissionStatusFailed - - # Mock the _advance_patch_idx function to avoid AttributeError - with patch("buttercup.orchestrator.scheduler.submissions._advance_patch_idx") as mock_advance: - # Simulate process_cycle - submissions.process_cycle() - - # Verify advance_patch_idx was called - mock_advance.assert_called_once_with(sample_submission_entry) - - # Verify state transition - assert sample_submission_entry.state == SubmissionEntry.SUBMIT_PATCH - - def test_wait_patch_pass_when_errored( - self, submissions, sample_submission_entry, mock_competition_api, mock_task_registry - ): - # Setup entry in WAIT_PATCH_PASS state - sample_submission_entry.state = SubmissionEntry.WAIT_PATCH_PASS - sample_submission_entry.competition_patch_id = "test-patch-123" - sample_submission_entry.patch_idx = 0 - sample_submission_entry.patches.append(SubmissionEntryPatch(patch="test patch content")) - submissions.entries = [sample_submission_entry] - - # Mock competition API to return ERRORED - mock_competition_api.get_patch_status.return_value = TypesSubmissionStatus.SubmissionStatusErrored - - # Simulate process_cycle - submissions.process_cycle() - - # Verify state transition - assert sample_submission_entry.state == SubmissionEntry.SUBMIT_PATCH - # Verify mark_errored was called - mock_task_registry.mark_errored.assert_called_once_with(sample_submission_entry.crashes[0].crash.target.task_id) - - def test_submit_bundle_successful(self, submissions, sample_submission_entry, mock_competition_api): - # Setup entry in SUBMIT_BUNDLE state - sample_submission_entry.state = SubmissionEntry.SUBMIT_BUNDLE - sample_submission_entry.pov_id = "test-pov-123" - sample_submission_entry.competition_patch_id = "test-patch-456" - submissions.entries = [sample_submission_entry] - - # Mock competition API to return successful bundle submission - mock_competition_api.submit_bundle.return_value = ( - "test-bundle-789", - TypesSubmissionStatus.SubmissionStatusAccepted, + """Test that patch request proceeds when _should_wait_for_patch_mitigation_merge returns False""" + # Create entry with passed POV that needs patch using the builder + entry = ( + SubmissionEntryBuilder() + .crash( + task_id="test-task-123", + competition_pov_id="test-pov-123", + result=SubmissionResult.PASSED, + harness_name="test_harness", + sanitizer="asan", + engine="libfuzzer", + ) + .build() ) + submissions.entries = [entry] - # Simulate process_cycle - submissions.process_cycle() + # Mock QueueFactory for patch requests + queue_mock = MagicMock() + with patch("buttercup.orchestrator.scheduler.submissions.QueueFactory") as queue_factory_mock: + queue_factory_mock.return_value.create.return_value = queue_mock - # Verify state transition and bundle ID - assert sample_submission_entry.state == SubmissionEntry.SUBMIT_MATCHING_SARIF - assert sample_submission_entry.bundle_id == "test-bundle-789" + # Mock _should_wait_for_patch_mitigation_merge to return False (no need to wait) + with patch.object(submissions, "_should_wait_for_patch_mitigation_merge", return_value=False) as mock_wait: + # Process cycle - should request patch + submissions.process_cycle() - def test_submit_bundle_to_stop_when_failed(self, submissions, sample_submission_entry, mock_competition_api): - # Setup entry in SUBMIT_BUNDLE state - sample_submission_entry.state = SubmissionEntry.SUBMIT_BUNDLE - sample_submission_entry.pov_id = "test-pov-123" - sample_submission_entry.competition_patch_id = "test-patch-456" - submissions.entries = [sample_submission_entry] + # Verify _should_wait_for_patch_mitigation_merge was called + mock_wait.assert_called_once_with(0, entry) - # Mock competition API to return failed bundle submission - mock_competition_api.submit_bundle.return_value = (None, TypesSubmissionStatus.SubmissionStatusFailed) + # Verify patch request was made + queue_mock.push.assert_called() + assert len(entry.patches) == 1 + assert entry.patches[0].internal_patch_id # Should have generated UUID - # Simulate process_cycle - submissions.process_cycle() + def test_patch_submission_when_ready(self, submissions, sample_submission_entry, mock_competition_api): + """Test patch submission when all conditions are met""" + # Create entry with passed POV and ready patch + entry = SubmissionEntry() + crash_with_id = CrashWithId() + crash_with_id.crash.CopyFrom(sample_submission_entry.crashes[0].crash) + crash_with_id.competition_pov_id = "test-pov-123" + crash_with_id.result = SubmissionResult.PASSED + entry.crashes.append(crash_with_id) - # Verify state transition to STOP - assert sample_submission_entry.state == SubmissionEntry.STOP + # Add patch with content + entry_patch = SubmissionEntryPatch() + entry_patch.internal_patch_id = "patch-uuid" + entry_patch.patch = "test patch content" + entry.patches.append(entry_patch) + entry.patch_idx = 0 + submissions.entries = [entry] - def test_submit_bundle_retry_when_errored(self, submissions, sample_submission_entry, mock_competition_api): - # Setup entry in SUBMIT_BUNDLE state - sample_submission_entry.state = SubmissionEntry.SUBMIT_BUNDLE - sample_submission_entry.pov_id = "test-pov-123" - sample_submission_entry.competition_patch_id = "test-patch-456" - submissions.entries = [sample_submission_entry] - - # Mock competition API to return error - mock_competition_api.submit_bundle.return_value = (None, TypesSubmissionStatus.SubmissionStatusErrored) - - # Simulate process_cycle - submissions.process_cycle() - - # Verify state remains the same (will retry on next cycle) - assert sample_submission_entry.state == SubmissionEntry.SUBMIT_BUNDLE - - def test_submit_matching_sarif_successful(self, submissions, sample_submission_entry, mock_competition_api): - # Setup entry in SUBMIT_MATCHING_SARIF state - sample_submission_entry.state = SubmissionEntry.SUBMIT_MATCHING_SARIF - sample_submission_entry.pov_id = "test-pov-123" - sample_submission_entry.competition_patch_id = "test-patch-456" - sample_submission_entry.bundle_id = "test-bundle-789" - submissions.entries = [sample_submission_entry] - - # Create a mock for finding a matching SARIF - with patch.object(submissions, "_submit_matching_sarif", autospec=True) as mock_submit_sarif: - # Configure the mock to set a sarif_id and advance state - def side_effect(i, e): - e.sarif_id = "test-sarif-123" - e.state = SubmissionEntry.SUBMIT_BUNDLE_PATCH - return - - mock_submit_sarif.side_effect = side_effect - - # Simulate process_cycle - submissions.process_cycle() - - # Verify state transition and SARIF ID - assert sample_submission_entry.state == SubmissionEntry.SUBMIT_BUNDLE_PATCH - assert sample_submission_entry.sarif_id == "test-sarif-123" - - def test_submit_bundle_patch_successful(self, submissions, sample_submission_entry, mock_competition_api): - # Setup entry in SUBMIT_BUNDLE_PATCH state - sample_submission_entry.state = SubmissionEntry.SUBMIT_BUNDLE_PATCH - sample_submission_entry.pov_id = "test-pov-123" - sample_submission_entry.competition_patch_id = "test-patch-456" - sample_submission_entry.bundle_id = "test-bundle-789" - sample_submission_entry.sarif_id = "test-sarif-123" - submissions.entries = [sample_submission_entry] - - # Mock competition API to return successful bundle patch - mock_competition_api.patch_bundle.return_value = (True, TypesSubmissionStatus.SubmissionStatusAccepted) - - # Simulate process_cycle - submissions.process_cycle() - - # Verify state transition to STOP (terminal state) - assert sample_submission_entry.state == SubmissionEntry.STOP - - def test_all_states_in_process_cycle(self, submissions, sample_submission_entry): - # Test that process_cycle handles all states by mocking the individual state handlers - submissions.entries = [sample_submission_entry] - - # Create mocks for all state handlers - submissions._submit_patch_request = Mock() - submissions._wait_pov_pass = Mock() - submissions._submit_patch = Mock() - submissions._wait_patch_pass = Mock() - submissions._submit_bundle = Mock() - submissions._submit_matching_sarif = Mock() - submissions._submit_bundle_patch = Mock() - - # Test each state - for state in [ - SubmissionEntry.SUBMIT_PATCH_REQUEST, - SubmissionEntry.WAIT_POV_PASS, - SubmissionEntry.SUBMIT_PATCH, - SubmissionEntry.WAIT_PATCH_PASS, - SubmissionEntry.SUBMIT_BUNDLE, - SubmissionEntry.SUBMIT_MATCHING_SARIF, - SubmissionEntry.SUBMIT_BUNDLE_PATCH, - SubmissionEntry.STOP, - ]: - # Reset mocks - for mock_method in [ - submissions._submit_patch_request, - submissions._wait_pov_pass, - submissions._submit_patch, - submissions._wait_patch_pass, - submissions._submit_bundle, - submissions._submit_matching_sarif, - submissions._submit_bundle_patch, - ]: - mock_method.reset_mock() - - # Set the state - sample_submission_entry.state = state - - # Run process_cycle - submissions.process_cycle() - - # Verify correct method was called based on state - if state == SubmissionEntry.SUBMIT_PATCH_REQUEST: - submissions._submit_patch_request.assert_called_once() - elif state == SubmissionEntry.WAIT_POV_PASS: - submissions._wait_pov_pass.assert_called_once() - elif state == SubmissionEntry.SUBMIT_PATCH: - submissions._submit_patch.assert_called_once() - elif state == SubmissionEntry.WAIT_PATCH_PASS: - submissions._wait_patch_pass.assert_called_once() - elif state == SubmissionEntry.SUBMIT_BUNDLE: - submissions._submit_bundle.assert_called_once() - elif state == SubmissionEntry.SUBMIT_MATCHING_SARIF: - submissions._submit_matching_sarif.assert_called_once() - elif state == SubmissionEntry.SUBMIT_BUNDLE_PATCH: - submissions._submit_bundle_patch.assert_called_once() - # For STOP state, none of the methods should be called - - def test_submit_patch_uses_task_id_correctly(self, submissions, sample_submission_entry, mock_competition_api): - """ - Test that the _submit_patch method correctly passes the task_id to the CompetitionAPI.submit_patch method, - not the TracedCrash object directly. - """ - # Setup entry in SUBMIT_PATCH state with a patch - task_id = "test-task-specific-id" - sample_submission_entry.state = SubmissionEntry.SUBMIT_PATCH - sample_submission_entry.patches.append(SubmissionEntryPatch(patch="test patch content")) - sample_submission_entry.patch_idx = 0 - sample_submission_entry.patch_submission_attempt = 0 - sample_submission_entry.crashes[0].crash.target.task_id = task_id - submissions.entries = [sample_submission_entry] - - # Mock the _task_id function to verify it's being called with the right arguments + # Mock POV mitigation check to return True (patch is good) + # Also mock _request_patched_builds_if_needed to avoid NODE_DATA_DIR dependency with ( - patch("buttercup.orchestrator.scheduler.submissions._task_id", return_value=task_id) as mock_task_id, - patch("buttercup.orchestrator.scheduler.submissions._have_more_patches", return_value=True), + patch.object(submissions, "_check_all_povs_are_mitigated", return_value=True), + patch.object(submissions, "_request_patched_builds_if_needed", return_value=False), ): # Mock competition API to return successful patch submission - mock_competition_api.submit_patch.return_value = ( - "test-patch-123", - TypesSubmissionStatus.SubmissionStatusAccepted, - ) + mock_competition_api.submit_patch.return_value = ("test-patch-456", SubmissionResult.ACCEPTED) - # Mock _check_all_povs_are_mitigated to return True so patch submission proceeds - submissions._check_all_povs_are_mitigated = Mock(return_value=True) - - # Simulate process_cycle + # Process cycle - should submit patch submissions.process_cycle() - # Verify _task_id was called with the submission entry - mock_task_id.assert_called_with(sample_submission_entry) + # Verify patch was submitted + mock_competition_api.submit_patch.assert_called_once_with( + entry.crashes[0].crash.crash.target.task_id, "test patch content" + ) + assert entry.patches[0].competition_patch_id == "test-patch-456" + assert entry.patches[0].result == SubmissionResult.ACCEPTED - # Verify competition_api.submit_patch was called with the task_id, not the crash object + def test_patch_status_update(self, submissions, sample_submission_entry, mock_competition_api): + """Test patch status updates work correctly""" + # Create entry with submitted patch + entry = SubmissionEntry() + crash_with_id = CrashWithId() + crash_with_id.crash.CopyFrom(sample_submission_entry.crashes[0].crash) + crash_with_id.competition_pov_id = "test-pov-123" + crash_with_id.result = SubmissionResult.PASSED + entry.crashes.append(crash_with_id) + + entry_patch = SubmissionEntryPatch() + entry_patch.internal_patch_id = "patch-uuid" + entry_patch.patch = "test patch content" + entry_patch.competition_patch_id = "test-patch-456" + entry_patch.result = SubmissionResult.ACCEPTED + entry.patches.append(entry_patch) + submissions.entries = [entry] + + # Mock competition API to return PASSED status + mock_competition_api.get_patch_status.return_value = SubmissionResult.PASSED + + # Process cycle - should update patch status + submissions.process_cycle() + + # Verify status was updated + mock_competition_api.get_patch_status.assert_called_once_with( + entry.crashes[0].crash.crash.target.task_id, "test-patch-456" + ) + assert entry.patches[0].result == SubmissionResult.PASSED + + def test_patch_advancement_on_failure(self, submissions, sample_submission_entry, mock_competition_api): + """Test that patch index advances when patch fails""" + # Create entry with failed patch and additional patches + entry = SubmissionEntry() + crash_with_id = CrashWithId() + crash_with_id.crash.CopyFrom(sample_submission_entry.crashes[0].crash) + crash_with_id.competition_pov_id = "test-pov-123" + crash_with_id.result = SubmissionResult.PASSED + entry.crashes.append(crash_with_id) + + # Add multiple patches + patch1 = SubmissionEntryPatch() + patch1.internal_patch_id = "patch-uuid-1" + patch1.patch = "test patch 1" + patch1.competition_patch_id = "test-patch-456" + patch1.result = SubmissionResult.ACCEPTED + entry.patches.append(patch1) + + patch2 = SubmissionEntryPatch() + patch2.internal_patch_id = "patch-uuid-2" + patch2.patch = "test patch 2" + entry.patches.append(patch2) + + entry.patch_idx = 0 # Currently on first patch + submissions.entries = [entry] + + # Mock competition API to return FAILED status + mock_competition_api.get_patch_status.return_value = SubmissionResult.FAILED + + # Process cycle - should advance patch index + submissions.process_cycle() + + # Verify patch index was advanced and status updated + assert entry.patches[0].result == SubmissionResult.FAILED + assert entry.patch_idx == 1 # Advanced to next patch + + def test_bundle_creation_when_patch_passes(self, submissions, sample_submission_entry, mock_competition_api): + """Test bundle creation when patch passes""" + # Create entry with passed POV and passed patch + entry = SubmissionEntry() + crash_with_id = CrashWithId() + crash_with_id.crash.CopyFrom(sample_submission_entry.crashes[0].crash) + crash_with_id.competition_pov_id = "test-pov-123" + crash_with_id.result = SubmissionResult.PASSED + entry.crashes.append(crash_with_id) + + entry_patch = SubmissionEntryPatch() + entry_patch.internal_patch_id = "patch-uuid" + entry_patch.patch = "test patch content" + entry_patch.competition_patch_id = "test-patch-456" + entry_patch.result = SubmissionResult.PASSED + entry.patches.append(entry_patch) + entry.patch_idx = 0 + submissions.entries = [entry] + + # Mock competition API to return successful bundle submission + mock_competition_api.submit_bundle.return_value = ("test-bundle-789", SubmissionResult.ACCEPTED) + + # Process cycle - should create bundle + submissions.process_cycle() + + # Verify bundle was created + mock_competition_api.submit_bundle.assert_called_once_with( + entry.crashes[0].crash.crash.target.task_id, "test-pov-123", "test-patch-456", "" + ) + assert len(entry.bundles) == 1 + assert entry.bundles[0].bundle_id == "test-bundle-789" + assert entry.bundles[0].competition_pov_id == "test-pov-123" + assert entry.bundles[0].competition_patch_id == "test-patch-456" + + def test_pov_resubmission_on_error(self, submissions, sample_submission_entry, mock_competition_api): + """Test POV resubmission when first attempt errors""" + # Create entry with errored POV + entry = SubmissionEntry() + crash_with_id = CrashWithId() + crash_with_id.crash.CopyFrom(sample_submission_entry.crashes[0].crash) + crash_with_id.competition_pov_id = "test-pov-123" + crash_with_id.result = SubmissionResult.ERRORED + entry.crashes.append(crash_with_id) + submissions.entries = [entry] + + # Mock competition API to return successful resubmission + mock_competition_api.submit_pov.return_value = ("test-pov-456", SubmissionResult.ACCEPTED) + + # Mock _request_patched_builds_if_needed to avoid NODE_DATA_DIR dependency + with patch.object(submissions, "_request_patched_builds_if_needed", return_value=False): + # Process cycle - should resubmit POV + submissions.process_cycle() + + # Verify POV was resubmitted with new ID + mock_competition_api.submit_pov.assert_called_once() + assert entry.crashes[0].competition_pov_id == "test-pov-456" + assert entry.crashes[0].result == SubmissionResult.ACCEPTED + + def test_patch_resubmission_on_error(self, submissions, sample_submission_entry, mock_competition_api): + """Test patch resubmission when submission errors""" + # Create entry with ready patch for resubmission + entry = SubmissionEntry() + crash_with_id = CrashWithId() + crash_with_id.crash.CopyFrom(sample_submission_entry.crashes[0].crash) + crash_with_id.competition_pov_id = "test-pov-123" + crash_with_id.result = SubmissionResult.PASSED + entry.crashes.append(crash_with_id) + + entry_patch = SubmissionEntryPatch() + entry_patch.internal_patch_id = "patch-uuid" + entry_patch.patch = "test patch content" + entry.patches.append(entry_patch) + entry.patch_idx = 0 + entry.patch_submission_attempts = 1 # Previous attempt failed + submissions.entries = [entry] + + # Mock POV mitigation check to return True + # Also mock _request_patched_builds_if_needed to avoid NODE_DATA_DIR dependency + with ( + patch.object(submissions, "_check_all_povs_are_mitigated", return_value=True), + patch.object(submissions, "_request_patched_builds_if_needed", return_value=False), + ): + # Mock competition API to return ERRORED then successful resubmission + mock_competition_api.submit_patch.return_value = (None, SubmissionResult.ERRORED) + + # Process cycle - should increment attempts and mark as errored + submissions.process_cycle() + + # Verify patch submission was attempted and errored mock_competition_api.submit_patch.assert_called_once() - args, kwargs = mock_competition_api.submit_patch.call_args + assert entry.patches[0].result == SubmissionResult.ERRORED + assert entry.patch_submission_attempts == 2 - # First argument should be the task_id, not the crash - assert args[0] == task_id - assert args[0] != sample_submission_entry.crashes[0] + def test_complete_workflow_integration(self, submissions, sample_submission_entry, mock_competition_api): + """Test complete workflow from vulnerability submission to bundle creation""" + # Start with fresh entry (simulating submit_vulnerability result) + entry = SubmissionEntry() + crash_with_id = CrashWithId() + crash_with_id.crash.CopyFrom(sample_submission_entry.crashes[0].crash) + entry.crashes.append(crash_with_id) + submissions.entries = [entry] - # Second argument should be the patch content - assert args[1] == "test patch content" + # Mock queue for patch requests + queue_mock = MagicMock() - # Verify state transition and patch ID set correctly - assert sample_submission_entry.state == SubmissionEntry.WAIT_PATCH_PASS - assert sample_submission_entry.competition_patch_id == "test-patch-123" - assert sample_submission_entry.patch_submission_attempt == 1 # Incremented + with ( + patch("buttercup.orchestrator.scheduler.submissions.QueueFactory") as queue_factory_mock, + patch.object(submissions, "_request_patched_builds_if_needed", return_value=False), + ): + queue_factory_mock.return_value.create.return_value = queue_mock + + # Step 1: Submit POV + mock_competition_api.submit_pov.return_value = ("test-pov-123", SubmissionResult.ACCEPTED) + submissions.process_cycle() + assert entry.crashes[0].competition_pov_id == "test-pov-123" + + # Step 2: POV status update to PASSED + mock_competition_api.get_pov_status.return_value = SubmissionResult.PASSED + submissions.process_cycle() + assert entry.crashes[0].result == SubmissionResult.PASSED + + # Step 3: Patch request should be made + assert len(entry.patches) == 1 + queue_mock.push.assert_called() + + # Step 4: Simulate patch being received + entry.patches[0].patch = "test patch content" + + # Step 5: Submit patch when ready + with ( + patch.object(submissions, "_check_all_povs_are_mitigated", return_value=True), + patch.object(submissions, "_request_patched_builds_if_needed", return_value=False), + ): + mock_competition_api.submit_patch.return_value = ("test-patch-456", SubmissionResult.ACCEPTED) + submissions.process_cycle() + assert entry.patches[0].competition_patch_id == "test-patch-456" + + # Step 6: Patch status update to PASSED + mock_competition_api.get_patch_status.return_value = SubmissionResult.PASSED + submissions.process_cycle() + assert entry.patches[0].result == SubmissionResult.PASSED + + # Step 7: Bundle creation + mock_competition_api.submit_bundle.return_value = ("test-bundle-789", SubmissionResult.ACCEPTED) + submissions.process_cycle() + assert len(entry.bundles) == 1 + assert entry.bundles[0].bundle_id == "test-bundle-789" # Tests for record_patched_build functionality class TestRecordPatchedBuild: def test_record_patched_build_successful(self, submissions, sample_submission_entry, sample_build_output): """Test successful recording of a build output to a patch.""" - # Setup submission entry with a patch + # Setup submission entry with a patch that has build output placeholders patch_entry = SubmissionEntryPatch() patch_entry.patch = "test patch content" patch_entry.internal_patch_id = "0" + + # Create build output placeholder that matches the sample_build_output + build_output_placeholder = BuildOutput() + build_output_placeholder.engine = sample_build_output.engine + build_output_placeholder.sanitizer = sample_build_output.sanitizer + build_output_placeholder.task_id = sample_build_output.task_id + build_output_placeholder.build_type = sample_build_output.build_type + build_output_placeholder.apply_diff = sample_build_output.apply_diff + build_output_placeholder.internal_patch_id = sample_build_output.internal_patch_id + # task_dir is empty as placeholder - this is what gets filled in + build_output_placeholder.task_dir = "" + + patch_entry.build_outputs.append(build_output_placeholder) sample_submission_entry.patches.append(patch_entry) submissions.entries = [sample_submission_entry] @@ -904,9 +3223,11 @@ class TestRecordPatchedBuild: # Verify success assert result is True - # Verify build output was added to the patch + # Verify build output was updated in the patch assert len(submissions.entries[0].patches) == 1 assert len(submissions.entries[0].patches[0].build_outputs) == 1 + # The task_dir should now be populated + assert submissions.entries[0].patches[0].build_outputs[0].task_dir == sample_build_output.task_dir assert submissions.entries[0].patches[0].build_outputs[0].internal_patch_id == "0" assert submissions.entries[0].patches[0].build_outputs[0].sanitizer == "test_sanitizer" assert submissions.entries[0].patches[0].build_outputs[0].engine == "libfuzzer" @@ -920,12 +3241,32 @@ class TestRecordPatchedBuild: def test_record_patched_build_multiple_outputs(self, submissions, sample_submission_entry): """Test recording multiple build outputs for the same patch.""" - # Setup submission entry with a patch - patch_entry = SubmissionEntryPatch() - patch_entry.patch = "test patch content" - patch_entry.internal_patch_id = "0" - sample_submission_entry.patches.append(patch_entry) - submissions.entries = [sample_submission_entry] + # Setup submission entry with a patch using builder + entry = ( + SubmissionEntryBuilder() + .crash(task_id="test-task-123") + .patch(internal_patch_id="0", patch_content="test patch content") + .build_output( + patch_internal_id="0", + task_id="test-task-123", + sanitizer="asan", + engine="libfuzzer", + build_type=BuildType.PATCH, + apply_diff=True, + task_dir="", # Empty indicates placeholder + ) + .build_output( + patch_internal_id="0", + task_id="test-task-123", + sanitizer="msan", + engine="afl", + build_type=BuildType.FUZZER, + apply_diff=False, + task_dir="", # Empty indicates placeholder + ) + .build() + ) + submissions.entries = [entry] # Create multiple build outputs with the same patch_idx but different properties build_output1 = BuildOutput() @@ -935,6 +3276,7 @@ class TestRecordPatchedBuild: build_output1.engine = "libfuzzer" build_output1.build_type = BuildType.PATCH build_output1.apply_diff = True + build_output1.task_dir = "/tmp/build/test-task-123-asan" build_output2 = BuildOutput() build_output2.internal_patch_id = "0" @@ -943,6 +3285,7 @@ class TestRecordPatchedBuild: build_output2.engine = "afl" build_output2.build_type = BuildType.FUZZER build_output2.apply_diff = False + build_output2.task_dir = "/tmp/build/test-task-123-msan" # Record both build outputs result1 = submissions.record_patched_build(build_output1) @@ -988,18 +3331,39 @@ class TestRecordPatchedBuild: def test_retrieve_build_outputs_from_patch(self, submissions, sample_submission_entry): """Test that we can retrieve build outputs from a patch after recording them.""" - # Setup submission entry with multiple patches - patch_entry1 = SubmissionEntryPatch() - patch_entry1.patch = "patch 1 content" - patch_entry1.internal_patch_id = "patch1" - sample_submission_entry.patches.append(patch_entry1) - - patch_entry2 = SubmissionEntryPatch() - patch_entry2.patch = "patch 2 content" - patch_entry2.internal_patch_id = "patch2" - sample_submission_entry.patches.append(patch_entry2) - - submissions.entries = [sample_submission_entry] + # Setup submission entry with multiple patches using builder + entry = ( + SubmissionEntryBuilder() + .crash(task_id="test-task-123") + .patch(internal_patch_id="patch1", patch_content="patch 1 content") + .build_output( + patch_internal_id="patch1", + task_id="test-task-123", + sanitizer="asan", + build_type=BuildType.PATCH, + apply_diff=True, + task_dir="", + ) + .build_output( + patch_internal_id="patch1", + task_id="test-task-123", + sanitizer="msan", + build_type=BuildType.FUZZER, + apply_diff=False, + task_dir="", + ) + .patch(internal_patch_id="patch2", patch_content="patch 2 content") + .build_output( + patch_internal_id="patch2", + task_id="test-task-123", + sanitizer="ubsan", + build_type=BuildType.COVERAGE, + apply_diff=True, + task_dir="", + ) + .build() + ) + submissions.entries = [entry] # Create different build outputs for different patches build_output_patch0_1 = BuildOutput() @@ -1008,6 +3372,7 @@ class TestRecordPatchedBuild: build_output_patch0_1.sanitizer = "asan" build_output_patch0_1.build_type = BuildType.PATCH build_output_patch0_1.apply_diff = True + build_output_patch0_1.task_dir = "/tmp/build/patch1-asan" build_output_patch0_2 = BuildOutput() build_output_patch0_2.internal_patch_id = "patch1" @@ -1015,6 +3380,7 @@ class TestRecordPatchedBuild: build_output_patch0_2.sanitizer = "msan" build_output_patch0_2.build_type = BuildType.FUZZER build_output_patch0_2.apply_diff = False + build_output_patch0_2.task_dir = "/tmp/build/patch1-msan" build_output_patch1 = BuildOutput() build_output_patch1.internal_patch_id = "patch2" @@ -1022,6 +3388,7 @@ class TestRecordPatchedBuild: build_output_patch1.sanitizer = "ubsan" build_output_patch1.build_type = BuildType.COVERAGE build_output_patch1.apply_diff = True + build_output_patch1.task_dir = "/tmp/build/patch2-ubsan" # Record all build outputs submissions.record_patched_build(build_output_patch0_1) @@ -1083,6 +3450,18 @@ class TestRecordPatchedBuild: patch_entry = SubmissionEntryPatch() patch_entry.patch = "test patch content" patch_entry.internal_patch_id = "duplicate-test" + + # Create placeholder + placeholder = BuildOutput() + placeholder.internal_patch_id = "duplicate-test" + placeholder.task_id = "test-task-123" + placeholder.sanitizer = "asan" + placeholder.engine = "libfuzzer" + placeholder.build_type = BuildType.PATCH + placeholder.apply_diff = True + placeholder.task_dir = "" # Empty indicates placeholder + + patch_entry.build_outputs.append(placeholder) sample_submission_entry.patches.append(patch_entry) submissions.entries = [sample_submission_entry] @@ -1094,6 +3473,7 @@ class TestRecordPatchedBuild: build_output.engine = "libfuzzer" build_output.build_type = BuildType.PATCH build_output.apply_diff = True + build_output.task_dir = "/tmp/build/duplicate-test" # Record the build output for the first time result1 = submissions.record_patched_build(build_output) @@ -1112,3 +3492,982 @@ class TestRecordPatchedBuild: # Verify persistence was only called once (for the first addition) assert submissions.redis.lset.call_count == 1 + + def test_merge_entries_by_patch_mitigation_no_merges(self, submissions): + """Test _merge_entries_by_patch_mitigation when no merges are needed.""" + # Create submissions with patches that don't mitigate other POVs + submissions.entries = [ + SubmissionEntryBuilder() + .crash(task_id="task-1", crash_input_path="/path/to/crash1.bin") + .patch(internal_patch_id="patch-1", patch_content="patch content 1") + .patch_idx(0) + .build(), + SubmissionEntryBuilder() + .crash(task_id="task-1", crash_input_path="/path/to/crash2.bin") + .patch(internal_patch_id="patch-2", patch_content="patch content 2") + .patch_idx(0) + .build(), + ] + + # Mock POV reproduction status to return no mitigation + def mock_pov_reproduce_patch_status(patch, crashes, task_id): + # Return None (pending) or True (did crash) for all POVs - no mitigation + return [None] * len(crashes) + + with patch.object(submissions, "_pov_reproduce_patch_status", side_effect=mock_pov_reproduce_patch_status): + # Call the method + submissions._merge_entries_by_patch_mitigation() + + # Verify no merges occurred - both submissions should still be active + assert len(submissions.entries) == 2 + assert not submissions.entries[0].stop + assert not submissions.entries[1].stop + + def test_merge_entries_by_patch_mitigation_successful_merge(self, submissions): + """Test _merge_entries_by_patch_mitigation when patches mitigate POVs and merging occurs.""" + # Create submissions where first has a patch that mitigates second's POVs + submissions.entries = [ + SubmissionEntryBuilder() + .crash(task_id="task-1", crash_input_path="/path/to/crash1.bin") + .patch(internal_patch_id="patch-1", patch_content="patch content 1") + .build_output(patch_internal_id="patch-1", task_dir="/build/path1", task_id="task-1") + .patch_idx(0) + .build(), + SubmissionEntryBuilder() + .crash(task_id="task-1", crash_input_path="/path/to/crash2.bin") + .patch(internal_patch_id="patch-2", patch_content="patch content 2") + .build_output(patch_internal_id="patch-2", task_dir="/build/path2", task_id="task-1") + .patch_idx(0) + .build(), + SubmissionEntryBuilder() + .crash(task_id="task-1", crash_input_path="/path/to/crash3.bin") + .build(), # No patches + ] + + # Ensure task registry doesn't filter out submissions + submissions.task_registry.should_stop_processing.return_value = False + + # Mock POV reproduction status + def mock_pov_reproduce_patch_status(patch, crashes, task_id): + if patch.internal_patch_id == "patch-1": + # patch-1 mitigates POVs from other submissions + return [Mock(did_crash=False)] * len(crashes) # Mitigated + else: + # Other patches don't mitigate + return [None] * len(crashes) # Pending + + # Mock _consolidate_similar_submissions to track calls + mock_consolidate = Mock() + + with ( + patch.object(submissions, "_pov_reproduce_patch_status", side_effect=mock_pov_reproduce_patch_status), + patch.object(submissions, "_consolidate_similar_submissions", mock_consolidate), + ): + # Call the method + submissions._merge_entries_by_patch_mitigation() + + # Verify consolidation was called + mock_consolidate.assert_called_once() + + # Verify the call arguments - should merge submissions 0, 1, and 2 + call_args = mock_consolidate.call_args[1] + similar_entries = call_args["similar_entries"] + assert len(similar_entries) == 3 # All three submissions + + # Verify the indices and entries + indices = [idx for idx, _ in similar_entries] + assert 0 in indices # First submission (target) + assert 1 in indices # Second submission (has POVs mitigated by patch-1) + assert 2 in indices # Third submission (has POVs mitigated by patch-1) + + def test_merge_entries_by_patch_mitigation_partial_mitigation(self, submissions): + """Test _merge_entries_by_patch_mitigation when patch mitigates some but not all POVs.""" + # Create submissions with multiple crashes + submissions.entries = [ + SubmissionEntryBuilder() + .crash(task_id="task-1", crash_input_path="/path/to/crash1.bin") + .patch(internal_patch_id="patch-1", patch_content="patch content 1") + .build_output(patch_internal_id="patch-1", task_dir="/build/path1", task_id="task-1") + .patch_idx(0) + .build(), + SubmissionEntryBuilder() + .crash(task_id="task-1", crash_input_path="/path/to/crash2.bin") + .crash(task_id="task-1", crash_input_path="/path/to/crash3.bin") + .build(), + ] + + # Ensure task registry doesn't filter out submissions + submissions.task_registry.should_stop_processing.return_value = False + + # Mock POV reproduction status - patch mitigates first crash but not second + def mock_pov_reproduce_patch_status(patch, crashes, task_id): + if patch.internal_patch_id == "patch-1": + # First crash mitigated, second not mitigated + return [Mock(did_crash=False), Mock(did_crash=True)] + return [None] * len(crashes) + + mock_consolidate = Mock() + + with ( + patch.object(submissions, "_pov_reproduce_patch_status", side_effect=mock_pov_reproduce_patch_status), + patch.object(submissions, "_consolidate_similar_submissions", mock_consolidate), + ): + # Call the method + submissions._merge_entries_by_patch_mitigation() + + # Verify consolidation was called (partial mitigation still triggers merge) + mock_consolidate.assert_called_once() + + # Verify both submissions are included + call_args = mock_consolidate.call_args[1] + similar_entries = call_args["similar_entries"] + assert len(similar_entries) == 2 + + def test_merge_entries_by_patch_mitigation_no_current_patch(self, submissions): + """Test _merge_entries_by_patch_mitigation when submissions have no current patch.""" + # Create submissions without patches or with patch_idx beyond available patches + submissions.entries = [ + SubmissionEntryBuilder() + .crash(task_id="task-1", crash_input_path="/path/to/crash1.bin") + .build(), # No patches + SubmissionEntryBuilder() + .crash(task_id="task-1", crash_input_path="/path/to/crash2.bin") + .patch(internal_patch_id="patch-1", patch_content="patch content 1") + .patch_idx(1) # Beyond available patches + .build(), + ] + + # Ensure task registry doesn't filter out submissions + submissions.task_registry.should_stop_processing.return_value = False + + mock_consolidate = Mock() + + with patch.object(submissions, "_consolidate_similar_submissions", mock_consolidate): + # Call the method + submissions._merge_entries_by_patch_mitigation() + + # Verify no consolidation occurred since no current patches + mock_consolidate.assert_not_called() + + def test_merge_entries_by_patch_mitigation_different_tasks(self, submissions): + """Test _merge_entries_by_patch_mitigation with submissions from different tasks.""" + # Create submissions from different tasks + submissions.entries = [ + SubmissionEntryBuilder() + .crash(task_id="task-1", crash_input_path="/path/to/crash1.bin") + .patch(internal_patch_id="patch-1", patch_content="patch content 1") + .patch_idx(0) + .build(), + SubmissionEntryBuilder().crash(task_id="task-2", crash_input_path="/path/to/crash2.bin").build(), + ] + + # Ensure task registry doesn't filter out submissions + submissions.task_registry.should_stop_processing.return_value = False + + # Mock POV reproduction status to return mitigation + def mock_pov_reproduce_patch_status(patch, crashes, task_id): + return [Mock(did_crash=False)] * len(crashes) + + mock_consolidate = Mock() + + with ( + patch.object(submissions, "_pov_reproduce_patch_status", side_effect=mock_pov_reproduce_patch_status), + patch.object(submissions, "_consolidate_similar_submissions", mock_consolidate), + ): + # Call the method + submissions._merge_entries_by_patch_mitigation() + + # Verify no consolidation occurred since submissions are from different tasks + mock_consolidate.assert_not_called() + + def test_merge_entries_by_patch_mitigation_stopped_submissions(self, submissions): + """Test _merge_entries_by_patch_mitigation skips stopped submissions.""" + # Create submissions with one stopped + submissions.entries = [ + SubmissionEntryBuilder() + .crash(task_id="task-1", crash_input_path="/path/to/crash1.bin") + .patch(internal_patch_id="patch-1", patch_content="patch content 1") + .patch_idx(0) + .build(), + SubmissionEntryBuilder().crash(task_id="task-1", crash_input_path="/path/to/crash2.bin").stopped().build(), + ] + + # Ensure task registry doesn't filter out submissions (stopped submissions are filtered by the stop flag, not task registry) + submissions.task_registry.should_stop_processing.return_value = False + + mock_consolidate = Mock() + + with patch.object(submissions, "_consolidate_similar_submissions", mock_consolidate): + # Call the method + submissions._merge_entries_by_patch_mitigation() + + # Verify no consolidation occurred since second submission is stopped + mock_consolidate.assert_not_called() + + def test_merge_entries_by_patch_mitigation_cancelled_tasks(self, submissions, mock_task_registry): + """Test _merge_entries_by_patch_mitigation skips submissions for cancelled tasks.""" + # Create submissions for a task that will be cancelled + submissions.entries = [ + SubmissionEntryBuilder() + .crash(task_id="cancelled-task", crash_input_path="/path/to/crash1.bin") + .patch(internal_patch_id="patch-1", patch_content="patch content 1") + .patch_idx(0) + .build(), + SubmissionEntryBuilder().crash(task_id="cancelled-task", crash_input_path="/path/to/crash2.bin").build(), + ] + + # Mock task registry to indicate task should stop processing + mock_task_registry.should_stop_processing.return_value = True + + mock_consolidate = Mock() + + with patch.object(submissions, "_consolidate_similar_submissions", mock_consolidate): + # Call the method + submissions._merge_entries_by_patch_mitigation() + + # Verify no consolidation occurred since task is cancelled + mock_consolidate.assert_not_called() + + def test_merge_entries_by_patch_mitigation_multiple_patches_same_submission(self, submissions): + """Test _merge_entries_by_patch_mitigation with multiple patches in the same submission.""" + # Create submission with multiple patches, current patch is the second one + submissions.entries = [ + SubmissionEntryBuilder() + .crash(task_id="task-1", crash_input_path="/path/to/crash1.bin") + .patch(internal_patch_id="patch-1a", patch_content="patch content 1a") + .patch(internal_patch_id="patch-1b", patch_content="patch content 1b") + .build_output(patch_internal_id="patch-1b", task_dir="/build/path1b", task_id="task-1") + .patch_idx(1) # Current patch is patch-1b + .build(), + SubmissionEntryBuilder().crash(task_id="task-1", crash_input_path="/path/to/crash2.bin").build(), + ] + + # Ensure task registry doesn't filter out submissions + submissions.task_registry.should_stop_processing.return_value = False + + # Mock POV reproduction status - only patch-1b (current patch) should be tested + def mock_pov_reproduce_patch_status(patch, crashes, task_id): + if patch.internal_patch_id == "patch-1b": + return [Mock(did_crash=False)] * len(crashes) # Mitigated + elif patch.internal_patch_id == "patch-1a": + # This shouldn't be called since it's not the current patch + raise AssertionError("Should not test non-current patch") + return [None] * len(crashes) + + mock_consolidate = Mock() + + with ( + patch.object(submissions, "_pov_reproduce_patch_status", side_effect=mock_pov_reproduce_patch_status), + patch.object(submissions, "_consolidate_similar_submissions", mock_consolidate), + ): + # Call the method + submissions._merge_entries_by_patch_mitigation() + + # Verify consolidation was called with the current patch + mock_consolidate.assert_called_once() + + def test_merge_entries_by_patch_mitigation_pending_status(self, submissions): + """Test _merge_entries_by_patch_mitigation when POV reproduction status is pending.""" + # Create submissions + submissions.entries = [ + SubmissionEntryBuilder() + .crash(task_id="task-1", crash_input_path="/path/to/crash1.bin") + .patch(internal_patch_id="patch-1", patch_content="patch content 1") + .patch_idx(0) + .build(), + SubmissionEntryBuilder().crash(task_id="task-1", crash_input_path="/path/to/crash2.bin").build(), + ] + + # Ensure task registry doesn't filter out submissions + submissions.task_registry.should_stop_processing.return_value = False + + # Mock POV reproduction status to return pending (None) + def mock_pov_reproduce_patch_status(patch, crashes, task_id): + return [None] * len(crashes) # All pending + + mock_consolidate = Mock() + + with ( + patch.object(submissions, "_pov_reproduce_patch_status", side_effect=mock_pov_reproduce_patch_status), + patch.object(submissions, "_consolidate_similar_submissions", mock_consolidate), + ): + # Call the method + submissions._merge_entries_by_patch_mitigation() + + # Verify no consolidation occurred since status is pending + mock_consolidate.assert_not_called() + + def test_merge_entries_by_patch_mitigation_mixed_statuses(self, submissions): + """Test _merge_entries_by_patch_mitigation with mixed POV reproduction statuses.""" + # Create submissions + submissions.entries = [ + SubmissionEntryBuilder() + .crash(task_id="task-1", crash_input_path="/path/to/crash1.bin") + .patch(internal_patch_id="patch-1", patch_content="patch content 1") + .build_output(patch_internal_id="patch-1", task_dir="/build/path1", task_id="task-1") + .patch_idx(0) + .build(), + SubmissionEntryBuilder() + .crash(task_id="task-1", crash_input_path="/path/to/crash2.bin") + .crash(task_id="task-1", crash_input_path="/path/to/crash3.bin") + .build(), + ] + + # Ensure task registry doesn't filter out submissions + submissions.task_registry.should_stop_processing.return_value = False + + # Mock POV reproduction status with mixed results + def mock_pov_reproduce_patch_status(patch, crashes, task_id): + if patch.internal_patch_id == "patch-1": + # First POV mitigated, second POV not mitigated + return [Mock(did_crash=False), Mock(did_crash=True)] + return [None] * len(crashes) + + mock_consolidate = Mock() + + with ( + patch.object(submissions, "_pov_reproduce_patch_status", side_effect=mock_pov_reproduce_patch_status), + patch.object(submissions, "_consolidate_similar_submissions", mock_consolidate), + ): + # Call the method + submissions._merge_entries_by_patch_mitigation() + + # Verify consolidation occurred since at least one POV was mitigated + mock_consolidate.assert_called_once() + + def test_merge_entries_by_patch_mitigation_self_exclusion(self, submissions): + """Test _merge_entries_by_patch_mitigation doesn't merge submission with itself.""" + # Create single submission + submissions.entries = [ + SubmissionEntryBuilder() + .crash(task_id="task-1", crash_input_path="/path/to/crash1.bin") + .patch(internal_patch_id="patch-1", patch_content="patch content 1") + .patch_idx(0) + .build(), + ] + + # Ensure task registry doesn't filter out submissions + submissions.task_registry.should_stop_processing.return_value = False + + # Mock POV reproduction status to return mitigation + def mock_pov_reproduce_patch_status(patch, crashes, task_id): + return [Mock(did_crash=False)] * len(crashes) + + mock_consolidate = Mock() + + with ( + patch.object(submissions, "_pov_reproduce_patch_status", side_effect=mock_pov_reproduce_patch_status), + patch.object(submissions, "_consolidate_similar_submissions", mock_consolidate), + ): + # Call the method + submissions._merge_entries_by_patch_mitigation() + + # Verify no consolidation occurred since there's only one submission + mock_consolidate.assert_not_called() + + def test_merge_entries_by_patch_mitigation_multiple_merges(self, submissions): + """Test _merge_entries_by_patch_mitigation with multiple independent merges.""" + # Create submissions where multiple patches can mitigate different sets of POVs + submissions.entries = [ + # First group - patch-1 mitigates submission 1 + SubmissionEntryBuilder() + .crash(task_id="task-1", crash_input_path="/path/to/crash1.bin") + .patch(internal_patch_id="patch-1", patch_content="patch content 1") + .build_output(patch_internal_id="patch-1", task_dir="/build/path1", task_id="task-1") + .patch_idx(0) + .build(), + SubmissionEntryBuilder().crash(task_id="task-1", crash_input_path="/path/to/crash2.bin").build(), + # Second group - patch-3 mitigates submission 3 + SubmissionEntryBuilder() + .crash(task_id="task-1", crash_input_path="/path/to/crash3.bin") + .patch(internal_patch_id="patch-3", patch_content="patch content 3") + .build_output(patch_internal_id="patch-3", task_dir="/build/path3", task_id="task-1") + .patch_idx(0) + .build(), + SubmissionEntryBuilder().crash(task_id="task-1", crash_input_path="/path/to/crash4.bin").build(), + ] + + # Ensure task registry doesn't filter out submissions + submissions.task_registry.should_stop_processing.return_value = False + + # Mock POV reproduction status + def mock_pov_reproduce_patch_status(patch, crashes, task_id): + if patch.internal_patch_id == "patch-1": + # patch-1 mitigates POVs from submission 1 only + return [Mock(did_crash=False)] * len(crashes) + elif patch.internal_patch_id == "patch-3": + # patch-3 mitigates POVs from submission 3 only + return [Mock(did_crash=False)] * len(crashes) + return [None] * len(crashes) + + mock_consolidate = Mock() + + with ( + patch.object(submissions, "_pov_reproduce_patch_status", side_effect=mock_pov_reproduce_patch_status), + patch.object(submissions, "_consolidate_similar_submissions", mock_consolidate), + ): + # Call the method + submissions._merge_entries_by_patch_mitigation() + + # Verify consolidation was called twice (once for each group) + assert mock_consolidate.call_count == 2 + + def test_merge_entries_by_patch_mitigation_error_handling(self, submissions): + """Test _merge_entries_by_patch_mitigation handles errors gracefully by logging them.""" + # Create submissions - need at least two submissions for the function to test mitigation + submissions.entries = [ + SubmissionEntryBuilder() + .crash(task_id="task-1", crash_input_path="/path/to/crash1.bin") + .patch(internal_patch_id="patch-1", patch_content="patch content 1") + .build_output(patch_internal_id="patch-1", task_dir="/build/path1", task_id="task-1") + .patch_idx(0) + .build(), + SubmissionEntryBuilder().crash(task_id="task-1", crash_input_path="/path/to/crash2.bin").build(), + ] + + # Ensure task registry doesn't filter out submissions + submissions.task_registry.should_stop_processing.return_value = False + + # Mock POV reproduction status to raise an exception + def mock_pov_reproduce_patch_status(patch, crashes, task_id): + raise Exception("Test error in POV reproduction") + + with ( + patch.object(submissions, "_pov_reproduce_patch_status", side_effect=mock_pov_reproduce_patch_status), + patch("buttercup.orchestrator.scheduler.submissions.logger") as mock_logger, + ): + # Call the method - should handle exception gracefully and log error + submissions._merge_entries_by_patch_mitigation() + + # Verify that the error was logged + mock_logger.error.assert_called_once() + error_call_args = mock_logger.error.call_args[0][0] # Get the first argument of the error call + assert "Error merging entries by patch mitigation" in error_call_args + assert "Test error in POV reproduction" in error_call_args + + def test_merge_entries_by_patch_mitigation_complex_scenario(self, submissions): + """Test _merge_entries_by_patch_mitigation with a complex scenario involving multiple tasks and patches.""" + # Create complex scenario with multiple tasks and patches + submissions.entries = [ + # Task 1 submissions + SubmissionEntryBuilder() + .crash(task_id="task-1", crash_input_path="/path/to/crash1.bin") + .patch(internal_patch_id="patch-1", patch_content="patch content 1") + .build_output(patch_internal_id="patch-1", task_dir="/build/path1", task_id="task-1") + .patch_idx(0) + .build(), + SubmissionEntryBuilder().crash(task_id="task-1", crash_input_path="/path/to/crash2.bin").build(), + SubmissionEntryBuilder().crash(task_id="task-1", crash_input_path="/path/to/crash3.bin").build(), + # Task 2 submissions (should be separate) + SubmissionEntryBuilder() + .crash(task_id="task-2", crash_input_path="/path/to/crash4.bin") + .patch(internal_patch_id="patch-4", patch_content="patch content 4") + .build_output(patch_internal_id="patch-4", task_dir="/build/path4", task_id="task-2") + .patch_idx(0) + .build(), + SubmissionEntryBuilder().crash(task_id="task-2", crash_input_path="/path/to/crash5.bin").build(), + ] + + # Ensure task registry doesn't filter out submissions + submissions.task_registry.should_stop_processing.return_value = False + + # Mock POV reproduction status + def mock_pov_reproduce_patch_status(patch, crashes, task_id): + if patch.internal_patch_id == "patch-1" and task_id == "task-1": + # patch-1 mitigates task-1 POVs + return [Mock(did_crash=False)] * len(crashes) + elif patch.internal_patch_id == "patch-4" and task_id == "task-2": + # patch-4 mitigates task-2 POVs + return [Mock(did_crash=False)] * len(crashes) + return [None] * len(crashes) + + mock_consolidate = Mock() + + with ( + patch.object(submissions, "_pov_reproduce_patch_status", side_effect=mock_pov_reproduce_patch_status), + patch.object(submissions, "_consolidate_similar_submissions", mock_consolidate), + ): + # Call the method + submissions._merge_entries_by_patch_mitigation() + + # Verify consolidation was called twice (once for each task) + assert mock_consolidate.call_count == 2 + + # Verify the consolidations were for the correct groups + call_args_list = [call[1]["similar_entries"] for call in mock_consolidate.call_args_list] + + # First call should be for task-1 submissions (indices 0, 1, 2) + first_call_indices = [idx for idx, _ in call_args_list[0]] + assert set(first_call_indices) == {0, 1, 2} + + # Second call should be for task-2 submissions (indices 3, 4) + second_call_indices = [idx for idx, _ in call_args_list[1]] + assert set(second_call_indices) == {3, 4} + + def test_merge_entries_by_patch_mitigation_incomplete_builds(self, submissions): + """Test _merge_entries_by_patch_mitigation when builds are not complete (some task_dir missing).""" + # Create submission with patch that has incomplete builds + submissions.entries = [ + SubmissionEntryBuilder() + .crash(task_id="task-1", crash_input_path="/path/to/crash1.bin") + .patch(internal_patch_id="patch-1", patch_content="patch content 1") + .build_output(patch_internal_id="patch-1", task_dir="", task_id="task-1") # Empty task_dir = incomplete + .patch_idx(0) + .build(), + SubmissionEntryBuilder().crash(task_id="task-1", crash_input_path="/path/to/crash2.bin").build(), + ] + + # Ensure task registry doesn't filter out submissions + submissions.task_registry.should_stop_processing.return_value = False + + # Mock _consolidate_similar_submissions to track calls + mock_consolidate = Mock() + + with ( + patch.object(submissions, "_consolidate_similar_submissions", mock_consolidate), + ): + # Call the method + submissions._merge_entries_by_patch_mitigation() + + # Verify consolidation was NOT called since builds are incomplete + mock_consolidate.assert_not_called() + + def test_merge_entries_by_patch_mitigation_no_build_outputs(self, submissions): + """Test _merge_entries_by_patch_mitigation when patch has no build outputs.""" + # Create submission with patch but no build outputs + submissions.entries = [ + SubmissionEntryBuilder() + .crash(task_id="task-1", crash_input_path="/path/to/crash1.bin") + .patch(internal_patch_id="patch-1", patch_content="patch content 1") + .patch_idx(0) + .build(), # No build outputs + SubmissionEntryBuilder().crash(task_id="task-1", crash_input_path="/path/to/crash2.bin").build(), + ] + + # Ensure task registry doesn't filter out submissions + submissions.task_registry.should_stop_processing.return_value = False + + # Mock _consolidate_similar_submissions to track calls + mock_consolidate = Mock() + + with ( + patch.object(submissions, "_consolidate_similar_submissions", mock_consolidate), + ): + # Call the method + submissions._merge_entries_by_patch_mitigation() + + # Verify consolidation was NOT called since no build outputs + mock_consolidate.assert_not_called() + + def test_merge_entries_by_patch_mitigation_empty_result_list(self, submissions): + """Test _merge_entries_by_patch_mitigation when POV reproduction returns empty results.""" + # Create submissions with normal crashes but mock reproduction to return empty list + submissions.entries = [ + SubmissionEntryBuilder() + .crash(task_id="task-1", crash_input_path="/path/to/crash1.bin") + .patch(internal_patch_id="patch-1", patch_content="patch content 1") + .build_output(patch_internal_id="patch-1", task_dir="/build/path1", task_id="task-1") + .patch_idx(0) + .build(), + SubmissionEntryBuilder().crash(task_id="task-1", crash_input_path="/path/to/crash2.bin").build(), + ] + + # Ensure task registry doesn't filter out submissions + submissions.task_registry.should_stop_processing.return_value = False + + # Mock POV reproduction status to return empty list (unusual but possible edge case) + def mock_pov_reproduce_patch_status(patch, crashes, task_id): + # Return empty list even though crashes exist (edge case) + return [] + + # Mock _consolidate_similar_submissions to track calls + mock_consolidate = Mock() + + with ( + patch.object(submissions, "_pov_reproduce_patch_status", side_effect=mock_pov_reproduce_patch_status), + patch.object(submissions, "_consolidate_similar_submissions", mock_consolidate), + ): + # Call the method + submissions._merge_entries_by_patch_mitigation() + + # Verify consolidation was NOT called since no results indicate mitigation + mock_consolidate.assert_not_called() + + def test_merge_entries_by_patch_mitigation_all_povs_pending(self, submissions): + """Test _merge_entries_by_patch_mitigation when all POVs are pending (None status).""" + # Create submissions + submissions.entries = [ + SubmissionEntryBuilder() + .crash(task_id="task-1", crash_input_path="/path/to/crash1.bin") + .patch(internal_patch_id="patch-1", patch_content="patch content 1") + .build_output(patch_internal_id="patch-1", task_dir="/build/path1", task_id="task-1") + .patch_idx(0) + .build(), + SubmissionEntryBuilder().crash(task_id="task-1", crash_input_path="/path/to/crash2.bin").build(), + ] + + # Ensure task registry doesn't filter out submissions + submissions.task_registry.should_stop_processing.return_value = False + + # Mock POV reproduction status - all pending + def mock_pov_reproduce_patch_status(patch, crashes, task_id): + return [None] * len(crashes) # All pending + + # Mock _consolidate_similar_submissions to track calls + mock_consolidate = Mock() + + with ( + patch.object(submissions, "_pov_reproduce_patch_status", side_effect=mock_pov_reproduce_patch_status), + patch.object(submissions, "_consolidate_similar_submissions", mock_consolidate), + ): + # Call the method + submissions._merge_entries_by_patch_mitigation() + + # Verify consolidation was NOT called since no POVs are mitigated + mock_consolidate.assert_not_called() + + def test_merge_entries_by_patch_mitigation_all_povs_not_mitigated(self, submissions): + """Test _merge_entries_by_patch_mitigation when all POVs are not mitigated (did_crash=True).""" + # Create submissions + submissions.entries = [ + SubmissionEntryBuilder() + .crash(task_id="task-1", crash_input_path="/path/to/crash1.bin") + .patch(internal_patch_id="patch-1", patch_content="patch content 1") + .build_output(patch_internal_id="patch-1", task_dir="/build/path1", task_id="task-1") + .patch_idx(0) + .build(), + SubmissionEntryBuilder().crash(task_id="task-1", crash_input_path="/path/to/crash2.bin").build(), + ] + + # Ensure task registry doesn't filter out submissions + submissions.task_registry.should_stop_processing.return_value = False + + # Mock POV reproduction status - all not mitigated + def mock_pov_reproduce_patch_status(patch, crashes, task_id): + return [Mock(did_crash=True)] * len(crashes) # All not mitigated + + # Mock _consolidate_similar_submissions to track calls + mock_consolidate = Mock() + + with ( + patch.object(submissions, "_pov_reproduce_patch_status", side_effect=mock_pov_reproduce_patch_status), + patch.object(submissions, "_consolidate_similar_submissions", mock_consolidate), + ): + # Call the method + submissions._merge_entries_by_patch_mitigation() + + # Verify consolidation was NOT called since no POVs are mitigated + mock_consolidate.assert_not_called() + + def test_merge_entries_by_patch_mitigation_single_submission(self, submissions): + """Test _merge_entries_by_patch_mitigation with only one submission.""" + # Create only one submission + submissions.entries = [ + SubmissionEntryBuilder() + .crash(task_id="task-1", crash_input_path="/path/to/crash1.bin") + .patch(internal_patch_id="patch-1", patch_content="patch content 1") + .build_output(patch_internal_id="patch-1", task_dir="/build/path1", task_id="task-1") + .patch_idx(0) + .build(), + ] + + # Ensure task registry doesn't filter out submissions + submissions.task_registry.should_stop_processing.return_value = False + + # Mock _consolidate_similar_submissions to track calls + mock_consolidate = Mock() + + with ( + patch.object(submissions, "_consolidate_similar_submissions", mock_consolidate), + ): + # Call the method + submissions._merge_entries_by_patch_mitigation() + + # Verify consolidation was NOT called since there's only one submission + mock_consolidate.assert_not_called() + + def test_merge_entries_by_patch_mitigation_patch_idx_out_of_bounds(self, submissions): + """Test _merge_entries_by_patch_mitigation when patch_idx is beyond available patches.""" + # Create submission with patch_idx that exceeds patch list + submissions.entries = [ + SubmissionEntryBuilder() + .crash(task_id="task-1", crash_input_path="/path/to/crash1.bin") + .patch(internal_patch_id="patch-1", patch_content="patch content 1") + .build_output(patch_internal_id="patch-1", task_dir="/build/path1", task_id="task-1") + .patch_idx(5) # Out of bounds - only one patch exists + .build(), + SubmissionEntryBuilder().crash(task_id="task-1", crash_input_path="/path/to/crash2.bin").build(), + ] + + # Ensure task registry doesn't filter out submissions + submissions.task_registry.should_stop_processing.return_value = False + + # Mock _consolidate_similar_submissions to track calls + mock_consolidate = Mock() + + with ( + patch.object(submissions, "_consolidate_similar_submissions", mock_consolidate), + ): + # Call the method + submissions._merge_entries_by_patch_mitigation() + + # Verify consolidation was NOT called since current_patch returns None + mock_consolidate.assert_not_called() + + def test_merge_entries_by_patch_mitigation_mixed_build_completion(self, submissions): + """Test _merge_entries_by_patch_mitigation with multiple build outputs where some are complete and some aren't.""" + # Create submission with mixed build completion status + entry = ( + SubmissionEntryBuilder() + .crash(task_id="task-1", crash_input_path="/path/to/crash1.bin") + .patch(internal_patch_id="patch-1", patch_content="patch content 1") + .patch_idx(0) + .build() + ) + + # Manually add multiple build outputs with mixed completion + from buttercup.common.datastructures.msg_pb2 import BuildOutput as BuildOutputMsg, BuildType + + build_output1 = BuildOutputMsg( + task_dir="/build/path1", # Complete + task_id="task-1", + build_type=BuildType.PATCH, + sanitizer="asan", + engine="libfuzzer", + ) + build_output2 = BuildOutputMsg( + task_dir="", # Incomplete + task_id="task-1", + build_type=BuildType.PATCH, + sanitizer="msan", + engine="libfuzzer", + ) + entry.patches[0].build_outputs.extend([build_output1, build_output2]) + + submissions.entries = [ + entry, + SubmissionEntryBuilder().crash(task_id="task-1", crash_input_path="/path/to/crash2.bin").build(), + ] + + # Ensure task registry doesn't filter out submissions + submissions.task_registry.should_stop_processing.return_value = False + + # Mock _consolidate_similar_submissions to track calls + mock_consolidate = Mock() + + with ( + patch.object(submissions, "_consolidate_similar_submissions", mock_consolidate), + ): + # Call the method + submissions._merge_entries_by_patch_mitigation() + + # Verify consolidation was NOT called since not all builds are complete + mock_consolidate.assert_not_called() + + +class TestShouldWaitForPatchMitigationMerge: + """Test cases for the _should_wait_for_patch_mitigation_merge method.""" + + def test_should_wait_for_patch_mitigation_merge_pending_evaluation(self, submissions): + """Test _should_wait_for_patch_mitigation_merge when POV evaluation is pending.""" + # Create submissions - one with a patch, one without + submissions.entries = [ + # Submission 0: Has POVs but no patch + SubmissionEntryBuilder() + .crash(task_id="task-1", crash_input_path="/path/to/crash1.bin") + .crash(task_id="task-1", crash_input_path="/path/to/crash2.bin") + .build(), + # Submission 1: Has a submitted patch + SubmissionEntryBuilder() + .crash(task_id="task-1", crash_input_path="/path/to/crash3.bin") + .patch(internal_patch_id="patch-1", patch_content="patch content", competition_patch_id="comp-patch-1") + .patch_idx(0) + .build(), + ] + + # Ensure task registry doesn't filter out submissions + submissions.task_registry.should_stop_processing.return_value = False + + # Mock POV reproduction status - return pending (None) for all POVs + def mock_pov_reproduce_patch_status(patch, crashes, task_id): + return [None] * len(crashes) # All pending + + with patch.object(submissions, "_pov_reproduce_patch_status", side_effect=mock_pov_reproduce_patch_status): + # Test submission 0 - should wait because evaluation is pending + result = submissions._should_wait_for_patch_mitigation_merge(0, submissions.entries[0]) + assert result is True + + def test_should_wait_for_patch_mitigation_merge_confirmed_mitigation(self, submissions): + """Test _should_wait_for_patch_mitigation_merge when POVs are confirmed mitigated.""" + # Create submissions - one with POVs, one with a patch that mitigates them + submissions.entries = [ + # Submission 0: Has POVs that will be mitigated + SubmissionEntryBuilder() + .crash(task_id="task-1", crash_input_path="/path/to/crash1.bin") + .crash(task_id="task-1", crash_input_path="/path/to/crash2.bin") + .build(), + # Submission 1: Has a submitted patch + SubmissionEntryBuilder() + .crash(task_id="task-1", crash_input_path="/path/to/crash3.bin") + .patch(internal_patch_id="patch-1", patch_content="patch content", competition_patch_id="comp-patch-1") + .patch_idx(0) + .build(), + ] + + # Ensure task registry doesn't filter out submissions + submissions.task_registry.should_stop_processing.return_value = False + + # Mock POV reproduction status - one mitigated, one not + def mock_pov_reproduce_patch_status(patch, crashes, task_id): + return [ + Mock(did_crash=False), # Mitigated + Mock(did_crash=True), # Not mitigated + ] + + with patch.object(submissions, "_pov_reproduce_patch_status", side_effect=mock_pov_reproduce_patch_status): + # Test submission 0 - should wait because at least one POV is mitigated + result = submissions._should_wait_for_patch_mitigation_merge(0, submissions.entries[0]) + assert result is True + + def test_should_wait_for_patch_mitigation_merge_no_mitigation(self, submissions): + """Test _should_wait_for_patch_mitigation_merge when no POVs are mitigated.""" + # Create submissions + submissions.entries = [ + # Submission 0: Has POVs that won't be mitigated + SubmissionEntryBuilder() + .crash(task_id="task-1", crash_input_path="/path/to/crash1.bin") + .crash(task_id="task-1", crash_input_path="/path/to/crash2.bin") + .build(), + # Submission 1: Has a submitted patch + SubmissionEntryBuilder() + .crash(task_id="task-1", crash_input_path="/path/to/crash3.bin") + .patch(internal_patch_id="patch-1", patch_content="patch content", competition_patch_id="comp-patch-1") + .patch_idx(0) + .build(), + ] + + # Ensure task registry doesn't filter out submissions + submissions.task_registry.should_stop_processing.return_value = False + + # Mock POV reproduction status - all not mitigated + def mock_pov_reproduce_patch_status(patch, crashes, task_id): + return [Mock(did_crash=True)] * len(crashes) # All not mitigated + + with patch.object(submissions, "_pov_reproduce_patch_status", side_effect=mock_pov_reproduce_patch_status): + # Test submission 0 - should not wait because no POVs are mitigated + result = submissions._should_wait_for_patch_mitigation_merge(0, submissions.entries[0]) + assert result is False + + def test_should_wait_for_patch_mitigation_merge_no_other_patches(self, submissions): + """Test _should_wait_for_patch_mitigation_merge when no other submissions have patches.""" + # Create submissions - none have patches + submissions.entries = [ + SubmissionEntryBuilder().crash(task_id="task-1", crash_input_path="/path/to/crash1.bin").build(), + SubmissionEntryBuilder().crash(task_id="task-1", crash_input_path="/path/to/crash2.bin").build(), + ] + + # Ensure task registry doesn't filter out submissions + submissions.task_registry.should_stop_processing.return_value = False + + # Test submission 0 - should not wait because no other submissions have patches + result = submissions._should_wait_for_patch_mitigation_merge(0, submissions.entries[0]) + assert result is False + + def test_should_wait_for_patch_mitigation_merge_unsubmitted_patch(self, submissions): + """Test _should_wait_for_patch_mitigation_merge when other submission has patch but not submitted.""" + # Create submissions + submissions.entries = [ + # Submission 0: Has POVs + SubmissionEntryBuilder().crash(task_id="task-1", crash_input_path="/path/to/crash1.bin").build(), + # Submission 1: Has patch but not submitted (no competition_patch_id) + SubmissionEntryBuilder() + .crash(task_id="task-1", crash_input_path="/path/to/crash2.bin") + .patch(internal_patch_id="patch-1", patch_content="patch content") # No competition_patch_id + .patch_idx(0) + .build(), + ] + + # Ensure task registry doesn't filter out submissions + submissions.task_registry.should_stop_processing.return_value = False + + # Test submission 0 - should not wait because other patch is not submitted + result = submissions._should_wait_for_patch_mitigation_merge(0, submissions.entries[0]) + assert result is False + + def test_should_wait_for_patch_mitigation_merge_different_tasks(self, submissions): + """Test _should_wait_for_patch_mitigation_merge with submissions from different tasks.""" + # Create submissions from different tasks + submissions.entries = [ + # Submission 0: Task 1 + SubmissionEntryBuilder().crash(task_id="task-1", crash_input_path="/path/to/crash1.bin").build(), + # Submission 1: Task 2 (different task) + SubmissionEntryBuilder() + .crash(task_id="task-2", crash_input_path="/path/to/crash2.bin") + .patch(internal_patch_id="patch-1", patch_content="patch content", competition_patch_id="comp-patch-1") + .patch_idx(0) + .build(), + ] + + # Ensure task registry doesn't filter out submissions + submissions.task_registry.should_stop_processing.return_value = False + + # Test submission 0 - should not wait because other submission is from different task + result = submissions._should_wait_for_patch_mitigation_merge(0, submissions.entries[0]) + assert result is False + + def test_should_wait_for_patch_mitigation_merge_self_exclusion(self, submissions): + """Test _should_wait_for_patch_mitigation_merge excludes checking against itself.""" + # Create submission that has both POVs and a patch + submissions.entries = [ + SubmissionEntryBuilder() + .crash(task_id="task-1", crash_input_path="/path/to/crash1.bin") + .patch(internal_patch_id="patch-1", patch_content="patch content", competition_patch_id="comp-patch-1") + .patch_idx(0) + .build(), + ] + + # Ensure task registry doesn't filter out submissions + submissions.task_registry.should_stop_processing.return_value = False + + # Test submission 0 - should not wait because it only checks against itself (which is excluded) + result = submissions._should_wait_for_patch_mitigation_merge(0, submissions.entries[0]) + assert result is False + + def test_should_wait_for_patch_mitigation_merge_multiple_other_submissions(self, submissions): + """Test _should_wait_for_patch_mitigation_merge with multiple other submissions with patches.""" + # Create submissions + submissions.entries = [ + # Submission 0: Has POVs to be checked + SubmissionEntryBuilder() + .crash(task_id="task-1", crash_input_path="/path/to/crash1.bin") + .crash(task_id="task-1", crash_input_path="/path/to/crash2.bin") + .build(), + # Submission 1: Has patch that doesn't mitigate + SubmissionEntryBuilder() + .crash(task_id="task-1", crash_input_path="/path/to/crash3.bin") + .patch(internal_patch_id="patch-1", patch_content="patch content 1", competition_patch_id="comp-patch-1") + .patch_idx(0) + .build(), + # Submission 2: Has patch that does mitigate + SubmissionEntryBuilder() + .crash(task_id="task-1", crash_input_path="/path/to/crash4.bin") + .patch(internal_patch_id="patch-2", patch_content="patch content 2", competition_patch_id="comp-patch-2") + .patch_idx(0) + .build(), + ] + + # Ensure task registry doesn't filter out submissions + submissions.task_registry.should_stop_processing.return_value = False + + # Mock POV reproduction status - first patch doesn't mitigate, second does + def mock_pov_reproduce_patch_status(patch, crashes, task_id): + if patch.internal_patch_id == "patch-1": + return [Mock(did_crash=True)] * len(crashes) # Doesn't mitigate + elif patch.internal_patch_id == "patch-2": + return [Mock(did_crash=False), Mock(did_crash=True)] # Partially mitigates + return [] + + with patch.object(submissions, "_pov_reproduce_patch_status", side_effect=mock_pov_reproduce_patch_status): + # Test submission 0 - should wait because second patch mitigates at least one POV + result = submissions._should_wait_for_patch_mitigation_merge(0, submissions.entries[0]) + assert result is True