From e826b535307c5ef521b5bdd548f33ca526194c7c Mon Sep 17 00:00:00 2001 From: Ronald Eytchison <58823072+reytchison@users.noreply.github.com> Date: Tue, 27 Jan 2026 17:13:34 -0500 Subject: [PATCH] Fix `CoverageBot.run_task` and add test cases (#458) * Use openlit <1.36.6 Openlit >=1.36.6 has a bug with langgraph instrumentation. Also add a test to seed-gen for when openlit is enabled * Link to issue * Fix CoverageBot.run_task and add test cases --------- Co-authored-by: Henrik Brodin <90325907+hbrodin@users.noreply.github.com> --- .../buttercup/fuzzing_infra/coverage_bot.py | 4 +- fuzzer/tests/test_coverage_bot.py | 139 ++++++++++++++++++ 2 files changed, 141 insertions(+), 2 deletions(-) diff --git a/fuzzer/src/buttercup/fuzzing_infra/coverage_bot.py b/fuzzer/src/buttercup/fuzzing_infra/coverage_bot.py index 2156593a..a31c5883 100644 --- a/fuzzer/src/buttercup/fuzzing_infra/coverage_bot.py +++ b/fuzzer/src/buttercup/fuzzing_infra/coverage_bot.py @@ -109,13 +109,13 @@ class CoverageBot(TaskLoop): yield (tmp_dir.path, remaining_files) - def run_task(self, task: WeightedHarness, builds: dict[BuildTypeHint, BuildOutput]) -> None: # type: ignore[invalid-method-override] + def run_task(self, task: WeightedHarness, builds: dict[BuildTypeHint, list[BuildOutput]]) -> None: # type: ignore[invalid-method-override] coverage_builds = builds[BuildType.COVERAGE] if not coverage_builds: logger.error(f"No coverage build found for {task.task_id}") return - coverage_build = coverage_builds # builds dict has wrong type + coverage_build = coverage_builds[0] logger.info(f"Coverage build: {coverage_build}") diff --git a/fuzzer/tests/test_coverage_bot.py b/fuzzer/tests/test_coverage_bot.py index 7f87ae53..dc2afe80 100644 --- a/fuzzer/tests/test_coverage_bot.py +++ b/fuzzer/tests/test_coverage_bot.py @@ -273,3 +273,142 @@ def test_submit_function_coverage_multiple_functions(coverage_bot, redis_client) # Create a set of function names for easier verification function_names = {coverage.function_name for coverage in stored_coverages} assert function_names == {"function1", "function2"} + + +def test_run_task_with_coverage_build(coverage_bot): + """Test that run_task correctly handles builds dict with list[BuildOutput] values.""" + from buttercup.common.datastructures.msg_pb2 import BuildOutput, BuildType, WeightedHarness + + # Create a mock WeightedHarness task + task = WeightedHarness() + task.task_id = "test_task_id" + task.harness_name = "test_harness" + task.package_name = "test_package" + task.weight = 1.0 + + # Create a mock BuildOutput with the required task_dir attribute + build_output = BuildOutput() + build_output.task_id = "test_task_id" + build_output.task_dir = "/tmp/test_task_dir" + build_output.build_type = BuildType.COVERAGE + + # The builds dict should have list[BuildOutput] as values, as per the base class signature + # dict[BuildType, list[BuildOutput]] + builds = {BuildType.COVERAGE: [build_output]} + + # Mock all the dependencies that run_task uses + with ( + patch("buttercup.fuzzing_infra.coverage_bot.ChallengeTask") as mock_challenge_task, + patch("buttercup.fuzzing_infra.coverage_bot.Corpus") as mock_corpus_class, + patch("buttercup.fuzzing_infra.coverage_bot.CoverageRunner") as mock_runner_class, + patch("buttercup.fuzzing_infra.coverage_bot.trace"), + ): + # Setup mock ChallengeTask + mock_task_instance = MagicMock() + mock_task_instance.task_meta.metadata = {} + mock_task_instance.project_name = "test_project" + mock_challenge_task.return_value = mock_task_instance + + # Setup mock local task from get_rw_copy + mock_local_task = MagicMock() + mock_local_task.task_meta.metadata = {} + mock_local_task.project_name = "test_project" + mock_task_instance.get_rw_copy.return_value.__enter__.return_value = mock_local_task + + # Setup mock Corpus + mock_corpus = MagicMock() + mock_corpus.path = "/tmp/corpus" + mock_corpus.local_corpus_size.return_value = 10 + mock_corpus_class.return_value = mock_corpus + + # Setup mock sample_corpus to return test files + with patch.object(coverage_bot, "_sample_corpus") as mock_sample_corpus: + mock_sample_corpus.return_value.__enter__.return_value = ("/tmp/sampled", ["file1", "file2"]) + + # Setup mock CoverageRunner + mock_runner = MagicMock() + mock_runner.run.return_value = [ + CoveredFunction(names="test_func", total_lines=100, covered_lines=50, function_paths=["path1"]), + ] + mock_runner_class.return_value = mock_runner + + coverage_bot.run_task(task, builds) + + # Verify ChallengeTask was called with the correct task_dir from the build_output + mock_challenge_task.assert_called_once_with(read_only_task_dir="/tmp/test_task_dir") + + +def test_run_task_with_empty_coverage_builds(coverage_bot): + """Test that run_task handles empty coverage builds list gracefully.""" + from buttercup.common.datastructures.msg_pb2 import BuildType, WeightedHarness + + task = WeightedHarness() + task.task_id = "test_task_id" + task.harness_name = "test_harness" + task.package_name = "test_package" + + # Empty list of builds + builds = {BuildType.COVERAGE: []} + + # This should return early without raising an error + coverage_bot.run_task(task, builds) + + +def test_run_task_with_multiple_coverage_builds(coverage_bot): + """Test that run_task uses the first build when multiple coverage builds exist.""" + from buttercup.common.datastructures.msg_pb2 import BuildOutput, BuildType, WeightedHarness + + task = WeightedHarness() + task.task_id = "test_task_id" + task.harness_name = "test_harness" + task.package_name = "test_package" + task.weight = 1.0 + + # Create multiple BuildOutput objects + build_output_1 = BuildOutput() + build_output_1.task_id = "test_task_id" + build_output_1.task_dir = "/tmp/first_task_dir" + build_output_1.build_type = BuildType.COVERAGE + + build_output_2 = BuildOutput() + build_output_2.task_id = "test_task_id" + build_output_2.task_dir = "/tmp/second_task_dir" + build_output_2.build_type = BuildType.COVERAGE + + # Multiple builds in the list + builds = {BuildType.COVERAGE: [build_output_1, build_output_2]} + + with ( + patch("buttercup.fuzzing_infra.coverage_bot.ChallengeTask") as mock_challenge_task, + patch("buttercup.fuzzing_infra.coverage_bot.Corpus") as mock_corpus_class, + patch("buttercup.fuzzing_infra.coverage_bot.CoverageRunner") as mock_runner_class, + patch("buttercup.fuzzing_infra.coverage_bot.trace"), + ): + mock_task_instance = MagicMock() + mock_task_instance.task_meta.metadata = {} + mock_task_instance.project_name = "test_project" + mock_challenge_task.return_value = mock_task_instance + + mock_local_task = MagicMock() + mock_local_task.task_meta.metadata = {} + mock_local_task.project_name = "test_project" + mock_task_instance.get_rw_copy.return_value.__enter__.return_value = mock_local_task + + mock_corpus = MagicMock() + mock_corpus.path = "/tmp/corpus" + mock_corpus.local_corpus_size.return_value = 10 + mock_corpus_class.return_value = mock_corpus + + with patch.object(coverage_bot, "_sample_corpus") as mock_sample_corpus: + mock_sample_corpus.return_value.__enter__.return_value = ("/tmp/sampled", ["file1"]) + + mock_runner = MagicMock() + mock_runner.run.return_value = [ + CoveredFunction(names="test_func", total_lines=100, covered_lines=50, function_paths=["path1"]), + ] + mock_runner_class.return_value = mock_runner + + coverage_bot.run_task(task, builds) + + # Verify it used the FIRST build's task_dir + mock_challenge_task.assert_called_once_with(read_only_task_dir="/tmp/first_task_dir")