From dbd047d93fc6a5342ca8d631847ca730d08d18d3 Mon Sep 17 00:00:00 2001 From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com> Date: Thu, 17 Mar 2022 12:11:50 -0700 Subject: [PATCH] Instead of just printing the total numbers of tests, passes, fails, and errors, print out the files that failed as well. --- .../ci/detection_testing_batch/summarize_json.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/bin/automated_detection_testing/ci/detection_testing_batch/summarize_json.py b/bin/automated_detection_testing/ci/detection_testing_batch/summarize_json.py index b9d5a9d822..01d1d4fdcf 100644 --- a/bin/automated_detection_testing/ci/detection_testing_batch/summarize_json.py +++ b/bin/automated_detection_testing/ci/detection_testing_batch/summarize_json.py @@ -61,9 +61,12 @@ def outputResultsJSON(output_filename:str, data:list[dict], baseline:OrderedDict fail_list = [os.path.join("security_content/detections",x['detection_file'] ) for x in data_sorted if x['success'] == False] if len(fail_list) > 0: - + print("FAILURES:") + for failed_test in fail_list: + print(f"\tfailed_test") failures_test_override = copy.deepcopy(summarization_reproduce_failure_config) - failures_test_override.update({"detections_list": fail_list, "no_interactive_failure":False, + #Force all tests to be interactive, even if they don't fail (because they failed on this test) + failures_test_override.update({"detections_list": fail_list, "no_interactive_failure":False, "interactive": True, "num_containers":1, "branch": baseline["branch"], "commit_hash":baseline["commit_hash"], "mode":"selected", "show_splunk_app_password": True}) with open(os.path.join(output_folder,failure_manifest_filename),"w") as failures: