From ec90ee19d551df03b3236b919285a2f5e05490d1 Mon Sep 17 00:00:00 2001 From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com> Date: Thu, 10 Feb 2022 09:17:23 -0800 Subject: [PATCH] Now, when one or more detection tests fail in the CI/CD, it will cause the workflow to fail. In turn, this would prevent a PR from being merged. However, this lets a developer know more easily (without needing to dive into the GitHub Actions logs) whether or not their detections passed successfully. It also upload an extra file called DetectionFailureManifest which is a test file for ONLY the failed tests. You can run this locally on your machine to quickly, interactively debug the failures only. --- .../ci/detection_testing_batch/summarize_json.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bin/automated_detection_testing/ci/detection_testing_batch/summarize_json.py b/bin/automated_detection_testing/ci/detection_testing_batch/summarize_json.py index d2912773cf..b9d5a9d822 100644 --- a/bin/automated_detection_testing/ci/detection_testing_batch/summarize_json.py +++ b/bin/automated_detection_testing/ci/detection_testing_batch/summarize_json.py @@ -87,9 +87,9 @@ def print_summary(test_count: int, pass_count:int, fail_count:int, error_count:i def exit_with_status(test_pass:bool, test_count: int, pass_count:int, fail_count:int, error_count:int)->None: if not test_pass: print("Result: FAIL") - print("DURING TESTING, THIS WILL STILL EXIT WITH AN EXIT CODE OF 0 (SUCCESS) TO ALLOW THE WORKFLOW " - "TO PASS AND CI/CD TO CONTINUE. THIS WILL BE CHANGED IN A FUTURE VERSION.") - sys.exit(0) + #print("DURING TESTING, THIS WILL STILL EXIT WITH AN EXIT CODE OF 0 (SUCCESS) TO ALLOW THE WORKFLOW " + # "TO PASS AND CI/CD TO CONTINUE. THIS WILL BE CHANGED IN A FUTURE VERSION.") + #sys.exit(0) sys.exit(1) else: print("Result: PASS!")