From af9e5e991d837045a5d15f8e07c800bfc640b457 Mon Sep 17 00:00:00 2001 From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com> Date: Wed, 23 Mar 2022 11:09:19 -0700 Subject: [PATCH 1/3] Added a pass rate output to the summary.json file AND the print output to make it easier to see the pass rate at a glance. This means we don't have to calculate it by hand and it's easier to integrate into our high-level metrics. --- bin/docker_detection_tester/summarize_json.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/bin/docker_detection_tester/summarize_json.py b/bin/docker_detection_tester/summarize_json.py index 430af344fe..4a8d79784a 100644 --- a/bin/docker_detection_tester/summarize_json.py +++ b/bin/docker_detection_tester/summarize_json.py @@ -44,8 +44,10 @@ def outputResultsJSON(output_filename:str, data:list[dict], baseline:OrderedDict else: result = "PASS for all %d detections"%(pass_count) + summary={"TOTAL_TESTS": test_count, "TESTS_PASSED": pass_count, "TOTAL_FAILURES": fail_count, "FAIL_ONLY": fail_without_error_count, + "PASS_RATE (%)": calculate_pass_rate(pass_count, test_count), "FAIL_AND_ERROR":fail_and_error_count } data_sorted = sorted(data, key = lambda k: (-k['error'], k['success'], k['detection_file'])) @@ -81,11 +83,21 @@ def outputResultsJSON(output_filename:str, data:list[dict], baseline:OrderedDict #note that total failures is fail_count, fail_and_error count is JUST errors (and every error is also a failure) return success, test_count, pass_count, fail_count, fail_and_error_count +def calculate_pass_rate(pass_count:int, test_count:int)->float: + if test_count == 0: + #Assume this means 100% pass rate to avoid divide by zero + pass_rate = 1 + else: + pass_rate = pass_count / test_count + return pass_rate + def print_summary(test_count: int, pass_count:int, fail_count:int, error_count:int)->None: + print("Summary:"\ - "\n\tTotal Tests: %d"\ - "\n\tTotal Pass : %d"\ - "\n\tTotal Fail : %d (%d of these were ERRORS)"%(test_count, pass_count, fail_count, error_count)) + f"\n\tTotal Tests: {test_count}"\ + f"\n\tTotal Pass : {pass_count}"\ + f"\n\tTotal Fail : {fail_count} ({error_count} of these were ERRORS))"\ + f"\n\tPass Rate : {calculate_pass_rate(pass_count, test_count):.3f}") def exit_with_status(test_pass:bool, test_count: int, pass_count:int, fail_count:int, error_count:int)->None: if not test_pass: From defa3d0f732468b7f5d4db6fb43caaae9c3ee075 Mon Sep 17 00:00:00 2001 From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com> Date: Wed, 23 Mar 2022 11:29:34 -0700 Subject: [PATCH 2/3] Path was wrong for SummaryTestResults. This error was introduced when the YML was updated during the detection testing code cleanup. This resulted in the entire folder being uploaded instead of a single json file. --- .github/workflows/detection-testing.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/detection-testing.yml b/.github/workflows/detection-testing.yml index 8a1de3cd03..80062fe601 100644 --- a/.github/workflows/detection-testing.yml +++ b/.github/workflows/detection-testing.yml @@ -278,7 +278,7 @@ jobs: with: name: SummaryTestResults path: | - bin/docker_detection_tester + bin/docker_detection_tester/summary_test_results.json - name: Upload Failures Manifest on Failure uses: actions/upload-artifact@v2 From 372f83b6736afde3857a0a4c4467ced8bd4dfbf3 Mon Sep 17 00:00:00 2001 From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com> Date: Wed, 23 Mar 2022 11:48:43 -0700 Subject: [PATCH 3/3] Updating name of a summary.json field --- bin/docker_detection_tester/summarize_json.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/docker_detection_tester/summarize_json.py b/bin/docker_detection_tester/summarize_json.py index 4a8d79784a..d57e109fc4 100644 --- a/bin/docker_detection_tester/summarize_json.py +++ b/bin/docker_detection_tester/summarize_json.py @@ -47,7 +47,7 @@ def outputResultsJSON(output_filename:str, data:list[dict], baseline:OrderedDict summary={"TOTAL_TESTS": test_count, "TESTS_PASSED": pass_count, "TOTAL_FAILURES": fail_count, "FAIL_ONLY": fail_without_error_count, - "PASS_RATE (%)": calculate_pass_rate(pass_count, test_count), + "PASS_RATE": calculate_pass_rate(pass_count, test_count), "FAIL_AND_ERROR":fail_and_error_count } data_sorted = sorted(data, key = lambda k: (-k['error'], k['success'], k['detection_file']))