Certain types of error would have missing fields in the results json, causing the summarization code to crash. Fixed that. Also sorted the output of the summarization to made it easier to look at a glance. Errors and failures will be at the top and the rest will be sorted alphabetically.

automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py

@@ -188,7 +188,7 @@ def generate_escu_app(persist_security_content: bool = False) -> str:
     return output_file_path_from_root
 
 
-def finish_mock(settings: dict, detections: list[str], output_file_template: str = "prior_config/config_tests_%d.json"):
+def finish_mock(settings: dict, detections: list[str], output_file_template: str = "prior_config/config_tests_%d.json")->bool:
     num_containers = settings['num_containers']
 
     try:
@@ -199,11 +199,11 @@ def finish_mock(settings: dict, detections: list[str], output_file_template: str
         os.makedirs("prior_config/apps")
     except FileExistsError as e:
         print("Directory priorconfig/apps exists, but we just deleted it!\m\tQuitting...", file=sys.stderr)
-        sys.exit(1)
+        return False
     except Exception as e:
         print("Some error occured when trying to make the configs folder: [%s]\n\tQuitting..." % (
             str(e)), file=sys.stderr)
-        sys.exit(1)
+        return False
 
     # Copy the apps to the appropriate local.  This will also update
     # the app paths in settings['local_apps']
@@ -252,22 +252,20 @@ def finish_mock(settings: dict, detections: list[str], output_file_template: str
                 if validated_settings is None:
                     print(
                         "There was an error validating the updated mock settings.\n\tQuitting...", file=sys.stderr)
-                    sys.exit(1)
+                    return False
 
         except Exception as e:
             print("Error writing config file %s: [%s]\n\tQuitting..." % (
                 fname, str(e)), file=sys.stderr)
-            sys.exit(1)
+            return False
 
-    sys.exit(0)
+    return True
 
 
 def main(args: list[str]):
-    try:
-        docker.client.from_env()
-    except Exception as e:
-        print("Error, failed to get docker client.  Is Docker Running?\n\t%s" % (str(e)))
-
+    #Disable insecure warnings.  We make a number of HTTPS requests to Splunk
+    #docker containers that we've set up.  Without this line, we get an 
+    #insecure warning every time due to invalid cert.
     requests.packages.urllib3.disable_warnings()
 
     start_datetime = datetime.now()
@@ -280,6 +278,16 @@ def main(args: list[str]):
         print("Unsupported action: [%s]" % (action), file=sys.stderr)
         sys.exit(1)
 
+    if settings['mock'] is False:
+        # If this is a real run, then make sure Docker is installed and running and usable
+        # If this is a mock, then that is not required.  By only checking on a non-mock
+        # run, we save ourselves the need to install docker in the CI for the manifest
+        # generation step.
+        try:
+            docker.client.from_env()
+        except Exception as e:
+            print("Error, failed to get docker client.  Is Docker Installed and Running?\n\t%s" % (str(e)))
+
     
     FULL_DOCKER_HUB_CONTAINER_NAME = "splunk/splunk:%s" % settings['container_tag']
 
@@ -304,6 +312,8 @@ def main(args: list[str]):
                                                    settings['detections_list'],
                                                    settings['detections_file'])
 
+
+    #Set up the directory that will be used to store the local apps/apps we build
     local_volume_absolute_path = os.path.abspath(
         os.path.join(os.getcwd(), "apps"))
     try:
@@ -317,41 +327,51 @@ def main(args: list[str]):
         print("Error creating the apps folder [%s]: [%s]\n\tQuitting..."
               % (local_volume_absolute_path, str(e)), file=sys.stderr)
         sys.exit(1)
+    #Add the info about the mount
+    mounts = [{"local_path": local_volume_absolute_path,
+               "container_path": "/tmp/apps", "type": "bind", "read_only": True}]
+
 
     # Check to see if we want to install ESCU and whether it was preeviously generated and we should use that file
     if 'SPLUNK_ES_CONTENT_UPDATE' in settings['local_apps'] and settings['local_apps']['SPLUNK_ES_CONTENT_UPDATE']['local_path'] is not None:
-        # Using a pregenerated ESCU, copy it to apps (unless it)
+        # Using a pregenerated ESCU, no need to build it
         pass
-        #file_path = os.path.expanduser(settings['local_apps']['SPLUNK_ES_CONTENT_UPDATE']['local_path'])
-        #source_path = file_path
-        # dest_path = os.path.join(
-        #    local_volume_absolute_path, os.path.basename(file_path))
-
     elif 'SPLUNK_ES_CONTENT_UPDATE' not in settings['local_apps']:
         print("%s was not found in %s.  We assume this is an error and shut down.\n\t"
               "Quitting..." % ('SPLUNK_ES_CONTENT_UPDATE', "settings['local_apps']"), file=sys.stderr)
         sys.exit(1)
     else:
-        # Need to generate that package
+        # Generate the ESCU package from this branch.
         source_path = generate_escu_app(settings['persist_security_content'])
         settings['local_apps']['SPLUNK_ES_CONTENT_UPDATE']['local_path'] = source_path
-        # dest_path = os.path.join(
-        #    local_volume_absolute_path, os.path.basename(source_path))
+        
 
+    # Copy all the apps, to include ESCU (whether pregenerated or just generated)
     copy_local_apps_to_directory(
         settings['local_apps'], local_volume_absolute_path)
 
-    if settings['mock']:
-        finish_mock(settings, all_test_files)
 
+    # If this is a mock run, finish it now
+    if settings['mock']:
+        #The function below 
+        if finish_mock(settings, all_test_files):
+            # mock was successful!
+            print("Mock successful!  Manifests generated!")
+            sys.exit(0)
+        else:
+            print("There was an unrecoverage error during the mock.\n\tQuitting...",file=sys.stderr)
+            sys.exit(1)
+
+
+
+    #Add some files that always need to be copied to to container to set up indexes and datamodels.
     files_to_copy_to_container = OrderedDict()
     files_to_copy_to_container["INDEXES"] = {
         "local_file_path": index_file_local_path, "container_file_path": index_file_container_path}
     files_to_copy_to_container["DATAMODELS"] = {
         "local_file_path": datamodel_file_local_path, "container_file_path": datamodel_file_container_path}
 
-    mounts = [{"local_path": local_volume_absolute_path,
-               "container_path": "/tmp/apps", "type": "bind", "read_only": True}]
+    
 
     cm = container_manager.ContainerManager(all_test_files,
                                             FULL_DOCKER_HUB_CONTAINER_NAME,

automated_detection_testing/ci/detection_testing_batch/modules/splunk_container.py

@@ -11,7 +11,7 @@ import requests
 import shutil
 from modules import splunk_sdk
 from modules import testing_service
-from modules import test_driver 
+from modules import test_driver
 import time
 import timeit
 from typing import Union
@@ -46,7 +46,7 @@ class SplunkContainer:
         self.container_password = container_password
         self.local_apps = local_apps
         self.splunkbase_apps = splunkbase_apps
-         
+
         self.files_to_copy_to_container = files_to_copy_to_container
         self.splunk_ip = splunk_ip
         self.container_name = container_name
@@ -74,15 +74,15 @@ class SplunkContainer:
     ) -> tuple[str, bool]:
         apps_to_install = []
         require_credentials = False
-        
+
         for app_name, app_info in self.local_apps.items():
-            
+
             app_file_name = os.path.basename(app_info['local_path'])
             app_file_container_path = os.path.join("/tmp/apps", app_file_name)
             apps_to_install.append(app_file_container_path)
 
         for app_name, app_info in self.splunkbase_apps.items():
-            
+
             if splunkbase_username is None or splunkbase_password is None:
                 raise Exception(
                     "Error: Requested app from Splunkbase but Splunkbase username and/or password were not supplied."
@@ -91,12 +91,10 @@ class SplunkContainer:
                 app_info["app_number"], app_info["app_version"])
             apps_to_install.append(target)
             require_credentials = True
-            #elif app["location"] == "local":
+            # elif app["location"] == "local":
             #    apps_to_install.append(app["container_path"])
         return ",".join(apps_to_install), require_credentials
 
-    
-
     def make_environment(
         self,
         local_apps: OrderedDict,
@@ -221,24 +219,24 @@ class SplunkContainer:
         if self.container_start_time == -1:
             total_time_string = "NOT STARTED"
         else:
-            total_time_rounded = datetime.timedelta( seconds =
-                round(current_time - self.container_start_time))
+            total_time_rounded = datetime.timedelta(
+                seconds=round(current_time - self.container_start_time))
             total_time_string = str(total_time_rounded)
 
         # Time that the container setup took
         if self.test_start_time == -1 or self.container_start_time == -1:
             setup_time_string = "NOT SET UP"
         else:
-            setup_secounds_rounded = datetime.timedelta(seconds =
-                round(self.test_start_time - self.container_start_time))
+            setup_secounds_rounded = datetime.timedelta(
+                seconds=round(self.test_start_time - self.container_start_time))
             setup_time_string = str(setup_secounds_rounded)
 
         # Time that the tests have been running
         if self.test_start_time == -1 or self.num_tests_completed == 0:
             testing_time_string = "NO TESTS COMPLETED"
         else:
-            testing_seconds_rounded = datetime.timedelta( seconds =
-                round(current_time - self.test_start_time))
+            testing_seconds_rounded = datetime.timedelta(
+                seconds=round(current_time - self.test_start_time))
 
             # Get the approximate time per test.  This is a clunky way to get rid of decimal
             # seconds.... but it works
@@ -246,14 +244,16 @@ class SplunkContainer:
             timedelta_per_test_rounded = timedelta_per_test - \
                 datetime.timedelta(
                     microseconds=timedelta_per_test.microseconds)
-            
-            testing_time_string = "%s (%d tests @ %s per test)"%(testing_seconds_rounded, self.num_tests_completed, timedelta_per_test_rounded)
+
+            testing_time_string = "%s (%d tests @ %s per test)" % (
+                testing_seconds_rounded, self.num_tests_completed, timedelta_per_test_rounded)
 
         summary_str = "Summary\n\t"\
                       "Total Time          : [%s]\n\t"\
                       "Container Start Time: [%s]\n\t"\
-                      "Test Execution Time : [%s]" %(total_time_string, setup_time_string, testing_time_string)
-        
+                      "Test Execution Time : [%s]" % (
+                          total_time_string, setup_time_string, testing_time_string)
+
         return summary_str
 
     def wait_for_splunk_ready(
@@ -364,12 +364,15 @@ class SplunkContainer:
                     "Warning - uncaught error in detection test for [%s] - this should not happen: [%s]"
                     % (detection_to_test, str(e))
                 )
+                # Fill in all the "Empty" fields with default values. Otherwise, we will not be able to 
+                # process the result correctly.  
                 self.synchronization_object.addError(
                     {"detection_file": detection_to_test,
                         "detection_error": str(e)}
+
+
                 )
-            self.num_tests_completed+=1
+            self.num_tests_completed += 1
 
             # Sleep for a small random time so that containers drift apart and don't synchronize their testing
             time.sleep(random.randint(1, 30))
-

automated_detection_testing/ci/detection_testing_batch/modules/test_driver.py

@@ -58,6 +58,15 @@ class TestDriver:
             self.lock.release()
 
     def addError(self, detection:dict)->None:
+        #Make sure that even errors have all of the required fields.
+        for required_field in ['search_string', 'diskUsage','runDuration', 'detection_name', 'scanCount']:
+            if required_field not in detection:
+                detection[required_field] = ""
+        if  'error' not in detection:
+            detection['error'] = True
+        if  'success' not in detection:
+            detection['success'] = False
+
         self.lock.acquire()
         try:
             self.errors.append(detection)

automated_detection_testing/ci/detection_testing_batch/summarize_json.py

@@ -5,14 +5,17 @@ import sys
 import json
 from modules import validate_args
 import os.path
+from operator import itemgetter
 
 def outputResultsJSON(output_filename:str, data:list[dict], baseline:OrderedDict)->bool:
     success = True
+    
     try:
         test_count = len(data)
         #Passed
         pass_count = len([x for x in data if x['success'] == True])
         
+        
         #A failure or an error
         fail_only_count = len([x for x in data if x['success'] == False])
         
@@ -42,8 +45,9 @@ def outputResultsJSON(output_filename:str, data:list[dict], baseline:OrderedDict
                  "TOTAL_FAILURES": fail_only_count, "FAIL_ONLY": fail_without_error_count, 
                  "FAIL_AND_ERROR":fail_and_error_count }
 
+        data_sorted = sorted(data, key = lambda k: (-k['error'], k['success'], k['detection_file']))
         with open(output_filename, "w") as jsonFile:
-            json.dump({'summary':summary, 'baseline': baseline, 'results':data}, jsonFile, indent="    ")
+            json.dump({'summary':summary, 'baseline': baseline, 'results':data_sorted}, jsonFile, indent="    ")
         
         
         #Generate a failure that the user can download to reproduce and test ONLY the failures locally.
@@ -51,7 +55,7 @@ def outputResultsJSON(output_filename:str, data:list[dict], baseline:OrderedDict
         #that succeeded!
         
         
-        fail_list = [os.path.join("security_content/detections",x['detection_file'] ) for x in data if x['success'] == False]
+        fail_list = [os.path.join("security_content/detections",x['detection_file'] ) for x in data_sorted if x['success'] == False]
 
         if len(fail_list) > 0:
             failures_test_override = {"detections_list": fail_list, "interactive_failure":True, 
@@ -61,6 +65,7 @@ def outputResultsJSON(output_filename:str, data:list[dict], baseline:OrderedDict
                 validate_args.validate_and_write(failures_test_override, failures)
     except Exception as e:
         print("There was an error generating [%s]: [%s]"%(output_filename, str(e)),file=sys.stderr)
+        print(data)
         raise(e)
         #success = False
         #return success, False