diff --git a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py index 59149f8bdc..80563e57c6 100644 --- a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py +++ b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py @@ -188,7 +188,7 @@ def generate_escu_app(persist_security_content: bool = False) -> str: return output_file_path_from_root -def finish_mock(settings: dict, detections: list[str], output_file_template: str = "prior_config/config_tests_%d.json"): +def finish_mock(settings: dict, detections: list[str], output_file_template: str = "prior_config/config_tests_%d.json")->bool: num_containers = settings['num_containers'] try: @@ -199,11 +199,11 @@ def finish_mock(settings: dict, detections: list[str], output_file_template: str os.makedirs("prior_config/apps") except FileExistsError as e: print("Directory priorconfig/apps exists, but we just deleted it!\m\tQuitting...", file=sys.stderr) - sys.exit(1) + return False except Exception as e: print("Some error occured when trying to make the configs folder: [%s]\n\tQuitting..." % ( str(e)), file=sys.stderr) - sys.exit(1) + return False # Copy the apps to the appropriate local. This will also update # the app paths in settings['local_apps'] @@ -252,22 +252,20 @@ def finish_mock(settings: dict, detections: list[str], output_file_template: str if validated_settings is None: print( "There was an error validating the updated mock settings.\n\tQuitting...", file=sys.stderr) - sys.exit(1) + return False except Exception as e: print("Error writing config file %s: [%s]\n\tQuitting..." % ( fname, str(e)), file=sys.stderr) - sys.exit(1) + return False - sys.exit(0) + return True def main(args: list[str]): - try: - docker.client.from_env() - except Exception as e: - print("Error, failed to get docker client. Is Docker Running?\n\t%s" % (str(e))) - + #Disable insecure warnings. We make a number of HTTPS requests to Splunk + #docker containers that we've set up. Without this line, we get an + #insecure warning every time due to invalid cert. requests.packages.urllib3.disable_warnings() start_datetime = datetime.now() @@ -280,6 +278,16 @@ def main(args: list[str]): print("Unsupported action: [%s]" % (action), file=sys.stderr) sys.exit(1) + if settings['mock'] is False: + # If this is a real run, then make sure Docker is installed and running and usable + # If this is a mock, then that is not required. By only checking on a non-mock + # run, we save ourselves the need to install docker in the CI for the manifest + # generation step. + try: + docker.client.from_env() + except Exception as e: + print("Error, failed to get docker client. Is Docker Installed and Running?\n\t%s" % (str(e))) + FULL_DOCKER_HUB_CONTAINER_NAME = "splunk/splunk:%s" % settings['container_tag'] @@ -304,6 +312,8 @@ def main(args: list[str]): settings['detections_list'], settings['detections_file']) + + #Set up the directory that will be used to store the local apps/apps we build local_volume_absolute_path = os.path.abspath( os.path.join(os.getcwd(), "apps")) try: @@ -317,41 +327,51 @@ def main(args: list[str]): print("Error creating the apps folder [%s]: [%s]\n\tQuitting..." % (local_volume_absolute_path, str(e)), file=sys.stderr) sys.exit(1) + #Add the info about the mount + mounts = [{"local_path": local_volume_absolute_path, + "container_path": "/tmp/apps", "type": "bind", "read_only": True}] + # Check to see if we want to install ESCU and whether it was preeviously generated and we should use that file if 'SPLUNK_ES_CONTENT_UPDATE' in settings['local_apps'] and settings['local_apps']['SPLUNK_ES_CONTENT_UPDATE']['local_path'] is not None: - # Using a pregenerated ESCU, copy it to apps (unless it) + # Using a pregenerated ESCU, no need to build it pass - #file_path = os.path.expanduser(settings['local_apps']['SPLUNK_ES_CONTENT_UPDATE']['local_path']) - #source_path = file_path - # dest_path = os.path.join( - # local_volume_absolute_path, os.path.basename(file_path)) - elif 'SPLUNK_ES_CONTENT_UPDATE' not in settings['local_apps']: print("%s was not found in %s. We assume this is an error and shut down.\n\t" "Quitting..." % ('SPLUNK_ES_CONTENT_UPDATE', "settings['local_apps']"), file=sys.stderr) sys.exit(1) else: - # Need to generate that package + # Generate the ESCU package from this branch. source_path = generate_escu_app(settings['persist_security_content']) settings['local_apps']['SPLUNK_ES_CONTENT_UPDATE']['local_path'] = source_path - # dest_path = os.path.join( - # local_volume_absolute_path, os.path.basename(source_path)) + + # Copy all the apps, to include ESCU (whether pregenerated or just generated) copy_local_apps_to_directory( settings['local_apps'], local_volume_absolute_path) - if settings['mock']: - finish_mock(settings, all_test_files) + # If this is a mock run, finish it now + if settings['mock']: + #The function below + if finish_mock(settings, all_test_files): + # mock was successful! + print("Mock successful! Manifests generated!") + sys.exit(0) + else: + print("There was an unrecoverage error during the mock.\n\tQuitting...",file=sys.stderr) + sys.exit(1) + + + + #Add some files that always need to be copied to to container to set up indexes and datamodels. files_to_copy_to_container = OrderedDict() files_to_copy_to_container["INDEXES"] = { "local_file_path": index_file_local_path, "container_file_path": index_file_container_path} files_to_copy_to_container["DATAMODELS"] = { "local_file_path": datamodel_file_local_path, "container_file_path": datamodel_file_container_path} - mounts = [{"local_path": local_volume_absolute_path, - "container_path": "/tmp/apps", "type": "bind", "read_only": True}] + cm = container_manager.ContainerManager(all_test_files, FULL_DOCKER_HUB_CONTAINER_NAME, diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/splunk_container.py b/automated_detection_testing/ci/detection_testing_batch/modules/splunk_container.py index 7a2d245d1e..f6d43ee6ce 100644 --- a/automated_detection_testing/ci/detection_testing_batch/modules/splunk_container.py +++ b/automated_detection_testing/ci/detection_testing_batch/modules/splunk_container.py @@ -11,7 +11,7 @@ import requests import shutil from modules import splunk_sdk from modules import testing_service -from modules import test_driver +from modules import test_driver import time import timeit from typing import Union @@ -46,7 +46,7 @@ class SplunkContainer: self.container_password = container_password self.local_apps = local_apps self.splunkbase_apps = splunkbase_apps - + self.files_to_copy_to_container = files_to_copy_to_container self.splunk_ip = splunk_ip self.container_name = container_name @@ -74,15 +74,15 @@ class SplunkContainer: ) -> tuple[str, bool]: apps_to_install = [] require_credentials = False - + for app_name, app_info in self.local_apps.items(): - + app_file_name = os.path.basename(app_info['local_path']) app_file_container_path = os.path.join("/tmp/apps", app_file_name) apps_to_install.append(app_file_container_path) for app_name, app_info in self.splunkbase_apps.items(): - + if splunkbase_username is None or splunkbase_password is None: raise Exception( "Error: Requested app from Splunkbase but Splunkbase username and/or password were not supplied." @@ -91,12 +91,10 @@ class SplunkContainer: app_info["app_number"], app_info["app_version"]) apps_to_install.append(target) require_credentials = True - #elif app["location"] == "local": + # elif app["location"] == "local": # apps_to_install.append(app["container_path"]) return ",".join(apps_to_install), require_credentials - - def make_environment( self, local_apps: OrderedDict, @@ -221,24 +219,24 @@ class SplunkContainer: if self.container_start_time == -1: total_time_string = "NOT STARTED" else: - total_time_rounded = datetime.timedelta( seconds = - round(current_time - self.container_start_time)) + total_time_rounded = datetime.timedelta( + seconds=round(current_time - self.container_start_time)) total_time_string = str(total_time_rounded) # Time that the container setup took if self.test_start_time == -1 or self.container_start_time == -1: setup_time_string = "NOT SET UP" else: - setup_secounds_rounded = datetime.timedelta(seconds = - round(self.test_start_time - self.container_start_time)) + setup_secounds_rounded = datetime.timedelta( + seconds=round(self.test_start_time - self.container_start_time)) setup_time_string = str(setup_secounds_rounded) # Time that the tests have been running if self.test_start_time == -1 or self.num_tests_completed == 0: testing_time_string = "NO TESTS COMPLETED" else: - testing_seconds_rounded = datetime.timedelta( seconds = - round(current_time - self.test_start_time)) + testing_seconds_rounded = datetime.timedelta( + seconds=round(current_time - self.test_start_time)) # Get the approximate time per test. This is a clunky way to get rid of decimal # seconds.... but it works @@ -246,14 +244,16 @@ class SplunkContainer: timedelta_per_test_rounded = timedelta_per_test - \ datetime.timedelta( microseconds=timedelta_per_test.microseconds) - - testing_time_string = "%s (%d tests @ %s per test)"%(testing_seconds_rounded, self.num_tests_completed, timedelta_per_test_rounded) + + testing_time_string = "%s (%d tests @ %s per test)" % ( + testing_seconds_rounded, self.num_tests_completed, timedelta_per_test_rounded) summary_str = "Summary\n\t"\ "Total Time : [%s]\n\t"\ "Container Start Time: [%s]\n\t"\ - "Test Execution Time : [%s]" %(total_time_string, setup_time_string, testing_time_string) - + "Test Execution Time : [%s]" % ( + total_time_string, setup_time_string, testing_time_string) + return summary_str def wait_for_splunk_ready( @@ -364,12 +364,15 @@ class SplunkContainer: "Warning - uncaught error in detection test for [%s] - this should not happen: [%s]" % (detection_to_test, str(e)) ) + # Fill in all the "Empty" fields with default values. Otherwise, we will not be able to + # process the result correctly. self.synchronization_object.addError( {"detection_file": detection_to_test, "detection_error": str(e)} + + ) - self.num_tests_completed+=1 + self.num_tests_completed += 1 # Sleep for a small random time so that containers drift apart and don't synchronize their testing time.sleep(random.randint(1, 30)) - diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/test_driver.py b/automated_detection_testing/ci/detection_testing_batch/modules/test_driver.py index b21d2ed960..171a524598 100644 --- a/automated_detection_testing/ci/detection_testing_batch/modules/test_driver.py +++ b/automated_detection_testing/ci/detection_testing_batch/modules/test_driver.py @@ -58,6 +58,15 @@ class TestDriver: self.lock.release() def addError(self, detection:dict)->None: + #Make sure that even errors have all of the required fields. + for required_field in ['search_string', 'diskUsage','runDuration', 'detection_name', 'scanCount']: + if required_field not in detection: + detection[required_field] = "" + if 'error' not in detection: + detection['error'] = True + if 'success' not in detection: + detection['success'] = False + self.lock.acquire() try: self.errors.append(detection) diff --git a/automated_detection_testing/ci/detection_testing_batch/summarize_json.py b/automated_detection_testing/ci/detection_testing_batch/summarize_json.py index a1084a4a44..0e80a3782d 100644 --- a/automated_detection_testing/ci/detection_testing_batch/summarize_json.py +++ b/automated_detection_testing/ci/detection_testing_batch/summarize_json.py @@ -5,14 +5,17 @@ import sys import json from modules import validate_args import os.path +from operator import itemgetter def outputResultsJSON(output_filename:str, data:list[dict], baseline:OrderedDict)->bool: success = True + try: test_count = len(data) #Passed pass_count = len([x for x in data if x['success'] == True]) + #A failure or an error fail_only_count = len([x for x in data if x['success'] == False]) @@ -42,8 +45,9 @@ def outputResultsJSON(output_filename:str, data:list[dict], baseline:OrderedDict "TOTAL_FAILURES": fail_only_count, "FAIL_ONLY": fail_without_error_count, "FAIL_AND_ERROR":fail_and_error_count } + data_sorted = sorted(data, key = lambda k: (-k['error'], k['success'], k['detection_file'])) with open(output_filename, "w") as jsonFile: - json.dump({'summary':summary, 'baseline': baseline, 'results':data}, jsonFile, indent=" ") + json.dump({'summary':summary, 'baseline': baseline, 'results':data_sorted}, jsonFile, indent=" ") #Generate a failure that the user can download to reproduce and test ONLY the failures locally. @@ -51,7 +55,7 @@ def outputResultsJSON(output_filename:str, data:list[dict], baseline:OrderedDict #that succeeded! - fail_list = [os.path.join("security_content/detections",x['detection_file'] ) for x in data if x['success'] == False] + fail_list = [os.path.join("security_content/detections",x['detection_file'] ) for x in data_sorted if x['success'] == False] if len(fail_list) > 0: failures_test_override = {"detections_list": fail_list, "interactive_failure":True, @@ -61,6 +65,7 @@ def outputResultsJSON(output_filename:str, data:list[dict], baseline:OrderedDict validate_args.validate_and_write(failures_test_override, failures) except Exception as e: print("There was an error generating [%s]: [%s]"%(output_filename, str(e)),file=sys.stderr) + print(data) raise(e) #success = False #return success, False