From c45c1335ae51996d28b21444c1fc02e2095ba85c Mon Sep 17 00:00:00 2001 From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com> Date: Wed, 9 Feb 2022 15:23:04 -0800 Subject: [PATCH 1/6] Two changes. First, removed the separate splunkbase_apps and local_apps sections of the config file. They are now unified under a single section called apps. If you don't provide a splunkbase username and password, it will simply use the HTTP target/local_paths. If you do provide those, then it will donwload from splunkbase. However, local_path will override both of these. --- .../detection_testing_execution.py | 52 ++-- .../modules/container_manager.py | 17 +- .../modules/splunk_container.py | 62 ++--- .../modules/validate_args.py | 252 +++++------------- 4 files changed, 137 insertions(+), 246 deletions(-) diff --git a/bin/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py b/bin/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py index ae4ae4ef2b..a648fc2e30 100644 --- a/bin/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py +++ b/bin/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py @@ -30,7 +30,7 @@ import modules.new_arguments2 from modules import (container_manager, new_arguments2, testing_service, validate_args) from modules.github_service import GithubService -from modules.validate_args import validate, validate_and_write +from modules.validate_args import validate, validate_and_write, ES_APP_NAME SPLUNK_CONTAINER_APPS_DIR = "/opt/splunk/etc/apps" index_file_local_path = "indexes.conf.tar" @@ -244,8 +244,8 @@ def finish_mock(settings: dict, detections: list[str], output_file_template: str return False # Copy the apps to the appropriate local. This will also update - # the app paths in settings['local_apps'] - copy_local_apps_to_directory(settings['local_apps'], "prior_config/apps") + # the app paths in settings['apps']['local_path'] if it exists + copy_local_apps_to_directory(settings['apps'], "prior_config/apps") for output_file_index in range(0, num_containers): fname = output_file_template % (output_file_index) @@ -331,23 +331,28 @@ def main(args: list[str]): credentials_needed = False credential_error = False - if len(settings['splunkbase_apps']) > 0: - credentials_needed = True - if settings['splunkbase_username'] == None and credentials_needed: - print("Error - you have listed apps to download from Splunkbase but have "\ - "not provided --splunkbase_username via the command line or config file.",file=sys.stderr) - credential_error = True + + if settings['splunkbase_username'] == None or settings['splunkbase_password'] == None: + missing_credentials = [] + if settings['splunkbase_username'] == None: + missing_credentials.append("--splunkbase_username") + if settings['splunkbase_password'] == None: + missing_credentials.append("--splunkbase_password") + + missing_credentials_string = ' and '.join(missing_credentials) + + + print(f"You have listed apps to install but have "\ + f"not provided {missing_credentials_string} via the command line or config file. " + f"We will download these files from S3 rather than Splunkbase.") + else: + + print(f"You have listed apps to install and provided Splunkbase credentials. "\ + f"These apps will be downloaded and installed from Splunkbase!") - if settings['splunkbase_password'] == None and credentials_needed: - print("Error - you have listed apps to download from Splunkbase but have "\ - "not provided --splunkbase_password via the command line or config file.",file=sys.stderr) - credential_error = True - if credential_error: - print("Please supply the required credentials to continue.\n\tQuitting...",file=sys.stderr) - sys.exit(1) FULL_DOCKER_HUB_CONTAINER_NAME = "splunk/splunk:%s" % settings['container_tag'] @@ -425,22 +430,22 @@ def main(args: list[str]): # Check to see if we want to install ESCU and whether it was preeviously generated and we should use that file - if 'SPLUNK_ES_CONTENT_UPDATE' in settings['local_apps'] and settings['local_apps']['SPLUNK_ES_CONTENT_UPDATE']['local_path'] is not None: + if ES_APP_NAME in settings['apps'] and settings['apps'][ES_APP_NAME]['local_path'] is not None: # Using a pregenerated ESCU, no need to build it pass - elif 'SPLUNK_ES_CONTENT_UPDATE' not in settings['local_apps']: - print("%s was not found in %s. We assume this is an error and shut down.\n\t" - "Quitting..." % ('SPLUNK_ES_CONTENT_UPDATE', "settings['local_apps']"), file=sys.stderr) + elif ES_APP_NAME not in settings['apps']: + print(f"{ES_APP_NAME} was not found in {settings['apps'].keys()}. We assume this is an error and shut down.\n\t" + "Quitting...", file=sys.stderr) sys.exit(1) else: # Generate the ESCU package from this branch. source_path = generate_escu_app(settings['persist_security_content']) - settings['local_apps']['SPLUNK_ES_CONTENT_UPDATE']['local_path'] = source_path + settings['apps']['SPLUNK_ES_CONTENT_UPDATE']['local_path'] = source_path # Copy all the apps, to include ESCU (whether pregenerated or just generated) copy_local_apps_to_directory( - settings['local_apps'], local_volume_absolute_path) + settings['apps'], local_volume_absolute_path) # If this is a mock run, finish it now @@ -472,8 +477,7 @@ def main(args: list[str]): FULL_DOCKER_HUB_CONTAINER_NAME, settings['local_base_container_name'], settings['num_containers'], - settings['local_apps'], - settings['splunkbase_apps'], + settings['apps'], settings['branch'], settings['commit_hash'], reproduce_test_config, diff --git a/bin/automated_detection_testing/ci/detection_testing_batch/modules/container_manager.py b/bin/automated_detection_testing/ci/detection_testing_batch/modules/container_manager.py index 6b25b19b7f..5279cd91b4 100644 --- a/bin/automated_detection_testing/ci/detection_testing_batch/modules/container_manager.py +++ b/bin/automated_detection_testing/ci/detection_testing_batch/modules/container_manager.py @@ -25,8 +25,7 @@ class ContainerManager: full_docker_hub_name: str, container_name_template: str, num_containers: int, - local_apps: OrderedDict, - splunkbase_apps:OrderedDict, + apps: OrderedDict, branch:str, commit_hash:str, summarization_reproduce_failure_config:dict, @@ -47,8 +46,8 @@ class ContainerManager: test_list, num_containers, summarization_reproduce_failure_config) self.mounts = self.create_mounts(mounts) - self.local_apps = local_apps - self.splunkbase_apps = splunkbase_apps + self.apps = apps + if container_password is None: self.container_password = self.get_random_password() @@ -93,11 +92,10 @@ class ContainerManager: self.baseline['TEST_FINISH_TIME'] = "TO BE UPDATED" self.baseline['TEST_DURATION'] = "TO BE UPDATED" - for key in self.local_apps: - self.baseline[key] = self.local_apps[key] + for key in self.apps: + self.baseline[key] = self.apps[key] - for key in self.splunkbase_apps: - self.baseline[key] = self.splunkbase_apps[key] + def run_test(self)->bool: @@ -170,8 +168,7 @@ class ContainerManager: self.synchronization_object, full_docker_hub_name, container_name, - self.local_apps, - self.splunkbase_apps, + self.apps, web_port_tuple, management_port_tuple, self.container_password, diff --git a/bin/automated_detection_testing/ci/detection_testing_batch/modules/splunk_container.py b/bin/automated_detection_testing/ci/detection_testing_batch/modules/splunk_container.py index fea6d69bad..23b1a81063 100644 --- a/bin/automated_detection_testing/ci/detection_testing_batch/modules/splunk_container.py +++ b/bin/automated_detection_testing/ci/detection_testing_batch/modules/splunk_container.py @@ -30,8 +30,7 @@ class SplunkContainer: synchronization_object: test_driver.TestDriver, full_docker_hub_path, container_name: str, - local_apps: OrderedDict, - splunkbase_apps: OrderedDict, + apps: OrderedDict, web_port_tuple: tuple[str, int], management_port_tuple: tuple[str, int], container_password: str, @@ -49,15 +48,15 @@ class SplunkContainer: self.client = docker.client.from_env() self.full_docker_hub_path = full_docker_hub_path self.container_password = container_password - self.local_apps = local_apps - self.splunkbase_apps = splunkbase_apps + + self.apps = apps self.files_to_copy_to_container = files_to_copy_to_container self.splunk_ip = splunk_ip self.container_name = container_name self.mounts = mounts self.environment = self.make_environment( - local_apps, splunkbase_apps, container_password, splunkbase_username, splunkbase_password + apps, container_password, splunkbase_username, splunkbase_password ) self.ports = self.make_ports(web_port_tuple, management_port_tuple) self.web_port = web_port_tuple[1] @@ -74,48 +73,51 @@ class SplunkContainer: def prepare_apps_path( self, - local_apps: OrderedDict, - splunkbase_apps: OrderedDict, + apps: OrderedDict, splunkbase_username: Union[str, None] = None, splunkbase_password: Union[str, None] = None, ) -> tuple[str, bool]: apps_to_install = [] + + #We don't require credentials unless we install at least one splunkbase app require_credentials = False - for app_name, app_info in self.local_apps.items(): - - if 'local_path' in app_info: + #If the username and password are supplied, then we will use splunkbase... + #assuming that the app_name and app_number are supplied. Note that if a + #local_path is supplied, then it should override this option! + if splunkbase_username is not None and splunkbase_password is not None: + use_splunkbase = True + else: + use_splunkbase = False + + for app_name, app_info in self.apps.items(): + if use_splunkbase is True and 'local_path' not in app_info: + target = SPLUNKBASE_URL % (app_info["app_number"], app_info["app_version"]) + apps_to_install.append(target) + #We will require credentials since we are installing at least one splunkbase app + require_credentials = True + elif 'local_path' in app_info: app_file_name = os.path.basename(app_info['local_path']) app_file_container_path = os.path.join("/tmp/apps", app_file_name) - apps_to_install.append(app_file_container_path) + apps_to_install.append(app_file_container_path) elif 'http_path' in app_info: apps_to_install.append(app_info['http_path']) + else: - print("Error, the app %s: %s has no http_path or local_path.\n\tQuitting..."%(app_name,app_info), file=sys.stderr) + if use_splunkbase is True: + print("Error, the app %s: %s could not be installed from Splunkbase because " + "--splunkbase_username and.or --splunkbase_password were not provided." + "\n\tQuitting..."%(app_name,app_info), file=sys.stderr) + else: + print("Error, the app %s: %s has no http_path or local_path.\n\tQuitting..."%(app_name,app_info), file=sys.stderr) sys.exit(1) - for app_name, app_info in self.splunkbase_apps.items(): - - if splunkbase_username is None or splunkbase_password is None: - raise Exception( - "Error: Requested app from Splunkbase but Splunkbase username and/or password were not supplied." - ) - target = SPLUNKBASE_URL % ( - app_info["app_number"], app_info["app_version"]) - apps_to_install.append(target) - require_credentials = True - # elif app["location"] == "local": - # apps_to_install.append(app["container_path"]) - #for printing out all the app paths we will install - #for num, name in zip(range(len(apps_to_install)),apps_to_install): - # print("%d: %s"%(num,name)) return ",".join(apps_to_install), require_credentials def make_environment( self, - local_apps: OrderedDict, - splunkbase_apps: OrderedDict, + apps: OrderedDict, container_password: str, splunkbase_username: Union[str, None] = None, splunkbase_password: Union[str, None] = None, @@ -124,7 +126,7 @@ class SplunkContainer: env["SPLUNK_START_ARGS"] = SPLUNK_START_ARGS env["SPLUNK_PASSWORD"] = container_password splunk_apps_url, require_credentials = self.prepare_apps_path( - local_apps, splunkbase_apps, splunkbase_username, splunkbase_password + apps, splunkbase_username, splunkbase_password ) if require_credentials: diff --git a/bin/automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py b/bin/automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py index a7398dd0d0..308c109e71 100644 --- a/bin/automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py +++ b/bin/automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py @@ -8,7 +8,7 @@ from typing import Union # If we want, we can easily add a description field to any of the objects here! - +ES_APP_NAME = "SPLUNK_ES_CONTENT_UPDATE" setup_schema = { "type": "object", "properties": { @@ -50,7 +50,7 @@ setup_schema = { }, - "local_apps": { + "apps": { "type": "object", "additionalProperties": False, "patternProperties": { @@ -59,131 +59,113 @@ setup_schema = { "additionalProperties": False, "properties": { "app_number": { - "type": [ - "integer", - "null" - ] + "type": ["integer","null"] }, "app_version": { - "type": [ - "string", - "null" - ] + "type": ["string","null"] }, "local_path": { - "type": [ - "string", - "null" - ] + "type": ["string","null"] }, "http_path": { - "type": [ - "string" - ] + "type": ["string", "null"] } }, - "oneOf": [ + "anyOf": [ {"required": ["local_path"]}, - {"required": ["http_path"]} + {"required": ["http_path"] }, + {"required": ["app_number", "app_version"] }, ] } }, "default": { - "SPLUNK_ES_CONTENT_UPDATE": { + ES_APP_NAME : { "app_number": 3449, "app_version": None, "local_path": None }, #The default apps below were taken from the attack_range loadout: https://github.com/splunk/attack_range/blob/develop/attack_range.conf.template - "SPLUNK_WINDOWS_TA": { - "app_number": 0, - "app_version": None, - "http_path": "https://attack-range-appbinaries.s3-us-west-2.amazonaws.com/splunk-add-on-for-microsoft-windows_812.tgz" + "SPLUNK_ADD_ON_FOR_MICROSOFT_WINDOWS": { + "app_number": 742, + "app_version": "8.4.0", + "http_path": "https://attack-range-appbinaries.s3.us-west-2.amazonaws.com/Latest/splunk-add-on-for-microsoft-windows_840.tgz" }, - "SPLUNK_ADD_ON_FOR_SYSMON_OLD": { - "app_number": 1, - "app_version": None, - "http_path": "https://attack-range-appbinaries.s3-us-west-2.amazonaws.com/splunk-add-on-for-microsoft-sysmon_1062.tgz" + "ADD_ON_FOR_LINUX_SYSMON": { + "app_number": 6176, + "app_version": "1.0.4", + "http_path": "https://attack-range-appbinaries.s3.us-west-2.amazonaws.com/Latest/add-on-for-linux-sysmon_104.tgz" }, - "SPLUNK_SYSMON_LINUX_TA_PATCHED": { - "app_number": 2, - "app_version": None, - "http_path": "https://attack-range-appbinaries.s3-us-west-2.amazonaws.com/add-on-for-linux-sysmon_103_PATCHED.tgz" + "SPLUNK_ADD_ON_FOR_SYSMON": { + "app_number": 5709, + "app_version": "1.0.1", + "http_path": "https://attack-range-appbinaries.s3.us-west-2.amazonaws.com/Latest/splunk-add-on-for-sysmon_101.tgz" }, - "SPLUNK_CIM_APP": { - "app_number": 3, - "app_version": None, - "http_path": "https://attack-range-appbinaries.s3-us-west-2.amazonaws.com/splunk-common-information-model-cim_4200.tgz" + "SPLUNK_COMMON_INFORMATION_MODEL": { + "app_number": 1621, + "app_version": "5.0.0", + "http_path": "https://attack-range-appbinaries.s3.us-west-2.amazonaws.com/Latest/splunk-common-information-model-cim_500.tgz" }, - "SPLUNK_AWS_TA": { - "app_number": 4, - "app_version": None, - "http_path": "https://attack-range-appbinaries.s3-us-west-2.amazonaws.com/splunk-add-on-for-amazon-web-services_510.tgz" + #Note - end of life on July 15, 2022 - https://splunkbase.splunk.com/app/1274/ + "SPLUNK_APP_FOR_AWS": { + "app_number": 1274, + "app_version": "6.0.3", + "http_path": "https://attack-range-appbinaries.s3.us-west-2.amazonaws.com/Latest/splunk-app-for-aws_603.tgz" }, - "SPLUNK_PYTHON_APP": { - "app_number": 5, - "app_version": None, - "http_path": "https://attack-range-appbinaries.s3-us-west-2.amazonaws.com/python-for-scientific-computing-for-linux-64-bit_202.tgz" + "PYTHON_FOR_SCIENTIFIC_COMPUTING_FOR_LINUX_64_BIT": { + "app_number": 2882, + "app_version": "3.0.2", + "http_path": "https://attack-range-appbinaries.s3.us-west-2.amazonaws.com/Latest/python-for-scientific-computing-for-linux-64-bit_302.tgz" }, - "SPLUNK_ASX_APP": { - "app_number": 17, - "app_version": None, - "http_path": "https://attack-range-appbinaries.s3-us-west-2.amazonaws.com/Splunk_ASX-latest.tar.gz" + "SPLUNK_MACHINE_LEARNING_TOOLKIT": { + "app_number": 2890, + "app_version": "5.3.1", + "http_path": "https://attack-range-appbinaries.s3.us-west-2.amazonaws.com/Latest/splunk-machine-learning-toolkit_531.tgz" }, - "SPLUNK_MLTK_APP": { - "app_number": 6, - "app_version": None, - "http_path": "https://attack-range-appbinaries.s3-us-west-2.amazonaws.com/splunk-machine-learning-toolkit_521.tgz" - }, - "SPLUNK_STREAM_APP": { - "app_number": 7, - "app_version": None, - "http_path": "https://attack-range-appbinaries.s3-us-west-2.amazonaws.com/splunk-app-for-stream_730.tgz" + "SPLUNK_APP_FOR_STREAM": { + "app_number": 1809, + "app_version": "8.0.1", + "http_path": "https://attack-range-appbinaries.s3.us-west-2.amazonaws.com/Latest/splunk-app-for-stream_801.tgz" }, "SPLUNK_ADD_ON_FOR_STREAM_WIRE_DATA": { - "app_number": 8, - "app_version": None, - "http_path": "https://attack-range-appbinaries.s3-us-west-2.amazonaws.com/splunk-add-on-for-stream-wire-data_730.tgz" + "app_number": 5234, + "app_version": "8.0.1", + "http_path": "https://attack-range-appbinaries.s3.us-west-2.amazonaws.com/Latest/splunk-add-on-for-stream-wire-data_801.tgz" }, "SPLUNK_ADD_ON_FOR_STREAM_FORWARDERS": { - "app_number": 9, - "app_version": None, - "http_path": "https://attack-range-appbinaries.s3-us-west-2.amazonaws.com/splunk-add-on-for-stream-forwarders_730.tgz" + "app_number": 5238, + "app_version": "8.0.1", + "http_path": "https://attack-range-appbinaries.s3.us-west-2.amazonaws.com/Latest/splunk-add-on-for-stream-forwarders_801.tgz" }, "SPLUNK_SECURITY_ESSENTIALS": { - "app_number": 10, - "app_version": None, - "http_path": "https://attack-range-appbinaries.s3-us-west-2.amazonaws.com/splunk-security-essentials_333.tgz" - }, - "SPLUNK_ADD_ON_FOR_ZEEK_AKA_BRO": { - "app_number": 11, - "app_version": None, - "http_path": "https://attack-range-appbinaries.s3-us-west-2.amazonaws.com/splunk-add-on-for-zeek-aka-bro_400.tgz" + "app_number": 3435, + "app_version": "3.4.0", + "http_path": "https://attack-range-appbinaries.s3.us-west-2.amazonaws.com/Latest/splunk-security-essentials_340.tgz" }, + "SPLUNK_ADD_ON_FOR_AMAZON_KINESIS_FIREHOSE": { - "app_number": 12, - "app_version": None, - "http_path": "https://attack-range-appbinaries.s3-us-west-2.amazonaws.com/splunk-add-on-for-amazon-kinesis-firehose_131r7d1d093.tgz" + "app_number": 3719, + "app_version": "1.3.2", + "http_path": "https://attack-range-appbinaries.s3.us-west-2.amazonaws.com/Latest/splunk-add-on-for-amazon-kinesis-firehose_132.tgz" }, "SPLUNK_ADD_ON_FOR_MICROSOFT_OFFICE_365": { - "app_number": 13, - "app_version": None, - "http_path": "https://attack-range-appbinaries.s3-us-west-2.amazonaws.com/splunk-add-on-for-microsoft-office-365_202.tgz" + "app_number": 4055, + "app_version": "2.2.0", + "http_path": "https://attack-range-appbinaries.s3.us-west-2.amazonaws.com/Latest/splunk-add-on-for-microsoft-office-365_220.tgz" }, - "SPLUNK_LINUX_TA": { - "app_number": 14, - "app_version": None, - "http_path": "https://attack-range-appbinaries.s3-us-west-2.amazonaws.com/splunk-add-on-for-unix-and-linux_820.tgz" + "SPLUNK_ADD_ON_FOR_UNIX_AND_LINUX": { + "app_number": 833, + "app_version": "8.4.0", + "http_path": "https://attack-range-appbinaries.s3.us-west-2.amazonaws.com/Latest/splunk-add-on-for-unix-and-linux_840.tgz" }, - "SPLUNK_NGINX_TA": { - "app_number": 15, - "app_version": None, - "http_path": "https://attack-range-appbinaries.s3-us-west-2.amazonaws.com/splunk-add-on-for-nginx_310.tgz" + "SPLUNK_ADD_ON_FOR_NGINX": { + "app_number": 3258, + "app_version": "3.1.0", + "http_path": "https://attack-range-appbinaries.s3.us-west-2.amazonaws.com/Latest/splunk-add-on-for-nginx_310.tgz" }, "SPLUNK_TA_FOR_ZEEK": { - "app_number": 16, - "app_version": None, - "http_path": "https://attack-range-appbinaries.s3-us-west-2.amazonaws.com/ta-for-zeek_105.tgz" + "app_number": 5466, + "app_version": "1.0.5", + "http_path": "https://attack-range-appbinaries.s3.us-west-2.amazonaws.com/Latest/ta-for-zeek_105.tgz" }, } @@ -226,100 +208,6 @@ setup_schema = { }, - - - "splunkbase_apps": { - "type": "object", - "patternProperties": { - "^.*$": { - "type": "object", - "additionalProperties": False, - "properties": { - "app_number": { - "type": "integer" - }, - "app_version": { - "type": "string" - } - } - } - }, - - "default": { - "SPLUNK_ADD_ON_FOR_SYSMON": { - "app_number": 5709, - "app_version": "1.0.1" - } - }, - - # "default": { - # "ADD-ON_FOR_LINUX_SYSMON": { - # "app_number": 6176, - # "app_version": "1.0.3" - # }, - # "SPLUNK_ADD_ON_FOR_AMAZON_WEB_SERVICES": { - # "app_number": 1876, - # "app_version": "5.2.1" - # }, - - # "SPLUNK_ADD_ON_FOR_MICROSOFT_OFFICE_365": - # { - # "app_number": 4055, - # "app_version": "2.2.0" - # }, - - # "SPLUNK_ADD_ON_FOR_AMAZON_KINESIS_FIREHOSE": { - # "app_number": 3719, - # "app_version": "1.3.2" - # }, - - # "SPLUNK_ANALYTIC_STORY_EXECUTION_APP": { - # "app_number": 4971, - # "app_version": "2.0.3" - # }, - - # "PYTHON_FOR_SCIENTIC_COMPUTING_LINUX_64_BIT": { - # "app_number": 2882, - # "app_version": "3.0.1" - # }, - - # "SPLUNK_MACHINE_LEARNING_TOOLKIT": { - # "app_number": 2890, - # "app_version": "5.3.0" - # }, - - # "SPLUNK_APP_FOR_STREAM": { - # "app_number": 1809, - # "app_version": "8.0.1" - # }, - # "SPLUNK_ADD_ON_FOR_STREAM_WIRE_DATA": { - # "app_number": 5234, - # "app_version": "8.0.1" - # }, - # "SPLUNK_ADD_ON_FOR_STREAM_FORWARDERS": { - # "app_number": 5238, - # "app_version": "8.0.1" - # }, - # "SPLUNK_ADD_ON_FOR_ZEEK_AKA_BRO": { - # "app_number": 1617, - # "app_version": "4.0.0" - # }, - # "SPLUNK_ADD_ON_FOR_UNIX_AND_LINUX": { - # "app_number": 833, - # "app_version": "8.4.0" - # }, - # "SPLUNK_ADD_ON_FOR_SYSMON": { - # "app_number": 5709, - # "app_version": "1.0.1" - # }, - # # According to https://docs.splunk.com/Documentation/ES/6.6.2/Install/Datamodels, these are included in ES. Don't install separately. - # "SPLUNK_COMMON_INFORMATION_MODEL": { - # "app_number": 1621, - # "app_version": "5.0.0" - # } - # } - }, - "splunkbase_username": { "type": ["string", "null"], "default": None From d0afcb458d1fe8228b7c8be2d44b294693d42a28 Mon Sep 17 00:00:00 2001 From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com> Date: Wed, 9 Feb 2022 15:37:34 -0800 Subject: [PATCH 2/6] Check to ensure that Splunkbase username and password are passed in the config or command line when one or more apps is lacking http_path or local_path. Before, this would be caught later on and not in the beginning. Also, the previous commit added new http_paths for all of the default apps. --- .../detection_testing_execution.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/bin/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py b/bin/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py index a648fc2e30..ffd9c1a3af 100644 --- a/bin/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py +++ b/bin/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py @@ -335,17 +335,29 @@ def main(args: list[str]): if settings['splunkbase_username'] == None or settings['splunkbase_password'] == None: + missing_credentials = [] if settings['splunkbase_username'] == None: missing_credentials.append("--splunkbase_username") if settings['splunkbase_password'] == None: missing_credentials.append("--splunkbase_password") - missing_credentials_string = ' and '.join(missing_credentials) + missing_credentials_string = '\n\t'.join(missing_credentials) + splunkbase_only_apps = [] + for app,content in settings['apps'].items(): + if 'local_path' not in content and 'http_path' not in content: + splunkbase_only_apps.append(app) + if len(splunkbase_only_apps) != 0: + print(f"Error - you have attempted to install the following apps: {splunkbase_only_apps}, " + "but you have not provided a local_path or an http_path in the config file. Normally, " + "we would download these from Splunkbase, but the following credentials are " + f"missing:\n\t{missing_credentials_string}\n Please provide them on the command line " + "or in the config file.\n\tQuitting...") + sys.exit(1) print(f"You have listed apps to install but have "\ - f"not provided {missing_credentials_string} via the command line or config file. " + f"not provided\n\t{missing_credentials_string} \nvia the command line or config file. " f"We will download these files from S3 rather than Splunkbase.") else: From 889f27064c3837a6b2dd38f07fdfb4196971b474 Mon Sep 17 00:00:00 2001 From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com> Date: Thu, 10 Feb 2022 15:50:25 -0800 Subject: [PATCH 3/6] Downloaded apps are now cached by default. This can save a huge amount of time on startup, especially for a large number of containers or for slower connections. Otherwise, each container that is set up needs over 600MB of throughput. --- .../detection_testing_execution.py | 145 ++++++++++-------- .../modules/splunk_container.py | 4 +- 2 files changed, 83 insertions(+), 66 deletions(-) diff --git a/bin/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py b/bin/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py index ffd9c1a3af..5fd68a55ab 100644 --- a/bin/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py +++ b/bin/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py @@ -1,6 +1,7 @@ import argparse import copy import csv +from ctypes.wintypes import tagRECT import json import os import queue @@ -45,27 +46,57 @@ datamodel_file_container_path = os.path.join( authorizations_file_local_path = "authorize.conf.tar" authorizations_file_container_path = "/opt/splunk/etc/system/local" +CONTAINER_APP_DIRECTORY = "apps" MAX_RECOMMENDED_CONTAINERS_BEFORE_WARNING = 2 -def download_file_from_http(url:str, target:str)->None: - #Will just overwrite an existing file +def download_file_from_http(url:str, destination_file:str, overwrite_file:bool=False)->None: + if os.path.exists(destination_file) and overwrite_file is False: + print(f"[{destination_file}] already exists...using cached version") + return + print(f"downloading to [{destination_file}]") file_to_download = requests.get(url, stream=True) - with open(target, "wb") as output: + with open(destination_file, "wb") as output: for piece in file_to_download.iter_content(chunk_size=(1024*1024)): output.write(piece) -def copy_local_apps_to_directory(apps: dict[str, dict], target_directory) -> None: - for key, item in apps.items(): +def copy_local_apps_to_directory(apps: dict[str, dict], splunkbase_username:tuple[str,None] = None, splunkbase_password:tuple[str,None] = None, mock:bool = False, target_directory:str = "apps") -> str: + if mock is True: + target_directory = os.path.join("prior_config", target_directory) + # Remove the apps directory or the prior config directory. If it's just an apps directory, then we don't want + #to remove that. + shutil.rmtree(target_directory, ignore_errors=True) + try: + # Make sure the directory exists. If it already did, that's okay. Don't delete anything from it + # We want to re-use previously downloaded apps + os.makedirs(target_directory, exist_ok = True) + + except Exception as e: + raise(Exception(f"Some error occured when trying to make the {target_directory}: [{str(e)}]")) + + + for key, item in apps.items(): + + # These apps are URLs that will be passed. The apps will be downloaded and installed by the container + # # Get the file from an http source + splunkbase_info = True if ('app_number' in item and item['app_number'] is not None and + 'app_version' in item and item['app_version'] is not None) else False + splunkbase_creds = True if (splunkbase_username is not None and + splunkbase_password is not None) else False + can_download_from_splunkbase = splunkbase_info and splunkbase_creds + + + #local apps can either have a local_path or an http_path if 'local_path' in item: source_path = os.path.abspath(os.path.expanduser(item['local_path'])) base_name = os.path.basename(source_path) dest_path = os.path.join(target_directory, base_name) try: + print(f"copying {os.path.relpath(source_path)} to {os.path.relpath(dest_path)}") shutil.copy(source_path, dest_path) item['local_path'] = dest_path except shutil.SameFileError as e: @@ -77,25 +108,34 @@ def copy_local_apps_to_directory(apps: dict[str, dict], target_directory) -> Non source_path, dest_path, str(e)), file=sys.stderr) sys.exit(1) - # These apps are URLs that will be passed. The apps will be downloaded and installed by the container - # # Get the file from an http source - # elif 'http_path' in item: - # http_path = item['http_path'] - # try: - # url_parse_obj = urlparse(http_path) - # path_after_host = url_parse_obj[2].rstrip('/') #removes / at the end, if applicable - # base_name = path_after_host.rpartition('/')[-1] #just get the file name - # dest_path = os.path.join(target_directory, base_name) #write the whole path - # download_file_from_http(http_path, dest_path) - # #we need to updat the local path because this is used to copy it into the container later - # item['local_path'] = dest_path - # except Exception as e: - # print("Error trying to download %s @ %s: [%s]. This app is required.\n\tQuitting..."%(key, http_path, str(e)),file=sys.stderr) - # sys.exit(1) - # else: - # print("Error - trying to install a local app that does not have 'local_path' or 'http_path'.\n\tQuitting...") - # sys.exit(1) + + elif can_download_from_splunkbase is True: + #Don't do anything, this will be downloaded from splunkbase + pass + elif splunkbase_info is True and splunkbase_creds is False and mock is True: + #Don't need to do anything, when this actually runs the apps will be downloaded from Splunkbase + #There is another opportunity to provide the creds then + pass + elif 'http_path' in item and can_download_from_splunkbase is False: + http_path = item['http_path'] + try: + url_parse_obj = urlparse(http_path) + path_after_host = url_parse_obj[2].rstrip('/') #removes / at the end, if applicable + base_name = path_after_host.rpartition('/')[-1] #just get the file name + dest_path = os.path.join(target_directory, base_name) #write the whole path + download_file_from_http(http_path, dest_path) + #we need to update the local path because this is used to copy it into the container later + item['local_path'] = dest_path + #Remove the HTTP Path, we will use the local_path instead + except Exception as e: + print("Error trying to download %s @ %s: [%s]. This app is required.\n\tQuitting..."%(key, http_path, str(e)),file=sys.stderr) + sys.exit(1) + elif splunkbase_info is False: + print(f"Error - trying to install an app [{key}] that does not have 'local_path', 'http_path', " + "or 'app_version' and 'app_number' for installing from Splunkbase.\n\tQuitting...") + sys.exit(1) + return target_directory def ensure_security_content(branch: str, commit_hash: Union[str,None], pr_number: Union[int, None], persist_security_content: bool) -> tuple[GithubService, bool]: @@ -183,7 +223,7 @@ def generate_escu_app(persist_security_content: bool = False) -> str: else: os.mkdir("slim_packaging") - os.mkdir("apps") + try: SPLUNK_PACKAGING_TOOLKIT_URL = "https://download.splunk.com/misc/packaging-toolkit/splunk-packaging-toolkit-0.9.0.tar.gz" SPLUNK_PACKAGING_TOOLKIT_FILENAME = 'splunk-packaging-toolkit-latest.tar.gz' @@ -216,7 +256,7 @@ def generate_escu_app(persist_security_content: bool = False) -> str: ret = subprocess.run("; ".join(commands), shell=True, capture_output=True) - if ret.returncode != 0: + if ret.returncode != 0: print("Command List:\n%s" % (commands)) print("Error generating new ESCU Package.\n\tQuitting and dumping error...\n[%s]" % ( ret.stderr.decode('utf-8')), file=sys.stderr) @@ -226,27 +266,10 @@ def generate_escu_app(persist_security_content: bool = False) -> str: return output_file_path_from_root + def finish_mock(settings: dict, detections: list[str], output_file_template: str = "prior_config/config_tests_%d.json")->bool: num_containers = settings['num_containers'] - try: - # Remove the prior config directory if it exists. If not, continue - shutil.rmtree("prior_config", ignore_errors=True) - - # We want to make the prior_config directory and the prior_config/apps directory - os.makedirs("prior_config/apps") - except FileExistsError as e: - print("Directory priorconfig/apps exists, but we just deleted it!\n\tQuitting...", file=sys.stderr) - return False - except Exception as e: - print("Some error occured when trying to make the configs folder: [%s]\n\tQuitting..." % ( - str(e)), file=sys.stderr) - return False - - # Copy the apps to the appropriate local. This will also update - # the app paths in settings['apps']['local_path'] if it exists - copy_local_apps_to_directory(settings['apps'], "prior_config/apps") - for output_file_index in range(0, num_containers): fname = output_file_template % (output_file_index) @@ -285,9 +308,8 @@ def finish_mock(settings: dict, detections: list[str], output_file_template: str # Make sure that it still validates after all of the changes try: - with open(fname, 'w') as cfg: - validated_settings, b = validate_and_write( - mock_settings, cfg) + with open(fname, 'w') as outfile: + validated_settings, b = validate_and_write(configuration=mock_settings, output_file = outfile, strip_credentials=True) if validated_settings is None: print( "There was an error validating the updated mock settings.\n\tQuitting...", file=sys.stderr) @@ -422,23 +444,7 @@ def main(args: list[str]): print("***This run will test [%d] detections!***"%(len(all_test_files))) - #Set up the directory that will be used to store the local apps/apps we build - local_volume_absolute_path = os.path.abspath( - os.path.join(os.getcwd(), "apps")) - try: - # remove the directory first - shutil.rmtree(local_volume_absolute_path, ignore_errors=True) - os.mkdir(local_volume_absolute_path) - except FileExistsError as e: - # Directory already exists, do nothing - pass - except Exception as e: - print("Error creating the apps folder [%s]: [%s]\n\tQuitting..." - % (local_volume_absolute_path, str(e)), file=sys.stderr) - sys.exit(1) - #Add the info about the mount - mounts = [{"local_path": local_volume_absolute_path, - "container_path": "/tmp/apps", "type": "bind", "read_only": True}] + # Check to see if we want to install ESCU and whether it was preeviously generated and we should use that file @@ -456,8 +462,17 @@ def main(args: list[str]): # Copy all the apps, to include ESCU (whether pregenerated or just generated) - copy_local_apps_to_directory( - settings['apps'], local_volume_absolute_path) + try: + relative_app_path = copy_local_apps_to_directory(settings['apps'], + splunkbase_username = settings['splunkbase_username'], + splunkbase_password = settings['splunkbase_password'], + mock=settings['mock'], target_directory = CONTAINER_APP_DIRECTORY) + + mounts = [{"local_path": os.path.abspath(relative_app_path), + "container_path": "/tmp/apps", "type": "bind", "read_only": True}] + except Exception as e: + print(f"Error occurred when copying apps to app folder: [{str(e)}]\n\tQuitting...", file=sys.stderr) + sys.exit(1) # If this is a mock run, finish it now diff --git a/bin/automated_detection_testing/ci/detection_testing_batch/modules/splunk_container.py b/bin/automated_detection_testing/ci/detection_testing_batch/modules/splunk_container.py index 23b1a81063..612e1d1056 100644 --- a/bin/automated_detection_testing/ci/detection_testing_batch/modules/splunk_container.py +++ b/bin/automated_detection_testing/ci/detection_testing_batch/modules/splunk_container.py @@ -23,7 +23,7 @@ SPLUNKBASE_URL = "https://splunkbase.splunk.com/app/%d/release/%s/download" SPLUNK_START_ARGS = "--accept-license" #Give ten minutes to start - this is probably enough time -MAX_CONTAINER_START_TIME_SECONDS = 60*10 +MAX_CONTAINER_START_TIME_SECONDS = 60*20 class SplunkContainer: def __init__( self, @@ -96,6 +96,8 @@ class SplunkContainer: apps_to_install.append(target) #We will require credentials since we are installing at least one splunkbase app require_credentials = True + #Some paths may have a local_path and an HTTP path defined. Default to the local_path first, + #mostly because we may have copied it before into the cache to speed up start time. elif 'local_path' in app_info: app_file_name = os.path.basename(app_info['local_path']) app_file_container_path = os.path.join("/tmp/apps", app_file_name) From 4f0622a5aaf0cd30b3fae49c2381c6d9c5a2549d Mon Sep 17 00:00:00 2001 From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com> Date: Thu, 10 Feb 2022 16:24:17 -0800 Subject: [PATCH 4/6] Update to latest Splunkbase Windows Sysmon --- .../ci/detection_testing_batch/modules/validate_args.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py b/bin/automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py index 308c109e71..0e31edc5db 100644 --- a/bin/automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py +++ b/bin/automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py @@ -97,8 +97,8 @@ setup_schema = { }, "SPLUNK_ADD_ON_FOR_SYSMON": { "app_number": 5709, - "app_version": "1.0.1", - "http_path": "https://attack-range-appbinaries.s3.us-west-2.amazonaws.com/Latest/splunk-add-on-for-sysmon_101.tgz" + "app_version": "2.0.0", + "http_path": "https://attack-range-appbinaries.s3.us-west-2.amazonaws.com/Latest/splunk-add-on-for-sysmon_200.tgz" }, "SPLUNK_COMMON_INFORMATION_MODEL": { "app_number": 1621, From 580dba242f126c1c8ddb383b427abaca803841e2 Mon Sep 17 00:00:00 2001 From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com> Date: Tue, 15 Feb 2022 15:37:17 -0800 Subject: [PATCH 5/6] Added ability to Ctrl-C/SIGINT the testing process and have it gracefully finish. This stops the containers and writes out partial progress to the output file. --- .../detection_testing_execution.py | 53 ++++++++++++++++++- .../modules/container_manager.py | 46 ++++++++++------ .../modules/splunk_container.py | 33 ++++++++++-- .../modules/test_driver.py | 11 +--- 4 files changed, 114 insertions(+), 29 deletions(-) diff --git a/bin/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py b/bin/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py index 5fd68a55ab..267f6af863 100644 --- a/bin/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py +++ b/bin/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py @@ -20,6 +20,8 @@ from tempfile import mkdtemp from timeit import default_timer as timer from typing import Union from urllib.parse import urlparse +import signal + import docker import requests @@ -51,6 +53,9 @@ CONTAINER_APP_DIRECTORY = "apps" MAX_RECOMMENDED_CONTAINERS_BEFORE_WARNING = 2 + + + def download_file_from_http(url:str, destination_file:str, overwrite_file:bool=False)->None: if os.path.exists(destination_file) and overwrite_file is False: print(f"[{destination_file}] already exists...using cached version") @@ -451,6 +456,7 @@ def main(args: list[str]): if ES_APP_NAME in settings['apps'] and settings['apps'][ES_APP_NAME]['local_path'] is not None: # Using a pregenerated ESCU, no need to build it pass + elif ES_APP_NAME not in settings['apps']: print(f"{ES_APP_NAME} was not found in {settings['apps'].keys()}. We assume this is an error and shut down.\n\t" "Quitting...", file=sys.stderr) @@ -499,6 +505,39 @@ def main(args: list[str]): + def shutdown_signal_handler_setup(sig, frame): + + print(f"Signal {sig} received... stopping all [{settings['num_containers']}] containers and shutting down...") + shutdown_client = docker.client.from_env() + errorCount = 0 + for container_number in range(settings['num_containers']): + container_name = settings['local_base_container_name']%container_number + print(f"Shutting down {container_name}...", file=sys.stderr, end='') + sys.stdout.flush() + try: + container = shutdown_client.containers.get(container_name) + #Note that stopping does not remove any of the volumes or logs, + #so stopping can be useful if we want to debug any container failure + container.stop(timeout=10) + print("done", file=sys.stderr) + except Exception as e: + print(f"Error trying to shut down {container_name}. It may have already shut down. Stop it youself with 'docker containter stop {container_name}", sys.stderr) + errorCount += 1 + if errorCount == 0: + print("All containers shut down successfully", file=sys.stderr) + else: + print(f"{errorCount} containers may still be running. Find out what is running with:\n\t'docker container ls'\nand shut them down with\n\t'docker container stop CONTAINER_NAME' ", file=sys.stderr) + + print("Quitting...",file=sys.stderr) + #We must use os._exit(1) because sys.exit(1) actually generates an exception which can be caught! And then we don't Quit! + os._exit(1) + + + + + #Setup requires a different teardown handler than during execution + signal.signal(signal.SIGINT, shutdown_signal_handler_setup) + try: cm = container_manager.ContainerManager(all_test_files, FULL_DOCKER_HUB_CONTAINER_NAME, @@ -523,6 +562,15 @@ def main(args: list[str]): print("Error - unrecoverable error trying to set up the containers: [%s].\n\tQuitting..."%(str(e)),file=sys.stderr) sys.exit(1) + def shutdown_signal_handler_execution(sig, frame): + #Set that a container has failed which will gracefully stop the other containers. + #This way we get our full cleanup routine, too! + print("Got a signal to shut down. Shutting down all containers, please wait...", file=sys.stderr) + cm.synchronization_object.containerFailure() + + #Update the signal handler + + signal.signal(signal.SIGINT, shutdown_signal_handler_execution) try: result = cm.run_test() except Exception as e: @@ -539,7 +587,10 @@ def main(args: list[str]): sys.exit(0) else: print("Test Execution Failed - review the logs for more details") - sys.exit(1) + #Because one or more of the threads could be stuck in a certain setup loop, like + #trying to copy files to a containers (which igonores errors), we must os._exit + #instead of sys.exit + os._exit(1) if __name__ == "__main__": diff --git a/bin/automated_detection_testing/ci/detection_testing_batch/modules/container_manager.py b/bin/automated_detection_testing/ci/detection_testing_batch/modules/container_manager.py index 5279cd91b4..975bf5d9e6 100644 --- a/bin/automated_detection_testing/ci/detection_testing_batch/modules/container_manager.py +++ b/bin/automated_detection_testing/ci/detection_testing_batch/modules/container_manager.py @@ -1,4 +1,5 @@ from collections import OrderedDict +from tabnanny import check import docker import datetime import docker.types @@ -42,6 +43,9 @@ class ContainerManager: interactive:bool=False ): + #Used to determine whether or not we should wait for container threads to finish when summarizing + self.all_tests_completed = False + self.synchronization_object = test_driver.TestDriver( test_list, num_containers, summarization_reproduce_failure_config) @@ -99,13 +103,22 @@ class ContainerManager: def run_test(self)->bool: - self.run_containers() self.run_status_thread() - for container in self.containers: - container.thread.join() - print(container.get_container_summary()) - print("Waiting for next summary thread printout to finish...") + self.run_containers() self.summary_thread.join() + + + + + for container in self.containers: + if self.all_tests_completed == True: + container.thread.join() + elif self.all_tests_completed == False: + #For some reason, we stopped early. So don't wait on the child threads to finish. Don't join, + #these threads may be stuck in their setup loops. Continue on. + pass + + print(container.get_container_summary()) print("All containers completed testing!") @@ -216,16 +229,19 @@ class ContainerManager: password = "".join(password_list) return password - def queue_status_thread(self, status_interval:int=60)->None: + def queue_status_thread(self, status_interval:int=60, num_steps:int=10)->None: while True: - if self.synchronization_object.checkContainerFailure(): - print("One of the containers has shut down prematurely and generated an exception. Shut down the rest of the containers.") - for container in self.containers: - container.stopContainer() - print("All containers stopped") - - return None + #This for loop lets us run the summarize print less often, but check for failure more often + for chunk in range(0, status_interval, int(status_interval/num_steps)): + if self.synchronization_object.checkContainerFailure(): + print("One of the containers has shut down prematurely or the test was halted. Ensuring all containers are stopped.") + for container in self.containers: + container.stopContainer() + print("All containers stopped") + self.all_tests_completed = False + return None + time.sleep(status_interval/num_steps) at_least_one_container_has_started_running_tests = False for container in self.containers: @@ -234,9 +250,9 @@ class ContainerManager: break if self.synchronization_object.summarize(testing_currently_active = at_least_one_container_has_started_running_tests) == False: #There are no more tests to run, so we can return from this thread - + self.all_tests_completed = True return None - time.sleep(status_interval) + def setup_image(self, reuse_images: bool, container_name: str) -> None: client = docker.client.from_env() diff --git a/bin/automated_detection_testing/ci/detection_testing_batch/modules/splunk_container.py b/bin/automated_detection_testing/ci/detection_testing_batch/modules/splunk_container.py index 612e1d1056..1c2c3c7f28 100644 --- a/bin/automated_detection_testing/ci/detection_testing_batch/modules/splunk_container.py +++ b/bin/automated_detection_testing/ci/detection_testing_batch/modules/splunk_container.py @@ -71,6 +71,7 @@ class SplunkContainer: self.num_tests_completed = 0 + def prepare_apps_path( self, apps: OrderedDict, @@ -318,10 +319,36 @@ class SplunkContainer: time.sleep(seconds_between_attempts) - @wrapt_timeout_decorator.timeout(MAX_CONTAINER_START_TIME_SECONDS, timeout_exception=RuntimeError) + #@wrapt_timeout_decorator.timeout(MAX_CONTAINER_START_TIME_SECONDS, timeout_exception=RuntimeError) def setup_container(self): - + self.container.start() + + + # def shutdown_signal_handler(sig, frame): + # shutdown_client = docker.client.from_env() + # errorCount = 0 + + # print(f"Shutting down {self.container_name}...", file=sys.stderr) + # try: + # container = shutdown_client.containers.get(self.container_name) + # #Note that stopping does not remove any of the volumes or logs, + # #so stopping can be useful if we want to debug any container failure + # container.stop(timeout=10) + # print(f"{self.container_name} shut down successfully", file=sys.stderr) + # except Exception as e: + # print(f"Error trying to shut down {self.container_name}. It may have already shut down. Stop it youself with 'docker containter stop {self.container_name}", sys.stderr) + + + # #We must use os._exit(1) because sys.exit(1) actually generates an exception which can be caught! And then we don't Quit! + # import os + # os._exit(1) + + + + # import signal + # signal.signal(signal.SIGINT, shutdown_signal_handler) + # By default, first copy the index file then the datamodel file for file_description, file_dict in self.files_to_copy_to_container.items(): self.extract_tar_file_to_container( @@ -423,7 +450,7 @@ class SplunkContainer: % (detection_to_test, str(e)) ) - traceback.print_exc() + #traceback.print_exc() #import pdb #pdb.set_trace() # Fill in all the "Empty" fields with default values. Otherwise, we will not be able to diff --git a/bin/automated_detection_testing/ci/detection_testing_batch/modules/test_driver.py b/bin/automated_detection_testing/ci/detection_testing_batch/modules/test_driver.py index a55dc9edb2..3bbbbcb6e1 100644 --- a/bin/automated_detection_testing/ci/detection_testing_batch/modules/test_driver.py +++ b/bin/automated_detection_testing/ci/detection_testing_batch/modules/test_driver.py @@ -235,7 +235,7 @@ class TestDriver: if self.checkContainerFailure(): - print("One or more containers crashed, so testing did not complete successfully. We wrote out all the results that we could") + print("One or more containers crashed or the test was HALTED early, so testing did not complete successfully. We wrote out all the results that we could") return False else: return success @@ -269,16 +269,7 @@ class TestDriver: def summarize(self,testing_currently_active:bool=False)->bool: - - if self.checkContainerFailure() == True: - print("Error running containers... shutting down", file=sys.stderr) - return False - - - - self.lock.acquire() - try: #Get a summary of some system stats From 05898d1c7285ca2ed06f132cdb77bcda8230d993 Mon Sep 17 00:00:00 2001 From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com> Date: Tue, 15 Feb 2022 15:43:36 -0800 Subject: [PATCH 6/6] Updated the test config used for github actions and the workflow to remove the use of the splunkbase username and password secrets. After this is merged, we can remove them from the repository. We will simply use the versions that we update in S3. --- .github/workflows/detection-testing.yml | 2 +- .../test_config_github_actions.json | 206 ++++++++---------- 2 files changed, 96 insertions(+), 112 deletions(-) diff --git a/.github/workflows/detection-testing.yml b/.github/workflows/detection-testing.yml index 0146f06aed..98ee2d736a 100644 --- a/.github/workflows/detection-testing.yml +++ b/.github/workflows/detection-testing.yml @@ -163,7 +163,7 @@ jobs: cd bin/automated_detection_testing/ci/detection_testing_batch source .venv/bin/activate - python3 detection_testing_execution.py run -c prior_config/${{ matrix.manifest_filename}} --splunkbase_username ${{ secrets.SPLUNKBASE_TESTING_USERNAME }} --splunkbase_password ${{ secrets.SPLUNKBASE_TESTING_KEY }} + python3 detection_testing_execution.py run -c prior_config/${{ matrix.manifest_filename}} - name: Upload Test Results Files diff --git a/bin/automated_detection_testing/ci/detection_testing_batch/test_config_github_actions.json b/bin/automated_detection_testing/ci/detection_testing_batch/test_config_github_actions.json index 70ba5473a2..fc72e21da8 100644 --- a/bin/automated_detection_testing/ci/detection_testing_batch/test_config_github_actions.json +++ b/bin/automated_detection_testing/ci/detection_testing_batch/test_config_github_actions.json @@ -1,113 +1,103 @@ { + "apps": { + "ADD_ON_FOR_LINUX_SYSMON": { + "app_number": 6176, + "app_version": "1.0.4", + "http_path": "https://attack-range-appbinaries.s3.us-west-2.amazonaws.com/Latest/add-on-for-linux-sysmon_104.tgz" + }, + "PYTHON_FOR_SCIENTIFIC_COMPUTING_FOR_LINUX_64_BIT": { + "app_number": 2882, + "app_version": "3.0.2", + "http_path": "https://attack-range-appbinaries.s3.us-west-2.amazonaws.com/Latest/python-for-scientific-computing-for-linux-64-bit_302.tgz" + }, + "SPLUNK_ADD_ON_FOR_AMAZON_KINESIS_FIREHOSE": { + "app_number": 3719, + "app_version": "1.3.2", + "http_path": "https://attack-range-appbinaries.s3.us-west-2.amazonaws.com/Latest/splunk-add-on-for-amazon-kinesis-firehose_132.tgz" + }, + "SPLUNK_ADD_ON_FOR_MICROSOFT_OFFICE_365": { + "app_number": 4055, + "app_version": "2.2.0", + "http_path": "https://attack-range-appbinaries.s3.us-west-2.amazonaws.com/Latest/splunk-add-on-for-microsoft-office-365_220.tgz" + }, + "SPLUNK_ADD_ON_FOR_MICROSOFT_WINDOWS": { + "app_number": 742, + "app_version": "8.4.0", + "http_path": "https://attack-range-appbinaries.s3.us-west-2.amazonaws.com/Latest/splunk-add-on-for-microsoft-windows_840.tgz" + }, + "SPLUNK_ADD_ON_FOR_NGINX": { + "app_number": 3258, + "app_version": "3.1.0", + "http_path": "https://attack-range-appbinaries.s3.us-west-2.amazonaws.com/Latest/splunk-add-on-for-nginx_310.tgz" + }, + "SPLUNK_ADD_ON_FOR_STREAM_FORWARDERS": { + "app_number": 5238, + "app_version": "8.0.1", + "http_path": "https://attack-range-appbinaries.s3.us-west-2.amazonaws.com/Latest/splunk-add-on-for-stream-forwarders_801.tgz" + }, + "SPLUNK_ADD_ON_FOR_STREAM_WIRE_DATA": { + "app_number": 5234, + "app_version": "8.0.1", + "http_path": "https://attack-range-appbinaries.s3.us-west-2.amazonaws.com/Latest/splunk-add-on-for-stream-wire-data_801.tgz" + }, + "SPLUNK_ADD_ON_FOR_SYSMON": { + "app_number": 5709, + "app_version": "2.0.0", + "http_path": "https://attack-range-appbinaries.s3.us-west-2.amazonaws.com/Latest/splunk-add-on-for-sysmon_200.tgz" + }, + "SPLUNK_ADD_ON_FOR_UNIX_AND_LINUX": { + "app_number": 833, + "app_version": "8.4.0", + "http_path": "https://attack-range-appbinaries.s3.us-west-2.amazonaws.com/Latest/splunk-add-on-for-unix-and-linux_840.tgz" + }, + "SPLUNK_APP_FOR_AWS": { + "app_number": 1274, + "app_version": "6.0.3", + "http_path": "https://attack-range-appbinaries.s3.us-west-2.amazonaws.com/Latest/splunk-app-for-aws_603.tgz" + }, + "SPLUNK_APP_FOR_STREAM": { + "app_number": 1809, + "app_version": "8.0.1", + "http_path": "https://attack-range-appbinaries.s3.us-west-2.amazonaws.com/Latest/splunk-app-for-stream_801.tgz" + }, + "SPLUNK_COMMON_INFORMATION_MODEL": { + "app_number": 1621, + "app_version": "5.0.0", + "http_path": "https://attack-range-appbinaries.s3.us-west-2.amazonaws.com/Latest/splunk-common-information-model-cim_500.tgz" + }, + "SPLUNK_ES_CONTENT_UPDATE": { + "app_number": 3449, + "app_version": null, + "local_path": null + }, + "SPLUNK_MACHINE_LEARNING_TOOLKIT": { + "app_number": 2890, + "app_version": "5.3.1", + "http_path": "https://attack-range-appbinaries.s3.us-west-2.amazonaws.com/Latest/splunk-machine-learning-toolkit_531.tgz" + }, + "SPLUNK_SECURITY_ESSENTIALS": { + "app_number": 3435, + "app_version": "3.4.0", + "http_path": "https://attack-range-appbinaries.s3.us-west-2.amazonaws.com/Latest/splunk-security-essentials_340.tgz" + }, + "SPLUNK_TA_FOR_ZEEK": { + "app_number": 5466, + "app_version": "1.0.5", + "http_path": "https://attack-range-appbinaries.s3.us-west-2.amazonaws.com/Latest/ta-for-zeek_105.tgz" + } + }, "branch": "BRANCH_DOES_NOT_EXIST_USE_CLI_ARGUMENT", "commit_hash": null, "container_tag": "latest", "detections_file": null, "detections_list": null, "folders": [ - "endpoint", - "cloud", - "network", - "web" + "endpoint", + "cloud", + "network", + "web" ], "interactive": false, - "local_apps": { - "SPLUNK_ADD_ON_FOR_AMAZON_KINESIS_FIREHOSE": { - "app_number": 12, - "app_version": null, - "http_path": "https://attack-range-appbinaries.s3-us-west-2.amazonaws.com/splunk-add-on-for-amazon-kinesis-firehose_131r7d1d093.tgz" - }, - "SPLUNK_ADD_ON_FOR_MICROSOFT_OFFICE_365": { - "app_number": 13, - "app_version": null, - "http_path": "https://attack-range-appbinaries.s3-us-west-2.amazonaws.com/splunk-add-on-for-microsoft-office-365_202.tgz" - }, - "SPLUNK_ADD_ON_FOR_STREAM_FORWARDERS": { - "app_number": 9, - "app_version": null, - "http_path": "https://attack-range-appbinaries.s3-us-west-2.amazonaws.com/splunk-add-on-for-stream-forwarders_730.tgz" - }, - "SPLUNK_ADD_ON_FOR_STREAM_WIRE_DATA": { - "app_number": 8, - "app_version": null, - "http_path": "https://attack-range-appbinaries.s3-us-west-2.amazonaws.com/splunk-add-on-for-stream-wire-data_730.tgz" - }, - "SPLUNK_ADD_ON_FOR_SYSMON_OLD": { - "app_number": 1, - "app_version": null, - "http_path": "https://attack-range-appbinaries.s3-us-west-2.amazonaws.com/splunk-add-on-for-microsoft-sysmon_1062.tgz" - }, - "SPLUNK_ADD_ON_FOR_ZEEK_AKA_BRO": { - "app_number": 11, - "app_version": null, - "http_path": "https://attack-range-appbinaries.s3-us-west-2.amazonaws.com/splunk-add-on-for-zeek-aka-bro_400.tgz" - }, - "SPLUNK_ASX_APP": { - "app_number": 17, - "app_version": null, - "http_path": "https://attack-range-appbinaries.s3-us-west-2.amazonaws.com/Splunk_ASX-latest.tar.gz" - }, - "SPLUNK_AWS_TA": { - "app_number": 4, - "app_version": null, - "http_path": "https://attack-range-appbinaries.s3-us-west-2.amazonaws.com/splunk-add-on-for-amazon-web-services_510.tgz" - }, - "SPLUNK_CIM_APP": { - "app_number": 3, - "app_version": null, - "http_path": "https://attack-range-appbinaries.s3-us-west-2.amazonaws.com/splunk-common-information-model-cim_4200.tgz" - }, - "SPLUNK_ES_CONTENT_UPDATE": { - "app_number": 3449, - "app_version": null, - "local_path": null - }, - "SPLUNK_LINUX_TA": { - "app_number": 14, - "app_version": null, - "http_path": "https://attack-range-appbinaries.s3-us-west-2.amazonaws.com/splunk-add-on-for-unix-and-linux_820.tgz" - }, - "SPLUNK_MLTK_APP": { - "app_number": 6, - "app_version": null, - "http_path": "https://attack-range-appbinaries.s3.us-west-2.amazonaws.com/splunk-machine-learning-toolkit_531.tgz" - }, - "SPLUNK_NGINX_TA": { - "app_number": 15, - "app_version": null, - "http_path": "https://attack-range-appbinaries.s3-us-west-2.amazonaws.com/splunk-add-on-for-nginx_310.tgz" - }, - "SPLUNK_PYTHON_APP": { - "app_number": 5, - "app_version": null, - "http_path": "https://attack-range-appbinaries.s3.us-west-2.amazonaws.com/python-for-scientific-computing-for-linux-64-bit_302.tgz" - }, - "SPLUNK_SECURITY_ESSENTIALS": { - "app_number": 10, - "app_version": null, - "http_path": "https://attack-range-appbinaries.s3-us-west-2.amazonaws.com/splunk-security-essentials_333.tgz" - }, - "SPLUNK_STREAM_APP": { - "app_number": 7, - "app_version": null, - "http_path": "https://attack-range-appbinaries.s3-us-west-2.amazonaws.com/splunk-app-for-stream_730.tgz" - }, - "SPLUNK_SYSMON_LINUX_TA_PATCHED": { - "app_number": 2, - "app_version": null, - "http_path": "https://attack-range-appbinaries.s3-us-west-2.amazonaws.com/add-on-for-linux-sysmon_103_PATCHED.tgz" - }, - "SPLUNK_TA_FOR_ZEEK": { - "app_number": 16, - "app_version": null, - "http_path": "https://attack-range-appbinaries.s3-us-west-2.amazonaws.com/ta-for-zeek_105.tgz" - }, - "SPLUNK_WINDOWS_TA": { - "app_number": 0, - "app_version": null, - "http_path": "https://attack-range-appbinaries.s3-us-west-2.amazonaws.com/splunk-add-on-for-microsoft-windows_812.tgz" - } - }, "local_base_container_name": "splunk_test_%d", "mock": false, "mode": "changes", @@ -119,17 +109,11 @@ "show_splunk_app_password": false, "splunk_app_password": null, "splunk_container_apps_directory": "/opt/splunk/etc/apps", - "splunkbase_apps": { - "SPLUNK_ADD_ON_FOR_SYSMON": { - "app_number": 5709, - "app_version": "1.0.1" - } - }, "splunkbase_password": null, "splunkbase_username": null, "types": [ - "Anomaly", - "Hunting", - "TTP" + "Anomaly", + "Hunting", + "TTP" ] -} +} \ No newline at end of file