diff --git a/bin/docker_detection_tester/modules/container_manager.py b/bin/docker_detection_tester/modules/container_manager.py index 975bf5d9e6..eaa1f8e733 100644 --- a/bin/docker_detection_tester/modules/container_manager.py +++ b/bin/docker_detection_tester/modules/container_manager.py @@ -137,9 +137,13 @@ class ContainerManager: def run_containers(self) -> None: - for container in self.containers: - #give a little time between container startup - time.sleep(15) + for container_number, container in enumerate(self.containers): + #give a little time between container startup if there is more than one container. + #Never wait on the first container. This gets us to testing as fast as possible + #for the most common case (one container) and gives us some extra time and + #reduces load when we are launching more than one container + if (container_number != 0): + time.sleep(10) container.thread.start() diff --git a/bin/docker_detection_tester/modules/splunk_sdk.py b/bin/docker_detection_tester/modules/splunk_sdk.py index 2ad652aae6..2dc6c36556 100644 --- a/bin/docker_detection_tester/modules/splunk_sdk.py +++ b/bin/docker_detection_tester/modules/splunk_sdk.py @@ -11,6 +11,7 @@ from typing import Union DEFAULT_EVENT_HOST = "ATTACK_DATA_HOST" DEFAULT_DATA_INDEX = "main" +FAILURE_SLEEP_INTERVAL_SECONDS = 60 def enable_delete_for_admin(splunk_host:str, splunk_port:int, splunk_password:str)->bool: try: @@ -225,7 +226,12 @@ def test_baseline_search(splunk_host, splunk_port, splunk_password, search, pass -def test_detection_search(splunk_host:str, splunk_port:int, splunk_password:str, search:str, pass_condition:str, detection_name:str, detection_file:str, earliest_time:str, latest_time:str)->dict: +def test_detection_search(splunk_host:str, splunk_port:int, splunk_password:str, search:str, pass_condition:str, + detection_name:str, detection_file:str, earliest_time:str, latest_time:str, attempts_remaining:int=4, + failure_sleep_interval_seconds:int=FAILURE_SLEEP_INTERVAL_SECONDS)->dict: + #Since this is an attempt, decrement the number of remaining attempts + attempts_remaining -= 1 + if search.startswith('|'): search = search else: @@ -294,8 +300,15 @@ def test_detection_search(splunk_host:str, splunk_port:int, splunk_password:str, #Should this be 1 for a pass, or should it be greater than 0? if int(job['resultCount']) != 1: #print("Test failed for detection: " + detection_name) - test_results['success'] = False - return test_results + if attempts_remaining > 0: + print(f"Execution of test failed for [{detection_name}]. Sleeping for [{failure_sleep_interval_seconds} seconds] and trying again...") + time.sleep(failure_sleep_interval_seconds) + return test_detection_search(splunk_host, splunk_port, splunk_password, search, pass_condition, detection_name, detection_file, + earliest_time, latest_time, attempts_remaining=attempts_remaining, + failure_sleep_interval_seconds=failure_sleep_interval_seconds) + else: + test_results['success'] = False + return test_results else: #print("Test successful for detection: " + detection_name) test_results['success'] = True