From f2b7f155eb51dd25d6290f531b384a5f99fc8000 Mon Sep 17 00:00:00 2001 From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com> Date: Thu, 7 Jul 2022 15:30:24 -0700 Subject: [PATCH 1/3] Faster container startup when launching a single container. Same wait time between containers when launching multiple containers. This gets us to startup much faster for the most common testing case, a single container. --- .../modules/container_manager.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/bin/docker_detection_tester/modules/container_manager.py b/bin/docker_detection_tester/modules/container_manager.py index 975bf5d9e6..eaa1f8e733 100644 --- a/bin/docker_detection_tester/modules/container_manager.py +++ b/bin/docker_detection_tester/modules/container_manager.py @@ -137,9 +137,13 @@ class ContainerManager: def run_containers(self) -> None: - for container in self.containers: - #give a little time between container startup - time.sleep(15) + for container_number, container in enumerate(self.containers): + #give a little time between container startup if there is more than one container. + #Never wait on the first container. This gets us to testing as fast as possible + #for the most common case (one container) and gives us some extra time and + #reduces load when we are launching more than one container + if (container_number != 0): + time.sleep(10) container.thread.start() From 81324ddb00b1014d8abbce0b474887d10b183b76 Mon Sep 17 00:00:00 2001 From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com> Date: Thu, 7 Jul 2022 15:58:46 -0700 Subject: [PATCH 2/3] A number of detections are passing locally but failing in CI/CD. I believe this is because they are not being given enough time to finish their data ingest completely. If a search fails, wait some time and run it a few more times to see if it will complete. --- .../modules/splunk_sdk.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/bin/docker_detection_tester/modules/splunk_sdk.py b/bin/docker_detection_tester/modules/splunk_sdk.py index 2ad652aae6..839f7bb2bb 100644 --- a/bin/docker_detection_tester/modules/splunk_sdk.py +++ b/bin/docker_detection_tester/modules/splunk_sdk.py @@ -11,6 +11,7 @@ from typing import Union DEFAULT_EVENT_HOST = "ATTACK_DATA_HOST" DEFAULT_DATA_INDEX = "main" +FAILURE_SLEEP_INTERVAL_SECONDS = 120 def enable_delete_for_admin(splunk_host:str, splunk_port:int, splunk_password:str)->bool: try: @@ -225,7 +226,12 @@ def test_baseline_search(splunk_host, splunk_port, splunk_password, search, pass -def test_detection_search(splunk_host:str, splunk_port:int, splunk_password:str, search:str, pass_condition:str, detection_name:str, detection_file:str, earliest_time:str, latest_time:str)->dict: +def test_detection_search(splunk_host:str, splunk_port:int, splunk_password:str, search:str, pass_condition:str, + detection_name:str, detection_file:str, earliest_time:str, latest_time:str, attempts_remaining:int=2, + failure_sleep_interval_seconds:int=FAILURE_SLEEP_INTERVAL_SECONDS)->dict: + #Since this is an attempt, decrement the number of remaining attempts + attempts_remaining -= 1 + if search.startswith('|'): search = search else: @@ -294,8 +300,15 @@ def test_detection_search(splunk_host:str, splunk_port:int, splunk_password:str, #Should this be 1 for a pass, or should it be greater than 0? if int(job['resultCount']) != 1: #print("Test failed for detection: " + detection_name) - test_results['success'] = False - return test_results + if attempts_remaining > 0: + print(f"Execution of test failed for [{detection_name}]. Sleeping for [{failure_sleep_interval_seconds} seconds] and triyng again...") + time.sleep(failure_sleep_interval_seconds) + return test_detection_search(splunk_host, splunk_port, splunk_password, search, pass_condition, detection_name, detection_file, + earliest_time, latest_time, attempts_remaining=attempts_remaining, + failure_sleep_interval_seconds=failure_sleep_interval_seconds) + else: + test_results['success'] = False + return test_results else: #print("Test successful for detection: " + detection_name) test_results['success'] = True From b1ba36517e10f1df1ed9ee1b17c91d39c6ebdc69 Mon Sep 17 00:00:00 2001 From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com> Date: Thu, 7 Jul 2022 17:09:51 -0700 Subject: [PATCH 3/3] Updated wait time and fixed typo in print statement. --- bin/docker_detection_tester/modules/splunk_sdk.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bin/docker_detection_tester/modules/splunk_sdk.py b/bin/docker_detection_tester/modules/splunk_sdk.py index 839f7bb2bb..2dc6c36556 100644 --- a/bin/docker_detection_tester/modules/splunk_sdk.py +++ b/bin/docker_detection_tester/modules/splunk_sdk.py @@ -11,7 +11,7 @@ from typing import Union DEFAULT_EVENT_HOST = "ATTACK_DATA_HOST" DEFAULT_DATA_INDEX = "main" -FAILURE_SLEEP_INTERVAL_SECONDS = 120 +FAILURE_SLEEP_INTERVAL_SECONDS = 60 def enable_delete_for_admin(splunk_host:str, splunk_port:int, splunk_password:str)->bool: try: @@ -227,7 +227,7 @@ def test_baseline_search(splunk_host, splunk_port, splunk_password, search, pass def test_detection_search(splunk_host:str, splunk_port:int, splunk_password:str, search:str, pass_condition:str, - detection_name:str, detection_file:str, earliest_time:str, latest_time:str, attempts_remaining:int=2, + detection_name:str, detection_file:str, earliest_time:str, latest_time:str, attempts_remaining:int=4, failure_sleep_interval_seconds:int=FAILURE_SLEEP_INTERVAL_SECONDS)->dict: #Since this is an attempt, decrement the number of remaining attempts attempts_remaining -= 1 @@ -301,7 +301,7 @@ def test_detection_search(splunk_host:str, splunk_port:int, splunk_password:str, if int(job['resultCount']) != 1: #print("Test failed for detection: " + detection_name) if attempts_remaining > 0: - print(f"Execution of test failed for [{detection_name}]. Sleeping for [{failure_sleep_interval_seconds} seconds] and triyng again...") + print(f"Execution of test failed for [{detection_name}]. Sleeping for [{failure_sleep_interval_seconds} seconds] and trying again...") time.sleep(failure_sleep_interval_seconds) return test_detection_search(splunk_host, splunk_port, splunk_password, search, pass_condition, detection_name, detection_file, earliest_time, latest_time, attempts_remaining=attempts_remaining,