From dc8b704332d89a32f7ae7ed2ad74f38767020b15 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Fri, 17 Sep 2021 15:14:17 -0700
Subject: [PATCH 001/166] Initial changes and testing for local dockerized
 detection testing service

---
 .../ansible/roles/update_escu/tasks/main.yml  |  13 +-
 .../detection_testing_execution.py            | 160 ++++++++++++++++--
 .../modules/testing_service.py                |  21 +--
 .../detection_testing_batch/requirements.txt  |   5 +-
 4 files changed, 168 insertions(+), 31 deletions(-)

diff --git a/automated_detection_testing/ci/detection_testing_batch/ansible/roles/update_escu/tasks/main.yml b/automated_detection_testing/ci/detection_testing_batch/ansible/roles/update_escu/tasks/main.yml
index 869e80ecc6..23fcbb9f24 100644
--- a/automated_detection_testing/ci/detection_testing_batch/ansible/roles/update_escu/tasks/main.yml
+++ b/automated_detection_testing/ci/detection_testing_batch/ansible/roles/update_escu/tasks/main.yml
@@ -14,8 +14,11 @@
     group: splunk
   become: yes
 
-- name: restart splunk
-  service:
-    name: splunkd
-    state: restarted
-  become: yes
\ No newline at end of file
+- name: restart containerized splunk
+  ansible.builtin.shell: /opt/splunk/bin/splunk restart
+  become: yes
+#- name: restart splunk
+#  service:
+#    name: splunkd
+#    state: restarted
+#  become: yes
\ No newline at end of file
diff --git a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
index a1f58721f5..508d68f1ef 100644
--- a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
+++ b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
@@ -3,9 +3,14 @@ import argparse
 import shutil
 import os
 import time
-
+import random
+import secrets
+import docker
+import threading
+import queue
 from modules.github_service import GithubService
 from modules import aws_service, testing_service
+import time
 
 
 DT_ATTACK_RANGE_STATE_STORE = "dt-attack-range-tf-state-store"
@@ -13,18 +18,47 @@ DT_ATTACK_RANGE_STATE = "dt-attack-range-state"
 REGION = "eu-central-1"
 NAME = "detection-testing-attack-range"
 
+PASSWORD_LENGTH=20
+MAX_RECOMMENDED_CONTAINERS_BEFORE_WARNING=2
+DOCKER_HUB_CONTAINER_PATH="splunk/splunk:latest"
+BASE_CONTAINER_NAME="splunk"
+
+DOCKER_COMMIT_NAME = "splunk_configured"
+
+
+BASE_CONTAINER_WEB_PORT=8000
+BASE_CONTAINER_MANAGEMENT_PORT=8089
+
+
+def wait_for_splunk_ready(splunk_container_name=None, splunk_web_port=None, max_seconds=30):
+    #The smarter version of this will try to hit one of the pages,
+    #probably the login page, and when that is available it means that
+    #splunk is fully started and ready to go.  Until then, we just
+    #use a simple sleep
+    time.sleep(max_seconds)
 
 def main(args):
 
     parser = argparse.ArgumentParser(description="CI Detection Testing")
-    parser.add_argument("-b", "--branch", required=True, help="security content branch")
-    parser.add_argument("-u", "--uuid", required=True, help="uuid for detection test")
-    parser.add_argument("-pr", "--pr-number", required=False, help="Pull Request Number")
+    parser.add_argument("-b", "--branch", type=str, required=True, help="security content branch")
+    parser.add_argument("-u", "--uuid", type=str, required=True, help="uuid for detection test")
+    parser.add_argument("-pr", "--pr-number", type=int, required=False, help="Pull Request Number")
+    parser.add_argument("-n", "--num_containers", required=False, type=int, default=1, help="The number of splunk docker containers to start and run for testing")
 
     args = parser.parse_args()
     branch = args.branch
     uuid_test = args.uuid
     pr_number = args.pr_number
+    num_containers = args.num_containers
+    if num_containers < 1:
+        #Perhaps this should be a mock-run - do the initial steps but don't do testing on the containers?
+        print("Error, requested 0 containers.  You must run with at least 1 container.")
+        sys.exit(1)
+    elif num_containers > MAX_RECOMMENDED_CONTAINERS_BEFORE_WARNING:
+        print("You requested to run with [%d] containers which may use a very large amount of resources \
+               as they all run in parallel.  The maximum suggested number of parallel. The maximum \
+               suggested number of containers is [%d].  We will do what you asked, but be warned!"%(num_containers, MAX_RECOMMENDED_CONTAINERS_BEFORE_WARNING))
+
 
     if pr_number:
         github_service = GithubService(branch, pr_number)
@@ -33,20 +67,124 @@ def main(args):
     test_files = github_service.get_changed_test_files()
     if len(test_files) == 0:
         print("No new detections to test.")
-        aws_service.dynamo_db_nothing_to_test(REGION, uuid_test, str(int(time.time())))
+        #aws_service.dynamo_db_nothing_to_test(REGION, uuid_test, str(int(time.time())))
         sys.exit(0)
 
-    dt_ar = aws_service.get_ar_information_from_dynamo_db(REGION, DT_ATTACK_RANGE_STATE)
-    splunk_instance = aws_service.get_splunk_instance(REGION, dt_ar['ssh_key_name'])
+    #dt_ar = aws_service.get_ar_information_from_dynamo_db(REGION, DT_ATTACK_RANGE_STATE)
+    #splunk_instance = aws_service.get_splunk_instance(REGION, dt_ar['ssh_key_name'])
 
-    splunk_ip = splunk_instance['NetworkInterfaces'][0]['Association']['PublicIp']
-    splunk_password = dt_ar['password']
-    ssh_key_name = dt_ar['ssh_key_name']
-    private_key = dt_ar['private_key']
+    #splunk_ip = splunk_instance['NetworkInterfaces'][0]['Association']['PublicIp']
+    #splunk_password = dt_ar['password']
+    #ssh_key_name = dt_ar['ssh_key_name']
+    #private_key = dt_ar['private_key']
+
+    #because this is only accessible to localhost, the password doesn't need to be particularly secure
+    #We can also share it between splunk on all containers
+    
+    splunk_password = secrets.token_urlsafe(PASSWORD_LENGTH)
+    splunk_container_manager_threads = []
+    
+
+    print("***Files to test: %d"%(len(test_files)))
+    test_file_queue = queue.Queue()
+    for filename in test_files:
+        test_file_queue.put(filename)
+    print("***Test files enqueued")
+
+    print("Getting docker client")
+    client = docker.client.from_env()
+    try:
+        print("Removing any existing containers called [%s]."%(BASE_CONTAINER_NAME))
+
+        c = client.containers.get(BASE_CONTAINER_NAME)
+        c.remove(v=True, force=True) #remove it even if it is running. remove volumes as well
+    except:
+        print("Container [%s] did not exist. No need to remove it"%(BASE_CONTAINER_NAME))
+
+    try:
+        try:
+            client.images.get(DOCKER_HUB_CONTAINER_PATH)
+            print("You already have an image named [%s]. We will not "
+                "download it again."%(DOCKER_HUB_CONTAINER_PATH))
+        except:
+            print("You did not have an image named [%s]. We will "
+                "download it now from the Docker Hub.  Please note "
+                "that this could take a long time depending on your "
+                "connection. It's around 2GB."%(DOCKER_HUB_CONTAINER_PATH))
+            client.images.pull(DOCKER_HUB_CONTAINER_PATH)
+            print("Finished downloading the image [%s]"%(DOCKER_HUB_CONTAINER_PATH))
+
+        try:
+            image = client.images.get(DOCKER_COMMIT_NAME)
+            print("Found an image called [%s]. We will remove it"%(DOCKER_COMMIT_NAME))
+            #Stop it if it's running, remove associated volumes too
+            image.remove(v=True, force=True)
+        except:
+            print("No image found named [%s]"%(DOCKER_COMMIT_NAME))
+        
+            
+
+        
+
+        print("Creating a new container called [%s]"%(BASE_CONTAINER_NAME))
+        environment = {"SPLUNK_START_ARGS": "--accept-license",
+                       "SPLUNK_PASSWORD"  : splunk_password }
+        ports= {"8000/tcp": BASE_CONTAINER_WEB_PORT - 1,
+                "8089/tcp": BASE_CONTAINER_MANAGEMENT_PORT - 1
+                }
+
+        base_container = client.containers.create("splunk/splunk:latest", ports=ports, environment=environment, name=BASE_CONTAINER_NAME, detach=True)
+        print("Running the new container called [%s]"%(BASE_CONTAINER_NAME))
+        base_container.start()
+        print("Container is running [%s]"%(BASE_CONTAINER_NAME))
+        print("Sleep for 60 seconds to allow the container to fully start up...")
+        wait_for_splunk_ready(max_seconds=60)
+        print("The container has fully started!")
+
+    except Exception as e:
+        print("There was an error getting the base container up and running.  "
+            "We cannot recover from this: [%s]\nGoodbye..."%(str(e)))
+        sys.exit(1)
+
+    print("Do the ESCU installation on this container. That way we don't have to "
+          "do it on every container that we then spin up.")
+
+    testing_service.prepare_detection_testing(BASE_CONTAINER_NAME, splunk_password)
+    print("Waiting for a few seconds for the splunk app to come up.")
+    wait_for_splunk_ready(max_seconds=30)
+    
+    print("Stopping the running container [%s]"%(BASE_CONTAINER_NAME))
+    base_container.stop()
+
+    print("Committing the configured container: [%s]--->[%s]"%(BASE_CONTAINER_NAME, DOCKER_COMMIT_NAME))
+    base_container.commit(repository=DOCKER_COMMIT_NAME)
+
+
+    '''
+    print("Removing container called splunk if it exists already...")
+    client.container.r
+
+    print("Build base docker container")
+
+
+    for container_index in range(num_containers):
+        container_name = "splunk_runner_%d"%(container_index)
+        t = threading.Thread(target=splunk_container_manager, args=(test_file_queue, container_name, splunk_password))
+        splunk_container_manager_threads.append(t)
+
+    splunk_container_name = "splunk"
 
     testing_service.prepare_detection_testing(ssh_key_name, private_key, splunk_ip, splunk_password)
     testing_service.test_detections(ssh_key_name, private_key, splunk_ip, splunk_password, test_files, uuid_test)
+    '''
 
+def splunk_container_manager(testing_queue, container_name, splunk_password):
+    #Is this going to be safe to use in different threads
+    docker_client = docker.client.from_env()
+    #start up the container from the base container
+    #Assume that the base container has already been fully built with
+    #escu etc
+    pass
 
 if __name__ == "__main__":
     main(sys.argv[1:])
\ No newline at end of file
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/testing_service.py b/automated_detection_testing/ci/detection_testing_batch/modules/testing_service.py
index cdb2502b3e..ab7511ffd5 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/testing_service.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/testing_service.py
@@ -10,20 +10,14 @@ from modules.DataManipulation import DataManipulation
 from modules import splunk_sdk, aws_service
 
 
-def prepare_detection_testing(ssh_key_name, private_key, splunk_ip, splunk_password):
-    with open(ssh_key_name, 'w') as file :
-        file.write(private_key)
-    os.chmod(ssh_key_name, 0o600)
-
-    sys.path.append(os.path.join(os.getcwd(),'security_content/bin'))
-
+def prepare_detection_testing(splunk_ip, splunk_password):
     try:
         module = __import__('generate')
         results = module.main(REPO_PATH = 'security_content' , OUTPUT_PATH = 'security_content/dist/escu', PRODUCT = 'ESCU', VERBOSE = 'False' )
     except Exception as e:
         print('Error: ' + str(e))
 
-    update_ESCU_app(splunk_ip, ssh_key_name, splunk_password)
+    update_ESCU_app(splunk_ip, splunk_password)
 
 
 def test_detections(ssh_key_name, private_key, splunk_ip, splunk_password, test_files, uuid_test):
@@ -128,22 +122,21 @@ def load_file(file_path):
     return file
 
 
-def update_ESCU_app(splunk_ip, ssh_key_name, splunk_password):
+def update_ESCU_app(container_name, splunk_password):
     print("Update ESCU App. This can take some time")
 
     ansible_vars = {}
-    ansible_vars['ansible_user'] = 'ubuntu'
-    ansible_vars['ansible_ssh_private_key_file'] = ssh_key_name
+    ansible_vars['ansible_user'] = 'ansible'
     ansible_vars['splunk_password'] = splunk_password
     ansible_vars['security_content_path'] = 'security_content'
-
-    cmdline = "-i %s, -u ubuntu" % (splunk_ip)
+    
+    cmdline = "--connection docker -i %s, -u %s" % (container_name, ansible_vars['ansible_user'])
     runner = ansible_runner.run(private_data_dir=os.path.join(os.path.dirname(__file__), '../'),
                                 cmdline=cmdline,
                                 roles_path=os.path.join(os.path.dirname(__file__), '../ansible/roles'),
                                 playbook=os.path.join(os.path.dirname(__file__), '../ansible/update_escu.yml'),
                                 extravars=ansible_vars)
-
+    print("Successfully updated the ESCU App!")
 
 
 def replay_attack_dataset(splunk_ip, splunk_password, ssh_key_name, folder_name, index, sourcetype, source, out):
diff --git a/automated_detection_testing/ci/detection_testing_batch/requirements.txt b/automated_detection_testing/ci/detection_testing_batch/requirements.txt
index 698a3b0abf..9fd488fd1f 100644
--- a/automated_detection_testing/ci/detection_testing_batch/requirements.txt
+++ b/automated_detection_testing/ci/detection_testing_batch/requirements.txt
@@ -7,4 +7,7 @@ Jinja2==3.0.0
 PyYAML==5.4
 requests==2.25.1
 six==1.16.0
-splunk-sdk==1.6.12
\ No newline at end of file
+splunk-sdk==1.6.12
+#newest version of docker for managing the splunk containers
+#we will freeze at a specific version later
+docker
\ No newline at end of file

From e5c87027c10d05822a8c7485093943031a5471a5 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Fri, 17 Sep 2021 16:45:05 -0700
Subject: [PATCH 002/166] Lots more changes building out the skeleton of the
 docker testing framework.  Needs testing, breakup into simpler files,
 documentation, etc. But a good start.

---
 .../detection_testing_execution.py            | 100 +++++++++++++++---
 .../modules/splunk_sdk.py                     |  12 +--
 .../modules/testing_service.py                |  56 +++++-----
 3 files changed, 115 insertions(+), 53 deletions(-)

diff --git a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
index 508d68f1ef..3ca89650c0 100644
--- a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
+++ b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
@@ -24,6 +24,7 @@ DOCKER_HUB_CONTAINER_PATH="splunk/splunk:latest"
 BASE_CONTAINER_NAME="splunk"
 
 DOCKER_COMMIT_NAME = "splunk_configured"
+RUNNER_BASE_NAME = "splunk_runner"
 
 
 BASE_CONTAINER_WEB_PORT=8000
@@ -156,35 +157,100 @@ def main(args):
     print("Stopping the running container [%s]"%(BASE_CONTAINER_NAME))
     base_container.stop()
 
+    #The part below does not seem to be working as expected. Will need to look into it
+    #When I create the new container, it fails to boot with 
+    '''The CA file specified (/opt/splunk/etc/auth/cacert.pem) does not exist. Cannot continue.
+    SSL certificate generation failed.
+
+
+    MSG:
+
+    non-zero return code
+    '''
+    #I am almost positive that I'm doing this wrong but it works for now...
     print("Committing the configured container: [%s]--->[%s]"%(BASE_CONTAINER_NAME, DOCKER_COMMIT_NAME))
     base_container.commit(repository=DOCKER_COMMIT_NAME)
+    
 
-
-    '''
-    print("Removing container called splunk if it exists already...")
-    client.container.r
-
-    print("Build base docker container")
-
-
+    print("Make all the threads...")
     for container_index in range(num_containers):
-        container_name = "splunk_runner_%d"%(container_index)
-        t = threading.Thread(target=splunk_container_manager, args=(test_file_queue, container_name, splunk_password))
+        container_name = "%s_%d"%(RUNNER_BASE_NAME, container_index)
+        web_port = BASE_CONTAINER_WEB_PORT  + container_index
+        management_port = BASE_CONTAINER_MANAGEMENT_PORT + container_index
+        print("Creating a new container called [%s]"%(container_name))
+        environment = {"SPLUNK_START_ARGS": "--accept-license",
+                       "SPLUNK_PASSWORD"  : splunk_password }
+        ports= {"8000/tcp": web_port,
+                "8089/tcp": management_port
+                }
+
+        test_container = client.containers.create(DOCKER_COMMIT_NAME, ports=ports, environment=environment, name=container_name, detach=True, volumes_from=[BASE_CONTAINER_NAME])
+        t = threading.Thread(target=splunk_container_manager, args=(test_file_queue, container_name, "127.0.0.1", splunk_password, management_port, uuid_test))
         splunk_container_manager_threads.append(t)
 
-    splunk_container_name = "splunk"
+    print("Start all the threads...")
+    for t in splunk_container_manager_threads:
+        t.start()
+    
+    #Try to join all the threads
+    for t in splunk_container_manager_threads:
+        t.join() #blocks on waiting to join
+        print("Joined a thread!")
 
-    testing_service.prepare_detection_testing(ssh_key_name, private_key, splunk_ip, splunk_password)
-    testing_service.test_detections(ssh_key_name, private_key, splunk_ip, splunk_password, test_files, uuid_test)
-    '''
+    print("DONE!")
+    #read all the results out from the output queue
+    try:
+        while True:
 
-def splunk_container_manager(testing_queue, container_name, splunk_password):
+            o = output_queue.get(block=False)
+            print("Got from queue:")
+            print(o)
+    except queue.Empty:
+        print("That's all the output!")
+    
+    #now we are done!
+
+
+    #detection testing service has already been prepared, no need to do it here!
+    #testing_service.prepare_detection_testing(ssh_key_name, private_key, splunk_ip, splunk_password)
+
+    #testing_service.test_detections(ssh_key_name, private_key, splunk_ip, splunk_password, test_files, uuid_test)
+    
+
+def splunk_container_manager(testing_queue, container_name, splunk_ip, splunk_password, splunk_port, uuid_test, output_queue):
+    print("Starting the container [%s] after a sleep"%(container_name))
     #Is this going to be safe to use in different threads
-    docker_client = docker.client.from_env()
+    client = docker.client.from_env()
     #start up the container from the base container
     #Assume that the base container has already been fully built with
     #escu etc
-    pass
+    #sleep for a little bit so that we don't all start at once...
+    time.sleep(random.randrange(0,120))
+
+    container = client.containers.get(container_name)
+    print("Starting the container [%s]"%(container_name))
+    container.start()
+    wait_for_splunk_ready(max_seconds=60)
+
+    index=0
+    try:
+        while True:
+            #Try to get something from the queue
+            detection_to_test = testing_queue.get(block=False)
+            
+            
+            #There is a detection to test
+            print("Container [%s]--->[%s]"%(container_name, detection_to_test))
+            
+            result = testing_service.test_detection_wrapper(container_name, splunk_ip, splunk_password, splunk_port, detection_to_test, index, uuid_test)
+            output_queue.put(result)
+            index=(index+1)%10
+    except queue.Empty:
+        print("Queue was empty, [%s] finished testing detections!"%(container_name))
+    
+    print("Shutting down the container [%s]"%(container_name))
+    container.stop()
+    print("Finished shutting down the container [%s]"&(container_name))
 
 if __name__ == "__main__":
     main(sys.argv[1:])
\ No newline at end of file
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/splunk_sdk.py b/automated_detection_testing/ci/detection_testing_batch/modules/splunk_sdk.py
index 159e43eecc..854ff47727 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/splunk_sdk.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/splunk_sdk.py
@@ -5,11 +5,11 @@ import splunklib.client as client
 import splunklib.results as results
 import requests
 
-def test_baseline_search(splunk_host, splunk_password, search, pass_condition, baseline_name, baseline_file, earliest_time, latest_time):
+def test_baseline_search(splunk_host, splunk_port, splunk_password, search, pass_condition, baseline_name, baseline_file, earliest_time, latest_time):
     try:
         service = client.connect(
             host=splunk_host,
-            port=8089,
+            port=splunk_port,
             username='admin',
             password=splunk_password
         )
@@ -54,11 +54,11 @@ def test_baseline_search(splunk_host, splunk_password, search, pass_condition, b
         return test_results
 
 
-def test_detection_search(splunk_host, splunk_password, search, pass_condition, detection_name, detection_file, earliest_time, latest_time):
+def test_detection_search(splunk_host, splunk_port, splunk_password, search, pass_condition, detection_name, detection_file, earliest_time, latest_time):
     try:
         service = client.connect(
             host=splunk_host,
-            port=8089,
+            port=splunk_port,
             username='admin',
             password=splunk_password
         )
@@ -103,11 +103,11 @@ def test_detection_search(splunk_host, splunk_password, search, pass_condition,
         return test_results
 
 
-def delete_attack_data(splunk_host, splunk_password):
+def delete_attack_data(splunk_host, splunk_password, splunk_port):
     try:
         service = client.connect(
             host=splunk_host,
-            port=8089,
+            port=splunk_port,
             username='admin',
             password=splunk_password
         )
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/testing_service.py b/automated_detection_testing/ci/detection_testing_batch/modules/testing_service.py
index ab7511ffd5..172536bc16 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/testing_service.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/testing_service.py
@@ -20,43 +20,40 @@ def prepare_detection_testing(splunk_ip, splunk_password):
     update_ESCU_app(splunk_ip, splunk_password)
 
 
-def test_detections(ssh_key_name, private_key, splunk_ip, splunk_password, test_files, uuid_test):
-    test_index = 1
-    result_tests = []
-    for test_file in test_files:
-        uuid_var = str(uuid.uuid4())
-        result_test = test_detection(ssh_key_name, private_key, splunk_ip, splunk_password, test_file, test_index, uuid_test, uuid_var)
-        result_tests.append(result_test)
-        if test_index == 10:
-            test_index = 1
-        else:
-            test_index = test_index + 1
+def test_detection_wrapper(container_name, splunk_ip, splunk_password, splunk_port, test_file, test_index, uuid_test):
 
-        # delete test data
-        splunk_sdk.delete_attack_data(splunk_ip, splunk_password)
+    uuid_var = str(uuid.uuid4())
+    result_test = test_detection(splunk_ip, splunk_port, container_name, splunk_password, test_file, test_index, uuid_test, uuid_var)
+    
 
-    for result_test in result_tests:
-        if result_test['detection_result']['error']:
-            print('Test failed for detection: ' + result_test['detection_result']['detection_name'] + ' ' + result_test['detection_result']['detection_file'])
-        else:
-            print('Test passed for detection: ' + result_test['detection_result']['detection_name'] + ' ' + result_test['detection_result']['detection_file'])
+    # delete test data
+    splunk_sdk.delete_attack_data(container_name, splunk_password, splunk_port)
 
-    return result_tests
+    
+    if result_test['detection_result']['error']:
+        print('Test failed for detection: ' + result_test['detection_result']['detection_name'] + ' ' + result_test['detection_result']['detection_file'])
+    else:
+        print('Test passed for detection: ' + result_test['detection_result']['detection_name'] + ' ' + result_test['detection_result']['detection_file'])
+
+    return result_test    
 
 
-def test_detection(ssh_key_name, private_key, splunk_ip, splunk_password, test_file, test_index, uuid_test, uuid_var):
+def test_detection(splunk_ip, splunk_port, container_name, splunk_password, test_file, test_index, uuid_test, uuid_var):
     try:
         test_file_obj = load_file("security_content/" + test_file[2:])
     except Exception as e:
-        print('Error: ' + str(e))
-        return
+        raise
+        #print('Error: ' + str(e))
+        #return
     
     if not test_file_obj:
+        print("Not test_file_obj!")
+        raise
         return
     #print(test_file_obj)
 
     # write entry dynamodb
-    aws_service.add_detection_results_in_dynamo_db('eu-central-1', uuid_var , uuid_test, test_file_obj['tests'][0]['name'], test_file_obj['tests'][0]['file'], str(int(time.time())))
+    #aws_service.add_detection_results_in_dynamo_db('eu-central-1', uuid_var , uuid_test, test_file_obj['tests'][0]['name'], test_file_obj['tests'][0]['file'], str(int(time.time())))
 
     epoch_time = str(int(time.time()))
     folder_name = "attack_data_" + epoch_time
@@ -74,7 +71,7 @@ def test_detection(ssh_key_name, private_key, splunk_ip, splunk_password, test_f
                 data_manipulation = DataManipulation()
                 data_manipulation.manipulate_timestamp(folder_name + '/' + attack_data['file_name'], attack_data['sourcetype'], attack_data['source'])
 
-        replay_attack_dataset(splunk_ip, splunk_password, ssh_key_name, folder_name, 'test' + str(test_index), attack_data['sourcetype'], attack_data['source'], attack_data['file_name'])
+        replay_attack_dataset(container_name, splunk_password, folder_name, 'test' + str(test_index), attack_data['sourcetype'], attack_data['source'], attack_data['file_name'])
 
     time.sleep(200)
 
@@ -89,12 +86,12 @@ def test_detection(ssh_key_name, private_key, splunk_ip, splunk_password, test_f
             result_obj = dict()
             result_obj['baseline'] = baseline_obj['name']
             result_obj['baseline_file'] = baseline_obj['file']
-            result = splunk_sdk.test_baseline_search(splunk_ip, splunk_password, baseline['search'], baseline_obj['pass_condition'], baseline['name'], baseline_obj['file'], baseline_obj['earliest_time'], baseline_obj['latest_time'])
+            result = splunk_sdk.test_baseline_search(splunk_ip, splunk_port, splunk_password, baseline['search'], baseline_obj['pass_condition'], baseline['name'], baseline_obj['file'], baseline_obj['earliest_time'], baseline_obj['latest_time'])
         result_test['baselines_result'] = results_baselines  
 
     detection_file_name = test['file']
     detection = load_file(os.path.join(os.path.dirname(__file__), '../security_content/detections', detection_file_name))
-    result_detection = splunk_sdk.test_detection_search(splunk_ip, splunk_password, detection['search'], test['pass_condition'], detection['name'], test['file'], test['earliest_time'], test['latest_time'])
+    result_detection = splunk_sdk.test_detection_search(splunk_ip, splunk_port, splunk_password, detection['search'], test['pass_condition'], detection['name'], test['file'], test['earliest_time'], test['latest_time'])
 
     result_detection['detection_name'] = test['name']
     result_detection['detection_file'] = test['file']
@@ -139,18 +136,17 @@ def update_ESCU_app(container_name, splunk_password):
     print("Successfully updated the ESCU App!")
 
 
-def replay_attack_dataset(splunk_ip, splunk_password, ssh_key_name, folder_name, index, sourcetype, source, out):
+def replay_attack_dataset(container_name, splunk_password, folder_name, index, sourcetype, source, out):
     ansible_vars = {}
     ansible_vars['folder_name'] = folder_name
-    ansible_vars['ansible_user'] = 'ubuntu'
-    ansible_vars['ansible_ssh_private_key_file'] = ssh_key_name
+    ansible_vars['ansible_user'] = 'ansible'
     ansible_vars['splunk_password'] = splunk_password
     ansible_vars['out'] = out
     ansible_vars['sourcetype'] = sourcetype
     ansible_vars['source'] = source
     ansible_vars['index'] = index
 
-    cmdline = "-i %s, -u ubuntu" % (splunk_ip)
+    cmdline = "--connection docker -i %s, -u %s" % (container_name, ansible_vars['ansible_user'])
     runner = ansible_runner.run(private_data_dir=os.path.join(os.path.dirname(__file__), '../'),
                                 cmdline=cmdline,
                                 roles_path=os.path.join(os.path.dirname(__file__), '../ansible/roles'),

From 4d1c456a337a5142e4fb790836079163ae4d3ca7 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Fri, 17 Sep 2021 16:49:05 -0700
Subject: [PATCH 003/166] Removed some aws calls which should no longer be made

---
 .../ci/detection_testing_batch/modules/testing_service.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/testing_service.py b/automated_detection_testing/ci/detection_testing_batch/modules/testing_service.py
index 172536bc16..6cee4cb149 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/testing_service.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/testing_service.py
@@ -7,7 +7,7 @@ import os
 import time
 import requests
 from modules.DataManipulation import DataManipulation
-from modules import splunk_sdk, aws_service
+from modules import splunk_sdk
 
 
 def prepare_detection_testing(splunk_ip, splunk_password):
@@ -98,9 +98,11 @@ def test_detection(splunk_ip, splunk_port, container_name, splunk_password, test
     result_test['detection_result'] = result_detection
 
     if result_detection['error']:
-        aws_service.update_detection_results_in_dynamo_db('eu-central-1', uuid_var, 'failed')
+        print("failed")
+        #aws_service.update_detection_results_in_dynamo_db('eu-central-1', uuid_var, 'failed')
     else:
-        aws_service.update_detection_results_in_dynamo_db('eu-central-1', uuid_var, 'passed')
+        print("passed")
+        #aws_service.update_detection_results_in_dynamo_db('eu-central-1', uuid_var, 'passed')
 
     return result_test
 

From ebb00af725ac332888168dc6ed05fd5e9b29b870 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Fri, 17 Sep 2021 16:58:14 -0700
Subject: [PATCH 004/166] Forgot to declare results_queue before using it.

---
 .../detection_testing_batch/detection_testing_execution.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
index 3ca89650c0..a5fe12c9a6 100644
--- a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
+++ b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
@@ -173,6 +173,7 @@ def main(args):
     
 
     print("Make all the threads...")
+    results_queue = queue.Queue()
     for container_index in range(num_containers):
         container_name = "%s_%d"%(RUNNER_BASE_NAME, container_index)
         web_port = BASE_CONTAINER_WEB_PORT  + container_index
@@ -185,7 +186,7 @@ def main(args):
                 }
 
         test_container = client.containers.create(DOCKER_COMMIT_NAME, ports=ports, environment=environment, name=container_name, detach=True, volumes_from=[BASE_CONTAINER_NAME])
-        t = threading.Thread(target=splunk_container_manager, args=(test_file_queue, container_name, "127.0.0.1", splunk_password, management_port, uuid_test))
+        t = threading.Thread(target=splunk_container_manager, args=(test_file_queue, container_name, "127.0.0.1", splunk_password, management_port, uuid_test, results_queue))
         splunk_container_manager_threads.append(t)
 
     print("Start all the threads...")
@@ -217,7 +218,7 @@ def main(args):
     #testing_service.test_detections(ssh_key_name, private_key, splunk_ip, splunk_password, test_files, uuid_test)
     
 
-def splunk_container_manager(testing_queue, container_name, splunk_ip, splunk_password, splunk_port, uuid_test, output_queue):
+def splunk_container_manager(testing_queue, container_name, splunk_ip, splunk_password, splunk_port, uuid_test, results_queue):
     print("Starting the container [%s] after a sleep"%(container_name))
     #Is this going to be safe to use in different threads
     client = docker.client.from_env()
@@ -243,7 +244,7 @@ def splunk_container_manager(testing_queue, container_name, splunk_ip, splunk_pa
             print("Container [%s]--->[%s]"%(container_name, detection_to_test))
             
             result = testing_service.test_detection_wrapper(container_name, splunk_ip, splunk_password, splunk_port, detection_to_test, index, uuid_test)
-            output_queue.put(result)
+            results_queue.put(result)
             index=(index+1)%10
     except queue.Empty:
         print("Queue was empty, [%s] finished testing detections!"%(container_name))

From 94f05e573c66d9bb230710d1c36f53d6c0cf8568 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Fri, 17 Sep 2021 17:33:20 -0700
Subject: [PATCH 005/166] Added some error handling so that the show can go on
 while we debug.  Fixed another bad variable naming error

---
 .../detection_testing_execution.py            |  2 +-
 .../modules/testing_service.py                | 51 ++++++++++---------
 2 files changed, 28 insertions(+), 25 deletions(-)

diff --git a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
index a5fe12c9a6..64c21bd350 100644
--- a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
+++ b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
@@ -203,7 +203,7 @@ def main(args):
     try:
         while True:
 
-            o = output_queue.get(block=False)
+            o = results_queue.get(block=False)
             print("Got from queue:")
             print(o)
     except queue.Empty:
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/testing_service.py b/automated_detection_testing/ci/detection_testing_batch/modules/testing_service.py
index 6cee4cb149..3a0f8cbbd0 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/testing_service.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/testing_service.py
@@ -77,33 +77,36 @@ def test_detection(splunk_ip, splunk_port, container_name, splunk_password, test
 
     result_test = {}
     test = test_file_obj['tests'][0]
+    try:
+        if 'baselines' in test:
+            results_baselines = []
+            for baseline_obj in test['baselines']:
+                baseline_file_name = baseline_obj['file']
+                baseline = load_file(os.path.join(os.path.dirname(__file__), '../security_content', baseline_file_name))
+                result_obj = dict()
+                result_obj['baseline'] = baseline_obj['name']
+                result_obj['baseline_file'] = baseline_obj['file']
+                result = splunk_sdk.test_baseline_search(splunk_ip, splunk_port, splunk_password, baseline['search'], baseline_obj['pass_condition'], baseline['name'], baseline_obj['file'], baseline_obj['earliest_time'], baseline_obj['latest_time'])
+            result_test['baselines_result'] = results_baselines  
 
-    if 'baselines' in test:
-        results_baselines = []
-        for baseline_obj in test['baselines']:
-            baseline_file_name = baseline_obj['file']
-            baseline = load_file(os.path.join(os.path.dirname(__file__), '../security_content', baseline_file_name))
-            result_obj = dict()
-            result_obj['baseline'] = baseline_obj['name']
-            result_obj['baseline_file'] = baseline_obj['file']
-            result = splunk_sdk.test_baseline_search(splunk_ip, splunk_port, splunk_password, baseline['search'], baseline_obj['pass_condition'], baseline['name'], baseline_obj['file'], baseline_obj['earliest_time'], baseline_obj['latest_time'])
-        result_test['baselines_result'] = results_baselines  
+        detection_file_name = test['file']
+        detection = load_file(os.path.join(os.path.dirname(__file__), '../security_content/detections', detection_file_name))
+        result_detection = splunk_sdk.test_detection_search(splunk_ip, splunk_port, splunk_password, detection['search'], test['pass_condition'], detection['name'], test['file'], test['earliest_time'], test['latest_time'])
 
-    detection_file_name = test['file']
-    detection = load_file(os.path.join(os.path.dirname(__file__), '../security_content/detections', detection_file_name))
-    result_detection = splunk_sdk.test_detection_search(splunk_ip, splunk_port, splunk_password, detection['search'], test['pass_condition'], detection['name'], test['file'], test['earliest_time'], test['latest_time'])
-
-    result_detection['detection_name'] = test['name']
-    result_detection['detection_file'] = test['file']
-    result_test['detection_result'] = result_detection
-
-    if result_detection['error']:
-        print("failed")
-        #aws_service.update_detection_results_in_dynamo_db('eu-central-1', uuid_var, 'failed')
-    else:
-        print("passed")
-        #aws_service.update_detection_results_in_dynamo_db('eu-central-1', uuid_var, 'passed')
+        result_detection['detection_name'] = test['name']
+        result_detection['detection_file'] = test['file']
+        result_test['detection_result'] = result_detection
 
+        if result_detection['error']:
+            print("failed")
+            #aws_service.update_detection_results_in_dynamo_db('eu-central-1', uuid_var, 'failed')
+        else:
+            print("passed")
+            #aws_service.update_detection_results_in_dynamo_db('eu-central-1', uuid_var, 'passed')
+    except Exception as e:
+        print("Caught some exception in test detection: [%s]"%(str(e)))
+        #just log the error itself for now so that we can continue
+        result_test = str(e)
     return result_test
 
 

From 5588eeabe1faf006490d7d74c9a5c8b4234433f5 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Fri, 17 Sep 2021 17:35:50 -0700
Subject: [PATCH 006/166] Fixed ugly multi line comment

---
 .../detection_testing_batch/detection_testing_execution.py  | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
index 64c21bd350..10d3c685be 100644
--- a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
+++ b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
@@ -56,9 +56,9 @@ def main(args):
         print("Error, requested 0 containers.  You must run with at least 1 container.")
         sys.exit(1)
     elif num_containers > MAX_RECOMMENDED_CONTAINERS_BEFORE_WARNING:
-        print("You requested to run with [%d] containers which may use a very large amount of resources \
-               as they all run in parallel.  The maximum suggested number of parallel. The maximum \
-               suggested number of containers is [%d].  We will do what you asked, but be warned!"%(num_containers, MAX_RECOMMENDED_CONTAINERS_BEFORE_WARNING))
+        print("You requested to run with [%d] containers which may use a very large amount of resources "
+               "as they all run in parallel.  The maximum suggested number of parallel. The maximum "
+               "suggested number of containers is [%d].  We will do what you asked, but be warned!"%(num_containers, MAX_RECOMMENDED_CONTAINERS_BEFORE_WARNING))
 
 
     if pr_number:

From 2a3e220249c4a4495574016b36b8ba0f92214496 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Fri, 17 Sep 2021 17:51:27 -0700
Subject: [PATCH 007/166] Made the splunk management port dynamic for uploading
 replay data.

---
 .../ansible/roles/attack_replay/tasks/main.yml               | 2 +-
 .../ci/detection_testing_batch/modules/testing_service.py    | 5 +++--
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/automated_detection_testing/ci/detection_testing_batch/ansible/roles/attack_replay/tasks/main.yml b/automated_detection_testing/ci/detection_testing_batch/ansible/roles/attack_replay/tasks/main.yml
index 5265bd86ff..1f1ccfe8e9 100644
--- a/automated_detection_testing/ci/detection_testing_batch/ansible/roles/attack_replay/tasks/main.yml
+++ b/automated_detection_testing/ci/detection_testing_batch/ansible/roles/attack_replay/tasks/main.yml
@@ -8,7 +8,7 @@
 
 - name: Call oneshot import
   uri:
-    url: https://localhost:8089/services/data/inputs/oneshot
+    url: https://localhost:{{ splunk_management_port }}/services/data/inputs/oneshot
     validate_certs: no
     method: POST
     user: admin
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/testing_service.py b/automated_detection_testing/ci/detection_testing_batch/modules/testing_service.py
index 3a0f8cbbd0..e785833828 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/testing_service.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/testing_service.py
@@ -71,7 +71,7 @@ def test_detection(splunk_ip, splunk_port, container_name, splunk_password, test
                 data_manipulation = DataManipulation()
                 data_manipulation.manipulate_timestamp(folder_name + '/' + attack_data['file_name'], attack_data['sourcetype'], attack_data['source'])
 
-        replay_attack_dataset(container_name, splunk_password, folder_name, 'test' + str(test_index), attack_data['sourcetype'], attack_data['source'], attack_data['file_name'])
+        replay_attack_dataset(container_name, splunk_port, splunk_password, folder_name, 'test' + str(test_index), attack_data['sourcetype'], attack_data['source'], attack_data['file_name'])
 
     time.sleep(200)
 
@@ -141,7 +141,7 @@ def update_ESCU_app(container_name, splunk_password):
     print("Successfully updated the ESCU App!")
 
 
-def replay_attack_dataset(container_name, splunk_password, folder_name, index, sourcetype, source, out):
+def replay_attack_dataset(container_name, splunk_port, splunk_password, folder_name, index, sourcetype, source, out):
     ansible_vars = {}
     ansible_vars['folder_name'] = folder_name
     ansible_vars['ansible_user'] = 'ansible'
@@ -150,6 +150,7 @@ def replay_attack_dataset(container_name, splunk_password, folder_name, index, s
     ansible_vars['sourcetype'] = sourcetype
     ansible_vars['source'] = source
     ansible_vars['index'] = index
+    ansible_vars['splunk_management_port'] = splunk_port
 
     cmdline = "--connection docker -i %s, -u %s" % (container_name, ansible_vars['ansible_user'])
     runner = ansible_runner.run(private_data_dir=os.path.join(os.path.dirname(__file__), '../'),

From ba78e372e2789faef60e81ce91c3f3f42ca0dc84 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Fri, 17 Sep 2021 18:12:02 -0700
Subject: [PATCH 008/166] Changed a port back to the original since it's run
 inside the docker container as part of a playbook

---
 .../ansible/roles/attack_replay/tasks/main.yml               | 2 +-
 .../ci/detection_testing_batch/modules/testing_service.py    | 5 ++---
 2 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/automated_detection_testing/ci/detection_testing_batch/ansible/roles/attack_replay/tasks/main.yml b/automated_detection_testing/ci/detection_testing_batch/ansible/roles/attack_replay/tasks/main.yml
index 1f1ccfe8e9..5265bd86ff 100644
--- a/automated_detection_testing/ci/detection_testing_batch/ansible/roles/attack_replay/tasks/main.yml
+++ b/automated_detection_testing/ci/detection_testing_batch/ansible/roles/attack_replay/tasks/main.yml
@@ -8,7 +8,7 @@
 
 - name: Call oneshot import
   uri:
-    url: https://localhost:{{ splunk_management_port }}/services/data/inputs/oneshot
+    url: https://localhost:8089/services/data/inputs/oneshot
     validate_certs: no
     method: POST
     user: admin
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/testing_service.py b/automated_detection_testing/ci/detection_testing_batch/modules/testing_service.py
index e785833828..3a0f8cbbd0 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/testing_service.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/testing_service.py
@@ -71,7 +71,7 @@ def test_detection(splunk_ip, splunk_port, container_name, splunk_password, test
                 data_manipulation = DataManipulation()
                 data_manipulation.manipulate_timestamp(folder_name + '/' + attack_data['file_name'], attack_data['sourcetype'], attack_data['source'])
 
-        replay_attack_dataset(container_name, splunk_port, splunk_password, folder_name, 'test' + str(test_index), attack_data['sourcetype'], attack_data['source'], attack_data['file_name'])
+        replay_attack_dataset(container_name, splunk_password, folder_name, 'test' + str(test_index), attack_data['sourcetype'], attack_data['source'], attack_data['file_name'])
 
     time.sleep(200)
 
@@ -141,7 +141,7 @@ def update_ESCU_app(container_name, splunk_password):
     print("Successfully updated the ESCU App!")
 
 
-def replay_attack_dataset(container_name, splunk_port, splunk_password, folder_name, index, sourcetype, source, out):
+def replay_attack_dataset(container_name, splunk_password, folder_name, index, sourcetype, source, out):
     ansible_vars = {}
     ansible_vars['folder_name'] = folder_name
     ansible_vars['ansible_user'] = 'ansible'
@@ -150,7 +150,6 @@ def replay_attack_dataset(container_name, splunk_port, splunk_password, folder_n
     ansible_vars['sourcetype'] = sourcetype
     ansible_vars['source'] = source
     ansible_vars['index'] = index
-    ansible_vars['splunk_management_port'] = splunk_port
 
     cmdline = "--connection docker -i %s, -u %s" % (container_name, ansible_vars['ansible_user'])
     runner = ansible_runner.run(private_data_dir=os.path.join(os.path.dirname(__file__), '../'),

From 0cd2cd850fad8f5c38610284415f6fa00d46cb8a Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Mon, 20 Sep 2021 11:01:06 -0700
Subject: [PATCH 009/166] Duplicate folder names were clobbering each other. 
 Also commented around initial steps to speed up testing time.  We will re use
 the docker containers that we initially built each time for testing.

---
 .../detection_testing_execution.py            | 20 +++++++++++++------
 .../modules/testing_service.py                |  4 ++--
 2 files changed, 16 insertions(+), 8 deletions(-)

diff --git a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
index 10d3c685be..bc8a2f8f69 100644
--- a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
+++ b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
@@ -115,6 +115,7 @@ def main(args):
             client.images.pull(DOCKER_HUB_CONTAINER_PATH)
             print("Finished downloading the image [%s]"%(DOCKER_HUB_CONTAINER_PATH))
 
+        '''
         try:
             image = client.images.get(DOCKER_COMMIT_NAME)
             print("Found an image called [%s]. We will remove it"%(DOCKER_COMMIT_NAME))
@@ -156,21 +157,28 @@ def main(args):
     
     print("Stopping the running container [%s]"%(BASE_CONTAINER_NAME))
     base_container.stop()
-
+    
     #The part below does not seem to be working as expected. Will need to look into it
     #When I create the new container, it fails to boot with 
-    '''The CA file specified (/opt/splunk/etc/auth/cacert.pem) does not exist. Cannot continue.
-    SSL certificate generation failed.
+    # The CA file specified (/opt/splunk/etc/auth/cacert.pem) does not exist. Cannot continue.
+    # SSL certificate generation failed.
 
 
-    MSG:
+    # MSG:
 
-    non-zero return code
-    '''
+    # non-zero return code
+    
     #I am almost positive that I'm doing this wrong but it works for now...
     print("Committing the configured container: [%s]--->[%s]"%(BASE_CONTAINER_NAME, DOCKER_COMMIT_NAME))
     base_container.commit(repository=DOCKER_COMMIT_NAME)
     
+    '''
+    
+    except Exception as e:
+        #Remove this since it exists in commented out section above.
+        print("There was an error getting the base container up and running.  "
+            "We cannot recover from this: [%s]\nGoodbye..."%(str(e)))
+        sys.exit(1)
 
     print("Make all the threads...")
     results_queue = queue.Queue()
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/testing_service.py b/automated_detection_testing/ci/detection_testing_batch/modules/testing_service.py
index 3a0f8cbbd0..131738174d 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/testing_service.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/testing_service.py
@@ -55,8 +55,8 @@ def test_detection(splunk_ip, splunk_port, container_name, splunk_password, test
     # write entry dynamodb
     #aws_service.add_detection_results_in_dynamo_db('eu-central-1', uuid_var , uuid_test, test_file_obj['tests'][0]['name'], test_file_obj['tests'][0]['file'], str(int(time.time())))
 
-    epoch_time = str(int(time.time()))
-    folder_name = "attack_data_" + epoch_time
+    #epoch_time = str(int(time.time()))
+    folder_name = "attack_data_%s"%(uuid.uuid4())
     os.mkdir(folder_name)
 
     for attack_data in test_file_obj['tests'][0]['attack_data']:

From f30a4c6997cc09bdd934eaaecc5639ee32cddbca Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Mon, 20 Sep 2021 11:05:17 -0700
Subject: [PATCH 010/166] Re-added rebuild code for full test.

---
 .../detection_testing_execution.py                     | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
index bc8a2f8f69..d897b73067 100644
--- a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
+++ b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
@@ -115,7 +115,7 @@ def main(args):
             client.images.pull(DOCKER_HUB_CONTAINER_PATH)
             print("Finished downloading the image [%s]"%(DOCKER_HUB_CONTAINER_PATH))
 
-        '''
+        
         try:
             image = client.images.get(DOCKER_COMMIT_NAME)
             print("Found an image called [%s]. We will remove it"%(DOCKER_COMMIT_NAME))
@@ -172,13 +172,9 @@ def main(args):
     print("Committing the configured container: [%s]--->[%s]"%(BASE_CONTAINER_NAME, DOCKER_COMMIT_NAME))
     base_container.commit(repository=DOCKER_COMMIT_NAME)
     
-    '''
     
-    except Exception as e:
-        #Remove this since it exists in commented out section above.
-        print("There was an error getting the base container up and running.  "
-            "We cannot recover from this: [%s]\nGoodbye..."%(str(e)))
-        sys.exit(1)
+    
+
 
     print("Make all the threads...")
     results_queue = queue.Queue()

From e0961ad0d31ec07c7cf7024a590a9cfe93a85bc2 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Mon, 20 Sep 2021 17:21:46 -0700
Subject: [PATCH 011/166] Still some issues when running more than one
 container in parallel.  Might be a management port issue, but needs
 debugging.

---
 .../detection_testing_execution.py            | 209 +++++++++++++-----
 .../modules/splunk_sdk.py                     |   2 +
 .../modules/testing_service.py                |  59 ++---
 3 files changed, 191 insertions(+), 79 deletions(-)

diff --git a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
index d897b73067..7e6e71316a 100644
--- a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
+++ b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
@@ -13,10 +13,6 @@ from modules import aws_service, testing_service
 import time
 
 
-DT_ATTACK_RANGE_STATE_STORE = "dt-attack-range-tf-state-store"
-DT_ATTACK_RANGE_STATE = "dt-attack-range-state"
-REGION = "eu-central-1"
-NAME = "detection-testing-attack-range"
 
 PASSWORD_LENGTH=20
 MAX_RECOMMENDED_CONTAINERS_BEFORE_WARNING=2
@@ -31,6 +27,8 @@ BASE_CONTAINER_WEB_PORT=8000
 BASE_CONTAINER_MANAGEMENT_PORT=8089
 
 
+
+
 def wait_for_splunk_ready(splunk_container_name=None, splunk_web_port=None, max_seconds=30):
     #The smarter version of this will try to hit one of the pages,
     #probably the login page, and when that is available it means that
@@ -38,6 +36,45 @@ def wait_for_splunk_ready(splunk_container_name=None, splunk_web_port=None, max_
     #use a simple sleep
     time.sleep(max_seconds)
 
+
+def remove_container(docker_client, container_name, force=True):
+    try:
+        container = docker_client.containers.get(container_name)
+    except Exception as e:
+        print("Could not find Docker Container [%s]. Container does not exist"%(container_name))
+        return True
+    try:
+        container.remove(v=True, force=force) #remove it even if it is running. remove volumes as well
+        print("Successfully removed Docker Container [%s]"%(container_name))
+    except Exception as e:
+        print("Could not remove Docker Container [%s]"%(container_name))
+        raise(Exception("CONTAINER REMOVE ERROR"))
+
+
+def stop_container(docker_client, container_name, force=True):
+    try:
+        container = docker_client.containers.get(container_name)
+    except:
+        print("Container with name [%s] does not exist"%(container_name))
+        return True
+    
+    try:
+        print("Checking to see if [%s] is running..."%(container_name), end='')
+        if container.status == 'exited':
+            print("NO")
+            return True
+        else:
+            print("YES (container.status is [%s])"%(container.status))
+            print("Stopping [%s]"%(container_name))
+            container.stop(force=force)
+            return True
+    except Exception as e:
+        print("Error trying to stop the container [%s]"%(container_name))
+        raise(Exception("CONTAINER STOP ERROR"))
+
+        
+
+
 def main(args):
 
     parser = argparse.ArgumentParser(description="CI Detection Testing")
@@ -45,12 +82,19 @@ def main(args):
     parser.add_argument("-u", "--uuid", type=str, required=True, help="uuid for detection test")
     parser.add_argument("-pr", "--pr-number", type=int, required=False, help="Pull Request Number")
     parser.add_argument("-n", "--num_containers", required=False, type=int, default=1, help="The number of splunk docker containers to start and run for testing")
+    
+    parser.add_argument("-i", "--reuse_images", required=False, type=bool, default=False, help="Should existing images be re-used, or should they be redownloaded?")
+    parser.add_argument("-c", "--reuse_containers", required=False, type=bool, default=False,  help="Should existing containers be re-used, or should they be rebuilt?")
 
     args = parser.parse_args()
     branch = args.branch
     uuid_test = args.uuid
     pr_number = args.pr_number
     num_containers = args.num_containers
+    reuse_containers = args.reuse_containers
+    reuse_images = args.reuse_images
+
+
     if num_containers < 1:
         #Perhaps this should be a mock-run - do the initial steps but don't do testing on the containers?
         print("Error, requested 0 containers.  You must run with at least 1 container.")
@@ -82,7 +126,10 @@ def main(args):
     #because this is only accessible to localhost, the password doesn't need to be particularly secure
     #We can also share it between splunk on all containers
     
-    splunk_password = secrets.token_urlsafe(PASSWORD_LENGTH)
+    #splunk_password = secrets.token_urlsafe(PASSWORD_LENGTH)
+
+    #Only accessible on local host, it's okay to expose the password for debugging
+    splunk_password = "123456qwerty!@#$%^QWERTY"
     splunk_container_manager_threads = []
     
 
@@ -94,70 +141,123 @@ def main(args):
 
     print("Getting docker client")
     client = docker.client.from_env()
+
     try:
         print("Removing any existing containers called [%s]."%(BASE_CONTAINER_NAME))
 
         c = client.containers.get(BASE_CONTAINER_NAME)
-        c.remove(v=True, force=True) #remove it even if it is running. remove volumes as well
     except:
-        print("Container [%s] did not exist. No need to remove it"%(BASE_CONTAINER_NAME))
+        print("Container [%s] did not exist. No need to remove it. It will; be created for you."%(BASE_CONTAINER_NAME))
+        c = None
+    
 
+    if (c and reuse_containers):
+        print("Found a container called [%s]. NOT removing it because you have specified --reuse_containers [%s]. "
+                "However, we must stop the container.  Stopping it now..."%(BASE_CONTAINER_NAME, reuse_containers))
+        stop_container(client, BASE_CONTAINER_NAME)
+        
+    elif c:
+        print("Found a container called [%s]. Removing it because you have specified --reuse_containers [%s]"%(BASE_CONTAINER_NAME, reuse_containers))
+        remove_container(client, BASE_CONTAINER_NAME)
+        
+
+
+
+    download_image = False
     try:
+        client.images.get(DOCKER_HUB_CONTAINER_PATH)
+        if reuse_images:
+            print("You already have an image named [%s]."%(DOCKER_HUB_CONTAINER_PATH))
+            download_image = False
+        else:
+            print("You already have an image named [%s]., "
+                    "but have speicified --reuse_images %s"%(DOCKER_HUB_CONTAINER_PATH, reuse_images))
+            download_image = True
+
+    except:
+        print("You did not have an image named [%s]."%(DOCKER_HUB_CONTAINER_PATH))
+        download_image = True
+
+    if download_image:        
         try:
-            client.images.get(DOCKER_HUB_CONTAINER_PATH)
-            print("You already have an image named [%s]. We will not "
-                "download it again."%(DOCKER_HUB_CONTAINER_PATH))
-        except:
-            print("You did not have an image named [%s]. We will "
-                "download it now from the Docker Hub.  Please note "
+            print("Downloading image [%s].  Please note "
                 "that this could take a long time depending on your "
                 "connection. It's around 2GB."%(DOCKER_HUB_CONTAINER_PATH))
             client.images.pull(DOCKER_HUB_CONTAINER_PATH)
             print("Finished downloading the image [%s]"%(DOCKER_HUB_CONTAINER_PATH))
+        except Exception as e: 
+            print("Unrecoverable error downloading image [%s]:[%s]"%(DOCKER_HUB_CONTAINER_PATH, str(e)))
+            sys.exit(1)
 
-        
+
+    remove_tag = False        
+    try:
+        image = client.images.get(DOCKER_COMMIT_NAME)
+        print("Found an image called [%s]"%(DOCKER_COMMIT_NAME))
+        if reuse_images == False:
+            print("We will remove the image [%s] because you have specificed --reuse_images %s"%(DOCKER_COMMIT_NAME, reuse_images))
+            remove_tag = True
+            build_Tag = True
+        else:
+            print("We will use the preexisting image for [%s]"%(DOCKER_COMMIT_NAME))
+            build_tag = False
+    except:
+        print("No image found named [%s]"%(DOCKER_COMMIT_NAME))
+        build_tag = True
+    
+
+    if remove_tag:
         try:
-            image = client.images.get(DOCKER_COMMIT_NAME)
-            print("Found an image called [%s]. We will remove it"%(DOCKER_COMMIT_NAME))
             #Stop it if it's running, remove associated volumes too
-            image.remove(v=True, force=True)
-        except:
-            print("No image found named [%s]"%(DOCKER_COMMIT_NAME))
-        
+            client.images.remove(image=DOCKER_COMMIT_NAME, force=True)
+            
+        except Exception as e:
+            print("Unrecoverable error removing [%s]: [%s]"%(DOCKER_COMMIT_NAME, str(e)))
+            sys.exit(1)
+            
+    
             
 
+
+    if not reuse_containers:
+        try:
+            print("Creating a new container called [%s]"%(BASE_CONTAINER_NAME))
         
 
-        print("Creating a new container called [%s]"%(BASE_CONTAINER_NAME))
-        environment = {"SPLUNK_START_ARGS": "--accept-license",
-                       "SPLUNK_PASSWORD"  : splunk_password }
-        ports= {"8000/tcp": BASE_CONTAINER_WEB_PORT - 1,
-                "8089/tcp": BASE_CONTAINER_MANAGEMENT_PORT - 1
-                }
+            environment = {"SPLUNK_START_ARGS": "--accept-license",
+                    "SPLUNK_PASSWORD"  : splunk_password }
+            ports= {"8000/tcp": BASE_CONTAINER_WEB_PORT - 1,
+            "8089/tcp": BASE_CONTAINER_MANAGEMENT_PORT - 1
+            }
+            base_container = client.containers.create("splunk/splunk:latest", ports=ports, environment=environment, name=BASE_CONTAINER_NAME, detach=True)
+            print("Running the new container called [%s]"%(BASE_CONTAINER_NAME))
+            base_container.start()
+            print("Container is running [%s]"%(BASE_CONTAINER_NAME))
+            print("Sleep for 60 seconds to allow the container to fully start up...")
+            wait_for_splunk_ready(max_seconds=60)
+            print("The container has fully started!")
 
-        base_container = client.containers.create("splunk/splunk:latest", ports=ports, environment=environment, name=BASE_CONTAINER_NAME, detach=True)
-        print("Running the new container called [%s]"%(BASE_CONTAINER_NAME))
-        base_container.start()
-        print("Container is running [%s]"%(BASE_CONTAINER_NAME))
-        print("Sleep for 60 seconds to allow the container to fully start up...")
-        wait_for_splunk_ready(max_seconds=60)
-        print("The container has fully started!")
+            print("Do the ESCU installation on this container. That way we don't have to "
+                    "do it on every container that we then spin up.")
 
-    except Exception as e:
-        print("There was an error getting the base container up and running.  "
-            "We cannot recover from this: [%s]\nGoodbye..."%(str(e)))
-        sys.exit(1)
+            testing_service.prepare_detection_testing(BASE_CONTAINER_NAME, splunk_password)
+            print("Waiting for a few seconds for the splunk app to come up.")
+            wait_for_splunk_ready(max_seconds=30)
+            
+            print("Stopping the running container [%s]"%(BASE_CONTAINER_NAME))
+            base_container.stop()
+            #I am almost positive that I'm doing this wrong but it works for now...
 
-    print("Do the ESCU installation on this container. That way we don't have to "
-          "do it on every container that we then spin up.")
-
-    testing_service.prepare_detection_testing(BASE_CONTAINER_NAME, splunk_password)
-    print("Waiting for a few seconds for the splunk app to come up.")
-    wait_for_splunk_ready(max_seconds=30)
-    
-    print("Stopping the running container [%s]"%(BASE_CONTAINER_NAME))
-    base_container.stop()
+            print("Committing the configured container: [%s]--->[%s]"%(BASE_CONTAINER_NAME, DOCKER_COMMIT_NAME))
+            base_container.commit(repository=DOCKER_COMMIT_NAME)
     
+
+        except Exception as e:
+            print("There was an error getting the base container up and running.  "
+                "We cannot recover from this: [%s]\nGoodbye..."%(str(e)))
+            sys.exit(1)
+
+     
     #The part below does not seem to be working as expected. Will need to look into it
     #When I create the new container, it fails to boot with 
     # The CA file specified (/opt/splunk/etc/auth/cacert.pem) does not exist. Cannot continue.
@@ -168,9 +268,6 @@ def main(args):
 
     # non-zero return code
     
-    #I am almost positive that I'm doing this wrong but it works for now...
-    print("Committing the configured container: [%s]--->[%s]"%(BASE_CONTAINER_NAME, DOCKER_COMMIT_NAME))
-    base_container.commit(repository=DOCKER_COMMIT_NAME)
     
     
     
@@ -238,6 +335,8 @@ def splunk_container_manager(testing_queue, container_name, splunk_ip, splunk_pa
     wait_for_splunk_ready(max_seconds=60)
 
     index=0
+    print("Inspect your containers, you have 5 minutes!")
+    wait_for_splunk_ready(max_seconds=60)
     try:
         while True:
             #Try to get something from the queue
@@ -246,9 +345,15 @@ def splunk_container_manager(testing_queue, container_name, splunk_ip, splunk_pa
             
             #There is a detection to test
             print("Container [%s]--->[%s]"%(container_name, detection_to_test))
-            
-            result = testing_service.test_detection_wrapper(container_name, splunk_ip, splunk_password, splunk_port, detection_to_test, index, uuid_test)
-            results_queue.put(result)
+            try:
+                pass
+                #result = testing_service.test_detection_wrapper(container_name, splunk_ip, splunk_password, splunk_port, detection_to_test, index, uuid_test)
+                #results_queue.put(result)
+            except Exception as e:
+                print("Caught some exception in test detection: [%s]"%(str(e)))
+                #just log the error itself for now so that we can continue
+                result_test = str(e)
+
             index=(index+1)%10
     except queue.Empty:
         print("Queue was empty, [%s] finished testing detections!"%(container_name))
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/splunk_sdk.py b/automated_detection_testing/ci/detection_testing_batch/modules/splunk_sdk.py
index 854ff47727..b372a9bd07 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/splunk_sdk.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/splunk_sdk.py
@@ -64,6 +64,7 @@ def test_detection_search(splunk_host, splunk_port, splunk_password, search, pas
         )
     except Exception as e:
         print("Unable to connect to Splunk instance: " + str(e))
+        raise(Exception("NO CONNECTION EXCEPTION"))
         return 1, {}
 
     # search and replace \\ with \\\
@@ -84,6 +85,7 @@ def test_detection_search(splunk_host, splunk_port, splunk_password, search, pas
         job = service.jobs.create(splunk_search, **kwargs)
     except Exception as e:
         print("Unable to execute detection: " + str(e))
+        raise(Exception("NO EXECUTION EXCEPTION"))
         return 1, {}
 
     test_results = dict()
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/testing_service.py b/automated_detection_testing/ci/detection_testing_batch/modules/testing_service.py
index 131738174d..d74ea2a073 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/testing_service.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/testing_service.py
@@ -77,36 +77,41 @@ def test_detection(splunk_ip, splunk_port, container_name, splunk_password, test
 
     result_test = {}
     test = test_file_obj['tests'][0]
-    try:
-        if 'baselines' in test:
-            results_baselines = []
-            for baseline_obj in test['baselines']:
-                baseline_file_name = baseline_obj['file']
-                baseline = load_file(os.path.join(os.path.dirname(__file__), '../security_content', baseline_file_name))
-                result_obj = dict()
-                result_obj['baseline'] = baseline_obj['name']
-                result_obj['baseline_file'] = baseline_obj['file']
-                result = splunk_sdk.test_baseline_search(splunk_ip, splunk_port, splunk_password, baseline['search'], baseline_obj['pass_condition'], baseline['name'], baseline_obj['file'], baseline_obj['earliest_time'], baseline_obj['latest_time'])
-            result_test['baselines_result'] = results_baselines  
+    
+    if 'baselines' in test:
+        results_baselines = []
+        for baseline_obj in test['baselines']:
+            baseline_file_name = baseline_obj['file']
+            baseline = load_file(os.path.join(os.path.dirname(__file__), '../security_content', baseline_file_name))
+            result_obj = dict()
+            result_obj['baseline'] = baseline_obj['name']
+            result_obj['baseline_file'] = baseline_obj['file']
+            print("Making test_baseline_search request to: [%s:%d]"%(splunk_ip, splunk_port))
+            result = splunk_sdk.test_baseline_search(splunk_ip, splunk_port, splunk_password, baseline['search'], baseline_obj['pass_condition'], baseline['name'], baseline_obj['file'], baseline_obj['earliest_time'], baseline_obj['latest_time'])
+            #we don't seem to be doing anything with this loop... are we supposed to have the following line belwo?
+            results_baselines.append(result)
 
-        detection_file_name = test['file']
-        detection = load_file(os.path.join(os.path.dirname(__file__), '../security_content/detections', detection_file_name))
-        result_detection = splunk_sdk.test_detection_search(splunk_ip, splunk_port, splunk_password, detection['search'], test['pass_condition'], detection['name'], test['file'], test['earliest_time'], test['latest_time'])
+        result_test['baselines_result'] = results_baselines  
 
-        result_detection['detection_name'] = test['name']
-        result_detection['detection_file'] = test['file']
-        result_test['detection_result'] = result_detection
+    detection_file_name = test['file']
+    detection = load_file(os.path.join(os.path.dirname(__file__), '../security_content/detections', detection_file_name))
+    print("Making test_detection_search request to: [%s:%d]"%(splunk_ip, splunk_port))
+    
+    result_detection = splunk_sdk.test_detection_search(splunk_ip, splunk_port, splunk_password, detection['search'], test['pass_condition'], detection['name'], test['file'], test['earliest_time'], test['latest_time'])
+    
+
+
+    result_detection['detection_name'] = test['name']
+    result_detection['detection_file'] = test['file']
+    result_test['detection_result'] = result_detection
+
+    if result_detection['error']:
+        print("failed")
+        #aws_service.update_detection_results_in_dynamo_db('eu-central-1', uuid_var, 'failed')
+    else:
+        print("passed")
+        #aws_service.update_detection_results_in_dynamo_db('eu-central-1', uuid_var, 'passed')
 
-        if result_detection['error']:
-            print("failed")
-            #aws_service.update_detection_results_in_dynamo_db('eu-central-1', uuid_var, 'failed')
-        else:
-            print("passed")
-            #aws_service.update_detection_results_in_dynamo_db('eu-central-1', uuid_var, 'passed')
-    except Exception as e:
-        print("Caught some exception in test detection: [%s]"%(str(e)))
-        #just log the error itself for now so that we can continue
-        result_test = str(e)
     return result_test
 
 

From 06a9c0f15ef82989fa00a1c82708a27bdd385fd6 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Mon, 11 Oct 2021 10:23:42 -0700
Subject: [PATCH 012/166] Lots more changes to how we run. Now based off of the
 splunk/splunk container on docker hub as much as possible.  We use ENV
 arguments to install required apps.  Still only at 50% pass rate, so we need
 more troubleshooting to figure out why.

---
 .../detection_testing_execution.py            | 350 +++++++++++-------
 .../detection_testing_batch/indexes.conf.tar  | Bin 0 -> 2560 bytes
 .../modules/splunk_sdk.py                     |   9 +-
 .../modules/testing_service.py                | 135 ++++++-
 4 files changed, 355 insertions(+), 139 deletions(-)
 create mode 100644 automated_detection_testing/ci/detection_testing_batch/indexes.conf.tar

diff --git a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
index 7e6e71316a..07acac2a62 100644
--- a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
+++ b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
@@ -11,16 +11,20 @@ import queue
 from modules.github_service import GithubService
 from modules import aws_service, testing_service
 import time
-
-
+import subprocess
+from datetime import datetime
+index_file_container_path = "/opt/splunk/etc/apps/search/"
+index_file_local_path = "indexes.conf.tar"
 
 PASSWORD_LENGTH=20
 MAX_RECOMMENDED_CONTAINERS_BEFORE_WARNING=2
 DOCKER_HUB_CONTAINER_PATH="splunk/splunk:latest"
 BASE_CONTAINER_NAME="splunk"
 
-DOCKER_COMMIT_NAME = "splunk_configured"
-RUNNER_BASE_NAME = "splunk_runner"
+
+
+#DOCKER_COMMIT_NAME = "splunk_configured"
+#RUNNER_BASE_NAME = "splunk_runner"
 
 
 BASE_CONTAINER_WEB_PORT=8000
@@ -76,7 +80,7 @@ def stop_container(docker_client, container_name, force=True):
 
 
 def main(args):
-
+    start_time = datetime.now()
     parser = argparse.ArgumentParser(description="CI Detection Testing")
     parser.add_argument("-b", "--branch", type=str, required=True, help="security content branch")
     parser.add_argument("-u", "--uuid", type=str, required=True, help="uuid for detection test")
@@ -101,8 +105,8 @@ def main(args):
         sys.exit(1)
     elif num_containers > MAX_RECOMMENDED_CONTAINERS_BEFORE_WARNING:
         print("You requested to run with [%d] containers which may use a very large amount of resources "
-               "as they all run in parallel.  The maximum suggested number of parallel. The maximum "
-               "suggested number of containers is [%d].  We will do what you asked, but be warned!"%(num_containers, MAX_RECOMMENDED_CONTAINERS_BEFORE_WARNING))
+               "as they all run in parallel.  The maximum suggested number of parallel containers is "
+               "[%d].  We will do what you asked, but be warned!"%(num_containers, MAX_RECOMMENDED_CONTAINERS_BEFORE_WARNING))
 
 
     if pr_number:
@@ -114,7 +118,30 @@ def main(args):
         print("No new detections to test.")
         #aws_service.dynamo_db_nothing_to_test(REGION, uuid_test, str(int(time.time())))
         sys.exit(0)
+    print("The files to test: %s", str(test_files))
+    
+    #Go into the security content directory
+    print("****GENERATE NEW CONTENT****")
+    os.chdir("security_content")
+    commands = ["python3 -m venv .venv", "source .venv/bin/activate", "python3 -m pip install -r requirements.txt", "python3 contentctl.py --path . --verbose generate --product ESCU --output dist/escu"]
+    ret = subprocess.run("; ".join(commands), shell=True, capture_output=False)
+    if ret.returncode != 0:
+        print("Error generating new content.  Exiting...")
+        sys.exit(1)
+    print("New content generated successfully")
+    os.chdir("..")
 
+    client = docker.client.from_env()
+    splunk_password = '123456qwertyQWERTY'
+    
+    
+    
+    
+    
+
+    #print("run it now!")
+    
+    #time.sleep(180)
     #dt_ar = aws_service.get_ar_information_from_dynamo_db(REGION, DT_ATTACK_RANGE_STATE)
     #splunk_instance = aws_service.get_splunk_instance(REGION, dt_ar['ssh_key_name'])
 
@@ -129,170 +156,202 @@ def main(args):
     #splunk_password = secrets.token_urlsafe(PASSWORD_LENGTH)
 
     #Only accessible on local host, it's okay to expose the password for debugging
-    splunk_password = "123456qwerty!@#$%^QWERTY"
+    #splunk_password = "123456qwerty!@#$%^QWERTY"
+    #splunk_password = '123456qwerty%^QWERTY'
     splunk_container_manager_threads = []
     
 
-    print("***Files to test: %d"%(len(test_files)))
+    # print("***Files to test: %d"%(len(test_files)))
     test_file_queue = queue.Queue()
     for filename in test_files:
         test_file_queue.put(filename)
-    print("***Test files enqueued")
+    # print("***Test files enqueued")
 
-    print("Getting docker client")
+    testing_service.test_detection_wrapper(None, None, splunk_password, None, test_file_queue.get(), 0%1, uuid_test)
+    
+    # print("Getting docker client")
     client = docker.client.from_env()
+    
+    # try:
+    #     print("Removing any existing containers called [%s]."%(BASE_CONTAINER_NAME))
 
-    try:
-        print("Removing any existing containers called [%s]."%(BASE_CONTAINER_NAME))
-
-        c = client.containers.get(BASE_CONTAINER_NAME)
-    except:
-        print("Container [%s] did not exist. No need to remove it. It will; be created for you."%(BASE_CONTAINER_NAME))
-        c = None
+    #     c = client.containers.get(BASE_CONTAINER_NAME)
+    # except:
+    #     print("Container [%s] did not exist. No need to remove it. It will; be created for you."%(BASE_CONTAINER_NAME))
+    #     c = None
     
 
-    if (c and reuse_containers):
-        print("Found a container called [%s]. NOT removing it because you have specified --reuse_containers [%s]. "
-                "However, we must stop the container.  Stopping it now..."%(BASE_CONTAINER_NAME, reuse_containers))
-        stop_container(client, BASE_CONTAINER_NAME)
+    # if (c and reuse_containers):
+    #     print("Found a container called [%s]. NOT removing it because you have specified --reuse_containers [%s]. "
+    #             "However, we must stop the container.  Stopping it now..."%(BASE_CONTAINER_NAME, reuse_containers))
+    #     stop_container(client, BASE_CONTAINER_NAME)
         
-    elif c:
-        print("Found a container called [%s]. Removing it because you have specified --reuse_containers [%s]"%(BASE_CONTAINER_NAME, reuse_containers))
-        remove_container(client, BASE_CONTAINER_NAME)
+    # elif c:
+    #     print("Found a container called [%s]. Removing it because you have specified --reuse_containers [%s]"%(BASE_CONTAINER_NAME, reuse_containers))
+    #     remove_container(client, BASE_CONTAINER_NAME)
         
 
 
 
-    download_image = False
-    try:
-        client.images.get(DOCKER_HUB_CONTAINER_PATH)
-        if reuse_images:
-            print("You already have an image named [%s]."%(DOCKER_HUB_CONTAINER_PATH))
-            download_image = False
-        else:
-            print("You already have an image named [%s]., "
-                    "but have speicified --reuse_images %s"%(DOCKER_HUB_CONTAINER_PATH, reuse_images))
-            download_image = True
+    # download_image = False
+    # try:
+    #     client.images.get(DOCKER_HUB_CONTAINER_PATH)
+    #     if reuse_images:
+    #         print("You already have an image named [%s]."%(DOCKER_HUB_CONTAINER_PATH))
+    #         download_image = False
+    #     else:
+    #         print("You already have an image named [%s]., "
+    #                 "but have speicified --reuse_images %s"%(DOCKER_HUB_CONTAINER_PATH, reuse_images))
+    #         download_image = True
 
-    except:
-        print("You did not have an image named [%s]."%(DOCKER_HUB_CONTAINER_PATH))
-        download_image = True
+    # except:
+    #     print("You did not have an image named [%s]."%(DOCKER_HUB_CONTAINER_PATH))
+    #     download_image = True
 
-    if download_image:        
-        try:
-            print("Downloading image [%s].  Please note "
-                "that this could take a long time depending on your "
-                "connection. It's around 2GB."%(DOCKER_HUB_CONTAINER_PATH))
-            client.images.pull(DOCKER_HUB_CONTAINER_PATH)
-            print("Finished downloading the image [%s]"%(DOCKER_HUB_CONTAINER_PATH))
-        except Exception as e: 
-            print("Unrecoverable error downloading image [%s]:[%s]"%(DOCKER_HUB_CONTAINER_PATH, str(e)))
-            sys.exit(1)
+    # if download_image:        
+    #     try:
+    #         print("Downloading image [%s].  Please note "
+    #             "that this could take a long time depending on your "
+    #             "connection. It's around 2GB."%(DOCKER_HUB_CONTAINER_PATH))
+    #         client.images.pull(DOCKER_HUB_CONTAINER_PATH)
+    #         print("Finished downloading the image [%s]"%(DOCKER_HUB_CONTAINER_PATH))
+    #     except Exception as e: 
+    #         print("Unrecoverable error downloading image [%s]:[%s]"%(DOCKER_HUB_CONTAINER_PATH, str(e)))
+    #         sys.exit(1)
 
 
-    remove_tag = False        
-    try:
-        image = client.images.get(DOCKER_COMMIT_NAME)
-        print("Found an image called [%s]"%(DOCKER_COMMIT_NAME))
-        if reuse_images == False:
-            print("We will remove the image [%s] because you have specificed --reuse_images %s"%(DOCKER_COMMIT_NAME, reuse_images))
-            remove_tag = True
-            build_Tag = True
-        else:
-            print("We will use the preexisting image for [%s]"%(DOCKER_COMMIT_NAME))
-            build_tag = False
-    except:
-        print("No image found named [%s]"%(DOCKER_COMMIT_NAME))
-        build_tag = True
+    # remove_tag = False        
+    # try:
+    #     image = client.images.get(DOCKER_COMMIT_NAME)
+    #     print("Found an image called [%s]"%(DOCKER_COMMIT_NAME))
+    #     if reuse_images == False:
+    #         print("We will remove the image [%s] because you have specificed --reuse_images %s"%(DOCKER_COMMIT_NAME, reuse_images))
+    #         remove_tag = True
+    #         build_Tag = True
+    #     else:
+    #         print("We will use the preexisting image for [%s]"%(DOCKER_COMMIT_NAME))
+    #         build_tag = False
+    # except:
+    #     print("No image found named [%s]"%(DOCKER_COMMIT_NAME))
+    #     build_tag = True
     
 
-    if remove_tag:
-        try:
-            #Stop it if it's running, remove associated volumes too
-            client.images.remove(image=DOCKER_COMMIT_NAME, force=True)
+    # if remove_tag:
+    #     try:
+    #         #Stop it if it's running, remove associated volumes too
+    #         client.images.remove(image=DOCKER_COMMIT_NAME, force=True)
             
-        except Exception as e:
-            print("Unrecoverable error removing [%s]: [%s]"%(DOCKER_COMMIT_NAME, str(e)))
-            sys.exit(1)
+    #     except Exception as e:
+    #         print("Unrecoverable error removing [%s]: [%s]"%(DOCKER_COMMIT_NAME, str(e)))
+    #         sys.exit(1)
             
     
             
 
+    # for ind in range(num_containers):
+    #     ind = str(ind)
+    #     if not reuse_containers:
+    #         try:
+    #             print("Creating a new container called [%s]"%(BASE_CONTAINER_NAME+ind))
+            
 
-    if not reuse_containers:
-        try:
-            print("Creating a new container called [%s]"%(BASE_CONTAINER_NAME))
+    #             environment = {"SPLUNK_START_ARGS": "--accept-license",
+    #                     "SPLUNK_PASSWORD"  : splunk_password }
+    #             ports= {"8000/tcp": BASE_CONTAINER_WEB_PORT - 1 + int(ind) + 1,
+    #             "8089/tcp": BASE_CONTAINER_MANAGEMENT_PORT - 1 + int(ind) + 1
+    #             }
+    #             base_container = client.containers.create("splunk/splunk:latest", ports=ports, environment=environment, name=BASE_CONTAINER_NAME+ind, detach=True)
+    #             print("Running the new container called [%s]"%(BASE_CONTAINER_NAME+ind))
+    #             base_container.start()
+    #             print("Container is running [%s]"%(BASE_CONTAINER_NAME+ind))
+    #             print("Sleep for 60 seconds to allow the container to fully start up...")
+    #             wait_for_splunk_ready(max_seconds=60)
+    #             print("The container has fully started!")
+
+    #             print("Do the ESCU installation on this container. That way we don't have to "
+    #                     "do it on every container that we then spin up.")
+
+    #             testing_service.prepare_detection_testing(BASE_CONTAINER_NAME+ind, splunk_password)
+    #             print("Waiting for a few seconds for the splunk app to come up.")
+    #             wait_for_splunk_ready(max_seconds=30)
+    #             print("Install the apps and enable accelerate")
+    #             wait_for_splunk_ready(max_seconds=180)
+    #             print("Stopping the running container [%s]"%(BASE_CONTAINER_NAME+ind))
+    #             base_container.stop()
+    #             #I am almost positive that I'm doing this wrong but it works for now...
+
+    #             #print("Committing the configured container: [%s]--->[%s]"%(BASE_CONTAINER_NAME, DOCKER_COMMIT_NAME))
+    #             #base_container.commit(repository=DOCKER_COMMIT_NAME)
         
 
-            environment = {"SPLUNK_START_ARGS": "--accept-license",
-                    "SPLUNK_PASSWORD"  : splunk_password }
-            ports= {"8000/tcp": BASE_CONTAINER_WEB_PORT - 1,
-            "8089/tcp": BASE_CONTAINER_MANAGEMENT_PORT - 1
-            }
-            base_container = client.containers.create("splunk/splunk:latest", ports=ports, environment=environment, name=BASE_CONTAINER_NAME, detach=True)
-            print("Running the new container called [%s]"%(BASE_CONTAINER_NAME))
-            base_container.start()
-            print("Container is running [%s]"%(BASE_CONTAINER_NAME))
-            print("Sleep for 60 seconds to allow the container to fully start up...")
-            wait_for_splunk_ready(max_seconds=60)
-            print("The container has fully started!")
-
-            print("Do the ESCU installation on this container. That way we don't have to "
-                    "do it on every container that we then spin up.")
-
-            testing_service.prepare_detection_testing(BASE_CONTAINER_NAME, splunk_password)
-            print("Waiting for a few seconds for the splunk app to come up.")
-            wait_for_splunk_ready(max_seconds=30)
-            
-            print("Stopping the running container [%s]"%(BASE_CONTAINER_NAME))
-            base_container.stop()
-            #I am almost positive that I'm doing this wrong but it works for now...
-
-            print("Committing the configured container: [%s]--->[%s]"%(BASE_CONTAINER_NAME, DOCKER_COMMIT_NAME))
-            base_container.commit(repository=DOCKER_COMMIT_NAME)
-    
-
-        except Exception as e:
-            print("There was an error getting the base container up and running.  "
-                "We cannot recover from this: [%s]\nGoodbye..."%(str(e)))
-            sys.exit(1)
+    #         except Exception as e:
+    #             print("There was an error getting the base container up and running.  "
+    #                 "We cannot recover from this: [%s]\nGoodbye..."%(str(e)))
+    #             sys.exit(1)
 
      
-    #The part below does not seem to be working as expected. Will need to look into it
-    #When I create the new container, it fails to boot with 
-    # The CA file specified (/opt/splunk/etc/auth/cacert.pem) does not exist. Cannot continue.
-    # SSL certificate generation failed.
+    # # # #The part below does not seem to be working as expected. Will need to look into it
+    # # # #When I create the new container, it fails to boot with 
+    # # # # The CA file specified (/opt/splunk/etc/auth/cacert.pem) does not exist. Cannot continue.
+    # # # # SSL certificate generation failed.
 
 
-    # MSG:
+    # # # # MSG:
 
-    # non-zero return code
+    # # # # non-zero return code
     
     
     
+
     
-
-
     print("Make all the threads...")
+    print("The number of detections we will test is [%d]"%(test_file_queue.qsize()))
+    
     results_queue = queue.Queue()
+    success_names_queue = queue.Queue()
+    failure_names_queue = queue.Queue()
     for container_index in range(num_containers):
-        container_name = "%s_%d"%(RUNNER_BASE_NAME, container_index)
+        container_name = "%s_%d"%(BASE_CONTAINER_NAME, container_index)
+        
         web_port = BASE_CONTAINER_WEB_PORT  + container_index
         management_port = BASE_CONTAINER_MANAGEMENT_PORT + container_index
-        print("Creating a new container called [%s]"%(container_name))
+
+        #docker run -p8089:8089 -p 8000:8000 -e "SPLUNK_START_ARGS=--accept-license" -e "SPLUNK_PASSWORD=123456qwertyQWERTY" -e "SPLUNK_APPS_URL=https://splunkbase.splunk.com/app/3435/release/3.3.4/download,https://splunkbase.splunk.com/app/5709/release/1.0.1/download,https://splunkbase.splunk.com/app/3449/release/3.29.0/download,https://splunkbase.splunk.com/app/1621/release/4.20.2/download" -e "SPLUNKBASE_USERNAME=ericmcginnistwo" -e "SPLUNKBASE_PASSWORD=splunkSecondAccount5@" -name splunktemplate splunk/splunk:latest 
         environment = {"SPLUNK_START_ARGS": "--accept-license",
-                       "SPLUNK_PASSWORD"  : splunk_password }
+                        "SPLUNK_PASSWORD"  : splunk_password, 
+                        "SPLUNK_APPS_URL"   : "https://splunkbase.splunk.com/app/3435/release/3.3.4/download,https://splunkbase.splunk.com/app/5709/release/1.0.1/download,https://splunkbase.splunk.com/app/3449/release/3.29.0/download,https://splunkbase.splunk.com/app/1621/release/4.20.2/download",
+                        "SPLUNKBASE_USERNAME" : "emcginnistwo",
+                        "SPLUNKBASE_PASSWORD" : "splunkSecondAccount5@"
+                        }
         ports= {"8000/tcp": web_port,
                 "8089/tcp": management_port
-                }
+               }
 
-        test_container = client.containers.create(DOCKER_COMMIT_NAME, ports=ports, environment=environment, name=container_name, detach=True, volumes_from=[BASE_CONTAINER_NAME])
-        t = threading.Thread(target=splunk_container_manager, args=(test_file_queue, container_name, "127.0.0.1", splunk_password, management_port, uuid_test, results_queue))
+        print("Creating CONTAINER: [%s]"%(container_name))
+        base_container = client.containers.create(DOCKER_HUB_CONTAINER_PATH, ports=ports, environment=environment, name=container_name, detach=True)
+        print("Created CONTAINER : [%s]"%(container_name))
+        #print("Creating a new container called [%s]"%(container_name))
+        #environment = {"SPLUNK_START_ARGS": "--accept-license",
+        #               "SPLUNK_PASSWORD"  : splunk_password }
+        #ports= {"8000/tcp": web_port,
+        #        "8089/tcp": management_port
+        #        }
+
+        #test_container = client.containers.create(DOCKER_COMMIT_NAME, ports=ports, environment=environment, name=container_name, detach=True, volumes_from=[BASE_CONTAINER_NAME])
+        
+        t = threading.Thread(target=splunk_container_manager, args=(test_file_queue, container_name, "127.0.0.1", splunk_password, management_port, uuid_test, results_queue, success_names_queue, failure_names_queue))
         splunk_container_manager_threads.append(t)
 
+    #add the queue status thread - there can be some error in one of the test threads, so this
+    #thread doesn't need to complete for the program to finish execution
+    status_thread = threading.Thread(target=queue_status_thread, args=(test_file_queue.qsize(), test_file_queue, results_queue, success_names_queue, failure_names_queue), daemon=True)
+    status_thread.start()
     print("Start all the threads...")
     for t in splunk_container_manager_threads:
         t.start()
+        #we need to start containers slowly.  Would be great it we could do all the setup and
+        #app install once (with Dockerfile?)
+        time.sleep(60)
     
     #Try to join all the threads
     for t in splunk_container_manager_threads:
@@ -305,38 +364,68 @@ def main(args):
         while True:
 
             o = results_queue.get(block=False)
-            print("Got from queue:")
             print(o)
     except queue.Empty:
         print("That's all the output!")
     
     #now we are done!
-
+    stop_time = datetime.now()
+    print("Total Execution Time: [%s]"%(stop_time-start_time))
+    
 
     #detection testing service has already been prepared, no need to do it here!
     #testing_service.prepare_detection_testing(ssh_key_name, private_key, splunk_ip, splunk_password)
 
     #testing_service.test_detections(ssh_key_name, private_key, splunk_ip, splunk_password, test_files, uuid_test)
     
+def queue_status_thread(total_tests_count, testing_queue, results_queue, success_names_queue, failure_names_queue):
+    while True:
+        print("***Progress Update:\n"\
+              "\tTests to run           : %d\n"\
+              "\tTests currently running: %d\n"\
+              "\tTests completed        : %d\n"\
+              "\t\tSuccess : %d\n"\
+              "\t\tFailure : %d"%(testing_queue.qsize(), total_tests_count - testing_queue.qsize() - results_queue.qsize(), results_queue.qsize(), success_names_queue.qsize(), failure_names_queue.qsize()))
+        if results_queue.qsize() == total_tests_count:
+            return
+        else:
+            time.sleep(10)
 
-def splunk_container_manager(testing_queue, container_name, splunk_ip, splunk_password, splunk_port, uuid_test, results_queue):
+def splunk_container_manager(testing_queue, container_name, splunk_ip, splunk_password, splunk_port, uuid_test, results_queue, success_names_queue, failure_names_queue):
     print("Starting the container [%s] after a sleep"%(container_name))
     #Is this going to be safe to use in different threads
     client = docker.client.from_env()
+    
     #start up the container from the base container
     #Assume that the base container has already been fully built with
     #escu etc
     #sleep for a little bit so that we don't all start at once...
-    time.sleep(random.randrange(0,120))
+    time.sleep(random.randrange(0,60))
 
     container = client.containers.get(container_name)
     print("Starting the container [%s]"%(container_name))
+
+    #need to use the low level client to put a file onto a container
+    apiclient = docker.APIClient()
+
     container.start()
-    wait_for_splunk_ready(max_seconds=60)
+    successful_copy = False
+    while not successful_copy:
+        try:
+            with open(index_file_local_path,"rb") as indexData:
+                #splunk will restart a few times will installation of apps takes place so it will reload its indexes...
+                time.sleep(10)
+                apiclient.put_archive(container=container_name, path=index_file_container_path, data=indexData)
+                successful_copy=True
+        except Exception as e:
+            print("Failed copy of index file to CONTAINER:[%s]...we will try again"%(container_name))
+            successful_copy=False
+            
+
+
+    wait_for_splunk_ready(max_seconds=120)
 
     index=0
-    print("Inspect your containers, you have 5 minutes!")
-    wait_for_splunk_ready(max_seconds=60)
     try:
         while True:
             #Try to get something from the queue
@@ -346,9 +435,12 @@ def splunk_container_manager(testing_queue, container_name, splunk_ip, splunk_pa
             #There is a detection to test
             print("Container [%s]--->[%s]"%(container_name, detection_to_test))
             try:
-                pass
-                #result = testing_service.test_detection_wrapper(container_name, splunk_ip, splunk_password, splunk_port, detection_to_test, index, uuid_test)
-                #results_queue.put(result)
+                result = testing_service.test_detection_wrapper(container_name, splunk_ip, splunk_password, splunk_port, detection_to_test, index%1, uuid_test)
+                if result['detection_result']['error']:
+                    failure_names_queue.put(result['detection_result']['detection_name'])
+                else:
+                    success_names_queue.put(result['detection_result']['detection_name'])
+                results_queue.put(result)
             except Exception as e:
                 print("Caught some exception in test detection: [%s]"%(str(e)))
                 #just log the error itself for now so that we can continue
@@ -360,7 +452,7 @@ def splunk_container_manager(testing_queue, container_name, splunk_ip, splunk_pa
     
     print("Shutting down the container [%s]"%(container_name))
     container.stop()
-    print("Finished shutting down the container [%s]"&(container_name))
+    print("Finished shutting down the container [%s]"%(container_name))
 
 if __name__ == "__main__":
     main(sys.argv[1:])
\ No newline at end of file
diff --git a/automated_detection_testing/ci/detection_testing_batch/indexes.conf.tar b/automated_detection_testing/ci/detection_testing_batch/indexes.conf.tar
new file mode 100644
index 0000000000000000000000000000000000000000..813a484efefb92eddfc9f2c81965778d79e48461
GIT binary patch
literal 2560
zcmeHF%WA_g5cJt!p+nEL{E%|$CC#PuL1@xbDMhh2u|Sr<-cYw+-}QqYN*zcb5U6w+
z(Cp5xW@KEhOcb2;aL#8!u%JUJd*n$rAl-F`#aW#4I3<vbFp){X#96l*#rF-WnpT4<
z-)U=gGclCPw%ZA=yB^>1g9l14i86iu+7{44bDcYT=#3{XPS(h5$otRyQ!anyFNB;0
zjGs8~sJ4He|0_a+{4LC#DV7RLcEhe#%ZHbz`}f5>>dC0%icJVsZ44|FsymCYt2KV!
zI*YY4)Wk=6-RPoyhGL(icD5%<SHYAt$yTaeJ5;6<Ug<A*oKu&G6Z$nFmg)m0_ZnD|
T2#-qK7kdZ11Kt7e06FjtM1*un

literal 0
HcmV?d00001

diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/splunk_sdk.py b/automated_detection_testing/ci/detection_testing_batch/modules/splunk_sdk.py
index b372a9bd07..adfd175baf 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/splunk_sdk.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/splunk_sdk.py
@@ -80,12 +80,15 @@ def test_detection_search(splunk_host, splunk_port, splunk_password, search, pas
               "dispatch.latest_time": "now"}
 
     splunk_search = search + ' ' + pass_condition
-
+    print("SEARCH:")
+    print(splunk_search)
+    print("Sleep for 30 seconds")
+    sleep(30)
     try:
         job = service.jobs.create(splunk_search, **kwargs)
     except Exception as e:
         print("Unable to execute detection: " + str(e))
-        raise(Exception("NO EXECUTION EXCEPTION"))
+        raise(Exception("***********NO EXECUTION EXCEPTION***********"))
         return 1, {}
 
     test_results = dict()
@@ -106,6 +109,8 @@ def test_detection_search(splunk_host, splunk_port, splunk_password, search, pas
 
 
 def delete_attack_data(splunk_host, splunk_password, splunk_port):
+    #print("DO NOT DELETE ANYTHING!")
+    #return None
     try:
         service = client.connect(
             host=splunk_host,
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/testing_service.py b/automated_detection_testing/ci/detection_testing_batch/modules/testing_service.py
index d74ea2a073..47dda5892b 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/testing_service.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/testing_service.py
@@ -1,4 +1,4 @@
-
+import re
 import ansible_runner
 import yaml
 import uuid
@@ -12,8 +12,10 @@ from modules import splunk_sdk
 
 def prepare_detection_testing(splunk_ip, splunk_password):
     try:
-        module = __import__('generate')
-        results = module.main(REPO_PATH = 'security_content' , OUTPUT_PATH = 'security_content/dist/escu', PRODUCT = 'ESCU', VERBOSE = 'False' )
+        pass
+        #import security_content.bin.generate as module
+        #module = __import__('security_content.bin.generate')
+        #results = module.main(REPO_PATH = 'security_content' , OUTPUT_PATH = 'security_content/dist/escu', PRODUCT = 'ESCU', VERBOSE = 'False' )
     except Exception as e:
         print('Error: ' + str(e))
 
@@ -27,7 +29,7 @@ def test_detection_wrapper(container_name, splunk_ip, splunk_password, splunk_po
     
 
     # delete test data
-    splunk_sdk.delete_attack_data(container_name, splunk_password, splunk_port)
+    #splunk_sdk.delete_attack_data(splunk_ip, splunk_password, splunk_port)
 
     
     if result_test['detection_result']['error']:
@@ -72,9 +74,9 @@ def test_detection(splunk_ip, splunk_port, container_name, splunk_password, test
                 data_manipulation.manipulate_timestamp(folder_name + '/' + attack_data['file_name'], attack_data['sourcetype'], attack_data['source'])
 
         replay_attack_dataset(container_name, splunk_password, folder_name, 'test' + str(test_index), attack_data['sourcetype'], attack_data['source'], attack_data['file_name'])
-
-    time.sleep(200)
-
+    print("START SLEEP AFTER REPLAY")
+    time.sleep(30)
+    print("DONE SLEEP AFTER REPLAY")
     result_test = {}
     test = test_file_obj['tests'][0]
     
@@ -117,6 +119,8 @@ def test_detection(splunk_ip, splunk_port, container_name, splunk_password, test
 
 def load_file(file_path):
     try:
+        print("Opening file path [%s]"%(file_path))
+        sys.exit(0)
         with open(file_path, 'r', encoding="utf-8") as stream:
             try:
                 file = list(yaml.safe_load_all(stream))[0]
@@ -133,7 +137,7 @@ def update_ESCU_app(container_name, splunk_password):
     print("Update ESCU App. This can take some time")
 
     ansible_vars = {}
-    ansible_vars['ansible_user'] = 'ansible'
+    ansible_vars['ansible_user'] = 'ansible_user'
     ansible_vars['splunk_password'] = splunk_password
     ansible_vars['security_content_path'] = 'security_content'
     
@@ -146,6 +150,121 @@ def update_ESCU_app(container_name, splunk_password):
     print("Successfully updated the ESCU App!")
 
 
+def configure_splunk_vm(container_name, splunk_password):
+    ansible_vars = {}
+    ansible_vars['splunk_password'] = splunk_password
+    ansible_vars['ansible_user'] = 'ansible'
+
+    #BEGIN BLOCK VARS FROM ATTACK_RANGE_LOCAL.CONF
+    splunk_url = "https://download.splunk.com/products/splunk/releases/8.0.2/linux/splunk-8.0.2-a7f645ddaf91-Linux-x86_64.tgz"
+    # Specify the download URL of Splunk Enterprise
+
+    splunk_binary = "splunk-8.0.2-a7f645ddaf91-Linux-x86_64.tgz"
+    # Specify the name of the Splunk Enterprise executable
+
+    s3_bucket_url = "https://attack-range-appbinaries.s3-us-west-2.amazonaws.com"
+    # Specify the S3 bucket url from which you want to download the Splunk Apps
+
+    splunk_windows_ta = "splunk-add-on-for-microsoft-windows_800.tgz"
+    # Specify the Splunk Windows TA
+
+    splunk_sysmon_ta = "splunk-add-on-for-microsoft-sysmon_1062.tgz"
+    # Specify the Splunk Sysmon TA
+
+    splunk_cim_app = "splunk-common-information-model-cim_4180.tgz"
+    # Specify the Splunk CIM App
+
+    splunk_escu_app = "DA-ESS-ContentUpdate-latest.tar.gz"
+    # Specify the Splunk ESCU App
+
+    splunk_asx_app = "Splunk_ASX-latest.tar.gz"
+    # Specify the Splunk ASX App
+
+    splunk_python_app = "python-for-scientific-computing-for-linux-64-bit_200.tgz"
+    # Specify the Splunk python for scientific computing dependency that is needed by the MLTK app
+
+    splunk_mltk_app = "splunk-machine-learning-toolkit_510.tgz"
+    # Specify the Splunk MLTK App
+
+    splunk_stream_app = "splunk-stream_720.tgz"
+    # Specify the Splunk Stream App
+
+    splunk_security_essentials_app = "splunk-security-essentials_310.tgz"
+    # Specify the Splunk SSE App
+
+    punchard_custom_visualization = "punchcard-custom-visualization_140.tgz"
+    # Specify the Punchard Custom Visualization App
+
+    status_indicator_custom_visualization = "status-indicator-custom-visualization_140.tgz"
+    # Specify the Status Indicator Custom Visualization App
+
+    splunk_attack_range_dashboard = "splunk_attack_range_reporting-1.0.5.tar.gz"
+    # Specify the Attack Range Dashboard App
+
+    timeline_custom_visualization = "timeline-custom-visualization_140.tgz"
+    # Specify the Timeline Custom Visualization App
+
+    splunk_aws_app = "splunk-add-on-for-amazon-web-services_500.tgz"
+    # Specify the Splunk AWS App
+    # Will be only installed when cloud_attack_range=1
+    #END BLOCK VARS FROM ATTACK_RANGE_LOCAL.CONF
+
+
+
+    splunk_es_app = 'splunk-enterprise-security_640.spl'
+    splunk_es_app_version = re.findall(r'\d+', splunk_es_app)[0]
+
+    #ansible_vars['ansible_python_interpreter'] = "/usr/bin/python3",
+    ansible_vars['splunk_admin_password'] = splunk_password
+    ansible_vars['splunk_url'] = splunk_url
+    ansible_vars['splunk_binary'] = splunk_binary
+    ansible_vars['s3_bucket_url'] = s3_bucket_url
+    ansible_vars['splunk_escu_app'] = splunk_escu_app
+    ansible_vars['splunk_asx_app'] = splunk_asx_app
+    ansible_vars['splunk_windows_ta'] = splunk_windows_ta
+    ansible_vars['splunk_cim_app'] = splunk_cim_app
+    ansible_vars['splunk_sysmon_ta'] = splunk_sysmon_ta
+    #ansible_vars['caldera_password'] = '{{ caldera_password }}'
+    ansible_vars['splunk_mltk_app'] = splunk_mltk_app
+    #ansible_vars['splunk_bots_dataset'] = '{{ splunk_bots_dataset }}'
+    ansible_vars['splunk_stream_app'] = splunk_stream_app
+    ansible_vars['splunk_python_app'] = splunk_python_app
+    #ansible_vars['phantom_app'] = '{{ phantom_app }}'
+    #ansible_vars['phantom_server'] = '{{ phantom_server }}'
+    #ansible_vars['phantom_server_private_ip'] = '{{ phantom_server_private_ip }}'
+    #ansible_vars['phantom_admin_password'] = '{{ phantom_admin_password }}'
+    ansible_vars['splunk_security_essentials_app'] = splunk_security_essentials_app
+    ansible_vars['punchard_custom_visualization'] = punchard_custom_visualization
+    ansible_vars['status_indicator_custom_visualization'] = status_indicator_custom_visualization
+    ansible_vars['splunk_attack_range_dashboard'] = splunk_attack_range_dashboard
+    ansible_vars['timeline_custom_visualization'] = timeline_custom_visualization
+    #ansible_vars['install_mission_control'] = install_mission_control
+    #ansible_vars['mission_control_app'] = mission_control_app
+    #ansible_vars['install_dsp'] = install_dsp
+    #ansible_vars['dsp_client_cert_path'] = dsp_client_cert_path
+    #ansible_vars['dsp_node'] = dsp_node
+    ansible_vars['splunk_server_private_ip'] = "127.0.0.1"
+    ansible_vars['cloud_attack_range'] = '0'
+
+    ansible_vars['install_es'] = '1'
+    ansible_vars['install_mltk'] = '0'
+    ansible_vars['install_mission_control'] = '0'
+    ansible_vars['install_dsp'] = '0'
+
+    ansible_vars['splunk_es_app'] = splunk_es_app
+    ansible_vars['splunk_es_app_version'] = splunk_es_app_version
+    print(ansible_vars['splunk_es_app_version'])
+
+
+
+
+    cmdline = "--connection docker -i %s, -u %s" % (container_name, ansible_vars['ansible_user'])
+    runner = ansible_runner.run(private_data_dir=os.path.join(os.path.dirname(__file__), '../'),
+                                cmdline=cmdline,
+                                roles_path=os.path.join(os.path.dirname(__file__), '../ansible/roles'),
+                                playbook=os.path.join(os.path.dirname(__file__), '../ansible/splunk_server.yml'),
+                                extravars=ansible_vars)
+
 def replay_attack_dataset(container_name, splunk_password, folder_name, index, sourcetype, source, out):
     ansible_vars = {}
     ansible_vars['folder_name'] = folder_name

From 2f06facd23e2f7a52c8a7cf4850fb2c3229ab8ae Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Mon, 11 Oct 2021 17:05:28 -0700
Subject: [PATCH 013/166] Lots of testing changes.... lots of progress.  Need
 to clean up and test entire set of detections properly with sleeps and
 diagnose how long sleep should be or if there is a way to block while input
 data is processed entirely.

---
 .../datamodels.conf.tar                       | Bin 0 -> 11776 bytes
 .../detection_testing_execution.py            |  88 ++++++++++++++----
 .../modules/github_service.py                 |  67 +++++++++++--
 .../modules/splunk_sdk.py                     |  36 +++++--
 .../modules/testing_service.py                |  22 +++--
 5 files changed, 173 insertions(+), 40 deletions(-)
 create mode 100644 automated_detection_testing/ci/detection_testing_batch/datamodels.conf.tar

diff --git a/automated_detection_testing/ci/detection_testing_batch/datamodels.conf.tar b/automated_detection_testing/ci/detection_testing_batch/datamodels.conf.tar
new file mode 100644
index 0000000000000000000000000000000000000000..d374ca68cdd4cc6c571541e0809efc5765d8ebf3
GIT binary patch
literal 11776
zcmeHNTW{Mo7|pYP1%bS51>M?-ooo+z2%0{0Xog_vx(8zrXzH^?gd#Ow?6|+aA9Zmq
zkYqQlqo%VA#IktgNau*(<<L@dCf^+FbQ=r?r{gg>2=6B+`8)1b@3@N}hNI!}a5xwa
zjz?rLJU$wo9+2@)ud4iVZyhsuX^`@{P)gW_f4WQnWW!iEoZrU!rbr9&hZ>T({xNeb
z)iFr>nrk)t&R2-koQy_I`X3z~jmD4kKRFr=4#?m;Uq|iN{m=D3tlN5rhvZy>aTfP#
zw%()|{kO+B2Lv)Fv?6ue8F9wL1KevSrCvr_#*y`D%8UR@>ToGjp2^6-)C(Eg{Ojv*
zmM0K<2~lQ*Ho~nCc}5a3Pr$mKV{;oV6X8G#>uRp?jAUH&dJVT4sK_!YcyW){jyWK|
zTtH?34?+BE!+Dc?XGQs0J7$X;t)hyG@;1-Naew^Vn=yG+ydLTR%t++kMNXu!KBGTO
zwR#AAP9bPe36xNCnrY)`W;6$Dse$~X1+x$wXt-o*SBfWi+u}1&pJ5P#RvEoZK)F)M
zM%f;qUW{g|4i$@G#=LY?C=1*hpj_gk<={4`Cl*4&9AcUxM9dg>+O(<k7Q>F11(BdJ
z`rOCt71|u!sI`(hX{iFY)aHhAou*#pB)4jjFg1s!nesLt?S%iM{`mB3qgt0U1k<GU
z%2DQH;V7s$(*i$Ma$w;FA$DP|`Irbx*5w#Um?cvHMdEeB=}ijaqjSaN$_l$zV2c^B
zr2_TOdR_trb>8YU^O*nNs|98fZB}jgNr{fu=p8HdtZABOp56t1hczH~VgWRT6nR-c
z+E%g5CfZSpbg_BJ*_hr_x&ULbW~5GQj7uKVI$a3ku-<D5NNW);uvwWh+ms#(++>aV
znMlVB>zKmw2ivwA`kx168i~l6#1UsHQX|1;j5n4-cRYT$YJGe!K3wgXyZVy#g$4Jf
z=QYHOcV(on=eakj#yjEpSzCjY-9~6FV0maTau&_T85usTTC48mTN*ja^Ri@jZN$5j
z3E3!U7<@5mc3Bc(&4cLz;}<>^*zY+y6B4sX(`;Htt@nsFWm8BlG4?UE?dJ7*l&RqU
zDbf0rPE;E8$G^Xz!gU)pj1TFDTwn!@6#!ZXIqS85dJ~kQz(;HRUWYBp*HqYG%!;`$
zyf&H%L5-e!G(HqV%L09`&k*-x$lVdIKykCm(4V49VX(chcB7Vme7bDa4##(_)$<mx
zl3`n{Ssbh;Y=|=<bBnL4U^RM9*s}$n$t4DI+(r&BnOri{&@z|dsKo4;%lg`C*dL$#
zO!4Q>!q6apfz{G)3_m42x`TJ3!V@ao<4#T4jn93h!*$V&G&pzV1gKP~z<Uk0S)srV
z8co$;9?>}A8C_&*Ygs-THbYOfHN01<l-N#0c|v&l!6(KQn3*=1*svk5t%=GxaSzuC
z^G_*Ey9-Ide8Qy!A%(*QEZh|`tUEmRo+``9<0>UCAz?<ZSvCrXHcq7LOnP-g&Vzs3
zs$&6+a$qMyzIs9F9aoGN`1^xAZ8xmH7Skgb<AUFSBbQ8}i<n}=*eXMTj_%NVEjBEG
zSH6i*T*RRt#~vH4#B}4-_&Z;CZO`~6rV|mKP;3pmG)4-8d?W3jyi{nqPs<qCIvXBg
zJCWfDsl8Jf9(UXu1^EZ2ts>xfC&YhkLg==j3SsAVMVBy*mdx=a{2HWJ`@^mM!94?e
N2KEf>8Tb|kz5p7CHva$s

literal 0
HcmV?d00001

diff --git a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
index 07acac2a62..128f44acef 100644
--- a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
+++ b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
@@ -13,8 +13,14 @@ from modules import aws_service, testing_service
 import time
 import subprocess
 from datetime import datetime
-index_file_container_path = "/opt/splunk/etc/apps/search/"
+
+SPLUNK_CONTAINER_APPS_DIR = "/opt/splunk/etc/apps"
 index_file_local_path = "indexes.conf.tar"
+index_file_container_path = os.path.join(SPLUNK_CONTAINER_APPS_DIR, "search")
+
+datamodel_file_local_path = "datamodels.conf.tar"
+datamodel_file_container_path = os.path.join(SPLUNK_CONTAINER_APPS_DIR, "Splunk_SA_CIM")
+
 
 PASSWORD_LENGTH=20
 MAX_RECOMMENDED_CONTAINERS_BEFORE_WARNING=2
@@ -90,6 +96,8 @@ def main(args):
     parser.add_argument("-i", "--reuse_images", required=False, type=bool, default=False, help="Should existing images be re-used, or should they be redownloaded?")
     parser.add_argument("-c", "--reuse_containers", required=False, type=bool, default=False,  help="Should existing containers be re-used, or should they be rebuilt?")
 
+    parser.add_argument("-s", "--success", type=str, required=False, help="File that contains previously successful runs that we don't need to test")
+
     args = parser.parse_args()
     branch = args.branch
     uuid_test = args.uuid
@@ -97,6 +105,13 @@ def main(args):
     num_containers = args.num_containers
     reuse_containers = args.reuse_containers
     reuse_images = args.reuse_images
+    success_file = args.success_file
+
+    if success_file is not None:
+        with open(success_file, "r") as successes:
+            success_tests = [x.strip() for x in successes.readlines()]
+    else:
+        success_tests = []
 
 
     if num_containers < 1:
@@ -119,11 +134,18 @@ def main(args):
         #aws_service.dynamo_db_nothing_to_test(REGION, uuid_test, str(int(time.time())))
         sys.exit(0)
     print("The files to test: %s", str(test_files))
+    new_test_files = []
+    for f in test_files:
+        if f not in success_tests:
+            new_test_files.append(f)
+        else:
+            print("Already found [%s] in success file, not testing it again"%(f))
+    test_files = new_test_files
     
     #Go into the security content directory
     print("****GENERATE NEW CONTENT****")
     os.chdir("security_content")
-    commands = ["python3 -m venv .venv", "source .venv/bin/activate", "python3 -m pip install -r requirements.txt", "python3 contentctl.py --path . --verbose generate --product ESCU --output dist/escu"]
+    commands = ["python3 -m venv .venv", "source .venv/bin/activate", "python3 -m pip install wheel", "python3 -m pip install -r requirements.txt", "python3 contentctl.py --path . --verbose generate --product ESCU --output dist/escu"]
     ret = subprocess.run("; ".join(commands), shell=True, capture_output=False)
     if ret.returncode != 0:
         print("Error generating new content.  Exiting...")
@@ -167,7 +189,6 @@ def main(args):
         test_file_queue.put(filename)
     # print("***Test files enqueued")
 
-    testing_service.test_detection_wrapper(None, None, splunk_password, None, test_file_queue.get(), 0%1, uuid_test)
     
     # print("Getting docker client")
     client = docker.client.from_env()
@@ -310,16 +331,28 @@ def main(args):
     results_queue = queue.Queue()
     success_names_queue = queue.Queue()
     failure_names_queue = queue.Queue()
+    
+
+    
     for container_index in range(num_containers):
         container_name = "%s_%d"%(BASE_CONTAINER_NAME, container_index)
         
         web_port = BASE_CONTAINER_WEB_PORT  + container_index
         management_port = BASE_CONTAINER_MANAGEMENT_PORT + container_index
 
+        SPLUNK_COMMON_INFORMATION_MODEL = "https://splunkbase.splunk.com/app/1621/release/4.20.2/download"        
+        SPLUNK_SECURITY_ESSENTIALS = "https://splunkbase.splunk.com/app/3435/release/3.3.4/download"
+        #SPLUNK_ADD_ON_FOR_SYSMON = "https://splunkbase.splunk.com/app/5709/release/1.0.1/download"
+        SPLUNK_ADD_ON_FOR_SYSMON = "https://splunkbase.splunk.com/app/1914/release/10.6.2/download"
+        SYSMON_APP_FOR_SPLUNK = "https://splunkbase.splunk.com/app/3544/release/2.0.0/download"
+        SPLUNK_ES_CONTENT_UPDATE = "https://splunkbase.splunk.com/app/3449/release/3.29.0/download"
+
+        SPLUNK_APPS = [SPLUNK_COMMON_INFORMATION_MODEL, SPLUNK_SECURITY_ESSENTIALS, SPLUNK_ADD_ON_FOR_SYSMON, SYSMON_APP_FOR_SPLUNK, SPLUNK_ES_CONTENT_UPDATE]
+
         #docker run -p8089:8089 -p 8000:8000 -e "SPLUNK_START_ARGS=--accept-license" -e "SPLUNK_PASSWORD=123456qwertyQWERTY" -e "SPLUNK_APPS_URL=https://splunkbase.splunk.com/app/3435/release/3.3.4/download,https://splunkbase.splunk.com/app/5709/release/1.0.1/download,https://splunkbase.splunk.com/app/3449/release/3.29.0/download,https://splunkbase.splunk.com/app/1621/release/4.20.2/download" -e "SPLUNKBASE_USERNAME=ericmcginnistwo" -e "SPLUNKBASE_PASSWORD=splunkSecondAccount5@" -name splunktemplate splunk/splunk:latest 
         environment = {"SPLUNK_START_ARGS": "--accept-license",
                         "SPLUNK_PASSWORD"  : splunk_password, 
-                        "SPLUNK_APPS_URL"   : "https://splunkbase.splunk.com/app/3435/release/3.3.4/download,https://splunkbase.splunk.com/app/5709/release/1.0.1/download,https://splunkbase.splunk.com/app/3449/release/3.29.0/download,https://splunkbase.splunk.com/app/1621/release/4.20.2/download",
+                        "SPLUNK_APPS_URL"   : ','.join(SPLUNK_APPS),
                         "SPLUNKBASE_USERNAME" : "emcginnistwo",
                         "SPLUNKBASE_PASSWORD" : "splunkSecondAccount5@"
                         }
@@ -360,6 +393,10 @@ def main(args):
 
     print("DONE!")
     #read all the results out from the output queue
+    strtime = str(int(time.time()))
+    #write success and failure
+    success_output = open("success_%s"%(strtime), "w") 
+    failure_output = open("failure_%s"%(strtime), "w")
     try:
         while True:
 
@@ -391,6 +428,23 @@ def queue_status_thread(total_tests_count, testing_queue, results_queue, success
         else:
             time.sleep(10)
 
+def copy_file_to_container(localFilePath, remoteFilePath, containerName, sleepTimeSeconds=5):
+    successful_copy = False
+    #need to use the low level client to put a file onto a container
+    apiclient = docker.APIClient()
+    while not successful_copy:
+        try:
+            with open(localFilePath,"rb") as fileData:
+                #splunk will restart a few times will installation of apps takes place so it will reload its indexes...
+                apiclient.put_archive(container=containerName, path=remoteFilePath, data=fileData)
+                successful_copy=True
+        except Exception as e:
+            print("Failed copy of [%s] file to CONTAINER:[%s]...we will try again"%(localFilePath, containerName))
+            time.sleep(10)
+            successful_copy=False
+    print("Successfully copied [%s] to [%s] on [%s]"%(localFilePath, remoteFilePath, containerName))
+            
+
 def splunk_container_manager(testing_queue, container_name, splunk_ip, splunk_password, splunk_port, uuid_test, results_queue, success_names_queue, failure_names_queue):
     print("Starting the container [%s] after a sleep"%(container_name))
     #Is this going to be safe to use in different threads
@@ -405,25 +459,22 @@ def splunk_container_manager(testing_queue, container_name, splunk_ip, splunk_pa
     container = client.containers.get(container_name)
     print("Starting the container [%s]"%(container_name))
 
-    #need to use the low level client to put a file onto a container
-    apiclient = docker.APIClient()
+    
 
     container.start()
-    successful_copy = False
-    while not successful_copy:
-        try:
-            with open(index_file_local_path,"rb") as indexData:
-                #splunk will restart a few times will installation of apps takes place so it will reload its indexes...
-                time.sleep(10)
-                apiclient.put_archive(container=container_name, path=index_file_container_path, data=indexData)
-                successful_copy=True
-        except Exception as e:
-            print("Failed copy of index file to CONTAINER:[%s]...we will try again"%(container_name))
-            successful_copy=False
-            
+    print("Start copying files to container")
+    copy_file_to_container(index_file_local_path, index_file_container_path, container_name)
+    copy_file_to_container(datamodel_file_local_path, datamodel_file_container_path, container_name)
+    print("Finished copying files to container!")
 
 
     wait_for_splunk_ready(max_seconds=120)
+    from modules.splunk_sdk import enable_delete_for_admin
+    if not enable_delete_for_admin(splunk_ip, splunk_port, splunk_password):
+        print("COULD NOT ENABLE DELETE FOR [%s].... quitting"%(container_name))
+        sys.exit(0)
+    
+    print("Successfully enabled DELETE for [%s]"%(container_name))
 
     index=0
     try:
@@ -431,7 +482,6 @@ def splunk_container_manager(testing_queue, container_name, splunk_ip, splunk_pa
             #Try to get something from the queue
             detection_to_test = testing_queue.get(block=False)
             
-            
             #There is a detection to test
             print("Container [%s]--->[%s]"%(container_name, detection_to_test))
             try:
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/github_service.py b/automated_detection_testing/ci/detection_testing_batch/modules/github_service.py
index 418f53460d..d113ae6d54 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/github_service.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/github_service.py
@@ -4,7 +4,7 @@ import os
 import logging
 import glob
 import subprocess
-
+import yaml
 
 # Logger
 logging.basicConfig(level=os.environ.get("LOGLEVEL", "INFO"))
@@ -33,7 +33,7 @@ class GithubService:
         branch2 = 'develop'
         g = git.Git('security_content')
         changed_test_files = []
-
+        changed_detection_files = []
         if branch1 != 'develop':
             differ = g.diff('--name-status', branch2 + '...' + branch1)
             changed_files = differ.splitlines()
@@ -49,12 +49,67 @@ class GithubService:
                     # changed detections
                     if 'detections' in file_path:
                         if not os.path.basename(file_path).startswith('ssa') and os.path.basename(file_path).endswith('.yml'):
-                            file_path_base = os.path.splitext(file_path)[0].replace('detections', 'tests') + '.test'
-                            file_path_new = file_path_base + '.yml'
-                            if file_path_new not in changed_test_files:
-                                changed_test_files.append(file_path_new)
+                            changed_detection_files.append(file_path)
+                            #file_path_base = os.path.splitext(file_path)[0].replace('detections', 'tests') + '.test'
+                            #file_path_new = file_path_base + '.yml'
+                            #if file_path_new not in changed_test_files:
+                            #    changed_test_files.append(file_path_new)
 
+        #all files have the format A\tFILENAME or M\tFILENAME.  Get rid of those leading characters
+        changed_test_files = [name.split('\t')[1] for name in changed_test_files if len(name.split('\t')) == 2]
+        changed_detection_files = [name.split('\t')[1] for name in changed_detection_files if len(name.split('\t')) == 2]
+
+
+        detections_to_test,_,_ = self.filter_test_types(changed_detection_files)
+        for f in detections_to_test:
+            file_path_base = os.path.splitext(f)[0].replace('detections', 'tests') + '.test'
+            file_path_new = file_path_base + '.yml'
+            if file_path_new not in changed_test_files:
+                changed_test_files.append(file_path_new)
+
+        
+        
+       
+        
+        print("Total things to test (test files and detection files changed): [%d]"%(len(changed_test_files)))
+        #for l in changed_test_files:
+        #    print(l)
+        #print(len(changed_test_files))
+        import time
+        time.sleep(5)
         return changed_test_files
 
+    def filter_test_types(self, test_files, test_types = ["Anomaly", "Hunting", "TTP"]):
+        files_to_test = []
+        files_not_to_test = []
+        error_files = []
+        for filename in test_files:
+            try:
+                with open(os.path.join("security_content", filename), "r") as fileData:
+                    yaml_dict = list(yaml.safe_load_all(fileData))[0]
+                    if 'type' not in yaml_dict.keys():
+                        print("Failed to find 'type' in the yaml for: [%s]"%(filename))
+                        error_files.append(filename)
+                    if yaml_dict['type'] in test_types:
+                        files_to_test.append(filename)
+                    else:
+                        files_not_to_test.append(filename)
+            except Exception as e:
+                print("Error on trying to scan [%s]: [%s]"%(filename, str(e)))
+                error_files.append(filename)
+        print("***Detection Information***\n"\
+              "\tTotal Files       : %d"
+              "\tFiles to test     : %d"
+              "\tFiles not to test : %d"
+              "\tError files       : %d"%(len(test_files), len(files_to_test), len(files_not_to_test), len(error_files)))
+        import time
+        time.sleep(5)
+        return files_to_test, files_not_to_test, error_files    
+
+
+
+                
+
+
 
 
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/splunk_sdk.py b/automated_detection_testing/ci/detection_testing_batch/modules/splunk_sdk.py
index adfd175baf..4ea12edd39 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/splunk_sdk.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/splunk_sdk.py
@@ -5,6 +5,31 @@ import splunklib.client as client
 import splunklib.results as results
 import requests
 
+
+def enable_delete_for_admin(splunk_host, splunk_port, splunk_password):
+    try:
+        service = client.connect(
+            host=splunk_host,
+            port=splunk_port,
+            username='admin',
+            password=splunk_password
+        )
+    except Exception as e:
+        print("Unable to connect to Splunk instance: " + str(e))
+        return 1, {}
+
+    # search and replace \\ with \\\
+    # search = search.replace('\\','\\\\')
+    role = service.roles['admin']
+    try:
+        role.grant('delete_by_keyword')
+    except Exception as e:
+        print("Error - failed trying to grant 'can_delete' privs to admin: [%s]"%(str(e)))
+        return False
+    return True
+
+
+
 def test_baseline_search(splunk_host, splunk_port, splunk_password, search, pass_condition, baseline_name, baseline_file, earliest_time, latest_time):
     try:
         service = client.connect(
@@ -82,8 +107,8 @@ def test_detection_search(splunk_host, splunk_port, splunk_password, search, pas
     splunk_search = search + ' ' + pass_condition
     print("SEARCH:")
     print(splunk_search)
-    print("Sleep for 30 seconds")
-    sleep(30)
+    #print("Sleep for 30 seconds")
+    #sleep(30)
     try:
         job = service.jobs.create(splunk_search, **kwargs)
     except Exception as e:
@@ -109,8 +134,7 @@ def test_detection_search(splunk_host, splunk_port, splunk_password, search, pas
 
 
 def delete_attack_data(splunk_host, splunk_password, splunk_port):
-    #print("DO NOT DELETE ANYTHING!")
-    #return None
+    print("Deleting test data!")
     try:
         service = client.connect(
             host=splunk_host,
@@ -122,8 +146,8 @@ def delete_attack_data(splunk_host, splunk_password, splunk_port):
         print("Unable to connect to Splunk instance: " + str(e))
         return 1, {}
 
-    splunk_search = 'search index=test* | delete'
-
+    #splunk_search = 'search index=test* | delete'
+    splunk_search = 'search index=main | delete'
     kwargs = {"exec_mode": "blocking",
               "dispatch.earliest_time": "-1d",
               "dispatch.latest_time": "now"}
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/testing_service.py b/automated_detection_testing/ci/detection_testing_batch/modules/testing_service.py
index 47dda5892b..6ba2fef846 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/testing_service.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/testing_service.py
@@ -28,8 +28,9 @@ def test_detection_wrapper(container_name, splunk_ip, splunk_password, splunk_po
     result_test = test_detection(splunk_ip, splunk_port, container_name, splunk_password, test_file, test_index, uuid_test, uuid_var)
     
 
+    #enter = input("Run some tests from [%s] on [%s] - we don't delete until you hit enter :)"%(container_name, test_file))
     # delete test data
-    #splunk_sdk.delete_attack_data(splunk_ip, splunk_password, splunk_port)
+    splunk_sdk.delete_attack_data(splunk_ip, splunk_password, splunk_port)
 
     
     if result_test['detection_result']['error']:
@@ -42,7 +43,7 @@ def test_detection_wrapper(container_name, splunk_ip, splunk_password, splunk_po
 
 def test_detection(splunk_ip, splunk_port, container_name, splunk_password, test_file, test_index, uuid_test, uuid_var):
     try:
-        test_file_obj = load_file("security_content/" + test_file[2:])
+        test_file_obj = load_file(os.path.join("security_content/", test_file))
     except Exception as e:
         raise
         #print('Error: ' + str(e))
@@ -64,18 +65,21 @@ def test_detection(splunk_ip, splunk_port, container_name, splunk_password, test
     for attack_data in test_file_obj['tests'][0]['attack_data']:
         url = attack_data['data']
         r = requests.get(url, allow_redirects=True)
-        open(folder_name + '/' + attack_data['file_name'], 'wb').write(r.content)
-        print(folder_name + '/' + attack_data['file_name'])
+        target_file = os.path.join(folder_name, attack_data['file_name'])
+        with open(target_file, 'wb') as target:
+            target.write(r.content)
+        print(target_file)
 
         # Update timestamps before replay
         if 'update_timestamp' in attack_data:
             if attack_data['update_timestamp'] == True:
                 data_manipulation = DataManipulation()
-                data_manipulation.manipulate_timestamp(folder_name + '/' + attack_data['file_name'], attack_data['sourcetype'], attack_data['source'])
-
-        replay_attack_dataset(container_name, splunk_password, folder_name, 'test' + str(test_index), attack_data['sourcetype'], attack_data['source'], attack_data['file_name'])
+                data_manipulation.manipulate_timestamp(target_file, attack_data['sourcetype'], attack_data['source'])
+        INDEX_TO_REPLAY_INTO = 'test' + str(test_index)
+        INDEX_TO_REPLAY_INTO = 'main'
+        replay_attack_dataset(container_name, splunk_password, folder_name, INDEX_TO_REPLAY_INTO, attack_data['sourcetype'], attack_data['source'], attack_data['file_name'])
     print("START SLEEP AFTER REPLAY")
-    time.sleep(30)
+    time.sleep(60)
     print("DONE SLEEP AFTER REPLAY")
     result_test = {}
     test = test_file_obj['tests'][0]
@@ -120,7 +124,7 @@ def test_detection(splunk_ip, splunk_port, container_name, splunk_password, test
 def load_file(file_path):
     try:
         print("Opening file path [%s]"%(file_path))
-        sys.exit(0)
+        
         with open(file_path, 'r', encoding="utf-8") as stream:
             try:
                 file = list(yaml.safe_load_all(stream))[0]

From 1a131ead59a74b75d7175e532b5674683039c873 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Tue, 12 Oct 2021 17:22:10 -0700
Subject: [PATCH 014/166] More changes for testing. Prep for long-term test of
 all detection with results output to file.

---
 .../detection_testing_execution.py            | 46 +++++++++++++++----
 .../modules/splunk_sdk.py                     |  4 +-
 2 files changed, 38 insertions(+), 12 deletions(-)

diff --git a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
index 128f44acef..321172a103 100644
--- a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
+++ b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
@@ -12,7 +12,10 @@ from modules.github_service import GithubService
 from modules import aws_service, testing_service
 import time
 import subprocess
-from datetime import datetime
+
+from timeit import default_timer as timer
+from datetime import timedelta
+
 
 SPLUNK_CONTAINER_APPS_DIR = "/opt/splunk/etc/apps"
 index_file_local_path = "indexes.conf.tar"
@@ -24,7 +27,7 @@ datamodel_file_container_path = os.path.join(SPLUNK_CONTAINER_APPS_DIR, "Splunk_
 
 PASSWORD_LENGTH=20
 MAX_RECOMMENDED_CONTAINERS_BEFORE_WARNING=2
-DOCKER_HUB_CONTAINER_PATH="splunk/splunk:latest"
+DOCKER_HUB_CONTAINER_PATH="splunk/splunk:8.1"
 BASE_CONTAINER_NAME="splunk"
 
 
@@ -86,7 +89,10 @@ def stop_container(docker_client, container_name, force=True):
 
 
 def main(args):
-    start_time = datetime.now()
+    
+    start_time = timer()
+    
+
     parser = argparse.ArgumentParser(description="CI Detection Testing")
     parser.add_argument("-b", "--branch", type=str, required=True, help="security content branch")
     parser.add_argument("-u", "--uuid", type=str, required=True, help="uuid for detection test")
@@ -96,7 +102,7 @@ def main(args):
     parser.add_argument("-i", "--reuse_images", required=False, type=bool, default=False, help="Should existing images be re-used, or should they be redownloaded?")
     parser.add_argument("-c", "--reuse_containers", required=False, type=bool, default=False,  help="Should existing containers be re-used, or should they be rebuilt?")
 
-    parser.add_argument("-s", "--success", type=str, required=False, help="File that contains previously successful runs that we don't need to test")
+    parser.add_argument("-s", "--success_file", type=str, required=False, help="File that contains previously successful runs that we don't need to test")
 
     args = parser.parse_args()
     branch = args.branch
@@ -107,11 +113,17 @@ def main(args):
     reuse_images = args.reuse_images
     success_file = args.success_file
 
+    success_tests = []
     if success_file is not None:
         with open(success_file, "r") as successes:
-            success_tests = [x.strip() for x in successes.readlines()]
+            for line in successes.readlines():
+                file_path_new = os.path.join("tests", os.path.splitext(line)[0]) + ".test.yml"
+                #file_path_base = os.path.splitext(line)[0].replace('detections', 'tests') + '.test'
+                #file_path_new = file_path_base + '.yml'
+                success_tests.append(file_path_new)
+                #success_tests = [x.strip() for x in successes.readlines()]
     else:
-        success_tests = []
+        pass
 
 
     if num_containers < 1:
@@ -141,6 +153,8 @@ def main(args):
         else:
             print("Already found [%s] in success file, not testing it again"%(f))
     test_files = new_test_files
+    print(test_files)
+    time.sleep(10)
     
     #Go into the security content directory
     print("****GENERATE NEW CONTENT****")
@@ -401,13 +415,23 @@ def main(args):
         while True:
 
             o = results_queue.get(block=False)
-            print(o)
+            o_result = o['detection_result']
+            if o_result['error'] is False:
+                success_output.write(o_result['detection_file']+'\n')
+            else:
+                failure_output.write(o_result['detection_file']+'\n')
+
+            print(o_result)
     except queue.Empty:
         print("That's all the output!")
     
+
+    success_output.close()
+    failure_output.close()
+
     #now we are done!
-    stop_time = datetime.now()
-    print("Total Execution Time: [%s]"%(stop_time-start_time))
+    stop_time = timer()
+    print("Total Execution Time: [%s]"%(timedelta(seconds=stop_time - start_time, microseconds=0)))
     
 
     #detection testing service has already been prepared, no need to do it here!
@@ -416,13 +440,15 @@ def main(args):
     #testing_service.test_detections(ssh_key_name, private_key, splunk_ip, splunk_password, test_files, uuid_test)
     
 def queue_status_thread(total_tests_count, testing_queue, results_queue, success_names_queue, failure_names_queue):
+    test_start_time = timer()
     while True:
         print("***Progress Update:\n"\
+              "\tElapsed Time           : %s\n"\
               "\tTests to run           : %d\n"\
               "\tTests currently running: %d\n"\
               "\tTests completed        : %d\n"\
               "\t\tSuccess : %d\n"\
-              "\t\tFailure : %d"%(testing_queue.qsize(), total_tests_count - testing_queue.qsize() - results_queue.qsize(), results_queue.qsize(), success_names_queue.qsize(), failure_names_queue.qsize()))
+              "\t\tFailure : %d"%(timedelta(seconds=timer() - test_start_time, microseconds=0), testing_queue.qsize(), total_tests_count - testing_queue.qsize() - results_queue.qsize(), results_queue.qsize(), success_names_queue.qsize(), failure_names_queue.qsize()))
         if results_queue.qsize() == total_tests_count:
             return
         else:
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/splunk_sdk.py b/automated_detection_testing/ci/detection_testing_batch/modules/splunk_sdk.py
index 4ea12edd39..60b9294406 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/splunk_sdk.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/splunk_sdk.py
@@ -107,8 +107,7 @@ def test_detection_search(splunk_host, splunk_port, splunk_password, search, pas
     splunk_search = search + ' ' + pass_condition
     print("SEARCH:")
     print(splunk_search)
-    #print("Sleep for 30 seconds")
-    #sleep(30)
+
     try:
         job = service.jobs.create(splunk_search, **kwargs)
     except Exception as e:
@@ -147,6 +146,7 @@ def delete_attack_data(splunk_host, splunk_password, splunk_port):
         return 1, {}
 
     #splunk_search = 'search index=test* | delete'
+    #_ = input("****************Press ENTER to DELETE****************")
     splunk_search = 'search index=main | delete'
     kwargs = {"exec_mode": "blocking",
               "dispatch.earliest_time": "-1d",

From b607f8a99d84b4492e507e71e11a128dd5433d2d Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Wed, 13 Oct 2021 12:31:02 -0700
Subject: [PATCH 015/166] Some small config updates

---
 .../detection_testing_execution.py            | 34 ++++++++++++++++---
 .../modules/github_service.py                 |  6 ++--
 2 files changed, 32 insertions(+), 8 deletions(-)

diff --git a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
index 321172a103..95f8f9980d 100644
--- a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
+++ b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
@@ -27,7 +27,7 @@ datamodel_file_container_path = os.path.join(SPLUNK_CONTAINER_APPS_DIR, "Splunk_
 
 PASSWORD_LENGTH=20
 MAX_RECOMMENDED_CONTAINERS_BEFORE_WARNING=2
-DOCKER_HUB_CONTAINER_PATH="splunk/splunk:8.1"
+DOCKER_HUB_CONTAINER_PATH="splunk/splunk:8.2.0"
 BASE_CONTAINER_NAME="splunk"
 
 
@@ -103,6 +103,8 @@ def main(args):
     parser.add_argument("-c", "--reuse_containers", required=False, type=bool, default=False,  help="Should existing containers be re-used, or should they be rebuilt?")
 
     parser.add_argument("-s", "--success_file", type=str, required=False, help="File that contains previously successful runs that we don't need to test")
+    parser.add_argument("-user", "--splunkbase_username", type=str, require=True, help="Splunkbase username for downloading Splunkbase apps")
+    parser.add_argument("-pw", "--splunkbase_password", type=str, require=True, help="Splunkbase password for downloading Splunkbase apps")
 
     args = parser.parse_args()
     branch = args.branch
@@ -112,6 +114,8 @@ def main(args):
     reuse_containers = args.reuse_containers
     reuse_images = args.reuse_images
     success_file = args.success_file
+    splunkbase_username = args.splunkbase_username
+    splunkbase_password = args.splunkbase_password
 
     success_tests = []
     if success_file is not None:
@@ -145,7 +149,7 @@ def main(args):
         print("No new detections to test.")
         #aws_service.dynamo_db_nothing_to_test(REGION, uuid_test, str(int(time.time())))
         sys.exit(0)
-    print("The files to test: %s", str(test_files))
+    #print("The files to test: %s", str(test_files))
     new_test_files = []
     for f in test_files:
         if f not in success_tests:
@@ -153,7 +157,25 @@ def main(args):
         else:
             print("Already found [%s] in success file, not testing it again"%(f))
     test_files = new_test_files
+    
+    test_files = [
+        #"tests/endpoint/disable_registry_tool.test.yml",
+        #"tests/endpoint/disable_show_hidden_files.test.yml",
+        #"tests/endpoint/disable_windows_behavior_monitoring.test.yml",
+        #"tests/endpoint/disable_windows_smartscreen_protection.test.yml",
+        #"tests/endpoint/disabling_cmd_application.test.yml",
+        #"tests/endpoint/disabling_controlpanel.test.yml",
+        #"tests/endpoint/disabling_folderoptions_windows_feature.test.yml"
+        #"tests/endpoint/disabling_norun_windows_app.test.yml",
+        #"tests/endpoint/disabling_systemrestore_in_registry.test.yml",
+        #"tests/endpoint/disabling_task_manager.test.yml"
+
+
+        "tests/endpoint/any_powershell_downloadfile.test.yml"
+        ]
+    
     print(test_files)
+    
     time.sleep(10)
     
     #Go into the security content directory
@@ -360,15 +382,17 @@ def main(args):
         SPLUNK_ADD_ON_FOR_SYSMON = "https://splunkbase.splunk.com/app/1914/release/10.6.2/download"
         SYSMON_APP_FOR_SPLUNK = "https://splunkbase.splunk.com/app/3544/release/2.0.0/download"
         SPLUNK_ES_CONTENT_UPDATE = "https://splunkbase.splunk.com/app/3449/release/3.29.0/download"
+        SPLUNK_ADD_ON_FOR_MICROSOFT_WINDOWS = "https://splunkbase.splunk.com/app/742/release/8.1.2/download"
 
-        SPLUNK_APPS = [SPLUNK_COMMON_INFORMATION_MODEL, SPLUNK_SECURITY_ESSENTIALS, SPLUNK_ADD_ON_FOR_SYSMON, SYSMON_APP_FOR_SPLUNK, SPLUNK_ES_CONTENT_UPDATE]
+
+        SPLUNK_APPS = [SPLUNK_COMMON_INFORMATION_MODEL, SPLUNK_SECURITY_ESSENTIALS, SPLUNK_ADD_ON_FOR_SYSMON, SYSMON_APP_FOR_SPLUNK, SPLUNK_ES_CONTENT_UPDATE, SPLUNK_ADD_ON_FOR_MICROSOFT_WINDOWS]
 
         #docker run -p8089:8089 -p 8000:8000 -e "SPLUNK_START_ARGS=--accept-license" -e "SPLUNK_PASSWORD=123456qwertyQWERTY" -e "SPLUNK_APPS_URL=https://splunkbase.splunk.com/app/3435/release/3.3.4/download,https://splunkbase.splunk.com/app/5709/release/1.0.1/download,https://splunkbase.splunk.com/app/3449/release/3.29.0/download,https://splunkbase.splunk.com/app/1621/release/4.20.2/download" -e "SPLUNKBASE_USERNAME=ericmcginnistwo" -e "SPLUNKBASE_PASSWORD=splunkSecondAccount5@" -name splunktemplate splunk/splunk:latest 
         environment = {"SPLUNK_START_ARGS": "--accept-license",
                         "SPLUNK_PASSWORD"  : splunk_password, 
                         "SPLUNK_APPS_URL"   : ','.join(SPLUNK_APPS),
-                        "SPLUNKBASE_USERNAME" : "emcginnistwo",
-                        "SPLUNKBASE_PASSWORD" : "splunkSecondAccount5@"
+                        "SPLUNKBASE_USERNAME" : splunkbase_username,
+                        "SPLUNKBASE_PASSWORD" : splunkbase_password
                         }
         ports= {"8000/tcp": web_port,
                 "8089/tcp": management_port
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/github_service.py b/automated_detection_testing/ci/detection_testing_batch/modules/github_service.py
index d113ae6d54..58330616d9 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/github_service.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/github_service.py
@@ -71,12 +71,12 @@ class GithubService:
         
        
         
-        print("Total things to test (test files and detection files changed): [%d]"%(len(changed_test_files)))
+        #print("Total things to test (test files and detection files changed): [%d]"%(len(changed_test_files)))
         #for l in changed_test_files:
         #    print(l)
         #print(len(changed_test_files))
-        import time
-        time.sleep(5)
+        #import time
+        #time.sleep(5)
         return changed_test_files
 
     def filter_test_types(self, test_files, test_types = ["Anomaly", "Hunting", "TTP"]):

From 75fe6a9f618d0c60e7362d530a57a54ce0b2930a Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Fri, 15 Oct 2021 13:55:46 -0700
Subject: [PATCH 016/166] More progress. Now build and upload apps from local
 file system as well as install from splunkbase.

---
 .../detection_testing_execution.py            | 117 +++++++++++++-----
 .../modules/github_service.py                 |  49 +++++++-
 .../modules/testing_service.py                |   2 +-
 3 files changed, 134 insertions(+), 34 deletions(-)

diff --git a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
index 95f8f9980d..ad68ebe662 100644
--- a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
+++ b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
@@ -103,8 +103,8 @@ def main(args):
     parser.add_argument("-c", "--reuse_containers", required=False, type=bool, default=False,  help="Should existing containers be re-used, or should they be rebuilt?")
 
     parser.add_argument("-s", "--success_file", type=str, required=False, help="File that contains previously successful runs that we don't need to test")
-    parser.add_argument("-user", "--splunkbase_username", type=str, require=True, help="Splunkbase username for downloading Splunkbase apps")
-    parser.add_argument("-pw", "--splunkbase_password", type=str, require=True, help="Splunkbase password for downloading Splunkbase apps")
+    parser.add_argument("-user", "--splunkbase_username", type=str, required=True, help="Splunkbase username for downloading Splunkbase apps")
+    parser.add_argument("-pw", "--splunkbase_password", type=str, required=True, help="Splunkbase password for downloading Splunkbase apps")
 
     args = parser.parse_args()
     branch = args.branch
@@ -157,23 +157,55 @@ def main(args):
         else:
             print("Already found [%s] in success file, not testing it again"%(f))
     test_files = new_test_files
-    
+    '''
     test_files = [
-        #"tests/endpoint/disable_registry_tool.test.yml",
-        #"tests/endpoint/disable_show_hidden_files.test.yml",
-        #"tests/endpoint/disable_windows_behavior_monitoring.test.yml",
-        #"tests/endpoint/disable_windows_smartscreen_protection.test.yml",
-        #"tests/endpoint/disabling_cmd_application.test.yml",
-        #"tests/endpoint/disabling_controlpanel.test.yml",
-        #"tests/endpoint/disabling_folderoptions_windows_feature.test.yml"
-        #"tests/endpoint/disabling_norun_windows_app.test.yml",
-        #"tests/endpoint/disabling_systemrestore_in_registry.test.yml",
-        #"tests/endpoint/disabling_task_manager.test.yml"
-
-
-        "tests/endpoint/any_powershell_downloadfile.test.yml"
-        ]
-    
+    "tests/endpoint/active_setup_registry_autostart.test.yml",
+    "tests/endpoint/change_default_file_association.test.yml",
+    "tests/endpoint/delete_shadowcopy_with_powershell.test.yml",
+    "tests/endpoint/detect_processes_used_for_system_network_configuration_discovery.test.yml",
+    "tests/endpoint/enable_rdp_in_other_port_number.test.yml",
+    "tests/endpoint/enable_wdigest_uselogoncredential_registry.test.yml",
+    "tests/endpoint/etw_registry_disabled.test.yml",
+    "tests/endpoint/eventvwr_uac_bypass.test.yml",
+    "tests/endpoint/get_notable_history.test.yml",
+    "tests/endpoint/get_parent_process_info.test.yml",
+    "tests/endpoint/get_process_info.test.yml",
+    "tests/endpoint/hide_user_account_from_sign_in_screen.test.yml",
+    "tests/endpoint/logon_script_event_trigger_execution.test.yml",
+    "tests/endpoint/mailsniper_invoke_functions.test.yml",
+    "tests/endpoint/malicious_inprocserver32_modification.test.yml",
+    "tests/endpoint/modification_of_wallpaper.test.yml",
+    "tests/endpoint/monitor_registry_keys_for_print_monitors.test.yml",
+    "tests/endpoint/net_profiler_uac_bypass.test.yml",
+    "tests/endpoint/powershell_disable_security_monitoring.test.yml",
+    "tests/endpoint/powershell_enable_smb1protocol_feature.test.yml",
+    "tests/endpoint/process_writing_dynamicwrapperx.test.yml",
+    "tests/endpoint/registry_keys_used_for_persistence.test.yml",
+    "tests/endpoint/registry_keys_used_for_privilege_escalation.test.yml",
+    "tests/endpoint/remcos_client_registry_install_entry.test.yml",
+    "tests/endpoint/revil_registry_entry.test.yml",
+    "tests/endpoint/screensaver_event_trigger_execution.test.yml",
+    "tests/endpoint/sdclt_uac_bypass.test.yml",
+    "tests/endpoint/secretdumps_offline_ntds_dumping_tool.test.yml",
+    "tests/endpoint/silentcleanup_uac_bypass.test.yml",
+    "tests/endpoint/slui_runas_elevated.test.yml", 
+    "tests/endpoint/disable_amsi_through_registry.test.yml",
+    "tests/endpoint/disable_etw_through_registry.test.yml",
+    "tests/endpoint/disable_registry_tool.test.yml",
+    "tests/endpoint/disable_security_logs_using_minint_registry.test.yml",
+    "tests/endpoint/disable_show_hidden_files.test.yml",
+    "tests/endpoint/disable_uac_remote_restriction.test.yml",
+    "tests/endpoint/disable_windows_app_hotkeys.test.yml",
+    "tests/endpoint/disable_windows_behavior_monitoring.test.yml",
+    "tests/endpoint/disable_windows_smartscreen_protection.test.yml",
+    "tests/endpoint/disabling_cmd_application.test.yml",
+    "tests/endpoint/disabling_controlpanel.test.yml",
+    "tests/endpoint/disabling_folderoptions_windows_feature.test.yml",
+    "tests/endpoint/disabling_norun_windows_app.test.yml",
+    "tests/endpoint/disabling_remote_user_account_control.test.yml",
+    "tests/endpoint/disabling_systemrestore_in_registry.test.yml",
+    "tests/endpoint/disabling_task_manager.test.yml"]
+    '''
     print(test_files)
     
     time.sleep(10)
@@ -181,14 +213,33 @@ def main(args):
     #Go into the security content directory
     print("****GENERATE NEW CONTENT****")
     os.chdir("security_content")
-    commands = ["python3 -m venv .venv", "source .venv/bin/activate", "python3 -m pip install wheel", "python3 -m pip install -r requirements.txt", "python3 contentctl.py --path . --verbose generate --product ESCU --output dist/escu"]
+    commands = ["python3 -m venv .venv", "source .venv/bin/activate", "python3 -m pip install wheel", "python3 -m pip install -r requirements.txt", "python3 contentctl.py --path . --verbose generate --product ESCU --output dist/escu", "tar -czf DA-ESS-ContentUpdate.spl -C dist/escu ."]
     ret = subprocess.run("; ".join(commands), shell=True, capture_output=False)
     if ret.returncode != 0:
         print("Error generating new content.  Exiting...")
         sys.exit(1)
-    print("New content generated successfully")
+    print("New content generated successfully")    
     os.chdir("..")
 
+    print("Generate new ESCU Package using new content")
+    commands = ["curl -Ls https://download.splunk.com/misc/packaging-toolkit/splunk-packaging-toolkit-0.9.0.tar.gz -o splunk-packaging-toolkit-latest.tar.gz", 
+    "rm -rf slim-latest",
+    "mkdir slim-latest", 
+    "tar -zxf splunk-packaging-toolkit-latest.tar.gz -C slim-latest --strip-components=1",
+    "cd slim-latest", 
+    "virtualenv --python=/usr/bin/python2.7 --clear venv",
+    "source venv/bin/activate",
+    "python2 -m pip install --upgrade pip",
+    "python2 -m pip install wheel",
+    "python2 -m pip install semantic_version",
+    "python2 -m pip install .",
+    "cp -R ../security_content/dist/escu DA-ESS-ContentUpdate"
+    "slim package -o upload DA-ESS-ContentUpdate",
+    "cp upload/DA-ESS-ContentUpdate*.tar.gz /tmp/apps/DA-ESS-ContentUpdate-latest.tar.gz"
+    ]
+
+    sys.exit(1)    
+
     client = docker.client.from_env()
     splunk_password = '123456qwertyQWERTY'
     
@@ -378,15 +429,16 @@ def main(args):
 
         SPLUNK_COMMON_INFORMATION_MODEL = "https://splunkbase.splunk.com/app/1621/release/4.20.2/download"        
         SPLUNK_SECURITY_ESSENTIALS = "https://splunkbase.splunk.com/app/3435/release/3.3.4/download"
-        #SPLUNK_ADD_ON_FOR_SYSMON = "https://splunkbase.splunk.com/app/5709/release/1.0.1/download"
-        SPLUNK_ADD_ON_FOR_SYSMON = "https://splunkbase.splunk.com/app/1914/release/10.6.2/download"
-        SYSMON_APP_FOR_SPLUNK = "https://splunkbase.splunk.com/app/3544/release/2.0.0/download"
-        SPLUNK_ES_CONTENT_UPDATE = "https://splunkbase.splunk.com/app/3449/release/3.29.0/download"
-        SPLUNK_ADD_ON_FOR_MICROSOFT_WINDOWS = "https://splunkbase.splunk.com/app/742/release/8.1.2/download"
-
-
-        SPLUNK_APPS = [SPLUNK_COMMON_INFORMATION_MODEL, SPLUNK_SECURITY_ESSENTIALS, SPLUNK_ADD_ON_FOR_SYSMON, SYSMON_APP_FOR_SPLUNK, SPLUNK_ES_CONTENT_UPDATE, SPLUNK_ADD_ON_FOR_MICROSOFT_WINDOWS]
-
+        #SPLUNK_ADD_ON_FOR_SYSMON_OLD = "https://splunkbase.splunk.com/app/1914/release/10.6.2/download"
+        #SPLUNK_ADD_ON_FOR_SYSMON_NEW = "https://splunkbase.splunk.com/app/5709/release/1.0.1/download"
+        #SYSMON_APP_FOR_SPLUNK = "https://splunkbase.splunk.com/app/3544/release/2.0.0/download"
+        #SPLUNK_ES_CONTENT_UPDATE = "https://splunkbase.splunk.com/app/3449/release/3.29.0/download"
+        #SPLUNK_ADD_ON_FOR_MICROSOFT_WINDOWS = "https://splunkbase.splunk.com/app/742/release/8.2.0/download"
+        LOCAL_GENERATED_ESCU_LATEST = os.path.join(os.getcwd(),"/tmp/apps/DA-ESS-ContentUpdate.tgz")
+        LOCAL_SPLUNK_ADD_ON_FOR_SYSMON_NEW  = os.path.join(os.getcwd(),"/tmp/apps/Splunk_TA_microsoft_sysmon-1.0.2-B1.spl")
+        
+        SPLUNK_APPS = [SPLUNK_COMMON_INFORMATION_MODEL, SPLUNK_SECURITY_ESSENTIALS, LOCAL_SPLUNK_ADD_ON_FOR_SYSMON_NEW, LOCAL_GENERATED_ESCU_LATEST]
+        #SPLUNK_APPS = [SPLUNK_COMMON_INFORMATION_MODEL, SPLUNK_SECURITY_ESSENTIALS  ,                                                 SPLUNK_ES_CONTENT_UPDATE, SPLUNK_ADD_ON_FOR_MICROSOFT_WINDOWS]
         #docker run -p8089:8089 -p 8000:8000 -e "SPLUNK_START_ARGS=--accept-license" -e "SPLUNK_PASSWORD=123456qwertyQWERTY" -e "SPLUNK_APPS_URL=https://splunkbase.splunk.com/app/3435/release/3.3.4/download,https://splunkbase.splunk.com/app/5709/release/1.0.1/download,https://splunkbase.splunk.com/app/3449/release/3.29.0/download,https://splunkbase.splunk.com/app/1621/release/4.20.2/download" -e "SPLUNKBASE_USERNAME=ericmcginnistwo" -e "SPLUNKBASE_PASSWORD=splunkSecondAccount5@" -name splunktemplate splunk/splunk:latest 
         environment = {"SPLUNK_START_ARGS": "--accept-license",
                         "SPLUNK_PASSWORD"  : splunk_password, 
@@ -397,9 +449,10 @@ def main(args):
         ports= {"8000/tcp": web_port,
                 "8089/tcp": management_port
                }
+        mounts = [docker.types.Mount(target = '/tmp/apps/', source = '/tmp/apps', type='bind', read_only=True)]
 
         print("Creating CONTAINER: [%s]"%(container_name))
-        base_container = client.containers.create(DOCKER_HUB_CONTAINER_PATH, ports=ports, environment=environment, name=container_name, detach=True)
+        base_container = client.containers.create(DOCKER_HUB_CONTAINER_PATH, ports=ports, environment=environment, name=container_name, mounts=mounts, detach=True)
         print("Created CONTAINER : [%s]"%(container_name))
         #print("Creating a new container called [%s]"%(container_name))
         #environment = {"SPLUNK_START_ARGS": "--accept-license",
@@ -504,7 +557,7 @@ def splunk_container_manager(testing_queue, container_name, splunk_ip, splunk_pa
     #Assume that the base container has already been fully built with
     #escu etc
     #sleep for a little bit so that we don't all start at once...
-    time.sleep(random.randrange(0,60))
+    #time.sleep(random.randrange(0,60))
 
     container = client.containers.get(container_name)
     print("Starting the container [%s]"%(container_name))
@@ -525,7 +578,7 @@ def splunk_container_manager(testing_queue, container_name, splunk_ip, splunk_pa
         sys.exit(0)
     
     print("Successfully enabled DELETE for [%s]"%(container_name))
-
+    time.sleep(60)
     index=0
     try:
         while True:
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/github_service.py b/automated_detection_testing/ci/detection_testing_batch/modules/github_service.py
index 58330616d9..64b22a1f44 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/github_service.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/github_service.py
@@ -58,7 +58,54 @@ class GithubService:
         #all files have the format A\tFILENAME or M\tFILENAME.  Get rid of those leading characters
         changed_test_files = [name.split('\t')[1] for name in changed_test_files if len(name.split('\t')) == 2]
         changed_detection_files = [name.split('\t')[1] for name in changed_detection_files if len(name.split('\t')) == 2]
-
+        
+        changed_detection_files = [
+        "detections/endpoint/active_setup_registry_autostart.yml",
+        "detections/endpoint/change_default_file_association.yml",
+        "detections/endpoint/delete_shadowcopy_with_powershell.yml",
+        "detections/endpoint/detect_processes_used_for_system_network_configuration_discovery.yml",
+        "detections/endpoint/enable_rdp_in_other_port_number.yml",
+        "detections/endpoint/enable_wdigest_uselogoncredential_registry.yml",
+        "detections/endpoint/etw_registry_disabled.yml",
+        "detections/endpoint/eventvwr_uac_bypass.yml",
+        "detections/endpoint/get_notable_history.yml",
+        "detections/endpoint/get_parent_process_info.yml",
+        "detections/endpoint/get_process_info.yml",
+        "detections/endpoint/hide_user_account_from_sign_in_screen.yml",
+        "detections/endpoint/logon_script_event_trigger_execution.yml",
+        "detections/endpoint/mailsniper_invoke_functions.yml",
+        "detections/endpoint/malicious_inprocserver32_modification.yml",
+        "detections/endpoint/modification_of_wallpaper.yml",
+        "detections/endpoint/monitor_registry_keys_for_print_monitors.yml",
+        "detections/endpoint/net_profiler_uac_bypass.yml",
+        "detections/endpoint/powershell_disable_security_monitoring.yml",
+        "detections/endpoint/powershell_enable_smb1protocol_feature.yml",
+        "detections/endpoint/process_writing_dynamicwrapperx.yml",
+        "detections/endpoint/registry_keys_used_for_persistence.yml",
+        "detections/endpoint/registry_keys_used_for_privilege_escalation.yml",
+        "detections/endpoint/remcos_client_registry_install_entry.yml",
+        "detections/endpoint/revil_registry_entry.yml",
+        "detections/endpoint/screensaver_event_trigger_execution.yml",
+        "detections/endpoint/sdclt_uac_bypass.yml",
+        "detections/endpoint/secretdumps_offline_ntds_dumping_tool.yml",
+        "detections/endpoint/silentcleanup_uac_bypass.yml",
+        "detections/endpoint/slui_runas_elevated.yml", 
+        "detections/endpoint/disable_amsi_through_registry.yml",
+        "detections/endpoint/disable_etw_through_registry.yml",
+        "detections/endpoint/disable_registry_tool.yml",
+        "detections/endpoint/disable_security_logs_using_minint_registry.yml",
+        "detections/endpoint/disable_show_hidden_files.yml",
+        "detections/endpoint/disable_uac_remote_restriction.yml",
+        "detections/endpoint/disable_windows_app_hotkeys.yml",
+        "detections/endpoint/disable_windows_behavior_monitoring.yml",
+        "detections/endpoint/disable_windows_smartscreen_protection.yml",
+        "detections/endpoint/disabling_cmd_application.yml",
+        "detections/endpoint/disabling_controlpanel.yml",
+        "detections/endpoint/disabling_folderoptions_windows_feature.yml",
+        "detections/endpoint/disabling_norun_windows_app.yml",
+        "detections/endpoint/disabling_remote_user_account_control.yml",
+        "detections/endpoint/disabling_systemrestore_in_registry.yml",
+        "detections/endpoint/disabling_task_manager.yml"]
 
         detections_to_test,_,_ = self.filter_test_types(changed_detection_files)
         for f in detections_to_test:
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/testing_service.py b/automated_detection_testing/ci/detection_testing_batch/modules/testing_service.py
index 6ba2fef846..a6c8ce7f7c 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/testing_service.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/testing_service.py
@@ -28,7 +28,7 @@ def test_detection_wrapper(container_name, splunk_ip, splunk_password, splunk_po
     result_test = test_detection(splunk_ip, splunk_port, container_name, splunk_password, test_file, test_index, uuid_test, uuid_var)
     
 
-    #enter = input("Run some tests from [%s] on [%s] - we don't delete until you hit enter :)"%(container_name, test_file))
+    enter = input("Run some tests from [%s] on [%s] - we don't delete until you hit enter :)"%(container_name, test_file))
     # delete test data
     splunk_sdk.delete_attack_data(splunk_ip, splunk_password, splunk_port)
 

From 9e76ab75fcf093768d6f256f1fcc5b35a0e92aee Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Tue, 19 Oct 2021 16:49:43 -0700
Subject: [PATCH 017/166] Working better during testing, but still needs a huge
 amount of cleanup. Lots of dead code and magic strings.

---
 .../detection_testing_execution.py            | 98 +++++++------------
 .../modules/github_service.py                 | 95 +++++++++---------
 .../modules/testing_service.py                | 18 ++--
 3 files changed, 91 insertions(+), 120 deletions(-)

diff --git a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
index ad68ebe662..7eb257fa90 100644
--- a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
+++ b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
@@ -144,69 +144,38 @@ def main(args):
         github_service = GithubService(branch, pr_number)
     else:
         github_service = GithubService(branch)
-    test_files = github_service.get_changed_test_files()
+    #test_files = github_service.get_all_tests_and_detections(folders=["endpoint"], previously_successful_tests=success_tests)
+    test_files = github_service.get_all_tests_and_detections(folders=["endpoint"])
+    print(len(test_files))
+    sys.exit(1)
+    #test_files2 = github_service.get_changed_test_files()
     if len(test_files) == 0:
         print("No new detections to test.")
         #aws_service.dynamo_db_nothing_to_test(REGION, uuid_test, str(int(time.time())))
         sys.exit(0)
-    #print("The files to test: %s", str(test_files))
+    
+    #print(test_files2[:5])
+    #sys.exit(0)
+
     new_test_files = []
+
+
     for f in test_files:
         if f not in success_tests:
             new_test_files.append(f)
         else:
+            #pass
             print("Already found [%s] in success file, not testing it again"%(f))
-    test_files = new_test_files
+
+    #test_files = new_test_files
     '''
-    test_files = [
-    "tests/endpoint/active_setup_registry_autostart.test.yml",
-    "tests/endpoint/change_default_file_association.test.yml",
-    "tests/endpoint/delete_shadowcopy_with_powershell.test.yml",
-    "tests/endpoint/detect_processes_used_for_system_network_configuration_discovery.test.yml",
-    "tests/endpoint/enable_rdp_in_other_port_number.test.yml",
-    "tests/endpoint/enable_wdigest_uselogoncredential_registry.test.yml",
-    "tests/endpoint/etw_registry_disabled.test.yml",
-    "tests/endpoint/eventvwr_uac_bypass.test.yml",
-    "tests/endpoint/get_notable_history.test.yml",
-    "tests/endpoint/get_parent_process_info.test.yml",
-    "tests/endpoint/get_process_info.test.yml",
-    "tests/endpoint/hide_user_account_from_sign_in_screen.test.yml",
-    "tests/endpoint/logon_script_event_trigger_execution.test.yml",
-    "tests/endpoint/mailsniper_invoke_functions.test.yml",
-    "tests/endpoint/malicious_inprocserver32_modification.test.yml",
-    "tests/endpoint/modification_of_wallpaper.test.yml",
-    "tests/endpoint/monitor_registry_keys_for_print_monitors.test.yml",
-    "tests/endpoint/net_profiler_uac_bypass.test.yml",
-    "tests/endpoint/powershell_disable_security_monitoring.test.yml",
-    "tests/endpoint/powershell_enable_smb1protocol_feature.test.yml",
-    "tests/endpoint/process_writing_dynamicwrapperx.test.yml",
-    "tests/endpoint/registry_keys_used_for_persistence.test.yml",
-    "tests/endpoint/registry_keys_used_for_privilege_escalation.test.yml",
-    "tests/endpoint/remcos_client_registry_install_entry.test.yml",
-    "tests/endpoint/revil_registry_entry.test.yml",
-    "tests/endpoint/screensaver_event_trigger_execution.test.yml",
-    "tests/endpoint/sdclt_uac_bypass.test.yml",
-    "tests/endpoint/secretdumps_offline_ntds_dumping_tool.test.yml",
-    "tests/endpoint/silentcleanup_uac_bypass.test.yml",
-    "tests/endpoint/slui_runas_elevated.test.yml", 
-    "tests/endpoint/disable_amsi_through_registry.test.yml",
-    "tests/endpoint/disable_etw_through_registry.test.yml",
-    "tests/endpoint/disable_registry_tool.test.yml",
-    "tests/endpoint/disable_security_logs_using_minint_registry.test.yml",
-    "tests/endpoint/disable_show_hidden_files.test.yml",
-    "tests/endpoint/disable_uac_remote_restriction.test.yml",
-    "tests/endpoint/disable_windows_app_hotkeys.test.yml",
-    "tests/endpoint/disable_windows_behavior_monitoring.test.yml",
-    "tests/endpoint/disable_windows_smartscreen_protection.test.yml",
-    "tests/endpoint/disabling_cmd_application.test.yml",
-    "tests/endpoint/disabling_controlpanel.test.yml",
-    "tests/endpoint/disabling_folderoptions_windows_feature.test.yml",
-    "tests/endpoint/disabling_norun_windows_app.test.yml",
-    "tests/endpoint/disabling_remote_user_account_control.test.yml",
-    "tests/endpoint/disabling_systemrestore_in_registry.test.yml",
-    "tests/endpoint/disabling_task_manager.test.yml"]
+    print('\n'.join(test_files))
+    for f in test_files:
+        with open('security_content/'+f,'r') as b:
+            data = b.read()
+            if 'baseline' in data.lower():
+                print("baseline found in [%s]"%(f))
     '''
-    print(test_files)
     
     time.sleep(10)
     
@@ -233,12 +202,18 @@ def main(args):
     "python2 -m pip install wheel",
     "python2 -m pip install semantic_version",
     "python2 -m pip install .",
-    "cp -R ../security_content/dist/escu DA-ESS-ContentUpdate"
+    "cp -R ../security_content/dist/escu DA-ESS-ContentUpdate",
     "slim package -o upload DA-ESS-ContentUpdate",
     "cp upload/DA-ESS-ContentUpdate*.tar.gz /tmp/apps/DA-ESS-ContentUpdate-latest.tar.gz"
     ]
 
-    sys.exit(1)    
+    ret = subprocess.run("; ".join(commands), shell=True, capture_output=False)
+    if ret.returncode != 0:
+        print("Error generating new ESCU Package.  Exiting...")
+        sys.exit(1)
+    print("New ESCU PAckage generated successfully")    
+
+    
 
     client = docker.client.from_env()
     splunk_password = '123456qwertyQWERTY'
@@ -429,15 +404,18 @@ def main(args):
 
         SPLUNK_COMMON_INFORMATION_MODEL = "https://splunkbase.splunk.com/app/1621/release/4.20.2/download"        
         SPLUNK_SECURITY_ESSENTIALS = "https://splunkbase.splunk.com/app/3435/release/3.3.4/download"
-        #SPLUNK_ADD_ON_FOR_SYSMON_OLD = "https://splunkbase.splunk.com/app/1914/release/10.6.2/download"
+        SPLUNK_ADD_ON_FOR_SYSMON_OLD = "https://splunkbase.splunk.com/app/1914/release/10.6.2/download"
         #SPLUNK_ADD_ON_FOR_SYSMON_NEW = "https://splunkbase.splunk.com/app/5709/release/1.0.1/download"
+        #LOCAL_SPLUNK_ADD_ON_FOR_SYSMON_NEW  = "/tmp/apps/Splunk_TA_microsoft_sysmon-1.0.2-B1.spl"
         #SYSMON_APP_FOR_SPLUNK = "https://splunkbase.splunk.com/app/3544/release/2.0.0/download"
         #SPLUNK_ES_CONTENT_UPDATE = "https://splunkbase.splunk.com/app/3449/release/3.29.0/download"
-        #SPLUNK_ADD_ON_FOR_MICROSOFT_WINDOWS = "https://splunkbase.splunk.com/app/742/release/8.2.0/download"
-        LOCAL_GENERATED_ESCU_LATEST = os.path.join(os.getcwd(),"/tmp/apps/DA-ESS-ContentUpdate.tgz")
-        LOCAL_SPLUNK_ADD_ON_FOR_SYSMON_NEW  = os.path.join(os.getcwd(),"/tmp/apps/Splunk_TA_microsoft_sysmon-1.0.2-B1.spl")
+        SPLUNK_ADD_ON_FOR_MICROSOFT_WINDOWS = "https://splunkbase.splunk.com/app/742/release/8.2.0/download"
+        LOCAL_GENERATED_ESCU_LATEST = "/tmp/apps/DA-ESS-ContentUpdate-latest.tar.gz"
         
-        SPLUNK_APPS = [SPLUNK_COMMON_INFORMATION_MODEL, SPLUNK_SECURITY_ESSENTIALS, LOCAL_SPLUNK_ADD_ON_FOR_SYSMON_NEW, LOCAL_GENERATED_ESCU_LATEST]
+        SPLUNK_APPS = [SPLUNK_COMMON_INFORMATION_MODEL, SPLUNK_SECURITY_ESSENTIALS, SPLUNK_ADD_ON_FOR_SYSMON_OLD, LOCAL_GENERATED_ESCU_LATEST, SPLUNK_ADD_ON_FOR_MICROSOFT_WINDOWS]
+        #SPLUNK_APPS = [SPLUNK_COMMON_INFORMATION_MODEL, SPLUNK_SECURITY_ESSENTIALS, LOCAL_SPLUNK_ADD_ON_FOR_SYSMON_NEW, LOCAL_GENERATED_ESCU_LATEST, SPLUNK_ADD_ON_FOR_MICROSOFT_WINDOWS]
+
+
         #SPLUNK_APPS = [SPLUNK_COMMON_INFORMATION_MODEL, SPLUNK_SECURITY_ESSENTIALS  ,                                                 SPLUNK_ES_CONTENT_UPDATE, SPLUNK_ADD_ON_FOR_MICROSOFT_WINDOWS]
         #docker run -p8089:8089 -p 8000:8000 -e "SPLUNK_START_ARGS=--accept-license" -e "SPLUNK_PASSWORD=123456qwertyQWERTY" -e "SPLUNK_APPS_URL=https://splunkbase.splunk.com/app/3435/release/3.3.4/download,https://splunkbase.splunk.com/app/5709/release/1.0.1/download,https://splunkbase.splunk.com/app/3449/release/3.29.0/download,https://splunkbase.splunk.com/app/1621/release/4.20.2/download" -e "SPLUNKBASE_USERNAME=ericmcginnistwo" -e "SPLUNKBASE_PASSWORD=splunkSecondAccount5@" -name splunktemplate splunk/splunk:latest 
         environment = {"SPLUNK_START_ARGS": "--accept-license",
@@ -557,7 +535,7 @@ def splunk_container_manager(testing_queue, container_name, splunk_ip, splunk_pa
     #Assume that the base container has already been fully built with
     #escu etc
     #sleep for a little bit so that we don't all start at once...
-    #time.sleep(random.randrange(0,60))
+    time.sleep(random.randrange(0,60))
 
     container = client.containers.get(container_name)
     print("Starting the container [%s]"%(container_name))
@@ -578,7 +556,7 @@ def splunk_container_manager(testing_queue, container_name, splunk_ip, splunk_pa
         sys.exit(0)
     
     print("Successfully enabled DELETE for [%s]"%(container_name))
-    time.sleep(60)
+    
     index=0
     try:
         while True:
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/github_service.py b/automated_detection_testing/ci/detection_testing_batch/modules/github_service.py
index 64b22a1f44..1d29e91a83 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/github_service.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/github_service.py
@@ -4,8 +4,9 @@ import os
 import logging
 import glob
 import subprocess
+from git.objects import base
 import yaml
-
+import pathlib
 # Logger
 logging.basicConfig(level=os.environ.get("LOGLEVEL", "INFO"))
 LOGGER = logging.getLogger(__name__)
@@ -28,6 +29,49 @@ class GithubService:
         repo_obj = git.Repo.clone_from(url, project, branch=branch)
         return repo_obj
 
+    
+    def get_all_tests_and_detections(self, folders=['endpoint', 'cloud', 'network'], ttps_to_test=["Anomaly","Hunting","TTP"], previously_successful_tests=[]):    
+        detections = []
+        for folder in folders:
+            detections.extend(self.get_all_files_in_folder(os.path.join("security_content/detections", folder), "*.yml"))
+            
+        
+        #Prune this down to only the subset of detections we can test
+        detections_to_test = []
+        tests=[]
+ 
+        
+        for detection in detections:
+            #Don't do anything with SSA stuff
+            if os.path.basename(detection).startswith("ssa"):
+                continue
+            with open(detection, "r") as d:
+                description = yaml.safe_load(d)
+                
+                test_filepath = os.path.splitext(detection)[0].replace('detections', 'tests') + '.test.yml'
+                test_filepath_without_security_content = str(pathlib.Path(*pathlib.Path(test_filepath).parts[1:]))
+                if 'type' in description and description['type'] in ttps_to_test:
+                    
+                    #print(description['type'])
+                    if not os.path.exists(test_filepath):
+                        print("Detection [%s] references [%s], but it does not exist"%(detection, test_filepath))
+                        #raise(Exception("Detection [%s] references [%s], but it does not exist"%(detection, test_filepath)))
+                    elif test_filepath_without_security_content in previously_successful_tests:
+                        print("Ignoring test [%s] before it has already passed previously"%(detection))
+                    else:
+                        #remove leading security_content/ from path
+                        tests.append(test_filepath_without_security_content)
+                else:
+                    #Don't do anything with these files
+                    pass
+
+        return tests
+        
+    def get_all_files_in_folder(self, foldername, extension):
+        filesnames =  glob.glob(os.path.join(foldername, extension))
+        return filesnames
+
+
     def get_changed_test_files(self):
         branch1 = self.security_content_branch
         branch2 = 'develop'
@@ -56,56 +100,9 @@ class GithubService:
                             #    changed_test_files.append(file_path_new)
 
         #all files have the format A\tFILENAME or M\tFILENAME.  Get rid of those leading characters
-        changed_test_files = [name.split('\t')[1] for name in changed_test_files if len(name.split('\t')) == 2]
+        changed_test_files      = [name.split('\t')[1] for name in changed_test_files      if len(name.split('\t')) == 2]
         changed_detection_files = [name.split('\t')[1] for name in changed_detection_files if len(name.split('\t')) == 2]
         
-        changed_detection_files = [
-        "detections/endpoint/active_setup_registry_autostart.yml",
-        "detections/endpoint/change_default_file_association.yml",
-        "detections/endpoint/delete_shadowcopy_with_powershell.yml",
-        "detections/endpoint/detect_processes_used_for_system_network_configuration_discovery.yml",
-        "detections/endpoint/enable_rdp_in_other_port_number.yml",
-        "detections/endpoint/enable_wdigest_uselogoncredential_registry.yml",
-        "detections/endpoint/etw_registry_disabled.yml",
-        "detections/endpoint/eventvwr_uac_bypass.yml",
-        "detections/endpoint/get_notable_history.yml",
-        "detections/endpoint/get_parent_process_info.yml",
-        "detections/endpoint/get_process_info.yml",
-        "detections/endpoint/hide_user_account_from_sign_in_screen.yml",
-        "detections/endpoint/logon_script_event_trigger_execution.yml",
-        "detections/endpoint/mailsniper_invoke_functions.yml",
-        "detections/endpoint/malicious_inprocserver32_modification.yml",
-        "detections/endpoint/modification_of_wallpaper.yml",
-        "detections/endpoint/monitor_registry_keys_for_print_monitors.yml",
-        "detections/endpoint/net_profiler_uac_bypass.yml",
-        "detections/endpoint/powershell_disable_security_monitoring.yml",
-        "detections/endpoint/powershell_enable_smb1protocol_feature.yml",
-        "detections/endpoint/process_writing_dynamicwrapperx.yml",
-        "detections/endpoint/registry_keys_used_for_persistence.yml",
-        "detections/endpoint/registry_keys_used_for_privilege_escalation.yml",
-        "detections/endpoint/remcos_client_registry_install_entry.yml",
-        "detections/endpoint/revil_registry_entry.yml",
-        "detections/endpoint/screensaver_event_trigger_execution.yml",
-        "detections/endpoint/sdclt_uac_bypass.yml",
-        "detections/endpoint/secretdumps_offline_ntds_dumping_tool.yml",
-        "detections/endpoint/silentcleanup_uac_bypass.yml",
-        "detections/endpoint/slui_runas_elevated.yml", 
-        "detections/endpoint/disable_amsi_through_registry.yml",
-        "detections/endpoint/disable_etw_through_registry.yml",
-        "detections/endpoint/disable_registry_tool.yml",
-        "detections/endpoint/disable_security_logs_using_minint_registry.yml",
-        "detections/endpoint/disable_show_hidden_files.yml",
-        "detections/endpoint/disable_uac_remote_restriction.yml",
-        "detections/endpoint/disable_windows_app_hotkeys.yml",
-        "detections/endpoint/disable_windows_behavior_monitoring.yml",
-        "detections/endpoint/disable_windows_smartscreen_protection.yml",
-        "detections/endpoint/disabling_cmd_application.yml",
-        "detections/endpoint/disabling_controlpanel.yml",
-        "detections/endpoint/disabling_folderoptions_windows_feature.yml",
-        "detections/endpoint/disabling_norun_windows_app.yml",
-        "detections/endpoint/disabling_remote_user_account_control.yml",
-        "detections/endpoint/disabling_systemrestore_in_registry.yml",
-        "detections/endpoint/disabling_task_manager.yml"]
 
         detections_to_test,_,_ = self.filter_test_types(changed_detection_files)
         for f in detections_to_test:
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/testing_service.py b/automated_detection_testing/ci/detection_testing_batch/modules/testing_service.py
index a6c8ce7f7c..b02c48d185 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/testing_service.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/testing_service.py
@@ -28,7 +28,7 @@ def test_detection_wrapper(container_name, splunk_ip, splunk_password, splunk_po
     result_test = test_detection(splunk_ip, splunk_port, container_name, splunk_password, test_file, test_index, uuid_test, uuid_var)
     
 
-    enter = input("Run some tests from [%s] on [%s] - we don't delete until you hit enter :)"%(container_name, test_file))
+    #enter = input("Run some tests from [%s] on [%s] - we don't delete until you hit enter :)"%(container_name, test_file))
     # delete test data
     splunk_sdk.delete_attack_data(splunk_ip, splunk_password, splunk_port)
 
@@ -45,14 +45,12 @@ def test_detection(splunk_ip, splunk_port, container_name, splunk_password, test
     try:
         test_file_obj = load_file(os.path.join("security_content/", test_file))
     except Exception as e:
-        raise
-        #print('Error: ' + str(e))
-        #return
+        print('Error: ' + str(e))
+        return None
     
     if not test_file_obj:
         print("Not test_file_obj!")
-        raise
-        return
+        return None
     #print(test_file_obj)
 
     # write entry dynamodb
@@ -79,7 +77,7 @@ def test_detection(splunk_ip, splunk_port, container_name, splunk_password, test
         INDEX_TO_REPLAY_INTO = 'main'
         replay_attack_dataset(container_name, splunk_password, folder_name, INDEX_TO_REPLAY_INTO, attack_data['sourcetype'], attack_data['source'], attack_data['file_name'])
     print("START SLEEP AFTER REPLAY")
-    time.sleep(60)
+    time.sleep(30)
     print("DONE SLEEP AFTER REPLAY")
     result_test = {}
     test = test_file_obj['tests'][0]
@@ -129,11 +127,9 @@ def load_file(file_path):
             try:
                 file = list(yaml.safe_load_all(stream))[0]
             except yaml.YAMLError as exc:
-                print("ERROR: reading {0}".format(file_path))
-                return False
+                raise(Exception("ERROR: reading {0}:[{1}]".format(file_path, str(exc))))
     except Exception as e:
-        print("ERROR: reading {0}".format(file_path))
-        return False
+        raise(Exception("ERROR: reading {0}:[{1}]".format(file_path, str(e))))
     return file
 
 

From 9995e9123dab2a3037662c96367c04cb5cb03160 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Tue, 19 Oct 2021 17:14:30 -0700
Subject: [PATCH 018/166] Added some additional arguments to control which
 detections we will test.

---
 .../detection_testing_execution.py            | 20 +++++++++++++------
 1 file changed, 14 insertions(+), 6 deletions(-)

diff --git a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
index 7eb257fa90..664fe54fb5 100644
--- a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
+++ b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
@@ -40,7 +40,8 @@ BASE_CONTAINER_WEB_PORT=8000
 BASE_CONTAINER_MANAGEMENT_PORT=8089
 
 
-
+DETECTION_TYPES = ['endpoint', 'cloud', 'network']
+DETECTION_MODES = ['new', 'all', 'selected']
 
 def wait_for_splunk_ready(splunk_container_name=None, splunk_web_port=None, max_seconds=30):
     #The smarter version of this will try to hit one of the pages,
@@ -105,6 +106,8 @@ def main(args):
     parser.add_argument("-s", "--success_file", type=str, required=False, help="File that contains previously successful runs that we don't need to test")
     parser.add_argument("-user", "--splunkbase_username", type=str, required=True, help="Splunkbase username for downloading Splunkbase apps")
     parser.add_argument("-pw", "--splunkbase_password", type=str, required=True, help="Splunkbase password for downloading Splunkbase apps")
+    parser.add_argument("-m", "--mode", type=str, choices=DETECTION_MODES, required=False, help="Whether to test new detections, specific detections, or all detections", default="all")
+    parser.add_argument("-t", "--types", type=str, required=False, nargs='+', help="Detection types to test. Can be one of more of %s"%(str(DETECTION_TYPES)), default=DETECTION_TYPES)
 
     args = parser.parse_args()
     branch = args.branch
@@ -116,6 +119,14 @@ def main(args):
     success_file = args.success_file
     splunkbase_username = args.splunkbase_username
     splunkbase_password = args.splunkbase_password
+    
+    mode = args.mode
+    folder_names = args.types
+    for t in args.types:
+        if t not in DETECTION_TYPES:
+            print("Error - requested test of [%s] but the only valid types are %s.\tQuitting..."%(t, str(DETECTION_TYPES)))
+            sys.exit(1)
+
 
     success_tests = []
     if success_file is not None:
@@ -144,11 +155,8 @@ def main(args):
         github_service = GithubService(branch, pr_number)
     else:
         github_service = GithubService(branch)
-    #test_files = github_service.get_all_tests_and_detections(folders=["endpoint"], previously_successful_tests=success_tests)
-    test_files = github_service.get_all_tests_and_detections(folders=["endpoint"])
-    print(len(test_files))
-    sys.exit(1)
-    #test_files2 = github_service.get_changed_test_files()
+    test_files = github_service.get_all_tests_and_detections(folders=["endpoint"], previously_successful_tests=success_tests)
+    
     if len(test_files) == 0:
         print("No new detections to test.")
         #aws_service.dynamo_db_nothing_to_test(REGION, uuid_test, str(int(time.time())))

From 505e2f7986b11ded0b28053b139f28d10c01299f Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Wed, 20 Oct 2021 15:13:50 -0700
Subject: [PATCH 019/166] Cleanup and restructuring of main logic.  Better
 prints, error handling, and readability.

---
 .../detection_testing_execution.py            | 149 +++++++++++++++++-
 1 file changed, 143 insertions(+), 6 deletions(-)

diff --git a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
index 664fe54fb5..8140cee7fb 100644
--- a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
+++ b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
@@ -8,6 +8,8 @@ import secrets
 import docker
 import threading
 import queue
+
+from docker.client import DockerClient
 from modules.github_service import GithubService
 from modules import aws_service, testing_service
 import time
@@ -27,8 +29,8 @@ datamodel_file_container_path = os.path.join(SPLUNK_CONTAINER_APPS_DIR, "Splunk_
 
 PASSWORD_LENGTH=20
 MAX_RECOMMENDED_CONTAINERS_BEFORE_WARNING=2
-DOCKER_HUB_CONTAINER_PATH="splunk/splunk:8.2.0"
-BASE_CONTAINER_NAME="splunk"
+DEFAULT_CONTAINER_TAG="latest"
+LOCAL_BASE_CONTAINER_NAME = "splunk_test_%d"
 
 
 
@@ -89,6 +91,104 @@ def stop_container(docker_client, container_name, force=True):
         
 
 
+def setup_image(client: DockerClient, reuse_images: bool, container_name: str) -> None:
+    if not reuse_images:
+        #Check to see if the image exists.  If it does, then remove it.  If it does not, then do nothing
+        docker_image = None
+        try:
+            docker_image = client.images.get(container_name)
+        except Exception as e:
+            #We don't need to do anything, the image did not exist on our system
+            print("Image named [%s] did not exist, so we don't need to try and remove it."%(container_name))
+        if docker_image != None:
+            #We found the image.  Let's try to delete it
+            print("Found docker image named [%s] and you have requested that we forcefully remove it"%(container_name))
+            try:
+                client.images.remove(image=container_name, force=True, noprune=False)
+                print("Docker image named [%s] forcefully removed"%(container_name))
+            except Exception as e:
+                print("Error forcefully removing [%s]"%(container_name))
+                raise(e)
+    
+    #See if the image exists.  If it doesn't, then pull it from Docker Hub
+    docker_image = None
+    try:
+        docker_image = client.images.get(container_name)
+        print("Docker image [%s] found, no need to download it."%(container_name))
+    except Exception as e:
+        #Image did not exist on the system
+        docker_image = None
+
+    if docker_image is None:
+        #We did not find the image, so pull it
+        try:
+            print("Downloading image [%s].  Please note "
+                 "that this could take a long time depending on your "
+                 "connection. It's around 2GB."%(container_name))
+            pull_start_time = timer()
+            client.images.pull(container_name)
+            pull_finish_time = timer()
+            print("Successfully pulled the docker image [%s] in %ss"%
+                  (container_name,
+                  timedelta(seconds=pull_finish_time - pull_start_time, microseconds=0) ))
+
+        except Exception as e:
+            print("There was an error trying to pull the image [%s]: [%s]"%(container_name,str(e)))
+            raise(e)
+
+def remove_existing_containers(client: DockerClient, reuse_containers: bool, container_template: str, num_containers: int, forceRemove: bool=True) -> bool:
+    if reuse_containers is True:
+        #Check to make sure that all of the requested containers exist
+        for index in range(0, num_containers):
+            container_name = container_template%(index)
+            print("Checking for the existence of container named [%s]"%(container_name))
+            try:
+                this_container = client.containers.get(container_name)
+            except Exception as e:
+                print("Failed to find a container named [%s]"%(container_name))
+                reuse_containers = False
+                break
+            try:
+                #Make sure that the container is stopped
+                print("Found [%s]. Stopping container..."%(container_name))
+                this_container.stop()
+            except Exception as e:
+                print("Failed to stop a container named [%s]"%(container_name))
+                reuse_containers = False
+                break
+        print("Found all of the containers, we will reuse them")
+        return True
+    
+    #Note that this variable can be changed by the block above, so don't
+    #convert this into an if/else
+    if reuse_containers is False:
+        for index in range(0,num_containers):
+            container_name = container_template%(index)
+            print("Trying to remove container [%s]"%(container_name))
+            try:
+                container = client.containers.get(container_name)
+            except Exception as e:
+                print("Could not find Docker Container [%s]. Container does not exist, so no need to remove it"%(container_name))
+                continue
+            try:
+                #container was found, so now we try to remove it
+                #v also removes volumes linked to the container
+                container.remove(v=True, force=forceRemove) #remove it even if it is running. remove volumes as well
+                print("Successfully removed Docker Container [%s]"%(container_name))
+            except Exception as e:
+                print("Could not remove Docker Container [%s]"%(container_name))
+                raise(Exception("CONTAINER REMOVE ERROR"))
+        return False
+
+
+
+        
+
+    
+
+    
+
+
 def main(args):
     
     start_time = timer()
@@ -100,14 +200,18 @@ def main(args):
     parser.add_argument("-pr", "--pr-number", type=int, required=False, help="Pull Request Number")
     parser.add_argument("-n", "--num_containers", required=False, type=int, default=1, help="The number of splunk docker containers to start and run for testing")
     
-    parser.add_argument("-i", "--reuse_images", required=False, type=bool, default=False, help="Should existing images be re-used, or should they be redownloaded?")
-    parser.add_argument("-c", "--reuse_containers", required=False, type=bool, default=False,  help="Should existing containers be re-used, or should they be rebuilt?")
 
+    parser.add_argument("-i", "--interactive_failure", required=False, default=False, action='store_true', help="If a test fails, should we pause before removing data so that the search can be debugged?")
+
+    parser.add_argument("-i", "--reuse_image", required=False, default=False, action='store_true', help="Should existing images be re-used, or should they be redownloaded?")
+    
+    parser.add_argument("-c", "--reuse_containers", required=False, default=False,  help="Should existing containers be re-used, or should they be rebuilt?")
     parser.add_argument("-s", "--success_file", type=str, required=False, help="File that contains previously successful runs that we don't need to test")
     parser.add_argument("-user", "--splunkbase_username", type=str, required=True, help="Splunkbase username for downloading Splunkbase apps")
     parser.add_argument("-pw", "--splunkbase_password", type=str, required=True, help="Splunkbase password for downloading Splunkbase apps")
     parser.add_argument("-m", "--mode", type=str, choices=DETECTION_MODES, required=False, help="Whether to test new detections, specific detections, or all detections", default="all")
     parser.add_argument("-t", "--types", type=str, required=False, nargs='+', help="Detection types to test. Can be one of more of %s"%(str(DETECTION_TYPES)), default=DETECTION_TYPES)
+    parser.add_argument("-ct", "--container_tag", type=str, required=False, help="The tag of the Splunk Container to use.  Tags are located at https://hub.docker.com/r/splunk/splunk/tags",default=DEFAULT_CONTAINER_TAG)
 
     args = parser.parse_args()
     branch = args.branch
@@ -115,11 +219,35 @@ def main(args):
     pr_number = args.pr_number
     num_containers = args.num_containers
     reuse_containers = args.reuse_containers
-    reuse_images = args.reuse_images
+    reuse_image = args.reuse_image
     success_file = args.success_file
     splunkbase_username = args.splunkbase_username
     splunkbase_password = args.splunkbase_password
+    full_docker_hub_container_name = "splunk/splunk:%s"%args.container_tag
     
+
+    client = docker.client.from_env()
+
+    #Remove containers that previously existed (if we are directed to do so)
+    try:
+        remove_existing_containers(client, reuse_containers, LOCAL_BASE_CONTAINER_NAME, num_containers)
+    except Exception as e:
+        print("Error tryting to remove existing containers.\n\tQuitting...")
+        sys.exit(1)
+
+    #Download and setup the image
+    try:
+        setup_image(client, reuse_image, full_docker_hub_container_name)
+    except Exception as e:
+        print("Error trying to set up the image.\n\tQuitting...")
+        sys.exit(1)
+    
+
+    sys.exit(0)
+
+
+
+
     mode = args.mode
     folder_names = args.types
     for t in args.types:
@@ -155,7 +283,16 @@ def main(args):
         github_service = GithubService(branch, pr_number)
     else:
         github_service = GithubService(branch)
-    test_files = github_service.get_all_tests_and_detections(folders=["endpoint"], previously_successful_tests=success_tests)
+    if args.mode == "new":
+        test_files = github_service.get_all_tests_and_detections(folders=args.types, previously_successful_tests=success_tests)
+    elif args.mode == "all":
+        pass
+    elif args.mode == "selected":
+        pass
+    else:
+        print("Unsupported mode [%s] chosen.  Supported modes are %s.\n\tQuitting..."%(args.mode, str(DETECTION_MODES)))
+        sys.exit(1)
+    
     
     if len(test_files) == 0:
         print("No new detections to test.")

From 4e52891726ec8e189657300065ac14d42b0b5a4c Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Wed, 20 Oct 2021 17:06:44 -0700
Subject: [PATCH 020/166] Better parsing of files to test, better error
 handling, more code reuse.

---
 .../detection_testing_execution.py            | 99 +++++++++++--------
 .../modules/github_service.py                 | 89 ++++++++++-------
 2 files changed, 109 insertions(+), 79 deletions(-)

diff --git a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
index 8140cee7fb..b59f4b862e 100644
--- a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
+++ b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
@@ -224,10 +224,54 @@ def main(args):
     splunkbase_username = args.splunkbase_username
     splunkbase_password = args.splunkbase_password
     full_docker_hub_container_name = "splunk/splunk:%s"%args.container_tag
+    interactive_failure = args.interactive_failure
+
+
+    #Read in all of the tests that we will ignore because they already passed
+    success_tests = []
+    if success_file is not None:
+        try:
+            with open(success_file, "r") as successes:
+                for line in successes.readlines():
+                    file_path_new = os.path.join("tests", os.path.splitext(line)[0]) + ".test.yml"
+                    success_tests.append(file_path_new)
+        except Exception as e:
+            print("Error - error reading success_file: [%s]"%(str(e)))
+            print("\n\tQuitting...")
+            sys.exit(1)
+                
+
     
 
+    #Ensure that a valid mode was chosen 
+    mode = args.mode
+    folder_names = args.types
+    for t in args.types:
+        if t not in DETECTION_TYPES:
+            print("Error - requested test of [%s] but the only valid types are %s.\tQuitting..."%(t, str(DETECTION_TYPES)))
+            sys.exit(1)
+
+
+    #Do some initial setup and validation of the containers and images
+
+    #If a user requests to use existing containers, they must also explicitly request to reuse existing images
+    if reuse_containers and not reuse_image:
+        print("Error - requested --reuse_containers but did not explicitly request --reuse_image.\n\tQuitting")
+        sys.exit(1)
+
+    if num_containers < 1:
+        #Perhaps this should be a mock-run - do the initial steps but don't do testing on the containers?
+        print("Error, requested 0 containers.  You must run with at least 1 container.")
+        sys.exit(1)
+    elif num_containers > MAX_RECOMMENDED_CONTAINERS_BEFORE_WARNING:
+        print("You requested to run with [%d] containers which may use a very large amount of resources "
+               "as they all run in parallel.  The maximum suggested number of parallel containers is "
+               "[%d].  We will do what you asked, but be warned!"%(num_containers, MAX_RECOMMENDED_CONTAINERS_BEFORE_WARNING))
+
     client = docker.client.from_env()
 
+    #Ensure that the images and containers are set up in a proper state
+
     #Remove containers that previously existed (if we are directed to do so)
     try:
         remove_existing_containers(client, reuse_containers, LOCAL_BASE_CONTAINER_NAME, num_containers)
@@ -243,65 +287,38 @@ def main(args):
         sys.exit(1)
     
 
-    sys.exit(0)
+    
 
 
 
 
-    mode = args.mode
-    folder_names = args.types
-    for t in args.types:
-        if t not in DETECTION_TYPES:
-            print("Error - requested test of [%s] but the only valid types are %s.\tQuitting..."%(t, str(DETECTION_TYPES)))
-            sys.exit(1)
+    
 
 
-    success_tests = []
-    if success_file is not None:
-        with open(success_file, "r") as successes:
-            for line in successes.readlines():
-                file_path_new = os.path.join("tests", os.path.splitext(line)[0]) + ".test.yml"
-                #file_path_base = os.path.splitext(line)[0].replace('detections', 'tests') + '.test'
-                #file_path_new = file_path_base + '.yml'
-                success_tests.append(file_path_new)
-                #success_tests = [x.strip() for x in successes.readlines()]
-    else:
-        pass
-
-
-    if num_containers < 1:
-        #Perhaps this should be a mock-run - do the initial steps but don't do testing on the containers?
-        print("Error, requested 0 containers.  You must run with at least 1 container.")
-        sys.exit(1)
-    elif num_containers > MAX_RECOMMENDED_CONTAINERS_BEFORE_WARNING:
-        print("You requested to run with [%d] containers which may use a very large amount of resources "
-               "as they all run in parallel.  The maximum suggested number of parallel containers is "
-               "[%d].  We will do what you asked, but be warned!"%(num_containers, MAX_RECOMMENDED_CONTAINERS_BEFORE_WARNING))
-
 
     if pr_number:
         github_service = GithubService(branch, pr_number)
     else:
         github_service = GithubService(branch)
     if args.mode == "new":
-        test_files = github_service.get_all_tests_and_detections(folders=args.types, previously_successful_tests=success_tests)
+        test_files = github_service.get_all_tests_and_detections(folders=args.types, 
+                                                                 previously_successful_tests=success_tests)
     elif args.mode == "all":
-        pass
-    elif args.mode == "selected":
-        pass
+        test_files = github_service.get_changed_test_files(folders=args.types,
+                                                           previously_successful_tests=success_tests)
+    #elif args.mode == "selected":
+    #    test_files = github_service.get_selected_test_files(folders=args.types,
+    #                                                       previously_successful_tests=success_tests)
+
     else:
         print("Unsupported mode [%s] chosen.  Supported modes are %s.\n\tQuitting..."%(args.mode, str(DETECTION_MODES)))
         sys.exit(1)
     
     
-    if len(test_files) == 0:
-        print("No new detections to test.")
-        #aws_service.dynamo_db_nothing_to_test(REGION, uuid_test, str(int(time.time())))
-        sys.exit(0)
-    
-    #print(test_files2[:5])
-    #sys.exit(0)
-
+    print("Number of detections to test: %d"%(len(test_files)))
+    time.sleep(5)
+    print(test_files)
+    sys.exit(0)
     new_test_files = []
 
 
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/github_service.py b/automated_detection_testing/ci/detection_testing_batch/modules/github_service.py
index 1d29e91a83..6854ba6607 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/github_service.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/github_service.py
@@ -30,20 +30,15 @@ class GithubService:
         return repo_obj
 
     
-    def get_all_tests_and_detections(self, folders=['endpoint', 'cloud', 'network'], ttps_to_test=["Anomaly","Hunting","TTP"], previously_successful_tests=[]):    
-        detections = []
-        for folder in folders:
-            detections.extend(self.get_all_files_in_folder(os.path.join("security_content/detections", folder), "*.yml"))
-            
-        
-        #Prune this down to only the subset of detections we can test
-        detections_to_test = []
-        tests=[]
- 
-        
-        for detection in detections:
-            #Don't do anything with SSA stuff
-            if os.path.basename(detection).startswith("ssa"):
+
+    def prune_detections(self, 
+                         detections_to_prune:list[str], 
+                         ttps_to_test:list[str], 
+                         previously_successful_tests:list[str],
+                         exclude_ssa:bool=True)->list[str]:
+        pruned_tests = []
+        for detection in detections_to_prune:
+            if os.path.basename(detection).startswith("ssa") and exclude_ssa:
                 continue
             with open(detection, "r") as d:
                 description = yaml.safe_load(d)
@@ -60,19 +55,31 @@ class GithubService:
                         print("Ignoring test [%s] before it has already passed previously"%(detection))
                     else:
                         #remove leading security_content/ from path
-                        tests.append(test_filepath_without_security_content)
+                        pruned_tests.append(test_filepath_without_security_content)
                 else:
                     #Don't do anything with these files
                     pass
+        return pruned_tests
 
-        return tests
+
+    def get_all_tests_and_detections(self, 
+                                     folders:list[str]=['endpoint', 'cloud', 'network'], 
+                                     ttps_to_test:list[str]=["Anomaly","Hunting","TTP"], 
+                                    previously_successful_tests:list[str]=[]) ->list[str]:    
+        detections = []
+        for folder in folders:
+            detections.extend(self.get_all_files_in_folder(os.path.join("security_content/detections", folder), "*.yml"))
+            
         
-    def get_all_files_in_folder(self, foldername, extension):
-        filesnames =  glob.glob(os.path.join(foldername, extension))
-        return filesnames
+        #Prune this down to only the subset of detections we can test
+        return self.prune_detections(detections, ttps_to_test, previously_successful_tests)
+        
+    def get_all_files_in_folder(self, foldername:str, extension:str)->list[str]:
+        filenames =  glob.glob(os.path.join(foldername, extension))
+        return filenames
 
 
-    def get_changed_test_files(self):
+    def get_changed_test_files(self, folders=['endpoint', 'cloud', 'network'], ttps_to_test=["Anomaly","Hunting","TTP"], previously_successful_tests=[]):
         branch1 = self.security_content_branch
         branch2 = 'develop'
         g = git.Git('security_content')
@@ -85,31 +92,37 @@ class GithubService:
             for file_path in changed_files:
                 # added or changed test files
                 if file_path.startswith('A') or file_path.startswith('M'):
-                    if 'tests' in file_path:
-                        if not os.path.basename(file_path).startswith('ssa') and os.path.basename(file_path).endswith('.test.yml'):
-                            if file_path not in changed_test_files:
-                                changed_test_files.append(file_path)
+                    if 'tests' in file_path and os.path.basename(file_path).endswith('.test.yml'):
+                            changed_test_files.append(file_path)
 
                     # changed detections
-                    if 'detections' in file_path:
-                        if not os.path.basename(file_path).startswith('ssa') and os.path.basename(file_path).endswith('.yml'):
+                    if 'detections' in file_path and os.path.basename(file_path).endswith('.yml'):
                             changed_detection_files.append(file_path)
-                            #file_path_base = os.path.splitext(file_path)[0].replace('detections', 'tests') + '.test'
-                            #file_path_new = file_path_base + '.yml'
-                            #if file_path_new not in changed_test_files:
-                            #    changed_test_files.append(file_path_new)
+                            
 
         #all files have the format A\tFILENAME or M\tFILENAME.  Get rid of those leading characters
         changed_test_files      = [name.split('\t')[1] for name in changed_test_files      if len(name.split('\t')) == 2]
         changed_detection_files = [name.split('\t')[1] for name in changed_detection_files if len(name.split('\t')) == 2]
-        
 
-        detections_to_test,_,_ = self.filter_test_types(changed_detection_files)
-        for f in detections_to_test:
-            file_path_base = os.path.splitext(f)[0].replace('detections', 'tests') + '.test'
-            file_path_new = file_path_base + '.yml'
-            if file_path_new not in changed_test_files:
-                changed_test_files.append(file_path_new)
+
+        #convert the test files to the detection file equivalent
+        converted_test_files = []
+        for test_filepath in changed_test_files:
+            detection_filename = str(pathlib.Path(*pathlib.Path(test_filepath).parts[-2:])).replace("tests", "detections",1)
+            converted_test_files.append(detection_filename)
+
+        for name in converted_test_files:
+            if name not in changed_detection_files:
+                changed_detection_files.append(name)
+
+        return self.prune_detections(changed_detection_files, ttps_to_test, previously_successful_tests)
+
+        #detections_to_test,_,_ = self.filter_test_types(changed_detection_files)
+        #for f in detections_to_test:
+        #    file_path_base = os.path.splitext(f)[0].replace('detections', 'tests') + '.test'
+        #    file_path_new = file_path_base + '.yml'
+        #    if file_path_new not in changed_test_files:
+        #        changed_test_files.append(file_path_new)
 
         
         
@@ -121,7 +134,7 @@ class GithubService:
         #print(len(changed_test_files))
         #import time
         #time.sleep(5)
-        return changed_test_files
+        
 
     def filter_test_types(self, test_files, test_types = ["Anomaly", "Hunting", "TTP"]):
         files_to_test = []

From 60262a57f82b617d14e4db58c7fb010bfbc0dcb0 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Wed, 20 Oct 2021 17:33:26 -0700
Subject: [PATCH 021/166] Added some more command line arguments to speed up
 testing and did some more smoketesting.

---
 .../detection_testing_execution.py            | 34 +++++++++----------
 .../modules/github_service.py                 |  4 +--
 2 files changed, 19 insertions(+), 19 deletions(-)

diff --git a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
index b59f4b862e..d019d6660f 100644
--- a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
+++ b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
@@ -203,16 +203,18 @@ def main(args):
 
     parser.add_argument("-i", "--interactive_failure", required=False, default=False, action='store_true', help="If a test fails, should we pause before removing data so that the search can be debugged?")
 
-    parser.add_argument("-i", "--reuse_image", required=False, default=False, action='store_true', help="Should existing images be re-used, or should they be redownloaded?")
+    parser.add_argument("-ri", "--reuse_image", required=False, default=False, action='store_true', help="Should existing images be re-used, or should they be redownloaded?")
     
-    parser.add_argument("-c", "--reuse_containers", required=False, default=False,  help="Should existing containers be re-used, or should they be rebuilt?")
+    parser.add_argument("-rc", "--reuse_containers", required=False, default=False,  help="Should existing containers be re-used, or should they be rebuilt?")
     parser.add_argument("-s", "--success_file", type=str, required=False, help="File that contains previously successful runs that we don't need to test")
     parser.add_argument("-user", "--splunkbase_username", type=str, required=True, help="Splunkbase username for downloading Splunkbase apps")
     parser.add_argument("-pw", "--splunkbase_password", type=str, required=True, help="Splunkbase password for downloading Splunkbase apps")
-    parser.add_argument("-m", "--mode", type=str, choices=DETECTION_MODES, required=False, help="Whether to test new detections, specific detections, or all detections", default="all")
-    parser.add_argument("-t", "--types", type=str, required=False, nargs='+', help="Detection types to test. Can be one of more of %s"%(str(DETECTION_TYPES)), default=DETECTION_TYPES)
+    parser.add_argument("-m", "--mode", type=str, choices=DETECTION_MODES, required=False, help="Whether to test new detections, specific detections, or all detections", default="new")
+    parser.add_argument("-t", "--types", type=str, required=False, help="Detection types to test. Can be one of more of %s"%(str(DETECTION_TYPES)), default=DETECTION_TYPES)
     parser.add_argument("-ct", "--container_tag", type=str, required=False, help="The tag of the Splunk Container to use.  Tags are located at https://hub.docker.com/r/splunk/splunk/tags",default=DEFAULT_CONTAINER_TAG)
-
+    parser.add_argument("-p", "--persist_security_content", required=False, default=False, action="store_true", help="Assumes security_content directory already exists.  Don't check it out and overwrite it again.  Saves "\
+                                                                                                                     "time and allows you to test a detection that you've updated.  Runs generate again in case you have "\
+                                                                                                                     "updated macros or anything else.  Especially useful for quick, local, iterative testing.")
     args = parser.parse_args()
     branch = args.branch
     uuid_test = args.uuid
@@ -245,14 +247,15 @@ def main(args):
 
     #Ensure that a valid mode was chosen 
     mode = args.mode
-    folder_names = args.types
-    for t in args.types:
+    folders = [a.strip() for a in args.types.split(',')]
+    for t in folders:
         if t not in DETECTION_TYPES:
             print("Error - requested test of [%s] but the only valid types are %s.\tQuitting..."%(t, str(DETECTION_TYPES)))
             sys.exit(1)
+    
+    
 
-
-    #Do some initial setup and validation of the containers and images
+    #Do some initial setup and validation of the containers and images  
 
     #If a user requests to use existing containers, they must also explicitly request to reuse existing images
     if reuse_containers and not reuse_image:
@@ -300,11 +303,11 @@ def main(args):
         github_service = GithubService(branch, pr_number)
     else:
         github_service = GithubService(branch)
-    if args.mode == "new":
-        test_files = github_service.get_all_tests_and_detections(folders=args.types, 
+    if args.mode == "all":
+        test_files = github_service.get_all_tests_and_detections(folders=folders, 
                                                                  previously_successful_tests=success_tests)
-    elif args.mode == "all":
-        test_files = github_service.get_changed_test_files(folders=args.types,
+    elif args.mode == "new":
+        test_files = github_service.get_changed_test_files(folders=folders,
                                                            previously_successful_tests=success_tests)
     #elif args.mode == "selected":
     #    test_files = github_service.get_selected_test_files(folders=args.types,
@@ -315,10 +318,7 @@ def main(args):
         sys.exit(1)
     
     
-    print("Number of detections to test: %d"%(len(test_files)))
-    time.sleep(5)
-    print(test_files)
-    sys.exit(0)
+    
     new_test_files = []
 
 
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/github_service.py b/automated_detection_testing/ci/detection_testing_batch/modules/github_service.py
index 6854ba6607..80b9c0043b 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/github_service.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/github_service.py
@@ -101,8 +101,8 @@ class GithubService:
                             
 
         #all files have the format A\tFILENAME or M\tFILENAME.  Get rid of those leading characters
-        changed_test_files      = [name.split('\t')[1] for name in changed_test_files      if len(name.split('\t')) == 2]
-        changed_detection_files = [name.split('\t')[1] for name in changed_detection_files if len(name.split('\t')) == 2]
+        changed_test_files      = [os.path.join("security_content",name.split('\t')[1]) for name in changed_test_files      if len(name.split('\t')) == 2]
+        changed_detection_files = [os.path.join("security_content",name.split('\t')[1]) for name in changed_detection_files if len(name.split('\t')) == 2]
 
 
         #convert the test files to the detection file equivalent

From 10d43aa599460a4ff07ca040c5d754475f1ea1c8 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Wed, 20 Oct 2021 18:17:07 -0700
Subject: [PATCH 022/166] Added functionality and checks to remove the
 security_content directory or persist the security_content directory.  This
 is useful because, on slow connections, it makes running subsequent tests A
 LOT faster.  It is also faster on a fast connection.  Finally, and most
 importantly, it lets you easily persist and re-test changes that you've made
 to detections.

---
 .../detection_testing_execution.py            | 28 +++++++++++++++----
 .../modules/github_service.py                 |  7 ++++-
 2 files changed, 29 insertions(+), 6 deletions(-)

diff --git a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
index d019d6660f..8fa5f8877a 100644
--- a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
+++ b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
@@ -228,6 +228,7 @@ def main(args):
     full_docker_hub_container_name = "splunk/splunk:%s"%args.container_tag
     interactive_failure = args.interactive_failure
 
+    persist_security_content = args.persist_security_content
 
     #Read in all of the tests that we will ignore because they already passed
     success_tests = []
@@ -297,12 +298,30 @@ def main(args):
 
     
 
+    if persist_security_content is True and os.path.exists("security_content"):
+        print("******You chose --persist_security_content and the security_content directory exists. We will not check out the repo again. Please be aware, this could cause issues if you're out of date.******")
+        github_service = GithubService(branch, existing_directory=persist_security_content)
 
-
-    if pr_number:
-        github_service = GithubService(branch, pr_number)
+    elif persist_security_content is True:
+        print("Error - you chose --persist_security_content but the security_content directory does not exist!\n\tQuitting...")
+        sys.exit(1)
     else:
-        github_service = GithubService(branch)
+        if os.path.exists("security_content/"):
+            print("Deleting the security_content directory")
+            try:
+                import shutil
+                shutil.rmtree("security_content/", ignore_errors=True)
+                print("Successfully removed security_content directory")
+            except Exception as e:
+                print("Error - could not remove the security_content directory: [%s].\n\tQuitting..."%(str(e)))
+                sys.exit(1)
+        
+        if pr_number:
+            github_service = GithubService(branch, pr_number)
+        else:
+            github_service = GithubService(branch)
+    
+    
     if args.mode == "all":
         test_files = github_service.get_all_tests_and_detections(folders=folders, 
                                                                  previously_successful_tests=success_tests)
@@ -318,7 +337,6 @@ def main(args):
         sys.exit(1)
     
     
-    
     new_test_files = []
 
 
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/github_service.py b/automated_detection_testing/ci/detection_testing_batch/modules/github_service.py
index 80b9c0043b..b2115b09d0 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/github_service.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/github_service.py
@@ -16,9 +16,14 @@ SECURITY_CONTENT_URL = "https://github.com/splunk/security_content"
 
 class GithubService:
 
-    def __init__(self, security_content_branch, PR_number = None):
+    def __init__(self, security_content_branch:str, PR_number:int = None, existing_directory:bool=False):
+        
         self.security_content_branch = security_content_branch
+        if existing_directory:
+            return
+        print("Checking out security_content!")
         self.security_content_repo_obj = self.clone_project(SECURITY_CONTENT_URL, f"security_content", f"develop")
+
         if PR_number:
             subprocess.call(["git", "-C", "security_content/", "fetch", "origin", "refs/pull/%d/head:%s"%(PR_number, security_content_branch)])
 

From 7ba61bc2b5c6c4dce4a105024eb252891bd727d8 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Thu, 21 Oct 2021 12:22:28 -0700
Subject: [PATCH 023/166] Generate good password for a container instead of
 always using a static password.  Note that a password can also be supplied on
 the command line.

---
 .../detection_testing_execution.py            | 128 +++++++++++-------
 1 file changed, 76 insertions(+), 52 deletions(-)

diff --git a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
index 8fa5f8877a..c3e58e4d31 100644
--- a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
+++ b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
@@ -17,7 +17,7 @@ import subprocess
 
 from timeit import default_timer as timer
 from datetime import timedelta
-
+import string
 
 SPLUNK_CONTAINER_APPS_DIR = "/opt/splunk/etc/apps"
 index_file_local_path = "indexes.conf.tar"
@@ -34,10 +34,6 @@ LOCAL_BASE_CONTAINER_NAME = "splunk_test_%d"
 
 
 
-#DOCKER_COMMIT_NAME = "splunk_configured"
-#RUNNER_BASE_NAME = "splunk_runner"
-
-
 BASE_CONTAINER_WEB_PORT=8000
 BASE_CONTAINER_MANAGEMENT_PORT=8089
 
@@ -45,6 +41,24 @@ BASE_CONTAINER_MANAGEMENT_PORT=8089
 DETECTION_TYPES = ['endpoint', 'cloud', 'network']
 DETECTION_MODES = ['new', 'all', 'selected']
 
+
+#taken from attack_range
+def get_random_password():
+    random_source = string.ascii_letters + string.digits
+    password = random.choice(string.ascii_lowercase)
+    password += random.choice(string.ascii_uppercase)
+    password += random.choice(string.digits)
+
+    
+    for i in range(random.randrange(16,26)):
+        password += random.choice(random_source)
+
+    password_list = list(password)
+    random.SystemRandom().shuffle(password_list)
+    password = ''.join(password_list)
+    return password
+
+
 def wait_for_splunk_ready(splunk_container_name=None, splunk_web_port=None, max_seconds=30):
     #The smarter version of this will try to hit one of the pages,
     #probably the login page, and when that is available it means that
@@ -200,7 +214,7 @@ def main(args):
     parser.add_argument("-pr", "--pr-number", type=int, required=False, help="Pull Request Number")
     parser.add_argument("-n", "--num_containers", required=False, type=int, default=1, help="The number of splunk docker containers to start and run for testing")
     
-
+    parser.add_argument("-cw", "--container_password", required=False, help="A password to use for the container.  If you don't choose one, a complex one will be generated for you.")
     parser.add_argument("-i", "--interactive_failure", required=False, default=False, action='store_true', help="If a test fails, should we pause before removing data so that the search can be debugged?")
 
     parser.add_argument("-ri", "--reuse_image", required=False, default=False, action='store_true', help="Should existing images be re-used, or should they be redownloaded?")
@@ -228,8 +242,16 @@ def main(args):
     full_docker_hub_container_name = "splunk/splunk:%s"%args.container_tag
     interactive_failure = args.interactive_failure
 
-    persist_security_content = args.persist_security_content
+    container_password = args.container_password
+    if container_password is None:
+        #Generate a sufficiently complex password 
+        container_password = get_random_password()
+        print("Generated the password: [%s]"%container_password)
+    else:
+        print("Since you supplied a password, we will not generate one for you.")
+    sys.exit(0)
 
+    persist_security_content = args.persist_security_content
     #Read in all of the tests that we will ignore because they already passed
     success_tests = []
     if success_file is not None:
@@ -337,62 +359,66 @@ def main(args):
         sys.exit(1)
     
     
-    new_test_files = []
-
-
-    for f in test_files:
-        if f not in success_tests:
-            new_test_files.append(f)
-        else:
-            #pass
-            print("Already found [%s] in success file, not testing it again"%(f))
-
-    #test_files = new_test_files
-    '''
-    print('\n'.join(test_files))
-    for f in test_files:
-        with open('security_content/'+f,'r') as b:
-            data = b.read()
-            if 'baseline' in data.lower():
-                print("baseline found in [%s]"%(f))
-    '''
     
-    time.sleep(10)
     
     #Go into the security content directory
     print("****GENERATE NEW CONTENT****")
     os.chdir("security_content")
-    commands = ["python3 -m venv .venv", "source .venv/bin/activate", "python3 -m pip install wheel", "python3 -m pip install -r requirements.txt", "python3 contentctl.py --path . --verbose generate --product ESCU --output dist/escu", "tar -czf DA-ESS-ContentUpdate.spl -C dist/escu ."]
-    ret = subprocess.run("; ".join(commands), shell=True, capture_output=False)
+    print(os.getcwd())
+    if persist_security_content is False:
+        commands = ["python3 -m venv .venv", "source .venv/bin/activate", "python3 -m pip install wheel", "python3 -m pip install -r requirements.txt", "python3 contentctl.py --path . --verbose generate --product ESCU --output dist/escu", "tar -czf DA-ESS-ContentUpdate.spl -C dist/escu ."]
+    else:
+        commands = ["source .venv/bin/activate", "python3 contentctl.py --path . --verbose generate --product ESCU --output dist/escu", "tar -czf DA-ESS-ContentUpdate.spl -C dist/escu ."]
+    ret = subprocess.run("; ".join(commands), shell=True, capture_output=True)
     if ret.returncode != 0:
         print("Error generating new content.  Exiting...")
         sys.exit(1)
     print("New content generated successfully")    
-    os.chdir("..")
+    
 
     print("Generate new ESCU Package using new content")
-    commands = ["curl -Ls https://download.splunk.com/misc/packaging-toolkit/splunk-packaging-toolkit-0.9.0.tar.gz -o splunk-packaging-toolkit-latest.tar.gz", 
-    "rm -rf slim-latest",
-    "mkdir slim-latest", 
-    "tar -zxf splunk-packaging-toolkit-latest.tar.gz -C slim-latest --strip-components=1",
-    "cd slim-latest", 
-    "virtualenv --python=/usr/bin/python2.7 --clear venv",
-    "source venv/bin/activate",
-    "python2 -m pip install --upgrade pip",
-    "python2 -m pip install wheel",
-    "python2 -m pip install semantic_version",
-    "python2 -m pip install .",
-    "cp -R ../security_content/dist/escu DA-ESS-ContentUpdate",
-    "slim package -o upload DA-ESS-ContentUpdate",
-    "cp upload/DA-ESS-ContentUpdate*.tar.gz /tmp/apps/DA-ESS-ContentUpdate-latest.tar.gz"
-    ]
+    if persist_security_content is True:
+        os.chdir("slim_packaging")
+        commands = ["cd slim-latest", 
+                    "source venv/bin/activate",
+                    "cp -R ../../dist/escu DA-ESS-ContentUpdate",
+                    "slim package -o upload DA-ESS-ContentUpdate",
+                    "cp upload/DA-ESS-ContentUpdate*.tar.gz ../apps/DA-ESS-ContentUpdate-latest.tar.gz"]
+    
+    else:
+        os.mkdir("slim_packaging")
+        os.chdir("slim_packaging")
+        os.mkdir("apps")
+        commands = ["curl -Ls https://download.splunk.com/misc/packaging-toolkit/splunk-packaging-toolkit-0.9.0.tar.gz -o splunk-packaging-toolkit-latest.tar.gz", 
+                    "rm -rf slim-latest",
+                    "mkdir slim-latest", 
+                    "tar -zxf splunk-packaging-toolkit-latest.tar.gz -C slim-latest --strip-components=1",
+                    "cd slim-latest", 
+                    "virtualenv --python=/usr/bin/python2.7 --clear venv",
+                    "source venv/bin/activate",
+                    "python2 -m pip install --upgrade pip",
+                    "python2 -m pip install wheel",
+                    "python2 -m pip install semantic_version",
+                    "python2 -m pip install .",
+                    "cp -R ../../dist/escu DA-ESS-ContentUpdate",
+                    "slim package -o upload DA-ESS-ContentUpdate",
+                    "cp upload/DA-ESS-ContentUpdate*.tar.gz ../apps/DA-ESS-ContentUpdate-latest.tar.gz"]
+        
+    
+    
 
     ret = subprocess.run("; ".join(commands), shell=True, capture_output=False)
     if ret.returncode != 0:
-        print("Error generating new ESCU Package.  Exiting...")
+        print("Error generating new ESCU Package.\n\tQuitting..."%())
         sys.exit(1)
-    print("New ESCU PAckage generated successfully")    
-
+    os.chdir("../..")
+    print("New ESCU Package generated successfully")    
+    
+    #Enqueue all of the test files for processing
+    test_file_queue = queue.Queue()
+    for filename in test_files:
+        test_file_queue.put(filename)
+    
     
 
     client = docker.client.from_env()
@@ -426,9 +452,7 @@ def main(args):
     
 
     # print("***Files to test: %d"%(len(test_files)))
-    test_file_queue = queue.Queue()
-    for filename in test_files:
-        test_file_queue.put(filename)
+  
     # print("***Test files enqueued")
 
     
@@ -607,7 +631,7 @@ def main(args):
         ports= {"8000/tcp": web_port,
                 "8089/tcp": management_port
                }
-        mounts = [docker.types.Mount(target = '/tmp/apps/', source = '/tmp/apps', type='bind', read_only=True)]
+        mounts = [docker.types.Mount(target = '/tmp/apps/', source = 'security_content/slim_packaging/apps', type='bind', read_only=True)]
 
         print("Creating CONTAINER: [%s]"%(container_name))
         base_container = client.containers.create(DOCKER_HUB_CONTAINER_PATH, ports=ports, environment=environment, name=container_name, mounts=mounts, detach=True)

From 365bcc82acb7173b9c638cad817faa4d8b636b00 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Thu, 21 Oct 2021 15:32:57 -0700
Subject: [PATCH 024/166] More cleanup and removal of dead code.  Created an
 object to synchronize all the threads and their status instead of having a
 massive number of arguments to each thread.  Includes a synchronization
 primitive.

---
 .../detection_testing_execution.py            | 365 +++++++-----------
 1 file changed, 148 insertions(+), 217 deletions(-)

diff --git a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
index c3e58e4d31..6c4a29e8e3 100644
--- a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
+++ b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
@@ -17,7 +17,9 @@ import subprocess
 
 from timeit import default_timer as timer
 from datetime import timedelta
+from datetime import datetime
 import string
+import shutil
 
 SPLUNK_CONTAINER_APPS_DIR = "/opt/splunk/etc/apps"
 index_file_local_path = "indexes.conf.tar"
@@ -43,7 +45,7 @@ DETECTION_MODES = ['new', 'all', 'selected']
 
 
 #taken from attack_range
-def get_random_password():
+def get_random_password()->str:
     random_source = string.ascii_letters + string.digits
     password = random.choice(string.ascii_lowercase)
     password += random.choice(string.ascii_uppercase)
@@ -215,6 +217,7 @@ def main(args):
     parser.add_argument("-n", "--num_containers", required=False, type=int, default=1, help="The number of splunk docker containers to start and run for testing")
     
     parser.add_argument("-cw", "--container_password", required=False, help="A password to use for the container.  If you don't choose one, a complex one will be generated for you.")
+    parser.add_argument("-show", "--show_password", required=False, default=False, action='store_true', help="Show the generated password to use to login to splunk.  For an CI/CD run, you probably don't want this.")
     parser.add_argument("-i", "--interactive_failure", required=False, default=False, action='store_true', help="If a test fails, should we pause before removing data so that the search can be debugged?")
 
     parser.add_argument("-ri", "--reuse_image", required=False, default=False, action='store_true', help="Should existing images be re-used, or should they be redownloaded?")
@@ -241,15 +244,16 @@ def main(args):
     splunkbase_password = args.splunkbase_password
     full_docker_hub_container_name = "splunk/splunk:%s"%args.container_tag
     interactive_failure = args.interactive_failure
-
-    container_password = args.container_password
-    if container_password is None:
+    show_password = args.show_password
+    splunk_password = args.container_password
+    if splunk_password is None:
         #Generate a sufficiently complex password 
-        container_password = get_random_password()
-        print("Generated the password: [%s]"%container_password)
+        splunk_password = get_random_password()
+        if show_password is True:
+            print("Generated the password: [%s]"%splunk_password)
     else:
         print("Since you supplied a password, we will not generate one for you.")
-    sys.exit(0)
+    
 
     persist_security_content = args.persist_security_content
     #Read in all of the tests that we will ignore because they already passed
@@ -331,7 +335,6 @@ def main(args):
         if os.path.exists("security_content/"):
             print("Deleting the security_content directory")
             try:
-                import shutil
                 shutil.rmtree("security_content/", ignore_errors=True)
                 print("Successfully removed security_content directory")
             except Exception as e:
@@ -407,7 +410,7 @@ def main(args):
     
     
 
-    ret = subprocess.run("; ".join(commands), shell=True, capture_output=False)
+    ret = subprocess.run("; ".join(commands), shell=True, capture_output=True)
     if ret.returncode != 0:
         print("Error generating new ESCU Package.\n\tQuitting..."%())
         sys.exit(1)
@@ -421,207 +424,47 @@ def main(args):
     
     
 
-    client = docker.client.from_env()
-    splunk_password = '123456qwertyQWERTY'
     
     
     
     
-    
-
-    #print("run it now!")
-    
-    #time.sleep(180)
-    #dt_ar = aws_service.get_ar_information_from_dynamo_db(REGION, DT_ATTACK_RANGE_STATE)
-    #splunk_instance = aws_service.get_splunk_instance(REGION, dt_ar['ssh_key_name'])
-
-    #splunk_ip = splunk_instance['NetworkInterfaces'][0]['Association']['PublicIp']
-    #splunk_password = dt_ar['password']
-    #ssh_key_name = dt_ar['ssh_key_name']
-    #private_key = dt_ar['private_key']
-
-    #because this is only accessible to localhost, the password doesn't need to be particularly secure
-    #We can also share it between splunk on all containers
-    
-    #splunk_password = secrets.token_urlsafe(PASSWORD_LENGTH)
-
-    #Only accessible on local host, it's okay to expose the password for debugging
-    #splunk_password = "123456qwerty!@#$%^QWERTY"
-    #splunk_password = '123456qwerty%^QWERTY'
+    #Create threads to manage all of the containers that we will start up
     splunk_container_manager_threads = []
     
 
-    # print("***Files to test: %d"%(len(test_files)))
-  
-    # print("***Test files enqueued")
-
-    
-    # print("Getting docker client")
-    client = docker.client.from_env()
-    
-    # try:
-    #     print("Removing any existing containers called [%s]."%(BASE_CONTAINER_NAME))
-
-    #     c = client.containers.get(BASE_CONTAINER_NAME)
-    # except:
-    #     print("Container [%s] did not exist. No need to remove it. It will; be created for you."%(BASE_CONTAINER_NAME))
-    #     c = None
-    
-
-    # if (c and reuse_containers):
-    #     print("Found a container called [%s]. NOT removing it because you have specified --reuse_containers [%s]. "
-    #             "However, we must stop the container.  Stopping it now..."%(BASE_CONTAINER_NAME, reuse_containers))
-    #     stop_container(client, BASE_CONTAINER_NAME)
-        
-    # elif c:
-    #     print("Found a container called [%s]. Removing it because you have specified --reuse_containers [%s]"%(BASE_CONTAINER_NAME, reuse_containers))
-    #     remove_container(client, BASE_CONTAINER_NAME)
-        
-
-
-
-    # download_image = False
-    # try:
-    #     client.images.get(DOCKER_HUB_CONTAINER_PATH)
-    #     if reuse_images:
-    #         print("You already have an image named [%s]."%(DOCKER_HUB_CONTAINER_PATH))
-    #         download_image = False
-    #     else:
-    #         print("You already have an image named [%s]., "
-    #                 "but have speicified --reuse_images %s"%(DOCKER_HUB_CONTAINER_PATH, reuse_images))
-    #         download_image = True
-
-    # except:
-    #     print("You did not have an image named [%s]."%(DOCKER_HUB_CONTAINER_PATH))
-    #     download_image = True
-
-    # if download_image:        
-    #     try:
-    #         print("Downloading image [%s].  Please note "
-    #             "that this could take a long time depending on your "
-    #             "connection. It's around 2GB."%(DOCKER_HUB_CONTAINER_PATH))
-    #         client.images.pull(DOCKER_HUB_CONTAINER_PATH)
-    #         print("Finished downloading the image [%s]"%(DOCKER_HUB_CONTAINER_PATH))
-    #     except Exception as e: 
-    #         print("Unrecoverable error downloading image [%s]:[%s]"%(DOCKER_HUB_CONTAINER_PATH, str(e)))
-    #         sys.exit(1)
-
-
-    # remove_tag = False        
-    # try:
-    #     image = client.images.get(DOCKER_COMMIT_NAME)
-    #     print("Found an image called [%s]"%(DOCKER_COMMIT_NAME))
-    #     if reuse_images == False:
-    #         print("We will remove the image [%s] because you have specificed --reuse_images %s"%(DOCKER_COMMIT_NAME, reuse_images))
-    #         remove_tag = True
-    #         build_Tag = True
-    #     else:
-    #         print("We will use the preexisting image for [%s]"%(DOCKER_COMMIT_NAME))
-    #         build_tag = False
-    # except:
-    #     print("No image found named [%s]"%(DOCKER_COMMIT_NAME))
-    #     build_tag = True
-    
-
-    # if remove_tag:
-    #     try:
-    #         #Stop it if it's running, remove associated volumes too
-    #         client.images.remove(image=DOCKER_COMMIT_NAME, force=True)
-            
-    #     except Exception as e:
-    #         print("Unrecoverable error removing [%s]: [%s]"%(DOCKER_COMMIT_NAME, str(e)))
-    #         sys.exit(1)
-            
-    
-            
-
-    # for ind in range(num_containers):
-    #     ind = str(ind)
-    #     if not reuse_containers:
-    #         try:
-    #             print("Creating a new container called [%s]"%(BASE_CONTAINER_NAME+ind))
-            
-
-    #             environment = {"SPLUNK_START_ARGS": "--accept-license",
-    #                     "SPLUNK_PASSWORD"  : splunk_password }
-    #             ports= {"8000/tcp": BASE_CONTAINER_WEB_PORT - 1 + int(ind) + 1,
-    #             "8089/tcp": BASE_CONTAINER_MANAGEMENT_PORT - 1 + int(ind) + 1
-    #             }
-    #             base_container = client.containers.create("splunk/splunk:latest", ports=ports, environment=environment, name=BASE_CONTAINER_NAME+ind, detach=True)
-    #             print("Running the new container called [%s]"%(BASE_CONTAINER_NAME+ind))
-    #             base_container.start()
-    #             print("Container is running [%s]"%(BASE_CONTAINER_NAME+ind))
-    #             print("Sleep for 60 seconds to allow the container to fully start up...")
-    #             wait_for_splunk_ready(max_seconds=60)
-    #             print("The container has fully started!")
-
-    #             print("Do the ESCU installation on this container. That way we don't have to "
-    #                     "do it on every container that we then spin up.")
-
-    #             testing_service.prepare_detection_testing(BASE_CONTAINER_NAME+ind, splunk_password)
-    #             print("Waiting for a few seconds for the splunk app to come up.")
-    #             wait_for_splunk_ready(max_seconds=30)
-    #             print("Install the apps and enable accelerate")
-    #             wait_for_splunk_ready(max_seconds=180)
-    #             print("Stopping the running container [%s]"%(BASE_CONTAINER_NAME+ind))
-    #             base_container.stop()
-    #             #I am almost positive that I'm doing this wrong but it works for now...
-
-    #             #print("Committing the configured container: [%s]--->[%s]"%(BASE_CONTAINER_NAME, DOCKER_COMMIT_NAME))
-    #             #base_container.commit(repository=DOCKER_COMMIT_NAME)
-        
-
-    #         except Exception as e:
-    #             print("There was an error getting the base container up and running.  "
-    #                 "We cannot recover from this: [%s]\nGoodbye..."%(str(e)))
-    #             sys.exit(1)
-
-     
-    # # # #The part below does not seem to be working as expected. Will need to look into it
-    # # # #When I create the new container, it fails to boot with 
-    # # # # The CA file specified (/opt/splunk/etc/auth/cacert.pem) does not exist. Cannot continue.
-    # # # # SSL certificate generation failed.
-
-
-    # # # # MSG:
-
-    # # # # non-zero return code
     
     
-    
-
-    
-    print("Make all the threads...")
-    print("The number of detections we will test is [%d]"%(test_file_queue.qsize()))
-    
-    results_queue = queue.Queue()
-    success_names_queue = queue.Queue()
-    failure_names_queue = queue.Queue()
-    
-
+    results_tracker = SynchronizedResultsTracker(test_files)
     
     for container_index in range(num_containers):
-        container_name = "%s_%d"%(BASE_CONTAINER_NAME, container_index)
+        container_name = LOCAL_BASE_CONTAINER_NAME%container_index
         
         web_port = BASE_CONTAINER_WEB_PORT  + container_index
         management_port = BASE_CONTAINER_MANAGEMENT_PORT + container_index
 
         SPLUNK_COMMON_INFORMATION_MODEL = "https://splunkbase.splunk.com/app/1621/release/4.20.2/download"        
         SPLUNK_SECURITY_ESSENTIALS = "https://splunkbase.splunk.com/app/3435/release/3.3.4/download"
-        SPLUNK_ADD_ON_FOR_SYSMON_OLD = "https://splunkbase.splunk.com/app/1914/release/10.6.2/download"
+        #SPLUNK_ADD_ON_FOR_SYSMON_OLD = "https://splunkbase.splunk.com/app/1914/release/10.6.2/download"
         #SPLUNK_ADD_ON_FOR_SYSMON_NEW = "https://splunkbase.splunk.com/app/5709/release/1.0.1/download"
-        #LOCAL_SPLUNK_ADD_ON_FOR_SYSMON_NEW  = "/tmp/apps/Splunk_TA_microsoft_sysmon-1.0.2-B1.spl"
+        
+        #Just a hack until we get the new version of system deployed and available from splunkbase
+        LOCAL_SPLUNK_ADD_ON_FOR_SYSMON_PATH  = os.path.expanduser("~/Downloads/Splunk_TA_microsoft_sysmon-1.0.2-B1.spl")    
+        LOCAL_SPLUNK_ADD_ON_FOR_SYSMON_VOLUME_PATH  = "/tmp/apps/Splunk_TA_microsoft_sysmon-1.0.2-B1.spl"
+        shutil.copyfile(LOCAL_SPLUNK_ADD_ON_FOR_SYSMON_PATH, "security_content/slim_packaging/apps/Splunk_TA_microsoft_sysmon-1.0.2-B1.spl")
+
+
         #SYSMON_APP_FOR_SPLUNK = "https://splunkbase.splunk.com/app/3544/release/2.0.0/download"
         #SPLUNK_ES_CONTENT_UPDATE = "https://splunkbase.splunk.com/app/3449/release/3.29.0/download"
         SPLUNK_ADD_ON_FOR_MICROSOFT_WINDOWS = "https://splunkbase.splunk.com/app/742/release/8.2.0/download"
         LOCAL_GENERATED_ESCU_LATEST = "/tmp/apps/DA-ESS-ContentUpdate-latest.tar.gz"
         
-        SPLUNK_APPS = [SPLUNK_COMMON_INFORMATION_MODEL, SPLUNK_SECURITY_ESSENTIALS, SPLUNK_ADD_ON_FOR_SYSMON_OLD, LOCAL_GENERATED_ESCU_LATEST, SPLUNK_ADD_ON_FOR_MICROSOFT_WINDOWS]
-        #SPLUNK_APPS = [SPLUNK_COMMON_INFORMATION_MODEL, SPLUNK_SECURITY_ESSENTIALS, LOCAL_SPLUNK_ADD_ON_FOR_SYSMON_NEW, LOCAL_GENERATED_ESCU_LATEST, SPLUNK_ADD_ON_FOR_MICROSOFT_WINDOWS]
+        SPLUNK_APPS = [SPLUNK_COMMON_INFORMATION_MODEL, 
+                       SPLUNK_SECURITY_ESSENTIALS, 
+                       LOCAL_SPLUNK_ADD_ON_FOR_SYSMON_VOLUME_PATH, 
+                       LOCAL_GENERATED_ESCU_LATEST, 
+                       SPLUNK_ADD_ON_FOR_MICROSOFT_WINDOWS]
+        
 
-
-        #SPLUNK_APPS = [SPLUNK_COMMON_INFORMATION_MODEL, SPLUNK_SECURITY_ESSENTIALS  ,                                                 SPLUNK_ES_CONTENT_UPDATE, SPLUNK_ADD_ON_FOR_MICROSOFT_WINDOWS]
-        #docker run -p8089:8089 -p 8000:8000 -e "SPLUNK_START_ARGS=--accept-license" -e "SPLUNK_PASSWORD=123456qwertyQWERTY" -e "SPLUNK_APPS_URL=https://splunkbase.splunk.com/app/3435/release/3.3.4/download,https://splunkbase.splunk.com/app/5709/release/1.0.1/download,https://splunkbase.splunk.com/app/3449/release/3.29.0/download,https://splunkbase.splunk.com/app/1621/release/4.20.2/download" -e "SPLUNKBASE_USERNAME=ericmcginnistwo" -e "SPLUNKBASE_PASSWORD=splunkSecondAccount5@" -name splunktemplate splunk/splunk:latest 
         environment = {"SPLUNK_START_ARGS": "--accept-license",
                         "SPLUNK_PASSWORD"  : splunk_password, 
                         "SPLUNK_APPS_URL"   : ','.join(SPLUNK_APPS),
@@ -634,37 +477,47 @@ def main(args):
         mounts = [docker.types.Mount(target = '/tmp/apps/', source = 'security_content/slim_packaging/apps', type='bind', read_only=True)]
 
         print("Creating CONTAINER: [%s]"%(container_name))
-        base_container = client.containers.create(DOCKER_HUB_CONTAINER_PATH, ports=ports, environment=environment, name=container_name, mounts=mounts, detach=True)
+        base_container = client.containers.create(full_docker_hub_container_name, ports=ports, environment=environment, name=container_name, mounts=mounts, detach=True)
         print("Created CONTAINER : [%s]"%(container_name))
-        #print("Creating a new container called [%s]"%(container_name))
-        #environment = {"SPLUNK_START_ARGS": "--accept-license",
-        #               "SPLUNK_PASSWORD"  : splunk_password }
-        #ports= {"8000/tcp": web_port,
-        #        "8089/tcp": management_port
-        #        }
-
-        #test_container = client.containers.create(DOCKER_COMMIT_NAME, ports=ports, environment=environment, name=container_name, detach=True, volumes_from=[BASE_CONTAINER_NAME])
         
-        t = threading.Thread(target=splunk_container_manager, args=(test_file_queue, container_name, "127.0.0.1", splunk_password, management_port, uuid_test, results_queue, success_names_queue, failure_names_queue))
+
+        
+        
+        t = threading.Thread(target=splunk_container_manager, 
+                             args=(results_tracker, 
+                                   container_name, 
+                                   "127.0.0.1", 
+                                   splunk_password, 
+                                   management_port, 
+                                   uuid_test, 
+                                   ))
+
         splunk_container_manager_threads.append(t)
 
     #add the queue status thread - there can be some error in one of the test threads, so this
     #thread doesn't need to complete for the program to finish execution
-    status_thread = threading.Thread(target=queue_status_thread, args=(test_file_queue.qsize(), test_file_queue, results_queue, success_names_queue, failure_names_queue), daemon=True)
+    status_thread = threading.Thread(target=queue_status_thread, 
+                                     args=(results_tracker,), 
+                                           daemon=True)
+    #Start this thread immediately
     status_thread.start()
-    print("Start all the threads...")
+
+
+    print("Start the testing threads")
     for t in splunk_container_manager_threads:
         t.start()
         #we need to start containers slowly.  Would be great it we could do all the setup and
-        #app install once (with Dockerfile?)
+        #app install once, but it looks like the container is unlikely to support that.
+        #We don't really want to fundamentally change this container, either, and will 
+        #keep it as close to production as possible
         time.sleep(60)
     
-    #Try to join all the threads
+    #Wait for all of the testing threads to complete
     for t in splunk_container_manager_threads:
         t.join() #blocks on waiting to join
-        print("Joined a thread!")
+        print("Testing thread completed execution")
 
-    print("DONE!")
+    print("All testing threads have completed execution")
     #read all the results out from the output queue
     strtime = str(int(time.time()))
     #write success and failure
@@ -698,20 +551,6 @@ def main(args):
 
     #testing_service.test_detections(ssh_key_name, private_key, splunk_ip, splunk_password, test_files, uuid_test)
     
-def queue_status_thread(total_tests_count, testing_queue, results_queue, success_names_queue, failure_names_queue):
-    test_start_time = timer()
-    while True:
-        print("***Progress Update:\n"\
-              "\tElapsed Time           : %s\n"\
-              "\tTests to run           : %d\n"\
-              "\tTests currently running: %d\n"\
-              "\tTests completed        : %d\n"\
-              "\t\tSuccess : %d\n"\
-              "\t\tFailure : %d"%(timedelta(seconds=timer() - test_start_time, microseconds=0), testing_queue.qsize(), total_tests_count - testing_queue.qsize() - results_queue.qsize(), results_queue.qsize(), success_names_queue.qsize(), failure_names_queue.qsize()))
-        if results_queue.qsize() == total_tests_count:
-            return
-        else:
-            time.sleep(10)
 
 def copy_file_to_container(localFilePath, remoteFilePath, containerName, sleepTimeSeconds=5):
     successful_copy = False
@@ -790,4 +629,96 @@ def splunk_container_manager(testing_queue, container_name, splunk_ip, splunk_pa
     print("Finished shutting down the container [%s]"%(container_name))
 
 if __name__ == "__main__":
-    main(sys.argv[1:])
\ No newline at end of file
+    main(sys.argv[1:])
+
+
+class SynchronizedResultsTracker:
+    def __init__(self, tests:list[str]):
+        #Create the queue and enque all of the tests
+        self.testing_queue = queue.Queue()
+        for test in tests:
+            self.testing_queue.put(test)
+        
+        self.total_number_of_tests = self.testing_queue.qsize()
+        #Creates a lock that will be used to synchronize access to this object
+        self.lock = threading.Lock()
+        self.start_time = timer()
+        self.failures = []
+        self.successes = []
+        self.errors = []
+    def addSuccess(self, result:dict)->None:
+        self.lock.acquire()
+        try:
+            self.successes.append(result)
+        finally:
+            self.lock.release()
+        
+
+    def addFailure(self, result:dict)->None:
+        self.lock.acquire()
+        try:
+            self.failures.append(result)
+        finally:
+            self.lock.release()
+
+    def addError(self, result:dict)->None:
+        self.lock.acquire()
+        try:
+            self.errors.append(result)
+        finally:
+            self.lock.release()
+    def outputResultsFiles(self)->None:
+        self.lock.acquire()
+        try:
+            pass
+        finally:
+            self.lock.release()
+    
+    def summarize(self)->None:
+        
+        self.lock.acquire()
+        try:
+            current_time = timer()
+            numberOfCompletedTests = len(self.successes) + len(self.failures) + len(self.errors)
+            remaining_tests = self.testing_queue.qsize()         
+            testsCurrentlyRunning = self.total_number_of_tests - remaining_tests
+            total_execution_time_seconds = current_time - self.start_time
+            
+            
+            if numberOfCompletedTests == 0:
+                estimated_seconds_to_finish_all_tests = "UNKNOWN"
+                estimated_completion_time_seconds = "UNKNOWN"
+            else:
+                average_time_per_test = total_execution_time_seconds / numberOfCompletedTests
+                estimated_seconds_to_finish_all_tests = average_time_per_test * remaining_tests
+                estimated_completion_time_seconds = timedelta(seconds=estimated_seconds_to_finish_all_tests)
+                
+            
+
+            
+                print("***Progress Update:\n"\
+                "\tElapsed Time             : %s\n"\
+                "\tEstimated Remaining Time : %s\n"\
+                "\tTests to run             : %d\n"\
+                "\tTests currently running  : %d\n"\
+                "\tTests completed          : %d\n"\
+                "\t\tSuccess : %d\n"\
+                "\t\tFailure : %d\n"\
+                "\t\tError   : %d"%(timedelta(total_execution_time_seconds), 
+                                    estimated_completion_time_seconds, 
+                                    remaining_tests, 
+                                    testsCurrentlyRunning,
+                                    numberOfCompletedTests, 
+                                    len(self.successes), 
+                                    len(self.failures), 
+                                    len(self.errors)))
+    
+        except Exception as e:
+            print("Error in printing execution summary: [%s]"%(str(e)))
+        finally:
+            self.lock.release()
+
+def queue_status_thread(status_object:SynchronizedResultsTracker)->None:
+    while True:      
+        status_object.summarize()
+        time.sleep(10)

From c5b5dfac8824a01c3816a77cfc5ff00b2228498c Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Thu, 21 Oct 2021 17:06:35 -0700
Subject: [PATCH 025/166] Lots more changes - adding more error checking,
 typing to more functions, and better logging for testing detections.  Initial
 cut at fixing up issues involving bad return types.  Still had not been
 run/tested yet.

---
 .../detection_testing_execution.py            | 144 ++++++++++--------
 .../modules/splunk_sdk.py                     | 107 +++++++------
 .../modules/testing_service.py                |  44 +++---
 3 files changed, 158 insertions(+), 137 deletions(-)

diff --git a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
index 6c4a29e8e3..363062badf 100644
--- a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
+++ b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
@@ -20,6 +20,7 @@ from datetime import timedelta
 from datetime import datetime
 import string
 import shutil
+from typing import Union
 
 SPLUNK_CONTAINER_APPS_DIR = "/opt/splunk/etc/apps"
 index_file_local_path = "indexes.conf.tar"
@@ -567,69 +568,6 @@ def copy_file_to_container(localFilePath, remoteFilePath, containerName, sleepTi
             time.sleep(10)
             successful_copy=False
     print("Successfully copied [%s] to [%s] on [%s]"%(localFilePath, remoteFilePath, containerName))
-            
-
-def splunk_container_manager(testing_queue, container_name, splunk_ip, splunk_password, splunk_port, uuid_test, results_queue, success_names_queue, failure_names_queue):
-    print("Starting the container [%s] after a sleep"%(container_name))
-    #Is this going to be safe to use in different threads
-    client = docker.client.from_env()
-    
-    #start up the container from the base container
-    #Assume that the base container has already been fully built with
-    #escu etc
-    #sleep for a little bit so that we don't all start at once...
-    time.sleep(random.randrange(0,60))
-
-    container = client.containers.get(container_name)
-    print("Starting the container [%s]"%(container_name))
-
-    
-
-    container.start()
-    print("Start copying files to container")
-    copy_file_to_container(index_file_local_path, index_file_container_path, container_name)
-    copy_file_to_container(datamodel_file_local_path, datamodel_file_container_path, container_name)
-    print("Finished copying files to container!")
-
-
-    wait_for_splunk_ready(max_seconds=120)
-    from modules.splunk_sdk import enable_delete_for_admin
-    if not enable_delete_for_admin(splunk_ip, splunk_port, splunk_password):
-        print("COULD NOT ENABLE DELETE FOR [%s].... quitting"%(container_name))
-        sys.exit(0)
-    
-    print("Successfully enabled DELETE for [%s]"%(container_name))
-    
-    index=0
-    try:
-        while True:
-            #Try to get something from the queue
-            detection_to_test = testing_queue.get(block=False)
-            
-            #There is a detection to test
-            print("Container [%s]--->[%s]"%(container_name, detection_to_test))
-            try:
-                result = testing_service.test_detection_wrapper(container_name, splunk_ip, splunk_password, splunk_port, detection_to_test, index%1, uuid_test)
-                if result['detection_result']['error']:
-                    failure_names_queue.put(result['detection_result']['detection_name'])
-                else:
-                    success_names_queue.put(result['detection_result']['detection_name'])
-                results_queue.put(result)
-            except Exception as e:
-                print("Caught some exception in test detection: [%s]"%(str(e)))
-                #just log the error itself for now so that we can continue
-                result_test = str(e)
-
-            index=(index+1)%10
-    except queue.Empty:
-        print("Queue was empty, [%s] finished testing detections!"%(container_name))
-    
-    print("Shutting down the container [%s]"%(container_name))
-    container.stop()
-    print("Finished shutting down the container [%s]"%(container_name))
-
-if __name__ == "__main__":
-    main(sys.argv[1:])
 
 
 class SynchronizedResultsTracker:
@@ -646,7 +584,14 @@ class SynchronizedResultsTracker:
         self.failures = []
         self.successes = []
         self.errors = []
+    def getTest(self)-> Union[str,None]:
+        try:
+            return self.testing_queue.get(block=False)
+        except Exception as e:
+            print("Testing queue empty!")
+            return None
     def addSuccess(self, result:dict)->None:
+        print("Test PASSED for detection: [%s --> %s"%(result['detection_result']['detection_name'], result['detection_result']['detection_file']))
         self.lock.acquire()
         try:
             self.successes.append(result)
@@ -655,6 +600,7 @@ class SynchronizedResultsTracker:
         
 
     def addFailure(self, result:dict)->None:
+        print("Test FAILED for detection: [%s --> %s"%(result['detection_result']['detection_name'], result['detection_result']['detection_file']))
         self.lock.acquire()
         try:
             self.failures.append(result)
@@ -717,6 +663,78 @@ class SynchronizedResultsTracker:
             print("Error in printing execution summary: [%s]"%(str(e)))
         finally:
             self.lock.release()
+    def addResult(self, result:dict)->None:
+        try:
+            if result['detection_result']['success'] is False:
+                #This is actually a failure of the detection, not an error. Naming is confusiong
+                self.addFailure(result)
+            elif result['detection_result']['success'] is True:
+                self.addSuccess(result)
+        except Exception as e:
+            #Neither a success or a failure, so add the object to the failures queue
+            self.addError(result)
+        
+
+
+def splunk_container_manager(testing_object:SynchronizedResultsTracker, container_name, splunk_ip, splunk_password, splunk_port, uuid_test):
+    print("Starting the container [%s] after a sleep"%(container_name))
+    #Is this going to be safe to use in different threads
+    client = docker.client.from_env()
+    
+    #start up the container from the base container
+    #Assume that the base container has already been fully built with
+    #escu etc
+    #sleep for a little bit so that we don't all start at once...
+    time.sleep(random.randrange(0,60))
+
+    container = client.containers.get(container_name)
+    print("Starting the container [%s]"%(container_name))
+
+    
+
+    container.start()
+    print("Start copying files to container")
+    copy_file_to_container(index_file_local_path, index_file_container_path, container_name)
+    copy_file_to_container(datamodel_file_local_path, datamodel_file_container_path, container_name)
+    print("Finished copying files to container!")
+
+
+    wait_for_splunk_ready(max_seconds=120)
+    from modules.splunk_sdk import enable_delete_for_admin
+    if not enable_delete_for_admin(splunk_ip, splunk_port, splunk_password):
+        print("COULD NOT ENABLE DELETE FOR [%s].... quitting"%(container_name))
+        sys.exit(0)
+    
+    print("Successfully enabled DELETE for [%s]"%(container_name))
+    
+    
+    
+    while True:
+        #Try to get something from the queue
+        detection_to_test = testing_object.getTest()
+        if detection_to_test is None:
+            print("Container [%s] has finished running detections, time to stop the container."%(container_name))
+            container.stop()
+            print("Container [%s] successfully stopped"%(container_name))
+            return None
+
+
+        
+        #There is a detection to test
+        print("Container [%s]--->[%s]"%(container_name, detection_to_test))
+        try:
+            result = testing_service.test_detection_wrapper(container_name, splunk_ip, splunk_password, splunk_port, detection_to_test, 0, uuid_test)
+            testing_object.addResult(result)
+        except Exception as e:
+            print("Warning - uncaught error in detection test for [%s] - this should not happen: [%s]"%(detection_to_test, str(e)))
+
+
+            
+if __name__ == "__main__":
+    main(sys.argv[1:])
+
+
+
 
 def queue_status_thread(status_object:SynchronizedResultsTracker)->None:
     while True:      
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/splunk_sdk.py b/automated_detection_testing/ci/detection_testing_batch/modules/splunk_sdk.py
index 60b9294406..9f8f356ee4 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/splunk_sdk.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/splunk_sdk.py
@@ -4,7 +4,7 @@ import splunklib.results as results
 import splunklib.client as client
 import splunklib.results as results
 import requests
-
+from typing import Union
 
 def enable_delete_for_admin(splunk_host, splunk_port, splunk_password):
     try:
@@ -79,22 +79,7 @@ def test_baseline_search(splunk_host, splunk_port, splunk_password, search, pass
         return test_results
 
 
-def test_detection_search(splunk_host, splunk_port, splunk_password, search, pass_condition, detection_name, detection_file, earliest_time, latest_time):
-    try:
-        service = client.connect(
-            host=splunk_host,
-            port=splunk_port,
-            username='admin',
-            password=splunk_password
-        )
-    except Exception as e:
-        print("Unable to connect to Splunk instance: " + str(e))
-        raise(Exception("NO CONNECTION EXCEPTION"))
-        return 1, {}
-
-    # search and replace \\ with \\\
-    # search = search.replace('\\','\\\\')
-
+def test_detection_search(splunk_host:str, splunk_port:int, splunk_password:str, search:str, pass_condition:str, detection_name:str, detection_file:str, earliest_time:str, latest_time:str)->dict:
     if search.startswith('|'):
         search = search
     else:
@@ -105,35 +90,10 @@ def test_detection_search(splunk_host, splunk_port, splunk_password, search, pas
               "dispatch.latest_time": "now"}
 
     splunk_search = search + ' ' + pass_condition
-    print("SEARCH:")
-    print(splunk_search)
-
-    try:
-        job = service.jobs.create(splunk_search, **kwargs)
-    except Exception as e:
-        print("Unable to execute detection: " + str(e))
-        raise(Exception("***********NO EXECUTION EXCEPTION***********"))
-        return 1, {}
+    
+    
 
     test_results = dict()
-    test_results['diskUsage'] = job['diskUsage']
-    test_results['runDuration'] = job['runDuration']
-    test_results['detection_name'] = detection_name
-    test_results['detection_file'] = detection_file
-    test_results['scanCount'] = job['scanCount']
-
-    if int(job['resultCount']) != 1:
-        print("Test failed for detection: " + detection_name)
-        test_results['error'] = True
-        return test_results
-    else:
-        print("Test successful for detection: " + detection_name)
-        test_results['error'] = False
-        return test_results
-
-
-def delete_attack_data(splunk_host, splunk_password, splunk_port):
-    print("Deleting test data!")
     try:
         service = client.connect(
             host=splunk_host,
@@ -143,10 +103,61 @@ def delete_attack_data(splunk_host, splunk_password, splunk_port):
         )
     except Exception as e:
         print("Unable to connect to Splunk instance: " + str(e))
-        return 1, {}
+        test_results['error'] = True
+        return test_results
+
+    # search and replace \\ with \\\
+    # search = search.replace('\\','\\\\')
+
+   
+
+    print("SEARCH: %s"%(splunk_search))
+
+    try:
+        job = service.jobs.create(splunk_search, **kwargs)
+    except Exception as e:
+        print("Unable to execute detection: " + str(e))
+        test_results['error'] = True
+        return test_results
+
+    
+    test_results['diskUsage'] = job['diskUsage']
+    test_results['runDuration'] = job['runDuration']
+    test_results['detection_name'] = detection_name
+    test_results['detection_file'] = detection_file
+    test_results['scanCount'] = job['scanCount']
+    test_results['search_string'] = splunk_search
+    
+    test_results['error'] = False #The search may have FAILED, but there was no error in the search
+
+    if int(job['resultCount']) != 1:
+        print("Test failed for detection: " + detection_name)
+        test_results['success'] = False
+        return test_results
+    else:
+        print("Test successful for detection: " + detection_name)
+        test_results['success'] = True
+        return test_results
+
+
+def delete_attack_data(splunk_host:str, splunk_password:str, splunk_port:int, wait_on_delete:bool, search_string:str)->bool:
+    
+    try:
+        service = client.connect(
+            host=splunk_host,
+            port=splunk_port,
+            username='admin',
+            password=splunk_password
+        )
+    except Exception as e:
+        print("Unable to connect to Splunk instance: " + str(e))
+        return False
 
     #splunk_search = 'search index=test* | delete'
-    #_ = input("****************Press ENTER to DELETE****************")
+    if wait_on_delete:
+        print    ("************Allowing time to debug search************")
+        print(search_string)
+        _ = input("****************Press ENTER to DELETE****************")
     splunk_search = 'search index=main | delete'
     kwargs = {"exec_mode": "blocking",
               "dispatch.earliest_time": "-1d",
@@ -156,4 +167,6 @@ def delete_attack_data(splunk_host, splunk_password, splunk_port):
         job = service.jobs.create(splunk_search, **kwargs)
     except Exception as e:
         print("Unable to execute search: " + str(e))
-        return 1, {}
\ No newline at end of file
+        return False
+    
+    return True
\ No newline at end of file
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/testing_service.py b/automated_detection_testing/ci/detection_testing_batch/modules/testing_service.py
index b02c48d185..9ef8228254 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/testing_service.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/testing_service.py
@@ -9,39 +9,34 @@ import requests
 from modules.DataManipulation import DataManipulation
 from modules import splunk_sdk
 
-
-def prepare_detection_testing(splunk_ip, splunk_password):
-    try:
-        pass
-        #import security_content.bin.generate as module
-        #module = __import__('security_content.bin.generate')
-        #results = module.main(REPO_PATH = 'security_content' , OUTPUT_PATH = 'security_content/dist/escu', PRODUCT = 'ESCU', VERBOSE = 'False' )
-    except Exception as e:
-        print('Error: ' + str(e))
-
-    update_ESCU_app(splunk_ip, splunk_password)
+from typing import Union
 
 
-def test_detection_wrapper(container_name, splunk_ip, splunk_password, splunk_port, test_file, test_index, uuid_test):
+
+def test_detection_wrapper(container_name:str, splunk_ip:str, splunk_password:str, splunk_port:int, test_file:str, test_index:int, uuid_test:str, wait_on_failure:bool=False)->dict:
 
     uuid_var = str(uuid.uuid4())
     result_test = test_detection(splunk_ip, splunk_port, container_name, splunk_password, test_file, test_index, uuid_test, uuid_var)
-    
+    if result_test is None:
+        #We failed so early in the process that we could not produce any meaningful result
+        raise(Exception("Test execution Error"))    
 
     #enter = input("Run some tests from [%s] on [%s] - we don't delete until you hit enter :)"%(container_name, test_file))
     # delete test data
-    splunk_sdk.delete_attack_data(splunk_ip, splunk_password, splunk_port)
-
+    search_string = result_test['detection_result']['search_string']
     
-    if result_test['detection_result']['error']:
-        print('Test failed for detection: ' + result_test['detection_result']['detection_name'] + ' ' + result_test['detection_result']['detection_file'])
+    #search failed if there was an error or the detection failed to produce the expected result
+    if wait_on_failure and (result_test['detection_result']['error'] or not result_test['detection_result']['success']):
+        wait_on_delete = True
     else:
-        print('Test passed for detection: ' + result_test['detection_result']['detection_name'] + ' ' + result_test['detection_result']['detection_file'])
+        wait_on_delete = False
+    splunk_sdk.delete_attack_data(splunk_ip, splunk_password, splunk_port, wait_on_delete, search_string)
+   
 
     return result_test    
 
 
-def test_detection(splunk_ip, splunk_port, container_name, splunk_password, test_file, test_index, uuid_test, uuid_var):
+def test_detection(splunk_ip, splunk_port, container_name, splunk_password, test_file, test_index, uuid_test, uuid_var)->Union[dict,None]:
     try:
         test_file_obj = load_file(os.path.join("security_content/", test_file))
     except Exception as e:
@@ -102,20 +97,15 @@ def test_detection(splunk_ip, splunk_port, container_name, splunk_password, test
     print("Making test_detection_search request to: [%s:%d]"%(splunk_ip, splunk_port))
     
     result_detection = splunk_sdk.test_detection_search(splunk_ip, splunk_port, splunk_password, detection['search'], test['pass_condition'], detection['name'], test['file'], test['earliest_time'], test['latest_time'])
-    
+    if result_detection['error']:
+        print("There was an error running the search: %s"%(result_detection['search_string']))
+
 
 
     result_detection['detection_name'] = test['name']
     result_detection['detection_file'] = test['file']
     result_test['detection_result'] = result_detection
 
-    if result_detection['error']:
-        print("failed")
-        #aws_service.update_detection_results_in_dynamo_db('eu-central-1', uuid_var, 'failed')
-    else:
-        print("passed")
-        #aws_service.update_detection_results_in_dynamo_db('eu-central-1', uuid_var, 'passed')
-
     return result_test
 
 

From 53dedd468f2396385b5e1ab52d767a9bf57f44c4 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Fri, 22 Oct 2021 12:57:38 -0700
Subject: [PATCH 026/166] Big changes to make the tool much more usable. 
 Actually useful for testing, but no nice output files just yet.

---
 .../detection_testing_execution.py            | 119 ++++++++++--------
 .../modules/DataManipulation.py               |   2 +-
 .../modules/splunk_sdk.py                     |  13 +-
 .../modules/testing_service.py                |  11 +-
 4 files changed, 80 insertions(+), 65 deletions(-)

diff --git a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
index 363062badf..8a7594c0b7 100644
--- a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
+++ b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
@@ -247,6 +247,10 @@ def main(args):
     interactive_failure = args.interactive_failure
     show_password = args.show_password
     splunk_password = args.container_password
+    
+    if splunk_password is None and reuse_containers is True:
+        print("Error - if you are going to reuse a container you MUST provide the password to it!")
+        sys.exit(1)
     if splunk_password is None:
         #Generate a sufficiently complex password 
         splunk_password = get_random_password()
@@ -475,7 +479,8 @@ def main(args):
         ports= {"8000/tcp": web_port,
                 "8089/tcp": management_port
                }
-        mounts = [docker.types.Mount(target = '/tmp/apps/', source = 'security_content/slim_packaging/apps', type='bind', read_only=True)]
+        source_path = os.path.join(os.getcwd(), "security_content", "slim_packaging","apps")
+        mounts = [docker.types.Mount(target = '/tmp/apps/', source = source_path, type='bind', read_only=True)]
 
         print("Creating CONTAINER: [%s]"%(container_name))
         base_container = client.containers.create(full_docker_hub_container_name, ports=ports, environment=environment, name=container_name, mounts=mounts, detach=True)
@@ -490,7 +495,8 @@ def main(args):
                                    "127.0.0.1", 
                                    splunk_password, 
                                    management_port, 
-                                   uuid_test, 
+                                   uuid_test,
+                                   interactive_failure 
                                    ))
 
         splunk_container_manager_threads.append(t)
@@ -521,26 +527,12 @@ def main(args):
     print("All testing threads have completed execution")
     #read all the results out from the output queue
     strtime = str(int(time.time()))
-    #write success and failure
-    success_output = open("success_%s"%(strtime), "w") 
-    failure_output = open("failure_%s"%(strtime), "w")
-    try:
-        while True:
 
-            o = results_queue.get(block=False)
-            o_result = o['detection_result']
-            if o_result['error'] is False:
-                success_output.write(o_result['detection_file']+'\n')
-            else:
-                failure_output.write(o_result['detection_file']+'\n')
 
-            print(o_result)
-    except queue.Empty:
-        print("That's all the output!")
+    #Generate all of the output information
+    results_tracker.outputResultsFiles()
+    
     
-
-    success_output.close()
-    failure_output.close()
 
     #now we are done!
     stop_time = timer()
@@ -564,7 +556,7 @@ def copy_file_to_container(localFilePath, remoteFilePath, containerName, sleepTi
                 apiclient.put_archive(container=containerName, path=remoteFilePath, data=fileData)
                 successful_copy=True
         except Exception as e:
-            print("Failed copy of [%s] file to CONTAINER:[%s]...we will try again"%(localFilePath, containerName))
+            #print("Failed copy of [%s] file to CONTAINER:[%s]...we will try again"%(localFilePath, containerName))
             time.sleep(10)
             successful_copy=False
     print("Successfully copied [%s] to [%s] on [%s]"%(localFilePath, remoteFilePath, containerName))
@@ -584,12 +576,14 @@ class SynchronizedResultsTracker:
         self.failures = []
         self.successes = []
         self.errors = []
+        self.container_ready_time = None
     def getTest(self)-> Union[str,None]:
         try:
             return self.testing_queue.get(block=False)
         except Exception as e:
             print("Testing queue empty!")
             return None
+        
     def addSuccess(self, result:dict)->None:
         print("Test PASSED for detection: [%s --> %s"%(result['detection_result']['detection_name'], result['detection_result']['detection_file']))
         self.lock.acquire()
@@ -607,10 +601,10 @@ class SynchronizedResultsTracker:
         finally:
             self.lock.release()
 
-    def addError(self, result:dict)->None:
+    def addError(self, detection:str, errorText:str)->None:
         self.lock.acquire()
         try:
-            self.errors.append(result)
+            self.errors.append((detection, errorText))
         finally:
             self.lock.release()
     def outputResultsFiles(self)->None:
@@ -623,34 +617,49 @@ class SynchronizedResultsTracker:
     def summarize(self)->None:
         
         self.lock.acquire()
+        
         try:
             current_time = timer()
-            numberOfCompletedTests = len(self.successes) + len(self.failures) + len(self.errors)
-            remaining_tests = self.testing_queue.qsize()         
-            testsCurrentlyRunning = self.total_number_of_tests - remaining_tests
-            total_execution_time_seconds = current_time - self.start_time
-            
-            
-            if numberOfCompletedTests == 0:
-                estimated_seconds_to_finish_all_tests = "UNKNOWN"
-                estimated_completion_time_seconds = "UNKNOWN"
+            if self.testing_queue.qsize() == self.total_number_of_tests:
+                #Testing has not started yet. We are setting up containers
+                print("***********PROGRESS UPDATE***********\n"\
+                      "\tWaiting for container setup: %s"%(timedelta(seconds=current_time - self.start_time)))
             else:
-                average_time_per_test = total_execution_time_seconds / numberOfCompletedTests
-                estimated_seconds_to_finish_all_tests = average_time_per_test * remaining_tests
-                estimated_completion_time_seconds = timedelta(seconds=estimated_seconds_to_finish_all_tests)
+                if self.container_ready_time is None:
+                    #This is the first status update since container setup has completed.  Get the current time.
+                    #This makes our remaining time estimates better since that estimate should not involve
+                    #the container setup time  
+                    self.container_ready_time = current_time
+
+                numberOfCompletedTests = len(self.successes) + len(self.failures) + len(self.errors)
+                remaining_tests = self.testing_queue.qsize()         
+                testsCurrentlyRunning = self.total_number_of_tests - remaining_tests - numberOfCompletedTests
+                total_execution_time_seconds = current_time - self.start_time
+
+                test_execution_time_seconds = current_time - self.container_ready_time
+                
+                
+                if numberOfCompletedTests == 0 or test_execution_time_seconds == 0:
+                    estimated_seconds_to_finish_all_tests = "UNKNOWN"
+                    estimated_completion_time_seconds = "UNKNOWN"
+                else:
+                    average_time_per_test = test_execution_time_seconds / numberOfCompletedTests
+                    #divide testsCurrentlyRunning by 2.0 because, on average, each running test will be 50% completed
+                    estimated_seconds_to_finish_all_tests = average_time_per_test * (remaining_tests + testsCurrentlyRunning/2.0)
+                    estimated_completion_time_seconds = timedelta(seconds=estimated_seconds_to_finish_all_tests)
+                    
                 
-            
 
             
-                print("***Progress Update:\n"\
-                "\tElapsed Time             : %s\n"\
-                "\tEstimated Remaining Time : %s\n"\
-                "\tTests to run             : %d\n"\
-                "\tTests currently running  : %d\n"\
-                "\tTests completed          : %d\n"\
+                print("***********PROGRESS UPDATE***********\n"\
+                "\tElapsed Time               : %s\n"\
+                "\tEstimated Remaining Time   : %s\n"\
+                "\tTests to run               : %d\n"\
+                "\tTests currently running    : %d\n"\
+                "\tTests completed            : %d\n"\
                 "\t\tSuccess : %d\n"\
                 "\t\tFailure : %d\n"\
-                "\t\tError   : %d"%(timedelta(total_execution_time_seconds), 
+                "\t\tError   : %d"%(timedelta(seconds=total_execution_time_seconds), 
                                     estimated_completion_time_seconds, 
                                     remaining_tests, 
                                     testsCurrentlyRunning,
@@ -663,6 +672,7 @@ class SynchronizedResultsTracker:
             print("Error in printing execution summary: [%s]"%(str(e)))
         finally:
             self.lock.release()
+        
     def addResult(self, result:dict)->None:
         try:
             if result['detection_result']['success'] is False:
@@ -672,11 +682,11 @@ class SynchronizedResultsTracker:
                 self.addSuccess(result)
         except Exception as e:
             #Neither a success or a failure, so add the object to the failures queue
-            self.addError(result)
+            self.addError("Unspecified Error", str(result))
         
 
 
-def splunk_container_manager(testing_object:SynchronizedResultsTracker, container_name, splunk_ip, splunk_password, splunk_port, uuid_test):
+def splunk_container_manager(testing_object:SynchronizedResultsTracker, container_name, splunk_ip, splunk_password, splunk_port, uuid_test, interactive_failure:bool=False):
     print("Starting the container [%s] after a sleep"%(container_name))
     #Is this going to be safe to use in different threads
     client = docker.client.from_env()
@@ -685,7 +695,7 @@ def splunk_container_manager(testing_object:SynchronizedResultsTracker, containe
     #Assume that the base container has already been fully built with
     #escu etc
     #sleep for a little bit so that we don't all start at once...
-    time.sleep(random.randrange(0,60))
+    
 
     container = client.containers.get(container_name)
     print("Starting the container [%s]"%(container_name))
@@ -693,10 +703,10 @@ def splunk_container_manager(testing_object:SynchronizedResultsTracker, containe
     
 
     container.start()
-    print("Start copying files to container")
+    print("Start copying files to container: [%s]"%(container_name))
     copy_file_to_container(index_file_local_path, index_file_container_path, container_name)
     copy_file_to_container(datamodel_file_local_path, datamodel_file_container_path, container_name)
-    print("Finished copying files to container!")
+    print("Finished copying files to container: [%s]"%(container_name))
 
 
     wait_for_splunk_ready(max_seconds=120)
@@ -723,12 +733,18 @@ def splunk_container_manager(testing_object:SynchronizedResultsTracker, containe
         #There is a detection to test
         print("Container [%s]--->[%s]"%(container_name, detection_to_test))
         try:
-            result = testing_service.test_detection_wrapper(container_name, splunk_ip, splunk_password, splunk_port, detection_to_test, 0, uuid_test)
+            result = testing_service.test_detection_wrapper(container_name, splunk_ip, splunk_password, splunk_port, detection_to_test, 0, uuid_test, wait_on_failure=interactive_failure)
             testing_object.addResult(result)
         except Exception as e:
             print("Warning - uncaught error in detection test for [%s] - this should not happen: [%s]"%(detection_to_test, str(e)))
+            testing_object.addError(detection_to_test,str(e))
 
-
+def queue_status_thread(status_object:SynchronizedResultsTracker)->None:
+    #This will run forever by design
+    print("start status")
+    while True:      
+        status_object.summarize()
+        time.sleep(10)
             
 if __name__ == "__main__":
     main(sys.argv[1:])
@@ -736,7 +752,4 @@ if __name__ == "__main__":
 
 
 
-def queue_status_thread(status_object:SynchronizedResultsTracker)->None:
-    while True:      
-        status_object.summarize()
-        time.sleep(10)
+
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/DataManipulation.py b/automated_detection_testing/ci/detection_testing_batch/modules/DataManipulation.py
index faef36565b..b8905a6a6d 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/DataManipulation.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/DataManipulation.py
@@ -10,7 +10,7 @@ class DataManipulation:
 
     def manipulate_timestamp(self, file_path, sourcetype, source):
 
-        print('Updating timestamps in attack_data before replaying')
+        #print('Updating timestamps in attack_data before replaying')
 
         if sourcetype == 'aws:cloudtrail':
             self.manipulate_timestamp_cloudtrail(file_path)
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/splunk_sdk.py b/automated_detection_testing/ci/detection_testing_batch/modules/splunk_sdk.py
index 9f8f356ee4..9c65c41d4a 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/splunk_sdk.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/splunk_sdk.py
@@ -131,16 +131,16 @@ def test_detection_search(splunk_host:str, splunk_port:int, splunk_password:str,
     test_results['error'] = False #The search may have FAILED, but there was no error in the search
 
     if int(job['resultCount']) != 1:
-        print("Test failed for detection: " + detection_name)
+        #print("Test failed for detection: " + detection_name)
         test_results['success'] = False
         return test_results
     else:
-        print("Test successful for detection: " + detection_name)
+        #print("Test successful for detection: " + detection_name)
         test_results['success'] = True
         return test_results
 
 
-def delete_attack_data(splunk_host:str, splunk_password:str, splunk_port:int, wait_on_delete:bool, search_string:str)->bool:
+def delete_attack_data(splunk_host:str, splunk_password:str, splunk_port:int, wait_on_delete:bool, search_string:str, detection_filename:str)->bool:
     
     try:
         service = client.connect(
@@ -155,9 +155,10 @@ def delete_attack_data(splunk_host:str, splunk_password:str, splunk_port:int, wa
 
     #splunk_search = 'search index=test* | delete'
     if wait_on_delete:
-        print    ("************Allowing time to debug search************")
-        print(search_string)
-        _ = input("****************Press ENTER to DELETE****************")
+        print("\n\n\n****SEARCH FAILURE: Allowing time to debug search****")
+        print("FILENAME : [%s]"%(detection_filename))
+        print("SEARCH   :\n%s"%(search_string))
+        _ = input("****************Press ENTER to DELETE****************\n\n\n")
     splunk_search = 'search index=main | delete'
     kwargs = {"exec_mode": "blocking",
               "dispatch.earliest_time": "-1d",
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/testing_service.py b/automated_detection_testing/ci/detection_testing_batch/modules/testing_service.py
index 9ef8228254..a94fbedc3b 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/testing_service.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/testing_service.py
@@ -30,7 +30,8 @@ def test_detection_wrapper(container_name:str, splunk_ip:str, splunk_password:st
         wait_on_delete = True
     else:
         wait_on_delete = False
-    splunk_sdk.delete_attack_data(splunk_ip, splunk_password, splunk_port, wait_on_delete, search_string)
+    
+    splunk_sdk.delete_attack_data(splunk_ip, splunk_password, splunk_port, wait_on_delete, search_string, test_file)
    
 
     return result_test    
@@ -61,7 +62,7 @@ def test_detection(splunk_ip, splunk_port, container_name, splunk_password, test
         target_file = os.path.join(folder_name, attack_data['file_name'])
         with open(target_file, 'wb') as target:
             target.write(r.content)
-        print(target_file)
+        #print(target_file)
 
         # Update timestamps before replay
         if 'update_timestamp' in attack_data:
@@ -71,9 +72,9 @@ def test_detection(splunk_ip, splunk_port, container_name, splunk_password, test
         INDEX_TO_REPLAY_INTO = 'test' + str(test_index)
         INDEX_TO_REPLAY_INTO = 'main'
         replay_attack_dataset(container_name, splunk_password, folder_name, INDEX_TO_REPLAY_INTO, attack_data['sourcetype'], attack_data['source'], attack_data['file_name'])
-    print("START SLEEP AFTER REPLAY")
+    
     time.sleep(30)
-    print("DONE SLEEP AFTER REPLAY")
+    
     result_test = {}
     test = test_file_obj['tests'][0]
     
@@ -111,7 +112,7 @@ def test_detection(splunk_ip, splunk_port, container_name, splunk_password, test
 
 def load_file(file_path):
     try:
-        print("Opening file path [%s]"%(file_path))
+        #print("Opening file path [%s]"%(file_path))
         
         with open(file_path, 'r', encoding="utf-8") as stream:
             try:

From 194de0c28a1626eb0796d4ead2afa5ac52514563 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Fri, 22 Oct 2021 15:41:18 -0700
Subject: [PATCH 027/166] Added ability to test individual detection by
 specifying them on the command line.

---
 .../detection_testing_execution.py            | 144 +++++++++---------
 .../modules/github_service.py                 |  10 +-
 2 files changed, 84 insertions(+), 70 deletions(-)

diff --git a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
index 8a7594c0b7..1459a31629 100644
--- a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
+++ b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
@@ -70,40 +70,8 @@ def wait_for_splunk_ready(splunk_container_name=None, splunk_web_port=None, max_
     time.sleep(max_seconds)
 
 
-def remove_container(docker_client, container_name, force=True):
-    try:
-        container = docker_client.containers.get(container_name)
-    except Exception as e:
-        print("Could not find Docker Container [%s]. Container does not exist"%(container_name))
-        return True
-    try:
-        container.remove(v=True, force=force) #remove it even if it is running. remove volumes as well
-        print("Successfully removed Docker Container [%s]"%(container_name))
-    except Exception as e:
-        print("Could not remove Docker Container [%s]"%(container_name))
-        raise(Exception("CONTAINER REMOVE ERROR"))
 
 
-def stop_container(docker_client, container_name, force=True):
-    try:
-        container = docker_client.containers.get(container_name)
-    except:
-        print("Container with name [%s] does not exist"%(container_name))
-        return True
-    
-    try:
-        print("Checking to see if [%s] is running..."%(container_name), end='')
-        if container.status == 'exited':
-            print("NO")
-            return True
-        else:
-            print("YES (container.status is [%s])"%(container.status))
-            print("Stopping [%s]"%(container_name))
-            container.stop(force=force)
-            return True
-    except Exception as e:
-        print("Error trying to stop the container [%s]"%(container_name))
-        raise(Exception("CONTAINER STOP ERROR"))
 
         
 
@@ -177,25 +145,32 @@ def remove_existing_containers(client: DockerClient, reuse_containers: bool, con
         return True
     
     #Note that this variable can be changed by the block above, so don't
-    #convert this into an if/else
+    #convert this into an if/else. Note that this IF is for verbosity:
+
     if reuse_containers is False:
         for index in range(0,num_containers):
             container_name = container_template%(index)
-            print("Trying to remove container [%s]"%(container_name))
-            try:
-                container = client.containers.get(container_name)
-            except Exception as e:
-                print("Could not find Docker Container [%s]. Container does not exist, so no need to remove it"%(container_name))
-                continue
-            try:
-                #container was found, so now we try to remove it
-                #v also removes volumes linked to the container
-                container.remove(v=True, force=forceRemove) #remove it even if it is running. remove volumes as well
-                print("Successfully removed Docker Container [%s]"%(container_name))
-            except Exception as e:
-                print("Could not remove Docker Container [%s]"%(container_name))
-                raise(Exception("CONTAINER REMOVE ERROR"))
+            removeContainer(client, container_name, forceRemove)
         return False
+    else:
+        raise(Exception("Error removing existing containers"))
+
+def removeContainer(client: DockerClient, container_name:str, forceRemove:bool=True)->bool:
+    print("Trying to remove container [%s]"%(container_name))
+    try:
+        container = client.containers.get(container_name)
+    except Exception as e:
+        print("Could not find Docker Container [%s]. Container does not exist, so no need to remove it"%(container_name))
+        return True
+    try:
+        #container was found, so now we try to remove it
+        #v also removes volumes linked to the container
+        container.remove(v=True, force=forceRemove) #remove it even if it is running. remove volumes as well
+        print("Successfully removed Docker Container [%s]"%(container_name))
+        return True
+    except Exception as e:
+        print("Could not remove Docker Container [%s]"%(container_name))
+        raise(Exception("CONTAINER REMOVE ERROR"))
 
 
 
@@ -223,11 +198,16 @@ def main(args):
 
     parser.add_argument("-ri", "--reuse_image", required=False, default=False, action='store_true', help="Should existing images be re-used, or should they be redownloaded?")
     
-    parser.add_argument("-rc", "--reuse_containers", required=False, default=False,  help="Should existing containers be re-used, or should they be rebuilt?")
+    #Allowing us to reuse containers is more trouble than it's worth (we may have rebuilt an app or, more likely, ESCU) and it is a pain to re-upload that
+    #instead of having the container restart and download/install itself.
+    #parser.add_argument("-rc", "--reuse_containers", required=False, default=False,  help="Should existing containers be re-used, or should they be rebuilt?")
+
     parser.add_argument("-s", "--success_file", type=str, required=False, help="File that contains previously successful runs that we don't need to test")
-    parser.add_argument("-user", "--splunkbase_username", type=str, required=True, help="Splunkbase username for downloading Splunkbase apps")
-    parser.add_argument("-pw", "--splunkbase_password", type=str, required=True, help="Splunkbase password for downloading Splunkbase apps")
+    parser.add_argument("-user", "--splunkbase_username", type=str, required=False, help="Splunkbase username for downloading Splunkbase apps")
+    parser.add_argument("-pw", "--splunkbase_password", type=str, required=False, help="Splunkbase password for downloading Splunkbase apps")
     parser.add_argument("-m", "--mode", type=str, choices=DETECTION_MODES, required=False, help="Whether to test new detections, specific detections, or all detections", default="new")
+    parser.add_argument("-tf","--test_files", type=str, required=False, help="The names of files that you want to test, separated by commas.")
+
     parser.add_argument("-t", "--types", type=str, required=False, help="Detection types to test. Can be one of more of %s"%(str(DETECTION_TYPES)), default=DETECTION_TYPES)
     parser.add_argument("-ct", "--container_tag", type=str, required=False, help="The tag of the Splunk Container to use.  Tags are located at https://hub.docker.com/r/splunk/splunk/tags",default=DEFAULT_CONTAINER_TAG)
     parser.add_argument("-p", "--persist_security_content", required=False, default=False, action="store_true", help="Assumes security_content directory already exists.  Don't check it out and overwrite it again.  Saves "\
@@ -238,7 +218,7 @@ def main(args):
     uuid_test = args.uuid
     pr_number = args.pr_number
     num_containers = args.num_containers
-    reuse_containers = args.reuse_containers
+    #reuse_containers = args.reuse_containers
     reuse_image = args.reuse_image
     success_file = args.success_file
     splunkbase_username = args.splunkbase_username
@@ -248,9 +228,17 @@ def main(args):
     show_password = args.show_password
     splunk_password = args.container_password
     
+    
+    
+        
+    
+        
+    
+    '''
     if splunk_password is None and reuse_containers is True:
         print("Error - if you are going to reuse a container you MUST provide the password to it!")
         sys.exit(1)
+    '''
     if splunk_password is None:
         #Generate a sufficiently complex password 
         splunk_password = get_random_password()
@@ -279,21 +267,28 @@ def main(args):
 
     #Ensure that a valid mode was chosen 
     mode = args.mode
+    if mode == "selected" and args.test_files == None:
+        print("Error - mode [%s] but did not provide any files to test.\nQuitting..."%(mode))
+    
+
     folders = [a.strip() for a in args.types.split(',')]
     for t in folders:
         if t not in DETECTION_TYPES:
             print("Error - requested test of [%s] but the only valid types are %s.\tQuitting..."%(t, str(DETECTION_TYPES)))
             sys.exit(1)
     
+
+
     
 
     #Do some initial setup and validation of the containers and images  
 
     #If a user requests to use existing containers, they must also explicitly request to reuse existing images
+    '''
     if reuse_containers and not reuse_image:
         print("Error - requested --reuse_containers but did not explicitly request --reuse_image.\n\tQuitting")
         sys.exit(1)
-
+    '''
     if num_containers < 1:
         #Perhaps this should be a mock-run - do the initial steps but don't do testing on the containers?
         print("Error, requested 0 containers.  You must run with at least 1 container.")
@@ -309,7 +304,7 @@ def main(args):
 
     #Remove containers that previously existed (if we are directed to do so)
     try:
-        remove_existing_containers(client, reuse_containers, LOCAL_BASE_CONTAINER_NAME, num_containers)
+        remove_existing_containers(client, False, LOCAL_BASE_CONTAINER_NAME, num_containers)
     except Exception as e:
         print("Error tryting to remove existing containers.\n\tQuitting...")
         sys.exit(1)
@@ -351,24 +346,29 @@ def main(args):
         else:
             github_service = GithubService(branch)
     
-    
-    if args.mode == "all":
-        test_files = github_service.get_all_tests_and_detections(folders=folders, 
-                                                                 previously_successful_tests=success_tests)
-    elif args.mode == "new":
-        test_files = github_service.get_changed_test_files(folders=folders,
-                                                           previously_successful_tests=success_tests)
-    #elif args.mode == "selected":
-    #    test_files = github_service.get_selected_test_files(folders=args.types,
-    #                                                       previously_successful_tests=success_tests)
+    try:
+        if mode == "all":
+            test_files = github_service.get_all_tests_and_detections(folders=folders, 
+                                                                    previously_successful_tests=success_tests)
+        elif mode == "new":
+            test_files = github_service.get_changed_test_files(folders=folders,
+                                                            previously_successful_tests=success_tests)
+        elif mode == "selected":
+            if set(folders) != set(DETECTION_TYPES):
+                print("You specified mode [%s] but also types: [%s]. We will ignore type restrictions and test all specified files"%(mode,str(folders)))
+            files_to_test = [name.strip() for name in args.test_files.split(',')]
+            test_files = github_service.get_selected_test_files(files_to_test,
+                                                            previously_successful_tests=success_tests)
 
-    else:
-        print("Unsupported mode [%s] chosen.  Supported modes are %s.\n\tQuitting..."%(args.mode, str(DETECTION_MODES)))
+        else:
+            print("Unsupported mode [%s] chosen.  Supported modes are %s.\n\tQuitting..."%(args.mode, str(DETECTION_MODES)))
+            sys.exit(1)
+    except Exception as e:
+        print("Error - Failed to read in detection files: [%s].\nQuitting..."%(str(e)))    
         sys.exit(1)
     
     
     
-    
     #Go into the security content directory
     print("****GENERATE NEW CONTENT****")
     os.chdir("security_content")
@@ -723,9 +723,15 @@ def splunk_container_manager(testing_object:SynchronizedResultsTracker, containe
         #Try to get something from the queue
         detection_to_test = testing_object.getTest()
         if detection_to_test is None:
-            print("Container [%s] has finished running detections, time to stop the container."%(container_name))
-            container.stop()
-            print("Container [%s] successfully stopped"%(container_name))
+            try:
+                print("Container [%s] has finished running detections, time to stop the container."%(container_name))
+                container.stop()
+                print("Container [%s] successfully stopped"%(container_name))
+                #remove the container
+                removeContainer(client, container_name, forceRemove=True)
+            except Exception as e:
+                print("Error stopping or removing the container: [%s]"%(str(e)))
+                
             return None
 
 
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/github_service.py b/automated_detection_testing/ci/detection_testing_batch/modules/github_service.py
index b2115b09d0..5b80bd0c58 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/github_service.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/github_service.py
@@ -67,10 +67,18 @@ class GithubService:
         return pruned_tests
 
 
+    def get_selected_test_files(self,
+                                detection_file_list:list[str], 
+                                ttps_to_test:list[str]=["Anomaly","Hunting","TTP"], 
+                                previously_successful_tests:list[str]=[]) ->list[str]:    
+        
+        return self.prune_detections(detection_file_list, ttps_to_test, previously_successful_tests)
+
+
     def get_all_tests_and_detections(self, 
                                      folders:list[str]=['endpoint', 'cloud', 'network'], 
                                      ttps_to_test:list[str]=["Anomaly","Hunting","TTP"], 
-                                    previously_successful_tests:list[str]=[]) ->list[str]:    
+                                     previously_successful_tests:list[str]=[]) ->list[str]:    
         detections = []
         for folder in folders:
             detections.extend(self.get_all_files_in_folder(os.path.join("security_content/detections", folder), "*.yml"))

From 0846bb6d0b1171a9a9a681f8bcb83440e55c09b5 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Fri, 22 Oct 2021 16:42:53 -0700
Subject: [PATCH 028/166] Putting download attack data into a 'temp dir' and
 removing attack data as soon as it's used.  We need to do this when we're
 testing on CI/CD, otherwise we will run out of space.  Should keep/remove
 attack data be an option instead of forced?

---
 .../detection_testing_execution.py            | 49 ++++++++++++++++---
 .../modules/testing_service.py                | 19 ++++---
 2 files changed, 55 insertions(+), 13 deletions(-)

diff --git a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
index 1459a31629..4e9cacd7de 100644
--- a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
+++ b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
@@ -22,6 +22,8 @@ import string
 import shutil
 from typing import Union
 
+from tempfile import mkdtemp
+
 SPLUNK_CONTAINER_APPS_DIR = "/opt/splunk/etc/apps"
 index_file_local_path = "indexes.conf.tar"
 index_file_container_path = os.path.join(SPLUNK_CONTAINER_APPS_DIR, "search")
@@ -528,9 +530,14 @@ def main(args):
     #read all the results out from the output queue
     strtime = str(int(time.time()))
 
+    print("Wait for the status thread to finish executing...")
+    status_thread.join()
+    print("Status thread finished executing.")
 
-    #Generate all of the output information
-    results_tracker.outputResultsFiles()
+    
+    #Remove the attack data and 
+    #generate all of the output information
+    results_tracker.finish()
     
     
 
@@ -577,6 +584,11 @@ class SynchronizedResultsTracker:
         self.successes = []
         self.errors = []
         self.container_ready_time = None
+        
+        #Just make a random folder to store attack data that we donwload
+        self.attack_data_root_folder = mkdtemp(prefix="attack_data_", dir=os.getcwd())
+        print("Attack data for this run will be stored at: [%s]"%(self.attack_data_root_folder))
+
     def getTest(self)-> Union[str,None]:
         try:
             return self.testing_queue.get(block=False)
@@ -614,7 +626,21 @@ class SynchronizedResultsTracker:
         finally:
             self.lock.release()
     
-    def summarize(self)->None:
+
+    def finish(self):
+        self.cleanup()
+        self.outputResultsFiles()
+        
+
+    def cleanup(self):
+        self.lock.acquire()
+        try:
+            print("Removing all attack data that was downloaded during this test at: [%s]"%(self.attack_data_root_folder))
+            shutil.rmtree(self.attack_data_root_folder)
+            print("Successfully removed all attack data")
+        finally:
+            self.lock.release()
+    def summarize(self)->bool:
         
         self.lock.acquire()
         
@@ -673,6 +699,11 @@ class SynchronizedResultsTracker:
         finally:
             self.lock.release()
         
+        return (self.total_number_of_tests - 
+                self.testing_queue.qsize() - 
+                (len(self.successes) + len(self.failures) + len(self.errors)) > 0)
+        
+        
     def addResult(self, result:dict)->None:
         try:
             if result['detection_result']['success'] is False:
@@ -731,7 +762,7 @@ def splunk_container_manager(testing_object:SynchronizedResultsTracker, containe
                 removeContainer(client, container_name, forceRemove=True)
             except Exception as e:
                 print("Error stopping or removing the container: [%s]"%(str(e)))
-                
+
             return None
 
 
@@ -739,8 +770,12 @@ def splunk_container_manager(testing_object:SynchronizedResultsTracker, containe
         #There is a detection to test
         print("Container [%s]--->[%s]"%(container_name, detection_to_test))
         try:
-            result = testing_service.test_detection_wrapper(container_name, splunk_ip, splunk_password, splunk_port, detection_to_test, 0, uuid_test, wait_on_failure=interactive_failure)
+            result = testing_service.test_detection_wrapper(container_name, splunk_ip, splunk_password, splunk_port, detection_to_test, 0, uuid_test, testing_object.attack_data_root_folder, wait_on_failure=interactive_failure)
             testing_object.addResult(result)
+            
+            #Remove the data from the test that we just ran.  We MUST do this when running on CI because otherwise, we will download
+            #a massive amount of data over the course of a long path and will run out of space on the relatively small CI runner drive
+            shutil.rmtree(result['attack_data_directory'])
         except Exception as e:
             print("Warning - uncaught error in detection test for [%s] - this should not happen: [%s]"%(detection_to_test, str(e)))
             testing_object.addError(detection_to_test,str(e))
@@ -749,7 +784,9 @@ def queue_status_thread(status_object:SynchronizedResultsTracker)->None:
     #This will run forever by design
     print("start status")
     while True:      
-        status_object.summarize()
+        if status_object.summarize() == False:
+            #There are no more tests to run, so we can return from this thread
+            return None
         time.sleep(10)
             
 if __name__ == "__main__":
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/testing_service.py b/automated_detection_testing/ci/detection_testing_batch/modules/testing_service.py
index a94fbedc3b..38008e9915 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/testing_service.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/testing_service.py
@@ -10,13 +10,14 @@ from modules.DataManipulation import DataManipulation
 from modules import splunk_sdk
 
 from typing import Union
+from os.path import relpath
+from tempfile import mkdtemp
 
 
-
-def test_detection_wrapper(container_name:str, splunk_ip:str, splunk_password:str, splunk_port:int, test_file:str, test_index:int, uuid_test:str, wait_on_failure:bool=False)->dict:
-
+def test_detection_wrapper(container_name:str, splunk_ip:str, splunk_password:str, splunk_port:int, test_file:str, test_index:int, uuid_test:str, attack_data_root_folder, wait_on_failure:bool=False)->dict:
+    
     uuid_var = str(uuid.uuid4())
-    result_test = test_detection(splunk_ip, splunk_port, container_name, splunk_password, test_file, test_index, uuid_test, uuid_var)
+    result_test = test_detection(splunk_ip, splunk_port, container_name, splunk_password, test_file, test_index, uuid_test, uuid_var, attack_data_root_folder)
     if result_test is None:
         #We failed so early in the process that we could not produce any meaningful result
         raise(Exception("Test execution Error"))    
@@ -37,7 +38,7 @@ def test_detection_wrapper(container_name:str, splunk_ip:str, splunk_password:st
     return result_test    
 
 
-def test_detection(splunk_ip, splunk_port, container_name, splunk_password, test_file, test_index, uuid_test, uuid_var)->Union[dict,None]:
+def test_detection(splunk_ip:str, splunk_port:int, container_name:str, splunk_password:str, test_file:str, test_index:int, uuid_test, uuid_var, attack_data_root_folder)->Union[dict,None]:
     try:
         test_file_obj = load_file(os.path.join("security_content/", test_file))
     except Exception as e:
@@ -53,8 +54,11 @@ def test_detection(splunk_ip, splunk_port, container_name, splunk_password, test
     #aws_service.add_detection_results_in_dynamo_db('eu-central-1', uuid_var , uuid_test, test_file_obj['tests'][0]['name'], test_file_obj['tests'][0]['file'], str(int(time.time())))
 
     #epoch_time = str(int(time.time()))
-    folder_name = "attack_data_%s"%(uuid.uuid4())
-    os.mkdir(folder_name)
+    
+
+    abs_folder_path = mkdtemp(prefix="DATA_", dir=attack_data_root_folder)
+    #The ansible playbook wants the relative path, so we convert it as required
+    folder_name = relpath(abs_folder_path, os.getcwd())
 
     for attack_data in test_file_obj['tests'][0]['attack_data']:
         url = attack_data['data']
@@ -106,6 +110,7 @@ def test_detection(splunk_ip, splunk_port, container_name, splunk_password, test
     result_detection['detection_name'] = test['name']
     result_detection['detection_file'] = test['file']
     result_test['detection_result'] = result_detection
+    result_test['attack_data_directory'] = abs_folder_path
 
     return result_test
 

From 83d6869c6c5bdab93ec268c119da0ae5cafbdedd Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Mon, 25 Oct 2021 11:24:16 -0700
Subject: [PATCH 029/166] Better support for different types of structured
 output files, including a summary of all the detections before they are even
 run.

---
 .../detection_testing_execution.py            | 45 +++++++++++++------
 .../modules/github_service.py                 | 25 ++++++++++-
 2 files changed, 56 insertions(+), 14 deletions(-)

diff --git a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
index 4e9cacd7de..47d94b813a 100644
--- a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
+++ b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
@@ -23,7 +23,7 @@ import shutil
 from typing import Union
 
 from tempfile import mkdtemp
-
+import csv
 SPLUNK_CONTAINER_APPS_DIR = "/opt/splunk/etc/apps"
 index_file_local_path = "indexes.conf.tar"
 index_file_container_path = os.path.join(SPLUNK_CONTAINER_APPS_DIR, "search")
@@ -597,7 +597,7 @@ class SynchronizedResultsTracker:
             return None
         
     def addSuccess(self, result:dict)->None:
-        print("Test PASSED for detection: [%s --> %s"%(result['detection_result']['detection_name'], result['detection_result']['detection_file']))
+        print("Test PASSED for detection: [%s --> %s"%(result['detection_name'], result['detection_file']))
         self.lock.acquire()
         try:
             self.successes.append(result)
@@ -606,26 +606,45 @@ class SynchronizedResultsTracker:
         
 
     def addFailure(self, result:dict)->None:
-        print("Test FAILED for detection: [%s --> %s"%(result['detection_result']['detection_name'], result['detection_result']['detection_file']))
+        print("Test FAILED for detection: [%s --> %s"%(result['detection_name'], result['detection_file']))
         self.lock.acquire()
         try:
             self.failures.append(result)
         finally:
             self.lock.release()
 
-    def addError(self, detection:str, errorText:str)->None:
+    def addError(self, detection:dict)->None:
         self.lock.acquire()
         try:
-            self.errors.append((detection, errorText))
+            self.errors.append(detection)
         finally:
             self.lock.release()
-    def outputResultsFiles(self)->None:
+    def outputResultsFile(self, column_names:list[str], output_filename:str, data:list[dict])->bool:
+        success = True
+        print("Generating %s"%(output_filename))
+        print(data)
         self.lock.acquire()
-        try:
-            pass
+        try:                        
+            with open(output_filename, 'w') as csvfile:
+                csv_writer = csv.DictWriter(csvfile, fieldnames=column_names)
+                csv_writer.writeheader()
+                for row in self.successes:
+                    csv_writer.writerow(row)
+
+        except Exception as e:
+            print("Failure writing to CSV file for [%s]"%str(e))
+            success = False
+
         finally:
             self.lock.release()
-    
+
+        return success
+
+    def outputResultsFiles(self)->bool:
+        res = self.outputResultsFile(['detection_name', 'detection_file', 'detection_result','runDuration','diskUsage', 'search_string', 'error', 'success', 'scanCount'],"success.csv", self.successes)
+        res |= self.outputResultsFile(['detection_name', 'detection_file', 'detection_result','runDuration','diskUsage', 'search_string', 'error', 'success', 'scanCount'], "failures.csv", self.failures)
+        res |= self.outputResultsFile(['detection_file', 'detection_error'], "errors.csv", self.errors)
+        return res
 
     def finish(self):
         self.cleanup()
@@ -708,12 +727,12 @@ class SynchronizedResultsTracker:
         try:
             if result['detection_result']['success'] is False:
                 #This is actually a failure of the detection, not an error. Naming is confusiong
-                self.addFailure(result)
+                self.addFailure(result['detection_result'])
             elif result['detection_result']['success'] is True:
-                self.addSuccess(result)
+                self.addSuccess(result['detection_result'])
         except Exception as e:
             #Neither a success or a failure, so add the object to the failures queue
-            self.addError("Unspecified Error", str(result))
+            self.addError({'detection_file':"Unknown File", "detection_error":str(result)})
         
 
 
@@ -778,7 +797,7 @@ def splunk_container_manager(testing_object:SynchronizedResultsTracker, containe
             shutil.rmtree(result['attack_data_directory'])
         except Exception as e:
             print("Warning - uncaught error in detection test for [%s] - this should not happen: [%s]"%(detection_to_test, str(e)))
-            testing_object.addError(detection_to_test,str(e))
+            testing_object.addError({"detection_file":detection_to_test,"detection_error":str(e)})
 
 def queue_status_thread(status_object:SynchronizedResultsTracker)->None:
     #This will run forever by design
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/github_service.py b/automated_detection_testing/ci/detection_testing_batch/modules/github_service.py
index 5b80bd0c58..61ac7b0122 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/github_service.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/github_service.py
@@ -7,6 +7,7 @@ import subprocess
 from git.objects import base
 import yaml
 import pathlib
+import csv
 # Logger
 logging.basicConfig(level=os.environ.get("LOGLEVEL", "INFO"))
 LOGGER = logging.getLogger(__name__)
@@ -40,8 +41,10 @@ class GithubService:
                          detections_to_prune:list[str], 
                          ttps_to_test:list[str], 
                          previously_successful_tests:list[str],
-                         exclude_ssa:bool=True)->list[str]:
+                         exclude_ssa:bool=True,
+                         summary_file:str=None)->list[str]:
         pruned_tests = []
+        csvlines = []
         for detection in detections_to_prune:
             if os.path.basename(detection).startswith("ssa") and exclude_ssa:
                 continue
@@ -61,9 +64,29 @@ class GithubService:
                     else:
                         #remove leading security_content/ from path
                         pruned_tests.append(test_filepath_without_security_content)
+                        if summary_file is not None:
+                            try:
+                                mitre_id = str(description['tags']['mitre_attack_id'])
+                            except:
+                                mitre_id = 'NONE'
+                            try:
+                                csvlines.append({'name':description['name'], 'description':description['description'], 'search':description['search'], 'mitre_attack_id':mitre_id, 'security_domain':description['tags']['security_domain'],'Relevant':'', 'Comments':''})
+                            except Exception as e:
+                                print("Error outputting summary for [%s]: [%s]"%(detection, str(e))) 
                 else:
                     #Don't do anything with these files
                     pass
+        
+        if summary_file is not None:
+            with open('detectionWork.csv', 'w') as csvfile:
+                fieldnames = ['name', 'description', 'search', 'mitre_attack_id', 'security_domain', 'Relevant', 'Comments']
+                writer = csv.DictWriter(csvfile, fieldnames=fieldnames, quoting=csv.QUOTE_ALL)
+                writer.writeheader()
+                for r in csvlines:
+                    writer.writerow(r)
+            
+        
+        
         return pruned_tests
 
 

From fd9e91d569f882e7c8ff69fd50b179d4aa5d6e7f Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Mon, 25 Oct 2021 14:28:06 -0700
Subject: [PATCH 030/166] Write out a summary csv and write environment
 iformation, to include datetime, splunk version, and installed splunk apps,
 to the csv files.

---
 .../detection_testing_execution.py            | 56 +++++++++++--------
 .../modules/github_service.py                 | 15 +++--
 2 files changed, 42 insertions(+), 29 deletions(-)

diff --git a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
index 47d94b813a..fb378345a0 100644
--- a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
+++ b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
@@ -20,7 +20,7 @@ from datetime import timedelta
 from datetime import datetime
 import string
 import shutil
-from typing import Union
+from typing import OrderedDict, Union
 
 from tempfile import mkdtemp
 import csv
@@ -215,6 +215,8 @@ def main(args):
     parser.add_argument("-p", "--persist_security_content", required=False, default=False, action="store_true", help="Assumes security_content directory already exists.  Don't check it out and overwrite it again.  Saves "\
                                                                                                                      "time and allows you to test a detection that you've updated.  Runs generate again in case you have "\
                                                                                                                      "updated macros or anything else.  Especially useful for quick, local, iterative testing.")
+    start_datetime = datetime.now()
+    
     args = parser.parse_args()
     branch = args.branch
     uuid_test = args.uuid
@@ -424,15 +426,10 @@ def main(args):
     os.chdir("../..")
     print("New ESCU Package generated successfully")    
     
-    #Enqueue all of the test files for processing
-    test_file_queue = queue.Queue()
-    for filename in test_files:
-        test_file_queue.put(filename)
     
     
 
-    
-    
+
     
     
     #Create threads to manage all of the containers that we will start up
@@ -441,7 +438,7 @@ def main(args):
 
     
     
-    results_tracker = SynchronizedResultsTracker(test_files)
+    results_tracker = SynchronizedResultsTracker(test_files[:3])
     
     for container_index in range(num_containers):
         container_name = LOCAL_BASE_CONTAINER_NAME%container_index
@@ -482,6 +479,7 @@ def main(args):
                 "8089/tcp": management_port
                }
         source_path = os.path.join(os.getcwd(), "security_content", "slim_packaging","apps")
+        
         mounts = [docker.types.Mount(target = '/tmp/apps/', source = source_path, type='bind', read_only=True)]
 
         print("Creating CONTAINER: [%s]"%(container_name))
@@ -537,7 +535,12 @@ def main(args):
     
     #Remove the attack data and 
     #generate all of the output information
-    results_tracker.finish()
+    baseline = OrderedDict()
+    baseline['SPLUNK_VERSION'] = full_docker_hub_container_name
+    baseline['SPLUNK_APPS'] = ','.join(SPLUNK_APPS)
+    baseline['TEST_START_TIME'] = str(start_datetime)
+
+    results_tracker.finish(baseline)
     
     
 
@@ -619,20 +622,25 @@ class SynchronizedResultsTracker:
             self.errors.append(detection)
         finally:
             self.lock.release()
-    def outputResultsFile(self, column_names:list[str], output_filename:str, data:list[dict])->bool:
+    def outputResultsFile(self, field_names:list[str], output_filename:str, data:list[dict], baseline:OrderedDict)->bool:
         success = True
-        print("Generating %s"%(output_filename))
-        print(data)
+        print("Generating %s..."%(output_filename), end='')
         self.lock.acquire()
+        
         try:                        
             with open(output_filename, 'w') as csvfile:
-                csv_writer = csv.DictWriter(csvfile, fieldnames=column_names)
+                header_writer = csv.writer(csvfile, quoting=csv.QUOTE_ALL)
+                for key in baseline:
+                    header_writer.writerow([key, baseline[key]])
+                header_writer.writerow(['',''])
+                csv_writer = csv.DictWriter(csvfile, fieldnames=field_names)
                 csv_writer.writeheader()
-                for row in self.successes:
+                for row in data:
                     csv_writer.writerow(row)
+            print("Done with [%d] detections"%(len(data)))
 
         except Exception as e:
-            print("Failure writing to CSV file for [%s]"%str(e))
+            print("Failure writing to CSV file for [%s]:"%(output_filename, str(e)))
             success = False
 
         finally:
@@ -640,15 +648,16 @@ class SynchronizedResultsTracker:
 
         return success
 
-    def outputResultsFiles(self)->bool:
-        res = self.outputResultsFile(['detection_name', 'detection_file', 'detection_result','runDuration','diskUsage', 'search_string', 'error', 'success', 'scanCount'],"success.csv", self.successes)
-        res |= self.outputResultsFile(['detection_name', 'detection_file', 'detection_result','runDuration','diskUsage', 'search_string', 'error', 'success', 'scanCount'], "failures.csv", self.failures)
-        res |= self.outputResultsFile(['detection_file', 'detection_error'], "errors.csv", self.errors)
+    def outputResultsFiles(self, baseline:OrderedDict, fields:list[str]=['detection_name', 'detection_file','runDuration','diskUsage', 'search_string', 'error', 'success', 'scanCount', 'detection_error'])->bool:
+        res = self.outputResultsFile(fields,"success.csv", self.successes, baseline)
+        res |= self.outputResultsFile(fields, "failures.csv", self.failures, baseline)
+        res |= self.outputResultsFile(fields, "errors.csv", self.errors, baseline)
+        res |= self.outputResultsFile(fields, "combined.csv", self.successes + self.failures + self.errors, baseline)
         return res
 
-    def finish(self):
+    def finish(self, baseline:OrderedDict):
         self.cleanup()
-        self.outputResultsFiles()
+        self.outputResultsFiles(baseline)
         
 
     def cleanup(self):
@@ -718,8 +727,8 @@ class SynchronizedResultsTracker:
         finally:
             self.lock.release()
         
+        #Return true while there are tests remaining
         return (self.total_number_of_tests - 
-                self.testing_queue.qsize() - 
                 (len(self.successes) + len(self.failures) + len(self.errors)) > 0)
         
         
@@ -748,10 +757,11 @@ def splunk_container_manager(testing_object:SynchronizedResultsTracker, containe
     
 
     container = client.containers.get(container_name)
+    
     print("Starting the container [%s]"%(container_name))
 
     
-
+    
     container.start()
     print("Start copying files to container: [%s]"%(container_name))
     copy_file_to_container(index_file_local_path, index_file_container_path, container_name)
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/github_service.py b/automated_detection_testing/ci/detection_testing_batch/modules/github_service.py
index 61ac7b0122..4559640228 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/github_service.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/github_service.py
@@ -53,8 +53,8 @@ class GithubService:
                 
                 test_filepath = os.path.splitext(detection)[0].replace('detections', 'tests') + '.test.yml'
                 test_filepath_without_security_content = str(pathlib.Path(*pathlib.Path(test_filepath).parts[1:]))
-                if 'type' in description and description['type'] in ttps_to_test:
-                    
+                #If no TTPS are provided, then we will get everything
+                if 'type' in description and (description['type'] in ttps_to_test or len(ttps_to_test) == 0):
                     #print(description['type'])
                     if not os.path.exists(test_filepath):
                         print("Detection [%s] references [%s], but it does not exist"%(detection, test_filepath))
@@ -70,7 +70,10 @@ class GithubService:
                             except:
                                 mitre_id = 'NONE'
                             try:
-                                csvlines.append({'name':description['name'], 'description':description['description'], 'search':description['search'], 'mitre_attack_id':mitre_id, 'security_domain':description['tags']['security_domain'],'Relevant':'', 'Comments':''})
+
+                                csvlines.append({'name':description['name'], 'filename':detection, 'description':description['description'], 
+                                                 'search':description['search'], 'mitre_attack_id':mitre_id, 'security_domain':description['tags']['security_domain'],
+                                                 'Relevant':'', 'Comments':'', "Runnable on SSA?": str(os.path.basename(detection).startswith("ssa"))})
                             except Exception as e:
                                 print("Error outputting summary for [%s]: [%s]"%(detection, str(e))) 
                 else:
@@ -78,15 +81,15 @@ class GithubService:
                     pass
         
         if summary_file is not None:
-            with open('detectionWork.csv', 'w') as csvfile:
-                fieldnames = ['name', 'description', 'search', 'mitre_attack_id', 'security_domain', 'Relevant', 'Comments']
+            print("writing")
+            with open(summary_file, 'w') as csvfile:
+                fieldnames = ['name', 'filename', 'description', 'search', 'mitre_attack_id', 'security_domain', 'Runnable on SSA?', 'Relevant', 'Comments']
                 writer = csv.DictWriter(csvfile, fieldnames=fieldnames, quoting=csv.QUOTE_ALL)
                 writer.writeheader()
                 for r in csvlines:
                     writer.writerow(r)
             
         
-        
         return pruned_tests
 
 

From 549bcac8a7b291bfd777c3792fb7c2ecf4d80f0a Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Tue, 26 Oct 2021 14:33:01 -0700
Subject: [PATCH 031/166] Changes to support Ubuntu, which doesn't have curl
 installed by default, and moving to python3 for slim.

---
 .../detection_testing_execution.py            | 105 ++++++++++++------
 .../modules/splunk_sdk.py                     |   6 +-
 2 files changed, 73 insertions(+), 38 deletions(-)

diff --git a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
index fb378345a0..63526ddfa0 100644
--- a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
+++ b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
@@ -24,6 +24,7 @@ from typing import OrderedDict, Union
 
 from tempfile import mkdtemp
 import csv
+from urllib.request import urlretrieve 
 SPLUNK_CONTAINER_APPS_DIR = "/opt/splunk/etc/apps"
 index_file_local_path = "indexes.conf.tar"
 index_file_container_path = os.path.join(SPLUNK_CONTAINER_APPS_DIR, "search")
@@ -378,7 +379,11 @@ def main(args):
     os.chdir("security_content")
     print(os.getcwd())
     if persist_security_content is False:
-        commands = ["python3 -m venv .venv", "source .venv/bin/activate", "python3 -m pip install wheel", "python3 -m pip install -r requirements.txt", "python3 contentctl.py --path . --verbose generate --product ESCU --output dist/escu", "tar -czf DA-ESS-ContentUpdate.spl -C dist/escu ."]
+        commands = ["python3 -m venv .venv", 
+                    ". ./.venv/bin/activate", 
+                    "python3 -m pip install wheel", 
+                    "python3 -m pip install -r requirements.txt", 
+                    "python3 contentctl.py --path . --verbose generate --product ESCU --output dist/escu", "tar -czf DA-ESS-ContentUpdate.spl -C dist/escu ."]
     else:
         commands = ["source .venv/bin/activate", "python3 contentctl.py --path . --verbose generate --product ESCU --output dist/escu", "tar -czf DA-ESS-ContentUpdate.spl -C dist/escu ."]
     ret = subprocess.run("; ".join(commands), shell=True, capture_output=True)
@@ -392,7 +397,7 @@ def main(args):
     if persist_security_content is True:
         os.chdir("slim_packaging")
         commands = ["cd slim-latest", 
-                    "source venv/bin/activate",
+                    ". ./.venv/bin/activate",
                     "cp -R ../../dist/escu DA-ESS-ContentUpdate",
                     "slim package -o upload DA-ESS-ContentUpdate",
                     "cp upload/DA-ESS-ContentUpdate*.tar.gz ../apps/DA-ESS-ContentUpdate-latest.tar.gz"]
@@ -401,17 +406,28 @@ def main(args):
         os.mkdir("slim_packaging")
         os.chdir("slim_packaging")
         os.mkdir("apps")
-        commands = ["curl -Ls https://download.splunk.com/misc/packaging-toolkit/splunk-packaging-toolkit-0.9.0.tar.gz -o splunk-packaging-toolkit-latest.tar.gz", 
-                    "rm -rf slim-latest",
+        
+        try:
+            SPLUNK_PACKAGING_TOOLKIT_URL = "https://download.splunk.com/misc/packaging-toolkit/splunk-packaging-toolkit-0.9.0.tar.gz"
+            print("Downloading the Splunk Packaging Toolkit from %s..."%(SPLUNK_PACKAGING_TOOLKIT_URL), end='')
+            urlretrieve(SPLUNK_PACKAGING_TOOLKIT_URL, 'splunk-packaging-toolkit-latest.tar.gz')
+            print("success")
+
+
+        except Exception as e:
+            print("FAILED")
+            print("Error downloading the Splunk Packaging Toolkit: [%s]"%(str(e)))
+            sys.exit(1)
+        
+        
+        commands = ["rm -rf slim-latest",
                     "mkdir slim-latest", 
                     "tar -zxf splunk-packaging-toolkit-latest.tar.gz -C slim-latest --strip-components=1",
                     "cd slim-latest", 
-                    "virtualenv --python=/usr/bin/python2.7 --clear venv",
-                    "source venv/bin/activate",
-                    "python2 -m pip install --upgrade pip",
-                    "python2 -m pip install wheel",
-                    "python2 -m pip install semantic_version",
-                    "python2 -m pip install .",
+                    "python3 -m venv .venv", 
+                    ". ./.venv/bin/activate", 
+                    "python3 -m pip install wheel", 
+                    "python3 -m pip install -r requirements.txt", 
                     "cp -R ../../dist/escu DA-ESS-ContentUpdate",
                     "slim package -o upload DA-ESS-ContentUpdate",
                     "cp upload/DA-ESS-ContentUpdate*.tar.gz ../apps/DA-ESS-ContentUpdate-latest.tar.gz"]
@@ -438,35 +454,46 @@ def main(args):
 
     
     
-    results_tracker = SynchronizedResultsTracker(test_files[:3])
+
+
+    results_tracker = SynchronizedResultsTracker(test_files)
     
+    local_volume_path = os.path.join(os.getcwd(), "security_content", "slim_packaging","apps")
+
+    SPLUNK_COMMON_INFORMATION_MODEL = "https://splunkbase.splunk.com/app/1621/release/4.20.2/download"        
+    SPLUNK_SECURITY_ESSENTIALS = "https://splunkbase.splunk.com/app/3435/release/3.3.4/download"
+    #SPLUNK_ADD_ON_FOR_SYSMON_OLD = "https://splunkbase.splunk.com/app/1914/release/10.6.2/download"
+    #SPLUNK_ADD_ON_FOR_SYSMON_NEW = "https://splunkbase.splunk.com/app/5709/release/1.0.1/download"
+    #SYSMON_APP_FOR_SPLUNK = "https://splunkbase.splunk.com/app/3544/release/2.0.0/download"
+    #SPLUNK_ES_CONTENT_UPDATE = "https://splunkbase.splunk.com/app/3449/release/3.29.0/download"
+
+    #Just a hack until we get the new version of system deployed and available from splunkbase
+    LOCAL_SPLUNK_ADD_ON_FOR_SYSMON_PATH  = os.path.expanduser("~/Downloads/Splunk_TA_microsoft_sysmon-1.0.2-B1.spl")    
+    shutil.copyfile(LOCAL_SPLUNK_ADD_ON_FOR_SYSMON_PATH, os.path.join(local_volume_path, "Splunk_TA_microsoft_sysmon-1.0.2-B1.spl"))
+    SPLUNK_ADD_ON_FOR_SYSMON_VOLUME_PATH  = "/tmp/apps/Splunk_TA_microsoft_sysmon-1.0.2-B1.spl"
+    
+
+
+    
+    SPLUNK_ADD_ON_FOR_MICROSOFT_WINDOWS = "https://splunkbase.splunk.com/app/742/release/8.2.0/download"
+    CONTAINER_VOLUME_PATH = '/tmp/apps/'
+    CONTAINER_GENERATED_ESCU_LATEST = os.path.join(CONTAINER_VOLUME_PATH, "DA-ESS-ContentUpdate-latest.tar.gz")
+    
+    SPLUNK_APPS = [SPLUNK_COMMON_INFORMATION_MODEL, 
+                    SPLUNK_SECURITY_ESSENTIALS, 
+                    SPLUNK_ADD_ON_FOR_SYSMON_VOLUME_PATH, 
+                    CONTAINER_GENERATED_ESCU_LATEST, 
+                    SPLUNK_ADD_ON_FOR_MICROSOFT_WINDOWS]
+
+
+
     for container_index in range(num_containers):
         container_name = LOCAL_BASE_CONTAINER_NAME%container_index
         
         web_port = BASE_CONTAINER_WEB_PORT  + container_index
         management_port = BASE_CONTAINER_MANAGEMENT_PORT + container_index
 
-        SPLUNK_COMMON_INFORMATION_MODEL = "https://splunkbase.splunk.com/app/1621/release/4.20.2/download"        
-        SPLUNK_SECURITY_ESSENTIALS = "https://splunkbase.splunk.com/app/3435/release/3.3.4/download"
-        #SPLUNK_ADD_ON_FOR_SYSMON_OLD = "https://splunkbase.splunk.com/app/1914/release/10.6.2/download"
-        #SPLUNK_ADD_ON_FOR_SYSMON_NEW = "https://splunkbase.splunk.com/app/5709/release/1.0.1/download"
         
-        #Just a hack until we get the new version of system deployed and available from splunkbase
-        LOCAL_SPLUNK_ADD_ON_FOR_SYSMON_PATH  = os.path.expanduser("~/Downloads/Splunk_TA_microsoft_sysmon-1.0.2-B1.spl")    
-        LOCAL_SPLUNK_ADD_ON_FOR_SYSMON_VOLUME_PATH  = "/tmp/apps/Splunk_TA_microsoft_sysmon-1.0.2-B1.spl"
-        shutil.copyfile(LOCAL_SPLUNK_ADD_ON_FOR_SYSMON_PATH, "security_content/slim_packaging/apps/Splunk_TA_microsoft_sysmon-1.0.2-B1.spl")
-
-
-        #SYSMON_APP_FOR_SPLUNK = "https://splunkbase.splunk.com/app/3544/release/2.0.0/download"
-        #SPLUNK_ES_CONTENT_UPDATE = "https://splunkbase.splunk.com/app/3449/release/3.29.0/download"
-        SPLUNK_ADD_ON_FOR_MICROSOFT_WINDOWS = "https://splunkbase.splunk.com/app/742/release/8.2.0/download"
-        LOCAL_GENERATED_ESCU_LATEST = "/tmp/apps/DA-ESS-ContentUpdate-latest.tar.gz"
-        
-        SPLUNK_APPS = [SPLUNK_COMMON_INFORMATION_MODEL, 
-                       SPLUNK_SECURITY_ESSENTIALS, 
-                       LOCAL_SPLUNK_ADD_ON_FOR_SYSMON_VOLUME_PATH, 
-                       LOCAL_GENERATED_ESCU_LATEST, 
-                       SPLUNK_ADD_ON_FOR_MICROSOFT_WINDOWS]
         
 
         environment = {"SPLUNK_START_ARGS": "--accept-license",
@@ -478,9 +505,10 @@ def main(args):
         ports= {"8000/tcp": web_port,
                 "8089/tcp": management_port
                }
-        source_path = os.path.join(os.getcwd(), "security_content", "slim_packaging","apps")
         
-        mounts = [docker.types.Mount(target = '/tmp/apps/', source = source_path, type='bind', read_only=True)]
+        
+        
+        mounts = [docker.types.Mount(target = CONTAINER_VOLUME_PATH, source = local_volume_path, type='bind', read_only=True)]
 
         print("Creating CONTAINER: [%s]"%(container_name))
         base_container = client.containers.create(full_docker_hub_container_name, ports=ports, environment=environment, name=container_name, mounts=mounts, detach=True)
@@ -517,7 +545,7 @@ def main(args):
         #app install once, but it looks like the container is unlikely to support that.
         #We don't really want to fundamentally change this container, either, and will 
         #keep it as close to production as possible
-        time.sleep(60)
+        time.sleep(5)
     
     #Wait for all of the testing threads to complete
     for t in splunk_container_manager_threads:
@@ -573,7 +601,7 @@ def copy_file_to_container(localFilePath, remoteFilePath, containerName, sleepTi
 
 
 class SynchronizedResultsTracker:
-    def __init__(self, tests:list[str]):
+    def __init__(self, num_containers:int, tests:list[str]):
         #Create the queue and enque all of the tests
         self.testing_queue = queue.Queue()
         for test in tests:
@@ -591,6 +619,7 @@ class SynchronizedResultsTracker:
         #Just make a random folder to store attack data that we donwload
         self.attack_data_root_folder = mkdtemp(prefix="attack_data_", dir=os.getcwd())
         print("Attack data for this run will be stored at: [%s]"%(self.attack_data_root_folder))
+        self.start_barrier = threading.Barrier(num_containers)
 
     def getTest(self)-> Union[str,None]:
         try:
@@ -777,9 +806,13 @@ def splunk_container_manager(testing_object:SynchronizedResultsTracker, containe
     
     print("Successfully enabled DELETE for [%s]"%(container_name))
     
-    
+    #Wait for all of the threads to join here
+    print("Container [%s] setup complete and waiting for other containers to be ready..."%(container_name))
+    testing_object.start_barrier.wait()
     
     while True:
+        #Sleep for a small random time so that containers drift apart and don't synchronize their testing
+        time.sleep(random.randint(1,30))
         #Try to get something from the queue
         detection_to_test = testing_object.getTest()
         if detection_to_test is None:
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/splunk_sdk.py b/automated_detection_testing/ci/detection_testing_batch/modules/splunk_sdk.py
index 9c65c41d4a..69218ed9be 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/splunk_sdk.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/splunk_sdk.py
@@ -90,10 +90,12 @@ def test_detection_search(splunk_host:str, splunk_port:int, splunk_password:str,
               "dispatch.latest_time": "now"}
 
     splunk_search = search + ' ' + pass_condition
+    test_results = dict()
     
+    test_results['search_string'] = splunk_search
     
 
-    test_results = dict()
+    
     try:
         service = client.connect(
             host=splunk_host,
@@ -126,7 +128,7 @@ def test_detection_search(splunk_host:str, splunk_port:int, splunk_password:str,
     test_results['detection_name'] = detection_name
     test_results['detection_file'] = detection_file
     test_results['scanCount'] = job['scanCount']
-    test_results['search_string'] = splunk_search
+    
     
     test_results['error'] = False #The search may have FAILED, but there was no error in the search
 

From 673c95673cd0aae8b49d8ed898fe8cd572bd6254 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Tue, 26 Oct 2021 14:47:01 -0700
Subject: [PATCH 032/166] Fixed slim install

---
 .../detection_testing_batch/detection_testing_execution.py   | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
index 63526ddfa0..9146a3c54d 100644
--- a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
+++ b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
@@ -427,7 +427,8 @@ def main(args):
                     "python3 -m venv .venv", 
                     ". ./.venv/bin/activate", 
                     "python3 -m pip install wheel", 
-                    "python3 -m pip install -r requirements.txt", 
+                    "python3 -m pip install semantic_version",
+                    "python3 -m pip install .",  
                     "cp -R ../../dist/escu DA-ESS-ContentUpdate",
                     "slim package -o upload DA-ESS-ContentUpdate",
                     "cp upload/DA-ESS-ContentUpdate*.tar.gz ../apps/DA-ESS-ContentUpdate-latest.tar.gz"]
@@ -435,7 +436,7 @@ def main(args):
     
     
 
-    ret = subprocess.run("; ".join(commands), shell=True, capture_output=True)
+    ret = subprocess.run("; ".join(commands), shell=True, capture_output=False)
     if ret.returncode != 0:
         print("Error generating new ESCU Package.\n\tQuitting..."%())
         sys.exit(1)

From b97fb1355531dad4814e9f90749aaf1490f39d6c Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Tue, 26 Oct 2021 15:25:26 -0700
Subject: [PATCH 033/166] Rolling back some changes that introduced slim
 installation errors

---
 .../detection_testing_execution.py                | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
index 9146a3c54d..494b54fca4 100644
--- a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
+++ b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
@@ -397,7 +397,7 @@ def main(args):
     if persist_security_content is True:
         os.chdir("slim_packaging")
         commands = ["cd slim-latest", 
-                    ". ./.venv/bin/activate",
+                    "source venv/bin/activate",
                     "cp -R ../../dist/escu DA-ESS-ContentUpdate",
                     "slim package -o upload DA-ESS-ContentUpdate",
                     "cp upload/DA-ESS-ContentUpdate*.tar.gz ../apps/DA-ESS-ContentUpdate-latest.tar.gz"]
@@ -424,11 +424,12 @@ def main(args):
                     "mkdir slim-latest", 
                     "tar -zxf splunk-packaging-toolkit-latest.tar.gz -C slim-latest --strip-components=1",
                     "cd slim-latest", 
-                    "python3 -m venv .venv", 
-                    ". ./.venv/bin/activate", 
-                    "python3 -m pip install wheel", 
-                    "python3 -m pip install semantic_version",
-                    "python3 -m pip install .",  
+                    "virtualenv --python=/usr/bin/python2.7 --clear .venv",
+                    "source venv/bin/activate",
+                    "python3 -m pip install --upgrade pip",
+                    "python2 -m pip install wheel",
+                    "python2 -m pip install semantic_version",
+                    "python2 -m pip install .",
                     "cp -R ../../dist/escu DA-ESS-ContentUpdate",
                     "slim package -o upload DA-ESS-ContentUpdate",
                     "cp upload/DA-ESS-ContentUpdate*.tar.gz ../apps/DA-ESS-ContentUpdate-latest.tar.gz"]
@@ -436,7 +437,7 @@ def main(args):
     
     
 
-    ret = subprocess.run("; ".join(commands), shell=True, capture_output=False)
+    ret = subprocess.run("; ".join(commands), shell=True, capture_output=True)
     if ret.returncode != 0:
         print("Error generating new ESCU Package.\n\tQuitting..."%())
         sys.exit(1)

From 5fc1b7585b01e872a25e97981745adec95442fcb Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Tue, 26 Oct 2021 15:55:07 -0700
Subject: [PATCH 034/166] Fixed the last source to . changes for building slim.
  Also added the number of containers argument to the Synchronization object.

---
 .../detection_testing_execution.py                        | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
index 494b54fca4..52a9caab98 100644
--- a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
+++ b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
@@ -385,7 +385,7 @@ def main(args):
                     "python3 -m pip install -r requirements.txt", 
                     "python3 contentctl.py --path . --verbose generate --product ESCU --output dist/escu", "tar -czf DA-ESS-ContentUpdate.spl -C dist/escu ."]
     else:
-        commands = ["source .venv/bin/activate", "python3 contentctl.py --path . --verbose generate --product ESCU --output dist/escu", "tar -czf DA-ESS-ContentUpdate.spl -C dist/escu ."]
+        commands = ["s. ./.venv/bin/activate", "python3 contentctl.py --path . --verbose generate --product ESCU --output dist/escu", "tar -czf DA-ESS-ContentUpdate.spl -C dist/escu ."]
     ret = subprocess.run("; ".join(commands), shell=True, capture_output=True)
     if ret.returncode != 0:
         print("Error generating new content.  Exiting...")
@@ -397,7 +397,7 @@ def main(args):
     if persist_security_content is True:
         os.chdir("slim_packaging")
         commands = ["cd slim-latest", 
-                    "source venv/bin/activate",
+                    ". ./.venv/bin/activate",
                     "cp -R ../../dist/escu DA-ESS-ContentUpdate",
                     "slim package -o upload DA-ESS-ContentUpdate",
                     "cp upload/DA-ESS-ContentUpdate*.tar.gz ../apps/DA-ESS-ContentUpdate-latest.tar.gz"]
@@ -425,7 +425,7 @@ def main(args):
                     "tar -zxf splunk-packaging-toolkit-latest.tar.gz -C slim-latest --strip-components=1",
                     "cd slim-latest", 
                     "virtualenv --python=/usr/bin/python2.7 --clear .venv",
-                    "source venv/bin/activate",
+                    ". ./.venv/bin/activate",
                     "python3 -m pip install --upgrade pip",
                     "python2 -m pip install wheel",
                     "python2 -m pip install semantic_version",
@@ -458,7 +458,7 @@ def main(args):
     
 
 
-    results_tracker = SynchronizedResultsTracker(test_files)
+    results_tracker = SynchronizedResultsTracker(test_files, num_containers=num_containers)
     
     local_volume_path = os.path.join(os.getcwd(), "security_content", "slim_packaging","apps")
 

From 710efec1a0b90f43b5cacae545e2ad198ac21e14 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Tue, 26 Oct 2021 16:19:37 -0700
Subject: [PATCH 035/166] Tweak how long we wait before starting tests with the
 containers.

---
 .../detection_testing_execution.py                   | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
index 52a9caab98..58dcfaf3ab 100644
--- a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
+++ b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
@@ -458,7 +458,7 @@ def main(args):
     
 
 
-    results_tracker = SynchronizedResultsTracker(test_files, num_containers=num_containers)
+    results_tracker = SynchronizedResultsTracker(test_files, num_containers)
     
     local_volume_path = os.path.join(os.getcwd(), "security_content", "slim_packaging","apps")
 
@@ -603,7 +603,7 @@ def copy_file_to_container(localFilePath, remoteFilePath, containerName, sleepTi
 
 
 class SynchronizedResultsTracker:
-    def __init__(self, num_containers:int, tests:list[str]):
+    def __init__(self, tests:list[str], num_containers:int):
         #Create the queue and enque all of the tests
         self.testing_queue = queue.Queue()
         for test in tests:
@@ -800,11 +800,11 @@ def splunk_container_manager(testing_object:SynchronizedResultsTracker, containe
     print("Finished copying files to container: [%s]"%(container_name))
 
 
-    wait_for_splunk_ready(max_seconds=120)
+    #wait_for_splunk_ready(max_seconds=120)
     from modules.splunk_sdk import enable_delete_for_admin
-    if not enable_delete_for_admin(splunk_ip, splunk_port, splunk_password):
-        print("COULD NOT ENABLE DELETE FOR [%s].... quitting"%(container_name))
-        sys.exit(0)
+    print("Enabling DELETE for [%s]"%(container_name))
+    while not enable_delete_for_admin(splunk_ip, splunk_port, splunk_password):
+        time.sleep(10)
     
     print("Successfully enabled DELETE for [%s]"%(container_name))
     

From afbc033601ac2df769e13f0d4a705881846fb0e7 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Tue, 26 Oct 2021 16:58:24 -0700
Subject: [PATCH 036/166] Wait some time after ready before starting all of the
 tests to allow the container to settle.

---
 .../detection_testing_execution.py                       | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
index 58dcfaf3ab..dd43d6a370 100644
--- a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
+++ b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
@@ -803,15 +803,18 @@ def splunk_container_manager(testing_object:SynchronizedResultsTracker, containe
     #wait_for_splunk_ready(max_seconds=120)
     from modules.splunk_sdk import enable_delete_for_admin
     print("Enabling DELETE for [%s]"%(container_name))
-    while not enable_delete_for_admin(splunk_ip, splunk_port, splunk_password):
-        time.sleep(10)
+    try:
+        while not enable_delete_for_admin(splunk_ip, splunk_port, splunk_password):
+            time.sleep(10)
+    except Exception as e:
+        print("Failure enabling DELETE for container [%s]: [%s].\n\tQuitting..."%(container_name, str(e)))
     
     print("Successfully enabled DELETE for [%s]"%(container_name))
     
     #Wait for all of the threads to join here
     print("Container [%s] setup complete and waiting for other containers to be ready..."%(container_name))
     testing_object.start_barrier.wait()
-    
+    wait_for_splunk_ready(max_seconds=60)
     while True:
         #Sleep for a small random time so that containers drift apart and don't synchronize their testing
         time.sleep(random.randint(1,30))

From 16d9cac1b5c9c65f34dbd57b106c5c4fd72fd6a6 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Wed, 27 Oct 2021 16:58:31 -0700
Subject: [PATCH 037/166] Added more apps that previously were not installed. 
 Changed the order - CIM is installed last. Since we require a file copy to a
 directory that does not exist until that app is installed, it prevents us
 from getting ahead of ourselves and starting tests on the container until all
 apps have installed.

---
 .../detection_testing_execution.py            | 104 +++++++++++++-----
 1 file changed, 76 insertions(+), 28 deletions(-)

diff --git a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
index dd43d6a370..c6d8fa1c23 100644
--- a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
+++ b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
@@ -24,7 +24,10 @@ from typing import OrderedDict, Union
 
 from tempfile import mkdtemp
 import csv
-from urllib.request import urlretrieve 
+
+from requests import get
+
+
 SPLUNK_CONTAINER_APPS_DIR = "/opt/splunk/etc/apps"
 index_file_local_path = "indexes.conf.tar"
 index_file_container_path = os.path.join(SPLUNK_CONTAINER_APPS_DIR, "search")
@@ -65,13 +68,30 @@ def get_random_password()->str:
     return password
 
 
-def wait_for_splunk_ready(splunk_container_name=None, splunk_web_port=None, max_seconds=30):
+def wait_for_splunk_ready(container_name:str, splunk_web_port:int, splunk_ip:str="127.0.0.1", max_seconds:int=300)->bool:
     #The smarter version of this will try to hit one of the pages,
     #probably the login page, and when that is available it means that
     #splunk is fully started and ready to go.  Until then, we just
     #use a simple sleep
-    time.sleep(max_seconds)
-
+    splunk_ready_url =  "http://%s:%d"%(splunk_ip, splunk_web_port)
+    print(splunk_ready_url)
+    start = timer()
+    while True:
+        try:
+            #Splunk container will not have proper ssl certificate
+            response = get(splunk_ready_url,timeout=5, verify=False)
+            response.raise_for_status()
+            print("\n\n\n*****CONTAINER GET WORKED OKAY******")
+            return True
+        except Exception as e:
+            elapsed = timer() - start
+            print(str(e))
+            if elapsed > max_seconds:
+                raise(Exception("Container [%s] took longer than maximum start time of [%d].\n\tQuitting..."%(container_name, max_seconds)))
+        print("Wait progress [%d of %d]"%(elapsed, max_seconds))
+        print(timer() - start)
+        time.sleep(5)
+    
 
 
 
@@ -217,7 +237,8 @@ def main(args):
                                                                                                                      "time and allows you to test a detection that you've updated.  Runs generate again in case you have "\
                                                                                                                      "updated macros or anything else.  Especially useful for quick, local, iterative testing.")
     start_datetime = datetime.now()
-    
+    import requests.packages.urllib3
+    requests.packages.urllib3.disable_warnings()
     args = parser.parse_args()
     branch = args.branch
     uuid_test = args.uuid
@@ -409,8 +430,13 @@ def main(args):
         
         try:
             SPLUNK_PACKAGING_TOOLKIT_URL = "https://download.splunk.com/misc/packaging-toolkit/splunk-packaging-toolkit-0.9.0.tar.gz"
+            SPLUNK_PACKAGING_TOOLKIT_FILENAME = 'splunk-packaging-toolkit-latest.tar.gz'
             print("Downloading the Splunk Packaging Toolkit from %s..."%(SPLUNK_PACKAGING_TOOLKIT_URL), end='')
-            urlretrieve(SPLUNK_PACKAGING_TOOLKIT_URL, 'splunk-packaging-toolkit-latest.tar.gz')
+            response = get(SPLUNK_PACKAGING_TOOLKIT_URL)
+            response.raise_for_status()
+            with open(SPLUNK_PACKAGING_TOOLKIT_FILENAME, 'wb') as slim_file:
+                slim_file.write(response.content)
+            
             print("success")
 
 
@@ -462,31 +488,51 @@ def main(args):
     
     local_volume_path = os.path.join(os.getcwd(), "security_content", "slim_packaging","apps")
 
-    SPLUNK_COMMON_INFORMATION_MODEL = "https://splunkbase.splunk.com/app/1621/release/4.20.2/download"        
-    SPLUNK_SECURITY_ESSENTIALS = "https://splunkbase.splunk.com/app/3435/release/3.3.4/download"
+          
+    
     #SPLUNK_ADD_ON_FOR_SYSMON_OLD = "https://splunkbase.splunk.com/app/1914/release/10.6.2/download"
     #SPLUNK_ADD_ON_FOR_SYSMON_NEW = "https://splunkbase.splunk.com/app/5709/release/1.0.1/download"
     #SYSMON_APP_FOR_SPLUNK = "https://splunkbase.splunk.com/app/3544/release/2.0.0/download"
     #SPLUNK_ES_CONTENT_UPDATE = "https://splunkbase.splunk.com/app/3449/release/3.29.0/download"
 
     #Just a hack until we get the new version of system deployed and available from splunkbase
-    LOCAL_SPLUNK_ADD_ON_FOR_SYSMON_PATH  = os.path.expanduser("~/Downloads/Splunk_TA_microsoft_sysmon-1.0.2-B1.spl")    
-    shutil.copyfile(LOCAL_SPLUNK_ADD_ON_FOR_SYSMON_PATH, os.path.join(local_volume_path, "Splunk_TA_microsoft_sysmon-1.0.2-B1.spl"))
-    SPLUNK_ADD_ON_FOR_SYSMON_VOLUME_PATH  = "/tmp/apps/Splunk_TA_microsoft_sysmon-1.0.2-B1.spl"
-    
-
-
-    
-    SPLUNK_ADD_ON_FOR_MICROSOFT_WINDOWS = "https://splunkbase.splunk.com/app/742/release/8.2.0/download"
     CONTAINER_VOLUME_PATH = '/tmp/apps/'
-    CONTAINER_GENERATED_ESCU_LATEST = os.path.join(CONTAINER_VOLUME_PATH, "DA-ESS-ContentUpdate-latest.tar.gz")
+    BETA_SPLUNK_ADD_ON_FOR_SYSMON_PATH  = os.path.expanduser("~/Downloads/Splunk_TA_microsoft_sysmon-1.0.2-B1.spl")    
+    shutil.copyfile(BETA_SPLUNK_ADD_ON_FOR_SYSMON_PATH, os.path.join(local_volume_path, "Splunk_TA_microsoft_sysmon-1.0.2-B1.spl"))
     
-    SPLUNK_APPS = [SPLUNK_COMMON_INFORMATION_MODEL, 
-                    SPLUNK_SECURITY_ESSENTIALS, 
-                    SPLUNK_ADD_ON_FOR_SYSMON_VOLUME_PATH, 
-                    CONTAINER_GENERATED_ESCU_LATEST, 
-                    SPLUNK_ADD_ON_FOR_MICROSOFT_WINDOWS]
+    BETA_SPLUNK_ADD_ON_FOR_SYSMON_CONTAINER_PATH  = "/tmp/apps/Splunk_TA_microsoft_sysmon-1.0.2-B1.spl"
+    GENERATED_SPLUNK_ES_CONTENT_UPDATE_CONTAINER_PATH = os.path.join(CONTAINER_VOLUME_PATH, "DA-ESS-ContentUpdate-latest.tar.gz")
 
+    SPLUNKBASE_URL = "https://splunkbase.splunk.com/app/%d/release/%s/download"
+    APPS_DICT = OrderedDict()
+    APPS_DICT['SPLUNK_ADD_ON_FOR_MICROSOFT_WINDOWS'] = {"app_number":742, 'app_version':"8.2.0", 'location':'splunkbase'}
+    APPS_DICT['SPLUNK_SECURITY_ESSENTIALS'] = {"app_number":3435, 'app_version':"3.3.4", 'location':'splunkbase'}
+    APPS_DICT['SPLUNK_ADD_ON_FOR_AMAZON_WEB_SERVICES'] = {"app_number":1876, 'app_version':"5.2.0", 'location':'splunkbase'}
+    APPS_DICT['SPLUNK_ADD_ON_FOR_MICROSOFT_OFFICE_365'] = {"app_number":4055, 'app_version':"2.2.0", 'location':'splunkbase'}
+    APPS_DICT['SPLUNK_ADD_ON_FOR_AMAZON_KINESIS_FIREHOSE'] = {"app_number":3719, 'app_version':"1.3.2", 'location':'splunkbase'}
+    APPS_DICT['SPLUNK_ANALYTIC_STORY_EXECUTION_APP'] = {"app_number":4971, 'app_version': "2.0.3", 'location':'splunkbase'}
+    APPS_DICT['PYTHON_FOR_SCIENTIC_COMPUTING_LINUX_64_BIT'] = {"app_number":2882, 'app_version':"2.0.2", 'location':'splunkbase'}
+    APPS_DICT['SPLUNK_MACHINE_LEARNING_TOOLKIT'] = {"app_number":2890, 'app_version':"5.2.2", 'location':'splunkbase'}
+    APPS_DICT['SPLUNK_APP_FOR_STREAM'] = {"app_number":1809, 'app_version':"8.0.1", 'location':'splunkbase'}
+    APPS_DICT['SPLUNK_ADD_ON_FOR_STREAM_WIRE_DATA'] = {"app_number":5234, 'app_version':"8.0.1", 'location':'splunkbase'}
+    APPS_DICT['SPLUNK_ADD_ON_FOR_STREAM_FORWARDERS'] = {"app_number":5238, 'app_version':"8.0.1", 'location':'splunkbase'}
+    APPS_DICT['SPLUNK_ADD_ON_FOR_ZEEK_AKA_BRO'] = {"app_number":1617, 'app_version':"4.0.0", 'location':'splunkbase'}
+    APPS_DICT['SPLUNK_ADD_ON_FOR_UNIX_AND_LINUX'] = {"app_number":833, 'app_version':"8.3.1", 'location':'splunkbase'}
+    APPS_DICT['GENERATED_SPLUNK_ES_CONTENT_UPDATE'] = {"app_number":3449, 'app_version':"Generated at %s"%(datetime.now()), 'location':GENERATED_SPLUNK_ES_CONTENT_UPDATE_CONTAINER_PATH}
+    APPS_DICT['BETA_SPLUNK_ADD_ON_FOR_SYSMON'] = {"app_number":3449, 'app_version':"Generated at %s"%(datetime.now()), 'location':BETA_SPLUNK_ADD_ON_FOR_SYSMON_CONTAINER_PATH}
+    APPS_DICT['SPLUNK_COMMON_INFORMATION_MODEL'] = {"app_number":1621, 'app_version':"4.20.2", 'location':'splunkbase'}
+    
+    
+
+    SPLUNK_APPS = []
+    for key, value in APPS_DICT.items():
+        if value['location'] == 'splunkbase':
+            #The app is on Splunkbase
+            SPLUNK_APPS.append(SPLUNKBASE_URL%(value['app_number'],value['app_version']))
+        else:
+            #The app is a file we generated locally
+            SPLUNK_APPS.append(value['location'])
+    
 
 
     for container_index in range(num_containers):
@@ -523,7 +569,8 @@ def main(args):
                              args=(results_tracker, 
                                    container_name, 
                                    "127.0.0.1", 
-                                   splunk_password, 
+                                   splunk_password,
+                                   web_port, 
                                    management_port, 
                                    uuid_test,
                                    interactive_failure 
@@ -776,7 +823,7 @@ class SynchronizedResultsTracker:
         
 
 
-def splunk_container_manager(testing_object:SynchronizedResultsTracker, container_name, splunk_ip, splunk_password, splunk_port, uuid_test, interactive_failure:bool=False):
+def splunk_container_manager(testing_object:SynchronizedResultsTracker, container_name, splunk_ip, splunk_password, splunk_web_port, splunk_management_port, uuid_test, interactive_failure:bool=False):
     print("Starting the container [%s] after a sleep"%(container_name))
     #Is this going to be safe to use in different threads
     client = docker.client.from_env()
@@ -800,11 +847,11 @@ def splunk_container_manager(testing_object:SynchronizedResultsTracker, containe
     print("Finished copying files to container: [%s]"%(container_name))
 
 
-    #wait_for_splunk_ready(max_seconds=120)
+    
     from modules.splunk_sdk import enable_delete_for_admin
     print("Enabling DELETE for [%s]"%(container_name))
     try:
-        while not enable_delete_for_admin(splunk_ip, splunk_port, splunk_password):
+        while not enable_delete_for_admin(splunk_ip, splunk_management_port, splunk_password):
             time.sleep(10)
     except Exception as e:
         print("Failure enabling DELETE for container [%s]: [%s].\n\tQuitting..."%(container_name, str(e)))
@@ -814,7 +861,8 @@ def splunk_container_manager(testing_object:SynchronizedResultsTracker, containe
     #Wait for all of the threads to join here
     print("Container [%s] setup complete and waiting for other containers to be ready..."%(container_name))
     testing_object.start_barrier.wait()
-    wait_for_splunk_ready(max_seconds=60)
+    wait_for_splunk_ready(container_name, splunk_web_port,splunk_ip, max_seconds=300)
+
     while True:
         #Sleep for a small random time so that containers drift apart and don't synchronize their testing
         time.sleep(random.randint(1,30))
@@ -837,7 +885,7 @@ def splunk_container_manager(testing_object:SynchronizedResultsTracker, containe
         #There is a detection to test
         print("Container [%s]--->[%s]"%(container_name, detection_to_test))
         try:
-            result = testing_service.test_detection_wrapper(container_name, splunk_ip, splunk_password, splunk_port, detection_to_test, 0, uuid_test, testing_object.attack_data_root_folder, wait_on_failure=interactive_failure)
+            result = testing_service.test_detection_wrapper(container_name, splunk_ip, splunk_password, splunk_management_port, detection_to_test, 0, uuid_test, testing_object.attack_data_root_folder, wait_on_failure=interactive_failure)
             testing_object.addResult(result)
             
             #Remove the data from the test that we just ran.  We MUST do this when running on CI because otherwise, we will download

From 820207dc033a707fe5c48cb06c65d95870fa660e Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Wed, 27 Oct 2021 16:59:15 -0700
Subject: [PATCH 038/166] See the previous commit message. Added some comments
 around the CIM app to explain why it is the last app to be installed.

---
 .../detection_testing_batch/detection_testing_execution.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
index c6d8fa1c23..fc7cecc072 100644
--- a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
+++ b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
@@ -520,6 +520,10 @@ def main(args):
     APPS_DICT['SPLUNK_ADD_ON_FOR_UNIX_AND_LINUX'] = {"app_number":833, 'app_version':"8.3.1", 'location':'splunkbase'}
     APPS_DICT['GENERATED_SPLUNK_ES_CONTENT_UPDATE'] = {"app_number":3449, 'app_version':"Generated at %s"%(datetime.now()), 'location':GENERATED_SPLUNK_ES_CONTENT_UPDATE_CONTAINER_PATH}
     APPS_DICT['BETA_SPLUNK_ADD_ON_FOR_SYSMON'] = {"app_number":3449, 'app_version':"Generated at %s"%(datetime.now()), 'location':BETA_SPLUNK_ADD_ON_FOR_SYSMON_CONTAINER_PATH}
+    
+    #CIM is last here for a reason!  Because we copy a file to a directory that does not exist until CIM
+    #has been installed, we use it to prevent the testing from beginning until the copy has succeeded.  
+    #KEEP THIS APP LAST!
     APPS_DICT['SPLUNK_COMMON_INFORMATION_MODEL'] = {"app_number":1621, 'app_version':"4.20.2", 'location':'splunkbase'}
     
     
@@ -843,6 +847,9 @@ def splunk_container_manager(testing_object:SynchronizedResultsTracker, containe
     container.start()
     print("Start copying files to container: [%s]"%(container_name))
     copy_file_to_container(index_file_local_path, index_file_container_path, container_name)
+    #The below copy will fail until the CIM app is installed.  This app MUST be installed last!
+    #If we install it earlier, we will get ahead of ourselves and start doing tests before the container
+    #is truly ready for testing and all apps have been installed
     copy_file_to_container(datamodel_file_local_path, datamodel_file_container_path, container_name)
     print("Finished copying files to container: [%s]"%(container_name))
 

From c212b20dd517737825a481c244686e830927fb42 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Wed, 27 Oct 2021 17:50:18 -0700
Subject: [PATCH 039/166] Joined detection types default argument from list to
 string.

---
 .../ci/detection_testing_batch/detection_testing_execution.py   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
index fc7cecc072..4a24d015f7 100644
--- a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
+++ b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
@@ -231,7 +231,7 @@ def main(args):
     parser.add_argument("-m", "--mode", type=str, choices=DETECTION_MODES, required=False, help="Whether to test new detections, specific detections, or all detections", default="new")
     parser.add_argument("-tf","--test_files", type=str, required=False, help="The names of files that you want to test, separated by commas.")
 
-    parser.add_argument("-t", "--types", type=str, required=False, help="Detection types to test. Can be one of more of %s"%(str(DETECTION_TYPES)), default=DETECTION_TYPES)
+    parser.add_argument("-t", "--types", type=str, required=False, help="Detection types to test. Can be one of more of %s"%(str(DETECTION_TYPES)), default=','.join(DETECTION_TYPES))
     parser.add_argument("-ct", "--container_tag", type=str, required=False, help="The tag of the Splunk Container to use.  Tags are located at https://hub.docker.com/r/splunk/splunk/tags",default=DEFAULT_CONTAINER_TAG)
     parser.add_argument("-p", "--persist_security_content", required=False, default=False, action="store_true", help="Assumes security_content directory already exists.  Don't check it out and overwrite it again.  Saves "\
                                                                                                                      "time and allows you to test a detection that you've updated.  Runs generate again in case you have "\

From a1521a1dce076bf5dbcd05e963feebf48eae83d9 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Mon, 1 Nov 2021 13:23:56 -0700
Subject: [PATCH 040/166] Created a GitHub Action YML for testing using docker
 container on GitHub actions. Right now, it's just testing a single, static
 detection since that's easier and faster.

---
 .../workflows/docker-detection-testing.yml    | 80 +++++++++++++++++++
 .../detection_testing_execution.py            | 55 ++++++++-----
 2 files changed, 113 insertions(+), 22 deletions(-)
 create mode 100644 .github/workflows/docker-detection-testing.yml

diff --git a/.github/workflows/docker-detection-testing.yml b/.github/workflows/docker-detection-testing.yml
new file mode 100644
index 0000000000..d5ee617342
--- /dev/null
+++ b/.github/workflows/docker-detection-testing.yml
@@ -0,0 +1,80 @@
+name: detection-testing
+on:
+  push:
+jobs:
+
+  validate-tag-if-present:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: TAGGED, Validate that the tag is in the correct format
+        
+        run: |
+          echo "The GITHUB_REF: $GITHUB_REF"
+          #First check to see if the release is a tag
+          if [[ $GITHUB_REF =~ refs/tags/* ]]; then
+            #Yes, this is a tag, so we need to test to make sure that the tag
+            #is in the correct format (like v1.10.20)
+            if [[ $GITHUB_REF =~ refs/tags/v[0-9]+.[0-9]+.[0-9]+ ]]; then
+              echo "PASS: Tagged release with good format"
+              exit 0
+            else
+              echo "FAIL: Tagged release with bad format"
+              exit 1
+            fi
+          else
+          echo "PASS: Not a tagged release"
+          exit 0
+          fi                    
+  
+  quit-for-dependabot:
+    runs-on: ubuntu-latest
+    if: github.actor != 'dependabot[bot]' && github.actor != 'dependabot-preview[bot]'
+    steps:
+      - name: "Placeholder"
+        run: |
+          echo "No, this is not a dependabot run!"
+
+
+  detection-testing:
+      runs-on: ubuntu-latest
+      needs: [validate-tag-if-present, quit-for-dependabot]
+      steps:  
+          - name: Get branch and PR required for detection testing main.py
+            id: vars
+            run: |
+              echo "::set-output name=branch::${GITHUB_REF#refs/heads/}"
+
+          - name: Checkout Repo
+            uses: actions/checkout@v2
+          
+          - name: Install Docker 
+            run: |
+              sudo apt update -qq
+              sudo apt install docker.io
+              sudo usermod -aG docker $USER
+
+              #python2.7 needed for slim, for now
+              sudo apt install python2.7
+              sudo apt install virtualenv python2.7-pip
+
+          - uses: actions/setup-python@v2
+            with:
+              python-version: '3.9' #Available versions here - https://github.com/actions/python-versions/releases  easy to change/make a matrix/use pypy
+              architecture: 'x64' # optional x64 or x86. Defaults to x64 if not specified          
+          
+          - name: Install Python Dependencies
+            run: |
+              cd automated_detection_testing/ci/detection_testing_batch
+              python3 -m venv .venv
+              source .venv/bin/activate
+              python3 -m pip install wheel
+              python3 -m pip install -r requirements.txt
+          
+          - name: Run the CI
+            run: |
+              cd automated_detection_testing/ci/detection_testing_batch
+              source .venv/bin/activate
+              python3 detection_testing_execution.py -b {{ steps.vars.outputs.branch }} -u 123456789 --splunkbase_username {{ secrets.SPLUNKBASE_TESTING_USERANME }} --splunkbase_password {{ secrets.SPLUNKBASE_TESTING_KEY }} -m selected -tf security_content/detections/endpoint/7zip_commandline_to_smb_share_path.yml -n1
+              echo "DONE!"
+          
diff --git a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
index 4a24d015f7..93e80fe3a0 100644
--- a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
+++ b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
@@ -81,15 +81,11 @@ def wait_for_splunk_ready(container_name:str, splunk_web_port:int, splunk_ip:str
             #Splunk container will not have proper ssl certificate
             response = get(splunk_ready_url,timeout=5, verify=False)
             response.raise_for_status()
-            print("\n\n\n*****CONTAINER GET WORKED OKAY******")
             return True
         except Exception as e:
             elapsed = timer() - start
-            print(str(e))
             if elapsed > max_seconds:
                 raise(Exception("Container [%s] took longer than maximum start time of [%d].\n\tQuitting..."%(container_name, max_seconds)))
-        print("Wait progress [%d of %d]"%(elapsed, max_seconds))
-        print(timer() - start)
         time.sleep(5)
     
 
@@ -497,30 +493,42 @@ def main(args):
 
     #Just a hack until we get the new version of system deployed and available from splunkbase
     CONTAINER_VOLUME_PATH = '/tmp/apps/'
-    BETA_SPLUNK_ADD_ON_FOR_SYSMON_PATH  = os.path.expanduser("~/Downloads/Splunk_TA_microsoft_sysmon-1.0.2-B1.spl")    
-    shutil.copyfile(BETA_SPLUNK_ADD_ON_FOR_SYSMON_PATH, os.path.join(local_volume_path, "Splunk_TA_microsoft_sysmon-1.0.2-B1.spl"))
     
-    BETA_SPLUNK_ADD_ON_FOR_SYSMON_CONTAINER_PATH  = "/tmp/apps/Splunk_TA_microsoft_sysmon-1.0.2-B1.spl"
     GENERATED_SPLUNK_ES_CONTENT_UPDATE_CONTAINER_PATH = os.path.join(CONTAINER_VOLUME_PATH, "DA-ESS-ContentUpdate-latest.tar.gz")
 
     SPLUNKBASE_URL = "https://splunkbase.splunk.com/app/%d/release/%s/download"
+    #Order that we install the apps is actually important
     APPS_DICT = OrderedDict()
     APPS_DICT['SPLUNK_ADD_ON_FOR_MICROSOFT_WINDOWS'] = {"app_number":742, 'app_version':"8.2.0", 'location':'splunkbase'}
     APPS_DICT['SPLUNK_SECURITY_ESSENTIALS'] = {"app_number":3435, 'app_version':"3.3.4", 'location':'splunkbase'}
-    APPS_DICT['SPLUNK_ADD_ON_FOR_AMAZON_WEB_SERVICES'] = {"app_number":1876, 'app_version':"5.2.0", 'location':'splunkbase'}
-    APPS_DICT['SPLUNK_ADD_ON_FOR_MICROSOFT_OFFICE_365'] = {"app_number":4055, 'app_version':"2.2.0", 'location':'splunkbase'}
-    APPS_DICT['SPLUNK_ADD_ON_FOR_AMAZON_KINESIS_FIREHOSE'] = {"app_number":3719, 'app_version':"1.3.2", 'location':'splunkbase'}
-    APPS_DICT['SPLUNK_ANALYTIC_STORY_EXECUTION_APP'] = {"app_number":4971, 'app_version': "2.0.3", 'location':'splunkbase'}
-    APPS_DICT['PYTHON_FOR_SCIENTIC_COMPUTING_LINUX_64_BIT'] = {"app_number":2882, 'app_version':"2.0.2", 'location':'splunkbase'}
-    APPS_DICT['SPLUNK_MACHINE_LEARNING_TOOLKIT'] = {"app_number":2890, 'app_version':"5.2.2", 'location':'splunkbase'}
-    APPS_DICT['SPLUNK_APP_FOR_STREAM'] = {"app_number":1809, 'app_version':"8.0.1", 'location':'splunkbase'}
-    APPS_DICT['SPLUNK_ADD_ON_FOR_STREAM_WIRE_DATA'] = {"app_number":5234, 'app_version':"8.0.1", 'location':'splunkbase'}
-    APPS_DICT['SPLUNK_ADD_ON_FOR_STREAM_FORWARDERS'] = {"app_number":5238, 'app_version':"8.0.1", 'location':'splunkbase'}
-    APPS_DICT['SPLUNK_ADD_ON_FOR_ZEEK_AKA_BRO'] = {"app_number":1617, 'app_version':"4.0.0", 'location':'splunkbase'}
-    APPS_DICT['SPLUNK_ADD_ON_FOR_UNIX_AND_LINUX'] = {"app_number":833, 'app_version':"8.3.1", 'location':'splunkbase'}
     APPS_DICT['GENERATED_SPLUNK_ES_CONTENT_UPDATE'] = {"app_number":3449, 'app_version':"Generated at %s"%(datetime.now()), 'location':GENERATED_SPLUNK_ES_CONTENT_UPDATE_CONTAINER_PATH}
-    APPS_DICT['BETA_SPLUNK_ADD_ON_FOR_SYSMON'] = {"app_number":3449, 'app_version':"Generated at %s"%(datetime.now()), 'location':BETA_SPLUNK_ADD_ON_FOR_SYSMON_CONTAINER_PATH}
     
+    
+    try:
+        raise
+        BETA_SPLUNK_ADD_ON_FOR_SYSMON_PATH  = os.path.expanduser("~/Downloads/Splunk_TA_microsoft_sysmon-1.0.2-B1.spl")    
+        shutil.copyfile(BETA_SPLUNK_ADD_ON_FOR_SYSMON_PATH, os.path.join(local_volume_path, "Splunk_TA_microsoft_sysmon-1.0.2-B1.spl"))
+    
+        BETA_SPLUNK_ADD_ON_FOR_SYSMON_CONTAINER_PATH  = "/tmp/apps/Splunk_TA_microsoft_sysmon-1.0.2-B1.spl"
+        APPS_DICT['BETA_SPLUNK_ADD_ON_FOR_SYSMON'] = {"app_number":5709, 'app_version':"Generated at %s"%(datetime.now()), 'location':BETA_SPLUNK_ADD_ON_FOR_SYSMON_CONTAINER_PATH}
+    except Exception as e:
+        print("Failed to grab beta sysmon at ~/Downloads/Splunk_TA_microsoft_sysmon-1.0.2-B1.spl. Using the one from Splunkbase")
+        APPS_DICT['SPLUNK_ADD_ON_FOR_SYSMON'] = {"app_number":5709, 'app_version':"1.0.1", 'location':'splunkbase'}
+    
+    if True:
+        APPS_DICT['SPLUNK_ADD_ON_FOR_AMAZON_WEB_SERVICES'] = {"app_number":1876, 'app_version':"5.2.0", 'location':'splunkbase'}
+        APPS_DICT['SPLUNK_ADD_ON_FOR_MICROSOFT_OFFICE_365'] = {"app_number":4055, 'app_version':"2.2.0", 'location':'splunkbase'}
+        APPS_DICT['SPLUNK_ADD_ON_FOR_AMAZON_KINESIS_FIREHOSE'] = {"app_number":3719, 'app_version':"1.3.2", 'location':'splunkbase'}
+        APPS_DICT['SPLUNK_ANALYTIC_STORY_EXECUTION_APP'] = {"app_number":4971, 'app_version': "2.0.3", 'location':'splunkbase'}
+        APPS_DICT['PYTHON_FOR_SCIENTIC_COMPUTING_LINUX_64_BIT'] = {"app_number":2882, 'app_version':"2.0.2", 'location':'splunkbase'}
+        APPS_DICT['SPLUNK_MACHINE_LEARNING_TOOLKIT'] = {"app_number":2890, 'app_version':"5.2.2", 'location':'splunkbase'}
+        APPS_DICT['SPLUNK_APP_FOR_STREAM'] = {"app_number":1809, 'app_version':"8.0.1", 'location':'splunkbase'}
+        APPS_DICT['SPLUNK_ADD_ON_FOR_STREAM_WIRE_DATA'] = {"app_number":5234, 'app_version':"8.0.1", 'location':'splunkbase'}
+        APPS_DICT['SPLUNK_ADD_ON_FOR_STREAM_FORWARDERS'] = {"app_number":5238, 'app_version':"8.0.1", 'location':'splunkbase'}
+        APPS_DICT['SPLUNK_ADD_ON_FOR_ZEEK_AKA_BRO'] = {"app_number":1617, 'app_version':"4.0.0", 'location':'splunkbase'}
+        APPS_DICT['SPLUNK_ADD_ON_FOR_UNIX_AND_LINUX'] = {"app_number":833, 'app_version':"8.3.1", 'location':'splunkbase'}
+    
+
     #CIM is last here for a reason!  Because we copy a file to a directory that does not exist until CIM
     #has been installed, we use it to prevent the testing from beginning until the copy has succeeded.  
     #KEEP THIS APP LAST!
@@ -528,6 +536,7 @@ def main(args):
     
     
 
+    
     SPLUNK_APPS = []
     for key, value in APPS_DICT.items():
         if value['location'] == 'splunkbase':
@@ -560,8 +569,10 @@ def main(args):
         
         
         
-        mounts = [docker.types.Mount(target = CONTAINER_VOLUME_PATH, source = local_volume_path, type='bind', read_only=True)]
 
+        
+        mounts = [docker.types.Mount(target = CONTAINER_VOLUME_PATH, source = local_volume_path, type='bind', read_only=True)]
+        
         print("Creating CONTAINER: [%s]"%(container_name))
         base_container = client.containers.create(full_docker_hub_container_name, ports=ports, environment=environment, name=container_name, mounts=mounts, detach=True)
         print("Created CONTAINER : [%s]"%(container_name))
@@ -852,7 +863,7 @@ def splunk_container_manager(testing_object:SynchronizedResultsTracker, containe
     #is truly ready for testing and all apps have been installed
     copy_file_to_container(datamodel_file_local_path, datamodel_file_container_path, container_name)
     print("Finished copying files to container: [%s]"%(container_name))
-
+    
 
     
     from modules.splunk_sdk import enable_delete_for_admin
@@ -869,7 +880,7 @@ def splunk_container_manager(testing_object:SynchronizedResultsTracker, containe
     print("Container [%s] setup complete and waiting for other containers to be ready..."%(container_name))
     testing_object.start_barrier.wait()
     wait_for_splunk_ready(container_name, splunk_web_port,splunk_ip, max_seconds=300)
-
+    
     while True:
         #Sleep for a small random time so that containers drift apart and don't synchronize their testing
         time.sleep(random.randint(1,30))

From bb22ea0604a6846500b5766451b81ef0bb862238 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Mon, 1 Nov 2021 13:31:37 -0700
Subject: [PATCH 041/166] Printing out version of containerd

---
 .github/workflows/docker-detection-testing.yml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/docker-detection-testing.yml b/.github/workflows/docker-detection-testing.yml
index d5ee617342..0945d6a1ef 100644
--- a/.github/workflows/docker-detection-testing.yml
+++ b/.github/workflows/docker-detection-testing.yml
@@ -1,4 +1,4 @@
-name: detection-testing
+name: docker-detection-testing
 on:
   push:
 jobs:
@@ -36,7 +36,7 @@ jobs:
           echo "No, this is not a dependabot run!"
 
 
-  detection-testing:
+  docker-detection-testing:
       runs-on: ubuntu-latest
       needs: [validate-tag-if-present, quit-for-dependabot]
       steps:  
@@ -51,6 +51,7 @@ jobs:
           - name: Install Docker 
             run: |
               sudo apt update -qq
+              which containerd
               sudo apt install docker.io
               sudo usermod -aG docker $USER
 

From ceb606a01f8d2e1ac061cf13ae6ca8fb51dff73e Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Mon, 1 Nov 2021 13:34:39 -0700
Subject: [PATCH 042/166] Explicitly installing containerd

---
 .github/workflows/docker-detection-testing.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/docker-detection-testing.yml b/.github/workflows/docker-detection-testing.yml
index 0945d6a1ef..63fe6d11fd 100644
--- a/.github/workflows/docker-detection-testing.yml
+++ b/.github/workflows/docker-detection-testing.yml
@@ -51,6 +51,7 @@ jobs:
           - name: Install Docker 
             run: |
               sudo apt update -qq
+              sudo apt install containerd
               which containerd
               sudo apt install docker.io
               sudo usermod -aG docker $USER

From c8615d208c0d82d5e38271f07fc4699b2735db87 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Mon, 1 Nov 2021 13:48:46 -0700
Subject: [PATCH 043/166] Docker is already installed on VM, no need to install
 it.

---
 .github/workflows/docker-detection-testing.yml | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/docker-detection-testing.yml b/.github/workflows/docker-detection-testing.yml
index 63fe6d11fd..4715c61658 100644
--- a/.github/workflows/docker-detection-testing.yml
+++ b/.github/workflows/docker-detection-testing.yml
@@ -50,11 +50,11 @@ jobs:
           
           - name: Install Docker 
             run: |
-              sudo apt update -qq
-              sudo apt install containerd
-              which containerd
-              sudo apt install docker.io
-              sudo usermod -aG docker $USER
+              #sudo apt update -qq
+              #sudo apt install runc
+              #sudo apt install containerd
+              #sudo apt install docker.io
+              #sudo usermod -aG docker $USER
 
               #python2.7 needed for slim, for now
               sudo apt install python2.7

From e04135b8ae9f3a8ede32a18a1c2eb4dcd93581c4 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Mon, 1 Nov 2021 14:02:55 -0700
Subject: [PATCH 044/166] Properly installing python2

---
 .github/workflows/docker-detection-testing.yml | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/docker-detection-testing.yml b/.github/workflows/docker-detection-testing.yml
index 4715c61658..c7c0a5bc22 100644
--- a/.github/workflows/docker-detection-testing.yml
+++ b/.github/workflows/docker-detection-testing.yml
@@ -50,15 +50,14 @@ jobs:
           
           - name: Install Docker 
             run: |
-              #sudo apt update -qq
-              #sudo apt install runc
-              #sudo apt install containerd
-              #sudo apt install docker.io
-              #sudo usermod -aG docker $USER
+              sudo apt update -qq
+              
 
               #python2.7 needed for slim, for now
-              sudo apt install python2.7
+              sudo apt install python2
               sudo apt install virtualenv python2.7-pip
+              curl https://bootstrap.pypa.io/pip/2.7/get-pip.py --output get-pip.py
+              sudo python2.7 get-pip.py
 
           - uses: actions/setup-python@v2
             with:

From 1bf59b0ca78e623fc1eca574c600bec5a9236529 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Mon, 1 Nov 2021 14:05:11 -0700
Subject: [PATCH 045/166] Installing pip by script, not by apt

---
 .github/workflows/docker-detection-testing.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/docker-detection-testing.yml b/.github/workflows/docker-detection-testing.yml
index c7c0a5bc22..ec1e343c1c 100644
--- a/.github/workflows/docker-detection-testing.yml
+++ b/.github/workflows/docker-detection-testing.yml
@@ -55,7 +55,7 @@ jobs:
 
               #python2.7 needed for slim, for now
               sudo apt install python2
-              sudo apt install virtualenv python2.7-pip
+              sudo apt install virtualenv 
               curl https://bootstrap.pypa.io/pip/2.7/get-pip.py --output get-pip.py
               sudo python2.7 get-pip.py
 

From 75bc9ee74c3f7a805f6e2a872b9aa408b3ed4257 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Mon, 1 Nov 2021 14:10:41 -0700
Subject: [PATCH 046/166] Badly formatted secrets fixed

---
 .github/workflows/docker-detection-testing.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/docker-detection-testing.yml b/.github/workflows/docker-detection-testing.yml
index ec1e343c1c..20122fc264 100644
--- a/.github/workflows/docker-detection-testing.yml
+++ b/.github/workflows/docker-detection-testing.yml
@@ -76,6 +76,6 @@ jobs:
             run: |
               cd automated_detection_testing/ci/detection_testing_batch
               source .venv/bin/activate
-              python3 detection_testing_execution.py -b {{ steps.vars.outputs.branch }} -u 123456789 --splunkbase_username {{ secrets.SPLUNKBASE_TESTING_USERANME }} --splunkbase_password {{ secrets.SPLUNKBASE_TESTING_KEY }} -m selected -tf security_content/detections/endpoint/7zip_commandline_to_smb_share_path.yml -n1
+              python3 detection_testing_execution.py -b ${{ steps.vars.outputs.branch }} -u 123456789 --splunkbase_username ${{ secrets.SPLUNKBASE_TESTING_USERNAME }} --splunkbase_password ${{ secrets.SPLUNKBASE_TESTING_KEY }} -m selected -tf security_content/detections/endpoint/7zip_commandline_to_smb_share_path.yml -n1
               echo "DONE!"
           

From e0dc0e6743dc53e0e0066a77287c09583f959482 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Mon, 1 Nov 2021 14:24:49 -0700
Subject: [PATCH 047/166] Testing 2 containers with a larger number of
 detections

---
 .github/workflows/docker-detection-testing.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/docker-detection-testing.yml b/.github/workflows/docker-detection-testing.yml
index 20122fc264..7406da3e14 100644
--- a/.github/workflows/docker-detection-testing.yml
+++ b/.github/workflows/docker-detection-testing.yml
@@ -76,6 +76,6 @@ jobs:
             run: |
               cd automated_detection_testing/ci/detection_testing_batch
               source .venv/bin/activate
-              python3 detection_testing_execution.py -b ${{ steps.vars.outputs.branch }} -u 123456789 --splunkbase_username ${{ secrets.SPLUNKBASE_TESTING_USERNAME }} --splunkbase_password ${{ secrets.SPLUNKBASE_TESTING_KEY }} -m selected -tf security_content/detections/endpoint/7zip_commandline_to_smb_share_path.yml -n1
+              python3 detection_testing_execution.py -b ${{ steps.vars.outputs.branch }} -u 123456789 --splunkbase_username ${{ secrets.SPLUNKBASE_TESTING_USERNAME }} --splunkbase_password ${{ secrets.SPLUNKBASE_TESTING_KEY }} -m new -n2
               echo "DONE!"
           

From 0c53c4a9ab354527c5c69080ad6f4df6137aa7bf Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Mon, 1 Nov 2021 14:45:57 -0700
Subject: [PATCH 048/166] Changed to test against a different branch for
 testing purposes

---
 .github/workflows/docker-detection-testing.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/docker-detection-testing.yml b/.github/workflows/docker-detection-testing.yml
index 7406da3e14..22d75da290 100644
--- a/.github/workflows/docker-detection-testing.yml
+++ b/.github/workflows/docker-detection-testing.yml
@@ -76,6 +76,7 @@ jobs:
             run: |
               cd automated_detection_testing/ci/detection_testing_batch
               source .venv/bin/activate
-              python3 detection_testing_execution.py -b ${{ steps.vars.outputs.branch }} -u 123456789 --splunkbase_username ${{ secrets.SPLUNKBASE_TESTING_USERNAME }} --splunkbase_password ${{ secrets.SPLUNKBASE_TESTING_KEY }} -m new -n2
+              #python3 detection_testing_execution.py -b ${{ steps.vars.outputs.branch }} -u 123456789 --splunkbase_username ${{ secrets.SPLUNKBASE_TESTING_USERNAME }} --splunkbase_password ${{ secrets.SPLUNKBASE_TESTING_KEY }} -m new -n2
+              python3 detection_testing_execution.py -b RegistryDetectionFixes -u 123456789 --splunkbase_username ${{ secrets.SPLUNKBASE_TESTING_USERNAME }} --splunkbase_password ${{ secrets.SPLUNKBASE_TESTING_KEY }} -m new -n2
               echo "DONE!"
           

From aadb588fff04b50e99fb71047e85f45a85cb2b64 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Mon, 1 Nov 2021 17:23:22 -0700
Subject: [PATCH 049/166] Changes to generate JSON artifacts as well as CSV and
 upload them after a test run on GH Actions.

---
 .../workflows/docker-detection-testing.yml    | 15 ++++-
 .../detection_testing_execution.py            | 61 ++++++++++++++++---
 .../modules/github_service.py                 |  5 +-
 3 files changed, 69 insertions(+), 12 deletions(-)

diff --git a/.github/workflows/docker-detection-testing.yml b/.github/workflows/docker-detection-testing.yml
index 22d75da290..d8a17d22f6 100644
--- a/.github/workflows/docker-detection-testing.yml
+++ b/.github/workflows/docker-detection-testing.yml
@@ -77,6 +77,19 @@ jobs:
               cd automated_detection_testing/ci/detection_testing_batch
               source .venv/bin/activate
               #python3 detection_testing_execution.py -b ${{ steps.vars.outputs.branch }} -u 123456789 --splunkbase_username ${{ secrets.SPLUNKBASE_TESTING_USERNAME }} --splunkbase_password ${{ secrets.SPLUNKBASE_TESTING_KEY }} -m new -n2
-              python3 detection_testing_execution.py -b RegistryDetectionFixes -u 123456789 --splunkbase_username ${{ secrets.SPLUNKBASE_TESTING_USERNAME }} --splunkbase_password ${{ secrets.SPLUNKBASE_TESTING_KEY }} -m new -n2
+              python3 detection_testing_execution.py -b RegistryDetectionFixes -u 123456789 --splunkbase_username ${{ secrets.SPLUNKBASE_TESTING_USERNAME }} --splunkbase_password ${{ secrets.SPLUNKBASE_TESTING_KEY }} -m new -n1
               echo "DONE!"
           
+          - name: Upload Test Results Files
+            uses: actions/upload-artifact@v2
+            with:
+              name: testing-results-files
+              path: |
+                automated_detection_testing/ci/detection_testing_batch/success.csv
+                automated_detection_testing/ci/detection_testing_batch/error.csv
+                automated_detection_testing/ci/detection_testing_batch/failure.csv
+                automated_detection_testing/ci/detection_testing_batch/combined.csv
+                automated_detection_testing/ci/detection_testing_batch/success.json
+                automated_detection_testing/ci/detection_testing_batch/error.json
+                automated_detection_testing/ci/detection_testing_batch/failure.json
+                automated_detection_testing/ci/detection_testing_batch/combined.json
\ No newline at end of file
diff --git a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
index 93e80fe3a0..9a8bdd341b 100644
--- a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
+++ b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
@@ -26,7 +26,7 @@ from tempfile import mkdtemp
 import csv
 
 from requests import get
-
+import json
 
 SPLUNK_CONTAINER_APPS_DIR = "/opt/splunk/etc/apps"
 index_file_local_path = "indexes.conf.tar"
@@ -389,6 +389,10 @@ def main(args):
         print("Error - Failed to read in detection files: [%s].\nQuitting..."%(str(e)))    
         sys.exit(1)
     
+    if len(test_files) == 0:
+        print("No files were found to be tested.  Returning an error (should this return success?).\n\tQuitting...")
+        sys.exit(1)
+    
     
     
     #Go into the security content directory
@@ -541,7 +545,8 @@ def main(args):
     for key, value in APPS_DICT.items():
         if value['location'] == 'splunkbase':
             #The app is on Splunkbase
-            SPLUNK_APPS.append(SPLUNKBASE_URL%(value['app_number'],value['app_version']))
+            target=SPLUNKBASE_URL%(value['app_number'],value['app_version'])
+            SPLUNK_APPS.append(target)
         else:
             #The app is a file we generated locally
             SPLUNK_APPS.append(value['location'])
@@ -627,17 +632,20 @@ def main(args):
     
     #Remove the attack data and 
     #generate all of the output information
+    stop_time = timer()
+    stop_datetime = datetime.now()
     baseline = OrderedDict()
     baseline['SPLUNK_VERSION'] = full_docker_hub_container_name
-    baseline['SPLUNK_APPS'] = ','.join(SPLUNK_APPS)
+    baseline['SPLUNK_APPS'] = APPS_DICT
     baseline['TEST_START_TIME'] = str(start_datetime)
+    baseline['TEST_FINISH_TIME'] = str(stop_datetime)
 
     results_tracker.finish(baseline)
     
     
 
     #now we are done!
-    stop_time = timer()
+    
     print("Total Execution Time: [%s]"%(timedelta(seconds=stop_time - start_time, microseconds=0)))
     
 
@@ -715,8 +723,11 @@ class SynchronizedResultsTracker:
             self.errors.append(detection)
         finally:
             self.lock.release()
-    def outputResultsFile(self, field_names:list[str], output_filename:str, data:list[dict], baseline:OrderedDict)->bool:
+    
+
+    def outputResultsCSV(self, field_names:list[str], output_filename:str, data:list[dict], baseline:OrderedDict)->bool:
         success = True
+
         print("Generating %s..."%(output_filename), end='')
         self.lock.acquire()
         
@@ -724,7 +735,19 @@ class SynchronizedResultsTracker:
             with open(output_filename, 'w') as csvfile:
                 header_writer = csv.writer(csvfile, quoting=csv.QUOTE_ALL)
                 for key in baseline:
-                    header_writer.writerow([key, baseline[key]])
+                    #Very basic support for pretty pritning dicts. Doesn't handle more than 1 nested dict
+                    if type(baseline[key]) is OrderedDict:
+                        header_writer.writerow([key, "-"])
+                        for nestedkey in baseline[key]:
+                            header_writer.writerow([nestedkey, baseline[key][nestedkey]])
+                    #Basic support for 1 layer nested list. Doesn't handle more than 1.
+                    elif type(baseline[key]) is list and len(baseline[key])>0:
+                        header_writer.writerow([key, baseline[key][0]])
+                        for i in range(1,len(baseline[key])):
+                            header_writer.writerow(['-', baseline[key][i]])
+                        
+                    else:
+                        header_writer.writerow([key, baseline[key]])
                 header_writer.writerow(['',''])
                 csv_writer = csv.DictWriter(csvfile, fieldnames=field_names)
                 csv_writer.writeheader()
@@ -741,11 +764,29 @@ class SynchronizedResultsTracker:
 
         return success
 
+    def outputResultsJSON(self, field_names:list[str], output_filename:str, data:list[dict], baseline:OrderedDict)->bool:
+        success = True
+        try:
+            with open(output_filename, "w") as jsonFile:
+                json.dump({'baseline': baseline, 'results':data}, jsonFile, indent="   ")
+        except Exception as e:
+            print("There was an error generating [%s]: [%s]"%(output_filename, str(e)))
+            success = False
+        return success
+    def outputResultsFile(self, field_names:list[str], output_filename:str, data:list[dict], baseline:OrderedDict, output_json:bool=True, output_csv:bool=True)->bool:
+        success = True
+        if output_csv:
+            success |= self.outputResultsCSV(field_names, output_filename + ".csv", data, baseline)
+        if output_json:
+            success |= self.outputResultsJSON(field_names, output_filename + ".json", data, baseline)
+        return success
+        
+
     def outputResultsFiles(self, baseline:OrderedDict, fields:list[str]=['detection_name', 'detection_file','runDuration','diskUsage', 'search_string', 'error', 'success', 'scanCount', 'detection_error'])->bool:
-        res = self.outputResultsFile(fields,"success.csv", self.successes, baseline)
-        res |= self.outputResultsFile(fields, "failures.csv", self.failures, baseline)
-        res |= self.outputResultsFile(fields, "errors.csv", self.errors, baseline)
-        res |= self.outputResultsFile(fields, "combined.csv", self.successes + self.failures + self.errors, baseline)
+        res = self.outputResultsFile(fields,"success", self.successes, baseline)
+        res |= self.outputResultsFile(fields, "failure", self.failures, baseline)
+        res |= self.outputResultsFile(fields, "error", self.errors, baseline)
+        res |= self.outputResultsFile(fields, "combined", self.successes + self.failures + self.errors, baseline)
         return res
 
     def finish(self, baseline:OrderedDict):
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/github_service.py b/automated_detection_testing/ci/detection_testing_batch/modules/github_service.py
index 4559640228..eba0fae6f8 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/github_service.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/github_service.py
@@ -118,7 +118,7 @@ class GithubService:
         return filenames
 
 
-    def get_changed_test_files(self, folders=['endpoint', 'cloud', 'network'], ttps_to_test=["Anomaly","Hunting","TTP"], previously_successful_tests=[]):
+    def get_changed_test_files(self, folders=['endpoint', 'cloud', 'network'], ttps_to_test=["Anomaly","Hunting","TTP"], previously_successful_tests=[])->list[str]:
         branch1 = self.security_content_branch
         branch2 = 'develop'
         g = git.Git('security_content')
@@ -137,6 +137,9 @@ class GithubService:
                     # changed detections
                     if 'detections' in file_path and os.path.basename(file_path).endswith('.yml'):
                             changed_detection_files.append(file_path)
+        else:
+            print("Looking for changed detections by diffing [%s] against [%s].  Of course none were returned."%(branch1, branch2))
+            return []
                             
 
         #all files have the format A\tFILENAME or M\tFILENAME.  Get rid of those leading characters

From 79833fccd036210940f0c2c35b32e27bcda37a82 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Mon, 1 Nov 2021 18:10:25 -0700
Subject: [PATCH 050/166] Fixed how OrderedDict is imported to hopefully fix
 CSV output format.

---
 .../ci/detection_testing_batch/detection_testing_execution.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
index 9a8bdd341b..3b96cc8329 100644
--- a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
+++ b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
@@ -20,8 +20,8 @@ from datetime import timedelta
 from datetime import datetime
 import string
 import shutil
-from typing import OrderedDict, Union
-
+from typing import Union
+from collections import OrderedDict
 from tempfile import mkdtemp
 import csv
 

From 8c501e789da92d9a037b60af126b74644cc7df65 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Wed, 3 Nov 2021 16:18:41 -0700
Subject: [PATCH 051/166] Distribute testing amongst multiple containers.

---
 .../workflows/docker-detection-testing.yml    |  86 +++++++-
 .../detection_testing_execution.py            | 203 +++++++++++-------
 2 files changed, 204 insertions(+), 85 deletions(-)

diff --git a/.github/workflows/docker-detection-testing.yml b/.github/workflows/docker-detection-testing.yml
index d8a17d22f6..d1c57204fc 100644
--- a/.github/workflows/docker-detection-testing.yml
+++ b/.github/workflows/docker-detection-testing.yml
@@ -36,7 +36,7 @@ jobs:
           echo "No, this is not a dependabot run!"
 
 
-  docker-detection-testing:
+  docker-detection-testing-setup:
       runs-on: ubuntu-latest
       needs: [validate-tag-if-present, quit-for-dependabot]
       steps:  
@@ -77,7 +77,7 @@ jobs:
               cd automated_detection_testing/ci/detection_testing_batch
               source .venv/bin/activate
               #python3 detection_testing_execution.py -b ${{ steps.vars.outputs.branch }} -u 123456789 --splunkbase_username ${{ secrets.SPLUNKBASE_TESTING_USERNAME }} --splunkbase_password ${{ secrets.SPLUNKBASE_TESTING_KEY }} -m new -n2
-              python3 detection_testing_execution.py -b RegistryDetectionFixes -u 123456789 --splunkbase_username ${{ secrets.SPLUNKBASE_TESTING_USERNAME }} --splunkbase_password ${{ secrets.SPLUNKBASE_TESTING_KEY }} -m new -n1
+              python3 detection_testing_execution.py -b RegistryDetectionFixes -u 123456789 --splunkbase_username ${{ secrets.SPLUNKBASE_TESTING_USERNAME }} --splunkbase_password ${{ secrets.SPLUNKBASE_TESTING_KEY }} -m new -n5 -split
               echo "DONE!"
           
           - name: Upload Test Results Files
@@ -85,11 +85,77 @@ jobs:
             with:
               name: testing-results-files
               path: |
-                automated_detection_testing/ci/detection_testing_batch/success.csv
-                automated_detection_testing/ci/detection_testing_batch/error.csv
-                automated_detection_testing/ci/detection_testing_batch/failure.csv
-                automated_detection_testing/ci/detection_testing_batch/combined.csv
-                automated_detection_testing/ci/detection_testing_batch/success.json
-                automated_detection_testing/ci/detection_testing_batch/error.json
-                automated_detection_testing/ci/detection_testing_batch/failure.json
-                automated_detection_testing/ci/detection_testing_batch/combined.json
\ No newline at end of file
+                automated_detection_testing/ci/detection_testing_batch/apps/DA-ESS-ContentUpdate-latest.tar.gz
+                automated_detection_testing/ci/detection_testing_batch/container_0_tests.txt
+                automated_detection_testing/ci/detection_testing_batch/container_1_tests.txt
+                automated_detection_testing/ci/detection_testing_batch/container_2_tests.txt
+                automated_detection_testing/ci/detection_testing_batch/container_3_tests.txt
+                automated_detection_testing/ci/detection_testing_batch/container_4_tests.txt
+
+
+  docker-detection-testing-execution:
+    runs-on: ubuntu-latest
+    needs: [validate-tag-if-present, quit-for-dependabot, docker-detection-testing-setup]
+    strategy:
+      matrix:
+        test_filename: ["container_0_tests.txt", "container_1_tests.txt", "container_2_tests.txt", "container_3_tests.txt", "container_4_tests.txt"]
+    steps:  
+        - name: Get branch and PR required for detection testing main.py
+          id: vars
+          run: |
+            echo "::set-output name=branch::${GITHUB_REF#refs/heads/}"
+
+        - name: Checkout Repo
+          uses: actions/checkout@v2
+        
+        - name: Download artifacts
+          uses: actions/download-artifact@v2
+          with:
+            name: testing-results-files
+            path: automated_detection_testing/ci/detection_testing_batch
+        - name: Install Docker 
+          run: |
+            sudo apt update -qq
+            
+
+            #python2.7 needed for slim, for now
+            sudo apt install python2
+            sudo apt install virtualenv 
+            curl https://bootstrap.pypa.io/pip/2.7/get-pip.py --output get-pip.py
+            sudo python2.7 get-pip.py
+
+        - uses: actions/setup-python@v2
+          with:
+            python-version: '3.9' #Available versions here - https://github.com/actions/python-versions/releases  easy to change/make a matrix/use pypy
+            architecture: 'x64' # optional x64 or x86. Defaults to x64 if not specified          
+        
+        - name: Install Python Dependencies
+          run: |
+            cd automated_detection_testing/ci/detection_testing_batch
+            python3 -m venv .venv
+            source .venv/bin/activate
+            python3 -m pip install wheel
+            python3 -m pip install -r requirements.txt
+        
+        - name: Run the CI
+          run: |
+            cd automated_detection_testing/ci/detection_testing_batch
+            source .venv/bin/activate
+            #python3 detection_testing_execution.py -b ${{ steps.vars.outputs.branch }} -u 123456789 --splunkbase_username ${{ secrets.SPLUNKBASE_TESTING_USERNAME }} --splunkbase_password ${{ secrets.SPLUNKBASE_TESTING_KEY }} -m new -n2
+            python3 detection_testing_execution.py -b RegistryDetectionFixes -u 123456789 --splunkbase_username ${{ secrets.SPLUNKBASE_TESTING_USERNAME }} --splunkbase_password ${{ secrets.SPLUNKBASE_TESTING_KEY }} -m selected -tff ${{ matrix.test_filename}} -n1 -e DA-ESS-ContentUpdate-latest.tar.gz
+                    
+        
+        - name: Upload Test Results Files
+          uses: actions/upload-artifact@v2
+          with:
+            name: testing-results-files
+            path: |
+              automated_detection_testing/ci/detection_testing_batch/success.csv
+              automated_detection_testing/ci/detection_testing_batch/error.csv
+              automated_detection_testing/ci/detection_testing_batch/failure.csv
+              automated_detection_testing/ci/detection_testing_batch/combined.csv
+              automated_detection_testing/ci/detection_testing_batch/success.json
+              automated_detection_testing/ci/detection_testing_batch/error.json
+              automated_detection_testing/ci/detection_testing_batch/failure.json
+              automated_detection_testing/ci/detection_testing_batch/combined.json
+              
\ No newline at end of file
diff --git a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
index 3b96cc8329..5bb27358f1 100644
--- a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
+++ b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
@@ -225,13 +225,19 @@ def main(args):
     parser.add_argument("-user", "--splunkbase_username", type=str, required=False, help="Splunkbase username for downloading Splunkbase apps")
     parser.add_argument("-pw", "--splunkbase_password", type=str, required=False, help="Splunkbase password for downloading Splunkbase apps")
     parser.add_argument("-m", "--mode", type=str, choices=DETECTION_MODES, required=False, help="Whether to test new detections, specific detections, or all detections", default="new")
-    parser.add_argument("-tf","--test_files", type=str, required=False, help="The names of files that you want to test, separated by commas.")
+    parser.add_argument("-tfl","--test_files_list", type=str, required=False, help="The names of files that you want to test, separated by commas.")
+    parser.add_argument("-tff","--test_files_file", type=argparse.FileType('r'), required=False, help="A file containing a list of detections to run, one per line")
+    parser.add_argument("-e","--escu_package", type=argparse.FileType('rb'), required=False, help="The ESCU file to use - will not generate a new ESCU package")
 
     parser.add_argument("-t", "--types", type=str, required=False, help="Detection types to test. Can be one of more of %s"%(str(DETECTION_TYPES)), default=','.join(DETECTION_TYPES))
     parser.add_argument("-ct", "--container_tag", type=str, required=False, help="The tag of the Splunk Container to use.  Tags are located at https://hub.docker.com/r/splunk/splunk/tags",default=DEFAULT_CONTAINER_TAG)
     parser.add_argument("-p", "--persist_security_content", required=False, default=False, action="store_true", help="Assumes security_content directory already exists.  Don't check it out and overwrite it again.  Saves "\
                                                                                                                      "time and allows you to test a detection that you've updated.  Runs generate again in case you have "\
                                                                                                                      "updated macros or anything else.  Especially useful for quick, local, iterative testing.")
+    
+
+    parser.add_argument("-split","--split_detections_then_stop", required=False, default=False, action='store_true', help="Should existing images be re-used, or should they be redownloaded?")
+
     start_datetime = datetime.now()
     import requests.packages.urllib3
     requests.packages.urllib3.disable_warnings()
@@ -249,7 +255,7 @@ def main(args):
     interactive_failure = args.interactive_failure
     show_password = args.show_password
     splunk_password = args.container_password
-    
+    pregenerated_escu_package = args.escu_package
     
     
         
@@ -289,9 +295,17 @@ def main(args):
 
     #Ensure that a valid mode was chosen 
     mode = args.mode
-    if mode == "selected" and args.test_files == None:
+    if mode == "selected" and args.test_files_list is None and args.test_files_file is None:
         print("Error - mode [%s] but did not provide any files to test.\nQuitting..."%(mode))
-    
+        sys.exit(1)
+    elif mode == "selected" and args.test_files_list is not None and args.test_files_file is not None:
+        print("Error - mode [%s] but you specified a list of detections to test AND a file of detections to test.\nQuitting..."%(mode))
+        sys.exit(1)
+    elif mode == "selected" and args.test_files_list is not None:
+        command_line_files_to_test = [name.strip() for name in args.test_files_list.split(',')]
+    elif mode == "selected" and args.test_files_file is not None:
+        lines = args.test_files_file.readlines()
+        command_line_files_to_test = [l.strip() for l in lines]
 
     folders = [a.strip() for a in args.types.split(',')]
     for t in folders:
@@ -345,6 +359,7 @@ def main(args):
 
 
     
+    
 
     if persist_security_content is True and os.path.exists("security_content"):
         print("******You chose --persist_security_content and the security_content directory exists. We will not check out the repo again. Please be aware, this could cause issues if you're out of date.******")
@@ -378,8 +393,8 @@ def main(args):
         elif mode == "selected":
             if set(folders) != set(DETECTION_TYPES):
                 print("You specified mode [%s] but also types: [%s]. We will ignore type restrictions and test all specified files"%(mode,str(folders)))
-            files_to_test = [name.strip() for name in args.test_files.split(',')]
-            test_files = github_service.get_selected_test_files(files_to_test,
+            
+            test_files = github_service.get_selected_test_files(command_line_files_to_test,
                                                             previously_successful_tests=success_tests)
 
         else:
@@ -393,85 +408,124 @@ def main(args):
         print("No files were found to be tested.  Returning an error (should this return success?).\n\tQuitting...")
         sys.exit(1)
     
-    
-    
-    #Go into the security content directory
-    print("****GENERATE NEW CONTENT****")
-    os.chdir("security_content")
-    print(os.getcwd())
-    if persist_security_content is False:
-        commands = ["python3 -m venv .venv", 
-                    ". ./.venv/bin/activate", 
-                    "python3 -m pip install wheel", 
-                    "python3 -m pip install -r requirements.txt", 
-                    "python3 contentctl.py --path . --verbose generate --product ESCU --output dist/escu", "tar -czf DA-ESS-ContentUpdate.spl -C dist/escu ."]
-    else:
-        commands = ["s. ./.venv/bin/activate", "python3 contentctl.py --path . --verbose generate --product ESCU --output dist/escu", "tar -czf DA-ESS-ContentUpdate.spl -C dist/escu ."]
-    ret = subprocess.run("; ".join(commands), shell=True, capture_output=True)
-    if ret.returncode != 0:
-        print("Error generating new content.  Exiting...")
-        sys.exit(1)
-    print("New content generated successfully")    
-    
-
-    print("Generate new ESCU Package using new content")
-    if persist_security_content is True:
-        os.chdir("slim_packaging")
-        commands = ["cd slim-latest", 
-                    ". ./.venv/bin/activate",
-                    "cp -R ../../dist/escu DA-ESS-ContentUpdate",
-                    "slim package -o upload DA-ESS-ContentUpdate",
-                    "cp upload/DA-ESS-ContentUpdate*.tar.gz ../apps/DA-ESS-ContentUpdate-latest.tar.gz"]
-    
-    else:
-        os.mkdir("slim_packaging")
-        os.chdir("slim_packaging")
+    local_volume_path = os.path.join(os.getcwd(), "apps")
+    try:
         os.mkdir("apps")
+    except FileExistsError as e:
+        #Directory already exists, do nothing
+        pass
+    except Exception as e:
+        print("Caught an error when copying the ESCU package [%s] to the apps folder [%s].\n\tQuitting..."%(pregenerated_escu_package.name, local_volume_path))
+        sys.exit(1)
+
+    if pregenerated_escu_package is None:
+        #Go into the security content directory
+        print("****GENERATE NEW CONTENT****")
+        os.chdir("security_content")
+        print(os.getcwd())
+        if persist_security_content is False:
+            commands = ["python3 -m venv .venv", 
+                        ". ./.venv/bin/activate", 
+                        "python3 -m pip install wheel", 
+                        "python3 -m pip install -r requirements.txt", 
+                        "python3 contentctl.py --path . --verbose generate --product ESCU --output dist/escu", "tar -czf DA-ESS-ContentUpdate.spl -C dist/escu ."]
+        else:
+            commands = ["s. ./.venv/bin/activate", "python3 contentctl.py --path . --verbose generate --product ESCU --output dist/escu", "tar -czf DA-ESS-ContentUpdate.spl -C dist/escu ."]
+        ret = subprocess.run("; ".join(commands), shell=True, capture_output=True)
+        if ret.returncode != 0:
+            print("Error generating new content.  Exiting...")
+            sys.exit(1)
+        print("New content generated successfully")    
         
-        try:
-            SPLUNK_PACKAGING_TOOLKIT_URL = "https://download.splunk.com/misc/packaging-toolkit/splunk-packaging-toolkit-0.9.0.tar.gz"
-            SPLUNK_PACKAGING_TOOLKIT_FILENAME = 'splunk-packaging-toolkit-latest.tar.gz'
-            print("Downloading the Splunk Packaging Toolkit from %s..."%(SPLUNK_PACKAGING_TOOLKIT_URL), end='')
-            response = get(SPLUNK_PACKAGING_TOOLKIT_URL)
-            response.raise_for_status()
-            with open(SPLUNK_PACKAGING_TOOLKIT_FILENAME, 'wb') as slim_file:
-                slim_file.write(response.content)
+
+        print("Generate new ESCU Package using new content")
+        if persist_security_content is True:
+            os.chdir("slim_packaging")
+            commands = ["cd slim-latest", 
+                        ". ./.venv/bin/activate",
+                        "cp -R ../../dist/escu DA-ESS-ContentUpdate",
+                        "slim package -o upload DA-ESS-ContentUpdate",
+                         "cp upload/DA-ESS-ContentUpdate*.tar.gz %s"%(os.path.join(local_volume_path, "DA-ESS-ContentUpdate-latest.tar.gz" ))]
+        
+        else:
+            os.mkdir("slim_packaging")
+            os.chdir("slim_packaging")
+            os.mkdir("apps")
             
-            print("success")
+            try:
+                SPLUNK_PACKAGING_TOOLKIT_URL = "https://download.splunk.com/misc/packaging-toolkit/splunk-packaging-toolkit-0.9.0.tar.gz"
+                SPLUNK_PACKAGING_TOOLKIT_FILENAME = 'splunk-packaging-toolkit-latest.tar.gz'
+                print("Downloading the Splunk Packaging Toolkit from %s..."%(SPLUNK_PACKAGING_TOOLKIT_URL), end='')
+                response = get(SPLUNK_PACKAGING_TOOLKIT_URL)
+                response.raise_for_status()
+                with open(SPLUNK_PACKAGING_TOOLKIT_FILENAME, 'wb') as slim_file:
+                    slim_file.write(response.content)
+                
+                print("success")
 
 
+            except Exception as e:
+                print("FAILED")
+                print("Error downloading the Splunk Packaging Toolkit: [%s]"%(str(e)))
+                sys.exit(1)
+            
+            
+            commands = ["rm -rf slim-latest",
+                        "mkdir slim-latest", 
+                        "tar -zxf splunk-packaging-toolkit-latest.tar.gz -C slim-latest --strip-components=1",
+                        "cd slim-latest", 
+                        "virtualenv --python=/usr/bin/python2.7 --clear .venv",
+                        ". ./.venv/bin/activate",
+                        "python3 -m pip install --upgrade pip",
+                        "python2 -m pip install wheel",
+                        "python2 -m pip install semantic_version",
+                        "python2 -m pip install .",
+                        "cp -R ../../dist/escu DA-ESS-ContentUpdate",
+                        "slim package -o upload DA-ESS-ContentUpdate",
+                        "cp upload/DA-ESS-ContentUpdate*.tar.gz %s"%(os.path.join(local_volume_path, "DA-ESS-ContentUpdate-latest.tar.gz" ))]
+            
+        
+        
+
+        ret = subprocess.run("; ".join(commands), shell=True, capture_output=True)
+        if ret.returncode != 0:
+            print("Error generating new ESCU Package.\n\tQuitting..."%())
+            sys.exit(1)
+        os.chdir("../..")
+        print("New ESCU Package generated successfully")
+    else:
+        print("Using previous generated ESCU package: [%s]"%(pregenerated_escu_package.name))    
+        try:
+            with open(os.path.join(local_volume_path, os.path.basename(pregenerated_escu_package.name)),'wb') as escu_package:
+                escu_package.write(pregenerated_escu_package.read())
         except Exception as e:
-            print("FAILED")
-            print("Error downloading the Splunk Packaging Toolkit: [%s]"%(str(e)))
+            print("Failure writing the ESCU package [%s] to [%s]: [%s].\n\tQuitting..."%(pregenerated_escu_package.name, local_volume_path, str(e)))
             sys.exit(1)
         
-        
-        commands = ["rm -rf slim-latest",
-                    "mkdir slim-latest", 
-                    "tar -zxf splunk-packaging-toolkit-latest.tar.gz -C slim-latest --strip-components=1",
-                    "cd slim-latest", 
-                    "virtualenv --python=/usr/bin/python2.7 --clear .venv",
-                    ". ./.venv/bin/activate",
-                    "python3 -m pip install --upgrade pip",
-                    "python2 -m pip install wheel",
-                    "python2 -m pip install semantic_version",
-                    "python2 -m pip install .",
-                    "cp -R ../../dist/escu DA-ESS-ContentUpdate",
-                    "slim package -o upload DA-ESS-ContentUpdate",
-                    "cp upload/DA-ESS-ContentUpdate*.tar.gz ../apps/DA-ESS-ContentUpdate-latest.tar.gz"]
+    
+    print("Wrote ESCU package to volume.")
         
     
     
+    if args.split_detections_then_stop:
+        for output_file_index in range(0,num_containers):
+            fname = "container_%d_tests.txt"%(output_file_index)
+            print("Writing tests to [%s]..."%(fname), end='')
+            with open(fname, "w") as output_file:
+                detection_tests = test_files[output_file_index::num_containers]
+                normalized_detection_names = []
+                for d in detection_tests:
+                    filename = os.path.basename(d)
+                    filename = filename.replace(".test.yml", ".yml")
+                    leading = os.path.split(d)[0]
+                    leading = leading.replace("tests/", "detections/")
+                    new_name = os.path.join("security_content", leading, filename)
+                    normalized_detection_names.append(new_name)
+                output_file.write('\n'.join(normalized_detection_names))
+            print("Done", end='')
+        sys.exit(0)
+
 
-    ret = subprocess.run("; ".join(commands), shell=True, capture_output=True)
-    if ret.returncode != 0:
-        print("Error generating new ESCU Package.\n\tQuitting..."%())
-        sys.exit(1)
-    os.chdir("../..")
-    print("New ESCU Package generated successfully")    
-    
-    
-    
 
 
     
@@ -486,7 +540,7 @@ def main(args):
 
     results_tracker = SynchronizedResultsTracker(test_files, num_containers)
     
-    local_volume_path = os.path.join(os.getcwd(), "security_content", "slim_packaging","apps")
+    
 
           
     
@@ -509,7 +563,6 @@ def main(args):
     
     
     try:
-        raise
         BETA_SPLUNK_ADD_ON_FOR_SYSMON_PATH  = os.path.expanduser("~/Downloads/Splunk_TA_microsoft_sysmon-1.0.2-B1.spl")    
         shutil.copyfile(BETA_SPLUNK_ADD_ON_FOR_SYSMON_PATH, os.path.join(local_volume_path, "Splunk_TA_microsoft_sysmon-1.0.2-B1.spl"))
     

From c4c1db1f2cf75b77df05ccd09f6b7fbcb0ef13de Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Wed, 3 Nov 2021 16:30:02 -0700
Subject: [PATCH 052/166] Fixed up the pathing related to the ESCU Package.

---
 .github/workflows/docker-detection-testing.yml            | 8 ++++----
 .../detection_testing_execution.py                        | 2 +-
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/docker-detection-testing.yml b/.github/workflows/docker-detection-testing.yml
index d1c57204fc..3eb30e0540 100644
--- a/.github/workflows/docker-detection-testing.yml
+++ b/.github/workflows/docker-detection-testing.yml
@@ -83,7 +83,7 @@ jobs:
           - name: Upload Test Results Files
             uses: actions/upload-artifact@v2
             with:
-              name: testing-results-files
+              name: testing-results-config
               path: |
                 automated_detection_testing/ci/detection_testing_batch/apps/DA-ESS-ContentUpdate-latest.tar.gz
                 automated_detection_testing/ci/detection_testing_batch/container_0_tests.txt
@@ -111,8 +111,8 @@ jobs:
         - name: Download artifacts
           uses: actions/download-artifact@v2
           with:
-            name: testing-results-files
-            path: automated_detection_testing/ci/detection_testing_batch
+            name: testing-results-config
+            path: automated_detection_testing/ci/detection_testing_batch/prior_config
         - name: Install Docker 
           run: |
             sudo apt update -qq
@@ -142,7 +142,7 @@ jobs:
             cd automated_detection_testing/ci/detection_testing_batch
             source .venv/bin/activate
             #python3 detection_testing_execution.py -b ${{ steps.vars.outputs.branch }} -u 123456789 --splunkbase_username ${{ secrets.SPLUNKBASE_TESTING_USERNAME }} --splunkbase_password ${{ secrets.SPLUNKBASE_TESTING_KEY }} -m new -n2
-            python3 detection_testing_execution.py -b RegistryDetectionFixes -u 123456789 --splunkbase_username ${{ secrets.SPLUNKBASE_TESTING_USERNAME }} --splunkbase_password ${{ secrets.SPLUNKBASE_TESTING_KEY }} -m selected -tff ${{ matrix.test_filename}} -n1 -e DA-ESS-ContentUpdate-latest.tar.gz
+            python3 detection_testing_execution.py -b RegistryDetectionFixes -u 123456789 --splunkbase_username ${{ secrets.SPLUNKBASE_TESTING_USERNAME }} --splunkbase_password ${{ secrets.SPLUNKBASE_TESTING_KEY }} -m selected -tff prior_config/${{ matrix.test_filename}} -n1 -e prior_config/apps/DA-ESS-ContentUpdate-latest.tar.gz
                     
         
         - name: Upload Test Results Files
diff --git a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
index 5bb27358f1..10deb2ed9e 100644
--- a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
+++ b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
@@ -522,7 +522,7 @@ def main(args):
                     new_name = os.path.join("security_content", leading, filename)
                     normalized_detection_names.append(new_name)
                 output_file.write('\n'.join(normalized_detection_names))
-            print("Done", end='')
+            print("Done")
         sys.exit(0)
 
 

From 6592124a12f861753651dc6afd3c705d9eef8eae Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Wed, 3 Nov 2021 16:49:57 -0700
Subject: [PATCH 053/166] Uploading the artifacts as different names so that
 they don't clobber each other.

---
 .github/workflows/docker-detection-testing.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/docker-detection-testing.yml b/.github/workflows/docker-detection-testing.yml
index 3eb30e0540..4ec66abf06 100644
--- a/.github/workflows/docker-detection-testing.yml
+++ b/.github/workflows/docker-detection-testing.yml
@@ -148,7 +148,7 @@ jobs:
         - name: Upload Test Results Files
           uses: actions/upload-artifact@v2
           with:
-            name: testing-results-files
+            name: ${{ matrix.test_filename}}.results
             path: |
               automated_detection_testing/ci/detection_testing_batch/success.csv
               automated_detection_testing/ci/detection_testing_batch/error.csv

From 522e2d03a650e61c527238920f8e24bd25837b74 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Wed, 3 Nov 2021 19:09:16 -0700
Subject: [PATCH 054/166] Changed the testing script to test everything.  This
 is a test to see how long it takes.

---
 .github/workflows/docker-detection-testing.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/docker-detection-testing.yml b/.github/workflows/docker-detection-testing.yml
index 4ec66abf06..5898369c64 100644
--- a/.github/workflows/docker-detection-testing.yml
+++ b/.github/workflows/docker-detection-testing.yml
@@ -77,7 +77,7 @@ jobs:
               cd automated_detection_testing/ci/detection_testing_batch
               source .venv/bin/activate
               #python3 detection_testing_execution.py -b ${{ steps.vars.outputs.branch }} -u 123456789 --splunkbase_username ${{ secrets.SPLUNKBASE_TESTING_USERNAME }} --splunkbase_password ${{ secrets.SPLUNKBASE_TESTING_KEY }} -m new -n2
-              python3 detection_testing_execution.py -b RegistryDetectionFixes -u 123456789 --splunkbase_username ${{ secrets.SPLUNKBASE_TESTING_USERNAME }} --splunkbase_password ${{ secrets.SPLUNKBASE_TESTING_KEY }} -m new -n5 -split
+              python3 detection_testing_execution.py -b RegistryDetectionFixes -u 123456789 --splunkbase_username ${{ secrets.SPLUNKBASE_TESTING_USERNAME }} --splunkbase_password ${{ secrets.SPLUNKBASE_TESTING_KEY }} -m all -n5 -split
               echo "DONE!"
           
           - name: Upload Test Results Files

From 22656ffe23460649495c1dff588d2441fa4c186a Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Fri, 5 Nov 2021 14:57:46 -0700
Subject: [PATCH 055/166] typo in NEEDS for test script.

---
 .../workflows/docker-detection-testing.yml    | 90 ++++++++++++++++++-
 1 file changed, 87 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/docker-detection-testing.yml b/.github/workflows/docker-detection-testing.yml
index 5898369c64..ae9cf277a4 100644
--- a/.github/workflows/docker-detection-testing.yml
+++ b/.github/workflows/docker-detection-testing.yml
@@ -77,7 +77,7 @@ jobs:
               cd automated_detection_testing/ci/detection_testing_batch
               source .venv/bin/activate
               #python3 detection_testing_execution.py -b ${{ steps.vars.outputs.branch }} -u 123456789 --splunkbase_username ${{ secrets.SPLUNKBASE_TESTING_USERNAME }} --splunkbase_password ${{ secrets.SPLUNKBASE_TESTING_KEY }} -m new -n2
-              python3 detection_testing_execution.py -b RegistryDetectionFixes -u 123456789 --splunkbase_username ${{ secrets.SPLUNKBASE_TESTING_USERNAME }} --splunkbase_password ${{ secrets.SPLUNKBASE_TESTING_KEY }} -m all -n5 -split
+              python3 detection_testing_execution.py -b RegistryDetectionFixes -u 123456789 --splunkbase_username ${{ secrets.SPLUNKBASE_TESTING_USERNAME }} --splunkbase_password ${{ secrets.SPLUNKBASE_TESTING_KEY }} -m new -n10 -split
               echo "DONE!"
           
           - name: Upload Test Results Files
@@ -91,6 +91,11 @@ jobs:
                 automated_detection_testing/ci/detection_testing_batch/container_2_tests.txt
                 automated_detection_testing/ci/detection_testing_batch/container_3_tests.txt
                 automated_detection_testing/ci/detection_testing_batch/container_4_tests.txt
+                automated_detection_testing/ci/detection_testing_batch/container_5_tests.txt
+                automated_detection_testing/ci/detection_testing_batch/container_6_tests.txt
+                automated_detection_testing/ci/detection_testing_batch/container_7_tests.txt
+                automated_detection_testing/ci/detection_testing_batch/container_8_tests.txt
+                automated_detection_testing/ci/detection_testing_batch/container_9_tests.txt
 
 
   docker-detection-testing-execution:
@@ -98,7 +103,16 @@ jobs:
     needs: [validate-tag-if-present, quit-for-dependabot, docker-detection-testing-setup]
     strategy:
       matrix:
-        test_filename: ["container_0_tests.txt", "container_1_tests.txt", "container_2_tests.txt", "container_3_tests.txt", "container_4_tests.txt"]
+        test_filename: ["container_0_tests.txt", 
+                        "container_1_tests.txt", 
+                        "container_2_tests.txt", 
+                        "container_3_tests.txt", 
+                        "container_4_tests.txt", 
+                        "container_5_tests.txt", 
+                        "container_6_tests.txt", 
+                        "container_7_tests.txt", 
+                        "container_8_tests.txt", 
+                        "container_9_tests.txt"]
     steps:  
         - name: Get branch and PR required for detection testing main.py
           id: vars
@@ -158,4 +172,74 @@ jobs:
               automated_detection_testing/ci/detection_testing_batch/error.json
               automated_detection_testing/ci/detection_testing_batch/failure.json
               automated_detection_testing/ci/detection_testing_batch/combined.json
-              
\ No newline at end of file
+          
+  docker-detection-testing-execution-merge-results:
+    runs-on: ubuntu-latest
+    needs: [validate-tag-if-present, quit-for-dependabot, docker-detection-testing-setup, docker-detection-testing-execution]
+ 
+    steps:  
+        - name: Get branch and PR required for detection testing main.py
+          id: vars
+          run: |
+            echo "::set-output name=branch::${GITHUB_REF#refs/heads/}"
+
+        - name: Checkout Repo
+          uses: actions/checkout@v2
+        
+        - name: Download artifacts
+          uses: actions/download-artifact@v2
+          with:
+            name: container_0_tests.txt.results
+            path: automated_detection_testing/ci/detection_testing_batch/results
+        - name: Download artifacts
+          uses: actions/download-artifact@v2
+          with:
+            name: container_1_tests.txt.results
+            path: automated_detection_testing/ci/detection_testing_batch/results
+        - name: Download artifacts
+          uses: actions/download-artifact@v2
+          with:
+            name: container_2_tests.txt.results
+            path: automated_detection_testing/ci/detection_testing_batch/results
+        - name: Download artifacts
+          uses: actions/download-artifact@v2
+          with:
+            name: container_3_tests.txt.results
+            path: automated_detection_testing/ci/detection_testing_batch/results
+        - name: Download artifacts
+          uses: actions/download-artifact@v2
+          with:
+            name: container_4_tests.txt.results
+            path: automated_detection_testing/ci/detection_testing_batch/results
+        - name: Download artifacts
+          uses: actions/download-artifact@v2
+          with:
+            name: container_5_tests.txt.results
+            path: automated_detection_testing/ci/detection_testing_batch/results
+        - name: Download artifacts
+          uses: actions/download-artifact@v2
+          with:
+            name: container_6_tests.txt.results
+            path: automated_detection_testing/ci/detection_testing_batch/results
+        - name: Download artifacts
+          uses: actions/download-artifact@v2
+          with:
+            name: container_7_tests.txt.results
+            path: automated_detection_testing/ci/detection_testing_batch/results
+        - name: Download artifacts
+          uses: actions/download-artifact@v2
+          with:
+            name: container_8_tests.txt.results
+            path: automated_detection_testing/ci/detection_testing_batch/results
+        - name: Download artifacts
+          uses: actions/download-artifact@v2
+          with:
+            name: container_9_tests.txt.results
+            path: automated_detection_testing/ci/detection_testing_batch/results
+        
+        - name: Merge Detections into single File
+          run: |
+            cd automated_detection_testing/ci/detection_testing_batch/results
+            ls -lah
+            echo "DONE!"
+            

From 89b2ac1e4d849fb5e17e92ab512dc47b8c74ac48 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Fri, 5 Nov 2021 15:32:28 -0700
Subject: [PATCH 056/166] Added a script to summarize the runs of multiple
 runners and merge them together. Results are uploaded into a single artifact.

---
 .../workflows/docker-detection-testing.yml    | 37 +++++++++-----
 .../detection_testing_batch/summarize_json.py | 51 +++++++++++++++++++
 2 files changed, 75 insertions(+), 13 deletions(-)
 create mode 100644 automated_detection_testing/ci/detection_testing_batch/summarize_json.py

diff --git a/.github/workflows/docker-detection-testing.yml b/.github/workflows/docker-detection-testing.yml
index ae9cf277a4..ef09efb5a4 100644
--- a/.github/workflows/docker-detection-testing.yml
+++ b/.github/workflows/docker-detection-testing.yml
@@ -190,56 +190,67 @@ jobs:
           uses: actions/download-artifact@v2
           with:
             name: container_0_tests.txt.results
-            path: automated_detection_testing/ci/detection_testing_batch/results
+            path: automated_detection_testing/ci/detection_testing_batch/results_0
         - name: Download artifacts
           uses: actions/download-artifact@v2
           with:
             name: container_1_tests.txt.results
-            path: automated_detection_testing/ci/detection_testing_batch/results
+            path: automated_detection_testing/ci/detection_testing_batch/results_1
         - name: Download artifacts
           uses: actions/download-artifact@v2
           with:
             name: container_2_tests.txt.results
-            path: automated_detection_testing/ci/detection_testing_batch/results
+            path: automated_detection_testing/ci/detection_testing_batch/results_2
         - name: Download artifacts
           uses: actions/download-artifact@v2
           with:
             name: container_3_tests.txt.results
-            path: automated_detection_testing/ci/detection_testing_batch/results
+            path: automated_detection_testing/ci/detection_testing_batch/results_3
         - name: Download artifacts
           uses: actions/download-artifact@v2
           with:
             name: container_4_tests.txt.results
-            path: automated_detection_testing/ci/detection_testing_batch/results
+            path: automated_detection_testing/ci/detection_testing_batch/results_4
         - name: Download artifacts
           uses: actions/download-artifact@v2
           with:
             name: container_5_tests.txt.results
-            path: automated_detection_testing/ci/detection_testing_batch/results
+            path: automated_detection_testing/ci/detection_testing_batch/results_5
         - name: Download artifacts
           uses: actions/download-artifact@v2
           with:
             name: container_6_tests.txt.results
-            path: automated_detection_testing/ci/detection_testing_batch/results
+            path: automated_detection_testing/ci/detection_testing_batch/results_6
         - name: Download artifacts
           uses: actions/download-artifact@v2
           with:
             name: container_7_tests.txt.results
-            path: automated_detection_testing/ci/detection_testing_batch/results
+            path: automated_detection_testing/ci/detection_testing_batch/results_7
         - name: Download artifacts
           uses: actions/download-artifact@v2
           with:
             name: container_8_tests.txt.results
-            path: automated_detection_testing/ci/detection_testing_batch/results
+            path: automated_detection_testing/ci/detection_testing_batch/results_8
         - name: Download artifacts
           uses: actions/download-artifact@v2
           with:
             name: container_9_tests.txt.results
-            path: automated_detection_testing/ci/detection_testing_batch/results
+            path: automated_detection_testing/ci/detection_testing_batch/results_9
         
+        - uses: actions/setup-python@v2
+          with:
+            python-version: '3.9' #Available versions here - https://github.com/actions/python-versions/releases  easy to change/make a matrix/use pypy
+            architecture: 'x64' # optional x64 or x86. Defaults to x64 if not specified    
+
         - name: Merge Detections into single File
           run: |
-            cd automated_detection_testing/ci/detection_testing_batch/results
-            ls -lah
-            echo "DONE!"
+            cd automated_detection_testing/ci/detection_testing_batch
+            python summarize_json.py --files results_*/combined.json --output_filename summary_test_results.json
             
+            
+        - name: Upload Summary Test Results JSON
+          uses: actions/upload-artifact@v2
+          with:
+            name: SummaryTestResults
+            path: |
+              automated_detection_testing/ci/detection_testing_batch/summary_test_results.json
\ No newline at end of file
diff --git a/automated_detection_testing/ci/detection_testing_batch/summarize_json.py b/automated_detection_testing/ci/detection_testing_batch/summarize_json.py
new file mode 100644
index 0000000000..102252495a
--- /dev/null
+++ b/automated_detection_testing/ci/detection_testing_batch/summarize_json.py
@@ -0,0 +1,51 @@
+import json
+from collections import OrderedDict
+import argparse
+import sys
+import json
+def outputResultsJSON(output_filename:str, data:list[dict], baseline:OrderedDict)->bool:
+    success = True
+    try:
+        with open(output_filename, "w") as jsonFile:
+            json.dump({'baseline': baseline, 'results':data}, jsonFile, indent="   ")
+    except Exception as e:
+        print("There was an error generating [%s]: [%s]"%(output_filename, str(e)))
+        success = False
+    return success
+
+
+
+parser = argparse.ArgumentParser(description="Results Merger")
+parser.add_argument('-f', '--files', type=argparse.FileType('r'), required=True, nargs='+', help="The json files you would like to combine into a single file")
+parser.add_argument('-o', '--output_filename', type=str, required=True, help="The name of the output file")
+args = parser.parse_args()
+
+all_data = OrderedDict()
+try:
+    for f in args.files:
+        if not f.name.endswith('.json'):
+            print("Error: passed in file must end in .json - you passed in [%s].\n\tQuitting..."%(f.name))
+            sys.exit(1)
+        data = json.loads(f.read())
+        if 'baseline' in all_data:
+            #everything has the same baseline, only need to do it once
+            pass
+        else:
+            all_data['baseline'] = data['baseline']
+        if 'results' in all_data:
+            #this is a list of dictionaries, so add to it
+            all_data['results'].extend(data['results'])
+        else:
+            all_data['results'] = data['results']
+
+    outputResultsJSON(args.output_filename, all_data['results'], all_data['baseline'])
+    print("Successfully summarized [%d] detections!"%(len(all_data['results'])))
+except Exception as e:
+    print("Error writing the summary file: [%s].\n\tQuitting..."%(str(e)))
+    sys.exit(1)
+
+
+
+    
+
+

From 0a3d51430656814ae53d1aab7e7b696291305942 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Wed, 10 Nov 2021 11:46:22 -0800
Subject: [PATCH 057/166] Small changes before large refactor

---
 .../detection_testing_batch/detection_testing_execution.py   | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
index 10deb2ed9e..cd06198e3a 100644
--- a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
+++ b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
@@ -252,6 +252,7 @@ def main(args):
     splunkbase_username = args.splunkbase_username
     splunkbase_password = args.splunkbase_password
     full_docker_hub_container_name = "splunk/splunk:%s"%args.container_tag
+    #full_docker_hub_container_name = "customimage"
     interactive_failure = args.interactive_failure
     show_password = args.show_password
     splunk_password = args.container_password
@@ -826,6 +827,7 @@ class SynchronizedResultsTracker:
             print("There was an error generating [%s]: [%s]"%(output_filename, str(e)))
             success = False
         return success
+        
     def outputResultsFile(self, field_names:list[str], output_filename:str, data:list[dict], baseline:OrderedDict, output_json:bool=True, output_csv:bool=True)->bool:
         success = True
         if output_csv:
@@ -974,7 +976,8 @@ def splunk_container_manager(testing_object:SynchronizedResultsTracker, containe
     print("Container [%s] setup complete and waiting for other containers to be ready..."%(container_name))
     testing_object.start_barrier.wait()
     wait_for_splunk_ready(container_name, splunk_web_port,splunk_ip, max_seconds=300)
-    
+    #print("\n\n\nLONG WAIT FOR THE EXPORT PLEASE \n\n\n\n")
+    #time.sleep(3600)
     while True:
         #Sleep for a small random time so that containers drift apart and don't synchronize their testing
         time.sleep(random.randint(1,30))

From b20d34c6a2652578f24c161ddf68a88535b8216a Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Fri, 12 Nov 2021 15:32:02 -0800
Subject: [PATCH 058/166] Huge changes to refactor docker based detection
 testing.  Broke down massive testing file into smaller files.  Not running or
 tested yet, but in progress.

---
 .../detection_testing_execution.py            | 441 +-----------------
 .../modules/container_manager.py              | 108 +++++
 .../modules/image_manager.py                  |  50 ++
 .../modules/splunk_container.py               | 338 ++++++++++++++
 .../modules/test_driver.py                    | 228 +++++++++
 .../detection_testing_batch/requirements.txt  |   3 +-
 6 files changed, 731 insertions(+), 437 deletions(-)
 create mode 100644 automated_detection_testing/ci/detection_testing_batch/modules/container_manager.py
 create mode 100644 automated_detection_testing/ci/detection_testing_batch/modules/image_manager.py
 create mode 100644 automated_detection_testing/ci/detection_testing_batch/modules/splunk_container.py
 create mode 100644 automated_detection_testing/ci/detection_testing_batch/modules/test_driver.py

diff --git a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
index cd06198e3a..a0f594e255 100644
--- a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
+++ b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
@@ -51,43 +51,8 @@ DETECTION_TYPES = ['endpoint', 'cloud', 'network']
 DETECTION_MODES = ['new', 'all', 'selected']
 
 
-#taken from attack_range
-def get_random_password()->str:
-    random_source = string.ascii_letters + string.digits
-    password = random.choice(string.ascii_lowercase)
-    password += random.choice(string.ascii_uppercase)
-    password += random.choice(string.digits)
 
     
-    for i in range(random.randrange(16,26)):
-        password += random.choice(random_source)
-
-    password_list = list(password)
-    random.SystemRandom().shuffle(password_list)
-    password = ''.join(password_list)
-    return password
-
-
-def wait_for_splunk_ready(container_name:str, splunk_web_port:int, splunk_ip:str="127.0.0.1", max_seconds:int=300)->bool:
-    #The smarter version of this will try to hit one of the pages,
-    #probably the login page, and when that is available it means that
-    #splunk is fully started and ready to go.  Until then, we just
-    #use a simple sleep
-    splunk_ready_url =  "http://%s:%d"%(splunk_ip, splunk_web_port)
-    print(splunk_ready_url)
-    start = timer()
-    while True:
-        try:
-            #Splunk container will not have proper ssl certificate
-            response = get(splunk_ready_url,timeout=5, verify=False)
-            response.raise_for_status()
-            return True
-        except Exception as e:
-            elapsed = timer() - start
-            if elapsed > max_seconds:
-                raise(Exception("Container [%s] took longer than maximum start time of [%d].\n\tQuitting..."%(container_name, max_seconds)))
-        time.sleep(5)
-    
 
 
 
@@ -95,102 +60,6 @@ def wait_for_splunk_ready(container_name:str, splunk_web_port:int, splunk_ip:str
         
 
 
-def setup_image(client: DockerClient, reuse_images: bool, container_name: str) -> None:
-    if not reuse_images:
-        #Check to see if the image exists.  If it does, then remove it.  If it does not, then do nothing
-        docker_image = None
-        try:
-            docker_image = client.images.get(container_name)
-        except Exception as e:
-            #We don't need to do anything, the image did not exist on our system
-            print("Image named [%s] did not exist, so we don't need to try and remove it."%(container_name))
-        if docker_image != None:
-            #We found the image.  Let's try to delete it
-            print("Found docker image named [%s] and you have requested that we forcefully remove it"%(container_name))
-            try:
-                client.images.remove(image=container_name, force=True, noprune=False)
-                print("Docker image named [%s] forcefully removed"%(container_name))
-            except Exception as e:
-                print("Error forcefully removing [%s]"%(container_name))
-                raise(e)
-    
-    #See if the image exists.  If it doesn't, then pull it from Docker Hub
-    docker_image = None
-    try:
-        docker_image = client.images.get(container_name)
-        print("Docker image [%s] found, no need to download it."%(container_name))
-    except Exception as e:
-        #Image did not exist on the system
-        docker_image = None
-
-    if docker_image is None:
-        #We did not find the image, so pull it
-        try:
-            print("Downloading image [%s].  Please note "
-                 "that this could take a long time depending on your "
-                 "connection. It's around 2GB."%(container_name))
-            pull_start_time = timer()
-            client.images.pull(container_name)
-            pull_finish_time = timer()
-            print("Successfully pulled the docker image [%s] in %ss"%
-                  (container_name,
-                  timedelta(seconds=pull_finish_time - pull_start_time, microseconds=0) ))
-
-        except Exception as e:
-            print("There was an error trying to pull the image [%s]: [%s]"%(container_name,str(e)))
-            raise(e)
-
-def remove_existing_containers(client: DockerClient, reuse_containers: bool, container_template: str, num_containers: int, forceRemove: bool=True) -> bool:
-    if reuse_containers is True:
-        #Check to make sure that all of the requested containers exist
-        for index in range(0, num_containers):
-            container_name = container_template%(index)
-            print("Checking for the existence of container named [%s]"%(container_name))
-            try:
-                this_container = client.containers.get(container_name)
-            except Exception as e:
-                print("Failed to find a container named [%s]"%(container_name))
-                reuse_containers = False
-                break
-            try:
-                #Make sure that the container is stopped
-                print("Found [%s]. Stopping container..."%(container_name))
-                this_container.stop()
-            except Exception as e:
-                print("Failed to stop a container named [%s]"%(container_name))
-                reuse_containers = False
-                break
-        print("Found all of the containers, we will reuse them")
-        return True
-    
-    #Note that this variable can be changed by the block above, so don't
-    #convert this into an if/else. Note that this IF is for verbosity:
-
-    if reuse_containers is False:
-        for index in range(0,num_containers):
-            container_name = container_template%(index)
-            removeContainer(client, container_name, forceRemove)
-        return False
-    else:
-        raise(Exception("Error removing existing containers"))
-
-def removeContainer(client: DockerClient, container_name:str, forceRemove:bool=True)->bool:
-    print("Trying to remove container [%s]"%(container_name))
-    try:
-        container = client.containers.get(container_name)
-    except Exception as e:
-        print("Could not find Docker Container [%s]. Container does not exist, so no need to remove it"%(container_name))
-        return True
-    try:
-        #container was found, so now we try to remove it
-        #v also removes volumes linked to the container
-        container.remove(v=True, force=forceRemove) #remove it even if it is running. remove volumes as well
-        print("Successfully removed Docker Container [%s]"%(container_name))
-        return True
-    except Exception as e:
-        print("Could not remove Docker Container [%s]"%(container_name))
-        raise(Exception("CONTAINER REMOVE ERROR"))
-
 
 
         
@@ -209,10 +78,11 @@ def main(args):
     parser.add_argument("-b", "--branch", type=str, required=True, help="security content branch")
     parser.add_argument("-u", "--uuid", type=str, required=True, help="uuid for detection test")
     parser.add_argument("-pr", "--pr-number", type=int, required=False, help="Pull Request Number")
+
     parser.add_argument("-n", "--num_containers", required=False, type=int, default=1, help="The number of splunk docker containers to start and run for testing")
     
     parser.add_argument("-cw", "--container_password", required=False, help="A password to use for the container.  If you don't choose one, a complex one will be generated for you.")
-    parser.add_argument("-show", "--show_password", required=False, default=False, action='store_true', help="Show the generated password to use to login to splunk.  For an CI/CD run, you probably don't want this.")
+    parser.add_argument("-show", "--show_password", required=False, default=False, action='store_true', help="Show the generated password to use to login to splunk.  For a CI/CD run, you probably don't want this.")
     parser.add_argument("-i", "--interactive_failure", required=False, default=False, action='store_true', help="If a test fails, should we pause before removing data so that the search can be debugged?")
 
     parser.add_argument("-ri", "--reuse_image", required=False, default=False, action='store_true', help="Should existing images be re-used, or should they be redownloaded?")
@@ -709,315 +579,14 @@ def main(args):
     #testing_service.test_detections(ssh_key_name, private_key, splunk_ip, splunk_password, test_files, uuid_test)
     
 
-def copy_file_to_container(localFilePath, remoteFilePath, containerName, sleepTimeSeconds=5):
-    successful_copy = False
-    #need to use the low level client to put a file onto a container
-    apiclient = docker.APIClient()
-    while not successful_copy:
-        try:
-            with open(localFilePath,"rb") as fileData:
-                #splunk will restart a few times will installation of apps takes place so it will reload its indexes...
-                apiclient.put_archive(container=containerName, path=remoteFilePath, data=fileData)
-                successful_copy=True
-        except Exception as e:
-            #print("Failed copy of [%s] file to CONTAINER:[%s]...we will try again"%(localFilePath, containerName))
-            time.sleep(10)
-            successful_copy=False
-    print("Successfully copied [%s] to [%s] on [%s]"%(localFilePath, remoteFilePath, containerName))
 
 
-class SynchronizedResultsTracker:
-    def __init__(self, tests:list[str], num_containers:int):
-        #Create the queue and enque all of the tests
-        self.testing_queue = queue.Queue()
-        for test in tests:
-            self.testing_queue.put(test)
-        
-        self.total_number_of_tests = self.testing_queue.qsize()
-        #Creates a lock that will be used to synchronize access to this object
-        self.lock = threading.Lock()
-        self.start_time = timer()
-        self.failures = []
-        self.successes = []
-        self.errors = []
-        self.container_ready_time = None
-        
-        #Just make a random folder to store attack data that we donwload
-        self.attack_data_root_folder = mkdtemp(prefix="attack_data_", dir=os.getcwd())
-        print("Attack data for this run will be stored at: [%s]"%(self.attack_data_root_folder))
-        self.start_barrier = threading.Barrier(num_containers)
-
-    def getTest(self)-> Union[str,None]:
-        try:
-            return self.testing_queue.get(block=False)
-        except Exception as e:
-            print("Testing queue empty!")
-            return None
-        
-    def addSuccess(self, result:dict)->None:
-        print("Test PASSED for detection: [%s --> %s"%(result['detection_name'], result['detection_file']))
-        self.lock.acquire()
-        try:
-            self.successes.append(result)
-        finally:
-            self.lock.release()
-        
-
-    def addFailure(self, result:dict)->None:
-        print("Test FAILED for detection: [%s --> %s"%(result['detection_name'], result['detection_file']))
-        self.lock.acquire()
-        try:
-            self.failures.append(result)
-        finally:
-            self.lock.release()
-
-    def addError(self, detection:dict)->None:
-        self.lock.acquire()
-        try:
-            self.errors.append(detection)
-        finally:
-            self.lock.release()
-    
-
-    def outputResultsCSV(self, field_names:list[str], output_filename:str, data:list[dict], baseline:OrderedDict)->bool:
-        success = True
-
-        print("Generating %s..."%(output_filename), end='')
-        self.lock.acquire()
-        
-        try:                        
-            with open(output_filename, 'w') as csvfile:
-                header_writer = csv.writer(csvfile, quoting=csv.QUOTE_ALL)
-                for key in baseline:
-                    #Very basic support for pretty pritning dicts. Doesn't handle more than 1 nested dict
-                    if type(baseline[key]) is OrderedDict:
-                        header_writer.writerow([key, "-"])
-                        for nestedkey in baseline[key]:
-                            header_writer.writerow([nestedkey, baseline[key][nestedkey]])
-                    #Basic support for 1 layer nested list. Doesn't handle more than 1.
-                    elif type(baseline[key]) is list and len(baseline[key])>0:
-                        header_writer.writerow([key, baseline[key][0]])
-                        for i in range(1,len(baseline[key])):
-                            header_writer.writerow(['-', baseline[key][i]])
-                        
-                    else:
-                        header_writer.writerow([key, baseline[key]])
-                header_writer.writerow(['',''])
-                csv_writer = csv.DictWriter(csvfile, fieldnames=field_names)
-                csv_writer.writeheader()
-                for row in data:
-                    csv_writer.writerow(row)
-            print("Done with [%d] detections"%(len(data)))
-
-        except Exception as e:
-            print("Failure writing to CSV file for [%s]:"%(output_filename, str(e)))
-            success = False
-
-        finally:
-            self.lock.release()
-
-        return success
-
-    def outputResultsJSON(self, field_names:list[str], output_filename:str, data:list[dict], baseline:OrderedDict)->bool:
-        success = True
-        try:
-            with open(output_filename, "w") as jsonFile:
-                json.dump({'baseline': baseline, 'results':data}, jsonFile, indent="   ")
-        except Exception as e:
-            print("There was an error generating [%s]: [%s]"%(output_filename, str(e)))
-            success = False
-        return success
-        
-    def outputResultsFile(self, field_names:list[str], output_filename:str, data:list[dict], baseline:OrderedDict, output_json:bool=True, output_csv:bool=True)->bool:
-        success = True
-        if output_csv:
-            success |= self.outputResultsCSV(field_names, output_filename + ".csv", data, baseline)
-        if output_json:
-            success |= self.outputResultsJSON(field_names, output_filename + ".json", data, baseline)
-        return success
-        
-
-    def outputResultsFiles(self, baseline:OrderedDict, fields:list[str]=['detection_name', 'detection_file','runDuration','diskUsage', 'search_string', 'error', 'success', 'scanCount', 'detection_error'])->bool:
-        res = self.outputResultsFile(fields,"success", self.successes, baseline)
-        res |= self.outputResultsFile(fields, "failure", self.failures, baseline)
-        res |= self.outputResultsFile(fields, "error", self.errors, baseline)
-        res |= self.outputResultsFile(fields, "combined", self.successes + self.failures + self.errors, baseline)
-        return res
-
-    def finish(self, baseline:OrderedDict):
-        self.cleanup()
-        self.outputResultsFiles(baseline)
-        
-
-    def cleanup(self):
-        self.lock.acquire()
-        try:
-            print("Removing all attack data that was downloaded during this test at: [%s]"%(self.attack_data_root_folder))
-            shutil.rmtree(self.attack_data_root_folder)
-            print("Successfully removed all attack data")
-        finally:
-            self.lock.release()
-    def summarize(self)->bool:
-        
-        self.lock.acquire()
-        
-        try:
-            current_time = timer()
-            if self.testing_queue.qsize() == self.total_number_of_tests:
-                #Testing has not started yet. We are setting up containers
-                print("***********PROGRESS UPDATE***********\n"\
-                      "\tWaiting for container setup: %s"%(timedelta(seconds=current_time - self.start_time)))
-            else:
-                if self.container_ready_time is None:
-                    #This is the first status update since container setup has completed.  Get the current time.
-                    #This makes our remaining time estimates better since that estimate should not involve
-                    #the container setup time  
-                    self.container_ready_time = current_time
-
-                numberOfCompletedTests = len(self.successes) + len(self.failures) + len(self.errors)
-                remaining_tests = self.testing_queue.qsize()         
-                testsCurrentlyRunning = self.total_number_of_tests - remaining_tests - numberOfCompletedTests
-                total_execution_time_seconds = current_time - self.start_time
-
-                test_execution_time_seconds = current_time - self.container_ready_time
-                
-                
-                if numberOfCompletedTests == 0 or test_execution_time_seconds == 0:
-                    estimated_seconds_to_finish_all_tests = "UNKNOWN"
-                    estimated_completion_time_seconds = "UNKNOWN"
-                else:
-                    average_time_per_test = test_execution_time_seconds / numberOfCompletedTests
-                    #divide testsCurrentlyRunning by 2.0 because, on average, each running test will be 50% completed
-                    estimated_seconds_to_finish_all_tests = average_time_per_test * (remaining_tests + testsCurrentlyRunning/2.0)
-                    estimated_completion_time_seconds = timedelta(seconds=estimated_seconds_to_finish_all_tests)
-                    
-                
-
-            
-                print("***********PROGRESS UPDATE***********\n"\
-                "\tElapsed Time               : %s\n"\
-                "\tEstimated Remaining Time   : %s\n"\
-                "\tTests to run               : %d\n"\
-                "\tTests currently running    : %d\n"\
-                "\tTests completed            : %d\n"\
-                "\t\tSuccess : %d\n"\
-                "\t\tFailure : %d\n"\
-                "\t\tError   : %d"%(timedelta(seconds=total_execution_time_seconds), 
-                                    estimated_completion_time_seconds, 
-                                    remaining_tests, 
-                                    testsCurrentlyRunning,
-                                    numberOfCompletedTests, 
-                                    len(self.successes), 
-                                    len(self.failures), 
-                                    len(self.errors)))
-    
-        except Exception as e:
-            print("Error in printing execution summary: [%s]"%(str(e)))
-        finally:
-            self.lock.release()
-        
-        #Return true while there are tests remaining
-        return (self.total_number_of_tests - 
-                (len(self.successes) + len(self.failures) + len(self.errors)) > 0)
-        
-        
-    def addResult(self, result:dict)->None:
-        try:
-            if result['detection_result']['success'] is False:
-                #This is actually a failure of the detection, not an error. Naming is confusiong
-                self.addFailure(result['detection_result'])
-            elif result['detection_result']['success'] is True:
-                self.addSuccess(result['detection_result'])
-        except Exception as e:
-            #Neither a success or a failure, so add the object to the failures queue
-            self.addError({'detection_file':"Unknown File", "detection_error":str(result)})
-        
-
-
-def splunk_container_manager(testing_object:SynchronizedResultsTracker, container_name, splunk_ip, splunk_password, splunk_web_port, splunk_management_port, uuid_test, interactive_failure:bool=False):
-    print("Starting the container [%s] after a sleep"%(container_name))
-    #Is this going to be safe to use in different threads
-    client = docker.client.from_env()
-    
-    #start up the container from the base container
-    #Assume that the base container has already been fully built with
-    #escu etc
-    #sleep for a little bit so that we don't all start at once...
-    
-
-    container = client.containers.get(container_name)
-    
-    print("Starting the container [%s]"%(container_name))
-
-    
-    
-    container.start()
-    print("Start copying files to container: [%s]"%(container_name))
-    copy_file_to_container(index_file_local_path, index_file_container_path, container_name)
-    #The below copy will fail until the CIM app is installed.  This app MUST be installed last!
-    #If we install it earlier, we will get ahead of ourselves and start doing tests before the container
-    #is truly ready for testing and all apps have been installed
-    copy_file_to_container(datamodel_file_local_path, datamodel_file_container_path, container_name)
-    print("Finished copying files to container: [%s]"%(container_name))
-    
-
-    
-    from modules.splunk_sdk import enable_delete_for_admin
-    print("Enabling DELETE for [%s]"%(container_name))
-    try:
-        while not enable_delete_for_admin(splunk_ip, splunk_management_port, splunk_password):
-            time.sleep(10)
-    except Exception as e:
-        print("Failure enabling DELETE for container [%s]: [%s].\n\tQuitting..."%(container_name, str(e)))
-    
-    print("Successfully enabled DELETE for [%s]"%(container_name))
-    
-    #Wait for all of the threads to join here
-    print("Container [%s] setup complete and waiting for other containers to be ready..."%(container_name))
-    testing_object.start_barrier.wait()
-    wait_for_splunk_ready(container_name, splunk_web_port,splunk_ip, max_seconds=300)
-    #print("\n\n\nLONG WAIT FOR THE EXPORT PLEASE \n\n\n\n")
-    #time.sleep(3600)
-    while True:
-        #Sleep for a small random time so that containers drift apart and don't synchronize their testing
-        time.sleep(random.randint(1,30))
-        #Try to get something from the queue
-        detection_to_test = testing_object.getTest()
-        if detection_to_test is None:
-            try:
-                print("Container [%s] has finished running detections, time to stop the container."%(container_name))
-                container.stop()
-                print("Container [%s] successfully stopped"%(container_name))
-                #remove the container
-                removeContainer(client, container_name, forceRemove=True)
-            except Exception as e:
-                print("Error stopping or removing the container: [%s]"%(str(e)))
-
-            return None
-
 
         
-        #There is a detection to test
-        print("Container [%s]--->[%s]"%(container_name, detection_to_test))
-        try:
-            result = testing_service.test_detection_wrapper(container_name, splunk_ip, splunk_password, splunk_management_port, detection_to_test, 0, uuid_test, testing_object.attack_data_root_folder, wait_on_failure=interactive_failure)
-            testing_object.addResult(result)
-            
-            #Remove the data from the test that we just ran.  We MUST do this when running on CI because otherwise, we will download
-            #a massive amount of data over the course of a long path and will run out of space on the relatively small CI runner drive
-            shutil.rmtree(result['attack_data_directory'])
-        except Exception as e:
-            print("Warning - uncaught error in detection test for [%s] - this should not happen: [%s]"%(detection_to_test, str(e)))
-            testing_object.addError({"detection_file":detection_to_test,"detection_error":str(e)})
 
-def queue_status_thread(status_object:SynchronizedResultsTracker)->None:
-    #This will run forever by design
-    print("start status")
-    while True:      
-        if status_object.summarize() == False:
-            #There are no more tests to run, so we can return from this thread
-            return None
-        time.sleep(10)
+
+
+
             
 if __name__ == "__main__":
     main(sys.argv[1:])
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/container_manager.py b/automated_detection_testing/ci/detection_testing_batch/modules/container_manager.py
new file mode 100644
index 0000000000..46dce75552
--- /dev/null
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/container_manager.py
@@ -0,0 +1,108 @@
+from collections import OrderedDict
+import docker
+import docker.types
+import random
+import splunk_container
+import string
+from typing import Union
+
+WEB_PORT_STRING = "8000/tcp"
+MANAGEMENT_PORT_STRING = "8089/tcp"
+
+
+class ContainerManager:
+    def __init__(
+        self,
+        container_name_template: str,
+        num_containers: int,
+        apps: list[dict],
+        files_to_copy_to_container:OrderedDict=OrderedDict(),
+        web_port_start: int = 8000,
+        management_port_start: int = 8089,
+        mounts: list[dict[str, Union[str, bool]]] = [],
+        container_password: Union[str, None] = None,
+        splunkbase_username: Union[str, None] = None,
+        splunkbase_password: Union[str, None] = None,
+    ):
+        self.mounts = self.create_mounts(mounts)
+        self.apps = apps
+        self.client = docker.from_env()
+
+        if container_password is None:
+            self.container_password = self.get_random_password()
+        else:
+            self.container_password = container_password
+
+        self.create_containers(
+            container_name_template,
+            num_containers,
+            web_port_start,
+            management_port_start,
+            splunkbase_username,
+            splunkbase_password,
+        )
+
+    def create_containers(
+        self,
+        container_name_template: str,
+        num_containers: int,
+        web_port_start: int,
+        management_port_start: int,
+        splunkbase_username: Union[str, None] = None,
+        splunkbase_password: Union[str, None] = None,
+    ) -> list[splunk_container.SplunkContainer]:
+        new_containers = []
+        for index in range(num_containers):
+            container_name = container_name_template % index
+            web_port = (WEB_PORT_STRING, web_port_start + index)
+            management_port = (MANAGEMENT_PORT_STRING, management_port_start + index)
+
+            new_containers.append(
+                splunk_container.SplunkContainer(
+                    self.client,
+                    container_name,
+                    self.apps,
+                    web_port,
+                    management_port,
+                    self.container_password,
+                    files_to_copy_to_container,
+                    self.mounts,
+                    splunkbase_username,
+                    splunkbase_password,
+                )
+            )
+
+        return new_containers
+
+    def create_mounts(
+        self, mounts: list[dict[str, Union[str, bool]]]
+    ) -> list[docker.types.Mount]:
+        new_mounts = []
+        for mount in mounts:
+            new_mounts.append(self.create_mount(mount))
+        return new_mounts
+
+    def create_mount(self, mount: dict[str, Union[str, bool]]) -> docker.types.Mount:
+        return docker.types.Mount(
+            source=mount["local_path"],
+            target=mount["container_path"],
+            type=mount["type"],
+            read_only=mount["read_only"],
+        )
+
+    # taken from attack_range
+    def get_random_password(
+        self, password_min_length: int = 16, password_max_length: int = 26
+    ) -> str:
+        random_source = string.ascii_letters + string.digits
+        password = random.choice(string.ascii_lowercase)
+        password += random.choice(string.ascii_uppercase)
+        password += random.choice(string.digits)
+
+        for i in range(random.randrange(password_min_length, password_max_length)):
+            password += random.choice(random_source)
+
+        password_list = list(password)
+        random.SystemRandom().shuffle(password_list)
+        password = "".join(password_list)
+        return password
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/image_manager.py b/automated_detection_testing/ci/detection_testing_batch/modules/image_manager.py
new file mode 100644
index 0000000000..0a48e97336
--- /dev/null
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/image_manager.py
@@ -0,0 +1,50 @@
+import datetime
+import docker
+import timeit
+
+def setup_image(client: docker.client.DockerClient, reuse_images: bool, container_name: str) -> None:
+    if not reuse_images:
+        #Check to see if the image exists.  If it does, then remove it.  If it does not, then do nothing
+        docker_image = None
+        try:
+            docker_image = client.images.get(container_name)
+        except Exception as e:
+            #We don't need to do anything, the image did not exist on our system
+            print("Image named [%s] did not exist, so we don't need to try and remove it."%(container_name))
+        if docker_image != None:
+            #We found the image.  Let's try to delete it
+            print("Found docker image named [%s] and you have requested that we forcefully remove it"%(container_name))
+            try:
+                client.images.remove(image=container_name, force=True, noprune=False)
+                print("Docker image named [%s] forcefully removed"%(container_name))
+            except Exception as e:
+                print("Error forcefully removing [%s]"%(container_name))
+                raise(e)
+    
+    #See if the image exists.  If it doesn't, then pull it from Docker Hub
+    docker_image = None
+    try:
+        docker_image = client.images.get(container_name)
+        print("Docker image [%s] found, no need to download it."%(container_name))
+    except Exception as e:
+        #Image did not exist on the system
+        docker_image = None
+
+    if docker_image is None:
+        #We did not find the image, so pull it
+        try:
+            print("Downloading image [%s].  Please note "
+                 "that this could take a long time depending on your "
+                 "connection. It's around 2GB."%(container_name))
+            pull_start_time = timeit.default_timer()
+            client.images.pull(container_name)
+            pull_finish_time = timeit.default_timer()
+            print("Successfully pulled the docker image [%s] in %ss"%
+                  (container_name,
+                  datetime.timedelta(seconds=pull_finish_time - pull_start_time, microseconds=0) ))
+
+        except Exception as e:
+            print("There was an error trying to pull the image [%s]: [%s]"%(container_name,str(e)))
+            raise(e)
+
+
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/splunk_container.py b/automated_detection_testing/ci/detection_testing_batch/modules/splunk_container.py
new file mode 100644
index 0000000000..c0a7e94279
--- /dev/null
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/splunk_container.py
@@ -0,0 +1,338 @@
+from collections import OrderedDict
+import docker
+import docker.types
+import docker.models
+import docker.models.resource
+import docker.models.containers
+import os.path
+import requests
+import shutil
+import splunk_sdk
+import testing_service
+import time
+import timeit
+from typing import Union
+
+
+SPLUNKBASE_URL = "https://splunkbase.splunk.com/app/%d/release/%s/download"
+SPLUNK_START_ARGS = "--accept-license"
+
+
+class SplunkContainer:
+    def __init__(
+        self,
+        synchronization_object,
+        full_docker_hub_path,
+        container_name: str,
+        apps: list[dict],
+        web_port: tuple[str, int],
+        management_port: tuple[str, int],
+        container_password: str,
+        files_to_copy_to_container: OrderedDict = OrderedDict(),
+        mounts: list[docker.types.Mount] = [],
+        splunkbase_username: Union[str, None] = None,
+        splunkbase_password: Union[str, None] = None,
+        splunk_ip:str = "127.0.0.1"
+    ):
+        self.synchronization_object = synchronization_object
+        self.client = docker.client.from_env()
+        self.full_docker_hub_path = full_docker_hub_path
+        self.container_password = container_password
+        self.apps = apps
+        self.files_to_copy_to_container = files_to_copy_to_container
+        self.splunk_ip = splunk_ip
+        self.container_name = container_name
+        self.mounts = mounts
+        self.environment = self.make_environment(
+            apps, container_password, splunkbase_username, splunkbase_password
+        )
+        self.ports = self.make_ports(web_port, management_port)
+        self.web_port = web_port
+        self.management_port = management_port
+        self.container = self.make_container()
+
+    def prepare_apps_path(
+        self,
+        apps: list[dict],
+        splunkbase_username: Union[str, None] = None,
+        splunkbase_password: Union[str, None] = None,
+    ) -> tuple[str, bool]:
+        apps_to_install = []
+        require_credentials = False
+        for app in self.apps:
+            if app["location"] == "splunkbase":
+                if splunkbase_username is None or splunkbase_password is None:
+                    raise Exception(
+                        "Error: Requested app from Splunkbase but Splunkbase username and/or password were not supplied."
+                    )
+                target = SPLUNKBASE_URL % (app["app_number"], app["app_version"])
+                apps_to_install.append(target)
+                require_credentials = True
+            elif app["location"] == "local":
+                apps_to_install.append(app["container_path"])
+        return ",".join(apps_to_install), require_credentials
+
+    def make_volume(
+        self,
+        local_path: str,
+        container_path: str,
+        type: str = "bind",
+        read_only: bool = True,
+    ) -> docker.types.Mount:
+        return docker.types.Mount(
+            source=local_path, target=container_path, type="bind", read_only=True
+        )
+
+    def make_environment(
+        self,
+        apps: list[dict],
+        container_password: str,
+        splunkbase_username: Union[str, None] = None,
+        splunkbase_password: Union[str, None] = None,
+    ) -> dict:
+        env = {}
+        env["SPLUNK_START_ARGS"] = SPLUNK_START_ARGS
+        env["SPLUNK_PASSWORD"] = container_password
+        splunk_apps_url, require_credentials = self.prepare_apps_path(
+            apps, splunkbase_username, splunkbase_password
+        )
+        if require_credentials:
+            env["SPLUNKBASE_USERNAME"] = splunkbase_username
+            env["SPLUNKBASE_PASSWORD"] = splunkbase_password
+        env["SPLUNK_APPS_URL"] = splunk_apps_url
+
+        return env
+
+    def make_ports(self, *ports: tuple[str, int]) -> dict[str, int]:
+        port_dict = {}
+        for port in ports:
+            port_dict[port[0]] = port[1]
+        return port_dict
+
+    def __str__(self) -> str:
+        container_string = (
+            "Container Name: %s\n\t"
+            "Docker Hub Path: %s\n\t"
+            "Apps: %s\n\t"
+            "Ports: %s\n\t"
+            "Mounts: %s\n\t"
+            % (
+                self.container_name,
+                self.full_docker_hub_path,
+                self.environment["SPLUNK_APPS_URL"],
+                self.ports,
+            )
+        )
+
+        return container_string
+
+    def make_container(self) -> docker.models.resource.Model:
+        container = self.client.containers.create(
+            self.full_docker_hub_path,
+            ports=self.ports,
+            environment=self.environment,
+            name=self.container_name,
+            mounts=self.mounts,
+            detach=True,
+        )
+
+        return container
+
+    def extract_tar_file_to_container(
+        self, local_file_path: str, container_file_path: str, sleepTimeSeconds: int = 5
+    ) -> bool:
+        # Check to make sure that the file ends in .tar.  If it doesn't raise an exception
+        if os.path.splitext(local_file_path)[1] != ".tar":
+            raise Exception(
+                "Error - Failed copy of file [%s] to container [%s].  Only "
+                "files ending in .tar can be copied to the container using this function."
+                % (local_file_path, self.container_name)
+            )
+        successful_copy = False
+        api_client = docker.APIClient()
+        # need to use the low level client to put a file onto a container
+        while not successful_copy:
+            try:
+                with open(local_file_path, "rb") as fileData:
+                    # splunk will restart a few times will installation of apps takes place so it will reload its indexes...
+
+                    api_client.put_archive(
+                        container=self.container_name,
+                        path=container_file_path,
+                        data=fileData,
+                    )
+                    successful_copy = True
+            except Exception as e:
+                # print("Failed copy of [%s] file to CONTAINER:[%s]...we will try again"%(localFilePath, containerName))
+                time.sleep(10)
+                successful_copy = False
+        print(
+            "Successfully copied [%s] to [%s] on [%s]"
+            % (local_file_path, container_file_path, self.container_name)
+        )
+        return successful_copy
+
+    def removeContainer(
+        self, removeVolumes: bool = True, forceRemove: bool = True
+    ) -> bool:
+        try:
+            container = self.client.containers.get(self.container_name)
+        except Exception as e:
+            # Container does not exist, no need to try and remove it
+            return True
+        try:
+            # container was found, so now we try to remove it
+            # v also removes volumes linked to the container
+            container.remove(
+                v=removeVolumes, force=forceRemove
+            )  # remove it even if it is running. remove volumes as well
+            # No need to print that the container has been removed, it is expected behavior
+            return True
+        except Exception as e:
+            print("Could not remove Docker Container [%s]" % (self.container_name))
+            raise (Exception("CONTAINER REMOVE ERROR"))
+
+    def wait_for_splunk_ready(
+        self,
+        max_seconds: int = 300,
+        seconds_between_attempts: int = 5,
+    ) -> bool:
+        # The smarter version of this will try to hit one of the pages,
+        # probably the login page, and when that is available it means that
+        # splunk is fully started and ready to go.  Until then, we just
+        # use a simple sleep
+        splunk_ready_url = "http://%s:%d" % (self.splunk_ip, self.web_port)
+        start = timeit.default_timer()
+        while True:
+            try:
+                # Splunk container will not have proper ssl certificate
+                response = requests.get(splunk_ready_url, timeout=5, verify=False)
+                response.raise_for_status()
+                return True
+            except Exception as e:
+                elapsed = timeit.default_timer() - start
+                if elapsed > max_seconds:
+                    raise (
+                        Exception(
+                            "Container [%s] took longer than maximum start time of [%d].\n\tQuitting..."
+                            % (self.container_name, max_seconds)
+                        )
+                    )
+            time.sleep(seconds_between_attempts)
+
+    def run_container(self) -> None:
+        print("Starting the container [%s]" % (self.container_name))
+        self.container.start()
+
+        # By default, first copy the index file then the datamodel file
+        for f in self.files_to_copy_to_container:
+            self.extract_tar_file_to_container(
+                f["local_file_path"], f["container_file_path"]
+            )
+
+        print("Finished copying files to [%s]" % (self.container_name))
+
+        try:
+            while not splunk_sdk.enable_delete_for_admin(
+                self.splunk_ip, self.management_port, self.container_password
+            ):
+                time.sleep(10)
+        except Exception as e:
+            print(
+                "Failure enabling DELETE for container [%s]: [%s].\n\tQuitting..."
+                % (self.container_name, str(e))
+            )
+
+
+        # Wait for all of the threads to join here
+        print(
+            "Container [%s] setup complete and waiting for other containers to be ready..."
+            % (self.container_name)
+        )
+        synchornization_object.start_barrier.wait()
+        self.wait_for_splunk_ready()
+        
+        while True:
+            # Sleep for a small random time so that containers drift apart and don't synchronize their testing
+            time.sleep(random.randint(1, 30))
+            # Try to get something from the queue
+            detection_to_test = testing_object.getTest()
+            if detection_to_test is None:
+                try:
+                    print(
+                        "Container [%s] has finished running detections, time to stop the container."
+                        % (container_name)
+                    )
+                    container.stop()
+                    print("Container [%s] successfully stopped" % (container_name))
+                    # remove the container
+                    removeContainer(client, container_name, forceRemove=True)
+                except Exception as e:
+                    print("Error stopping or removing the container: [%s]" % (str(e)))
+
+                return None
+
+            # There is a detection to test
+            print("Container [%s]--->[%s]" % (container_name, detection_to_test))
+            try:
+                result = testing_service.test_detection_wrapper(
+                    container_name,
+                    splunk_ip,
+                    splunk_password,
+                    splunk_management_port,
+                    detection_to_test,
+                    0,
+                    uuid_test,
+                    testing_object.attack_data_root_folder,
+                    wait_on_failure=interactive_failure,
+                )
+                testing_object.addResult(result)
+
+                # Remove the data from the test that we just ran.  We MUST do this when running on CI because otherwise, we will download
+                # a massive amount of data over the course of a long path and will run out of space on the relatively small CI runner drive
+                shutil.rmtree(result["attack_data_directory"])
+            except Exception as e:
+                print(
+                    "Warning - uncaught error in detection test for [%s] - this should not happen: [%s]"
+                    % (detection_to_test, str(e))
+                )
+                testing_object.addError(
+                    {"detection_file": detection_to_test, "detection_error": str(e)}
+                )
+
+
+"""
+def remove_existing_containers(client: docker.client.DockerClient, reuse_containers: bool, container_template: str, num_containers: int, forceRemove: bool=True) -> bool:
+    if reuse_containers is True:
+        #Check to make sure that all of the requested containers exist
+        for index in range(0, num_containers):
+            container_name = container_template%(index)
+            print("Checking for the existence of container named [%s]"%(container_name))
+            try:
+                this_container = client.containers.get(container_name)
+            except Exception as e:
+                print("Failed to find a container named [%s]"%(container_name))
+                reuse_containers = False
+                break
+            try:
+                #Make sure that the container is stopped
+                print("Found [%s]. Stopping container..."%(container_name))
+                this_container.stop()
+            except Exception as e:
+                print("Failed to stop a container named [%s]"%(container_name))
+                reuse_containers = False
+                break
+        print("Found all of the containers, we will reuse them")
+        return True
+    
+    #Note that this variable can be changed by the block above, so don't
+    #convert this into an if/else. Note that this IF is for verbosity:
+
+    if reuse_containers is False:
+        for index in range(0,num_containers):
+            container_name = container_template%(index)
+            removeContainer(client, container_name, forceRemove)
+        return False
+    else:
+        raise(Exception("Error removing existing containers"))
+   """
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/test_driver.py b/automated_detection_testing/ci/detection_testing_batch/modules/test_driver.py
new file mode 100644
index 0000000000..71015d0b29
--- /dev/null
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/test_driver.py
@@ -0,0 +1,228 @@
+from collections import OrderedDict
+import csv
+import datetime
+import json
+import os
+import queue
+import shutil
+import tempfile
+import threading
+import time
+import timeit
+from typing import Union
+
+
+class TestDriver:
+    def __init__(self, tests:list[str], num_containers:int):
+        #Create the queue and enque all of the tests
+        self.testing_queue = queue.Queue()
+        for test in tests:
+            self.testing_queue.put(test)
+        
+        self.total_number_of_tests = self.testing_queue.qsize()
+        #Creates a lock that will be used to synchronize access to this object
+        self.lock = threading.Lock()
+        self.start_time = timeit.default_timer()
+        self.failures = []
+        self.successes = []
+        self.errors = []
+        self.container_ready_time = None
+        
+        #Just make a random folder to store attack data that we donwload
+        self.attack_data_root_folder = tempfile.mkdtemp(prefix="attack_data_", dir=os.getcwd())
+        print("Attack data for this run will be stored at: [%s]"%(self.attack_data_root_folder))
+        self.start_barrier = threading.Barrier(num_containers)
+
+    def getTest(self)-> Union[str,None]:
+        try:
+            return self.testing_queue.get(block=False)
+        except Exception as e:
+            print("Testing queue empty!")
+            return None
+        
+    def addSuccess(self, result:dict)->None:
+        print("Test PASSED for detection: [%s --> %s"%(result['detection_name'], result['detection_file']))
+        self.lock.acquire()
+        try:
+            self.successes.append(result)
+        finally:
+            self.lock.release()
+        
+
+    def addFailure(self, result:dict)->None:
+        print("Test FAILED for detection: [%s --> %s"%(result['detection_name'], result['detection_file']))
+        self.lock.acquire()
+        try:
+            self.failures.append(result)
+        finally:
+            self.lock.release()
+
+    def addError(self, detection:dict)->None:
+        self.lock.acquire()
+        try:
+            self.errors.append(detection)
+        finally:
+            self.lock.release()
+    
+
+    def outputResultsCSV(self, field_names:list[str], output_filename:str, data:list[dict], baseline:OrderedDict)->bool:
+        success = True
+
+        print("Generating %s..."%(output_filename), end='')
+        self.lock.acquire()
+        
+        try:                        
+            with open(output_filename, 'w') as csvfile:
+                header_writer = csv.writer(csvfile, quoting=csv.QUOTE_ALL)
+                for key in baseline:
+                    #Very basic support for pretty pritning dicts. Doesn't handle more than 1 nested dict
+                    if type(baseline[key]) is OrderedDict:
+                        header_writer.writerow([key, "-"])
+                        for nestedkey in baseline[key]:
+                            header_writer.writerow([nestedkey, baseline[key][nestedkey]])
+                    #Basic support for 1 layer nested list. Doesn't handle more than 1.
+                    elif type(baseline[key]) is list and len(baseline[key])>0:
+                        header_writer.writerow([key, baseline[key][0]])
+                        for i in range(1,len(baseline[key])):
+                            header_writer.writerow(['-', baseline[key][i]])
+                        
+                    else:
+                        header_writer.writerow([key, baseline[key]])
+                header_writer.writerow(['',''])
+                csv_writer = csv.DictWriter(csvfile, fieldnames=field_names)
+                csv_writer.writeheader()
+                for row in data:
+                    csv_writer.writerow(row)
+            print("Done with [%d] detections"%(len(data)))
+
+        except Exception as e:
+            print("Failure writing to CSV file for [%s]:"%(output_filename, str(e)))
+            success = False
+
+        finally:
+            self.lock.release()
+
+        return success
+
+    def outputResultsJSON(self, field_names:list[str], output_filename:str, data:list[dict], baseline:OrderedDict)->bool:
+        success = True
+        try:
+            with open(output_filename, "w") as jsonFile:
+                json.dump({'baseline': baseline, 'results':data}, jsonFile, indent="   ")
+        except Exception as e:
+            print("There was an error generating [%s]: [%s]"%(output_filename, str(e)))
+            success = False
+        return success
+        
+    def outputResultsFile(self, field_names:list[str], output_filename:str, data:list[dict], baseline:OrderedDict, output_json:bool=True, output_csv:bool=True)->bool:
+        success = True
+        if output_csv:
+            success |= self.outputResultsCSV(field_names, output_filename + ".csv", data, baseline)
+        if output_json:
+            success |= self.outputResultsJSON(field_names, output_filename + ".json", data, baseline)
+        return success
+        
+
+    def outputResultsFiles(self, baseline:OrderedDict, fields:list[str]=['detection_name', 'detection_file','runDuration','diskUsage', 'search_string', 'error', 'success', 'scanCount', 'detection_error'])->bool:
+        res = self.outputResultsFile(fields,"success", self.successes, baseline)
+        res |= self.outputResultsFile(fields, "failure", self.failures, baseline)
+        res |= self.outputResultsFile(fields, "error", self.errors, baseline)
+        res |= self.outputResultsFile(fields, "combined", self.successes + self.failures + self.errors, baseline)
+        return res
+
+    def finish(self, baseline:OrderedDict):
+        self.cleanup()
+        self.outputResultsFiles(baseline)
+        
+
+    def cleanup(self):
+        self.lock.acquire()
+        try:
+            print("Removing all attack data that was downloaded during this test at: [%s]"%(self.attack_data_root_folder))
+            shutil.rmtree(self.attack_data_root_folder)
+            print("Successfully removed all attack data")
+        finally:
+            self.lock.release()
+    def summarize(self)->bool:
+        
+        self.lock.acquire()
+        
+        try:
+            current_time = timeit.default_timer()
+            if self.testing_queue.qsize() == self.total_number_of_tests:
+                #Testing has not started yet. We are setting up containers
+                print("***********PROGRESS UPDATE***********\n"\
+                      "\tWaiting for container setup: %s"%(datetime.timedelta(seconds=current_time - self.start_time)))
+            else:
+                if self.container_ready_time is None:
+                    #This is the first status update since container setup has completed.  Get the current time.
+                    #This makes our remaining time estimates better since that estimate should not involve
+                    #the container setup time  
+                    self.container_ready_time = current_time
+
+                numberOfCompletedTests = len(self.successes) + len(self.failures) + len(self.errors)
+                remaining_tests = self.testing_queue.qsize()         
+                testsCurrentlyRunning = self.total_number_of_tests - remaining_tests - numberOfCompletedTests
+                total_execution_time_seconds = current_time - self.start_time
+
+                test_execution_time_seconds = current_time - self.container_ready_time
+                
+                
+                if numberOfCompletedTests == 0 or test_execution_time_seconds == 0:
+                    estimated_seconds_to_finish_all_tests = "UNKNOWN"
+                    estimated_completion_time_seconds = "UNKNOWN"
+                else:
+                    average_time_per_test = test_execution_time_seconds / numberOfCompletedTests
+                    #divide testsCurrentlyRunning by 2.0 because, on average, each running test will be 50% completed
+                    estimated_seconds_to_finish_all_tests = average_time_per_test * (remaining_tests + testsCurrentlyRunning/2.0)
+                    estimated_completion_time_seconds = datetime.timedelta(seconds=estimated_seconds_to_finish_all_tests)
+                    
+                
+
+            
+                print("***********PROGRESS UPDATE***********\n"\
+                "\tElapsed Time               : %s\n"\
+                "\tEstimated Remaining Time   : %s\n"\
+                "\tTests to run               : %d\n"\
+                "\tTests currently running    : %d\n"\
+                "\tTests completed            : %d\n"\
+                "\t\tSuccess : %d\n"\
+                "\t\tFailure : %d\n"\
+                "\t\tError   : %d"%(datetime.timedelta(seconds=total_execution_time_seconds), 
+                                    estimated_completion_time_seconds, 
+                                    remaining_tests, 
+                                    testsCurrentlyRunning,
+                                    numberOfCompletedTests, 
+                                    len(self.successes), 
+                                    len(self.failures), 
+                                    len(self.errors)))
+    
+        except Exception as e:
+            print("Error in printing execution summary: [%s]"%(str(e)))
+        finally:
+            self.lock.release()
+        
+        #Return true while there are tests remaining
+        return (self.total_number_of_tests - 
+                (len(self.successes) + len(self.failures) + len(self.errors)) > 0)
+        
+        
+    def addResult(self, result:dict)->None:
+        try:
+            if result['detection_result']['success'] is False:
+                #This is actually a failure of the detection, not an error. Naming is confusiong
+                self.addFailure(result['detection_result'])
+            elif result['detection_result']['success'] is True:
+                self.addSuccess(result['detection_result'])
+        except Exception as e:
+            #Neither a success or a failure, so add the object to the failures queue
+            self.addError({'detection_file':"Unknown File", "detection_error":str(result)})
+
+def queue_status_thread(status_object:TestingSynchronization)->None:
+    #This will run forever by design
+    print("start status")
+    while True:      
+        if status_object.summarize() == False:
+            #There are no more tests to run, so we can return from this thread
+            return None
+        time.sleep(10)
\ No newline at end of file
diff --git a/automated_detection_testing/ci/detection_testing_batch/requirements.txt b/automated_detection_testing/ci/detection_testing_batch/requirements.txt
index 9fd488fd1f..d4a44f878c 100644
--- a/automated_detection_testing/ci/detection_testing_batch/requirements.txt
+++ b/automated_detection_testing/ci/detection_testing_batch/requirements.txt
@@ -8,6 +8,7 @@ PyYAML==5.4
 requests==2.25.1
 six==1.16.0
 splunk-sdk==1.6.12
+splunk-packaging-toolkit==1.0.1
 #newest version of docker for managing the splunk containers
 #we will freeze at a specific version later
-docker
\ No newline at end of file
+docker==5.0.3

From 4e16ab139d467cfc65926bcb5f4d775c6ecdce8c Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Fri, 12 Nov 2021 16:37:05 -0800
Subject: [PATCH 059/166] More maintenance to make the testing code more
 modular and better documented.

---
 .../modules/container_manager.py              | 38 ++++++++---
 .../modules/splunk_container.py               | 65 ++++++++++---------
 .../modules/test_driver.py                    |  2 +-
 .../modules/testing_service.py                | 10 ++-
 4 files changed, 69 insertions(+), 46 deletions(-)

diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/container_manager.py b/automated_detection_testing/ci/detection_testing_batch/modules/container_manager.py
index 46dce75552..d28d2f25a0 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/container_manager.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/container_manager.py
@@ -4,6 +4,7 @@ import docker.types
 import random
 import splunk_container
 import string
+import test_driver
 from typing import Union
 
 WEB_PORT_STRING = "8000/tcp"
@@ -13,10 +14,12 @@ MANAGEMENT_PORT_STRING = "8089/tcp"
 class ContainerManager:
     def __init__(
         self,
+        test_list: list[str],
+        full_docker_hub_name,
         container_name_template: str,
         num_containers: int,
-        apps: list[dict],
-        files_to_copy_to_container:OrderedDict=OrderedDict(),
+        apps: OrderedDict,
+        files_to_copy_to_container: OrderedDict = OrderedDict(),
         web_port_start: int = 8000,
         management_port_start: int = 8089,
         mounts: list[dict[str, Union[str, bool]]] = [],
@@ -24,46 +27,61 @@ class ContainerManager:
         splunkbase_username: Union[str, None] = None,
         splunkbase_password: Union[str, None] = None,
     ):
+        self.synchronization_object = test_driver.TestDriver(test_list, num_containers)
+
         self.mounts = self.create_mounts(mounts)
         self.apps = apps
-        self.client = docker.from_env()
 
         if container_password is None:
             self.container_password = self.get_random_password()
         else:
             self.container_password = container_password
 
-        self.create_containers(
+        self.containers = self.create_containers(
+            full_docker_hub_name,
             container_name_template,
             num_containers,
             web_port_start,
             management_port_start,
             splunkbase_username,
             splunkbase_password,
+            files_to_copy_to_container,
         )
 
+        self.run_containers()
+
+    def run_containers(self) -> None:
+        for container in self.containers:
+            container.thread.run()
+
     def create_containers(
         self,
+        full_docker_hub_name: str,
         container_name_template: str,
         num_containers: int,
         web_port_start: int,
         management_port_start: int,
         splunkbase_username: Union[str, None] = None,
         splunkbase_password: Union[str, None] = None,
+        files_to_copy_to_container: OrderedDict = OrderedDict(),
     ) -> list[splunk_container.SplunkContainer]:
         new_containers = []
         for index in range(num_containers):
             container_name = container_name_template % index
-            web_port = (WEB_PORT_STRING, web_port_start + index)
-            management_port = (MANAGEMENT_PORT_STRING, management_port_start + index)
-
+            web_port_tuple = (WEB_PORT_STRING, web_port_start + index)
+            management_port_tuple = (
+                MANAGEMENT_PORT_STRING,
+                management_port_start + index,
+            )
+            #Get a new client for this container
             new_containers.append(
                 splunk_container.SplunkContainer(
-                    self.client,
+                    self.synchronization_object,
+                    full_docker_hub_name,
                     container_name,
                     self.apps,
-                    web_port,
-                    management_port,
+                    web_port_tuple,
+                    management_port_tuple,
                     self.container_password,
                     files_to_copy_to_container,
                     self.mounts,
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/splunk_container.py b/automated_detection_testing/ci/detection_testing_batch/modules/splunk_container.py
index c0a7e94279..d7b11fa80e 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/splunk_container.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/splunk_container.py
@@ -5,14 +5,16 @@ import docker.models
 import docker.models.resource
 import docker.models.containers
 import os.path
+import random
 import requests
 import shutil
 import splunk_sdk
 import testing_service
+import test_driver
 import time
 import timeit
 from typing import Union
-
+import threading
 
 SPLUNKBASE_URL = "https://splunkbase.splunk.com/app/%d/release/%s/download"
 SPLUNK_START_ARGS = "--accept-license"
@@ -21,19 +23,21 @@ SPLUNK_START_ARGS = "--accept-license"
 class SplunkContainer:
     def __init__(
         self,
-        synchronization_object,
+        synchronization_object:test_driver.TestDriver,
         full_docker_hub_path,
         container_name: str,
-        apps: list[dict],
-        web_port: tuple[str, int],
-        management_port: tuple[str, int],
+        apps: OrderedDict,
+        web_port_tuple: tuple[str, int],
+        management_port_tuple: tuple[str, int],
         container_password: str,
         files_to_copy_to_container: OrderedDict = OrderedDict(),
         mounts: list[docker.types.Mount] = [],
         splunkbase_username: Union[str, None] = None,
         splunkbase_password: Union[str, None] = None,
-        splunk_ip:str = "127.0.0.1"
+        splunk_ip:str = "127.0.0.1",
+        interactive_failure:bool = False
     ):
+        self.interactive_failure = interactive_failure
         self.synchronization_object = synchronization_object
         self.client = docker.client.from_env()
         self.full_docker_hub_path = full_docker_hub_path
@@ -46,20 +50,24 @@ class SplunkContainer:
         self.environment = self.make_environment(
             apps, container_password, splunkbase_username, splunkbase_password
         )
-        self.ports = self.make_ports(web_port, management_port)
-        self.web_port = web_port
-        self.management_port = management_port
+        self.ports = self.make_ports(web_port_tuple, management_port_tuple)
+        self.web_port = web_port_tuple[1]
+        self.management_port = management_port_tuple[1]
         self.container = self.make_container()
 
+        self.thread = threading.Thread(target=self.run_container)
+
+
     def prepare_apps_path(
         self,
-        apps: list[dict],
+        apps: OrderedDict,
         splunkbase_username: Union[str, None] = None,
         splunkbase_password: Union[str, None] = None,
     ) -> tuple[str, bool]:
         apps_to_install = []
         require_credentials = False
-        for app in self.apps:
+        for app_name in self.apps:
+            app = self.apps[app_name]
             if app["location"] == "splunkbase":
                 if splunkbase_username is None or splunkbase_password is None:
                     raise Exception(
@@ -85,7 +93,7 @@ class SplunkContainer:
 
     def make_environment(
         self,
-        apps: list[dict],
+        apps: OrderedDict,
         container_password: str,
         splunkbase_username: Union[str, None] = None,
         splunkbase_password: Union[str, None] = None,
@@ -249,44 +257,43 @@ class SplunkContainer:
             "Container [%s] setup complete and waiting for other containers to be ready..."
             % (self.container_name)
         )
-        synchornization_object.start_barrier.wait()
+        
+        self.synchronization_object.start_barrier.wait()
         self.wait_for_splunk_ready()
         
         while True:
             # Sleep for a small random time so that containers drift apart and don't synchronize their testing
             time.sleep(random.randint(1, 30))
             # Try to get something from the queue
-            detection_to_test = testing_object.getTest()
+            detection_to_test = self.synchronization_object.getTest()
             if detection_to_test is None:
                 try:
                     print(
                         "Container [%s] has finished running detections, time to stop the container."
-                        % (container_name)
+                        % (self.container_name)
                     )
-                    container.stop()
-                    print("Container [%s] successfully stopped" % (container_name))
+                    self.container.stop()
+                    print("Container [%s] successfully stopped" % (self.container_name))
                     # remove the container
-                    removeContainer(client, container_name, forceRemove=True)
+                    self.removeContainer()
                 except Exception as e:
                     print("Error stopping or removing the container: [%s]" % (str(e)))
 
                 return None
 
             # There is a detection to test
-            print("Container [%s]--->[%s]" % (container_name, detection_to_test))
+            print("Container [%s]--->[%s]" % (self.container_name, detection_to_test))
             try:
                 result = testing_service.test_detection_wrapper(
-                    container_name,
-                    splunk_ip,
-                    splunk_password,
-                    splunk_management_port,
+                    self.container_name,
+                    self.splunk_ip,
+                    self.container_password,
+                    self.management_port,
                     detection_to_test,
-                    0,
-                    uuid_test,
-                    testing_object.attack_data_root_folder,
-                    wait_on_failure=interactive_failure,
+                    self.synchronization_object.attack_data_root_folder,
+                    wait_on_failure=self.interactive_failure,
                 )
-                testing_object.addResult(result)
+                self.synchronization_object.addResult(result)
 
                 # Remove the data from the test that we just ran.  We MUST do this when running on CI because otherwise, we will download
                 # a massive amount of data over the course of a long path and will run out of space on the relatively small CI runner drive
@@ -296,7 +303,7 @@ class SplunkContainer:
                     "Warning - uncaught error in detection test for [%s] - this should not happen: [%s]"
                     % (detection_to_test, str(e))
                 )
-                testing_object.addError(
+                self.synchronization_object.addError(
                     {"detection_file": detection_to_test, "detection_error": str(e)}
                 )
 
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/test_driver.py b/automated_detection_testing/ci/detection_testing_batch/modules/test_driver.py
index 71015d0b29..0277e0a18a 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/test_driver.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/test_driver.py
@@ -218,7 +218,7 @@ class TestDriver:
             #Neither a success or a failure, so add the object to the failures queue
             self.addError({'detection_file':"Unknown File", "detection_error":str(result)})
 
-def queue_status_thread(status_object:TestingSynchronization)->None:
+def queue_status_thread(status_object:TestDriver)->None:
     #This will run forever by design
     print("start status")
     while True:      
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/testing_service.py b/automated_detection_testing/ci/detection_testing_batch/modules/testing_service.py
index 38008e9915..4134a87c78 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/testing_service.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/testing_service.py
@@ -14,10 +14,10 @@ from os.path import relpath
 from tempfile import mkdtemp
 
 
-def test_detection_wrapper(container_name:str, splunk_ip:str, splunk_password:str, splunk_port:int, test_file:str, test_index:int, uuid_test:str, attack_data_root_folder, wait_on_failure:bool=False)->dict:
+def test_detection_wrapper(container_name:str, splunk_ip:str, splunk_password:str, splunk_port:int, test_file:str, attack_data_root_folder, wait_on_failure:bool=False)->dict:
     
     uuid_var = str(uuid.uuid4())
-    result_test = test_detection(splunk_ip, splunk_port, container_name, splunk_password, test_file, test_index, uuid_test, uuid_var, attack_data_root_folder)
+    result_test = test_detection(splunk_ip, splunk_port, container_name, splunk_password, test_file, uuid_var, attack_data_root_folder)
     if result_test is None:
         #We failed so early in the process that we could not produce any meaningful result
         raise(Exception("Test execution Error"))    
@@ -38,7 +38,7 @@ def test_detection_wrapper(container_name:str, splunk_ip:str, splunk_password:st
     return result_test    
 
 
-def test_detection(splunk_ip:str, splunk_port:int, container_name:str, splunk_password:str, test_file:str, test_index:int, uuid_test, uuid_var, attack_data_root_folder)->Union[dict,None]:
+def test_detection(splunk_ip:str, splunk_port:int, container_name:str, splunk_password:str, test_file:str, uuid_var, attack_data_root_folder)->Union[dict,None]:
     try:
         test_file_obj = load_file(os.path.join("security_content/", test_file))
     except Exception as e:
@@ -73,9 +73,7 @@ def test_detection(splunk_ip:str, splunk_port:int, container_name:str, splunk_pa
             if attack_data['update_timestamp'] == True:
                 data_manipulation = DataManipulation()
                 data_manipulation.manipulate_timestamp(target_file, attack_data['sourcetype'], attack_data['source'])
-        INDEX_TO_REPLAY_INTO = 'test' + str(test_index)
-        INDEX_TO_REPLAY_INTO = 'main'
-        replay_attack_dataset(container_name, splunk_password, folder_name, INDEX_TO_REPLAY_INTO, attack_data['sourcetype'], attack_data['source'], attack_data['file_name'])
+        replay_attack_dataset(container_name, splunk_password, folder_name, "main", attack_data['sourcetype'], attack_data['source'], attack_data['file_name'])
     
     time.sleep(30)
     

From f64fef09d757701aad4c83b1a64f80a45d883c97 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Fri, 12 Nov 2021 17:11:10 -0800
Subject: [PATCH 060/166] Still needs a lot of rework and cleanup - can't run
 right now.  Looking to finish cleanup on Monday.

---
 .../detection_testing_execution.py            | 18 +------
 .../modules/container_manager.py              |  4 +-
 .../modules/splunk_container.py               | 52 +++----------------
 3 files changed, 10 insertions(+), 64 deletions(-)

diff --git a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
index a0f594e255..3c478d8191 100644
--- a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
+++ b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
@@ -36,7 +36,7 @@ datamodel_file_local_path = "datamodels.conf.tar"
 datamodel_file_container_path = os.path.join(SPLUNK_CONTAINER_APPS_DIR, "Splunk_SA_CIM")
 
 
-PASSWORD_LENGTH=20
+
 MAX_RECOMMENDED_CONTAINERS_BEFORE_WARNING=2
 DEFAULT_CONTAINER_TAG="latest"
 LOCAL_BASE_CONTAINER_NAME = "splunk_test_%d"
@@ -52,22 +52,6 @@ DETECTION_MODES = ['new', 'all', 'selected']
 
 
 
-    
-
-
-
-
-        
-
-
-
-
-        
-
-    
-
-    
-
 
 def main(args):
     
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/container_manager.py b/automated_detection_testing/ci/detection_testing_batch/modules/container_manager.py
index d28d2f25a0..896c976f89 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/container_manager.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/container_manager.py
@@ -15,7 +15,7 @@ class ContainerManager:
     def __init__(
         self,
         test_list: list[str],
-        full_docker_hub_name,
+        full_docker_hub_name: str,
         container_name_template: str,
         num_containers: int,
         apps: OrderedDict,
@@ -48,8 +48,6 @@ class ContainerManager:
             files_to_copy_to_container,
         )
 
-        self.run_containers()
-
     def run_containers(self) -> None:
         for container in self.containers:
             container.thread.run()
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/splunk_container.py b/automated_detection_testing/ci/detection_testing_batch/modules/splunk_container.py
index d7b11fa80e..f1c65f234c 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/splunk_container.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/splunk_container.py
@@ -23,7 +23,7 @@ SPLUNK_START_ARGS = "--accept-license"
 class SplunkContainer:
     def __init__(
         self,
-        synchronization_object:test_driver.TestDriver,
+        synchronization_object: test_driver.TestDriver,
         full_docker_hub_path,
         container_name: str,
         apps: OrderedDict,
@@ -34,8 +34,8 @@ class SplunkContainer:
         mounts: list[docker.types.Mount] = [],
         splunkbase_username: Union[str, None] = None,
         splunkbase_password: Union[str, None] = None,
-        splunk_ip:str = "127.0.0.1",
-        interactive_failure:bool = False
+        splunk_ip: str = "127.0.0.1",
+        interactive_failure: bool = False,
     ):
         self.interactive_failure = interactive_failure
         self.synchronization_object = synchronization_object
@@ -57,7 +57,6 @@ class SplunkContainer:
 
         self.thread = threading.Thread(target=self.run_container)
 
-
     def prepare_apps_path(
         self,
         apps: OrderedDict,
@@ -135,6 +134,9 @@ class SplunkContainer:
         return container_string
 
     def make_container(self) -> docker.models.resource.Model:
+        # First, make sure that the container has been removed if it already existed
+        self.removeContainer()
+        
         container = self.client.containers.create(
             self.full_docker_hub_path,
             ports=self.ports,
@@ -251,16 +253,15 @@ class SplunkContainer:
                 % (self.container_name, str(e))
             )
 
-
         # Wait for all of the threads to join here
         print(
             "Container [%s] setup complete and waiting for other containers to be ready..."
             % (self.container_name)
         )
-        
+
         self.synchronization_object.start_barrier.wait()
         self.wait_for_splunk_ready()
-        
+
         while True:
             # Sleep for a small random time so that containers drift apart and don't synchronize their testing
             time.sleep(random.randint(1, 30))
@@ -306,40 +307,3 @@ class SplunkContainer:
                 self.synchronization_object.addError(
                     {"detection_file": detection_to_test, "detection_error": str(e)}
                 )
-
-
-"""
-def remove_existing_containers(client: docker.client.DockerClient, reuse_containers: bool, container_template: str, num_containers: int, forceRemove: bool=True) -> bool:
-    if reuse_containers is True:
-        #Check to make sure that all of the requested containers exist
-        for index in range(0, num_containers):
-            container_name = container_template%(index)
-            print("Checking for the existence of container named [%s]"%(container_name))
-            try:
-                this_container = client.containers.get(container_name)
-            except Exception as e:
-                print("Failed to find a container named [%s]"%(container_name))
-                reuse_containers = False
-                break
-            try:
-                #Make sure that the container is stopped
-                print("Found [%s]. Stopping container..."%(container_name))
-                this_container.stop()
-            except Exception as e:
-                print("Failed to stop a container named [%s]"%(container_name))
-                reuse_containers = False
-                break
-        print("Found all of the containers, we will reuse them")
-        return True
-    
-    #Note that this variable can be changed by the block above, so don't
-    #convert this into an if/else. Note that this IF is for verbosity:
-
-    if reuse_containers is False:
-        for index in range(0,num_containers):
-            container_name = container_template%(index)
-            removeContainer(client, container_name, forceRemove)
-        return False
-    else:
-        raise(Exception("Error removing existing containers"))
-   """

From 9e435bbc437d89c4e6f687c9c0b8a89164450b0a Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Mon, 15 Nov 2021 14:47:24 -0800
Subject: [PATCH 061/166] Lots more changes as we break into separate files for
 more readability and maintainability.  Big changes around printing status and
 managing different threads as they run.

---
 .../modules/container_manager.py              | 45 ++++++++++-
 .../modules/splunk_container.py               | 81 ++++++++++++++++---
 .../modules/test_driver.py                    |  8 --
 3 files changed, 114 insertions(+), 20 deletions(-)

diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/container_manager.py b/automated_detection_testing/ci/detection_testing_batch/modules/container_manager.py
index 896c976f89..3cf7595deb 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/container_manager.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/container_manager.py
@@ -1,10 +1,13 @@
 from collections import OrderedDict
 import docker
+import datetime
 import docker.types
 import random
 import splunk_container
 import string
 import test_driver
+import threading
+import time
 from typing import Union
 
 WEB_PORT_STRING = "8000/tcp"
@@ -27,7 +30,8 @@ class ContainerManager:
         splunkbase_username: Union[str, None] = None,
         splunkbase_password: Union[str, None] = None,
     ):
-        self.synchronization_object = test_driver.TestDriver(test_list, num_containers)
+        self.synchronization_object = test_driver.TestDriver(
+            test_list, num_containers)
 
         self.mounts = self.create_mounts(mounts)
         self.apps = apps
@@ -47,10 +51,39 @@ class ContainerManager:
             splunkbase_password,
             files_to_copy_to_container,
         )
+        self.summary_thread = threading.Thread(target=self.queue_status_thread,args=())
+
+        #Construct the baseline from the splunk version and the apps to be installed
+        self.baseline = OrderedDict()
+        #Get a datetime and add it as the first entry in the baseline
+        self.baseline['DATE'] = str(datetime.datetime.now().replace(microsecond=0))
+        self.baseline['SPLUNK_VERSION'] = full_docker_hub_name
+        for key in self.apps:
+            self.baseline[key] = self.apps[key]
+
+    def run_test(self):
+        self.run_containers()
+        self.run_status_thread()
+        for container in self.containers:
+            container.thread.join()
+            print(container.get_container_summary())
+        self.summary_thread.join()
+        print("All containers completed testing!")
+        
+        
+        self.synchronization_object.finish(self.baseline)
+
+
+
 
     def run_containers(self) -> None:
         for container in self.containers:
             container.thread.run()
+    
+    def run_status_thread(self) -> None:
+        self.queue_status_thread.run()
+        
+
 
     def create_containers(
         self,
@@ -71,7 +104,7 @@ class ContainerManager:
                 MANAGEMENT_PORT_STRING,
                 management_port_start + index,
             )
-            #Get a new client for this container
+            # Get a new client for this container
             new_containers.append(
                 splunk_container.SplunkContainer(
                     self.synchronization_object,
@@ -122,3 +155,11 @@ class ContainerManager:
         random.SystemRandom().shuffle(password_list)
         password = "".join(password_list)
         return password
+
+    def queue_status_thread(self)->None:
+        #This will run fo
+        while True:      
+            if self.synchronization_object.summarize() == False:
+                #There are no more tests to run, so we can return from this thread
+                return None
+            time.sleep(10)
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/splunk_container.py b/automated_detection_testing/ci/detection_testing_batch/modules/splunk_container.py
index f1c65f234c..e28370c101 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/splunk_container.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/splunk_container.py
@@ -1,4 +1,5 @@
 from collections import OrderedDict
+import datetime
 import docker
 import docker.types
 import docker.models
@@ -57,6 +58,10 @@ class SplunkContainer:
 
         self.thread = threading.Thread(target=self.run_container)
 
+        self.container_start_time = 0
+        self.test_start_time = 0
+        self.num_tests_completed = 0
+
     def prepare_apps_path(
         self,
         apps: OrderedDict,
@@ -72,7 +77,8 @@ class SplunkContainer:
                     raise Exception(
                         "Error: Requested app from Splunkbase but Splunkbase username and/or password were not supplied."
                     )
-                target = SPLUNKBASE_URL % (app["app_number"], app["app_version"])
+                target = SPLUNKBASE_URL % (
+                    app["app_number"], app["app_version"])
                 apps_to_install.append(target)
                 require_credentials = True
             elif app["location"] == "local":
@@ -136,7 +142,7 @@ class SplunkContainer:
     def make_container(self) -> docker.models.resource.Model:
         # First, make sure that the container has been removed if it already existed
         self.removeContainer()
-        
+
         container = self.client.containers.create(
             self.full_docker_hub_path,
             ports=self.ports,
@@ -199,9 +205,53 @@ class SplunkContainer:
             # No need to print that the container has been removed, it is expected behavior
             return True
         except Exception as e:
-            print("Could not remove Docker Container [%s]" % (self.container_name))
+            print("Could not remove Docker Container [%s]" % (
+                self.container_name))
             raise (Exception("CONTAINER REMOVE ERROR"))
 
+    def get_container_summary(self) -> str:
+        current_time = timeit.default_timer()
+        # Get rid of the decimal (microseconds) so that we have whole seconds
+        if self.container_start_time is None or self.test_start_time is None:
+            print(self.container_start_time)
+
+        # Total time the container has been running
+        if self.container_start_time == -1:
+            total_time_string = "NOT STARTED"
+        else:
+            total_time_rounded = datetime.timedelta(
+                round(current_time - self.container_start_time))
+            total_time_string = str(total_time_rounded)
+
+        # Time that the container setup took
+        if self.test_start_time == -1 or self.container_start_time == -1:
+            setup_time_string = "NOT SET UP"
+        else:
+            setup_secounds_rounded = datetime.timedelta(
+                round(self.test_start_time - self.container_start_time))
+            setup_time_string = str(setup_secounds_rounded)
+
+        # Time that the tests have been running
+        if self.test_start_time == -1 or self.num_tests_completed == 0:
+            testing_time_string = "NO TESTS COMPLETED"
+        else:
+            testing_seconds_rounded = datetime.timedelta(
+                round(current_time - self.test_start_time))
+
+            # Get the approximate time per test.  This is a clunky way to get rid of decimal
+            # seconds.... but it works
+            timedelta_per_test = testing_seconds_rounded/self.num_tests_completed
+            timedelta_per_test_rounded = timedelta_per_test - \
+                datetime.timedelta(
+                    microseconds=timedelta_per_test.microseconds)
+
+            testing_time_string = "%s per test (%d tests)"%(timedelta_per_test_rounded, str(testing_seconds_rounded))
+
+        summary_str = "[%s] Summary\n\t"\
+                      "Total Time          :"\
+                      "Container Start Time:"\
+                      "Test Execution Time :" %(total_time_string, setup_time_string, testing_time_string)
+
     def wait_for_splunk_ready(
         self,
         max_seconds: int = 300,
@@ -216,7 +266,8 @@ class SplunkContainer:
         while True:
             try:
                 # Splunk container will not have proper ssl certificate
-                response = requests.get(splunk_ready_url, timeout=5, verify=False)
+                response = requests.get(
+                    splunk_ready_url, timeout=5, verify=False)
                 response.raise_for_status()
                 return True
             except Exception as e:
@@ -232,6 +283,7 @@ class SplunkContainer:
 
     def run_container(self) -> None:
         print("Starting the container [%s]" % (self.container_name))
+        self.container_start_time = timeit.default_timer()
         self.container.start()
 
         # By default, first copy the index file then the datamodel file
@@ -262,9 +314,10 @@ class SplunkContainer:
         self.synchronization_object.start_barrier.wait()
         self.wait_for_splunk_ready()
 
+        # Sleep for a small random time so that containers drift apart and don't synchronize their testing
+        time.sleep(random.randint(1, 30))
+        self.test_start_time = timeit.default_timer()
         while True:
-            # Sleep for a small random time so that containers drift apart and don't synchronize their testing
-            time.sleep(random.randint(1, 30))
             # Try to get something from the queue
             detection_to_test = self.synchronization_object.getTest()
             if detection_to_test is None:
@@ -274,16 +327,19 @@ class SplunkContainer:
                         % (self.container_name)
                     )
                     self.container.stop()
-                    print("Container [%s] successfully stopped" % (self.container_name))
+                    print("Container [%s] successfully stopped" %
+                          (self.container_name))
                     # remove the container
                     self.removeContainer()
                 except Exception as e:
-                    print("Error stopping or removing the container: [%s]" % (str(e)))
+                    print(
+                        "Error stopping or removing the container: [%s]" % (str(e)))
 
                 return None
 
             # There is a detection to test
-            print("Container [%s]--->[%s]" % (self.container_name, detection_to_test))
+            print("Container [%s]--->[%s]" %
+                  (self.container_name, detection_to_test))
             try:
                 result = testing_service.test_detection_wrapper(
                     self.container_name,
@@ -305,5 +361,10 @@ class SplunkContainer:
                     % (detection_to_test, str(e))
                 )
                 self.synchronization_object.addError(
-                    {"detection_file": detection_to_test, "detection_error": str(e)}
+                    {"detection_file": detection_to_test,
+                        "detection_error": str(e)}
                 )
+            self.num_tests_completed+=1
+
+            # Sleep for a small random time so that containers drift apart and don't synchronize their testing
+            time.sleep(random.randint(1, 30))
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/test_driver.py b/automated_detection_testing/ci/detection_testing_batch/modules/test_driver.py
index 0277e0a18a..b21d2ed960 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/test_driver.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/test_driver.py
@@ -218,11 +218,3 @@ class TestDriver:
             #Neither a success or a failure, so add the object to the failures queue
             self.addError({'detection_file':"Unknown File", "detection_error":str(result)})
 
-def queue_status_thread(status_object:TestDriver)->None:
-    #This will run forever by design
-    print("start status")
-    while True:      
-        if status_object.summarize() == False:
-            #There are no more tests to run, so we can return from this thread
-            return None
-        time.sleep(10)
\ No newline at end of file

From 7aca8f62038c4b0f1942a14f9df1be15bec1c298 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Mon, 15 Nov 2021 16:11:17 -0800
Subject: [PATCH 062/166] Removed the image_manager.py file - it was simple and
 its functionality was integrated into the container_manager.py.

---
 .../modules/container_manager.py              | 71 ++++++++++++++++++-
 .../modules/image_manager.py                  | 50 -------------
 .../modules/splunk_container.py               |  3 +
 3 files changed, 72 insertions(+), 52 deletions(-)
 delete mode 100644 automated_detection_testing/ci/detection_testing_batch/modules/image_manager.py

diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/container_manager.py b/automated_detection_testing/ci/detection_testing_batch/modules/container_manager.py
index 3cf7595deb..b703872057 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/container_manager.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/container_manager.py
@@ -8,6 +8,7 @@ import string
 import test_driver
 import threading
 import time
+import timeit
 from typing import Union
 
 WEB_PORT_STRING = "8000/tcp"
@@ -29,6 +30,7 @@ class ContainerManager:
         container_password: Union[str, None] = None,
         splunkbase_username: Union[str, None] = None,
         splunkbase_password: Union[str, None] = None,
+        reuse_image:bool = True
     ):
         self.synchronization_object = test_driver.TestDriver(
             test_list, num_containers)
@@ -50,14 +52,20 @@ class ContainerManager:
             splunkbase_username,
             splunkbase_password,
             files_to_copy_to_container,
+            reuse_image
         )
         self.summary_thread = threading.Thread(target=self.queue_status_thread,args=())
 
         #Construct the baseline from the splunk version and the apps to be installed
         self.baseline = OrderedDict()
         #Get a datetime and add it as the first entry in the baseline
-        self.baseline['DATE'] = str(datetime.datetime.now().replace(microsecond=0))
+        self.start_time = datetime.datetime.now()
         self.baseline['SPLUNK_VERSION'] = full_docker_hub_name
+        #Added here first to preserve ordering for OrderedDict
+        self.baseline['TEST_START_TIME'] = "TO BE UPDATED"
+        self.baseline['TEST_FINISH_TIME'] = "TO BE UPDATED"
+        self.baseline['TEST_DURATION'] = "TO BE UPDATED"
+
         for key in self.apps:
             self.baseline[key] = self.apps[key]
 
@@ -71,6 +79,15 @@ class ContainerManager:
         print("All containers completed testing!")
         
         
+        stop_time = datetime.datetime.now()
+        x = stop_time - self.start_time
+
+        self.baseline['TEST_START_TIME'] = str(self.start_time)
+        self.baseline['TEST_FINISH_TIME'] =  str(stop_time)
+        
+        duration = stop_time - self.start_time
+        self.baseline['TEST_DURATION'] = duration - datetime.timedelta(microseconds=duration.microseconds)
+
         self.synchronization_object.finish(self.baseline)
 
 
@@ -95,7 +112,11 @@ class ContainerManager:
         splunkbase_username: Union[str, None] = None,
         splunkbase_password: Union[str, None] = None,
         files_to_copy_to_container: OrderedDict = OrderedDict(),
+        reuse_image = True
     ) -> list[splunk_container.SplunkContainer]:
+        #First make sure that the image exists and has been downloaded
+        self.setup_image(reuse_image, full_docker_hub_name)
+
         new_containers = []
         for index in range(num_containers):
             container_name = container_name_template % index
@@ -104,7 +125,7 @@ class ContainerManager:
                 MANAGEMENT_PORT_STRING,
                 management_port_start + index,
             )
-            # Get a new client for this container
+            
             new_containers.append(
                 splunk_container.SplunkContainer(
                     self.synchronization_object,
@@ -163,3 +184,49 @@ class ContainerManager:
                 #There are no more tests to run, so we can return from this thread
                 return None
             time.sleep(10)
+
+    def setup_image(self, reuse_images: bool, container_name: str) -> None:
+        client = docker.client.from_env()
+        if not reuse_images:
+            #Check to see if the image exists.  If it does, then remove it.  If it does not, then do nothing
+            docker_image = None
+            try:
+                docker_image = client.images.get(container_name)
+            except Exception as e:
+                #We don't need to do anything, the image did not exist on our system
+                #print("Image named [%s] did not exist, so we don't need to try and remove it."%(container_name))
+                pass
+            if docker_image != None:
+                #We found the image.  Let's try to delete it
+                print("Found docker image named [%s] and you have requested that we forcefully remove it"%(container_name))
+                try:
+                    client.images.remove(image=container_name, force=True, noprune=False)
+                    print("Docker image named [%s] forcefully removed"%(container_name))
+                except Exception as e:
+                    print("Error forcefully removing [%s]"%(container_name))
+                    raise(e)
+        
+        #See if the image exists.  If it doesn't, then pull it from Docker Hub
+        try:
+            docker_image = client.images.get(container_name)
+            print("Docker image [%s] found, no need to download it."%(container_name))
+        except Exception as e:
+            #Image did not exist on the system
+            docker_image = None
+
+        if docker_image is None:
+            #We did not find the image, so pull it
+            try:
+                print("Downloading image [%s].  Please note "
+                    "that this could take a long time depending on your "
+                    "connection. It's around 2GB."%(container_name))
+                pull_start_time = timeit.default_timer()
+                client.images.pull(container_name)
+                pull_finish_time = timeit.default_timer()
+                print("Successfully pulled the docker image [%s] in %ss"%
+                    (container_name,
+                    datetime.timedelta(seconds=pull_finish_time - pull_start_time, microseconds=0) ))
+
+            except Exception as e:
+                print("There was an error trying to pull the image [%s]: [%s]"%(container_name,str(e)))
+                raise(e)
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/image_manager.py b/automated_detection_testing/ci/detection_testing_batch/modules/image_manager.py
deleted file mode 100644
index 0a48e97336..0000000000
--- a/automated_detection_testing/ci/detection_testing_batch/modules/image_manager.py
+++ /dev/null
@@ -1,50 +0,0 @@
-import datetime
-import docker
-import timeit
-
-def setup_image(client: docker.client.DockerClient, reuse_images: bool, container_name: str) -> None:
-    if not reuse_images:
-        #Check to see if the image exists.  If it does, then remove it.  If it does not, then do nothing
-        docker_image = None
-        try:
-            docker_image = client.images.get(container_name)
-        except Exception as e:
-            #We don't need to do anything, the image did not exist on our system
-            print("Image named [%s] did not exist, so we don't need to try and remove it."%(container_name))
-        if docker_image != None:
-            #We found the image.  Let's try to delete it
-            print("Found docker image named [%s] and you have requested that we forcefully remove it"%(container_name))
-            try:
-                client.images.remove(image=container_name, force=True, noprune=False)
-                print("Docker image named [%s] forcefully removed"%(container_name))
-            except Exception as e:
-                print("Error forcefully removing [%s]"%(container_name))
-                raise(e)
-    
-    #See if the image exists.  If it doesn't, then pull it from Docker Hub
-    docker_image = None
-    try:
-        docker_image = client.images.get(container_name)
-        print("Docker image [%s] found, no need to download it."%(container_name))
-    except Exception as e:
-        #Image did not exist on the system
-        docker_image = None
-
-    if docker_image is None:
-        #We did not find the image, so pull it
-        try:
-            print("Downloading image [%s].  Please note "
-                 "that this could take a long time depending on your "
-                 "connection. It's around 2GB."%(container_name))
-            pull_start_time = timeit.default_timer()
-            client.images.pull(container_name)
-            pull_finish_time = timeit.default_timer()
-            print("Successfully pulled the docker image [%s] in %ss"%
-                  (container_name,
-                  datetime.timedelta(seconds=pull_finish_time - pull_start_time, microseconds=0) ))
-
-        except Exception as e:
-            print("There was an error trying to pull the image [%s]: [%s]"%(container_name,str(e)))
-            raise(e)
-
-
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/splunk_container.py b/automated_detection_testing/ci/detection_testing_batch/modules/splunk_container.py
index e28370c101..d958ff1e81 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/splunk_container.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/splunk_container.py
@@ -251,6 +251,8 @@ class SplunkContainer:
                       "Total Time          :"\
                       "Container Start Time:"\
                       "Test Execution Time :" %(total_time_string, setup_time_string, testing_time_string)
+        
+        return summary_str
 
     def wait_for_splunk_ready(
         self,
@@ -368,3 +370,4 @@ class SplunkContainer:
 
             # Sleep for a small random time so that containers drift apart and don't synchronize their testing
             time.sleep(random.randint(1, 30))
+

From 4b42586bfd99ac30b994f0898007abec50a7cd16 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Tue, 16 Nov 2021 12:47:42 -0800
Subject: [PATCH 063/166] Created a new file to develop the arguments parsing
 logic.

---
 .../detection_testing_execution.py            |  2 +-
 .../detection_testing_batch/new_arguments.py  | 51 +++++++++++++++++++
 2 files changed, 52 insertions(+), 1 deletion(-)
 create mode 100644 automated_detection_testing/ci/detection_testing_batch/new_arguments.py

diff --git a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
index 3c478d8191..992d20e9c3 100644
--- a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
+++ b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
@@ -422,7 +422,7 @@ def main(args):
         shutil.copyfile(BETA_SPLUNK_ADD_ON_FOR_SYSMON_PATH, os.path.join(local_volume_path, "Splunk_TA_microsoft_sysmon-1.0.2-B1.spl"))
     
         BETA_SPLUNK_ADD_ON_FOR_SYSMON_CONTAINER_PATH  = "/tmp/apps/Splunk_TA_microsoft_sysmon-1.0.2-B1.spl"
-        APPS_DICT['BETA_SPLUNK_ADD_ON_FOR_SYSMON'] = {"app_number":5709, 'app_version':"Generated at %s"%(datetime.now()), 'location':BETA_SPLUNK_ADD_ON_FOR_SYSMON_CONTAINER_PATH}
+        APPS_DICT['BETA_SPLUNK_ADD_ON_FOR_SYSMON'] = {"app_number":5709, 'app_version':"Generated at %s"%(datetime.now()), 'location':"local", "container_path": BETA_SPLUNK_ADD_ON_FOR_SYSMON_CONTAINER_PATH}
     except Exception as e:
         print("Failed to grab beta sysmon at ~/Downloads/Splunk_TA_microsoft_sysmon-1.0.2-B1.spl. Using the one from Splunkbase")
         APPS_DICT['SPLUNK_ADD_ON_FOR_SYSMON'] = {"app_number":5709, 'app_version':"1.0.1", 'location':'splunkbase'}
diff --git a/automated_detection_testing/ci/detection_testing_batch/new_arguments.py b/automated_detection_testing/ci/detection_testing_batch/new_arguments.py
new file mode 100644
index 0000000000..a486dcecb2
--- /dev/null
+++ b/automated_detection_testing/ci/detection_testing_batch/new_arguments.py
@@ -0,0 +1,51 @@
+import argparse
+import sys
+
+
+def main(args):
+    parser = argparse.ArgumentParser(
+        description="Use 'SOME_PROGRAM_NAME_STRING --help' to get help with the arguments")
+    actions_parser = parser.add_subparsers(title="test action")
+
+    configure_parser = actions_parser.add_parser(
+        "configure", help="configure a test run")
+
+    configure_parser.add_argument(
+        '-c', '--context', required=True, help="Some help as a test")
+
+    test_parser = actions_parser.add_parser("test", help="run a test")
+    test_parser.add_argument('-b', '--branch', required=True,
+                             help="The branch whose detections you would like to test.  "\
+                                  "In order to calculate new/changed detections, the detections "\
+                                  "in this branch will be diffed against those in the 'develop' branch")
+    test_parser.add_argument(
+        '-pr', '--pull_request_number', required=False, help="Pull request number.")
+
+    mode_parser = test_parser.add_subparsers(title="Test Modes", required=True)
+    new_parser = mode_parser.add_parser("new", 
+                                        #aliases=['changed'], 
+                                        help="Test only the new or changed detections")
+    selected_parser = mode_parser.add_parser("selected", help="Test only the detections from the target branch that "\
+                                                              " are passed on the command line.  These can be given as "\
+                                                              "a list of files or as a file containing a list of files.")
+    selected_group = selected_parser.add_mutually_exclusive_group(required=True)
+    selected_group.add_argument('-df', '--detections_file', type=argparse.FileType('r'), 
+                                required=False, help="A file containing a list of detections to run, one per line")
+    selected_group.add_argument('-dl', '--detections_list', 
+                                required=False, help="The names of files that you want to test, separated by commas.  "\
+                                                     "Do not include spaces between the detections!")
+    
+
+
+    
+    all_parser = mode_parser.add_parser("all", 
+                                        #aliases=['everything'], 
+                                        help="Test all of the detections in the target branch.  "\
+                                             "Note that this could take a very long time.")
+
+
+    parser.parse_args()
+
+
+if __name__ == "__main__":
+    main(sys.argv[1:])

From 80a1b9ad9b4095dafb880440685b35291f5064f4 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Tue, 16 Nov 2021 17:19:43 -0800
Subject: [PATCH 064/166] More progress adding and parsing new arguments.

---
 .../detection_testing_batch/new_arguments.py  | 48 ++++++++++++++++---
 1 file changed, 41 insertions(+), 7 deletions(-)

diff --git a/automated_detection_testing/ci/detection_testing_batch/new_arguments.py b/automated_detection_testing/ci/detection_testing_batch/new_arguments.py
index a486dcecb2..037a12454e 100644
--- a/automated_detection_testing/ci/detection_testing_batch/new_arguments.py
+++ b/automated_detection_testing/ci/detection_testing_batch/new_arguments.py
@@ -3,6 +3,9 @@ import sys
 
 
 def main(args):
+
+    default_args = {}
+
     parser = argparse.ArgumentParser(
         description="Use 'SOME_PROGRAM_NAME_STRING --help' to get help with the arguments")
     actions_parser = parser.add_subparsers(title="test action")
@@ -21,10 +24,41 @@ def main(args):
     test_parser.add_argument(
         '-pr', '--pull_request_number', required=False, help="Pull request number.")
 
+    VALID_DETECTION_TYPES = ['endpoint', 'cloud', 'network']
+
+    #Common Test Arguments
+    test_parser.add_argument('-t', '--types', type=str, action="append", 
+                             help="Detection types to test. Can be one or more of %s"%(VALID_DETECTION_TYPES))
+    
+    
+    test_parser.add_argument('-e', '--escu_package', type=argparse.FileType('rb'), required=False, 
+                                        help="A previously generated ESCU PAcklage to use.  If you pass this "\
+                                             "argument, a new ESCU package will not be generated.  Note that this "\
+                                             "may cause newly-written detections to fail (for example, if they "\
+                                             "leverage macros that have been added or modified).")
+   
+    test_parser.add_argument('-p','--persist_security_content', required=False, action="store_true",
+                             help="Assumes security_content directory already exists.  Don't check it out and overwrite it again.  Saves "\
+                             "time and allows you to test a detection that you've updated.  Runs generate again in case you have "\
+                             "updated macros or anything else.  Especially useful for quick, local, iterative testing.")
+
+
+    test_parser.add_argument('tag', '--container_tag', required=False, default = default_args['container_tag'], 
+                             help="The tag of the Splunk Container to use.  Tags are located "\
+                                  "at https://hub.docker.com/r/splunk/splunk/tags")
+
+    test_parser.add_argument("-show", "--show_password", required=False, default=False, action='store_true', 
+                             help="Show the generated password to use to login to splunk.  For a CI/CD run, "\
+                            "you probably don't want this.")
+
+    
+    #Mode settings
     mode_parser = test_parser.add_subparsers(title="Test Modes", required=True)
-    new_parser = mode_parser.add_parser("new", 
-                                        #aliases=['changed'], 
+    #NEW
+    new_parser = mode_parser.add_parser("changes", 
                                         help="Test only the new or changed detections")
+
+    #SELECTED
     selected_parser = mode_parser.add_parser("selected", help="Test only the detections from the target branch that "\
                                                               " are passed on the command line.  These can be given as "\
                                                               "a list of files or as a file containing a list of files.")
@@ -35,16 +69,16 @@ def main(args):
                                 required=False, help="The names of files that you want to test, separated by commas.  "\
                                                      "Do not include spaces between the detections!")
     
-
-
-    
+    #ALL
     all_parser = mode_parser.add_parser("all", 
-                                        #aliases=['everything'], 
                                         help="Test all of the detections in the target branch.  "\
                                              "Note that this could take a very long time.")
 
 
-    parser.parse_args()
+    a = parser.parse_args()
+    
+    print(a)
+    print(a.__dict__)
 
 
 if __name__ == "__main__":

From 56f4223ecdbe86cd5d7ad5df6588651791e1c37f Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Wed, 17 Nov 2021 14:09:28 -0800
Subject: [PATCH 065/166] Added some supporting code that will make generating
 config files and validating arguments easier.

---
 .../modules/jsonschema_errorprinter.py        | 134 +++++++++++++
 .../modules/validate_args.py                  | 181 ++++++++++++++++++
 .../detection_testing_batch/new_arguments.py  |  35 +++-
 .../detection_testing_batch/requirements.txt  |   3 +
 4 files changed, 345 insertions(+), 8 deletions(-)
 create mode 100644 automated_detection_testing/ci/detection_testing_batch/modules/jsonschema_errorprinter.py
 create mode 100644 automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py

diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/jsonschema_errorprinter.py b/automated_detection_testing/ci/detection_testing_batch/modules/jsonschema_errorprinter.py
new file mode 100644
index 0000000000..438657e8f4
--- /dev/null
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/jsonschema_errorprinter.py
@@ -0,0 +1,134 @@
+"""
+Courtesy https://github.com/ccpgames/jsonschema-errorprinter with minor
+updates to support Python 3 (changed cStringIO to io), to print out
+multiple errors, and a few other small changes.
+
+Licensed under the MIT License, reproduced below:
+Copyright © 2015 CCP hf.
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be included
+in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 
+THERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
+OR OTHER DEALINGS IN THE SOFTWARE.
+"""
+
+"""
+    Json Schema Validation Error Pretty-printer.
+    ---------------------------------------------------
+
+    Makes a user friendly error message from a ValidationError.
+
+"""
+import json
+import io
+
+
+
+import jsonschema
+
+
+def check_json(json_object, schema, context=None):
+    try:
+        validator = jsonschema.Draft7Validator(schema, jsonschema.FormatChecker())
+        errors_formatted = []
+        for error in sorted(validator.iter_errors(json_object), key=str):
+
+            #validate(json_object, schema, format_checker=FormatChecker())
+            #except jsonschema.ValidationError as e:
+            report = generate_validation_error_report(error, json_object)
+            #note = "\n*** Note - If there is more than one error, only the first error is shown ***\n\n"
+            if context:
+                errors_formatted.append("Schema check failed for '{}'\n{}".format(context, report))
+                #return note + "Schema check failed for '{}'\n{}".format(context, report)
+            else:
+                errors_formatted.append("Schema check failed.\n{}".format(report))
+                #return note + "Schema check failed.\n{}".format(report)
+        return errors_formatted
+    except Exception as e:
+        #Some error occurred, probably related to the schema itself
+        raise(Exception("Error validating the JSON Schema: %s"%(str(e))))
+            
+
+
+def generate_validation_error_report(
+    e,
+    json_object,
+    lines_before=7,
+    lines_after=7
+    ):
+    """
+    Generate a detailed report of a schema validation error.
+
+    'e' is a jsonschema.ValidationError exception that errored on
+    'json_object'.
+
+    Steps to discover the location of the validation error:
+    1. Traverse the json object using the 'path' in the validation exception
+       and replace the offending value with a special marker.
+    2. Pretty-print the json object indendented json text.
+    3. Search for the special marker in the json text to find the actual
+       line number of the error.
+    4. Make a report by showing the error line with a context of
+      'lines_before' and 'lines_after' number of lines on each side.
+    """
+
+    if json_object is None:
+        return "'json_object' cannot be None."
+    if not e.path:
+        return str(e)
+    marker = "3fb539deef7c4e2991f265c0a982f5ea"
+
+    # Find the object that is erroring, and replace it with the marker.
+    ob_tmp = json_object
+    for entry in list(e.path)[:-1]:
+        ob_tmp = ob_tmp[entry]
+
+    orig, ob_tmp[e.path[-1]] = ob_tmp[e.path[-1]], marker
+
+    # Pretty print the object and search for the marker.
+    json_error = json.dumps(json_object, indent=4)
+    string_io_instance = io.StringIO(json_error)
+    errline = None
+
+    for lineno, text in enumerate(string_io_instance):
+            if marker in text:
+                errline = lineno
+                break
+
+    if errline is not None:
+        # Re-create report.
+        report = []
+        ob_tmp[e.path[-1]] = orig
+        json_error = json.dumps(json_object, indent=4)
+        string_io_instance = io.StringIO(json_error)
+
+        for lineno, text in enumerate(string_io_instance):
+            if lineno == errline:
+                line_text = "{:4}: >>>".format(lineno+1)
+            else:
+                line_text = "{:4}:    ".format(lineno+1)
+            report.append(line_text + text.rstrip("\n"))
+
+        report = report[max(0, errline-lines_before):errline+1+lines_after]
+
+        s = "Error in line {}:\n".format(errline+1)
+        s += "\n".join(report)
+        s+= '\n\tREASON:' + str(e).split('\n')[0]
+        #s += "\n\n" + str(e).replace("u'", "'")
+    else:
+        s = str(e)
+    return s
\ No newline at end of file
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py b/automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py
new file mode 100644
index 0000000000..df1195aa62
--- /dev/null
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py
@@ -0,0 +1,181 @@
+import argparse
+import io
+import json
+import jsonschema_errorprinter
+
+
+setup_schema = {
+    "type": "object",
+    "properties": {
+        "action": {
+            "type": "string",
+                    "enum": ["configure", "test"]
+            
+        },
+
+        "branch": {
+            "type":"string"
+        },
+
+        "container_tag": {
+            "type": "string"
+        },
+        
+        "interactive_failure": {
+            "type": "boolean"
+        },
+
+        "local_apps": {
+            "type": "array",
+            "items": {
+                "type": "object",
+                "properties": {
+                    "local_path": {
+                        "type": "string"
+                    },
+                    "app_name": {
+                        "type": "string"
+                    },
+                    "app_number": {
+                        "type": "integer"
+                    },
+                    "app_version": {
+                        "type": "string"
+                    }
+                }
+            }
+        },
+
+        "mode": {
+            "type":"string",
+            "enum": ["changes", "selected", "new"]
+        },
+
+        "num_containers": {
+            "type": "integer",
+            "minimum": 1
+        },
+
+        "persist_security_content": {
+            "type": "boolean"
+        },
+
+        "pr_number": {
+            "type":"integer"
+        },
+
+        "reuse_image": {
+            "type": "boolean"
+        },
+
+        "show_password": {
+            "type": "boolean"
+        },
+
+        "splunkbase_apps": {
+            "type": "array",
+            "items": {
+                "type": "object",
+                "properties": {
+                    "app_name": {
+                        "type": "string"
+                    },
+                    "app_number": {
+                        "type": "integer"
+                    },
+                    "app_version": {
+                        "type": "string"
+                    },
+                    "app_name": {
+                        "type": "string"
+                    }
+                }
+            }
+        },
+
+        "types": {
+            "type": "array",
+            "enum": ["endpoint", "cloud", "network"],
+            "maxItems": 3,
+            "maxItemsssss":2
+        },
+        
+
+        
+        
+
+        
+
+    }
+}
+
+import jsonschema
+import jsonschema.exceptions
+import sys
+
+def v(configuration:dict)->bool:
+    #v = jsonschema.Draft201909Validator(argument_schema)
+    test = {"action":"tests", "branch":15}
+    try:
+        validation_results = jsonschema_errorprinter.check_json(test, setup_schema)
+        if len(validation_results) == 0:
+            print("Input configuration successfully validated!")
+            return True
+        else:
+            print("[%d] failures detected during validation of the configuration!"%(len(validation_results)))
+            for error in validation_results:
+                print(error,end="\n\n", file=sys.stderr)
+            return False            
+    except Exception as e:
+        print(str(e))
+        return False
+
+
+    '''
+    try:
+        v.validate({"action":"doot", "branch":"15"}  )
+    except jsonschema.exceptions.ValidationError as e:
+        print("Error validating the json", file=sys.stderr)
+        print(e)
+        return False
+        
+    except jsonschema.exceptions.SchemaError as e:
+        print("Error validating the schema", file=sys.stderr)
+    '''
+if __name__ == "__main__":
+    v()
+'''
+def load(json_settings: io.TextIOWrapper) -> dict:
+    default_settings = json.load(json_settings)
+    return default_settings
+
+
+def load_and_validate(json_settings: io.TextIOWrapper) -> dict:
+    settings = load(json_settings)
+    validate(settings)
+    return settings
+
+
+def validate(args: dict) -> bool:
+    validate_common_arguments()
+
+    validate_mode()
+
+    return True
+
+
+def validate_mode(args: dict) -> bool:
+    return True
+
+
+def validate_mode_selected(args: dict) -> bool:
+    return True
+
+
+def validate_mode_changes(args: dict) -> -bool:
+    return True
+
+
+def validate_mode_all(args: dict) -> bool:
+    return True
+'''
\ No newline at end of file
diff --git a/automated_detection_testing/ci/detection_testing_batch/new_arguments.py b/automated_detection_testing/ci/detection_testing_batch/new_arguments.py
index 037a12454e..40a3f38409 100644
--- a/automated_detection_testing/ci/detection_testing_batch/new_arguments.py
+++ b/automated_detection_testing/ci/detection_testing_batch/new_arguments.py
@@ -1,10 +1,18 @@
 import argparse
+import json
+from modules import validate_args
 import sys
 
-
+DEFAULT_CONFIG_FILE = "defaults.json"
 def main(args):
 
-    default_args = {}
+    try:
+        with open(DEFAULT_CONFIG_FILE, 'r') as settings_file:
+            default_settings = json.load(settings_file)
+    except Exception as e:
+        print("Error loading settings file %s: %s"%(DEFAULT_CONFIG_FILE, str(e)), file=sys.stderr)
+        sys.exit(1)
+    
 
     parser = argparse.ArgumentParser(
         description="Use 'SOME_PROGRAM_NAME_STRING --help' to get help with the arguments")
@@ -14,7 +22,7 @@ def main(args):
         "configure", help="configure a test run")
 
     configure_parser.add_argument(
-        '-c', '--context', required=True, help="Some help as a test")
+        '-o', '--output_config', required=True, help="Name of config file to generate")
 
     test_parser = actions_parser.add_parser("test", help="run a test")
     test_parser.add_argument('-b', '--branch', required=True,
@@ -43,7 +51,7 @@ def main(args):
                              "updated macros or anything else.  Especially useful for quick, local, iterative testing.")
 
 
-    test_parser.add_argument('tag', '--container_tag', required=False, default = default_args['container_tag'], 
+    test_parser.add_argument('-tag', '--container_tag', required=False, default = default_args['container_tag'], 
                              help="The tag of the Splunk Container to use.  Tags are located "\
                                   "at https://hub.docker.com/r/splunk/splunk/tags")
 
@@ -51,6 +59,13 @@ def main(args):
                              help="Show the generated password to use to login to splunk.  For a CI/CD run, "\
                             "you probably don't want this.")
 
+    test_parser.add_argument('-r','--reuse_image', required=False, default=True, action='store_true', 
+                             help="Should existing images be re-used, or should they be redownloaded?")
+
+    test_parser.add_argument('-i', '--interactive_failure', required=False, default=False, action='store_true',
+                            help="If a test fails, should we pause before removing data so that the search can be debugged?")
+    
+
     
     #Mode settings
     mode_parser = test_parser.add_subparsers(title="Test Modes", required=True)
@@ -75,10 +90,14 @@ def main(args):
                                              "Note that this could take a very long time.")
 
 
-    a = parser.parse_args()
-    
-    print(a)
-    print(a.__dict__)
+    args = parser.parse_args()
+    try:
+        validate_args.validate(args.__dict__)
+
+    except Exception as e:
+        print("Error validating command line arguments: [%s]"%(str(e)))
+        sys.exit(1)
+
 
 
 if __name__ == "__main__":
diff --git a/automated_detection_testing/ci/detection_testing_batch/requirements.txt b/automated_detection_testing/ci/detection_testing_batch/requirements.txt
index d4a44f878c..defe6be649 100644
--- a/automated_detection_testing/ci/detection_testing_batch/requirements.txt
+++ b/automated_detection_testing/ci/detection_testing_batch/requirements.txt
@@ -12,3 +12,6 @@ splunk-packaging-toolkit==1.0.1
 #newest version of docker for managing the splunk containers
 #we will freeze at a specific version later
 docker==5.0.3
+
+#For help getting and parsing the configuration
+jsonschema

From 82d53ccbd9978a7a862b65a148ba1ed23b7bc27f Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Wed, 17 Nov 2021 16:23:39 -0800
Subject: [PATCH 066/166] Moving to json based configuration files and a
 framework to generate that file, too

---
 .../modules/jsonschema_errorprinter.py        |  79 ++++++---
 .../modules/new_arguments2.py                 | 138 +++++++++++++++
 .../modules/validate_args.py                  | 166 ++++++++++++------
 .../detection_testing_batch/new_arguments.py  |   2 +
 4 files changed, 308 insertions(+), 77 deletions(-)
 create mode 100644 automated_detection_testing/ci/detection_testing_batch/modules/new_arguments2.py

diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/jsonschema_errorprinter.py b/automated_detection_testing/ci/detection_testing_batch/modules/jsonschema_errorprinter.py
index 438657e8f4..afb07ebdf1 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/jsonschema_errorprinter.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/jsonschema_errorprinter.py
@@ -1,7 +1,8 @@
 """
 Courtesy https://github.com/ccpgames/jsonschema-errorprinter with minor
 updates to support Python 3 (changed cStringIO to io), to print out
-multiple errors, and a few other small changes.
+multiple errors, the ability to place default values, and a few
+other small changes.
 
 Licensed under the MIT License, reproduced below:
 Copyright © 2015 CCP hf.
@@ -33,35 +34,69 @@ OR OTHER DEALINGS IN THE SOFTWARE.
     Makes a user friendly error message from a ValidationError.
 
 """
-import json
+
+
+
 import io
-
-
-
+import json
 import jsonschema
+import jsonschema.validators
+
+# The 'default' field is really just for documentation in the
+# json schema.  We would like to use it to actually fill in
+# values when they aren't supplied.  This code is provided
+# by the jsonschema project itself because this behavior
+# is not part of the default jsonschema definition
+# https://python-jsonschema.readthedocs.io/en/latest/faq/
+def extend_with_default(validator_class):
+    validate_properties = validator_class.VALIDATORS["properties"]
+
+    def set_defaults(validator, properties, instance, schema):
+        for property, subschema in properties.items():
+            if "default" in subschema:
+                instance.setdefault(property, subschema["default"])
+
+        for error in validate_properties(
+            validator, properties, instance, schema,
+        ):
+            yield error
+
+    return jsonschema.validators.extend(
+        validator_class, {"properties": set_defaults},
+    )
 
 
-def check_json(json_object, schema, context=None):
+def check_json(json_object, schema, context=None) -> tuple[list[str], dict]:
     try:
-        validator = jsonschema.Draft7Validator(schema, jsonschema.FormatChecker())
+        DefaultValidatingDraft7Validator = extend_with_default(
+                jsonschema.Draft7Validator)
+        
+        validator = DefaultValidatingDraft7Validator(schema, jsonschema.FormatChecker())
         errors_formatted = []
         for error in sorted(validator.iter_errors(json_object), key=str):
 
             #validate(json_object, schema, format_checker=FormatChecker())
-            #except jsonschema.ValidationError as e:
+            # except jsonschema.ValidationError as e:
             report = generate_validation_error_report(error, json_object)
             #note = "\n*** Note - If there is more than one error, only the first error is shown ***\n\n"
             if context:
-                errors_formatted.append("Schema check failed for '{}'\n{}".format(context, report))
-                #return note + "Schema check failed for '{}'\n{}".format(context, report)
+                errors_formatted.append(
+                    "Schema check failed for '{}'\n{}".format(context, report))
+                # return note + "Schema check failed for '{}'\n{}".format(context, report)
             else:
-                errors_formatted.append("Schema check failed.\n{}".format(report))
-                #return note + "Schema check failed.\n{}".format(report)
-        return errors_formatted
+                errors_formatted.append(
+                    "Schema check failed.\n{}".format(report))
+                # return note + "Schema check failed.\n{}".format(report)
+        if len(errors_formatted) == 0:
+            #DefaultValidatingDraft7Validator = extend_with_default(
+            #    jsonschema.Draft7Validator)
+            #DefaultValidatingDraft7Validator(schema).validate(json_object)
+            return (errors_formatted, json_object)
+        else:
+            return (errors_formatted, {})
     except Exception as e:
-        #Some error occurred, probably related to the schema itself
-        raise(Exception("Error validating the JSON Schema: %s"%(str(e))))
-            
+        # Some error occurred, probably related to the schema itself
+        raise(Exception("Error validating the JSON Schema: %s" % (str(e))))
 
 
 def generate_validation_error_report(
@@ -69,7 +104,7 @@ def generate_validation_error_report(
     json_object,
     lines_before=7,
     lines_after=7
-    ):
+):
     """
     Generate a detailed report of a schema validation error.
 
@@ -105,9 +140,9 @@ def generate_validation_error_report(
     errline = None
 
     for lineno, text in enumerate(string_io_instance):
-            if marker in text:
-                errline = lineno
-                break
+        if marker in text:
+            errline = lineno
+            break
 
     if errline is not None:
         # Re-create report.
@@ -127,8 +162,8 @@ def generate_validation_error_report(
 
         s = "Error in line {}:\n".format(errline+1)
         s += "\n".join(report)
-        s+= '\n\tREASON:' + str(e).split('\n')[0]
+        s += '\n\tREASON:' + str(e).split('\n')[0]
         #s += "\n\n" + str(e).replace("u'", "'")
     else:
         s = str(e)
-    return s
\ No newline at end of file
+    return s
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/new_arguments2.py b/automated_detection_testing/ci/detection_testing_batch/modules/new_arguments2.py
new file mode 100644
index 0000000000..1dba7e68bf
--- /dev/null
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/new_arguments2.py
@@ -0,0 +1,138 @@
+import argparse
+import json
+from typing import OrderedDict
+import validate_args
+import sys
+
+
+def configure_action(args):
+    print("WE ARE CONFIGURING!")
+    settings = OrderedDict()
+    
+    settings = validate_args.v(validate_args.setup_schema)
+    if settings is False:
+        print("Failure while processing settings.\n\tQuitting...", file=sys.stderr)
+        sys.exit(1)
+    for arg in settings:
+        choice = input("%s [%s]:"%(arg,settings[arg]))
+        
+
+    
+
+DEFAULT_CONFIG_FILE = "defaults.json"
+def main(args):
+    '''
+    try:
+        with open(DEFAULT_CONFIG_FILE, 'r') as settings_file:
+            default_settings = json.load(settings_file)
+    except Exception as e:
+        print("Error loading settings file %s: %s"%(DEFAULT_CONFIG_FILE, str(e)), file=sys.stderr)
+        sys.exit(1)
+    '''
+
+    parser = argparse.ArgumentParser(
+        description="Use 'SOME_PROGRAM_NAME_STRING --help' to get help with the arguments")
+    parser.set_defaults(func=lambda _: parser.print_help())
+    
+    actions_parser = parser.add_subparsers(title="Action")
+
+    configure_parser = actions_parser.add_parser(
+        "configure", help="Configure a test run")
+    configure_parser.set_defaults(func=configure_action)
+    configure_parser.add_argument('-i', '--input_config_file', required=False, type=argparse.FileType('r'), default=DEFAULT_CONFIG_FILE, help="The config file to base the configuration off of.")
+    configure_parser.add_argument('-o', '--output_config_file', required=False, type=argparse.FileType('w'), help="The config file to write the configuration off of.")
+    
+
+
+    test_parser = actions_parser.add_parser(
+        "run", help="Run a test")
+
+    args = parser.parse_args()
+    #Run the appropriate parser
+
+    args.func(args)
+
+    '''
+
+    configure_parser.add_argument(
+        '-o', '--output_config', required=True, help="Name of config file to generate")
+
+    test_parser = actions_parser.add_parser("test", help="run a test")
+    test_parser.add_argument('-b', '--branch', required=True,
+                             help="The branch whose detections you would like to test.  "\
+                                  "In order to calculate new/changed detections, the detections "\
+                                  "in this branch will be diffed against those in the 'develop' branch")
+    test_parser.add_argument(
+        '-pr', '--pull_request_number', required=False, help="Pull request number.")
+
+    VALID_DETECTION_TYPES = ['endpoint', 'cloud', 'network']
+
+    #Common Test Arguments
+    test_parser.add_argument('-t', '--types', type=str, action="append", 
+                             help="Detection types to test. Can be one or more of %s"%(VALID_DETECTION_TYPES))
+    
+    
+    test_parser.add_argument('-e', '--escu_package', type=argparse.FileType('rb'), required=False, 
+                                        help="A previously generated ESCU PAcklage to use.  If you pass this "\
+                                             "argument, a new ESCU package will not be generated.  Note that this "\
+                                             "may cause newly-written detections to fail (for example, if they "\
+                                             "leverage macros that have been added or modified).")
+   
+    test_parser.add_argument('-p','--persist_security_content', required=False, action="store_true",
+                             help="Assumes security_content directory already exists.  Don't check it out and overwrite it again.  Saves "\
+                             "time and allows you to test a detection that you've updated.  Runs generate again in case you have "\
+                             "updated macros or anything else.  Especially useful for quick, local, iterative testing.")
+
+
+    test_parser.add_argument('-tag', '--container_tag', required=False, default = default_args['container_tag'], 
+                             help="The tag of the Splunk Container to use.  Tags are located "\
+                                  "at https://hub.docker.com/r/splunk/splunk/tags")
+
+    test_parser.add_argument("-show", "--show_password", required=False, default=False, action='store_true', 
+                             help="Show the generated password to use to login to splunk.  For a CI/CD run, "\
+                            "you probably don't want this.")
+
+    test_parser.add_argument('-r','--reuse_image', required=False, default=True, action='store_true', 
+                             help="Should existing images be re-used, or should they be redownloaded?")
+
+    test_parser.add_argument('-i', '--interactive_failure', required=False, default=False, action='store_true',
+                            help="If a test fails, should we pause before removing data so that the search can be debugged?")
+    
+
+    
+    #Mode settings
+    mode_parser = test_parser.add_subparsers(title="Test Modes", required=True)
+    #NEW
+    new_parser = mode_parser.add_parser("changes", 
+                                        help="Test only the new or changed detections")
+
+    #SELECTED
+
+    selected_parser = mode_parser.add_parser("selected", help="Test only the detections from the target branch that "\
+                                                              " are passed on the command line.  These can be given as "\
+                                                              "a list of files or as a file containing a list of files.")
+    selected_group = selected_parser.add_mutually_exclusive_group(required=True)
+    selected_group.add_argument('-df', '--detections_file', type=argparse.FileType('r'), 
+                                required=False, help="A file containing a list of detections to run, one per line")
+    selected_group.add_argument('-dl', '--detections_list', 
+                                required=False, help="The names of files that you want to test, separated by commas.  "\
+                                                     "Do not include spaces between the detections!")
+    
+    #ALL
+    all_parser = mode_parser.add_parser("all", 
+                                        help="Test all of the detections in the target branch.  "\
+                                             "Note that this could take a very long time.")
+
+
+    args = parser.parse_args()
+    try:
+        validate_args.validate(args.__dict__)
+
+    except Exception as e:
+        print("Error validating command line arguments: [%s]"%(str(e)))
+        sys.exit(1)
+    '''
+
+
+if __name__ == "__main__":
+    main(sys.argv[1:])
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py b/automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py
index df1195aa62..7664dae569 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py
@@ -1,28 +1,32 @@
+import sys
+import jsonschema.exceptions
+import jsonschema
 import argparse
 import io
 import json
 import jsonschema_errorprinter
+from typing import Union
 
 
+# If we want, we can easily add a description field to any of the objects here!
 setup_schema = {
     "type": "object",
     "properties": {
-        "action": {
-            "type": "string",
-                    "enum": ["configure", "test"]
-            
-        },
+        
 
         "branch": {
-            "type":"string"
+            "type": "string",
+            "default": "develop"
         },
 
         "container_tag": {
-            "type": "string"
+            "type": "string",
+            "default": "latest"
         },
-        
+
         "interactive_failure": {
-            "type": "boolean"
+            "type": "boolean",
+            "default": False
         },
 
         "local_apps": {
@@ -30,9 +34,6 @@ setup_schema = {
             "items": {
                 "type": "object",
                 "properties": {
-                    "local_path": {
-                        "type": "string"
-                    },
                     "app_name": {
                         "type": "string"
                     },
@@ -41,35 +42,46 @@ setup_schema = {
                     },
                     "app_version": {
                         "type": "string"
-                    }
-                }
+                    },
+                    "local_path": {
+                        "type": "string"
+                    },
+                },
+                "default": []
             }
         },
 
         "mode": {
-            "type":"string",
-            "enum": ["changes", "selected", "new"]
+            "type": "string",
+            "enum": ["changes", "selected", "new"],
+            "default": "changes"
         },
 
         "num_containers": {
             "type": "integer",
-            "minimum": 1
+            "minimum": 1,
+            "default": 1
         },
 
         "persist_security_content": {
-            "type": "boolean"
+            "type": "boolean",
+            "default": False
         },
 
         "pr_number": {
-            "type":"integer"
+            "type": ["integer", "null"],
+            "default": None
         },
 
         "reuse_image": {
-            "type": "boolean"
+            "type": "boolean",
+            "default": True
         },
 
         "show_password": {
-            "type": "boolean"
+            "type": "boolean",
+            "default": False
+
         },
 
         "splunkbase_apps": {
@@ -85,53 +97,90 @@ setup_schema = {
                     },
                     "app_version": {
                         "type": "string"
-                    },
-                    "app_name": {
-                        "type": "string"
                     }
-                }
-            }
+                },
+            },
+            "default":  [
+                {"app_name": "SPLUNK_ADD_ON_FOR_AMAZON_WEB_SERVICES",
+                    "app_number": 1876, "app_version": "5.2.0"},
+                {"app_name": "SPLUNK_ADD_ON_FOR_MICROSOFT_OFFICE_365",
+                 "app_number": 4055, "app_version": "2.2.0"},
+                {"app_name": "SPLUNK_ADD_ON_FOR_AMAZON_KINESIS_FIREHOSE",
+                 "app_number": 3719, "app_version": "1.3.2"},
+                {"app_name": "SPLUNK_ANALYTIC_STORY_EXECUTION_APP",
+                 "app_number": 4971, "app_version": "2.0.3"},
+                {"app_name": "PYTHON_FOR_SCIENTIC_COMPUTING_LINUX_64_BIT",
+                 "app_number": 2882, "app_version": "2.0.2"},
+                {"app_name": "SPLUNK_MACHINE_LEARNING_TOOLKIT",
+                 "app_number": 2890, "app_version": "5.2.2"},
+                {"app_name": "SPLUNK_APP_FOR_STREAM",
+                 "app_number": 1809, "app_version": "8.0.1"},
+                {"app_name": "SPLUNK_ADD_ON_FOR_STREAM_WIRE_DATA",
+                 "app_number": 5234, "app_version": "8.0.1"},
+                {"app_name": "SPLUNK_ADD_ON_FOR_STREAM_FORWARDERS",
+                 "app_number": 5238, "app_version": "8.0.1"},
+                {"app_name": "SPLUNK_ADD_ON_FOR_ZEEK_AKA_BRO",
+                 "app_number": 1617, "app_version": "4.0.0"},
+                {"app_name": "SPLUNK_ADD_ON_FOR_UNIX_AND_LINUX",
+                 "app_number": 833, "app_version": "8.3.1"},
+                {"app_name": "SPLUNK_COMMON_INFORMATION_MODEL",
+                 "app_number": 1621, "app_version": "4.20.2"}
+            ]
+        },
+        "splunkbase_username": {
+            "type": ["string","null"],
+            "default": None
+        },
+        "splunkbase_password": {
+            "type": ["string", "null"],
+            "default": None
+        },
+        "splunk_container_apps_directory":{
+            "type":"string",
+            "default": "/opt/splunk/etc/apps"
+        },
+        "local_base_container_name": {
+            "type": "string",
+            "default": "splunk_test_%d"
+        },
+        
+        "mock": {
+            "type": "boolean",
+            "default": False
         },
 
         "types": {
             "type": "array",
-            "enum": ["endpoint", "cloud", "network"],
-            "maxItems": 3,
-            "maxItemsssss":2
+            "items": {
+                "type": "string",
+                "enum": ["endpoint", "cloud", "network"]
+            },
+            "default": ["endpoint", "cloud", "network"]
         },
-        
-
-        
-        
-
-        
-
     }
 }
 
-import jsonschema
-import jsonschema.exceptions
-import sys
 
-def v(configuration:dict)->bool:
+def v(configuration: dict) -> Union[bool, dict]:
     #v = jsonschema.Draft201909Validator(argument_schema)
-    test = {"action":"tests", "branch":15}
+
     try:
-        validation_results = jsonschema_errorprinter.check_json(test, setup_schema)
-        if len(validation_results) == 0:
+        validation_errors, validated_json = jsonschema_errorprinter.check_json(
+            configuration, setup_schema)
+        if len(validation_errors) == 0:
             print("Input configuration successfully validated!")
-            return True
+            return validated_json
         else:
-            print("[%d] failures detected during validation of the configuration!"%(len(validation_results)))
-            for error in validation_results:
-                print(error,end="\n\n", file=sys.stderr)
-            return False            
+            print("[%d] failures detected during validation of the configuration!" % (
+                len(validation_errors)))
+            for error in validation_errors:
+                print(error, end="\n\n", file=sys.stderr)
+            return False
     except Exception as e:
-        print(str(e))
+        print(str(e), file=sys.stderr)
         return False
 
-
-    '''
+    """
     try:
         v.validate({"action":"doot", "branch":"15"}  )
     except jsonschema.exceptions.ValidationError as e:
@@ -141,10 +190,17 @@ def v(configuration:dict)->bool:
         
     except jsonschema.exceptions.SchemaError as e:
         print("Error validating the schema", file=sys.stderr)
-    '''
+    """
+
+
 if __name__ == "__main__":
-    v()
-'''
+    c = v({"action": "test", "branch": "wow"})
+    print(c)
+    if c is False:
+        print("whoops")
+    else:
+        print(c.keys())
+"""
 def load(json_settings: io.TextIOWrapper) -> dict:
     default_settings = json.load(json_settings)
     return default_settings
@@ -178,4 +234,4 @@ def validate_mode_changes(args: dict) -> -bool:
 
 def validate_mode_all(args: dict) -> bool:
     return True
-'''
\ No newline at end of file
+"""
diff --git a/automated_detection_testing/ci/detection_testing_batch/new_arguments.py b/automated_detection_testing/ci/detection_testing_batch/new_arguments.py
index 40a3f38409..b485bd9d69 100644
--- a/automated_detection_testing/ci/detection_testing_batch/new_arguments.py
+++ b/automated_detection_testing/ci/detection_testing_batch/new_arguments.py
@@ -3,6 +3,7 @@ import json
 from modules import validate_args
 import sys
 
+
 DEFAULT_CONFIG_FILE = "defaults.json"
 def main(args):
 
@@ -74,6 +75,7 @@ def main(args):
                                         help="Test only the new or changed detections")
 
     #SELECTED
+
     selected_parser = mode_parser.add_parser("selected", help="Test only the detections from the target branch that "\
                                                               " are passed on the command line.  These can be given as "\
                                                               "a list of files or as a file containing a list of files.")

From cf8d5f2dffa3aaead48b98a3690b52c166a97002 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Thu, 18 Nov 2021 13:01:55 -0800
Subject: [PATCH 067/166] Much better configuration parsing, validation, and
 substitution of defaults using jsonschema.

---
 .../modules/jsonschema_errorprinter.py        |  4 ++
 .../modules/new_arguments2.py                 | 55 ++++++++++++++++---
 .../modules/validate_args.py                  | 19 +++----
 3 files changed, 61 insertions(+), 17 deletions(-)

diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/jsonschema_errorprinter.py b/automated_detection_testing/ci/detection_testing_batch/modules/jsonschema_errorprinter.py
index afb07ebdf1..9a1eb91888 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/jsonschema_errorprinter.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/jsonschema_errorprinter.py
@@ -71,13 +71,17 @@ def check_json(json_object, schema, context=None) -> tuple[list[str], dict]:
         DefaultValidatingDraft7Validator = extend_with_default(
                 jsonschema.Draft7Validator)
         
+        
         validator = DefaultValidatingDraft7Validator(schema, jsonschema.FormatChecker())
+        #validator = jsonschema.Draft7Validator(schema, jsonschema.FormatChecker())
         errors_formatted = []
+        
         for error in sorted(validator.iter_errors(json_object), key=str):
 
             #validate(json_object, schema, format_checker=FormatChecker())
             # except jsonschema.ValidationError as e:
             report = generate_validation_error_report(error, json_object)
+            
             #note = "\n*** Note - If there is more than one error, only the first error is shown ***\n\n"
             if context:
                 errors_formatted.append(
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/new_arguments2.py b/automated_detection_testing/ci/detection_testing_batch/modules/new_arguments2.py
index 1dba7e68bf..7a49308aad 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/new_arguments2.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/new_arguments2.py
@@ -6,15 +6,56 @@ import sys
 
 
 def configure_action(args):
-    print("WE ARE CONFIGURING!")
     settings = OrderedDict()
-    
-    settings = validate_args.v(validate_args.setup_schema)
-    if settings is False:
+    if args.input_config_file is None:
+        settings,schema = validate_args.validate({})
+    else:
+        try:
+            cfg = json.loads(args.input_config_file.read())
+        except Exception as e:
+            raise(e)
+        settings,schema = validate_args.validate(cfg)
+        
+            
+
+    if settings == None:
         print("Failure while processing settings.\n\tQuitting...", file=sys.stderr)
         sys.exit(1)
+    
+    new_config = {}
     for arg in settings:
-        choice = input("%s [%s]:"%(arg,settings[arg]))
+        default = settings[arg]
+        default_string = str(default).replace("'", '"')
+        choice = input("%s [default: %s]: "%(arg,default_string))
+        choice = choice.strip()
+        if len(choice) == 0:
+            print("\tNothing entered, using default:")
+            new_config[arg] = default
+        else:
+            if choice.lower() in ["true", "false"] and schema['properties'][arg]['type'] == "boolean" :
+                new_config[arg] = json.loads(choice.lower())
+            else:
+                if choice in ['true','false'] or (choice.isdigit() and schema['properties'][arg]['type'] != "integer"):
+                    choice = '"' + choice + '"'
+                # replace all single quotes with doubles quotes to make valid json
+                if "'" in choice:
+                    print('''Found %d single quotes (') in input... we will convert these to double quotes (") to ensure valida json.'''%(choice.count("'")))
+                    choice = choice.replace("'",'"')
+                new_config[arg] = json.loads(choice)
+        print("\t{0}\n".format(new_config[arg]))
+
+
+    #Now parse the new config and make sure it's good
+    validated_new_settings, schema = validate_args.validate(new_config)
+    if validate_args == None:
+        print("Error in the new settings!")
+    else:
+        print("New settings worked great.  Writing results to : %s"%(args.output_config_file.name))
+        args.output_config_file.write(json.dumps(validated_new_settings, sort_keys=True, indent=4))
+
+    
+
+
         
 
     
@@ -39,8 +80,8 @@ def main(args):
     configure_parser = actions_parser.add_parser(
         "configure", help="Configure a test run")
     configure_parser.set_defaults(func=configure_action)
-    configure_parser.add_argument('-i', '--input_config_file', required=False, type=argparse.FileType('r'), default=DEFAULT_CONFIG_FILE, help="The config file to base the configuration off of.")
-    configure_parser.add_argument('-o', '--output_config_file', required=False, type=argparse.FileType('w'), help="The config file to write the configuration off of.")
+    configure_parser.add_argument('-i', '--input_config_file', required=False, type=argparse.FileType('r'), help="The config file to base the configuration off of.")
+    configure_parser.add_argument('-o', '--output_config_file', required=True, type=argparse.FileType('w'), help="The config file to write the configuration off of.")
     
 
 
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py b/automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py
index 7664dae569..4ca67b8ddf 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py
@@ -1,10 +1,11 @@
-import sys
-import jsonschema.exceptions
-import jsonschema
 import argparse
+import copy
 import io
 import json
+import jsonschema
+import jsonschema.exceptions
 import jsonschema_errorprinter
+import sys
 from typing import Union
 
 
@@ -12,8 +13,6 @@ from typing import Union
 setup_schema = {
     "type": "object",
     "properties": {
-        
-
         "branch": {
             "type": "string",
             "default": "develop"
@@ -161,24 +160,24 @@ setup_schema = {
 }
 
 
-def v(configuration: dict) -> Union[bool, dict]:
+def validate(configuration: dict) -> tuple[Union[dict,None],dict]:
     #v = jsonschema.Draft201909Validator(argument_schema)
-
+    
     try:
         validation_errors, validated_json = jsonschema_errorprinter.check_json(
             configuration, setup_schema)
         if len(validation_errors) == 0:
             print("Input configuration successfully validated!")
-            return validated_json
+            return validated_json, setup_schema
         else:
             print("[%d] failures detected during validation of the configuration!" % (
                 len(validation_errors)))
             for error in validation_errors:
                 print(error, end="\n\n", file=sys.stderr)
-            return False
+            return None, setup_schema
     except Exception as e:
         print(str(e), file=sys.stderr)
-        return False
+        return None, setup_schema
 
     """
     try:

From aca21ef2857d5e0fb9664028e5e35a02e16fc097 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Thu, 18 Nov 2021 15:44:01 -0800
Subject: [PATCH 068/166] Changes to the interface used to construct the config
 file and perform additional validation of the file

---
 .../modules/new_arguments2.py                 | 143 ++++++++++++----
 .../modules/validate_args.py                  | 157 ++++++++++++++----
 2 files changed, 232 insertions(+), 68 deletions(-)

diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/new_arguments2.py b/automated_detection_testing/ci/detection_testing_batch/modules/new_arguments2.py
index 7a49308aad..4375b08e4a 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/new_arguments2.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/new_arguments2.py
@@ -4,63 +4,100 @@ from typing import OrderedDict
 import validate_args
 import sys
 
+DEFAULT_CONFIG_FILE = "test_config.json"
 
-def configure_action(args):
+def configure_action(args) -> bool:
     settings = OrderedDict()
     if args.input_config_file is None:
-        settings,schema = validate_args.validate({})
+        settings, schema = validate_args.validate({})
     else:
-        try:
-            cfg = json.loads(args.input_config_file.read())
-        except Exception as e:
-            raise(e)
-        settings,schema = validate_args.validate(cfg)
-        
-            
+        settings, schema = validate_args.validate_file(args.input_config_file)
 
-    if settings == None:
+    if settings is None:
         print("Failure while processing settings.\n\tQuitting...", file=sys.stderr)
         sys.exit(1)
-    
+
     new_config = {}
     for arg in settings:
         default = settings[arg]
         default_string = str(default).replace("'", '"')
-        choice = input("%s [default: %s]: "%(arg,default_string))
+        
+        if 'enum' in schema['properties'][arg]:
+            choice = input("%s [default: %s | choices: {%s}]: " % (arg, default_string,','.join(schema['properties'][arg]['enum'])))
+        else:
+            choice = input("%s [default: %s]: " % (arg, default_string))
         choice = choice.strip()
         if len(choice) == 0:
             print("\tNothing entered, using default:")
             new_config[arg] = default
+            formatted_print = default
         else:
-            if choice.lower() in ["true", "false"] and schema['properties'][arg]['type'] == "boolean" :
+            if choice.lower() in ["true", "false"] and schema['properties'][arg]['type'] == "boolean":
                 new_config[arg] = json.loads(choice.lower())
+                formatted_print = choice.lower()
             else:
-                if choice in ['true','false'] or (choice.isdigit() and schema['properties'][arg]['type'] != "integer"):
+                if choice in ['true', 'false'] or (choice.isdigit() and schema['properties'][arg]['type'] != "integer"):
                     choice = '"' + choice + '"'
                 # replace all single quotes with doubles quotes to make valid json
-                if "'" in choice:
-                    print('''Found %d single quotes (') in input... we will convert these to double quotes (") to ensure valida json.'''%(choice.count("'")))
-                    choice = choice.replace("'",'"')
+                elif "'" in choice:
+                    print('''Found %d single quotes (') in input... we will convert these to double quotes (") to ensure valida json.''' % (
+                        choice.count("'")))
+                    choice = choice.replace("'", '"')
+                elif '"' in choice:
+                    #Do nothing
+                    pass
+                else:
+                    choice = '"' + choice + '"'
+
                 new_config[arg] = json.loads(choice)
-        print("\t{0}\n".format(new_config[arg]))
+                formatted_print = choice
+        #We print out choice instead of new_config[arg] because the json.loads() messes up the quotation marks again
+        print("\t{0}\n".format(formatted_print))
 
-
-    #Now parse the new config and make sure it's good
+    # Now parse the new config and make sure it's good
     validated_new_settings, schema = validate_args.validate(new_config)
-    if validate_args == None:
+    if validated_new_settings == None:
         print("Error in the new settings!")
+        return False
     else:
-        print("New settings worked great.  Writing results to : %s"%(args.output_config_file.name))
-        args.output_config_file.write(json.dumps(validated_new_settings, sort_keys=True, indent=4))
+        print("New settings worked great.  Writing results to: %s" %
+              (args.output_config_file.name))
+        args.output_config_file.write(json.dumps(
+            validated_new_settings, sort_keys=True, indent=4))
+        return True
+
+
+def update_config_with_cli_arguments(args_dict:dict)->dict:
+    #First load the config file
+
+    settings,_ = validate_args.validate_file(args_dict['config_file'])
+    if settings is None:
+        print("Failure while processing settings in [%s].\n\tQuitting..."%(args_dict['config_file'].name), file=sys.stderr)
+        sys.exit(1)
+    
+    #Then update it with the values that were passed as command line arguments
+    for key, value in args_dict.items():
+        if key in settings:
+            settings[key] = value
+    
+    #Validate again to make sure we didn't break anything
+    settings,_ = validate_args.validate(settings)
+    if settings is None:
+        print("Failure while processing updated settings from command line.\n\tQuitting...", file=sys.stderr)
+        sys.exit(1)
+    
+    return settings
 
     
 
+def run_action(args) -> bool:
 
-        
+    config = update_config_with_cli_arguments(args.__dict__)
+
+
+    return True
 
-    
 
-DEFAULT_CONFIG_FILE = "defaults.json"
 def main(args):
     '''
     try:
@@ -74,24 +111,62 @@ def main(args):
     parser = argparse.ArgumentParser(
         description="Use 'SOME_PROGRAM_NAME_STRING --help' to get help with the arguments")
     parser.set_defaults(func=lambda _: parser.print_help())
-    
+
     actions_parser = parser.add_subparsers(title="Action")
 
+    # Configure parser
     configure_parser = actions_parser.add_parser(
         "configure", help="Configure a test run")
     configure_parser.set_defaults(func=configure_action)
-    configure_parser.add_argument('-i', '--input_config_file', required=False, type=argparse.FileType('r'), help="The config file to base the configuration off of.")
-    configure_parser.add_argument('-o', '--output_config_file', required=True, type=argparse.FileType('w'), help="The config file to write the configuration off of.")
-    
+    configure_parser.add_argument('-i', '--input_config_file', required=False,
+                                  type=argparse.FileType('r'), help="The config file to base the configuration off of.")
+    configure_parser.add_argument('-o', '--output_config_file', required=False, default=DEFAULT_CONFIG_FILE, 
+                                  type=argparse.FileType('w'), help="The config file to write the configuration off of.")
 
-
-    test_parser = actions_parser.add_parser(
+    # Run parser
+    run_parser = actions_parser.add_parser(
         "run", help="Run a test")
+    run_parser.set_defaults(func=run_action)
+    run_parser.add_argument('-c', '--config_file', required=False,
+                            type=argparse.FileType('r'),
+                            default = DEFAULT_CONFIG_FILE,
+                            help="The config file for the test.  Note that this file "\
+                            "cannot be changed (except for credentials that can be "\
+                            "entered on the command line).")
+
+    run_parser.add_argument('-user', '--splunkbase_username', required=False, type=str,
+                            help="Username for login to splunkbase.  This is required "
+                            "if downloading packages from Splunkbase.  While this can "
+                            "be stored in the config file, it is strongly recommended "
+                            "to enter it at runtime.")
+    run_parser.add_argument('-pass', '--splunkbase_password', required=False, type=str,
+                            help="Password for login to splunkbase.  This is required if "
+                            "downloading packages from Splunkbase.  While this can be "
+                            "stored in the config file, it is strongly recommended "
+                            "to enter it at runtime.")
+
+    run_parser.add_argument('-splunkpass', '--splunk_app_password', required=False, type=str,
+                            help="Password for login to the splunk app.  If you don't "
+                            "provide one here or in the config, it will be generated "
+                            "automatically for you.")
+    run_parser.add_argument("-show_pass", "--show_splunk_app_password", required=False, 
+                            action="store_true",
+                            help="The password to login to the Splunk Server.  ")
 
     args = parser.parse_args()
-    #Run the appropriate parser
+    
+    # Run the appropriate parser
 
-    args.func(args)
+    try:
+        if args.func(args):
+            print("Success!")
+            sys.exit(0)
+        else:
+            print("Fail")
+            sys.exit(1)
+    except Exception as e:
+        print("Unknown Error - [%s]" % (str(e)))
+        sys.exit(1)
 
     '''
 
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py b/automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py
index 4ca67b8ddf..624ae90e5e 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py
@@ -10,6 +10,7 @@ from typing import Union
 
 
 # If we want, we can easily add a description field to any of the objects here!
+
 setup_schema = {
     "type": "object",
     "properties": {
@@ -27,6 +28,19 @@ setup_schema = {
             "type": "boolean",
             "default": False
         },
+            
+        "detections_list": {
+            "type":["array"],
+            "items": {
+                "type":"string"
+            },
+            "default":[],
+        },
+
+        "detections_file":{
+            "type": ["string","null"],
+            "default": None
+        },
 
         "local_apps": {
             "type": "array",
@@ -52,7 +66,7 @@ setup_schema = {
 
         "mode": {
             "type": "string",
-            "enum": ["changes", "selected", "new"],
+            "enum": ["changes", "selected", "all"],
             "default": "changes"
         },
 
@@ -77,9 +91,9 @@ setup_schema = {
             "default": True
         },
 
-        "show_password": {
+        "show_splunk_app_password": {
             "type": "boolean",
-            "default": False
+            "default": True
 
         },
 
@@ -100,49 +114,89 @@ setup_schema = {
                 },
             },
             "default":  [
-                {"app_name": "SPLUNK_ADD_ON_FOR_AMAZON_WEB_SERVICES",
-                    "app_number": 1876, "app_version": "5.2.0"},
-                {"app_name": "SPLUNK_ADD_ON_FOR_MICROSOFT_OFFICE_365",
-                 "app_number": 4055, "app_version": "2.2.0"},
-                {"app_name": "SPLUNK_ADD_ON_FOR_AMAZON_KINESIS_FIREHOSE",
-                 "app_number": 3719, "app_version": "1.3.2"},
-                {"app_name": "SPLUNK_ANALYTIC_STORY_EXECUTION_APP",
-                 "app_number": 4971, "app_version": "2.0.3"},
-                {"app_name": "PYTHON_FOR_SCIENTIC_COMPUTING_LINUX_64_BIT",
-                 "app_number": 2882, "app_version": "2.0.2"},
-                {"app_name": "SPLUNK_MACHINE_LEARNING_TOOLKIT",
-                 "app_number": 2890, "app_version": "5.2.2"},
-                {"app_name": "SPLUNK_APP_FOR_STREAM",
-                 "app_number": 1809, "app_version": "8.0.1"},
-                {"app_name": "SPLUNK_ADD_ON_FOR_STREAM_WIRE_DATA",
-                 "app_number": 5234, "app_version": "8.0.1"},
-                {"app_name": "SPLUNK_ADD_ON_FOR_STREAM_FORWARDERS",
-                 "app_number": 5238, "app_version": "8.0.1"},
-                {"app_name": "SPLUNK_ADD_ON_FOR_ZEEK_AKA_BRO",
-                 "app_number": 1617, "app_version": "4.0.0"},
-                {"app_name": "SPLUNK_ADD_ON_FOR_UNIX_AND_LINUX",
-                 "app_number": 833, "app_version": "8.3.1"},
-                {"app_name": "SPLUNK_COMMON_INFORMATION_MODEL",
-                 "app_number": 1621, "app_version": "4.20.2"}
+                {
+                    "app_name": "SPLUNK_ADD_ON_FOR_AMAZON_WEB_SERVICES",
+                    "app_number": 1876,
+                    "app_version": "5.2.0"
+                },
+                {
+                    "app_name": "SPLUNK_ADD_ON_FOR_MICROSOFT_OFFICE_365",
+                    "app_number": 4055,
+                    "app_version": "2.2.0"
+                },
+                {
+                    "app_name": "SPLUNK_ADD_ON_FOR_AMAZON_KINESIS_FIREHOSE",
+                    "app_number": 3719,
+                    "app_version": "1.3.2"
+                },
+                {
+                    "app_name": "SPLUNK_ANALYTIC_STORY_EXECUTION_APP",
+                    "app_number": 4971,
+                    "app_version": "2.0.3"
+                },
+                {
+                    "app_name": "PYTHON_FOR_SCIENTIC_COMPUTING_LINUX_64_BIT",
+                    "app_number": 2882,
+                    "app_version": "2.0.2"
+                },
+                {
+                    "app_name": "SPLUNK_MACHINE_LEARNING_TOOLKIT",
+                    "app_number": 2890,
+                    "app_version": "5.2.2"
+                },
+                {
+                    "app_name": "SPLUNK_APP_FOR_STREAM",
+                    "app_number": 1809,
+                    "app_version": "8.0.1"
+                },
+                {
+                    "app_name": "SPLUNK_ADD_ON_FOR_STREAM_WIRE_DATA",
+                    "app_number": 5234,
+                    "app_version": "8.0.1"
+                },
+                {
+                    "app_name": "SPLUNK_ADD_ON_FOR_STREAM_FORWARDERS",
+                    "app_number": 5238,
+                    "app_version": "8.0.1"
+                },
+                {
+                    "app_name": "SPLUNK_ADD_ON_FOR_ZEEK_AKA_BRO",
+                    "app_number": 1617,
+                    "app_version": "4.0.0"
+                },
+                {
+                    "app_name": "SPLUNK_ADD_ON_FOR_UNIX_AND_LINUX",
+                    "app_number": 833,
+                    "app_version": "8.3.1"
+                },
+                {
+                    "app_name": "SPLUNK_COMMON_INFORMATION_MODEL",
+                    "app_number": 1621,
+                    "app_version": "4.20.2"
+                }
             ]
         },
         "splunkbase_username": {
-            "type": ["string","null"],
+            "type": ["string", "null"],
             "default": None
         },
         "splunkbase_password": {
             "type": ["string", "null"],
             "default": None
         },
-        "splunk_container_apps_directory":{
-            "type":"string",
+        "splunk_app_password": {
+            "type": ["string", "null"],
+            "default": None
+        },
+        "splunk_container_apps_directory": {
+            "type": "string",
             "default": "/opt/splunk/etc/apps"
         },
         "local_base_container_name": {
             "type": "string",
             "default": "splunk_test_%d"
         },
-        
+
         "mock": {
             "type": "boolean",
             "default": False
@@ -160,15 +214,50 @@ setup_schema = {
 }
 
 
-def validate(configuration: dict) -> tuple[Union[dict,None],dict]:
-    #v = jsonschema.Draft201909Validator(argument_schema)
+def validate_file(file:io.TextIOWrapper)->tuple[Union[dict, None], dict]:
+    try:
+        settings = json.loads(file.read())
+        return validate(settings)
+    except Exception as e:
+        raise(e)
     
+
+
+
+def check_dependencies(settings: dict)->bool:
+    #Check complex mode dependencies
+    error_free = True
+    if settings['mode'] == 'selected':
+        #Make sure that exactly one of the following fields is populated
+        if settings['detections_file'] == None and settings['detections_list'] == []:
+            print("Error - mode was 'selected' but no detections_list or detections_file were supplied.",file=sys.stderr)
+            error_free = False
+        elif settings['detections_file'] != None and settings['detections_list'] != []:
+            print("Error - mode was 'selected' but detections_list and detections_file were supplied.",file=sys.stderr)
+            error_free = False
+    if settings['mode'] != 'selected'and settings['detections_file'] != None:
+        print("Error - mode was not 'selected' but detections_file was supplied.",file=sys.stderr)
+        error_free = False
+    elif settings['mode'] != 'selected' and settings['detections_list'] != []:
+        print("Error - mode was not 'selected' but detections_list was supplied.",file=sys.stderr)
+        error_free = False
+
+    #Returns true if there are not errors
+    return error_free
+
+def validate(configuration: dict) -> tuple[Union[dict, None], dict]:
+    #v = jsonschema.Draft201909Validator(argument_schema)
+
     try:
         validation_errors, validated_json = jsonschema_errorprinter.check_json(
             configuration, setup_schema)
-        if len(validation_errors) == 0:
+        no_complex_errors = check_dependencies(validated_json)
+        if len(validation_errors) == 0 and no_complex_errors:
             print("Input configuration successfully validated!")
             return validated_json, setup_schema
+        elif no_complex_errors == False:
+            print("Failed due to error(s) listed above.", file=sys.stderr)
+            return None,setup_schema
         else:
             print("[%d] failures detected during validation of the configuration!" % (
                 len(validation_errors)))

From 1f5386b7762cd8d826b64be21ec5487b10fa9b70 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Thu, 18 Nov 2021 16:56:46 -0800
Subject: [PATCH 069/166] Mostly done with overhaul to arguments.  Initial
 testing show that it looks good.  Now to integrate the newly parsed arguments
 in with the rewritten run behavior.

---
 .../detection_testing_execution.py            | 34 ++++++-----
 .../modules/new_arguments2.py                 | 34 +++++------
 .../modules/validate_args.py                  | 59 +++++++++++--------
 3 files changed, 70 insertions(+), 57 deletions(-)

diff --git a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
index 992d20e9c3..8e4654c115 100644
--- a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
+++ b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
@@ -27,6 +27,9 @@ import csv
 
 from requests import get
 import json
+import requests.packages.urllib3
+
+import modules.new_arguments2
 
 SPLUNK_CONTAINER_APPS_DIR = "/opt/splunk/etc/apps"
 index_file_local_path = "indexes.conf.tar"
@@ -38,25 +41,28 @@ datamodel_file_container_path = os.path.join(SPLUNK_CONTAINER_APPS_DIR, "Splunk_
 
 
 MAX_RECOMMENDED_CONTAINERS_BEFORE_WARNING=2
-DEFAULT_CONTAINER_TAG="latest"
-LOCAL_BASE_CONTAINER_NAME = "splunk_test_%d"
 
 
 
-BASE_CONTAINER_WEB_PORT=8000
-BASE_CONTAINER_MANAGEMENT_PORT=8089
-
-
-DETECTION_TYPES = ['endpoint', 'cloud', 'network']
-DETECTION_MODES = ['new', 'all', 'selected']
 
 
 
 
 def main(args):
-    
-    start_time = timer()
-    
+    requests.packages.urllib3.disable_warnings()
+
+    start_datetime = datetime.now()
+    action, settings = modules.new_arguments2.parse(args)
+    if action == "configure":
+        #Done, nothing else to do
+        print("Configuration complete!")
+        sys.exit(0)
+    elif action != "run":
+        print("Unsupported action: [%s]"%(action), file=sys.stderr)
+        sys.exit(1)
+    print("time to run the test!")
+    sys.exit(0)
+
 
     parser = argparse.ArgumentParser(description="CI Detection Testing")
     parser.add_argument("-b", "--branch", type=str, required=True, help="security content branch")
@@ -92,9 +98,9 @@ def main(args):
 
     parser.add_argument("-split","--split_detections_then_stop", required=False, default=False, action='store_true', help="Should existing images be re-used, or should they be redownloaded?")
 
-    start_datetime = datetime.now()
-    import requests.packages.urllib3
-    requests.packages.urllib3.disable_warnings()
+    
+    
+    
     args = parser.parse_args()
     branch = args.branch
     uuid_test = args.uuid
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/new_arguments2.py b/automated_detection_testing/ci/detection_testing_batch/modules/new_arguments2.py
index 4375b08e4a..460d9af287 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/new_arguments2.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/new_arguments2.py
@@ -1,12 +1,12 @@
 import argparse
 import json
-from typing import OrderedDict
-import validate_args
+from typing import OrderedDict, Union
+import modules.validate_args as validate_args
 import sys
 
 DEFAULT_CONFIG_FILE = "test_config.json"
 
-def configure_action(args) -> bool:
+def configure_action(args) -> tuple[str, dict]:
     settings = OrderedDict()
     if args.input_config_file is None:
         settings, schema = validate_args.validate({})
@@ -58,16 +58,18 @@ def configure_action(args) -> bool:
     validated_new_settings, schema = validate_args.validate(new_config)
     if validated_new_settings == None:
         print("Error in the new settings!")
-        return False
+        sys.exit(1)
+        
     else:
-        print("New settings worked great.  Writing results to: %s" %
+        print("New settings successful.  Writing results to: %s" %
               (args.output_config_file.name))
         args.output_config_file.write(json.dumps(
             validated_new_settings, sort_keys=True, indent=4))
-        return True
+    
+    return ("configure", validated_new_settings)
 
 
-def update_config_with_cli_arguments(args_dict:dict)->dict:
+def update_config_with_cli_arguments(args_dict:dict)->tuple[str, dict]:
     #First load the config file
 
     settings,_ = validate_args.validate_file(args_dict['config_file'])
@@ -86,19 +88,19 @@ def update_config_with_cli_arguments(args_dict:dict)->dict:
         print("Failure while processing updated settings from command line.\n\tQuitting...", file=sys.stderr)
         sys.exit(1)
     
-    return settings
+    return ("run", settings)
 
     
 
-def run_action(args) -> bool:
+def run_action(args) -> tuple[str,dict]:
 
     config = update_config_with_cli_arguments(args.__dict__)
 
 
-    return True
+    return config
 
 
-def main(args):
+def parse(args)->tuple[str,dict]:
     '''
     try:
         with open(DEFAULT_CONFIG_FILE, 'r') as settings_file:
@@ -158,12 +160,8 @@ def main(args):
     # Run the appropriate parser
 
     try:
-        if args.func(args):
-            print("Success!")
-            sys.exit(0)
-        else:
-            print("Fail")
-            sys.exit(1)
+        action, settings = args.func(args)
+        return action, settings
     except Exception as e:
         print("Unknown Error - [%s]" % (str(e)))
         sys.exit(1)
@@ -251,4 +249,4 @@ def main(args):
 
 
 if __name__ == "__main__":
-    main(sys.argv[1:])
+    parse(sys.argv[1:])
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py b/automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py
index 624ae90e5e..af57245555 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py
@@ -4,7 +4,7 @@ import io
 import json
 import jsonschema
 import jsonschema.exceptions
-import jsonschema_errorprinter
+import modules.jsonschema_errorprinter as jsonschema_errorprinter
 import sys
 from typing import Union
 
@@ -28,17 +28,17 @@ setup_schema = {
             "type": "boolean",
             "default": False
         },
-            
+
         "detections_list": {
-            "type":["array"],
+            "type": ["array"],
             "items": {
-                "type":"string"
+                "type": "string"
             },
-            "default":[],
+            "default": [],
         },
 
-        "detections_file":{
-            "type": ["string","null"],
+        "detections_file": {
+            "type": ["string", "null"],
             "default": None
         },
 
@@ -57,11 +57,20 @@ setup_schema = {
                         "type": "string"
                     },
                     "local_path": {
-                        "type": "string"
+                        "type": ["string", "null"],
+                        "default": None
                     },
-                },
-                "default": []
-            }
+                }
+            },
+            "default": [
+                {
+                    "app_name": "SPLUNK_ES_CONTENT_UPDATE",
+                    "app_number": 3449,
+                    "app_version": "GENERATED",
+                    'local_path': None
+                }
+            ]
+
         },
 
         "mode": {
@@ -214,53 +223,53 @@ setup_schema = {
 }
 
 
-def validate_file(file:io.TextIOWrapper)->tuple[Union[dict, None], dict]:
+def validate_file(file: io.TextIOWrapper) -> tuple[Union[dict, None], dict]:
     try:
         settings = json.loads(file.read())
         return validate(settings)
     except Exception as e:
         raise(e)
-    
 
 
-
-def check_dependencies(settings: dict)->bool:
-    #Check complex mode dependencies
+def check_dependencies(settings: dict) -> bool:
+    # Check complex mode dependencies
     error_free = True
     if settings['mode'] == 'selected':
-        #Make sure that exactly one of the following fields is populated
+        # Make sure that exactly one of the following fields is populated
         if settings['detections_file'] == None and settings['detections_list'] == []:
-            print("Error - mode was 'selected' but no detections_list or detections_file were supplied.",file=sys.stderr)
+            print("Error - mode was 'selected' but no detections_list or detections_file were supplied.", file=sys.stderr)
             error_free = False
         elif settings['detections_file'] != None and settings['detections_list'] != []:
-            print("Error - mode was 'selected' but detections_list and detections_file were supplied.",file=sys.stderr)
+            print("Error - mode was 'selected' but detections_list and detections_file were supplied.", file=sys.stderr)
             error_free = False
-    if settings['mode'] != 'selected'and settings['detections_file'] != None:
-        print("Error - mode was not 'selected' but detections_file was supplied.",file=sys.stderr)
+    if settings['mode'] != 'selected' and settings['detections_file'] != None:
+        print("Error - mode was not 'selected' but detections_file was supplied.", file=sys.stderr)
         error_free = False
     elif settings['mode'] != 'selected' and settings['detections_list'] != []:
-        print("Error - mode was not 'selected' but detections_list was supplied.",file=sys.stderr)
+        print("Error - mode was not 'selected' but detections_list was supplied.", file=sys.stderr)
         error_free = False
 
-    #Returns true if there are not errors
+    # Returns true if there are not errors
     return error_free
 
+
 def validate(configuration: dict) -> tuple[Union[dict, None], dict]:
     #v = jsonschema.Draft201909Validator(argument_schema)
 
     try:
         validation_errors, validated_json = jsonschema_errorprinter.check_json(
             configuration, setup_schema)
+        
         no_complex_errors = check_dependencies(validated_json)
         if len(validation_errors) == 0 and no_complex_errors:
             print("Input configuration successfully validated!")
             return validated_json, setup_schema
         elif no_complex_errors == False:
             print("Failed due to error(s) listed above.", file=sys.stderr)
-            return None,setup_schema
+            return None, setup_schema
         else:
             print("[%d] failures detected during validation of the configuration!" % (
-                len(validation_errors)))
+                len(validation_errors)),file=sys.stderr)
             for error in validation_errors:
                 print(error, end="\n\n", file=sys.stderr)
             return None, setup_schema

From 844435a821f00cf89db97d0c415d35f22609dfe7 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Fri, 19 Nov 2021 11:21:31 -0800
Subject: [PATCH 070/166] Added a field to the jsonschema and renamed another
 field (types and folders).  Continuing to rework the main detection_testing
 file.

---
 .../detection_testing_execution.py            | 503 +++++++-----------
 .../modules/github_service.py                 | 226 ++++----
 .../modules/validate_args.py                  |  15 +-
 3 files changed, 336 insertions(+), 408 deletions(-)

diff --git a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
index 8e4654c115..ab50adf94a 100644
--- a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
+++ b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
@@ -36,16 +36,41 @@ index_file_local_path = "indexes.conf.tar"
 index_file_container_path = os.path.join(SPLUNK_CONTAINER_APPS_DIR, "search")
 
 datamodel_file_local_path = "datamodels.conf.tar"
-datamodel_file_container_path = os.path.join(SPLUNK_CONTAINER_APPS_DIR, "Splunk_SA_CIM")
+datamodel_file_container_path = os.path.join(
+    SPLUNK_CONTAINER_APPS_DIR, "Splunk_SA_CIM")
 
 
-
-MAX_RECOMMENDED_CONTAINERS_BEFORE_WARNING=2
+MAX_RECOMMENDED_CONTAINERS_BEFORE_WARNING = 2
 
 
+def ensure_security_content(branch: str, pr_number: Union[int, None], persist_security_content: bool) -> GithubService:
+    if persist_security_content is True and os.path.exists("security_content"):
+        print("****** You chose --persist_security_content and the security_content directory exists. "
+              "We will not check out the repo again. Please be aware, this could cause issues if you're "
+              "out of date. ******")
+        github_service = GithubService(
+            branch, existing_directory=persist_security_content)
 
+    elif persist_security_content is True:
+        print("Error - you chose --persist_security_content but the security_content directory does not exist!\n\tQuitting...")
+        sys.exit(1)
+    else:
+        if os.path.exists("security_content/"):
+            print("Deleting the security_content directory")
+            try:
+                shutil.rmtree("security_content/", ignore_errors=True)
+                print("Successfully removed security_content directory")
+            except Exception as e:
+                print(
+                    "Error - could not remove the security_content directory: [%s].\n\tQuitting..." % (str(e)))
+                sys.exit(1)
 
+        if pr_number:
+            github_service = GithubService(branch, pr_number)
+        else:
+            github_service = GithubService(branch)
 
+    return github_service
 
 
 def main(args):
@@ -54,16 +79,15 @@ def main(args):
     start_datetime = datetime.now()
     action, settings = modules.new_arguments2.parse(args)
     if action == "configure":
-        #Done, nothing else to do
+        # Done, nothing else to do
         print("Configuration complete!")
         sys.exit(0)
     elif action != "run":
-        print("Unsupported action: [%s]"%(action), file=sys.stderr)
+        print("Unsupported action: [%s]" % (action), file=sys.stderr)
         sys.exit(1)
     print("time to run the test!")
-    sys.exit(0)
-
 
+    '''
     parser = argparse.ArgumentParser(description="CI Detection Testing")
     parser.add_argument("-b", "--branch", type=str, required=True, help="security content branch")
     parser.add_argument("-u", "--uuid", type=str, required=True, help="uuid for detection test")
@@ -102,6 +126,8 @@ def main(args):
     
     
     args = parser.parse_args()
+    '''
+    '''
     branch = args.branch
     uuid_test = args.uuid
     pr_number = args.pr_number
@@ -117,224 +143,99 @@ def main(args):
     show_password = args.show_password
     splunk_password = args.container_password
     pregenerated_escu_package = args.escu_package
-    
-    
-        
-    
-        
-    
     '''
-    if splunk_password is None and reuse_containers is True:
-        print("Error - if you are going to reuse a container you MUST provide the password to it!")
-        sys.exit(1)
-    '''
-    if splunk_password is None:
-        #Generate a sufficiently complex password 
-        splunk_password = get_random_password()
-        if show_password is True:
-            print("Generated the password: [%s]"%splunk_password)
-    else:
-        print("Since you supplied a password, we will not generate one for you.")
-    
 
-    persist_security_content = args.persist_security_content
-    #Read in all of the tests that we will ignore because they already passed
-    success_tests = []
-    if success_file is not None:
-        try:
-            with open(success_file, "r") as successes:
-                for line in successes.readlines():
-                    file_path_new = os.path.join("tests", os.path.splitext(line)[0]) + ".test.yml"
-                    success_tests.append(file_path_new)
-        except Exception as e:
-            print("Error - error reading success_file: [%s]"%(str(e)))
-            print("\n\tQuitting...")
-            sys.exit(1)
-                
+    FULL_DOCKER_HUB_CONTAINER_NAME = "splunk/splunk:%s" % settings['container_tag']
 
-    
-
-    #Ensure that a valid mode was chosen 
-    mode = args.mode
-    if mode == "selected" and args.test_files_list is None and args.test_files_file is None:
-        print("Error - mode [%s] but did not provide any files to test.\nQuitting..."%(mode))
-        sys.exit(1)
-    elif mode == "selected" and args.test_files_list is not None and args.test_files_file is not None:
-        print("Error - mode [%s] but you specified a list of detections to test AND a file of detections to test.\nQuitting..."%(mode))
-        sys.exit(1)
-    elif mode == "selected" and args.test_files_list is not None:
-        command_line_files_to_test = [name.strip() for name in args.test_files_list.split(',')]
-    elif mode == "selected" and args.test_files_file is not None:
-        lines = args.test_files_file.readlines()
-        command_line_files_to_test = [l.strip() for l in lines]
-
-    folders = [a.strip() for a in args.types.split(',')]
-    for t in folders:
-        if t not in DETECTION_TYPES:
-            print("Error - requested test of [%s] but the only valid types are %s.\tQuitting..."%(t, str(DETECTION_TYPES)))
-            sys.exit(1)
-    
-
-
-    
-
-    #Do some initial setup and validation of the containers and images  
-
-    #If a user requests to use existing containers, they must also explicitly request to reuse existing images
-    '''
-    if reuse_containers and not reuse_image:
-        print("Error - requested --reuse_containers but did not explicitly request --reuse_image.\n\tQuitting")
-        sys.exit(1)
-    '''
-    if num_containers < 1:
-        #Perhaps this should be a mock-run - do the initial steps but don't do testing on the containers?
-        print("Error, requested 0 containers.  You must run with at least 1 container.")
-        sys.exit(1)
-    elif num_containers > MAX_RECOMMENDED_CONTAINERS_BEFORE_WARNING:
+    if settings['num_containers'] > MAX_RECOMMENDED_CONTAINERS_BEFORE_WARNING:
         print("You requested to run with [%d] containers which may use a very large amount of resources "
-               "as they all run in parallel.  The maximum suggested number of parallel containers is "
-               "[%d].  We will do what you asked, but be warned!"%(num_containers, MAX_RECOMMENDED_CONTAINERS_BEFORE_WARNING))
+              "as they all run in parallel.  The maximum suggested number of parallel containers is "
+              "[%d].  We will do what you asked, but be warned!" % (settings['num_containers'], MAX_RECOMMENDED_CONTAINERS_BEFORE_WARNING))
 
-    client = docker.client.from_env()
+    # Check out security content if required
+    github_service = ensure_security_content(
+        settings['branch'], settings['pr_number'], settings['persist_security_content'])
 
-    #Ensure that the images and containers are set up in a proper state
-
-    #Remove containers that previously existed (if we are directed to do so)
-    try:
-        remove_existing_containers(client, False, LOCAL_BASE_CONTAINER_NAME, num_containers)
-    except Exception as e:
-        print("Error tryting to remove existing containers.\n\tQuitting...")
-        sys.exit(1)
-
-    #Download and setup the image
-    try:
-        setup_image(client, reuse_image, full_docker_hub_container_name)
-    except Exception as e:
-        print("Error trying to set up the image.\n\tQuitting...")
-        sys.exit(1)
-    
+    all_test_files = github_service.get_test_files(settings['mode'],
+                                                   settings['folders'],
+                                                   settings['ttps'],
+                                                   settings['detections_list'],
+                                                   settings['detections_file'])
 
     
 
-
-
-
-    
-    
-
-    if persist_security_content is True and os.path.exists("security_content"):
-        print("******You chose --persist_security_content and the security_content directory exists. We will not check out the repo again. Please be aware, this could cause issues if you're out of date.******")
-        github_service = GithubService(branch, existing_directory=persist_security_content)
-
-    elif persist_security_content is True:
-        print("Error - you chose --persist_security_content but the security_content directory does not exist!\n\tQuitting...")
-        sys.exit(1)
-    else:
-        if os.path.exists("security_content/"):
-            print("Deleting the security_content directory")
-            try:
-                shutil.rmtree("security_content/", ignore_errors=True)
-                print("Successfully removed security_content directory")
-            except Exception as e:
-                print("Error - could not remove the security_content directory: [%s].\n\tQuitting..."%(str(e)))
-                sys.exit(1)
-        
-        if pr_number:
-            github_service = GithubService(branch, pr_number)
-        else:
-            github_service = GithubService(branch)
-    
-    try:
-        if mode == "all":
-            test_files = github_service.get_all_tests_and_detections(folders=folders, 
-                                                                    previously_successful_tests=success_tests)
-        elif mode == "new":
-            test_files = github_service.get_changed_test_files(folders=folders,
-                                                            previously_successful_tests=success_tests)
-        elif mode == "selected":
-            if set(folders) != set(DETECTION_TYPES):
-                print("You specified mode [%s] but also types: [%s]. We will ignore type restrictions and test all specified files"%(mode,str(folders)))
-            
-            test_files = github_service.get_selected_test_files(command_line_files_to_test,
-                                                            previously_successful_tests=success_tests)
-
-        else:
-            print("Unsupported mode [%s] chosen.  Supported modes are %s.\n\tQuitting..."%(args.mode, str(DETECTION_MODES)))
-            sys.exit(1)
-    except Exception as e:
-        print("Error - Failed to read in detection files: [%s].\nQuitting..."%(str(e)))    
-        sys.exit(1)
-    
     if len(test_files) == 0:
         print("No files were found to be tested.  Returning an error (should this return success?).\n\tQuitting...")
         sys.exit(1)
-    
+
     local_volume_path = os.path.join(os.getcwd(), "apps")
     try:
         os.mkdir("apps")
     except FileExistsError as e:
-        #Directory already exists, do nothing
+        # Directory already exists, do nothing
         pass
     except Exception as e:
-        print("Caught an error when copying the ESCU package [%s] to the apps folder [%s].\n\tQuitting..."%(pregenerated_escu_package.name, local_volume_path))
+        print("Caught an error when copying the ESCU package [%s] to the apps folder [%s].\n\tQuitting..." % (
+            pregenerated_escu_package.name, local_volume_path))
         sys.exit(1)
 
     if pregenerated_escu_package is None:
-        #Go into the security content directory
+        # Go into the security content directory
         print("****GENERATE NEW CONTENT****")
         os.chdir("security_content")
         print(os.getcwd())
         if persist_security_content is False:
-            commands = ["python3 -m venv .venv", 
-                        ". ./.venv/bin/activate", 
-                        "python3 -m pip install wheel", 
-                        "python3 -m pip install -r requirements.txt", 
+            commands = ["python3 -m venv .venv",
+                        ". ./.venv/bin/activate",
+                        "python3 -m pip install wheel",
+                        "python3 -m pip install -r requirements.txt",
                         "python3 contentctl.py --path . --verbose generate --product ESCU --output dist/escu", "tar -czf DA-ESS-ContentUpdate.spl -C dist/escu ."]
         else:
-            commands = ["s. ./.venv/bin/activate", "python3 contentctl.py --path . --verbose generate --product ESCU --output dist/escu", "tar -czf DA-ESS-ContentUpdate.spl -C dist/escu ."]
-        ret = subprocess.run("; ".join(commands), shell=True, capture_output=True)
+            commands = ["s. ./.venv/bin/activate", "python3 contentctl.py --path . --verbose generate --product ESCU --output dist/escu",
+                        "tar -czf DA-ESS-ContentUpdate.spl -C dist/escu ."]
+        ret = subprocess.run("; ".join(commands),
+                             shell=True, capture_output=True)
         if ret.returncode != 0:
             print("Error generating new content.  Exiting...")
             sys.exit(1)
-        print("New content generated successfully")    
-        
+        print("New content generated successfully")
 
         print("Generate new ESCU Package using new content")
         if persist_security_content is True:
             os.chdir("slim_packaging")
-            commands = ["cd slim-latest", 
+            commands = ["cd slim-latest",
                         ". ./.venv/bin/activate",
                         "cp -R ../../dist/escu DA-ESS-ContentUpdate",
                         "slim package -o upload DA-ESS-ContentUpdate",
-                         "cp upload/DA-ESS-ContentUpdate*.tar.gz %s"%(os.path.join(local_volume_path, "DA-ESS-ContentUpdate-latest.tar.gz" ))]
-        
+                        "cp upload/DA-ESS-ContentUpdate*.tar.gz %s" % (os.path.join(local_volume_path, "DA-ESS-ContentUpdate-latest.tar.gz"))]
+
         else:
             os.mkdir("slim_packaging")
             os.chdir("slim_packaging")
             os.mkdir("apps")
-            
+
             try:
                 SPLUNK_PACKAGING_TOOLKIT_URL = "https://download.splunk.com/misc/packaging-toolkit/splunk-packaging-toolkit-0.9.0.tar.gz"
                 SPLUNK_PACKAGING_TOOLKIT_FILENAME = 'splunk-packaging-toolkit-latest.tar.gz'
-                print("Downloading the Splunk Packaging Toolkit from %s..."%(SPLUNK_PACKAGING_TOOLKIT_URL), end='')
+                print("Downloading the Splunk Packaging Toolkit from %s..." %
+                      (SPLUNK_PACKAGING_TOOLKIT_URL), end='')
                 response = get(SPLUNK_PACKAGING_TOOLKIT_URL)
                 response.raise_for_status()
                 with open(SPLUNK_PACKAGING_TOOLKIT_FILENAME, 'wb') as slim_file:
                     slim_file.write(response.content)
-                
-                print("success")
 
+                print("success")
 
             except Exception as e:
                 print("FAILED")
-                print("Error downloading the Splunk Packaging Toolkit: [%s]"%(str(e)))
+                print(
+                    "Error downloading the Splunk Packaging Toolkit: [%s]" % (str(e)))
                 sys.exit(1)
-            
-            
+
             commands = ["rm -rf slim-latest",
-                        "mkdir slim-latest", 
+                        "mkdir slim-latest",
                         "tar -zxf splunk-packaging-toolkit-latest.tar.gz -C slim-latest --strip-components=1",
-                        "cd slim-latest", 
+                        "cd slim-latest",
                         "virtualenv --python=/usr/bin/python2.7 --clear .venv",
                         ". ./.venv/bin/activate",
                         "python3 -m pip install --upgrade pip",
@@ -343,35 +244,32 @@ def main(args):
                         "python2 -m pip install .",
                         "cp -R ../../dist/escu DA-ESS-ContentUpdate",
                         "slim package -o upload DA-ESS-ContentUpdate",
-                        "cp upload/DA-ESS-ContentUpdate*.tar.gz %s"%(os.path.join(local_volume_path, "DA-ESS-ContentUpdate-latest.tar.gz" ))]
-            
-        
-        
+                        "cp upload/DA-ESS-ContentUpdate*.tar.gz %s" % (os.path.join(local_volume_path, "DA-ESS-ContentUpdate-latest.tar.gz"))]
 
-        ret = subprocess.run("; ".join(commands), shell=True, capture_output=True)
+        ret = subprocess.run("; ".join(commands),
+                             shell=True, capture_output=True)
         if ret.returncode != 0:
-            print("Error generating new ESCU Package.\n\tQuitting..."%())
+            print("Error generating new ESCU Package.\n\tQuitting..." % ())
             sys.exit(1)
         os.chdir("../..")
         print("New ESCU Package generated successfully")
     else:
-        print("Using previous generated ESCU package: [%s]"%(pregenerated_escu_package.name))    
+        print("Using previous generated ESCU package: [%s]" % (
+            pregenerated_escu_package.name))
         try:
-            with open(os.path.join(local_volume_path, os.path.basename(pregenerated_escu_package.name)),'wb') as escu_package:
+            with open(os.path.join(local_volume_path, os.path.basename(pregenerated_escu_package.name)), 'wb') as escu_package:
                 escu_package.write(pregenerated_escu_package.read())
         except Exception as e:
-            print("Failure writing the ESCU package [%s] to [%s]: [%s].\n\tQuitting..."%(pregenerated_escu_package.name, local_volume_path, str(e)))
+            print("Failure writing the ESCU package [%s] to [%s]: [%s].\n\tQuitting..." % (
+                pregenerated_escu_package.name, local_volume_path, str(e)))
             sys.exit(1)
-        
-    
+
     print("Wrote ESCU package to volume.")
-        
-    
-    
+
     if args.split_detections_then_stop:
-        for output_file_index in range(0,num_containers):
-            fname = "container_%d_tests.txt"%(output_file_index)
-            print("Writing tests to [%s]..."%(fname), end='')
+        for output_file_index in range(0, num_containers):
+            fname = "container_%d_tests.txt" % (output_file_index)
+            print("Writing tests to [%s]..." % (fname), end='')
             with open(fname, "w") as output_file:
                 detection_tests = test_files[output_file_index::num_containers]
                 normalized_detection_names = []
@@ -380,172 +278,163 @@ def main(args):
                     filename = filename.replace(".test.yml", ".yml")
                     leading = os.path.split(d)[0]
                     leading = leading.replace("tests/", "detections/")
-                    new_name = os.path.join("security_content", leading, filename)
+                    new_name = os.path.join(
+                        "security_content", leading, filename)
                     normalized_detection_names.append(new_name)
                 output_file.write('\n'.join(normalized_detection_names))
             print("Done")
         sys.exit(0)
 
-
-
-
-    
-    
-    #Create threads to manage all of the containers that we will start up
+    # Create threads to manage all of the containers that we will start up
     splunk_container_manager_threads = []
-    
-
-    
-    
-
 
     results_tracker = SynchronizedResultsTracker(test_files, num_containers)
-    
-    
 
-          
-    
     #SPLUNK_ADD_ON_FOR_SYSMON_OLD = "https://splunkbase.splunk.com/app/1914/release/10.6.2/download"
     #SPLUNK_ADD_ON_FOR_SYSMON_NEW = "https://splunkbase.splunk.com/app/5709/release/1.0.1/download"
     #SYSMON_APP_FOR_SPLUNK = "https://splunkbase.splunk.com/app/3544/release/2.0.0/download"
     #SPLUNK_ES_CONTENT_UPDATE = "https://splunkbase.splunk.com/app/3449/release/3.29.0/download"
 
-    #Just a hack until we get the new version of system deployed and available from splunkbase
+    # Just a hack until we get the new version of system deployed and available from splunkbase
     CONTAINER_VOLUME_PATH = '/tmp/apps/'
-    
-    GENERATED_SPLUNK_ES_CONTENT_UPDATE_CONTAINER_PATH = os.path.join(CONTAINER_VOLUME_PATH, "DA-ESS-ContentUpdate-latest.tar.gz")
+
+    GENERATED_SPLUNK_ES_CONTENT_UPDATE_CONTAINER_PATH = os.path.join(
+        CONTAINER_VOLUME_PATH, "DA-ESS-ContentUpdate-latest.tar.gz")
 
     SPLUNKBASE_URL = "https://splunkbase.splunk.com/app/%d/release/%s/download"
-    #Order that we install the apps is actually important
+    # Order that we install the apps is actually important
     APPS_DICT = OrderedDict()
-    APPS_DICT['SPLUNK_ADD_ON_FOR_MICROSOFT_WINDOWS'] = {"app_number":742, 'app_version':"8.2.0", 'location':'splunkbase'}
-    APPS_DICT['SPLUNK_SECURITY_ESSENTIALS'] = {"app_number":3435, 'app_version':"3.3.4", 'location':'splunkbase'}
-    APPS_DICT['GENERATED_SPLUNK_ES_CONTENT_UPDATE'] = {"app_number":3449, 'app_version':"Generated at %s"%(datetime.now()), 'location':GENERATED_SPLUNK_ES_CONTENT_UPDATE_CONTAINER_PATH}
-    
-    
+    APPS_DICT['SPLUNK_ADD_ON_FOR_MICROSOFT_WINDOWS'] = {
+        "app_number": 742, 'app_version': "8.2.0", 'location': 'splunkbase'}
+    APPS_DICT['SPLUNK_SECURITY_ESSENTIALS'] = {
+        "app_number": 3435, 'app_version': "3.3.4", 'location': 'splunkbase'}
+    APPS_DICT['GENERATED_SPLUNK_ES_CONTENT_UPDATE'] = {"app_number": 3449, 'app_version': "Generated at %s" % (
+        datetime.now()), 'location': GENERATED_SPLUNK_ES_CONTENT_UPDATE_CONTAINER_PATH}
+
     try:
-        BETA_SPLUNK_ADD_ON_FOR_SYSMON_PATH  = os.path.expanduser("~/Downloads/Splunk_TA_microsoft_sysmon-1.0.2-B1.spl")    
-        shutil.copyfile(BETA_SPLUNK_ADD_ON_FOR_SYSMON_PATH, os.path.join(local_volume_path, "Splunk_TA_microsoft_sysmon-1.0.2-B1.spl"))
-    
-        BETA_SPLUNK_ADD_ON_FOR_SYSMON_CONTAINER_PATH  = "/tmp/apps/Splunk_TA_microsoft_sysmon-1.0.2-B1.spl"
-        APPS_DICT['BETA_SPLUNK_ADD_ON_FOR_SYSMON'] = {"app_number":5709, 'app_version':"Generated at %s"%(datetime.now()), 'location':"local", "container_path": BETA_SPLUNK_ADD_ON_FOR_SYSMON_CONTAINER_PATH}
+        BETA_SPLUNK_ADD_ON_FOR_SYSMON_PATH = os.path.expanduser(
+            "~/Downloads/Splunk_TA_microsoft_sysmon-1.0.2-B1.spl")
+        shutil.copyfile(BETA_SPLUNK_ADD_ON_FOR_SYSMON_PATH, os.path.join(
+            local_volume_path, "Splunk_TA_microsoft_sysmon-1.0.2-B1.spl"))
+
+        BETA_SPLUNK_ADD_ON_FOR_SYSMON_CONTAINER_PATH = "/tmp/apps/Splunk_TA_microsoft_sysmon-1.0.2-B1.spl"
+        APPS_DICT['BETA_SPLUNK_ADD_ON_FOR_SYSMON'] = {"app_number": 5709, 'app_version': "Generated at %s" % (
+            datetime.now()), 'location': "local", "container_path": BETA_SPLUNK_ADD_ON_FOR_SYSMON_CONTAINER_PATH}
     except Exception as e:
         print("Failed to grab beta sysmon at ~/Downloads/Splunk_TA_microsoft_sysmon-1.0.2-B1.spl. Using the one from Splunkbase")
-        APPS_DICT['SPLUNK_ADD_ON_FOR_SYSMON'] = {"app_number":5709, 'app_version':"1.0.1", 'location':'splunkbase'}
-    
+        APPS_DICT['SPLUNK_ADD_ON_FOR_SYSMON'] = {
+            "app_number": 5709, 'app_version': "1.0.1", 'location': 'splunkbase'}
+
     if True:
-        APPS_DICT['SPLUNK_ADD_ON_FOR_AMAZON_WEB_SERVICES'] = {"app_number":1876, 'app_version':"5.2.0", 'location':'splunkbase'}
-        APPS_DICT['SPLUNK_ADD_ON_FOR_MICROSOFT_OFFICE_365'] = {"app_number":4055, 'app_version':"2.2.0", 'location':'splunkbase'}
-        APPS_DICT['SPLUNK_ADD_ON_FOR_AMAZON_KINESIS_FIREHOSE'] = {"app_number":3719, 'app_version':"1.3.2", 'location':'splunkbase'}
-        APPS_DICT['SPLUNK_ANALYTIC_STORY_EXECUTION_APP'] = {"app_number":4971, 'app_version': "2.0.3", 'location':'splunkbase'}
-        APPS_DICT['PYTHON_FOR_SCIENTIC_COMPUTING_LINUX_64_BIT'] = {"app_number":2882, 'app_version':"2.0.2", 'location':'splunkbase'}
-        APPS_DICT['SPLUNK_MACHINE_LEARNING_TOOLKIT'] = {"app_number":2890, 'app_version':"5.2.2", 'location':'splunkbase'}
-        APPS_DICT['SPLUNK_APP_FOR_STREAM'] = {"app_number":1809, 'app_version':"8.0.1", 'location':'splunkbase'}
-        APPS_DICT['SPLUNK_ADD_ON_FOR_STREAM_WIRE_DATA'] = {"app_number":5234, 'app_version':"8.0.1", 'location':'splunkbase'}
-        APPS_DICT['SPLUNK_ADD_ON_FOR_STREAM_FORWARDERS'] = {"app_number":5238, 'app_version':"8.0.1", 'location':'splunkbase'}
-        APPS_DICT['SPLUNK_ADD_ON_FOR_ZEEK_AKA_BRO'] = {"app_number":1617, 'app_version':"4.0.0", 'location':'splunkbase'}
-        APPS_DICT['SPLUNK_ADD_ON_FOR_UNIX_AND_LINUX'] = {"app_number":833, 'app_version':"8.3.1", 'location':'splunkbase'}
-    
+        APPS_DICT['SPLUNK_ADD_ON_FOR_AMAZON_WEB_SERVICES'] = {
+            "app_number": 1876, 'app_version': "5.2.0", 'location': 'splunkbase'}
+        APPS_DICT['SPLUNK_ADD_ON_FOR_MICROSOFT_OFFICE_365'] = {
+            "app_number": 4055, 'app_version': "2.2.0", 'location': 'splunkbase'}
+        APPS_DICT['SPLUNK_ADD_ON_FOR_AMAZON_KINESIS_FIREHOSE'] = {
+            "app_number": 3719, 'app_version': "1.3.2", 'location': 'splunkbase'}
+        APPS_DICT['SPLUNK_ANALYTIC_STORY_EXECUTION_APP'] = {
+            "app_number": 4971, 'app_version': "2.0.3", 'location': 'splunkbase'}
+        APPS_DICT['PYTHON_FOR_SCIENTIC_COMPUTING_LINUX_64_BIT'] = {
+            "app_number": 2882, 'app_version': "2.0.2", 'location': 'splunkbase'}
+        APPS_DICT['SPLUNK_MACHINE_LEARNING_TOOLKIT'] = {
+            "app_number": 2890, 'app_version': "5.2.2", 'location': 'splunkbase'}
+        APPS_DICT['SPLUNK_APP_FOR_STREAM'] = {
+            "app_number": 1809, 'app_version': "8.0.1", 'location': 'splunkbase'}
+        APPS_DICT['SPLUNK_ADD_ON_FOR_STREAM_WIRE_DATA'] = {
+            "app_number": 5234, 'app_version': "8.0.1", 'location': 'splunkbase'}
+        APPS_DICT['SPLUNK_ADD_ON_FOR_STREAM_FORWARDERS'] = {
+            "app_number": 5238, 'app_version': "8.0.1", 'location': 'splunkbase'}
+        APPS_DICT['SPLUNK_ADD_ON_FOR_ZEEK_AKA_BRO'] = {
+            "app_number": 1617, 'app_version': "4.0.0", 'location': 'splunkbase'}
+        APPS_DICT['SPLUNK_ADD_ON_FOR_UNIX_AND_LINUX'] = {
+            "app_number": 833, 'app_version': "8.3.1", 'location': 'splunkbase'}
 
-    #CIM is last here for a reason!  Because we copy a file to a directory that does not exist until CIM
-    #has been installed, we use it to prevent the testing from beginning until the copy has succeeded.  
-    #KEEP THIS APP LAST!
-    APPS_DICT['SPLUNK_COMMON_INFORMATION_MODEL'] = {"app_number":1621, 'app_version':"4.20.2", 'location':'splunkbase'}
-    
-    
+    # CIM is last here for a reason!  Because we copy a file to a directory that does not exist until CIM
+    # has been installed, we use it to prevent the testing from beginning until the copy has succeeded.
+    # KEEP THIS APP LAST!
+    APPS_DICT['SPLUNK_COMMON_INFORMATION_MODEL'] = {
+        "app_number": 1621, 'app_version': "4.20.2", 'location': 'splunkbase'}
 
-    
     SPLUNK_APPS = []
     for key, value in APPS_DICT.items():
         if value['location'] == 'splunkbase':
-            #The app is on Splunkbase
-            target=SPLUNKBASE_URL%(value['app_number'],value['app_version'])
+            # The app is on Splunkbase
+            target = SPLUNKBASE_URL % (
+                value['app_number'], value['app_version'])
             SPLUNK_APPS.append(target)
         else:
-            #The app is a file we generated locally
+            # The app is a file we generated locally
             SPLUNK_APPS.append(value['location'])
-    
-
 
     for container_index in range(num_containers):
-        container_name = LOCAL_BASE_CONTAINER_NAME%container_index
-        
-        web_port = BASE_CONTAINER_WEB_PORT  + container_index
+        container_name = LOCAL_BASE_CONTAINER_NAME % container_index
+
+        web_port = BASE_CONTAINER_WEB_PORT + container_index
         management_port = BASE_CONTAINER_MANAGEMENT_PORT + container_index
 
-        
-        
-
         environment = {"SPLUNK_START_ARGS": "--accept-license",
-                        "SPLUNK_PASSWORD"  : splunk_password, 
-                        "SPLUNK_APPS_URL"   : ','.join(SPLUNK_APPS),
-                        "SPLUNKBASE_USERNAME" : splunkbase_username,
-                        "SPLUNKBASE_PASSWORD" : splunkbase_password
-                        }
-        ports= {"8000/tcp": web_port,
-                "8089/tcp": management_port
-               }
-        
-        
-        
+                       "SPLUNK_PASSWORD": splunk_password,
+                       "SPLUNK_APPS_URL": ','.join(SPLUNK_APPS),
+                       "SPLUNKBASE_USERNAME": splunkbase_username,
+                       "SPLUNKBASE_PASSWORD": splunkbase_password
+                       }
+        ports = {"8000/tcp": web_port,
+                 "8089/tcp": management_port
+                 }
 
-        
-        mounts = [docker.types.Mount(target = CONTAINER_VOLUME_PATH, source = local_volume_path, type='bind', read_only=True)]
-        
-        print("Creating CONTAINER: [%s]"%(container_name))
-        base_container = client.containers.create(full_docker_hub_container_name, ports=ports, environment=environment, name=container_name, mounts=mounts, detach=True)
-        print("Created CONTAINER : [%s]"%(container_name))
-        
+        mounts = [docker.types.Mount(
+            target=CONTAINER_VOLUME_PATH, source=local_volume_path, type='bind', read_only=True)]
 
-        
-        
-        t = threading.Thread(target=splunk_container_manager, 
-                             args=(results_tracker, 
-                                   container_name, 
-                                   "127.0.0.1", 
+        print("Creating CONTAINER: [%s]" % (container_name))
+        base_container = client.containers.create(
+            full_docker_hub_container_name, ports=ports, environment=environment, name=container_name, mounts=mounts, detach=True)
+        print("Created CONTAINER : [%s]" % (container_name))
+
+        t = threading.Thread(target=splunk_container_manager,
+                             args=(results_tracker,
+                                   container_name,
+                                   "127.0.0.1",
                                    splunk_password,
-                                   web_port, 
-                                   management_port, 
+                                   web_port,
+                                   management_port,
                                    uuid_test,
-                                   interactive_failure 
+                                   interactive_failure
                                    ))
 
         splunk_container_manager_threads.append(t)
 
-    #add the queue status thread - there can be some error in one of the test threads, so this
-    #thread doesn't need to complete for the program to finish execution
-    status_thread = threading.Thread(target=queue_status_thread, 
-                                     args=(results_tracker,), 
-                                           daemon=True)
-    #Start this thread immediately
+    # add the queue status thread - there can be some error in one of the test threads, so this
+    # thread doesn't need to complete for the program to finish execution
+    status_thread = threading.Thread(target=queue_status_thread,
+                                     args=(results_tracker,),
+                                     daemon=True)
+    # Start this thread immediately
     status_thread.start()
 
-
     print("Start the testing threads")
     for t in splunk_container_manager_threads:
         t.start()
-        #we need to start containers slowly.  Would be great it we could do all the setup and
-        #app install once, but it looks like the container is unlikely to support that.
-        #We don't really want to fundamentally change this container, either, and will 
-        #keep it as close to production as possible
+        # we need to start containers slowly.  Would be great it we could do all the setup and
+        # app install once, but it looks like the container is unlikely to support that.
+        # We don't really want to fundamentally change this container, either, and will
+        # keep it as close to production as possible
         time.sleep(5)
-    
-    #Wait for all of the testing threads to complete
+
+    # Wait for all of the testing threads to complete
     for t in splunk_container_manager_threads:
-        t.join() #blocks on waiting to join
+        t.join()  # blocks on waiting to join
         print("Testing thread completed execution")
 
     print("All testing threads have completed execution")
-    #read all the results out from the output queue
+    # read all the results out from the output queue
     strtime = str(int(time.time()))
 
     print("Wait for the status thread to finish executing...")
     status_thread.join()
     print("Status thread finished executing.")
 
-    
-    #Remove the attack data and 
-    #generate all of the output information
+    # Remove the attack data and
+    # generate all of the output information
     stop_time = timer()
     stop_datetime = datetime.now()
     baseline = OrderedDict()
@@ -555,33 +444,17 @@ def main(args):
     baseline['TEST_FINISH_TIME'] = str(stop_datetime)
 
     results_tracker.finish(baseline)
-    
-    
 
-    #now we are done!
-    
-    print("Total Execution Time: [%s]"%(timedelta(seconds=stop_time - start_time, microseconds=0)))
-    
+    # now we are done!
 
-    #detection testing service has already been prepared, no need to do it here!
+    print("Total Execution Time: [%s]" % (
+        timedelta(seconds=stop_time - start_time, microseconds=0)))
+
+    # detection testing service has already been prepared, no need to do it here!
     #testing_service.prepare_detection_testing(ssh_key_name, private_key, splunk_ip, splunk_password)
 
     #testing_service.test_detections(ssh_key_name, private_key, splunk_ip, splunk_password, test_files, uuid_test)
-    
 
 
-
-
-        
-
-
-
-
-            
 if __name__ == "__main__":
     main(sys.argv[1:])
-
-
-
-
-
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/github_service.py b/automated_detection_testing/ci/detection_testing_batch/modules/github_service.py
index eba0fae6f8..efb4826ce5 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/github_service.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/github_service.py
@@ -1,13 +1,18 @@
 
+import csv
+import glob
+import logging
+import os
+import pathlib
+import subprocess
+import sys
+from typing import Union
+from docker import types
+
 import git
-import os
-import logging
-import glob
-import subprocess
-from git.objects import base
 import yaml
-import pathlib
-import csv
+from git.objects import base
+
 # Logger
 logging.basicConfig(level=os.environ.get("LOGLEVEL", "INFO"))
 LOGGER = logging.getLogger(__name__)
@@ -17,16 +22,18 @@ SECURITY_CONTENT_URL = "https://github.com/splunk/security_content"
 
 class GithubService:
 
-    def __init__(self, security_content_branch:str, PR_number:int = None, existing_directory:bool=False):
-        
+    def __init__(self, security_content_branch: str, PR_number: int = None, existing_directory: bool = False):
+
         self.security_content_branch = security_content_branch
         if existing_directory:
             return
         print("Checking out security_content!")
-        self.security_content_repo_obj = self.clone_project(SECURITY_CONTENT_URL, f"security_content", f"develop")
+        self.security_content_repo_obj = self.clone_project(
+            SECURITY_CONTENT_URL, f"security_content", f"develop")
 
         if PR_number:
-            subprocess.call(["git", "-C", "security_content/", "fetch", "origin", "refs/pull/%d/head:%s"%(PR_number, security_content_branch)])
+            subprocess.call(["git", "-C", "security_content/", "fetch", "origin",
+                            "refs/pull/%d/head:%s" % (PR_number, security_content_branch)])
 
         self.security_content_repo_obj.git.checkout(security_content_branch)
 
@@ -35,14 +42,12 @@ class GithubService:
         repo_obj = git.Repo.clone_from(url, project, branch=branch)
         return repo_obj
 
-    
-
-    def prune_detections(self, 
-                         detections_to_prune:list[str], 
-                         ttps_to_test:list[str], 
-                         previously_successful_tests:list[str],
-                         exclude_ssa:bool=True,
-                         summary_file:str=None)->list[str]:
+    def prune_detections(self,
+                         detections_to_prune: list[str],
+                         ttps_to_test: list[str],
+                         previously_successful_tests: list[str],
+                         exclude_ssa: bool = True,
+                         summary_file: str = None) -> list[str]:
         pruned_tests = []
         csvlines = []
         for detection in detections_to_prune:
@@ -50,75 +55,125 @@ class GithubService:
                 continue
             with open(detection, "r") as d:
                 description = yaml.safe_load(d)
-                
-                test_filepath = os.path.splitext(detection)[0].replace('detections', 'tests') + '.test.yml'
-                test_filepath_without_security_content = str(pathlib.Path(*pathlib.Path(test_filepath).parts[1:]))
-                #If no TTPS are provided, then we will get everything
+
+                test_filepath = os.path.splitext(detection)[0].replace(
+                    'detections', 'tests') + '.test.yml'
+                test_filepath_without_security_content = str(
+                    pathlib.Path(*pathlib.Path(test_filepath).parts[1:]))
+                # If no TTPS are provided, then we will get everything
                 if 'type' in description and (description['type'] in ttps_to_test or len(ttps_to_test) == 0):
-                    #print(description['type'])
+                    # print(description['type'])
                     if not os.path.exists(test_filepath):
-                        print("Detection [%s] references [%s], but it does not exist"%(detection, test_filepath))
+                        print("Detection [%s] references [%s], but it does not exist" % (
+                            detection, test_filepath))
                         #raise(Exception("Detection [%s] references [%s], but it does not exist"%(detection, test_filepath)))
                     elif test_filepath_without_security_content in previously_successful_tests:
-                        print("Ignoring test [%s] before it has already passed previously"%(detection))
+                        print(
+                            "Ignoring test [%s] before it has already passed previously" % (detection))
                     else:
-                        #remove leading security_content/ from path
-                        pruned_tests.append(test_filepath_without_security_content)
+                        # remove leading security_content/ from path
+                        pruned_tests.append(
+                            test_filepath_without_security_content)
                         if summary_file is not None:
                             try:
-                                mitre_id = str(description['tags']['mitre_attack_id'])
+                                mitre_id = str(
+                                    description['tags']['mitre_attack_id'])
                             except:
                                 mitre_id = 'NONE'
                             try:
 
-                                csvlines.append({'name':description['name'], 'filename':detection, 'description':description['description'], 
-                                                 'search':description['search'], 'mitre_attack_id':mitre_id, 'security_domain':description['tags']['security_domain'],
-                                                 'Relevant':'', 'Comments':'', "Runnable on SSA?": str(os.path.basename(detection).startswith("ssa"))})
+                                csvlines.append({'name': description['name'], 'filename': detection, 'description': description['description'],
+                                                 'search': description['search'], 'mitre_attack_id': mitre_id, 'security_domain': description['tags']['security_domain'],
+                                                 'Relevant': '', 'Comments': '', "Runnable on SSA?": str(os.path.basename(detection).startswith("ssa"))})
                             except Exception as e:
-                                print("Error outputting summary for [%s]: [%s]"%(detection, str(e))) 
+                                print("Error outputting summary for [%s]: [%s]" % (
+                                    detection, str(e)))
                 else:
-                    #Don't do anything with these files
+                    # Don't do anything with these files
                     pass
-        
+
         if summary_file is not None:
             print("writing")
             with open(summary_file, 'w') as csvfile:
-                fieldnames = ['name', 'filename', 'description', 'search', 'mitre_attack_id', 'security_domain', 'Runnable on SSA?', 'Relevant', 'Comments']
-                writer = csv.DictWriter(csvfile, fieldnames=fieldnames, quoting=csv.QUOTE_ALL)
+                fieldnames = ['name', 'filename', 'description', 'search', 'mitre_attack_id',
+                              'security_domain', 'Runnable on SSA?', 'Relevant', 'Comments']
+                writer = csv.DictWriter(
+                    csvfile, fieldnames=fieldnames, quoting=csv.QUOTE_ALL)
                 writer.writeheader()
                 for r in csvlines:
                     writer.writerow(r)
-            
-        
+
         return pruned_tests
 
+    def get_test_files(self, mode: str, folders:list[str], ttps:list[str],
+                        detections_list: Union[list[str], None], 
+                       detections_file=Union[str, None]) -> list[str]:
+        if mode == "changes":
+            tests = self.get_changed_test_files(folders, ttps)
+        elif mode == "selected":
+            if detections_list is None and detections_file is None:
+                #It's actually valid to supply an EMPTY list of files and the test should pass.
+                #This can occur when we try to test, for example, 1 detection but start 2 containers.
+                #We still want this to pass testing, so we shouldn't fail there!
+                print("Trying to test a list of files, but None were provided", file=sys.stderr)
+                sys.exit(1)
+
+            if detections_list is not None and detections_file is not None:
+                print("Both detections_list [%s] and detections_file [%s] were provided.  "\
+                      "Because these confilect, we cannot test.\n\tQuitting..."%
+                      (detections_list, detections_file), file=sys.stderr)
+                sys.exit(1)
+            if detections_list is not None:
+                tests = self.get_selected_test_files(detections_list, folders, ttps)
+            if detections_file is not None:
+                try:
+                    with open(detections_file,'r') as f:
+                        data = f.readlines()
+                    #Strip all whitespace from lines and exclude lines that are just whitespace
+                    files_to_test = [line.strip() for line in data if len(line.strip()) > 0]
+                except Exception as e:
+                    print("There was an error reading the input file [%s]: [%s].\n\t"\
+                          "Quitting..."%(detections_file, str(e)))
+                    sys.exit(1)
+                
+                tests = self.get_selected_test_files(files_to_test, folders, ttps)
+
+        elif mode == "all":
+            tests = self.get_all_tests_and_detections(folders,ttps)
+        else:
+            print(
+                "Error, unsupported mode [%s].  Mode must be one of %s", file=sys.stderr)
+            sys.exit(1)
+
+        return []
 
     def get_selected_test_files(self,
-                                detection_file_list:list[str], 
-                                ttps_to_test:list[str]=["Anomaly","Hunting","TTP"], 
-                                previously_successful_tests:list[str]=[]) ->list[str]:    
-        
+                                detection_file_list: list[str],
+                                ttps_to_test: list[str] = [
+                                    "Anomaly", "Hunting", "TTP"],
+                                previously_successful_tests: list[str] = []) -> list[str]:
+
         return self.prune_detections(detection_file_list, ttps_to_test, previously_successful_tests)
 
-
-    def get_all_tests_and_detections(self, 
-                                     folders:list[str]=['endpoint', 'cloud', 'network'], 
-                                     ttps_to_test:list[str]=["Anomaly","Hunting","TTP"], 
-                                     previously_successful_tests:list[str]=[]) ->list[str]:    
+    def get_all_tests_and_detections(self,
+                                     folders: list[str] = [
+                                         'endpoint', 'cloud', 'network'],
+                                     ttps_to_test: list[str] = [
+                                         "Anomaly", "Hunting", "TTP"],
+                                     previously_successful_tests: list[str] = []) -> list[str]:
         detections = []
         for folder in folders:
-            detections.extend(self.get_all_files_in_folder(os.path.join("security_content/detections", folder), "*.yml"))
-            
-        
-        #Prune this down to only the subset of detections we can test
+            detections.extend(self.get_all_files_in_folder(
+                os.path.join("security_content/detections", folder), "*.yml"))
+
+        # Prune this down to only the subset of detections we can test
         return self.prune_detections(detections, ttps_to_test, previously_successful_tests)
-        
-    def get_all_files_in_folder(self, foldername:str, extension:str)->list[str]:
-        filenames =  glob.glob(os.path.join(foldername, extension))
+
+    def get_all_files_in_folder(self, foldername: str, extension: str) -> list[str]:
+        filenames = glob.glob(os.path.join(foldername, extension))
         return filenames
 
-
-    def get_changed_test_files(self, folders=['endpoint', 'cloud', 'network'], ttps_to_test=["Anomaly","Hunting","TTP"], previously_successful_tests=[])->list[str]:
+    def get_changed_test_files(self, folders=['endpoint', 'cloud', 'network'], ttps_to_test=["Anomaly", "Hunting", "TTP"], previously_successful_tests=[]) -> list[str]:
         branch1 = self.security_content_branch
         branch2 = 'develop'
         g = git.Git('security_content')
@@ -132,25 +187,27 @@ class GithubService:
                 # added or changed test files
                 if file_path.startswith('A') or file_path.startswith('M'):
                     if 'tests' in file_path and os.path.basename(file_path).endswith('.test.yml'):
-                            changed_test_files.append(file_path)
+                        changed_test_files.append(file_path)
 
                     # changed detections
                     if 'detections' in file_path and os.path.basename(file_path).endswith('.yml'):
-                            changed_detection_files.append(file_path)
+                        changed_detection_files.append(file_path)
         else:
-            print("Looking for changed detections by diffing [%s] against [%s].  Of course none were returned."%(branch1, branch2))
+            print("Looking for changed detections by diffing [%s] against [%s].  Of course none were returned." % (
+                branch1, branch2))
             return []
-                            
 
-        #all files have the format A\tFILENAME or M\tFILENAME.  Get rid of those leading characters
-        changed_test_files      = [os.path.join("security_content",name.split('\t')[1]) for name in changed_test_files      if len(name.split('\t')) == 2]
-        changed_detection_files = [os.path.join("security_content",name.split('\t')[1]) for name in changed_detection_files if len(name.split('\t')) == 2]
+        # all files have the format A\tFILENAME or M\tFILENAME.  Get rid of those leading characters
+        changed_test_files = [os.path.join("security_content", name.split(
+            '\t')[1]) for name in changed_test_files if len(name.split('\t')) == 2]
+        changed_detection_files = [os.path.join("security_content", name.split(
+            '\t')[1]) for name in changed_detection_files if len(name.split('\t')) == 2]
 
-
-        #convert the test files to the detection file equivalent
+        # convert the test files to the detection file equivalent
         converted_test_files = []
         for test_filepath in changed_test_files:
-            detection_filename = str(pathlib.Path(*pathlib.Path(test_filepath).parts[-2:])).replace("tests", "detections",1)
+            detection_filename = str(pathlib.Path(
+                *pathlib.Path(test_filepath).parts[-2:])).replace("tests", "detections", 1)
             converted_test_files.append(detection_filename)
 
         for name in converted_test_files:
@@ -160,25 +217,20 @@ class GithubService:
         return self.prune_detections(changed_detection_files, ttps_to_test, previously_successful_tests)
 
         #detections_to_test,_,_ = self.filter_test_types(changed_detection_files)
-        #for f in detections_to_test:
+        # for f in detections_to_test:
         #    file_path_base = os.path.splitext(f)[0].replace('detections', 'tests') + '.test'
         #    file_path_new = file_path_base + '.yml'
         #    if file_path_new not in changed_test_files:
         #        changed_test_files.append(file_path_new)
 
-        
-        
-       
-        
         #print("Total things to test (test files and detection files changed): [%d]"%(len(changed_test_files)))
-        #for l in changed_test_files:
+        # for l in changed_test_files:
         #    print(l)
-        #print(len(changed_test_files))
+        # print(len(changed_test_files))
         #import time
-        #time.sleep(5)
-        
+        # time.sleep(5)
 
-    def filter_test_types(self, test_files, test_types = ["Anomaly", "Hunting", "TTP"]):
+    def filter_test_types(self, test_files, test_types=["Anomaly", "Hunting", "TTP"]):
         files_to_test = []
         files_not_to_test = []
         error_files = []
@@ -187,28 +239,22 @@ class GithubService:
                 with open(os.path.join("security_content", filename), "r") as fileData:
                     yaml_dict = list(yaml.safe_load_all(fileData))[0]
                     if 'type' not in yaml_dict.keys():
-                        print("Failed to find 'type' in the yaml for: [%s]"%(filename))
+                        print(
+                            "Failed to find 'type' in the yaml for: [%s]" % (filename))
                         error_files.append(filename)
                     if yaml_dict['type'] in test_types:
                         files_to_test.append(filename)
                     else:
                         files_not_to_test.append(filename)
             except Exception as e:
-                print("Error on trying to scan [%s]: [%s]"%(filename, str(e)))
+                print("Error on trying to scan [%s]: [%s]" % (
+                    filename, str(e)))
                 error_files.append(filename)
-        print("***Detection Information***\n"\
+        print("***Detection Information***\n"
               "\tTotal Files       : %d"
               "\tFiles to test     : %d"
               "\tFiles not to test : %d"
-              "\tError files       : %d"%(len(test_files), len(files_to_test), len(files_not_to_test), len(error_files)))
+              "\tError files       : %d" % (len(test_files), len(files_to_test), len(files_not_to_test), len(error_files)))
         import time
         time.sleep(5)
-        return files_to_test, files_not_to_test, error_files    
-
-
-
-                
-
-
-
-
+        return files_to_test, files_not_to_test, error_files
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py b/automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py
index af57245555..db587a93a0 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py
@@ -30,11 +30,11 @@ setup_schema = {
         },
 
         "detections_list": {
-            "type": ["array"],
+            "type": ["array","null"],
             "items": {
                 "type": "string"
             },
-            "default": [],
+            "default": None,
         },
 
         "detections_file": {
@@ -211,7 +211,7 @@ setup_schema = {
             "default": False
         },
 
-        "types": {
+        "folders": {
             "type": "array",
             "items": {
                 "type": "string",
@@ -219,6 +219,15 @@ setup_schema = {
             },
             "default": ["endpoint", "cloud", "network"]
         },
+
+        "types": {
+            "type": "array",
+            "items": {
+                "type": "string",
+                "enum": ["Anomaly", "Hutning", "TTP"]
+            },
+            "default": ["Anomaly", "Hutning", "TTP"]
+        },
     }
 }
 

From 53b01c192f9a967930952ddb518c1eec499878ca Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Fri, 19 Nov 2021 13:00:23 -0800
Subject: [PATCH 071/166] More changes to the detection_testing_exeuction file
 and arg validation file to make it simpler.

---
 .../detection_testing_execution.py            | 207 ++++++++++--------
 .../modules/validate_args.py                  |  15 +-
 2 files changed, 125 insertions(+), 97 deletions(-)

diff --git a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
index ab50adf94a..1db14dd986 100644
--- a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
+++ b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
@@ -73,6 +73,83 @@ def ensure_security_content(branch: str, pr_number: Union[int, None], persist_se
     return github_service
 
 
+def generate_escu_app(persist_security_content:bool=False)->str:
+    # Go into the security content directory
+        print("****GENERATING ESCU APP****")
+        os.chdir("security_content")
+        print(os.getcwd())
+        if persist_security_content is False:
+            commands = ["python3 -m venv .venv",
+                        ". ./.venv/bin/activate",
+                        "python3 -m pip install wheel",
+                        "python3 -m pip install -r requirements.txt",
+                        "python3 contentctl.py --path . --verbose generate --product ESCU --output dist/escu", "tar -czf DA-ESS-ContentUpdate.spl -C dist/escu ."]
+        else:
+            commands = ["s. ./.venv/bin/activate", "python3 contentctl.py --path . --verbose generate --product ESCU --output dist/escu",
+                        "tar -czf DA-ESS-ContentUpdate.spl -C dist/escu ."]
+        ret = subprocess.run("; ".join(commands),
+                             shell=True, capture_output=True)
+        if ret.returncode != 0:
+            print("Error generating new content.  Exiting...")
+            sys.exit(1)
+        
+
+        
+        output_file_name = "DA-ESS-ContentUpdate-latest.tar.gz"
+        output_file_path_from_root = os.path.join("security_content", "slim_packaging", "apps", "upload", output_file_name)
+        output_file_path_from_apps = os.path.join("upload", output_file_name)
+
+        if persist_security_content is True:
+            os.chdir("slim_packaging")
+            commands = ["cd slim-latest",
+                        ". ./.venv/bin/activate",
+                        "cp -R ../../dist/escu DA-ESS-ContentUpdate",
+                        "slim package -o upload DA-ESS-ContentUpdate",
+                        "cp upload/DA-ESS-ContentUpdate*.tar.gz %s" % (output_file_path_from_apps)]
+
+        else:
+            os.mkdir("slim_packaging")
+            os.chdir("slim_packaging")
+            os.mkdir("apps")
+
+            try:
+                SPLUNK_PACKAGING_TOOLKIT_URL = "https://download.splunk.com/misc/packaging-toolkit/splunk-packaging-toolkit-0.9.0.tar.gz"
+                SPLUNK_PACKAGING_TOOLKIT_FILENAME = 'splunk-packaging-toolkit-latest.tar.gz'
+                print("Downloading the Splunk Packaging Toolkit from %s..." %
+                      (SPLUNK_PACKAGING_TOOLKIT_URL), end='')
+                response = get(SPLUNK_PACKAGING_TOOLKIT_URL)
+                response.raise_for_status()
+                with open(SPLUNK_PACKAGING_TOOLKIT_FILENAME, 'wb') as slim_file:
+                    slim_file.write(response.content)
+            except Exception as e:
+                print("Error downloading the Splunk Packaging Toolkit: [%s].\n\tQuitting..." % 
+                      (str(e)), file=sys.stderr)
+                sys.exit(1)
+
+            commands = ["rm -rf slim-latest",
+                        "mkdir slim-latest",
+                        "tar -zxf splunk-packaging-toolkit-latest.tar.gz -C slim-latest --strip-components=1",
+                        "cd slim-latest",
+                        "virtualenv --python=/usr/bin/python2.7 --clear .venv",
+                        ". ./.venv/bin/activate",
+                        "python3 -m pip install --upgrade pip",
+                        "python2 -m pip install wheel",
+                        "python2 -m pip install semantic_version",
+                        "python2 -m pip install .",
+                        "cp -R ../../dist/escu DA-ESS-ContentUpdate",
+                        "slim package -o upload DA-ESS-ContentUpdate",
+                        "cp upload/DA-ESS-ContentUpdate*.tar.gz %s" % (output_file_path_from_apps)]
+
+        ret = subprocess.run("; ".join(commands),
+                             shell=True, capture_output=True)
+        if ret.returncode != 0:
+            print("Error generating new ESCU Package.\n\tQuitting..." % ())
+            sys.exit(1)
+        os.chdir("../..")
+
+        return output_file_path_from_root
+
+
 def main(args):
     requests.packages.urllib3.disable_warnings()
 
@@ -162,11 +239,14 @@ def main(args):
                                                    settings['detections_list'],
                                                    settings['detections_file'])
 
+
     
 
-    if len(test_files) == 0:
-        print("No files were found to be tested.  Returning an error (should this return success?).\n\tQuitting...")
-        sys.exit(1)
+#    if len(all_test_files) == 0:
+#        print("No files were found to be tested.  While this could be due to an error, "\
+#              "this could be correct if there were no changes to detections.  We will "\
+#              "exit with success.\n\tQuitting...")
+#        sys.exit(0)
 
     local_volume_path = os.path.join(os.getcwd(), "apps")
     try:
@@ -175,97 +255,46 @@ def main(args):
         # Directory already exists, do nothing
         pass
     except Exception as e:
-        print("Caught an error when copying the ESCU package [%s] to the apps folder [%s].\n\tQuitting..." % (
-            pregenerated_escu_package.name, local_volume_path))
+        print("Error creating the apps folder [%s]: [%s]\n\tQuitting..." 
+              % (local_volume_path, str(e)), file=sys.stderr)
         sys.exit(1)
 
-    if pregenerated_escu_package is None:
-        # Go into the security content directory
-        print("****GENERATE NEW CONTENT****")
-        os.chdir("security_content")
-        print(os.getcwd())
-        if persist_security_content is False:
-            commands = ["python3 -m venv .venv",
-                        ". ./.venv/bin/activate",
-                        "python3 -m pip install wheel",
-                        "python3 -m pip install -r requirements.txt",
-                        "python3 contentctl.py --path . --verbose generate --product ESCU --output dist/escu", "tar -czf DA-ESS-ContentUpdate.spl -C dist/escu ."]
-        else:
-            commands = ["s. ./.venv/bin/activate", "python3 contentctl.py --path . --verbose generate --product ESCU --output dist/escu",
-                        "tar -czf DA-ESS-ContentUpdate.spl -C dist/escu ."]
-        ret = subprocess.run("; ".join(commands),
-                             shell=True, capture_output=True)
-        if ret.returncode != 0:
-            print("Error generating new content.  Exiting...")
-            sys.exit(1)
-        print("New content generated successfully")
+    #Check to see if we want to install ESCU and whether it was preeviously generated and we should use that file
+    if 'SPLUNK_ES_CONTENT_UPDATE' in settings['local_apps'] and  settings['local_apps']['SPLUNK_ES_CONTENT_UPDATE'] is not None:
+        #Using a pregenerated ESCU, copy it to apps (unless it)
 
-        print("Generate new ESCU Package using new content")
-        if persist_security_content is True:
-            os.chdir("slim_packaging")
-            commands = ["cd slim-latest",
-                        ". ./.venv/bin/activate",
-                        "cp -R ../../dist/escu DA-ESS-ContentUpdate",
-                        "slim package -o upload DA-ESS-ContentUpdate",
-                        "cp upload/DA-ESS-ContentUpdate*.tar.gz %s" % (os.path.join(local_volume_path, "DA-ESS-ContentUpdate-latest.tar.gz"))]
-
-        else:
-            os.mkdir("slim_packaging")
-            os.chdir("slim_packaging")
-            os.mkdir("apps")
-
-            try:
-                SPLUNK_PACKAGING_TOOLKIT_URL = "https://download.splunk.com/misc/packaging-toolkit/splunk-packaging-toolkit-0.9.0.tar.gz"
-                SPLUNK_PACKAGING_TOOLKIT_FILENAME = 'splunk-packaging-toolkit-latest.tar.gz'
-                print("Downloading the Splunk Packaging Toolkit from %s..." %
-                      (SPLUNK_PACKAGING_TOOLKIT_URL), end='')
-                response = get(SPLUNK_PACKAGING_TOOLKIT_URL)
-                response.raise_for_status()
-                with open(SPLUNK_PACKAGING_TOOLKIT_FILENAME, 'wb') as slim_file:
-                    slim_file.write(response.content)
-
-                print("success")
-
-            except Exception as e:
-                print("FAILED")
-                print(
-                    "Error downloading the Splunk Packaging Toolkit: [%s]" % (str(e)))
-                sys.exit(1)
-
-            commands = ["rm -rf slim-latest",
-                        "mkdir slim-latest",
-                        "tar -zxf splunk-packaging-toolkit-latest.tar.gz -C slim-latest --strip-components=1",
-                        "cd slim-latest",
-                        "virtualenv --python=/usr/bin/python2.7 --clear .venv",
-                        ". ./.venv/bin/activate",
-                        "python3 -m pip install --upgrade pip",
-                        "python2 -m pip install wheel",
-                        "python2 -m pip install semantic_version",
-                        "python2 -m pip install .",
-                        "cp -R ../../dist/escu DA-ESS-ContentUpdate",
-                        "slim package -o upload DA-ESS-ContentUpdate",
-                        "cp upload/DA-ESS-ContentUpdate*.tar.gz %s" % (os.path.join(local_volume_path, "DA-ESS-ContentUpdate-latest.tar.gz"))]
-
-        ret = subprocess.run("; ".join(commands),
-                             shell=True, capture_output=True)
-        if ret.returncode != 0:
-            print("Error generating new ESCU Package.\n\tQuitting..." % ())
-            sys.exit(1)
-        os.chdir("../..")
-        print("New ESCU Package generated successfully")
+        source_path = settings['local_apps']['SPLUNK_ES_CONTENT_UPDATE']
+        dest_path = os.path.join(local_volume_path, os.path.basename(settings['local_apps']['SPLUNK_ES_CONTENT_UPDATE']))
+        
+    elif 'SPLUNK_ES_CONTENT_UPDATE' not in settings['local_apps']:
+        print("%s was not found in %s.  We assume this is an error and shut down.\n\t"\
+              "Quitting..."%('SPLUNK_ES_CONTENT_UPDATE', "settings['local_apps']"), file=sys.stderr)
+        sys.exit(1)
     else:
-        print("Using previous generated ESCU package: [%s]" % (
-            pregenerated_escu_package.name))
-        try:
-            with open(os.path.join(local_volume_path, os.path.basename(pregenerated_escu_package.name)), 'wb') as escu_package:
-                escu_package.write(pregenerated_escu_package.read())
-        except Exception as e:
-            print("Failure writing the ESCU package [%s] to [%s]: [%s].\n\tQuitting..." % (
-                pregenerated_escu_package.name, local_volume_path, str(e)))
-            sys.exit(1)
+        #Need to generate that package
+        source_path = generate_escu_app(settings['persist_security_content'])
+        dest_path = os.path.join(local_volume_path, os.path.basename(source_path))
 
-    print("Wrote ESCU package to volume.")
 
+    #Now write out the package, whether it was previously generated or
+    #we just generated it
+    try:
+        shutil.copy(source_path, dest_path)
+    except shutil.SameFileError as e:
+        #Same file, not a real error.  The copy just doesn't happen
+        pass
+    except Exception as e:
+        print("Error copying ESCU Package [%s] to [%s].\n\tQuitting..."%(source_path, dest_path), file=sys.stderr)
+        sys.exit(1)
+
+
+    print("Wrote ESCU package to volume folder.")
+
+    if settings['mock']:
+        print("Okay, just a mock")
+        sys.exit(0)
+    print("More than a mock")        
+    '''
     if args.split_detections_then_stop:
         for output_file_index in range(0, num_containers):
             fname = "container_%d_tests.txt" % (output_file_index)
@@ -454,7 +483,7 @@ def main(args):
     #testing_service.prepare_detection_testing(ssh_key_name, private_key, splunk_ip, splunk_password)
 
     #testing_service.test_detections(ssh_key_name, private_key, splunk_ip, splunk_password, test_files, uuid_test)
-
+    '''
 
 if __name__ == "__main__":
     main(sys.argv[1:])
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py b/automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py
index db587a93a0..4592279aa0 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py
@@ -54,7 +54,7 @@ setup_schema = {
                         "type": "integer"
                     },
                     "app_version": {
-                        "type": "string"
+                        "type": ["string","null"]
                     },
                     "local_path": {
                         "type": ["string", "null"],
@@ -66,7 +66,7 @@ setup_schema = {
                 {
                     "app_name": "SPLUNK_ES_CONTENT_UPDATE",
                     "app_number": 3449,
-                    "app_version": "GENERATED",
+                    "app_version": None,
                     'local_path': None
                 }
             ]
@@ -224,9 +224,9 @@ setup_schema = {
             "type": "array",
             "items": {
                 "type": "string",
-                "enum": ["Anomaly", "Hutning", "TTP"]
+                "enum": ["Anomaly", "Hunting", "TTP"]
             },
-            "default": ["Anomaly", "Hutning", "TTP"]
+            "default": ["Anomaly", "Hunting", "TTP"]
         },
     }
 }
@@ -245,16 +245,16 @@ def check_dependencies(settings: dict) -> bool:
     error_free = True
     if settings['mode'] == 'selected':
         # Make sure that exactly one of the following fields is populated
-        if settings['detections_file'] == None and settings['detections_list'] == []:
+        if settings['detections_file'] == None and settings['detections_list'] == None:
             print("Error - mode was 'selected' but no detections_list or detections_file were supplied.", file=sys.stderr)
             error_free = False
-        elif settings['detections_file'] != None and settings['detections_list'] != []:
+        elif settings['detections_file'] != None and settings['detections_list'] != None:
             print("Error - mode was 'selected' but detections_list and detections_file were supplied.", file=sys.stderr)
             error_free = False
     if settings['mode'] != 'selected' and settings['detections_file'] != None:
         print("Error - mode was not 'selected' but detections_file was supplied.", file=sys.stderr)
         error_free = False
-    elif settings['mode'] != 'selected' and settings['detections_list'] != []:
+    elif settings['mode'] != 'selected' and settings['detections_list'] != None:
         print("Error - mode was not 'selected' but detections_list was supplied.", file=sys.stderr)
         error_free = False
 
@@ -271,7 +271,6 @@ def validate(configuration: dict) -> tuple[Union[dict, None], dict]:
         
         no_complex_errors = check_dependencies(validated_json)
         if len(validation_errors) == 0 and no_complex_errors:
-            print("Input configuration successfully validated!")
             return validated_json, setup_schema
         elif no_complex_errors == False:
             print("Failed due to error(s) listed above.", file=sys.stderr)

From bc4a4c0f458dc80e6113b5119be54a9a895e8fb2 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Fri, 19 Nov 2021 14:19:14 -0800
Subject: [PATCH 072/166] More changes around the generation of the ESCU
 package and testing around persist_security_content option.

---
 .../detection_testing_execution.py            | 57 ++++++++++++-------
 .../modules/github_service.py                 | 32 +++++------
 2 files changed, 52 insertions(+), 37 deletions(-)

diff --git a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
index 1db14dd986..bb529e259c 100644
--- a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
+++ b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
@@ -83,35 +83,47 @@ def generate_escu_app(persist_security_content:bool=False)->str:
                         ". ./.venv/bin/activate",
                         "python3 -m pip install wheel",
                         "python3 -m pip install -r requirements.txt",
-                        "python3 contentctl.py --path . --verbose generate --product ESCU --output dist/escu", "tar -czf DA-ESS-ContentUpdate.spl -C dist/escu ."]
+                        "python3 contentctl.py --path . --verbose generate --product ESCU --output dist/escu", 
+                        "tar -czf DA-ESS-ContentUpdate.spl -C dist/escu ."]
         else:
-            commands = ["s. ./.venv/bin/activate", "python3 contentctl.py --path . --verbose generate --product ESCU --output dist/escu",
+            commands = [". ./.venv/bin/activate", 
+                        "python3 contentctl.py --path . --verbose generate --product ESCU --output dist/escu",
                         "tar -czf DA-ESS-ContentUpdate.spl -C dist/escu ."]
         ret = subprocess.run("; ".join(commands),
                              shell=True, capture_output=True)
         if ret.returncode != 0:
-            print("Error generating new content.  Exiting...")
+            print("Error generating new content.\n\tQuitting and dumping error...\n[%s]" % (ret.stderr))
             sys.exit(1)
         
 
         
         output_file_name = "DA-ESS-ContentUpdate-latest.tar.gz"
-        output_file_path_from_root = os.path.join("security_content", "slim_packaging", "apps", "upload", output_file_name)
-        output_file_path_from_apps = os.path.join("upload", output_file_name)
+        output_file_path_from_slim_latest = os.path.join("upload", output_file_name)
+        output_file_path_from_security_content = os.path.join("slim_packaging", "slim_latest", output_file_path_from_slim_latest)
+        output_file_path_from_root = os.path.join("security_content", output_file_path_from_security_content
+        
+        
 
         if persist_security_content is True:
-            os.chdir("slim_packaging")
-            commands = ["cd slim-latest",
+            try:
+                os.remove(output_file_path_from_slim_latest)
+            except FileNotFoundError:
+                #No problem if we fail to remove it, that just means it wasn't there and we didn't need to
+                pass
+            except Exception as e:
+                print("Error deleting the (possibly) existing old ESCU File: [%s]"%(str(e)), file=sys.stderr)
+                sys.exit(1)
+
+            #There remove the latest file if it exists
+            commands = ["cd slim_packaging/slim_latest",
                         ". ./.venv/bin/activate",
                         "cp -R ../../dist/escu DA-ESS-ContentUpdate",
                         "slim package -o upload DA-ESS-ContentUpdate",
-                        "cp upload/DA-ESS-ContentUpdate*.tar.gz %s" % (output_file_path_from_apps)]
+                        "cp upload/DA-ESS-ContentUpdate*.tar.gz %s" % (output_file_path_from_slim_latest)]
 
         else:
             os.mkdir("slim_packaging")
-            os.chdir("slim_packaging")
             os.mkdir("apps")
-
             try:
                 SPLUNK_PACKAGING_TOOLKIT_URL = "https://download.splunk.com/misc/packaging-toolkit/splunk-packaging-toolkit-0.9.0.tar.gz"
                 SPLUNK_PACKAGING_TOOLKIT_FILENAME = 'splunk-packaging-toolkit-latest.tar.gz'
@@ -126,10 +138,11 @@ def generate_escu_app(persist_security_content:bool=False)->str:
                       (str(e)), file=sys.stderr)
                 sys.exit(1)
 
-            commands = ["rm -rf slim-latest",
-                        "mkdir slim-latest",
-                        "tar -zxf splunk-packaging-toolkit-latest.tar.gz -C slim-latest --strip-components=1",
-                        "cd slim-latest",
+            commands = ["rm -rf slim_packaging/slim_latest",
+                        "mkdir slim_packaging/slim_latest",
+                        "cd splim_packaging",
+                        "tar -zxf splunk-packaging-toolkit-latest.tar.gz -C slim_latest --strip-components=1",
+                        "cd slim_latest",
                         "virtualenv --python=/usr/bin/python2.7 --clear .venv",
                         ". ./.venv/bin/activate",
                         "python3 -m pip install --upgrade pip",
@@ -138,14 +151,15 @@ def generate_escu_app(persist_security_content:bool=False)->str:
                         "python2 -m pip install .",
                         "cp -R ../../dist/escu DA-ESS-ContentUpdate",
                         "slim package -o upload DA-ESS-ContentUpdate",
-                        "cp upload/DA-ESS-ContentUpdate*.tar.gz %s" % (output_file_path_from_apps)]
+                        "cp upload/DA-ESS-ContentUpdate*.tar.gz %s" % (output_file_path_from_slim_latest)]
 
         ret = subprocess.run("; ".join(commands),
                              shell=True, capture_output=True)
         if ret.returncode != 0:
-            print("Error generating new ESCU Package.\n\tQuitting..." % ())
+            print("Command List:\n%s"%(commands))
+            print("Error generating new ESCU Package.\n\tQuitting and dumping error...\n[%s]" % (ret.stderr.decode('utf-8')),file=sys.stderr)
             sys.exit(1)
-        os.chdir("../..")
+        os.chdir("../")
 
         return output_file_path_from_root
 
@@ -162,7 +176,7 @@ def main(args):
     elif action != "run":
         print("Unsupported action: [%s]" % (action), file=sys.stderr)
         sys.exit(1)
-    print("time to run the test!")
+    
 
     '''
     parser = argparse.ArgumentParser(description="CI Detection Testing")
@@ -235,7 +249,7 @@ def main(args):
 
     all_test_files = github_service.get_test_files(settings['mode'],
                                                    settings['folders'],
-                                                   settings['ttps'],
+                                                   settings['types'],
                                                    settings['detections_list'],
                                                    settings['detections_file'])
 
@@ -266,7 +280,7 @@ def main(args):
         source_path = settings['local_apps']['SPLUNK_ES_CONTENT_UPDATE']
         dest_path = os.path.join(local_volume_path, os.path.basename(settings['local_apps']['SPLUNK_ES_CONTENT_UPDATE']))
         
-    elif 'SPLUNK_ES_CONTENT_UPDATE' not in settings['local_apps']:
+    elif 'SPLUNK_ES_CONTENT_UPDATE' not in [app['app_name'] for app in settings['local_apps']]:
         print("%s was not found in %s.  We assume this is an error and shut down.\n\t"\
               "Quitting..."%('SPLUNK_ES_CONTENT_UPDATE', "settings['local_apps']"), file=sys.stderr)
         sys.exit(1)
@@ -284,7 +298,8 @@ def main(args):
         #Same file, not a real error.  The copy just doesn't happen
         pass
     except Exception as e:
-        print("Error copying ESCU Package [%s] to [%s].\n\tQuitting..."%(source_path, dest_path), file=sys.stderr)
+        print(os.getcwd())
+        print("Error copying ESCU Package [%s] to [%s]: [%s].\n\tQuitting..."%(source_path, dest_path, str(e)), file=sys.stderr)
         sys.exit(1)
 
 
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/github_service.py b/automated_detection_testing/ci/detection_testing_batch/modules/github_service.py
index efb4826ce5..dad73ed194 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/github_service.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/github_service.py
@@ -44,7 +44,7 @@ class GithubService:
 
     def prune_detections(self,
                          detections_to_prune: list[str],
-                         ttps_to_test: list[str],
+                           types_to_test: list[str],
                          previously_successful_tests: list[str],
                          exclude_ssa: bool = True,
                          summary_file: str = None) -> list[str]:
@@ -60,8 +60,8 @@ class GithubService:
                     'detections', 'tests') + '.test.yml'
                 test_filepath_without_security_content = str(
                     pathlib.Path(*pathlib.Path(test_filepath).parts[1:]))
-                # If no TTPS are provided, then we will get everything
-                if 'type' in description and (description['type'] in ttps_to_test or len(ttps_to_test) == 0):
+                # If no   types are provided, then we will get everything
+                if 'type' in description and (description['type'] in   types_to_test or len(  types_to_test) == 0):
                     # print(description['type'])
                     if not os.path.exists(test_filepath):
                         print("Detection [%s] references [%s], but it does not exist" % (
@@ -105,11 +105,11 @@ class GithubService:
 
         return pruned_tests
 
-    def get_test_files(self, mode: str, folders:list[str], ttps:list[str],
+    def get_test_files(self, mode: str, folders:list[str],   types:list[str],
                         detections_list: Union[list[str], None], 
                        detections_file=Union[str, None]) -> list[str]:
         if mode == "changes":
-            tests = self.get_changed_test_files(folders, ttps)
+            tests = self.get_changed_test_files(folders,   types)
         elif mode == "selected":
             if detections_list is None and detections_file is None:
                 #It's actually valid to supply an EMPTY list of files and the test should pass.
@@ -124,7 +124,7 @@ class GithubService:
                       (detections_list, detections_file), file=sys.stderr)
                 sys.exit(1)
             if detections_list is not None:
-                tests = self.get_selected_test_files(detections_list, folders, ttps)
+                tests = self.get_selected_test_files(detections_list, folders,   types)
             if detections_file is not None:
                 try:
                     with open(detections_file,'r') as f:
@@ -136,10 +136,10 @@ class GithubService:
                           "Quitting..."%(detections_file, str(e)))
                     sys.exit(1)
                 
-                tests = self.get_selected_test_files(files_to_test, folders, ttps)
+                tests = self.get_selected_test_files(files_to_test, folders,   types)
 
         elif mode == "all":
-            tests = self.get_all_tests_and_detections(folders,ttps)
+            tests = self.get_all_tests_and_detections(folders,  types)
         else:
             print(
                 "Error, unsupported mode [%s].  Mode must be one of %s", file=sys.stderr)
@@ -149,16 +149,16 @@ class GithubService:
 
     def get_selected_test_files(self,
                                 detection_file_list: list[str],
-                                ttps_to_test: list[str] = [
+                                  types_to_test: list[str] = [
                                     "Anomaly", "Hunting", "TTP"],
                                 previously_successful_tests: list[str] = []) -> list[str]:
 
-        return self.prune_detections(detection_file_list, ttps_to_test, previously_successful_tests)
+        return self.prune_detections(detection_file_list,   types_to_test, previously_successful_tests)
 
     def get_all_tests_and_detections(self,
                                      folders: list[str] = [
                                          'endpoint', 'cloud', 'network'],
-                                     ttps_to_test: list[str] = [
+                                       types_to_test: list[str] = [
                                          "Anomaly", "Hunting", "TTP"],
                                      previously_successful_tests: list[str] = []) -> list[str]:
         detections = []
@@ -167,13 +167,13 @@ class GithubService:
                 os.path.join("security_content/detections", folder), "*.yml"))
 
         # Prune this down to only the subset of detections we can test
-        return self.prune_detections(detections, ttps_to_test, previously_successful_tests)
+        return self.prune_detections(detections,   types_to_test, previously_successful_tests)
 
     def get_all_files_in_folder(self, foldername: str, extension: str) -> list[str]:
         filenames = glob.glob(os.path.join(foldername, extension))
         return filenames
 
-    def get_changed_test_files(self, folders=['endpoint', 'cloud', 'network'], ttps_to_test=["Anomaly", "Hunting", "TTP"], previously_successful_tests=[]) -> list[str]:
+    def get_changed_test_files(self, folders=['endpoint', 'cloud', 'network'],   types_to_test=["Anomaly", "Hunting", "TTP"], previously_successful_tests=[]) -> list[str]:
         branch1 = self.security_content_branch
         branch2 = 'develop'
         g = git.Git('security_content')
@@ -193,8 +193,8 @@ class GithubService:
                     if 'detections' in file_path and os.path.basename(file_path).endswith('.yml'):
                         changed_detection_files.append(file_path)
         else:
-            print("Looking for changed detections by diffing [%s] against [%s].  Of course none were returned." % (
-                branch1, branch2))
+            print("Looking for changed detections by diffing [%s] against [%s].  They are the same branch, so none were returned." % (
+                branch1, branch2), file=sys.stderr)
             return []
 
         # all files have the format A\tFILENAME or M\tFILENAME.  Get rid of those leading characters
@@ -214,7 +214,7 @@ class GithubService:
             if name not in changed_detection_files:
                 changed_detection_files.append(name)
 
-        return self.prune_detections(changed_detection_files, ttps_to_test, previously_successful_tests)
+        return self.prune_detections(changed_detection_files,   types_to_test, previously_successful_tests)
 
         #detections_to_test,_,_ = self.filter_test_types(changed_detection_files)
         # for f in detections_to_test:

From c0bdcc6c310ac7e300d66b970a86648889c08907 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Fri, 19 Nov 2021 14:38:45 -0800
Subject: [PATCH 073/166] Successful behavior generating escu package
 regardless of whether --persist-security-content is passed

---
 .../detection_testing_execution.py                     | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
index bb529e259c..8eba06e44c 100644
--- a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
+++ b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
@@ -77,7 +77,6 @@ def generate_escu_app(persist_security_content:bool=False)->str:
     # Go into the security content directory
         print("****GENERATING ESCU APP****")
         os.chdir("security_content")
-        print(os.getcwd())
         if persist_security_content is False:
             commands = ["python3 -m venv .venv",
                         ". ./.venv/bin/activate",
@@ -100,15 +99,15 @@ def generate_escu_app(persist_security_content:bool=False)->str:
         output_file_name = "DA-ESS-ContentUpdate-latest.tar.gz"
         output_file_path_from_slim_latest = os.path.join("upload", output_file_name)
         output_file_path_from_security_content = os.path.join("slim_packaging", "slim_latest", output_file_path_from_slim_latest)
-        output_file_path_from_root = os.path.join("security_content", output_file_path_from_security_content
+        output_file_path_from_root = os.path.join("security_content", output_file_path_from_security_content)
         
         
 
         if persist_security_content is True:
             try:
-                os.remove(output_file_path_from_slim_latest)
+                os.remove(output_file_path_from_security_content)
             except FileNotFoundError:
-                #No problem if we fail to remove it, that just means it wasn't there and we didn't need to
+                #No problem if we fail to remove it, that just means it wasn't there and we didn't need to    
                 pass
             except Exception as e:
                 print("Error deleting the (possibly) existing old ESCU File: [%s]"%(str(e)), file=sys.stderr)
@@ -140,7 +139,7 @@ def generate_escu_app(persist_security_content:bool=False)->str:
 
             commands = ["rm -rf slim_packaging/slim_latest",
                         "mkdir slim_packaging/slim_latest",
-                        "cd splim_packaging",
+                        "cd slim_packaging",
                         "tar -zxf splunk-packaging-toolkit-latest.tar.gz -C slim_latest --strip-components=1",
                         "cd slim_latest",
                         "virtualenv --python=/usr/bin/python2.7 --clear .venv",
@@ -298,7 +297,6 @@ def main(args):
         #Same file, not a real error.  The copy just doesn't happen
         pass
     except Exception as e:
-        print(os.getcwd())
         print("Error copying ESCU Package [%s] to [%s]: [%s].\n\tQuitting..."%(source_path, dest_path, str(e)), file=sys.stderr)
         sys.exit(1)
 

From 6a9b5504aaba54d9117adddf6169f6c8884f324e Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Fri, 19 Nov 2021 16:06:04 -0800
Subject: [PATCH 074/166] Converted the app listings in the jsonschema from
 arrays to dicts. they are easier to search for apps that way. also some more
 testing of the escu app building and persist_security_content. looking good.

---
 .../detection_testing_execution.py            |  59 +++++++-
 .../modules/new_arguments2.py                 |  13 +-
 .../modules/validate_args.py                  | 133 ++++++++++--------
 3 files changed, 130 insertions(+), 75 deletions(-)

diff --git a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
index 8eba06e44c..b7c7fe7ab5 100644
--- a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
+++ b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
@@ -10,6 +10,8 @@ import threading
 import queue
 
 from docker.client import DockerClient
+from modules import validate_args
+from modules.validate_args import validate
 from modules.github_service import GithubService
 from modules import aws_service, testing_service
 import time
@@ -163,7 +165,7 @@ def generate_escu_app(persist_security_content:bool=False)->str:
         return output_file_path_from_root
 
 
-def main(args):
+def main(args:list[str]):
     requests.packages.urllib3.disable_warnings()
 
     start_datetime = datetime.now()
@@ -273,13 +275,14 @@ def main(args):
         sys.exit(1)
 
     #Check to see if we want to install ESCU and whether it was preeviously generated and we should use that file
-    if 'SPLUNK_ES_CONTENT_UPDATE' in settings['local_apps'] and  settings['local_apps']['SPLUNK_ES_CONTENT_UPDATE'] is not None:
+    if 'SPLUNK_ES_CONTENT_UPDATE' in settings['local_apps'] and settings['local_apps']['SPLUNK_ES_CONTENT_UPDATE']['local_path'] is not None:
         #Using a pregenerated ESCU, copy it to apps (unless it)
 
-        source_path = settings['local_apps']['SPLUNK_ES_CONTENT_UPDATE']
-        dest_path = os.path.join(local_volume_path, os.path.basename(settings['local_apps']['SPLUNK_ES_CONTENT_UPDATE']))
+        file_path = settings['local_apps']['SPLUNK_ES_CONTENT_UPDATE']['local_path']
+        source_path = file_path
+        dest_path = os.path.join(local_volume_path, os.path.basename(file_path))
         
-    elif 'SPLUNK_ES_CONTENT_UPDATE' not in [app['app_name'] for app in settings['local_apps']]:
+    elif 'SPLUNK_ES_CONTENT_UPDATE' not in settings['local_apps']:
         print("%s was not found in %s.  We assume this is an error and shut down.\n\t"\
               "Quitting..."%('SPLUNK_ES_CONTENT_UPDATE', "settings['local_apps']"), file=sys.stderr)
         sys.exit(1)
@@ -304,8 +307,50 @@ def main(args):
     print("Wrote ESCU package to volume folder.")
 
     if settings['mock']:
-        print("Okay, just a mock")
-        sys.exit(0)
+        def finish_mock(num_containers:int, detections:list[str], output_file_template:str="config_tests_%d.json"):
+            for output_file_index in range(0, num_containers):
+                fname = output_file_template % (output_file_index)
+                
+                #Get the n'th detection for this file
+                detection_tests = detections[output_file_index::num_containers]
+                normalized_detection_names = []
+                #Normalize the test filename to the name of the detection instead.
+                #These are what we should write to the file
+                for d in detection_tests:
+                    filename = os.path.basename(d)
+                    filename = filename.replace(".test.yml", ".yml")
+                    leading = os.path.split(d)[0]
+                    leading = leading.replace("tests/", "detections/")
+                    new_name = os.path.join(
+                        "security_content", leading, filename)
+                    normalized_detection_names.append(new_name)
+                
+                #Generate an appropriate config file for this test
+                import copy
+                mock_settings = copy.deepcopy(settings)
+                #This may be able to support as many as 2 for GitHub Actions...
+                #we will have to determine in testing.
+                mock_settings['num_containers'] = 1
+                
+                #Must be selected since we are passing in a list of detections
+                mock_settings['mode'] = 'selected'
+            
+                #Pass in the list of detections to run
+                mock_settings['detections_list'] = normalized_detection_names
+
+                #We want to persist security content and run with the escu package that we created
+                mock_settings['persist_security_content'] = True
+
+                #mock_settings['persist_security_content'][]
+
+
+                
+
+                
+            print("Done")
+
+        
+            sys.exit(0)
     print("More than a mock")        
     '''
     if args.split_detections_then_stop:
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/new_arguments2.py b/automated_detection_testing/ci/detection_testing_batch/modules/new_arguments2.py
index 460d9af287..808aa26f89 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/new_arguments2.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/new_arguments2.py
@@ -55,17 +55,12 @@ def configure_action(args) -> tuple[str, dict]:
         print("\t{0}\n".format(formatted_print))
 
     # Now parse the new config and make sure it's good
-    validated_new_settings, schema = validate_args.validate(new_config)
-    if validated_new_settings == None:
-        print("Error in the new settings!")
+    validated_new_settings, schema = validate_args.validate_and_write(new_config, args.output_config_file)
+    if validated_new_settings == None:        
+        print("Could not update settings.\n\tQuitting...", file=sys.stderr)
         sys.exit(1)
-        
-    else:
-        print("New settings successful.  Writing results to: %s" %
-              (args.output_config_file.name))
-        args.output_config_file.write(json.dumps(
-            validated_new_settings, sort_keys=True, indent=4))
     
+
     return ("configure", validated_new_settings)
 
 
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py b/automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py
index 4592279aa0..3ae4c5b12a 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py
@@ -30,7 +30,7 @@ setup_schema = {
         },
 
         "detections_list": {
-            "type": ["array","null"],
+            "type": ["array", "null"],
             "items": {
                 "type": "string"
             },
@@ -43,36 +43,37 @@ setup_schema = {
         },
 
         "local_apps": {
-            "type": "array",
-            "items": {
-                "type": "object",
-                "properties": {
-                    "app_name": {
+            "type": "object",
+            "properties": {
+                "value": {
+                    "type": "string",
+                    "properties": {
+                         "app_name": {
                         "type": "string"
                     },
                     "app_number": {
                         "type": "integer"
                     },
                     "app_version": {
-                        "type": ["string","null"]
+                        "type": ["string", "null"]
                     },
                     "local_path": {
                         "type": ["string", "null"],
                         "default": None
                     },
                 }
-            },
-            "default": [
-                {
-                    "app_name": "SPLUNK_ES_CONTENT_UPDATE",
-                    "app_number": 3449,
-                    "app_version": None,
-                    'local_path': None
                 }
-            ]
-
+            },
+            "default": {
+               "SPLUNK_ES_CONTENT_UPDATE": {
+                   "app_number": 3449,
+                   "app_version": None,
+                   'local_path': None
+               }
+            }
         },
 
+
         "mode": {
             "type": "string",
             "enum": ["changes", "selected", "all"],
@@ -107,84 +108,80 @@ setup_schema = {
         },
 
         "splunkbase_apps": {
-            "type": "array",
-            "items": {
-                "type": "object",
-                "properties": {
-                    "app_name": {
-                        "type": "string"
-                    },
-                    "app_number": {
-                        "type": "integer"
-                    },
-                    "app_version": {
-                        "type": "string"
-                    }
-                },
-            },
-            "default":  [
+            "type": "object",
+            "properies": {
+               "value":
                 {
-                    "app_name": "SPLUNK_ADD_ON_FOR_AMAZON_WEB_SERVICES",
+                   "type": "string",
+                    "properties": {
+                        "app_number": {
+                         "type": "integer"
+                     },
+                     "app_version": {
+                         "type": "string"
+                    }
+                    }
+                }
+            },
+            "default": {
+                "SPLUNK_ADD_ON_FOR_AMAZON_WEB_SERVICES":{
                     "app_number": 1876,
                     "app_version": "5.2.0"
                 },
+
+                "SPLUNK_ADD_ON_FOR_MICROSOFT_OFFICE_365":
                 {
-                    "app_name": "SPLUNK_ADD_ON_FOR_MICROSOFT_OFFICE_365",
                     "app_number": 4055,
                     "app_version": "2.2.0"
                 },
-                {
-                    "app_name": "SPLUNK_ADD_ON_FOR_AMAZON_KINESIS_FIREHOSE",
+
+                "SPLUNK_ADD_ON_FOR_AMAZON_KINESIS_FIREHOSE": {
                     "app_number": 3719,
                     "app_version": "1.3.2"
                 },
-                {
-                    "app_name": "SPLUNK_ANALYTIC_STORY_EXECUTION_APP",
+
+                "SPLUNK_ANALYTIC_STORY_EXECUTION_APP": {
                     "app_number": 4971,
                     "app_version": "2.0.3"
                 },
-                {
-                    "app_name": "PYTHON_FOR_SCIENTIC_COMPUTING_LINUX_64_BIT",
+
+                "PYTHON_FOR_SCIENTIC_COMPUTING_LINUX_64_BIT": {
                     "app_number": 2882,
                     "app_version": "2.0.2"
                 },
-                {
-                    "app_name": "SPLUNK_MACHINE_LEARNING_TOOLKIT",
+
+                "SPLUNK_MACHINE_LEARNING_TOOLKIT": {
                     "app_number": 2890,
                     "app_version": "5.2.2"
                 },
-                {
-                    "app_name": "SPLUNK_APP_FOR_STREAM",
+
+                "SPLUNK_APP_FOR_STREAM": {
                     "app_number": 1809,
                     "app_version": "8.0.1"
                 },
-                {
-                    "app_name": "SPLUNK_ADD_ON_FOR_STREAM_WIRE_DATA",
+                "SPLUNK_ADD_ON_FOR_STREAM_WIRE_DATA": {
                     "app_number": 5234,
                     "app_version": "8.0.1"
                 },
-                {
-                    "app_name": "SPLUNK_ADD_ON_FOR_STREAM_FORWARDERS",
+                "SPLUNK_ADD_ON_FOR_STREAM_FORWARDERS": {
                     "app_number": 5238,
                     "app_version": "8.0.1"
                 },
-                {
-                    "app_name": "SPLUNK_ADD_ON_FOR_ZEEK_AKA_BRO",
+                "SPLUNK_ADD_ON_FOR_ZEEK_AKA_BRO": {
                     "app_number": 1617,
                     "app_version": "4.0.0"
                 },
-                {
-                    "app_name": "SPLUNK_ADD_ON_FOR_UNIX_AND_LINUX",
+                "SPLUNK_ADD_ON_FOR_UNIX_AND_LINUX": {
                     "app_number": 833,
                     "app_version": "8.3.1"
                 },
-                {
-                    "app_name": "SPLUNK_COMMON_INFORMATION_MODEL",
+                "SPLUNK_COMMON_INFORMATION_MODEL": {
                     "app_number": 1621,
                     "app_version": "4.20.2"
                 }
-            ]
+            }
         },
+
         "splunkbase_username": {
             "type": ["string", "null"],
             "default": None
@@ -262,13 +259,31 @@ def check_dependencies(settings: dict) -> bool:
     return error_free
 
 
+def validate_and_write(configuration: dict, output_file: io.TextIOWrapper) -> tuple[Union[dict, None], dict]:
+    validated_json, setup_schema = validate(configuration)
+    if validated_json == None:
+        print("Error in the new settings! No output file written")
+    else:
+        print("Settings updated.  Writing results to: %s" %
+              (output_file.name))
+        try:
+            output_file.write(json.dumps(
+                validated_json, sort_keys=True, indent=4))
+        except Exception as e:
+            print("Error writing settings to %s: [%s]" % (
+                output_file.name, str(e)), file=sys.stderr)
+            return None, setup_schema
+
+    return validated_json, setup_schema
+
+
 def validate(configuration: dict) -> tuple[Union[dict, None], dict]:
-    #v = jsonschema.Draft201909Validator(argument_schema)
+    # v = jsonschema.Draft201909Validator(argument_schema)
 
     try:
         validation_errors, validated_json = jsonschema_errorprinter.check_json(
             configuration, setup_schema)
-        
+
         no_complex_errors = check_dependencies(validated_json)
         if len(validation_errors) == 0 and no_complex_errors:
             return validated_json, setup_schema
@@ -277,7 +292,7 @@ def validate(configuration: dict) -> tuple[Union[dict, None], dict]:
             return None, setup_schema
         else:
             print("[%d] failures detected during validation of the configuration!" % (
-                len(validation_errors)),file=sys.stderr)
+                len(validation_errors)), file=sys.stderr)
             for error in validation_errors:
                 print(error, end="\n\n", file=sys.stderr)
             return None, setup_schema
@@ -292,7 +307,7 @@ def validate(configuration: dict) -> tuple[Union[dict, None], dict]:
         print("Error validating the json", file=sys.stderr)
         print(e)
         return False
-        
+
     except jsonschema.exceptions.SchemaError as e:
         print("Error validating the schema", file=sys.stderr)
     """

From 3f1b02590898b140cda56d96d0f16ad8a1be7e57 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Fri, 19 Nov 2021 17:07:27 -0800
Subject: [PATCH 075/166] Mock is now working and splits the tests as intended.
  Appropriate mock config.json files are created, validated, and written.

---
 .../detection_testing_execution.py            | 96 ++++++++++++-------
 .../modules/github_service.py                 | 17 ++--
 .../modules/new_arguments2.py                 |  6 ++
 3 files changed, 77 insertions(+), 42 deletions(-)

diff --git a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
index b7c7fe7ab5..a80bb1a575 100644
--- a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
+++ b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
@@ -1,37 +1,33 @@
-import sys
 import argparse
-import shutil
+import copy
+import csv
+import json
 import os
-import time
+import queue
 import random
 import secrets
-import docker
-import threading
-import queue
-
-from docker.client import DockerClient
-from modules import validate_args
-from modules.validate_args import validate
-from modules.github_service import GithubService
-from modules import aws_service, testing_service
-import time
-import subprocess
-
-from timeit import default_timer as timer
-from datetime import timedelta
-from datetime import datetime
-import string
 import shutil
-from typing import Union
+import string
+import subprocess
+import sys
+import threading
+import time
 from collections import OrderedDict
+from datetime import datetime, timedelta
 from tempfile import mkdtemp
-import csv
+from timeit import default_timer as timer
+from typing import Union
 
-from requests import get
-import json
+import docker
 import requests.packages.urllib3
+from docker.client import DockerClient
+from requests import get
+from modules.validate_args import validate_and_write
 
 import modules.new_arguments2
+from modules import aws_service, testing_service, validate_args
+from modules.github_service import GithubService
+from modules.validate_args import validate
 
 SPLUNK_CONTAINER_APPS_DIR = "/opt/splunk/etc/apps"
 index_file_local_path = "indexes.conf.tar"
@@ -52,12 +48,13 @@ def ensure_security_content(branch: str, pr_number: Union[int, None], persist_se
               "out of date. ******")
         github_service = GithubService(
             branch, existing_directory=persist_security_content)
-
-    elif persist_security_content is True:
-        print("Error - you chose --persist_security_content but the security_content directory does not exist!\n\tQuitting...")
-        sys.exit(1)
+    
     else:
-        if os.path.exists("security_content/"):
+        if persist_security_content is True and not os.path.exists("security_content"):
+            print("Error - you chose --persist_security_content but the security_content directory does not exist!"\
+                  "  We will check it out for you.\n\tQuitting...")
+        
+        elif os.path.exists("security_content/"):
             print("Deleting the security_content directory")
             try:
                 shutil.rmtree("security_content/", ignore_errors=True)
@@ -178,7 +175,8 @@ def main(args:list[str]):
         print("Unsupported action: [%s]" % (action), file=sys.stderr)
         sys.exit(1)
     
-
+    
+    
     '''
     parser = argparse.ArgumentParser(description="CI Detection Testing")
     parser.add_argument("-b", "--branch", type=str, required=True, help="security content branch")
@@ -248,6 +246,7 @@ def main(args:list[str]):
     github_service = ensure_security_content(
         settings['branch'], settings['pr_number'], settings['persist_security_content'])
 
+
     all_test_files = github_service.get_test_files(settings['mode'],
                                                    settings['folders'],
                                                    settings['types'],
@@ -307,7 +306,18 @@ def main(args:list[str]):
     print("Wrote ESCU package to volume folder.")
 
     if settings['mock']:
-        def finish_mock(num_containers:int, detections:list[str], output_file_template:str="config_tests_%d.json"):
+        def finish_mock(settings:dict, detections:list[str], output_file_template:str="prior_config/config_tests_%d.json"):
+            num_containers = settings['num_containers']
+
+            
+            try:
+                os.mkdir("prior_config")
+            except FileExistsError:
+                pass
+            except Exception as e:
+                print("Some error occured when trying to make the configs folder: [%s]\n\tQuitting..."%(str(e)), file=sys.stderr)
+                sys.exit(1)
+
             for output_file_index in range(0, num_containers):
                 fname = output_file_template % (output_file_index)
                 
@@ -326,7 +336,6 @@ def main(args:list[str]):
                     normalized_detection_names.append(new_name)
                 
                 #Generate an appropriate config file for this test
-                import copy
                 mock_settings = copy.deepcopy(settings)
                 #This may be able to support as many as 2 for GitHub Actions...
                 #we will have to determine in testing.
@@ -341,16 +350,33 @@ def main(args:list[str]):
                 #We want to persist security content and run with the escu package that we created
                 mock_settings['persist_security_content'] = True
 
-                #mock_settings['persist_security_content'][]
-
+                mock_settings['local_apps'] = { 
+                    "SPLUNK_ES_CONTENT_UPDATE": {
+                        "app_number": 3449,
+                        "app_version": None,
+                        'local_path': "prior_config/apps/DA-ESS-ContentUpdate-latest.tar.gz"}
+                    }
+                
+                mock_settings['mock'] = False
+                
+                #Make sure that it still validates after all of the changes
 
                 
-
                 
-            print("Done")
+                try:
+                    with open(fname,'w') as cfg:
+                        validated_settings,b = validate_and_write(mock_settings,cfg)
+                        if validated_settings is None:
+                            print("There was an error validating the updated mock settings.\n\tQuitting...",file=sys.stderr)
+                            sys.exit(1)
 
+                except Exception as e:
+                    print("Error writing config file %s: [%s]\n\tQuitting..."%(fname,str(e)),file=sys.stderr)
+                    sys.exit(1)
         
             sys.exit(0)
+        finish_mock(settings,all_test_files)
+
     print("More than a mock")        
     '''
     if args.split_detections_then_stop:
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/github_service.py b/automated_detection_testing/ci/detection_testing_batch/modules/github_service.py
index dad73ed194..5450279021 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/github_service.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/github_service.py
@@ -109,7 +109,7 @@ class GithubService:
                         detections_list: Union[list[str], None], 
                        detections_file=Union[str, None]) -> list[str]:
         if mode == "changes":
-            tests = self.get_changed_test_files(folders,   types)
+            tests = self.get_changed_test_files(folders, types)
         elif mode == "selected":
             if detections_list is None and detections_file is None:
                 #It's actually valid to supply an EMPTY list of files and the test should pass.
@@ -118,14 +118,14 @@ class GithubService:
                 print("Trying to test a list of files, but None were provided", file=sys.stderr)
                 sys.exit(1)
 
-            if detections_list is not None and detections_file is not None:
+            elif detections_list is not None and detections_file is not None:
                 print("Both detections_list [%s] and detections_file [%s] were provided.  "\
                       "Because these confilect, we cannot test.\n\tQuitting..."%
                       (detections_list, detections_file), file=sys.stderr)
                 sys.exit(1)
-            if detections_list is not None:
+            elif detections_list is not None:
                 tests = self.get_selected_test_files(detections_list, folders,   types)
-            if detections_file is not None:
+            elif detections_file is not None:
                 try:
                     with open(detections_file,'r') as f:
                         data = f.readlines()
@@ -137,15 +137,18 @@ class GithubService:
                     sys.exit(1)
                 
                 tests = self.get_selected_test_files(files_to_test, folders,   types)
+            else:
+                #impossible to get here
+                print("Impossible to get here.  Just kept to make the if/elif more self describing",file=sys.stderr)
+                sys.exit(1)
 
         elif mode == "all":
             tests = self.get_all_tests_and_detections(folders,  types)
         else:
-            print(
-                "Error, unsupported mode [%s].  Mode must be one of %s", file=sys.stderr)
+            print("Error, unsupported mode [%s].  Mode must be one of %s", file=sys.stderr)
             sys.exit(1)
 
-        return []
+        return tests
 
     def get_selected_test_files(self,
                                 detection_file_list: list[str],
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/new_arguments2.py b/automated_detection_testing/ci/detection_testing_batch/modules/new_arguments2.py
index 808aa26f89..ff1cf6c19f 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/new_arguments2.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/new_arguments2.py
@@ -131,6 +131,8 @@ def parse(args)->tuple[str,dict]:
                             "cannot be changed (except for credentials that can be "\
                             "entered on the command line).")
 
+    
+
     run_parser.add_argument('-user', '--splunkbase_username', required=False, type=str,
                             help="Username for login to splunkbase.  This is required "
                             "if downloading packages from Splunkbase.  While this can "
@@ -149,6 +151,10 @@ def parse(args)->tuple[str,dict]:
     run_parser.add_argument("-show_pass", "--show_splunk_app_password", required=False, 
                             action="store_true",
                             help="The password to login to the Splunk Server.  ")
+    
+    run_parser.add_argument("-m", "--mock", required=False, 
+                            action="store_true",
+                            help="Split into multiple configs, don't actually run the tests.")
 
     args = parser.parse_args()
     

From 3cd1e42ada50a7d7f27abf243a8f84eaa73d2258 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Mon, 22 Nov 2021 16:36:21 -0800
Subject: [PATCH 076/166] Huge refactor is almost complete.  Able to launch and
 run multiple containers now as intended

---
 .../detection_testing_execution.py            | 322 ++++++++++--------
 .../modules/container_manager.py              |  53 ++-
 .../modules/new_arguments2.py                 |  36 +-
 .../modules/splunk_container.py               |  70 ++--
 .../modules/validate_args.py                  |  34 +-
 5 files changed, 309 insertions(+), 206 deletions(-)

diff --git a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
index a80bb1a575..72ce7f2084 100644
--- a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
+++ b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
@@ -23,6 +23,7 @@ import requests.packages.urllib3
 from docker.client import DockerClient
 from requests import get
 from modules.validate_args import validate_and_write
+from modules import container_manager
 
 import modules.new_arguments2
 from modules import aws_service, testing_service, validate_args
@@ -33,6 +34,7 @@ SPLUNK_CONTAINER_APPS_DIR = "/opt/splunk/etc/apps"
 index_file_local_path = "indexes.conf.tar"
 index_file_container_path = os.path.join(SPLUNK_CONTAINER_APPS_DIR, "search")
 
+# Should be the last one we copy.
 datamodel_file_local_path = "datamodels.conf.tar"
 datamodel_file_container_path = os.path.join(
     SPLUNK_CONTAINER_APPS_DIR, "Splunk_SA_CIM")
@@ -48,12 +50,12 @@ def ensure_security_content(branch: str, pr_number: Union[int, None], persist_se
               "out of date. ******")
         github_service = GithubService(
             branch, existing_directory=persist_security_content)
-    
+
     else:
         if persist_security_content is True and not os.path.exists("security_content"):
-            print("Error - you chose --persist_security_content but the security_content directory does not exist!"\
+            print("Error - you chose --persist_security_content but the security_content directory does not exist!"
                   "  We will check it out for you.\n\tQuitting...")
-        
+
         elif os.path.exists("security_content/"):
             print("Deleting the security_content directory")
             try:
@@ -72,97 +74,99 @@ def ensure_security_content(branch: str, pr_number: Union[int, None], persist_se
     return github_service
 
 
-def generate_escu_app(persist_security_content:bool=False)->str:
+def generate_escu_app(persist_security_content: bool = False) -> str:
     # Go into the security content directory
-        print("****GENERATING ESCU APP****")
-        os.chdir("security_content")
-        if persist_security_content is False:
-            commands = ["python3 -m venv .venv",
-                        ". ./.venv/bin/activate",
-                        "python3 -m pip install wheel",
-                        "python3 -m pip install -r requirements.txt",
-                        "python3 contentctl.py --path . --verbose generate --product ESCU --output dist/escu", 
-                        "tar -czf DA-ESS-ContentUpdate.spl -C dist/escu ."]
-        else:
-            commands = [". ./.venv/bin/activate", 
-                        "python3 contentctl.py --path . --verbose generate --product ESCU --output dist/escu",
-                        "tar -czf DA-ESS-ContentUpdate.spl -C dist/escu ."]
-        ret = subprocess.run("; ".join(commands),
-                             shell=True, capture_output=True)
-        if ret.returncode != 0:
-            print("Error generating new content.\n\tQuitting and dumping error...\n[%s]" % (ret.stderr))
+    print("****GENERATING ESCU APP****")
+    os.chdir("security_content")
+    if persist_security_content is False:
+        commands = ["python3 -m venv .venv",
+                    ". ./.venv/bin/activate",
+                    "python3 -m pip install wheel",
+                    "python3 -m pip install -r requirements.txt",
+                    "python3 contentctl.py --path . --verbose generate --product ESCU --output dist/escu",
+                    "tar -czf DA-ESS-ContentUpdate.spl -C dist/escu ."]
+    else:
+        commands = [". ./.venv/bin/activate",
+                    "python3 contentctl.py --path . --verbose generate --product ESCU --output dist/escu",
+                    "tar -czf DA-ESS-ContentUpdate.spl -C dist/escu ."]
+    ret = subprocess.run("; ".join(commands),
+                         shell=True, capture_output=True)
+    if ret.returncode != 0:
+        print("Error generating new content.\n\tQuitting and dumping error...\n[%s]" % (
+            ret.stderr))
+        sys.exit(1)
+
+    output_file_name = "DA-ESS-ContentUpdate-latest.tar.gz"
+    output_file_path_from_slim_latest = os.path.join(
+        "upload", output_file_name)
+    output_file_path_from_security_content = os.path.join(
+        "slim_packaging", "slim_latest", output_file_path_from_slim_latest)
+    output_file_path_from_root = os.path.join(
+        "security_content", output_file_path_from_security_content)
+
+    if persist_security_content is True:
+        try:
+            os.remove(output_file_path_from_security_content)
+        except FileNotFoundError:
+            # No problem if we fail to remove it, that just means it wasn't there and we didn't need to
+            pass
+        except Exception as e:
+            print("Error deleting the (possibly) existing old ESCU File: [%s]" % (
+                str(e)), file=sys.stderr)
             sys.exit(1)
-        
 
-        
-        output_file_name = "DA-ESS-ContentUpdate-latest.tar.gz"
-        output_file_path_from_slim_latest = os.path.join("upload", output_file_name)
-        output_file_path_from_security_content = os.path.join("slim_packaging", "slim_latest", output_file_path_from_slim_latest)
-        output_file_path_from_root = os.path.join("security_content", output_file_path_from_security_content)
-        
-        
+        # There remove the latest file if it exists
+        commands = ["cd slim_packaging/slim_latest",
+                    ". ./.venv/bin/activate",
+                    "cp -R ../../dist/escu DA-ESS-ContentUpdate",
+                    "slim package -o upload DA-ESS-ContentUpdate",
+                    "cp upload/DA-ESS-ContentUpdate*.tar.gz %s" % (output_file_path_from_slim_latest)]
 
-        if persist_security_content is True:
-            try:
-                os.remove(output_file_path_from_security_content)
-            except FileNotFoundError:
-                #No problem if we fail to remove it, that just means it wasn't there and we didn't need to    
-                pass
-            except Exception as e:
-                print("Error deleting the (possibly) existing old ESCU File: [%s]"%(str(e)), file=sys.stderr)
-                sys.exit(1)
-
-            #There remove the latest file if it exists
-            commands = ["cd slim_packaging/slim_latest",
-                        ". ./.venv/bin/activate",
-                        "cp -R ../../dist/escu DA-ESS-ContentUpdate",
-                        "slim package -o upload DA-ESS-ContentUpdate",
-                        "cp upload/DA-ESS-ContentUpdate*.tar.gz %s" % (output_file_path_from_slim_latest)]
-
-        else:
-            os.mkdir("slim_packaging")
-            os.mkdir("apps")
-            try:
-                SPLUNK_PACKAGING_TOOLKIT_URL = "https://download.splunk.com/misc/packaging-toolkit/splunk-packaging-toolkit-0.9.0.tar.gz"
-                SPLUNK_PACKAGING_TOOLKIT_FILENAME = 'splunk-packaging-toolkit-latest.tar.gz'
-                print("Downloading the Splunk Packaging Toolkit from %s..." %
-                      (SPLUNK_PACKAGING_TOOLKIT_URL), end='')
-                response = get(SPLUNK_PACKAGING_TOOLKIT_URL)
-                response.raise_for_status()
-                with open(SPLUNK_PACKAGING_TOOLKIT_FILENAME, 'wb') as slim_file:
-                    slim_file.write(response.content)
-            except Exception as e:
-                print("Error downloading the Splunk Packaging Toolkit: [%s].\n\tQuitting..." % 
-                      (str(e)), file=sys.stderr)
-                sys.exit(1)
-
-            commands = ["rm -rf slim_packaging/slim_latest",
-                        "mkdir slim_packaging/slim_latest",
-                        "cd slim_packaging",
-                        "tar -zxf splunk-packaging-toolkit-latest.tar.gz -C slim_latest --strip-components=1",
-                        "cd slim_latest",
-                        "virtualenv --python=/usr/bin/python2.7 --clear .venv",
-                        ". ./.venv/bin/activate",
-                        "python3 -m pip install --upgrade pip",
-                        "python2 -m pip install wheel",
-                        "python2 -m pip install semantic_version",
-                        "python2 -m pip install .",
-                        "cp -R ../../dist/escu DA-ESS-ContentUpdate",
-                        "slim package -o upload DA-ESS-ContentUpdate",
-                        "cp upload/DA-ESS-ContentUpdate*.tar.gz %s" % (output_file_path_from_slim_latest)]
-
-        ret = subprocess.run("; ".join(commands),
-                             shell=True, capture_output=True)
-        if ret.returncode != 0:
-            print("Command List:\n%s"%(commands))
-            print("Error generating new ESCU Package.\n\tQuitting and dumping error...\n[%s]" % (ret.stderr.decode('utf-8')),file=sys.stderr)
+    else:
+        os.mkdir("slim_packaging")
+        os.mkdir("apps")
+        try:
+            SPLUNK_PACKAGING_TOOLKIT_URL = "https://download.splunk.com/misc/packaging-toolkit/splunk-packaging-toolkit-0.9.0.tar.gz"
+            SPLUNK_PACKAGING_TOOLKIT_FILENAME = 'splunk-packaging-toolkit-latest.tar.gz'
+            print("Downloading the Splunk Packaging Toolkit from %s..." %
+                  (SPLUNK_PACKAGING_TOOLKIT_URL), end='')
+            response = get(SPLUNK_PACKAGING_TOOLKIT_URL)
+            response.raise_for_status()
+            with open(SPLUNK_PACKAGING_TOOLKIT_FILENAME, 'wb') as slim_file:
+                slim_file.write(response.content)
+        except Exception as e:
+            print("Error downloading the Splunk Packaging Toolkit: [%s].\n\tQuitting..." %
+                  (str(e)), file=sys.stderr)
             sys.exit(1)
-        os.chdir("../")
 
-        return output_file_path_from_root
+        commands = ["rm -rf slim_packaging/slim_latest",
+                    "mkdir slim_packaging/slim_latest",
+                    "cd slim_packaging",
+                    "tar -zxf splunk-packaging-toolkit-latest.tar.gz -C slim_latest --strip-components=1",
+                    "cd slim_latest",
+                    "virtualenv --python=/usr/bin/python2.7 --clear .venv",
+                    ". ./.venv/bin/activate",
+                    "python3 -m pip install --upgrade pip",
+                    "python2 -m pip install wheel",
+                    "python2 -m pip install semantic_version",
+                    "python2 -m pip install .",
+                    "cp -R ../../dist/escu DA-ESS-ContentUpdate",
+                    "slim package -o upload DA-ESS-ContentUpdate",
+                    "cp upload/DA-ESS-ContentUpdate*.tar.gz %s" % (output_file_path_from_slim_latest)]
+
+    ret = subprocess.run("; ".join(commands),
+                         shell=True, capture_output=True)
+    if ret.returncode != 0:
+        print("Command List:\n%s" % (commands))
+        print("Error generating new ESCU Package.\n\tQuitting and dumping error...\n[%s]" % (
+            ret.stderr.decode('utf-8')), file=sys.stderr)
+        sys.exit(1)
+    os.chdir("../")
+
+    return output_file_path_from_root
 
 
-def main(args:list[str]):
+def main(args: list[str]):
     requests.packages.urllib3.disable_warnings()
 
     start_datetime = datetime.now()
@@ -174,9 +178,8 @@ def main(args:list[str]):
     elif action != "run":
         print("Unsupported action: [%s]" % (action), file=sys.stderr)
         sys.exit(1)
-    
-    
-    
+
+
     '''
     parser = argparse.ArgumentParser(description="CI Detection Testing")
     parser.add_argument("-b", "--branch", type=str, required=True, help="security content branch")
@@ -246,7 +249,6 @@ def main(args:list[str]):
     github_service = ensure_security_content(
         settings['branch'], settings['pr_number'], settings['persist_security_content'])
 
-
     all_test_files = github_service.get_test_files(settings['mode'],
                                                    settings['folders'],
                                                    settings['types'],
@@ -254,78 +256,83 @@ def main(args:list[str]):
                                                    settings['detections_file'])
 
 
-    
-
 #    if len(all_test_files) == 0:
 #        print("No files were found to be tested.  While this could be due to an error, "\
 #              "this could be correct if there were no changes to detections.  We will "\
 #              "exit with success.\n\tQuitting...")
 #        sys.exit(0)
 
-    local_volume_path = os.path.join(os.getcwd(), "apps")
+    local_volume_absolute_path = os.path.abspath(
+        os.path.join(os.getcwd(), "apps"))
     try:
         os.mkdir("apps")
     except FileExistsError as e:
         # Directory already exists, do nothing
         pass
     except Exception as e:
-        print("Error creating the apps folder [%s]: [%s]\n\tQuitting..." 
-              % (local_volume_path, str(e)), file=sys.stderr)
+        print("Error creating the apps folder [%s]: [%s]\n\tQuitting..."
+              % (local_volume_absolute_path, str(e)), file=sys.stderr)
         sys.exit(1)
 
-    #Check to see if we want to install ESCU and whether it was preeviously generated and we should use that file
+    # Check to see if we want to install ESCU and whether it was preeviously generated and we should use that file
     if 'SPLUNK_ES_CONTENT_UPDATE' in settings['local_apps'] and settings['local_apps']['SPLUNK_ES_CONTENT_UPDATE']['local_path'] is not None:
-        #Using a pregenerated ESCU, copy it to apps (unless it)
+        # Using a pregenerated ESCU, copy it to apps (unless it)
 
-        file_path = settings['local_apps']['SPLUNK_ES_CONTENT_UPDATE']['local_path']
+        file_path = os.path.expanduser(settings['local_apps']['SPLUNK_ES_CONTENT_UPDATE']['local_path'])
         source_path = file_path
-        dest_path = os.path.join(local_volume_path, os.path.basename(file_path))
-        
+        dest_path = os.path.join(
+            local_volume_absolute_path, os.path.basename(file_path))
+
     elif 'SPLUNK_ES_CONTENT_UPDATE' not in settings['local_apps']:
-        print("%s was not found in %s.  We assume this is an error and shut down.\n\t"\
-              "Quitting..."%('SPLUNK_ES_CONTENT_UPDATE', "settings['local_apps']"), file=sys.stderr)
+        print("%s was not found in %s.  We assume this is an error and shut down.\n\t"
+              "Quitting..." % ('SPLUNK_ES_CONTENT_UPDATE', "settings['local_apps']"), file=sys.stderr)
         sys.exit(1)
     else:
-        #Need to generate that package
+        # Need to generate that package
         source_path = generate_escu_app(settings['persist_security_content'])
-        dest_path = os.path.join(local_volume_path, os.path.basename(source_path))
+        dest_path = os.path.join(
+            local_volume_absolute_path, os.path.basename(source_path))
+    COPY_ALL_LOCAL_FILES_NOT_JUST_ESCU
+    
 
-
-    #Now write out the package, whether it was previously generated or
-    #we just generated it
+    # Now write out the package, whether it was previously generated or
+    # we just generated it
     try:
         shutil.copy(source_path, dest_path)
+        #Update apps path for the ESCU package we just built
+        settings['local_apps']['SPLUNK_ES_CONTENT_UPDATE']['local_path'] = dest_path
+
     except shutil.SameFileError as e:
-        #Same file, not a real error.  The copy just doesn't happen
+        # Same file, not a real error.  The copy just doesn't happen
         pass
     except Exception as e:
-        print("Error copying ESCU Package [%s] to [%s]: [%s].\n\tQuitting..."%(source_path, dest_path, str(e)), file=sys.stderr)
+        print("Error copying ESCU Package [%s] to [%s]: [%s].\n\tQuitting..." % (
+            source_path, dest_path, str(e)), file=sys.stderr)
         sys.exit(1)
 
-
     print("Wrote ESCU package to volume folder.")
 
     if settings['mock']:
-        def finish_mock(settings:dict, detections:list[str], output_file_template:str="prior_config/config_tests_%d.json"):
+        def finish_mock(settings: dict, detections: list[str], output_file_template: str = "prior_config/config_tests_%d.json"):
             num_containers = settings['num_containers']
 
-            
             try:
                 os.mkdir("prior_config")
             except FileExistsError:
                 pass
             except Exception as e:
-                print("Some error occured when trying to make the configs folder: [%s]\n\tQuitting..."%(str(e)), file=sys.stderr)
+                print("Some error occured when trying to make the configs folder: [%s]\n\tQuitting..." % (
+                    str(e)), file=sys.stderr)
                 sys.exit(1)
 
             for output_file_index in range(0, num_containers):
                 fname = output_file_template % (output_file_index)
-                
-                #Get the n'th detection for this file
+
+                # Get the n'th detection for this file
                 detection_tests = detections[output_file_index::num_containers]
                 normalized_detection_names = []
-                #Normalize the test filename to the name of the detection instead.
-                #These are what we should write to the file
+                # Normalize the test filename to the name of the detection instead.
+                # These are what we should write to the file
                 for d in detection_tests:
                     filename = os.path.basename(d)
                     filename = filename.replace(".test.yml", ".yml")
@@ -334,50 +341,80 @@ def main(args:list[str]):
                     new_name = os.path.join(
                         "security_content", leading, filename)
                     normalized_detection_names.append(new_name)
-                
-                #Generate an appropriate config file for this test
+
+                # Generate an appropriate config file for this test
                 mock_settings = copy.deepcopy(settings)
-                #This may be able to support as many as 2 for GitHub Actions...
-                #we will have to determine in testing.
+                # This may be able to support as many as 2 for GitHub Actions...
+                # we will have to determine in testing.
                 mock_settings['num_containers'] = 1
-                
-                #Must be selected since we are passing in a list of detections
+
+                # Must be selected since we are passing in a list of detections
                 mock_settings['mode'] = 'selected'
-            
-                #Pass in the list of detections to run
+
+                # Pass in the list of detections to run
                 mock_settings['detections_list'] = normalized_detection_names
 
-                #We want to persist security content and run with the escu package that we created
+                # We want to persist security content and run with the escu package that we created
                 mock_settings['persist_security_content'] = True
 
-                mock_settings['local_apps'] = { 
+                mock_settings['local_apps'] = {
                     "SPLUNK_ES_CONTENT_UPDATE": {
                         "app_number": 3449,
                         "app_version": None,
                         'local_path': "prior_config/apps/DA-ESS-ContentUpdate-latest.tar.gz"}
-                    }
-                
-                mock_settings['mock'] = False
-                
-                #Make sure that it still validates after all of the changes
+                }
+
+                mock_settings['mock'] = False
+
+                # Make sure that it still validates after all of the changes
 
-                
-                
                 try:
-                    with open(fname,'w') as cfg:
-                        validated_settings,b = validate_and_write(mock_settings,cfg)
+                    with open(fname, 'w') as cfg:
+                        validated_settings, b = validate_and_write(
+                            mock_settings, cfg)
                         if validated_settings is None:
-                            print("There was an error validating the updated mock settings.\n\tQuitting...",file=sys.stderr)
+                            print(
+                                "There was an error validating the updated mock settings.\n\tQuitting...", file=sys.stderr)
                             sys.exit(1)
 
                 except Exception as e:
-                    print("Error writing config file %s: [%s]\n\tQuitting..."%(fname,str(e)),file=sys.stderr)
+                    print("Error writing config file %s: [%s]\n\tQuitting..." % (
+                        fname, str(e)), file=sys.stderr)
                     sys.exit(1)
-        
-            sys.exit(0)
-        finish_mock(settings,all_test_files)
 
-    print("More than a mock")        
+            sys.exit(0)
+        finish_mock(settings, all_test_files)
+
+    files_to_copy_to_container = OrderedDict()
+    files_to_copy_to_container["INDEXES"] = {
+        "local_file_path": index_file_local_path, "container_file_path": index_file_container_path}
+    files_to_copy_to_container["DATAMODELS"] = {
+        "local_file_path": datamodel_file_local_path, "container_file_path": datamodel_file_container_path}
+
+    mounts = [{"local_path": local_volume_absolute_path,
+               "container_path": "/tmp/apps", "type": "bind", "read_only": True}]
+    
+    cm = container_manager.ContainerManager(all_test_files,
+                                            FULL_DOCKER_HUB_CONTAINER_NAME,
+                                            settings['local_base_container_name'],
+                                            settings['num_containers'],
+                                            settings['local_apps'],
+                                            settings['splunkbase_apps'],
+                                            files_to_copy_to_container=files_to_copy_to_container,
+                                            web_port_start=8000,
+                                            management_port_start=8089,
+                                            mounts=mounts,
+                                            show_container_password=settings['show_splunk_app_password'],
+                                            container_password=settings['splunk_app_password'],
+                                            splunkbase_username=settings['splunkbase_username'],
+                                            splunkbase_password=settings['splunkbase_password'],
+                                            reuse_image=settings['reuse_image'],
+                                            interactive_failure=settings['interactive_failure'])
+
+
+    print(cm.containers[0].environment)
+    cm.run_test()
+
     '''
     if args.split_detections_then_stop:
         for output_file_index in range(0, num_containers):
@@ -569,5 +606,6 @@ def main(args:list[str]):
     #testing_service.test_detections(ssh_key_name, private_key, splunk_ip, splunk_password, test_files, uuid_test)
     '''
 
+
 if __name__ == "__main__":
     main(sys.argv[1:])
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/container_manager.py b/automated_detection_testing/ci/detection_testing_batch/modules/container_manager.py
index b703872057..a7136ea937 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/container_manager.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/container_manager.py
@@ -2,13 +2,16 @@ from collections import OrderedDict
 import docker
 import datetime
 import docker.types
+import os
 import random
-import splunk_container
+
+from modules import splunk_container
 import string
-import test_driver
+from modules import test_driver
 import threading
 import time
 import timeit
+
 from typing import Union
 
 WEB_PORT_STRING = "8000/tcp"
@@ -22,26 +25,35 @@ class ContainerManager:
         full_docker_hub_name: str,
         container_name_template: str,
         num_containers: int,
-        apps: OrderedDict,
+        local_apps: OrderedDict,
+        splunkbase_apps:OrderedDict,
         files_to_copy_to_container: OrderedDict = OrderedDict(),
         web_port_start: int = 8000,
         management_port_start: int = 8089,
-        mounts: list[dict[str, Union[str, bool]]] = [],
+        mounts: list[dict[str, str]] = [],
+        show_container_password:bool=True,
         container_password: Union[str, None] = None,
         splunkbase_username: Union[str, None] = None,
         splunkbase_password: Union[str, None] = None,
-        reuse_image:bool = True
+        reuse_image:bool = True,
+        interactive_failure:bool=False
+
     ):
         self.synchronization_object = test_driver.TestDriver(
             test_list, num_containers)
 
         self.mounts = self.create_mounts(mounts)
-        self.apps = apps
+        self.local_apps = local_apps
+        self.splunkbase_apps = splunkbase_apps
 
         if container_password is None:
             self.container_password = self.get_random_password()
         else:
             self.container_password = container_password
+        
+        if show_container_password:
+            print("Splunk App Password: [%s]"%(self.container_password))
+        
 
         self.containers = self.create_containers(
             full_docker_hub_name,
@@ -52,7 +64,8 @@ class ContainerManager:
             splunkbase_username,
             splunkbase_password,
             files_to_copy_to_container,
-            reuse_image
+            reuse_image,
+            interactive_failure
         )
         self.summary_thread = threading.Thread(target=self.queue_status_thread,args=())
 
@@ -66,8 +79,12 @@ class ContainerManager:
         self.baseline['TEST_FINISH_TIME'] = "TO BE UPDATED"
         self.baseline['TEST_DURATION'] = "TO BE UPDATED"
 
-        for key in self.apps:
-            self.baseline[key] = self.apps[key]
+        for key in self.local_apps:
+            self.baseline[key] = self.local_apps[key]
+
+        for key in self.splunkbase_apps:
+            self.baseline[key] = self.splunkbase_apps[key]
+
 
     def run_test(self):
         self.run_containers()
@@ -95,10 +112,11 @@ class ContainerManager:
 
     def run_containers(self) -> None:
         for container in self.containers:
-            container.thread.run()
+            container.thread.start()
+        
     
     def run_status_thread(self) -> None:
-        self.queue_status_thread.run()
+        self.summary_thread.start()
         
 
 
@@ -112,7 +130,8 @@ class ContainerManager:
         splunkbase_username: Union[str, None] = None,
         splunkbase_password: Union[str, None] = None,
         files_to_copy_to_container: OrderedDict = OrderedDict(),
-        reuse_image = True
+        reuse_image:bool = True,
+        interactive_failure:bool = True
     ) -> list[splunk_container.SplunkContainer]:
         #First make sure that the image exists and has been downloaded
         self.setup_image(reuse_image, full_docker_hub_name)
@@ -131,7 +150,8 @@ class ContainerManager:
                     self.synchronization_object,
                     full_docker_hub_name,
                     container_name,
-                    self.apps,
+                    self.local_apps,
+                    self.splunkbase_apps,
                     web_port_tuple,
                     management_port_tuple,
                     self.container_password,
@@ -139,22 +159,23 @@ class ContainerManager:
                     self.mounts,
                     splunkbase_username,
                     splunkbase_password,
+                    interactive_failure=interactive_failure
                 )
             )
 
         return new_containers
 
     def create_mounts(
-        self, mounts: list[dict[str, Union[str, bool]]]
+        self, mounts: list[dict[str, str]]
     ) -> list[docker.types.Mount]:
         new_mounts = []
         for mount in mounts:
             new_mounts.append(self.create_mount(mount))
         return new_mounts
 
-    def create_mount(self, mount: dict[str, Union[str, bool]]) -> docker.types.Mount:
+    def create_mount(self, mount: dict[str, str]) -> docker.types.Mount:
         return docker.types.Mount(
-            source=mount["local_path"],
+            source=os.path.abspath(mount["local_path"]),
             target=mount["container_path"],
             type=mount["type"],
             read_only=mount["read_only"],
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/new_arguments2.py b/automated_detection_testing/ci/detection_testing_batch/modules/new_arguments2.py
index ff1cf6c19f..978fcdafd8 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/new_arguments2.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/new_arguments2.py
@@ -14,7 +14,7 @@ def configure_action(args) -> tuple[str, dict]:
         settings, schema = validate_args.validate_file(args.input_config_file)
 
     if settings is None:
-        print("Failure while processing settings.\n\tQuitting...", file=sys.stderr)
+        print("Failure while processing settings\n\tQuitting...", file=sys.stderr)
         sys.exit(1)
 
     new_config = {}
@@ -36,6 +36,7 @@ def configure_action(args) -> tuple[str, dict]:
                 new_config[arg] = json.loads(choice.lower())
                 formatted_print = choice.lower()
             else:
+
                 if choice in ['true', 'false'] or (choice.isdigit() and schema['properties'][arg]['type'] != "integer"):
                     choice = '"' + choice + '"'
                 # replace all single quotes with doubles quotes to make valid json
@@ -46,6 +47,8 @@ def configure_action(args) -> tuple[str, dict]:
                 elif '"' in choice:
                     #Do nothing
                     pass
+                elif choice.isdigit():
+                    pass
                 else:
                     choice = '"' + choice + '"'
 
@@ -148,20 +151,43 @@ def parse(args)->tuple[str,dict]:
                             help="Password for login to the splunk app.  If you don't "
                             "provide one here or in the config, it will be generated "
                             "automatically for you.")
-    run_parser.add_argument("-show_pass", "--show_splunk_app_password", required=False, 
+
+    run_parser.add_argument("-show_pass", "--show_splunk_app_password", required=False,
                             action="store_true",
-                            help="The password to login to the Splunk Server.  ")
+                            help="The password to login to the Splunk Server.  If the config "\
+                            "file is set to true, it will override the default False for this.  True "\
+                            "will override the default value in the config file.")
     
     run_parser.add_argument("-m", "--mock", required=False, 
                             action="store_true",
-                            help="Split into multiple configs, don't actually run the tests.")
+                            help="Split into multiple configs, don't actually run the tests. If the config "\
+                            "file is set to true, it will override the default False for this.  True "\
+                            "will override the default value in the config file.")
 
     args = parser.parse_args()
     
+    
     # Run the appropriate parser
-
     try:
+        #If an argument is not passed on the command line, don't overwrite its config 
+        #file value with None - keep the config file value
+        keys = list(args.__dict__.keys())
+        for key in keys:
+            if args.__dict__[key] is None and key in ["show_pass", "mock"]:
+                del args.__dict__[key]
+
         action, settings = args.func(args)
+        
+        '''
+        default_settings,_ = validate_args.validate({})
+        if default_settings is None:
+            print("Somehow default settings were None.\n\tQuitting...",file=sys.stderr)
+            sys.exit(1)
+        #Fix up the show_app_password and mock arguments, as shown in the documentation
+        #for those args
+        settings['show_splunk_app_password'] |= default_settings['show_splunk_app_password']
+        settings['mock'] |= default_settings['mock']
+        '''
         return action, settings
     except Exception as e:
         print("Unknown Error - [%s]" % (str(e)))
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/splunk_container.py b/automated_detection_testing/ci/detection_testing_batch/modules/splunk_container.py
index d958ff1e81..1318eda110 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/splunk_container.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/splunk_container.py
@@ -9,9 +9,9 @@ import os.path
 import random
 import requests
 import shutil
-import splunk_sdk
-import testing_service
-import test_driver
+from modules import splunk_sdk
+from modules import testing_service
+from modules import test_driver 
 import time
 import timeit
 from typing import Union
@@ -27,7 +27,8 @@ class SplunkContainer:
         synchronization_object: test_driver.TestDriver,
         full_docker_hub_path,
         container_name: str,
-        apps: OrderedDict,
+        local_apps: OrderedDict,
+        splunkbase_apps: OrderedDict,
         web_port_tuple: tuple[str, int],
         management_port_tuple: tuple[str, int],
         container_password: str,
@@ -43,13 +44,15 @@ class SplunkContainer:
         self.client = docker.client.from_env()
         self.full_docker_hub_path = full_docker_hub_path
         self.container_password = container_password
-        self.apps = apps
+        self.local_apps = local_apps
+        self.splunkbase_apps = splunkbase_apps
+         
         self.files_to_copy_to_container = files_to_copy_to_container
         self.splunk_ip = splunk_ip
         self.container_name = container_name
         self.mounts = mounts
         self.environment = self.make_environment(
-            apps, container_password, splunkbase_username, splunkbase_password
+            local_apps, splunkbase_apps, container_password, splunkbase_username, splunkbase_password
         )
         self.ports = self.make_ports(web_port_tuple, management_port_tuple)
         self.web_port = web_port_tuple[1]
@@ -64,41 +67,40 @@ class SplunkContainer:
 
     def prepare_apps_path(
         self,
-        apps: OrderedDict,
+        local_apps: OrderedDict,
+        splunkbase_apps: OrderedDict,
         splunkbase_username: Union[str, None] = None,
         splunkbase_password: Union[str, None] = None,
     ) -> tuple[str, bool]:
         apps_to_install = []
         require_credentials = False
-        for app_name in self.apps:
-            app = self.apps[app_name]
-            if app["location"] == "splunkbase":
-                if splunkbase_username is None or splunkbase_password is None:
-                    raise Exception(
-                        "Error: Requested app from Splunkbase but Splunkbase username and/or password were not supplied."
-                    )
-                target = SPLUNKBASE_URL % (
-                    app["app_number"], app["app_version"])
-                apps_to_install.append(target)
-                require_credentials = True
-            elif app["location"] == "local":
-                apps_to_install.append(app["container_path"])
+        
+        for app_name, app_info in self.local_apps.items():
+            
+            app_file_name = os.path.basename(app_info['local_path'])
+            app_file_container_path = os.path.join("/tmp/apps", app_file_name)
+            apps_to_install.append(app_file_container_path)
+
+        for app_name, app_info in self.splunkbase_apps.items():
+            
+            if splunkbase_username is None or splunkbase_password is None:
+                raise Exception(
+                    "Error: Requested app from Splunkbase but Splunkbase username and/or password were not supplied."
+                )
+            target = SPLUNKBASE_URL % (
+                app_info["app_number"], app_info["app_version"])
+            apps_to_install.append(target)
+            require_credentials = True
+            #elif app["location"] == "local":
+            #    apps_to_install.append(app["container_path"])
         return ",".join(apps_to_install), require_credentials
 
-    def make_volume(
-        self,
-        local_path: str,
-        container_path: str,
-        type: str = "bind",
-        read_only: bool = True,
-    ) -> docker.types.Mount:
-        return docker.types.Mount(
-            source=local_path, target=container_path, type="bind", read_only=True
-        )
+    
 
     def make_environment(
         self,
-        apps: OrderedDict,
+        local_apps: OrderedDict,
+        splunkbase_apps: OrderedDict,
         container_password: str,
         splunkbase_username: Union[str, None] = None,
         splunkbase_password: Union[str, None] = None,
@@ -107,7 +109,7 @@ class SplunkContainer:
         env["SPLUNK_START_ARGS"] = SPLUNK_START_ARGS
         env["SPLUNK_PASSWORD"] = container_password
         splunk_apps_url, require_credentials = self.prepare_apps_path(
-            apps, splunkbase_username, splunkbase_password
+            local_apps, splunkbase_apps, splunkbase_username, splunkbase_password
         )
         if require_credentials:
             env["SPLUNKBASE_USERNAME"] = splunkbase_username
@@ -289,9 +291,9 @@ class SplunkContainer:
         self.container.start()
 
         # By default, first copy the index file then the datamodel file
-        for f in self.files_to_copy_to_container:
+        for file_description, file_dict in self.files_to_copy_to_container.items():
             self.extract_tar_file_to_container(
-                f["local_file_path"], f["container_file_path"]
+                file_dict["local_file_path"], file_dict["container_file_path"]
             )
 
         print("Finished copying files to [%s]" % (self.container_name))
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py b/automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py
index 3ae4c5b12a..92a89471b4 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py
@@ -69,6 +69,11 @@ setup_schema = {
                    "app_number": 3449,
                    "app_version": None,
                    'local_path': None
+               },
+               "BETA_SPLUNK_ADD_ON_FOR_SYSMON": {
+                   "app_number": 5709,
+                   "app_version": None,
+                   "local_path": "~/Downloads/Splunk_TA_microsoft_sysmon-1.0.2-B1.spl"
                }
             }
         },
@@ -178,6 +183,10 @@ setup_schema = {
                 "SPLUNK_COMMON_INFORMATION_MODEL": {
                     "app_number": 1621,
                     "app_version": "4.20.2"
+                },
+                "SPLUNK_ADD_ON_FOR_SYSMON": {
+                    "app_number": 5709,
+                    "app_version": "1.0.1"
                 }
             }
         },
@@ -240,8 +249,11 @@ def validate_file(file: io.TextIOWrapper) -> tuple[Union[dict, None], dict]:
 def check_dependencies(settings: dict) -> bool:
     # Check complex mode dependencies
     error_free = True
+    
+    
     if settings['mode'] == 'selected':
         # Make sure that exactly one of the following fields is populated
+    
         if settings['detections_file'] == None and settings['detections_list'] == None:
             print("Error - mode was 'selected' but no detections_list or detections_file were supplied.", file=sys.stderr)
             error_free = False
@@ -254,7 +266,7 @@ def check_dependencies(settings: dict) -> bool:
     elif settings['mode'] != 'selected' and settings['detections_list'] != None:
         print("Error - mode was not 'selected' but detections_list was supplied.", file=sys.stderr)
         error_free = False
-
+    
     # Returns true if there are not errors
     return error_free
 
@@ -281,23 +293,27 @@ def validate(configuration: dict) -> tuple[Union[dict, None], dict]:
     # v = jsonschema.Draft201909Validator(argument_schema)
 
     try:
+        
         validation_errors, validated_json = jsonschema_errorprinter.check_json(
             configuration, setup_schema)
-
-        no_complex_errors = check_dependencies(validated_json)
-        if len(validation_errors) == 0 and no_complex_errors:
-            return validated_json, setup_schema
-        elif no_complex_errors == False:
-            print("Failed due to error(s) listed above.", file=sys.stderr)
-            return None, setup_schema
+        
+        if len(validation_errors) == 0:
+            #check to make sure there were no complex errors
+            no_complex_errors = check_dependencies(validated_json)
+            if no_complex_errors:
+                return validated_json, setup_schema
+            else:
+                print("Validation failed due to error(s) listed above.", file=sys.stderr)
+                return None, setup_schema
         else:
             print("[%d] failures detected during validation of the configuration!" % (
                 len(validation_errors)), file=sys.stderr)
             for error in validation_errors:
                 print(error, end="\n\n", file=sys.stderr)
             return None, setup_schema
+            
     except Exception as e:
-        print(str(e), file=sys.stderr)
+        print("There was an error validation the configuration: [%s]"%(str(e)), file=sys.stderr)
         return None, setup_schema
 
     """

From 914f49d6f7d1a8f3373b546a06ca53c53e056fbd Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Tue, 23 Nov 2021 11:59:01 -0800
Subject: [PATCH 077/166] Delete intermediate testing artifacts/container
 config files in CI.  Better job packaging all local apps, not just escu, into
 the same folder for --mock.

---
 .../workflows/docker-detection-testing.yml    | 17 ++++-
 .../detection_testing_execution.py            | 70 ++++++++++++++-----
 .../modules/validate_args.py                  |  8 +--
 3 files changed, 71 insertions(+), 24 deletions(-)

diff --git a/.github/workflows/docker-detection-testing.yml b/.github/workflows/docker-detection-testing.yml
index ef09efb5a4..52b18d4c41 100644
--- a/.github/workflows/docker-detection-testing.yml
+++ b/.github/workflows/docker-detection-testing.yml
@@ -253,4 +253,19 @@ jobs:
           with:
             name: SummaryTestResults
             path: |
-              automated_detection_testing/ci/detection_testing_batch/summary_test_results.json
\ No newline at end of file
+              automated_detection_testing/ci/detection_testing_batch/summary_test_results.json
+        
+        - name: Clean up intermediate Files
+          uses: geekyeggo/delete-artifact@v1
+          with:
+              name: |
+                container_0_tests.txt.results
+                container_1_tests.txt.results
+                container_2_tests.txt.results
+                container_3_tests.txt.results
+                container_4_tests.txt.results
+                container_5_tests.txt.results
+                container_6_tests.txt.results
+                container_7_tests.txt.results
+                container_8_tests.txt.results
+                container_9_tests.txt.results
\ No newline at end of file
diff --git a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
index 72ce7f2084..2a25ce999f 100644
--- a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
+++ b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
@@ -3,6 +3,7 @@ import copy
 import csv
 import json
 import os
+from posixpath import basename
 import queue
 import random
 import secrets
@@ -42,6 +43,24 @@ datamodel_file_container_path = os.path.join(
 
 MAX_RECOMMENDED_CONTAINERS_BEFORE_WARNING = 2
 
+def copy_local_apps_to_directory(apps: dict[str,dict], target_directory)->None:
+    for key, item in apps.items():
+        source_path = os.path.abspath(os.path.expanduser(item['local_path']))
+        base_name = os.path.basename(source_path)
+        dest_path = os.path.join(target_directory, base_name)
+        try:
+            shutil.copy(source_path, dest_path)
+            item['local_path'] = dest_path
+            print("Copied %s to apps"%(base_name))     
+        except shutil.SameFileError as e:
+            # Same file, not a real error.  The copy just doesn't happen
+            print("err:%s"%(str(e)))
+            pass
+        except Exception as e:
+            print("Error copying ESCU Package [%s] to [%s]: [%s].\n\tQuitting..." % (
+                source_path, dest_path, str(e)), file=sys.stderr)
+            sys.exit(1)
+
 
 def ensure_security_content(branch: str, pr_number: Union[int, None], persist_security_content: bool) -> GithubService:
     if persist_security_content is True and os.path.exists("security_content"):
@@ -265,7 +284,9 @@ def main(args: list[str]):
     local_volume_absolute_path = os.path.abspath(
         os.path.join(os.getcwd(), "apps"))
     try:
-        os.mkdir("apps")
+        #remove the directory first
+        shutil.rmtree(local_volume_absolute_path,ignore_errors=True)
+        os.mkdir(local_volume_absolute_path)
     except FileExistsError as e:
         # Directory already exists, do nothing
         pass
@@ -277,11 +298,11 @@ def main(args: list[str]):
     # Check to see if we want to install ESCU and whether it was preeviously generated and we should use that file
     if 'SPLUNK_ES_CONTENT_UPDATE' in settings['local_apps'] and settings['local_apps']['SPLUNK_ES_CONTENT_UPDATE']['local_path'] is not None:
         # Using a pregenerated ESCU, copy it to apps (unless it)
-
-        file_path = os.path.expanduser(settings['local_apps']['SPLUNK_ES_CONTENT_UPDATE']['local_path'])
-        source_path = file_path
-        dest_path = os.path.join(
-            local_volume_absolute_path, os.path.basename(file_path))
+        pass
+        #file_path = os.path.expanduser(settings['local_apps']['SPLUNK_ES_CONTENT_UPDATE']['local_path'])
+        #source_path = file_path
+        #dest_path = os.path.join(
+        #    local_volume_absolute_path, os.path.basename(file_path))
 
     elif 'SPLUNK_ES_CONTENT_UPDATE' not in settings['local_apps']:
         print("%s was not found in %s.  We assume this is an error and shut down.\n\t"
@@ -290,11 +311,18 @@ def main(args: list[str]):
     else:
         # Need to generate that package
         source_path = generate_escu_app(settings['persist_security_content'])
-        dest_path = os.path.join(
-            local_volume_absolute_path, os.path.basename(source_path))
-    COPY_ALL_LOCAL_FILES_NOT_JUST_ESCU
+        settings['local_apps']['SPLUNK_ES_CONTENT_UPDATE']['local_path'] = source_path
+        #dest_path = os.path.join(
+        #    local_volume_absolute_path, os.path.basename(source_path))
+    
     
 
+    copy_local_apps_to_directory(settings['local_apps'], local_volume_absolute_path)
+    
+
+
+    
+    '''
     # Now write out the package, whether it was previously generated or
     # we just generated it
     try:
@@ -311,20 +339,31 @@ def main(args: list[str]):
         sys.exit(1)
 
     print("Wrote ESCU package to volume folder.")
+    '''
 
     if settings['mock']:
         def finish_mock(settings: dict, detections: list[str], output_file_template: str = "prior_config/config_tests_%d.json"):
             num_containers = settings['num_containers']
 
             try:
-                os.mkdir("prior_config")
-            except FileExistsError:
-                pass
+                #Remove the prior config directory if it exists.  If not, continue
+                shutil.rmtree("prior_config", ignore_errors=True)
+                
+                #We want to make the prior_config directory and the prior_config/apps directory
+                os.makedirs("prior_config/apps")
+            except FileExistsError as e:
+                print("Directory priorconfig/apps exists, but we just deleted it!\m\tQuitting...",file=sys.stderr)
+                sys.exit(1)
             except Exception as e:
                 print("Some error occured when trying to make the configs folder: [%s]\n\tQuitting..." % (
                     str(e)), file=sys.stderr)
                 sys.exit(1)
 
+            
+            #Copy the apps to the appropriate local.  This will also update
+            #the app paths in settings['local_apps']
+            copy_local_apps_to_directory(settings['local_apps'], "prior_config/apps")
+
             for output_file_index in range(0, num_containers):
                 fname = output_file_template % (output_file_index)
 
@@ -357,13 +396,6 @@ def main(args: list[str]):
                 # We want to persist security content and run with the escu package that we created
                 mock_settings['persist_security_content'] = True
 
-                mock_settings['local_apps'] = {
-                    "SPLUNK_ES_CONTENT_UPDATE": {
-                        "app_number": 3449,
-                        "app_version": None,
-                        'local_path': "prior_config/apps/DA-ESS-ContentUpdate-latest.tar.gz"}
-                }
-
                 mock_settings['mock'] = False
 
                 # Make sure that it still validates after all of the changes
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py b/automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py
index 92a89471b4..3682607218 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py
@@ -180,13 +180,13 @@ setup_schema = {
                     "app_number": 833,
                     "app_version": "8.3.1"
                 },
-                "SPLUNK_COMMON_INFORMATION_MODEL": {
-                    "app_number": 1621,
-                    "app_version": "4.20.2"
-                },
                 "SPLUNK_ADD_ON_FOR_SYSMON": {
                     "app_number": 5709,
                     "app_version": "1.0.1"
+                },
+                "SPLUNK_COMMON_INFORMATION_MODEL": {
+                    "app_number": 1621,
+                    "app_version": "4.20.2"
                 }
             }
         },

From dab746bb0c7ebd92b41af4628087382bd8f76cec Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Tue, 23 Nov 2021 14:30:54 -0800
Subject: [PATCH 078/166] Save out a json file showing the config, to include
 the command line arguments with credentials removed, whenever a test is
 actually run.  This makes it trivial to reproduce the test run on another
 machine or again on the same machine.

---
 .../detection_testing_execution.py            | 23 ++++++++++---------
 .../modules/container_manager.py              |  2 +-
 .../modules/github_service.py                 | 11 +++++----
 .../modules/new_arguments2.py                 | 21 ++++++++++++++++-
 .../modules/splunk_container.py               | 20 ++++++++--------
 5 files changed, 49 insertions(+), 28 deletions(-)

diff --git a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
index 2a25ce999f..b3ef52ac68 100644
--- a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
+++ b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
@@ -48,10 +48,10 @@ def copy_local_apps_to_directory(apps: dict[str,dict], target_directory)->None:
         source_path = os.path.abspath(os.path.expanduser(item['local_path']))
         base_name = os.path.basename(source_path)
         dest_path = os.path.join(target_directory, base_name)
+        
         try:
             shutil.copy(source_path, dest_path)
             item['local_path'] = dest_path
-            print("Copied %s to apps"%(base_name))     
         except shutil.SameFileError as e:
             # Same file, not a real error.  The copy just doesn't happen
             print("err:%s"%(str(e)))
@@ -153,6 +153,7 @@ def generate_escu_app(persist_security_content: bool = False) -> str:
             response.raise_for_status()
             with open(SPLUNK_PACKAGING_TOOLKIT_FILENAME, 'wb') as slim_file:
                 slim_file.write(response.content)
+            print("Done")
         except Exception as e:
             print("Error downloading the Splunk Packaging Toolkit: [%s].\n\tQuitting..." %
                   (str(e)), file=sys.stderr)
@@ -186,6 +187,11 @@ def generate_escu_app(persist_security_content: bool = False) -> str:
 
 
 def main(args: list[str]):
+    try:
+        docker.client.from_env()
+    except Exception as e:
+        print("Error, failed to get docker client.  Is Docker Running?\n\t%s"%(str(e)))
+
     requests.packages.urllib3.disable_warnings()
 
     start_datetime = datetime.now()
@@ -197,8 +203,8 @@ def main(args: list[str]):
     elif action != "run":
         print("Unsupported action: [%s]" % (action), file=sys.stderr)
         sys.exit(1)
-
-
+    
+    
     '''
     parser = argparse.ArgumentParser(description="CI Detection Testing")
     parser.add_argument("-b", "--branch", type=str, required=True, help="security content branch")
@@ -274,13 +280,8 @@ def main(args: list[str]):
                                                    settings['detections_list'],
                                                    settings['detections_file'])
 
-
-#    if len(all_test_files) == 0:
-#        print("No files were found to be tested.  While this could be due to an error, "\
-#              "this could be correct if there were no changes to detections.  We will "\
-#              "exit with success.\n\tQuitting...")
-#        sys.exit(0)
-
+    
+    
     local_volume_absolute_path = os.path.abspath(
         os.path.join(os.getcwd(), "apps"))
     try:
@@ -444,7 +445,7 @@ def main(args: list[str]):
                                             interactive_failure=settings['interactive_failure'])
 
 
-    print(cm.containers[0].environment)
+    
     cm.run_test()
 
     '''
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/container_manager.py b/automated_detection_testing/ci/detection_testing_batch/modules/container_manager.py
index a7136ea937..4e45a9010d 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/container_manager.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/container_manager.py
@@ -103,7 +103,7 @@ class ContainerManager:
         self.baseline['TEST_FINISH_TIME'] =  str(stop_time)
         
         duration = stop_time - self.start_time
-        self.baseline['TEST_DURATION'] = duration - datetime.timedelta(microseconds=duration.microseconds)
+        self.baseline['TEST_DURATION'] = str(duration - datetime.timedelta(microseconds=duration.microseconds))
 
         self.synchronization_object.finish(self.baseline)
 
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/github_service.py b/automated_detection_testing/ci/detection_testing_batch/modules/github_service.py
index 5450279021..f1e88a3dae 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/github_service.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/github_service.py
@@ -50,6 +50,7 @@ class GithubService:
                          summary_file: str = None) -> list[str]:
         pruned_tests = []
         csvlines = []
+        
         for detection in detections_to_prune:
             if os.path.basename(detection).startswith("ssa") and exclude_ssa:
                 continue
@@ -61,8 +62,8 @@ class GithubService:
                 test_filepath_without_security_content = str(
                     pathlib.Path(*pathlib.Path(test_filepath).parts[1:]))
                 # If no   types are provided, then we will get everything
-                if 'type' in description and (description['type'] in   types_to_test or len(  types_to_test) == 0):
-                    # print(description['type'])
+                if 'type' in description and (description['type'] in types_to_test or len(types_to_test) == 0):
+                    
                     if not os.path.exists(test_filepath):
                         print("Detection [%s] references [%s], but it does not exist" % (
                             detection, test_filepath))
@@ -124,7 +125,7 @@ class GithubService:
                       (detections_list, detections_file), file=sys.stderr)
                 sys.exit(1)
             elif detections_list is not None:
-                tests = self.get_selected_test_files(detections_list, folders,   types)
+                tests = self.get_selected_test_files(detections_list, types)
             elif detections_file is not None:
                 try:
                     with open(detections_file,'r') as f:
@@ -156,7 +157,7 @@ class GithubService:
                                     "Anomaly", "Hunting", "TTP"],
                                 previously_successful_tests: list[str] = []) -> list[str]:
 
-        return self.prune_detections(detection_file_list,   types_to_test, previously_successful_tests)
+        return self.prune_detections(detection_file_list, types_to_test, previously_successful_tests)
 
     def get_all_tests_and_detections(self,
                                      folders: list[str] = [
@@ -170,7 +171,7 @@ class GithubService:
                 os.path.join("security_content/detections", folder), "*.yml"))
 
         # Prune this down to only the subset of detections we can test
-        return self.prune_detections(detections,   types_to_test, previously_successful_tests)
+        return self.prune_detections(detections, types_to_test, previously_successful_tests)
 
     def get_all_files_in_folder(self, foldername: str, extension: str) -> list[str]:
         filenames = glob.glob(os.path.join(foldername, extension))
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/new_arguments2.py b/automated_detection_testing/ci/detection_testing_batch/modules/new_arguments2.py
index 978fcdafd8..2ec757b9a9 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/new_arguments2.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/new_arguments2.py
@@ -1,4 +1,6 @@
 import argparse
+import copy
+import datetime
 import json
 from typing import OrderedDict, Union
 import modules.validate_args as validate_args
@@ -86,6 +88,16 @@ def update_config_with_cli_arguments(args_dict:dict)->tuple[str, dict]:
         print("Failure while processing updated settings from command line.\n\tQuitting...", file=sys.stderr)
         sys.exit(1)
     
+    now = datetime.datetime.now()
+    configname = now.strftime('%Y-%m-%dT%H:%M:%S.%f%z') + '-test-run.json' 
+    with open(configname,'w') as test_config:
+        settings_with_creds_stripped = copy.deepcopy(settings)
+        #strip out credentials
+        settings_with_creds_stripped['splunkbase_password'] = None
+        settings_with_creds_stripped['splunkbase_username'] = None
+        settings_with_creds_stripped['container_password'] = None
+        validate_args.validate_and_write(settings_with_creds_stripped, test_config)
+
     return ("run", settings)
 
     
@@ -141,6 +153,13 @@ def parse(args)->tuple[str,dict]:
                             "if downloading packages from Splunkbase.  While this can "
                             "be stored in the config file, it is strongly recommended "
                             "to enter it at runtime.")
+
+    run_parser.add_argument('-b', '--branch', required=False, type=str,
+                            help="The branch to run the tests on.")
+    
+    run_parser.add_argument('-m', '--mode', required=False, type=str,
+                            help="The mode all, changes, or selected for the testing.")
+
     run_parser.add_argument('-pass', '--splunkbase_password', required=False, type=str,
                             help="Password for login to splunkbase.  This is required if "
                             "downloading packages from Splunkbase.  While this can be "
@@ -158,7 +177,7 @@ def parse(args)->tuple[str,dict]:
                             "file is set to true, it will override the default False for this.  True "\
                             "will override the default value in the config file.")
     
-    run_parser.add_argument("-m", "--mock", required=False, 
+    run_parser.add_argument("-mock", "--mock", required=False, 
                             action="store_true",
                             help="Split into multiple configs, don't actually run the tests. If the config "\
                             "file is set to true, it will override the default False for this.  True "\
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/splunk_container.py b/automated_detection_testing/ci/detection_testing_batch/modules/splunk_container.py
index 1318eda110..1c9f3ff0f0 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/splunk_container.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/splunk_container.py
@@ -59,7 +59,7 @@ class SplunkContainer:
         self.management_port = management_port_tuple[1]
         self.container = self.make_container()
 
-        self.thread = threading.Thread(target=self.run_container)
+        self.thread = threading.Thread(target=self.run_container, )
 
         self.container_start_time = 0
         self.test_start_time = 0
@@ -221,7 +221,7 @@ class SplunkContainer:
         if self.container_start_time == -1:
             total_time_string = "NOT STARTED"
         else:
-            total_time_rounded = datetime.timedelta(
+            total_time_rounded = datetime.timedelta( seconds =
                 round(current_time - self.container_start_time))
             total_time_string = str(total_time_rounded)
 
@@ -229,7 +229,7 @@ class SplunkContainer:
         if self.test_start_time == -1 or self.container_start_time == -1:
             setup_time_string = "NOT SET UP"
         else:
-            setup_secounds_rounded = datetime.timedelta(
+            setup_secounds_rounded = datetime.timedelta(seconds =
                 round(self.test_start_time - self.container_start_time))
             setup_time_string = str(setup_secounds_rounded)
 
@@ -237,7 +237,7 @@ class SplunkContainer:
         if self.test_start_time == -1 or self.num_tests_completed == 0:
             testing_time_string = "NO TESTS COMPLETED"
         else:
-            testing_seconds_rounded = datetime.timedelta(
+            testing_seconds_rounded = datetime.timedelta( seconds =
                 round(current_time - self.test_start_time))
 
             # Get the approximate time per test.  This is a clunky way to get rid of decimal
@@ -246,13 +246,13 @@ class SplunkContainer:
             timedelta_per_test_rounded = timedelta_per_test - \
                 datetime.timedelta(
                     microseconds=timedelta_per_test.microseconds)
+            
+            testing_time_string = "%s per test (%d tests)"%(timedelta_per_test_rounded, self.num_tests_completed)
 
-            testing_time_string = "%s per test (%d tests)"%(timedelta_per_test_rounded, str(testing_seconds_rounded))
-
-        summary_str = "[%s] Summary\n\t"\
-                      "Total Time          :"\
-                      "Container Start Time:"\
-                      "Test Execution Time :" %(total_time_string, setup_time_string, testing_time_string)
+        summary_str = "Summary\n\t"\
+                      "Total Time          : [%s]\n\t"\
+                      "Container Start Time: [%s]\n\t"\
+                      "Test Execution Time : [%s]" %(total_time_string, setup_time_string, testing_time_string)
         
         return summary_str
 

From fab525a6821a51f8d841dd37f9cce139055c422d Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Tue, 23 Nov 2021 14:48:37 -0800
Subject: [PATCH 079/166] Added support for more command line arguments for
 number containers and mode and branch.

---
 .../ci/detection_testing_batch/modules/new_arguments2.py     | 4 ++++
 .../ci/detection_testing_batch/modules/validate_args.py      | 5 -----
 2 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/new_arguments2.py b/automated_detection_testing/ci/detection_testing_batch/modules/new_arguments2.py
index 2ec757b9a9..fa8a8ead97 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/new_arguments2.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/new_arguments2.py
@@ -183,6 +183,10 @@ def parse(args)->tuple[str,dict]:
                             "file is set to true, it will override the default False for this.  True "\
                             "will override the default value in the config file.")
 
+    run_parser.add_argument("-n", "--num_containers", required=False, type=int,
+                            help="The number of Splunk containers to run or mock")
+    
+
     args = parser.parse_args()
     
     
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py b/automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py
index 3682607218..625fed0390 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py
@@ -69,11 +69,6 @@ setup_schema = {
                    "app_number": 3449,
                    "app_version": None,
                    'local_path': None
-               },
-               "BETA_SPLUNK_ADD_ON_FOR_SYSMON": {
-                   "app_number": 5709,
-                   "app_version": None,
-                   "local_path": "~/Downloads/Splunk_TA_microsoft_sysmon-1.0.2-B1.spl"
                }
             }
         },

From 1015abc30aec3060b8737988c423c14b6f4ffb8d Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Tue, 23 Nov 2021 15:10:28 -0800
Subject: [PATCH 080/166] Finally ready to try the full integration test after
 modifying the docker-detection-testing.yml with new names, paths, and
 arguments.  Fingers crossed...

---
 .../workflows/docker-detection-testing.yml    | 95 +++++++++----------
 .../modules/new_arguments2.py                 | 20 +++-
 2 files changed, 64 insertions(+), 51 deletions(-)

diff --git a/.github/workflows/docker-detection-testing.yml b/.github/workflows/docker-detection-testing.yml
index 52b18d4c41..1ccff1eaa7 100644
--- a/.github/workflows/docker-detection-testing.yml
+++ b/.github/workflows/docker-detection-testing.yml
@@ -76,43 +76,41 @@ jobs:
             run: |
               cd automated_detection_testing/ci/detection_testing_batch
               source .venv/bin/activate
-              #python3 detection_testing_execution.py -b ${{ steps.vars.outputs.branch }} -u 123456789 --splunkbase_username ${{ secrets.SPLUNKBASE_TESTING_USERNAME }} --splunkbase_password ${{ secrets.SPLUNKBASE_TESTING_KEY }} -m new -n2
-              python3 detection_testing_execution.py -b RegistryDetectionFixes -u 123456789 --splunkbase_username ${{ secrets.SPLUNKBASE_TESTING_USERNAME }} --splunkbase_password ${{ secrets.SPLUNKBASE_TESTING_KEY }} -m new -n10 -split
-              echo "DONE!"
+              
+              python3 detection_testing_execution.py run --branch RegistryDetectionFixes  --mode selected --num_containers 10 --mock 
           
           - name: Upload Test Results Files
             uses: actions/upload-artifact@v2
             with:
               name: testing-results-config
               path: |
-                automated_detection_testing/ci/detection_testing_batch/apps/DA-ESS-ContentUpdate-latest.tar.gz
-                automated_detection_testing/ci/detection_testing_batch/container_0_tests.txt
-                automated_detection_testing/ci/detection_testing_batch/container_1_tests.txt
-                automated_detection_testing/ci/detection_testing_batch/container_2_tests.txt
-                automated_detection_testing/ci/detection_testing_batch/container_3_tests.txt
-                automated_detection_testing/ci/detection_testing_batch/container_4_tests.txt
-                automated_detection_testing/ci/detection_testing_batch/container_5_tests.txt
-                automated_detection_testing/ci/detection_testing_batch/container_6_tests.txt
-                automated_detection_testing/ci/detection_testing_batch/container_7_tests.txt
-                automated_detection_testing/ci/detection_testing_batch/container_8_tests.txt
-                automated_detection_testing/ci/detection_testing_batch/container_9_tests.txt
-
+                automated_detection_testing/ci/detection_testing_batch/prior_config/apps/DA-ESS-ContentUpdate-latest.tar.gz
+                automated_detection_testing/ci/detection_testing_batch/prior_config/config_tests_0.json
+                automated_detection_testing/ci/detection_testing_batch/prior_config/config_tests_1.json
+                automated_detection_testing/ci/detection_testing_batch/prior_config/config_tests_2.json
+                automated_detection_testing/ci/detection_testing_batch/prior_config/config_tests_3.json
+                automated_detection_testing/ci/detection_testing_batch/prior_config/config_tests_4.json
+                automated_detection_testing/ci/detection_testing_batch/prior_config/config_tests_5.json
+                automated_detection_testing/ci/detection_testing_batch/prior_config/config_tests_6.json
+                automated_detection_testing/ci/detection_testing_batch/prior_config/config_tests_7.json
+                automated_detection_testing/ci/detection_testing_batch/prior_config/config_tests_8.json
+                automated_detection_testing/ci/detection_testing_batch/prior_config/config_tests_9.json
 
   docker-detection-testing-execution:
     runs-on: ubuntu-latest
     needs: [validate-tag-if-present, quit-for-dependabot, docker-detection-testing-setup]
     strategy:
       matrix:
-        test_filename: ["container_0_tests.txt", 
-                        "container_1_tests.txt", 
-                        "container_2_tests.txt", 
-                        "container_3_tests.txt", 
-                        "container_4_tests.txt", 
-                        "container_5_tests.txt", 
-                        "container_6_tests.txt", 
-                        "container_7_tests.txt", 
-                        "container_8_tests.txt", 
-                        "container_9_tests.txt"]
+        manifest_filename: ["config_tests_0.json", 
+                        "config_tests_1.json",
+                        "config_tests_2.json",
+                        "config_tests_3.json",
+                        "config_tests_4.json",
+                        "config_tests_5.json",
+                        "config_tests_6.json",
+                        "config_tests_7.json",
+                        "config_tests_8.json",
+                        "config_tests_9.json"]
     steps:  
         - name: Get branch and PR required for detection testing main.py
           id: vars
@@ -155,14 +153,14 @@ jobs:
           run: |
             cd automated_detection_testing/ci/detection_testing_batch
             source .venv/bin/activate
-            #python3 detection_testing_execution.py -b ${{ steps.vars.outputs.branch }} -u 123456789 --splunkbase_username ${{ secrets.SPLUNKBASE_TESTING_USERNAME }} --splunkbase_password ${{ secrets.SPLUNKBASE_TESTING_KEY }} -m new -n2
-            python3 detection_testing_execution.py -b RegistryDetectionFixes -u 123456789 --splunkbase_username ${{ secrets.SPLUNKBASE_TESTING_USERNAME }} --splunkbase_password ${{ secrets.SPLUNKBASE_TESTING_KEY }} -m selected -tff prior_config/${{ matrix.test_filename}} -n1 -e prior_config/apps/DA-ESS-ContentUpdate-latest.tar.gz
+            
+            python3 detection_testing_execution.py run -c prior_config/${{ matrix.manifest_filename}} --splunkbase_username ${{ secrets.SPLUNKBASE_TESTING_USERNAME }} --splunkbase_password ${{ secrets.SPLUNKBASE_TESTING_KEY }} 
                     
         
         - name: Upload Test Results Files
           uses: actions/upload-artifact@v2
           with:
-            name: ${{ matrix.test_filename}}.results
+            name: ${{ matrix.manifest_filename}}.results
             path: |
               automated_detection_testing/ci/detection_testing_batch/success.csv
               automated_detection_testing/ci/detection_testing_batch/error.csv
@@ -189,52 +187,52 @@ jobs:
         - name: Download artifacts
           uses: actions/download-artifact@v2
           with:
-            name: container_0_tests.txt.results
+            name: config_tests_0.json.results
             path: automated_detection_testing/ci/detection_testing_batch/results_0
         - name: Download artifacts
           uses: actions/download-artifact@v2
           with:
-            name: container_1_tests.txt.results
+            name: config_tests_1.json.results
             path: automated_detection_testing/ci/detection_testing_batch/results_1
         - name: Download artifacts
           uses: actions/download-artifact@v2
           with:
-            name: container_2_tests.txt.results
+            name: config_tests_2.json.results
             path: automated_detection_testing/ci/detection_testing_batch/results_2
         - name: Download artifacts
           uses: actions/download-artifact@v2
           with:
-            name: container_3_tests.txt.results
+            name: config_tests_3.json.txt.results
             path: automated_detection_testing/ci/detection_testing_batch/results_3
         - name: Download artifacts
           uses: actions/download-artifact@v2
           with:
-            name: container_4_tests.txt.results
+            name: config_tests_4.json.results
             path: automated_detection_testing/ci/detection_testing_batch/results_4
         - name: Download artifacts
           uses: actions/download-artifact@v2
           with:
-            name: container_5_tests.txt.results
+            name: config_tests_5.json.results
             path: automated_detection_testing/ci/detection_testing_batch/results_5
         - name: Download artifacts
           uses: actions/download-artifact@v2
           with:
-            name: container_6_tests.txt.results
+            name: config_tests_6.json.results
             path: automated_detection_testing/ci/detection_testing_batch/results_6
         - name: Download artifacts
           uses: actions/download-artifact@v2
           with:
-            name: container_7_tests.txt.results
+            name: config_tests_7.json.results
             path: automated_detection_testing/ci/detection_testing_batch/results_7
         - name: Download artifacts
           uses: actions/download-artifact@v2
           with:
-            name: container_8_tests.txt.results
+            name: config_tests_8.json.results
             path: automated_detection_testing/ci/detection_testing_batch/results_8
         - name: Download artifacts
           uses: actions/download-artifact@v2
           with:
-            name: container_9_tests.txt.results
+            name: config_tests_9.json.results
             path: automated_detection_testing/ci/detection_testing_batch/results_9
         
         - uses: actions/setup-python@v2
@@ -259,13 +257,14 @@ jobs:
           uses: geekyeggo/delete-artifact@v1
           with:
               name: |
-                container_0_tests.txt.results
-                container_1_tests.txt.results
-                container_2_tests.txt.results
-                container_3_tests.txt.results
-                container_4_tests.txt.results
-                container_5_tests.txt.results
-                container_6_tests.txt.results
-                container_7_tests.txt.results
-                container_8_tests.txt.results
-                container_9_tests.txt.results
\ No newline at end of file
+                config_tests_0.json.results
+                config_tests_1.json.results
+                config_tests_2.json.results
+                config_tests_3.json.results
+                config_tests_4.json.results
+                config_tests_5.json.results
+                config_tests_6.json.results
+                config_tests_7.json.results
+                config_tests_8.json.results
+                config_tests_9.json.results
+                
\ No newline at end of file
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/new_arguments2.py b/automated_detection_testing/ci/detection_testing_batch/modules/new_arguments2.py
index fa8a8ead97..409bc56ad5 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/new_arguments2.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/new_arguments2.py
@@ -3,7 +3,8 @@ import copy
 import datetime
 import json
 from typing import OrderedDict, Union
-import modules.validate_args as validate_args
+#import modules.validate_args as validate_args
+from modules import validate_args
 import sys
 
 DEFAULT_CONFIG_FILE = "test_config.json"
@@ -120,6 +121,14 @@ def parse(args)->tuple[str,dict]:
         sys.exit(1)
     '''
 
+    import os
+    #if there is no default config file, then generate one
+    if not os.path.exists(DEFAULT_CONFIG_FILE):
+        print("No default configuration file [%s] found.  Creating one..."%(DEFAULT_CONFIG_FILE))
+        with open(DEFAULT_CONFIG_FILE,'w') as cfg:
+            validate_args.validate_and_write({},cfg)
+
+
     parser = argparse.ArgumentParser(
         description="Use 'SOME_PROGRAM_NAME_STRING --help' to get help with the arguments")
     parser.set_defaults(func=lambda _: parser.print_help())
@@ -186,8 +195,13 @@ def parse(args)->tuple[str,dict]:
     run_parser.add_argument("-n", "--num_containers", required=False, type=int,
                             help="The number of Splunk containers to run or mock")
     
-
-    args = parser.parse_args()
+    try:
+        args = parser.parse_args()
+    except Exception as e:
+        print(str(e))
+        print(dir(e))
+        print("doot")
+        sys.exit(1)
     
     
     # Run the appropriate parser

From b0e7c33c4d3592d56ed82356d7c2ba63c4c1ce7d Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Tue, 23 Nov 2021 15:14:49 -0800
Subject: [PATCH 081/166] Changing mode for CI PR test from selected to
 changes.

---
 .github/workflows/docker-detection-testing.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/docker-detection-testing.yml b/.github/workflows/docker-detection-testing.yml
index 1ccff1eaa7..39093040a1 100644
--- a/.github/workflows/docker-detection-testing.yml
+++ b/.github/workflows/docker-detection-testing.yml
@@ -77,7 +77,7 @@ jobs:
               cd automated_detection_testing/ci/detection_testing_batch
               source .venv/bin/activate
               
-              python3 detection_testing_execution.py run --branch RegistryDetectionFixes  --mode selected --num_containers 10 --mock 
+              python3 detection_testing_execution.py run --branch RegistryDetectionFixes  --num_containers 10 --mock 
           
           - name: Upload Test Results Files
             uses: actions/upload-artifact@v2

From 451d2500db598f09d207537a07defe52f8824f92 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Tue, 23 Nov 2021 15:22:11 -0800
Subject: [PATCH 082/166] Fixed overwriting mode with null if nothing is passed
 on the command line.

---
 .../modules/new_arguments2.py                        | 12 ++++--------
 1 file changed, 4 insertions(+), 8 deletions(-)

diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/new_arguments2.py b/automated_detection_testing/ci/detection_testing_batch/modules/new_arguments2.py
index 409bc56ad5..b71dae52ac 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/new_arguments2.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/new_arguments2.py
@@ -195,13 +195,9 @@ def parse(args)->tuple[str,dict]:
     run_parser.add_argument("-n", "--num_containers", required=False, type=int,
                             help="The number of Splunk containers to run or mock")
     
-    try:
-        args = parser.parse_args()
-    except Exception as e:
-        print(str(e))
-        print(dir(e))
-        print("doot")
-        sys.exit(1)
+    
+    args = parser.parse_args()
+    
     
     
     # Run the appropriate parser
@@ -210,7 +206,7 @@ def parse(args)->tuple[str,dict]:
         #file value with None - keep the config file value
         keys = list(args.__dict__.keys())
         for key in keys:
-            if args.__dict__[key] is None and key in ["show_pass", "mock"]:
+            if args.__dict__[key] is None and key in ["show_pass", "mock", "mode"]:
                 del args.__dict__[key]
 
         action, settings = args.func(args)

From 1afbaa3a1d8531a98e8b777daa9d2a483626e656 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Tue, 23 Nov 2021 15:35:05 -0800
Subject: [PATCH 083/166] Fixed bad value substitution into the schema coming
 from command line arguments.

---
 .../ci/detection_testing_batch/modules/new_arguments2.py    | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/new_arguments2.py b/automated_detection_testing/ci/detection_testing_batch/modules/new_arguments2.py
index b71dae52ac..ab125b4b67 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/new_arguments2.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/new_arguments2.py
@@ -202,11 +202,13 @@ def parse(args)->tuple[str,dict]:
     
     # Run the appropriate parser
     try:
-        #If an argument is not passed on the command line, don't overwrite its config 
+        #If one of these arguments is not passed on the command line, don't overwrite its config 
         #file value with None - keep the config file value
         keys = list(args.__dict__.keys())
         for key in keys:
-            if args.__dict__[key] is None and key in ["show_pass", "mock", "mode"]:
+            if args.__dict__[key] is None and key in ["splunkbase_username","branch","mode",
+                                                      "splunkbase_password","splunk_app_password",
+                                                      "mock","num_containers"]:
                 del args.__dict__[key]
 
         action, settings = args.func(args)

From 1a39321be6fbfc59a974747fedcca5875d7190c4 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Tue, 23 Nov 2021 15:54:12 -0800
Subject: [PATCH 084/166] Small typo in artifact filename.

---
 .github/workflows/docker-detection-testing.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/docker-detection-testing.yml b/.github/workflows/docker-detection-testing.yml
index 39093040a1..ebc2e5a817 100644
--- a/.github/workflows/docker-detection-testing.yml
+++ b/.github/workflows/docker-detection-testing.yml
@@ -202,7 +202,7 @@ jobs:
         - name: Download artifacts
           uses: actions/download-artifact@v2
           with:
-            name: config_tests_3.json.txt.results
+            name: config_tests_3.json.results
             path: automated_detection_testing/ci/detection_testing_batch/results_3
         - name: Download artifacts
           uses: actions/download-artifact@v2

From ae7fb140856e7d9afea755af4cb09f12a04304b1 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Tue, 23 Nov 2021 16:38:22 -0800
Subject: [PATCH 085/166] Updated summarize_json.py to provide a total
 success/failure field and also return a nonzero return code if at least one
 test fails.

---
 .../detection_testing_batch/summarize_json.py | 54 +++++++++++++++++--
 1 file changed, 49 insertions(+), 5 deletions(-)

diff --git a/automated_detection_testing/ci/detection_testing_batch/summarize_json.py b/automated_detection_testing/ci/detection_testing_batch/summarize_json.py
index 102252495a..6cc49684bc 100644
--- a/automated_detection_testing/ci/detection_testing_batch/summarize_json.py
+++ b/automated_detection_testing/ci/detection_testing_batch/summarize_json.py
@@ -6,11 +6,47 @@ import json
 def outputResultsJSON(output_filename:str, data:list[dict], baseline:OrderedDict)->bool:
     success = True
     try:
+        test_count = len(data)
+        #Passed
+        pass_count = len([x for x in data if x['success'] == True])
+        
+        #A failure or an error
+        fail_only_count = len([x for x in data if x['success'] == False])
+        
+        #An error (every error is also a failure)
+        fail_and_error_count = len([x for x in data if x['error'] == True])
+        
+        #A failure without an error
+        fail_without_error_count = len([x for x in data if x['success'] == False and x['error'] == False])
+        
+        #This number should always be zero...
+        error_and_success_count = len([x for x in data if x['success'] == True and x['error'] == True])
+        if error_and_success_count > 0:
+            print("Error - a test was successful, but also included an error. This should be impossible.",file=sys.stderr)
+            success = False
+            
+        if test_count != (pass_count + fail_only_count):
+            print("Error - the total tests [%d] does not equal the pass[%d]/fails[%d]"%(test_count, pass_count,fail_only_count))
+            success=False
+
+        if fail_only_count > 0:
+            result = "FAIL for %d detections"%(fail_only_count)
+            success = False
+        else:
+            result = "PASS for all %d detections"%(pass_count)
+
+        summary={"TOTAL_TESTS": test_count, "TESTS_PASSED": pass_count, 
+                 "TOTAL_FAILURES": fail_only_count, "FAIL_ONLY": fail_without_error_count, 
+                 "FAIL_AND_ERROR":fail_and_error_count }
+
         with open(output_filename, "w") as jsonFile:
-            json.dump({'baseline': baseline, 'results':data}, jsonFile, indent="   ")
+            json.dump({'summary':summary, 'baseline': baseline, 'results':data}, jsonFile, indent="   ")
     except Exception as e:
-        print("There was an error generating [%s]: [%s]"%(output_filename, str(e)))
-        success = False
+        print("There was an error generating [%s]: [%s]"%(output_filename, str(e)),file=sys.stderr)
+        raise(e)
+        #success = False
+        #return success, False
+
     return success
 
 
@@ -38,8 +74,16 @@ try:
         else:
             all_data['results'] = data['results']
 
-    outputResultsJSON(args.output_filename, all_data['results'], all_data['baseline'])
-    print("Successfully summarized [%d] detections!"%(len(all_data['results'])))
+    test_pass = outputResultsJSON(args.output_filename, all_data['results'], all_data['baseline'])
+    print("Successfully summarized [%d] detections"%(len(all_data['results'])))
+    if not test_pass:
+        print("Result: FAIL")
+        sys.exit(1)
+    else:
+        print("Result: PASS!")
+        sys.exit(0)
+
+    
 except Exception as e:
     print("Error writing the summary file: [%s].\n\tQuitting..."%(str(e)))
     sys.exit(1)

From a302d952b41642b11d440753b138a2a8903ea7f4 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Tue, 23 Nov 2021 17:12:25 -0800
Subject: [PATCH 086/166] tiny ci change to test everything overnight

---
 .github/workflows/docker-detection-testing.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/docker-detection-testing.yml b/.github/workflows/docker-detection-testing.yml
index ebc2e5a817..0370c36f34 100644
--- a/.github/workflows/docker-detection-testing.yml
+++ b/.github/workflows/docker-detection-testing.yml
@@ -77,7 +77,7 @@ jobs:
               cd automated_detection_testing/ci/detection_testing_batch
               source .venv/bin/activate
               
-              python3 detection_testing_execution.py run --branch RegistryDetectionFixes  --num_containers 10 --mock 
+              python3 detection_testing_execution.py run --branch develop  --mode all --num_containers 10 --mock 
           
           - name: Upload Test Results Files
             uses: actions/upload-artifact@v2

From 6b292ba2c1a8d9432c40c34d62f5949e0b2ae0af Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Tue, 23 Nov 2021 17:13:44 -0800
Subject: [PATCH 087/166] upload artifacts even if summarize fails.

---
 .github/workflows/docker-detection-testing.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/docker-detection-testing.yml b/.github/workflows/docker-detection-testing.yml
index 0370c36f34..e704469387 100644
--- a/.github/workflows/docker-detection-testing.yml
+++ b/.github/workflows/docker-detection-testing.yml
@@ -248,6 +248,7 @@ jobs:
             
         - name: Upload Summary Test Results JSON
           uses: actions/upload-artifact@v2
+          if: always()
           with:
             name: SummaryTestResults
             path: |
@@ -255,6 +256,7 @@ jobs:
         
         - name: Clean up intermediate Files
           uses: geekyeggo/delete-artifact@v1
+          if: always()
           with:
               name: |
                 config_tests_0.json.results

From 3cd3ca6692df66bf68e39a5cc561730a0dc1cb3f Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Wed, 24 Nov 2021 09:54:27 -0800
Subject: [PATCH 088/166] Another CI change to list directory structure during
 summarize step.

---
 .github/workflows/docker-detection-testing.yml                 | 3 ++-
 .../ci/detection_testing_batch/summarize_json.py               | 2 +-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/docker-detection-testing.yml b/.github/workflows/docker-detection-testing.yml
index e704469387..6a8a5b929e 100644
--- a/.github/workflows/docker-detection-testing.yml
+++ b/.github/workflows/docker-detection-testing.yml
@@ -77,7 +77,7 @@ jobs:
               cd automated_detection_testing/ci/detection_testing_batch
               source .venv/bin/activate
               
-              python3 detection_testing_execution.py run --branch develop  --mode all --num_containers 10 --mock 
+              python3 detection_testing_execution.py run --branch develop  --mode changes --num_containers 10 --mock 
           
           - name: Upload Test Results Files
             uses: actions/upload-artifact@v2
@@ -243,6 +243,7 @@ jobs:
         - name: Merge Detections into single File
           run: |
             cd automated_detection_testing/ci/detection_testing_batch
+            ls -lah
             python summarize_json.py --files results_*/combined.json --output_filename summary_test_results.json
             
             
diff --git a/automated_detection_testing/ci/detection_testing_batch/summarize_json.py b/automated_detection_testing/ci/detection_testing_batch/summarize_json.py
index 6cc49684bc..38ddb61a16 100644
--- a/automated_detection_testing/ci/detection_testing_batch/summarize_json.py
+++ b/automated_detection_testing/ci/detection_testing_batch/summarize_json.py
@@ -40,7 +40,7 @@ def outputResultsJSON(output_filename:str, data:list[dict], baseline:OrderedDict
                  "FAIL_AND_ERROR":fail_and_error_count }
 
         with open(output_filename, "w") as jsonFile:
-            json.dump({'summary':summary, 'baseline': baseline, 'results':data}, jsonFile, indent="   ")
+            json.dump({'summary':summary, 'baseline': baseline, 'results':data}, jsonFile, indent="    ")
     except Exception as e:
         print("There was an error generating [%s]: [%s]"%(output_filename, str(e)),file=sys.stderr)
         raise(e)

From 5012b0f76647623b4f8c7eceda04004b8e313938 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Wed, 24 Nov 2021 10:10:27 -0800
Subject: [PATCH 089/166] Another run to check and see if the summarization
 works and confirm the directory structure of the artifacts.

---
 .github/workflows/docker-detection-testing.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/docker-detection-testing.yml b/.github/workflows/docker-detection-testing.yml
index 6a8a5b929e..fa47136773 100644
--- a/.github/workflows/docker-detection-testing.yml
+++ b/.github/workflows/docker-detection-testing.yml
@@ -243,8 +243,8 @@ jobs:
         - name: Merge Detections into single File
           run: |
             cd automated_detection_testing/ci/detection_testing_batch
-            ls -lah
-            python summarize_json.py --files results_*/combined.json --output_filename summary_test_results.json
+            ls -lahR results_*
+            python summarize_json.py --files results_*/*/combined.json --output_filename summary_test_results.json
             
             
         - name: Upload Summary Test Results JSON

From 54a34608feeabdbb873f8462cb912f2108a64ee1 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Wed, 24 Nov 2021 10:36:33 -0800
Subject: [PATCH 090/166] Fixed results paths once more for summary.

---
 .github/workflows/docker-detection-testing.yml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.github/workflows/docker-detection-testing.yml b/.github/workflows/docker-detection-testing.yml
index fa47136773..edcb1e588a 100644
--- a/.github/workflows/docker-detection-testing.yml
+++ b/.github/workflows/docker-detection-testing.yml
@@ -243,8 +243,7 @@ jobs:
         - name: Merge Detections into single File
           run: |
             cd automated_detection_testing/ci/detection_testing_batch
-            ls -lahR results_*
-            python summarize_json.py --files results_*/*/combined.json --output_filename summary_test_results.json
+            python summarize_json.py --files results_*/combined.json --output_filename summary_test_results.json
             
             
         - name: Upload Summary Test Results JSON

From ccc29005fa7ad13c3023df62bf7c8ff33c3596d7 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Wed, 24 Nov 2021 11:12:19 -0800
Subject: [PATCH 091/166] Still diagnosing path issues for summarize.

---
 .github/workflows/docker-detection-testing.yml | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/.github/workflows/docker-detection-testing.yml b/.github/workflows/docker-detection-testing.yml
index edcb1e588a..0c75b7d2ec 100644
--- a/.github/workflows/docker-detection-testing.yml
+++ b/.github/workflows/docker-detection-testing.yml
@@ -243,6 +243,13 @@ jobs:
         - name: Merge Detections into single File
           run: |
             cd automated_detection_testing/ci/detection_testing_batch
+            echo "res0"
+            ls 
+            echo "res1"
+            ls
+            echo "res2"
+            ls results_*
+
             python summarize_json.py --files results_*/combined.json --output_filename summary_test_results.json
             
             

From f32f1bed2a55ea6fb8a31baf06a6dbc16d08ef44 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Wed, 24 Nov 2021 11:45:52 -0800
Subject: [PATCH 092/166] More verbose debug messages in summarize_json.

---
 .github/workflows/docker-detection-testing.yml             | 7 -------
 .../ci/detection_testing_batch/summarize_json.py           | 3 +++
 2 files changed, 3 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/docker-detection-testing.yml b/.github/workflows/docker-detection-testing.yml
index 0c75b7d2ec..edcb1e588a 100644
--- a/.github/workflows/docker-detection-testing.yml
+++ b/.github/workflows/docker-detection-testing.yml
@@ -243,13 +243,6 @@ jobs:
         - name: Merge Detections into single File
           run: |
             cd automated_detection_testing/ci/detection_testing_batch
-            echo "res0"
-            ls 
-            echo "res1"
-            ls
-            echo "res2"
-            ls results_*
-
             python summarize_json.py --files results_*/combined.json --output_filename summary_test_results.json
             
             
diff --git a/automated_detection_testing/ci/detection_testing_batch/summarize_json.py b/automated_detection_testing/ci/detection_testing_batch/summarize_json.py
index 38ddb61a16..f648d444f5 100644
--- a/automated_detection_testing/ci/detection_testing_batch/summarize_json.py
+++ b/automated_detection_testing/ci/detection_testing_batch/summarize_json.py
@@ -58,11 +58,14 @@ args = parser.parse_args()
 
 all_data = OrderedDict()
 try:
+    print("We will summarize the files: %s"%(str(args.files)))
     for f in args.files:
         if not f.name.endswith('.json'):
             print("Error: passed in file must end in .json - you passed in [%s].\n\tQuitting..."%(f.name))
             sys.exit(1)
         data = json.loads(f.read())
+        print(f.name)
+        print(data)
         if 'baseline' in all_data:
             #everything has the same baseline, only need to do it once
             pass

From 4f6be1058119b5e60fe8623f7211856b68d2eeef Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Wed, 24 Nov 2021 12:00:02 -0800
Subject: [PATCH 093/166] Testing with the proper mode for a specific branch.

---
 .github/workflows/docker-detection-testing.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/docker-detection-testing.yml b/.github/workflows/docker-detection-testing.yml
index edcb1e588a..4cbcad53da 100644
--- a/.github/workflows/docker-detection-testing.yml
+++ b/.github/workflows/docker-detection-testing.yml
@@ -77,7 +77,7 @@ jobs:
               cd automated_detection_testing/ci/detection_testing_batch
               source .venv/bin/activate
               
-              python3 detection_testing_execution.py run --branch develop  --mode changes --num_containers 10 --mock 
+              python3 detection_testing_execution.py run --branch RegistryDetectionFixes  --mode changes --num_containers 10 --mock 
           
           - name: Upload Test Results Files
             uses: actions/upload-artifact@v2

From 88977a9fb8e37f7682f9395acd4015e207e93859 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Wed, 24 Nov 2021 13:58:32 -0800
Subject: [PATCH 094/166] Changes to template and summary to support passing a
 commit_hash parameter, not just a branch parameter, so that even old tests
 can be replicated.

---
 .../workflows/docker-detection-testing.yml    |   9 +
 .../detection_testing_execution.py            | 438 ++++--------------
 .../modules/container_manager.py              |   4 +
 .../modules/github_service.py                 |  27 +-
 .../modules/new_arguments2.py                 |  24 +-
 .../modules/validate_args.py                  |  80 +---
 .../detection_testing_batch/summarize_json.py |  19 +-
 7 files changed, 174 insertions(+), 427 deletions(-)

diff --git a/.github/workflows/docker-detection-testing.yml b/.github/workflows/docker-detection-testing.yml
index 4cbcad53da..9473fdd9bb 100644
--- a/.github/workflows/docker-detection-testing.yml
+++ b/.github/workflows/docker-detection-testing.yml
@@ -253,6 +253,15 @@ jobs:
             name: SummaryTestResults
             path: |
               automated_detection_testing/ci/detection_testing_batch/summary_test_results.json
+
+        - name: Upload Failures Manifest on Failure
+          uses: actions/upload-artifact@v2
+          if: failure()
+          with:
+            name: DetectionFailureManifest
+            path: |
+              automated_detection_testing/ci/detection_testing_batch/detection_failure_manifest.json
+            
         
         - name: Clean up intermediate Files
           uses: geekyeggo/delete-artifact@v1
diff --git a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
index b3ef52ac68..fc3529e564 100644
--- a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
+++ b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
@@ -23,6 +23,7 @@ import docker
 import requests.packages.urllib3
 from docker.client import DockerClient
 from requests import get
+from modules import new_arguments2
 from modules.validate_args import validate_and_write
 from modules import container_manager
 
@@ -62,13 +63,13 @@ def copy_local_apps_to_directory(apps: dict[str,dict], target_directory)->None:
             sys.exit(1)
 
 
-def ensure_security_content(branch: str, pr_number: Union[int, None], persist_security_content: bool) -> GithubService:
+def ensure_security_content(branch: str, commit_hash:str, pr_number: Union[int, None], persist_security_content: bool) -> GithubService:
     if persist_security_content is True and os.path.exists("security_content"):
         print("****** You chose --persist_security_content and the security_content directory exists. "
               "We will not check out the repo again. Please be aware, this could cause issues if you're "
               "out of date. ******")
         github_service = GithubService(
-            branch, existing_directory=persist_security_content)
+            branch, commit_hash, existing_directory=persist_security_content)
 
     else:
         if persist_security_content is True and not os.path.exists("security_content"):
@@ -86,9 +87,9 @@ def ensure_security_content(branch: str, pr_number: Union[int, None], persist_se
                 sys.exit(1)
 
         if pr_number:
-            github_service = GithubService(branch, pr_number)
+            github_service = GithubService(branch, commit_hash, pr_number)
         else:
-            github_service = GithubService(branch)
+            github_service = GithubService(branch, commit_hash)
 
     return github_service
 
@@ -185,6 +186,80 @@ def generate_escu_app(persist_security_content: bool = False) -> str:
 
     return output_file_path_from_root
 
+def finish_mock(settings: dict, detections: list[str], output_file_template: str = "prior_config/config_tests_%d.json"):
+    num_containers = settings['num_containers']
+
+    try:
+        #Remove the prior config directory if it exists.  If not, continue
+        shutil.rmtree("prior_config", ignore_errors=True)
+        
+        #We want to make the prior_config directory and the prior_config/apps directory
+        os.makedirs("prior_config/apps")
+    except FileExistsError as e:
+        print("Directory priorconfig/apps exists, but we just deleted it!\m\tQuitting...",file=sys.stderr)
+        sys.exit(1)
+    except Exception as e:
+        print("Some error occured when trying to make the configs folder: [%s]\n\tQuitting..." % (
+            str(e)), file=sys.stderr)
+        sys.exit(1)
+
+    
+    #Copy the apps to the appropriate local.  This will also update
+    #the app paths in settings['local_apps']
+    copy_local_apps_to_directory(settings['local_apps'], "prior_config/apps")
+
+    for output_file_index in range(0, num_containers):
+        fname = output_file_template % (output_file_index)
+
+        # Get the n'th detection for this file
+        detection_tests = detections[output_file_index::num_containers]
+        normalized_detection_names = []
+        # Normalize the test filename to the name of the detection instead.
+        # These are what we should write to the file
+        for d in detection_tests:
+            filename = os.path.basename(d)
+            filename = filename.replace(".test.yml", ".yml")
+            leading = os.path.split(d)[0]
+            leading = leading.replace("tests/", "detections/")
+            new_name = os.path.join(
+                "security_content", leading, filename)
+            normalized_detection_names.append(new_name)
+
+        # Generate an appropriate config file for this test
+        mock_settings = copy.deepcopy(settings)
+        # This may be able to support as many as 2 for GitHub Actions...
+        # we will have to determine in testing.
+        mock_settings['num_containers'] = 1
+
+        # Must be selected since we are passing in a list of detections
+        mock_settings['mode'] = 'selected'
+
+        # Pass in the list of detections to run
+        mock_settings['detections_list'] = normalized_detection_names
+
+        # We want to persist security content and run with the escu package that we created
+        mock_settings['persist_security_content'] = True
+
+        mock_settings['mock'] = False
+
+        # Make sure that it still validates after all of the changes
+
+        try:
+            with open(fname, 'w') as cfg:
+                validated_settings, b = validate_and_write(
+                    mock_settings, cfg)
+                if validated_settings is None:
+                    print(
+                        "There was an error validating the updated mock settings.\n\tQuitting...", file=sys.stderr)
+                    sys.exit(1)
+
+        except Exception as e:
+            print("Error writing config file %s: [%s]\n\tQuitting..." % (
+                fname, str(e)), file=sys.stderr)
+            sys.exit(1)
+
+    sys.exit(0)
+
 
 def main(args: list[str]):
     try:
@@ -204,64 +279,6 @@ def main(args: list[str]):
         print("Unsupported action: [%s]" % (action), file=sys.stderr)
         sys.exit(1)
     
-    
-    '''
-    parser = argparse.ArgumentParser(description="CI Detection Testing")
-    parser.add_argument("-b", "--branch", type=str, required=True, help="security content branch")
-    parser.add_argument("-u", "--uuid", type=str, required=True, help="uuid for detection test")
-    parser.add_argument("-pr", "--pr-number", type=int, required=False, help="Pull Request Number")
-
-    parser.add_argument("-n", "--num_containers", required=False, type=int, default=1, help="The number of splunk docker containers to start and run for testing")
-    
-    parser.add_argument("-cw", "--container_password", required=False, help="A password to use for the container.  If you don't choose one, a complex one will be generated for you.")
-    parser.add_argument("-show", "--show_password", required=False, default=False, action='store_true', help="Show the generated password to use to login to splunk.  For a CI/CD run, you probably don't want this.")
-    parser.add_argument("-i", "--interactive_failure", required=False, default=False, action='store_true', help="If a test fails, should we pause before removing data so that the search can be debugged?")
-
-    parser.add_argument("-ri", "--reuse_image", required=False, default=False, action='store_true', help="Should existing images be re-used, or should they be redownloaded?")
-    
-    #Allowing us to reuse containers is more trouble than it's worth (we may have rebuilt an app or, more likely, ESCU) and it is a pain to re-upload that
-    #instead of having the container restart and download/install itself.
-    #parser.add_argument("-rc", "--reuse_containers", required=False, default=False,  help="Should existing containers be re-used, or should they be rebuilt?")
-
-    parser.add_argument("-s", "--success_file", type=str, required=False, help="File that contains previously successful runs that we don't need to test")
-    parser.add_argument("-user", "--splunkbase_username", type=str, required=False, help="Splunkbase username for downloading Splunkbase apps")
-    parser.add_argument("-pw", "--splunkbase_password", type=str, required=False, help="Splunkbase password for downloading Splunkbase apps")
-    parser.add_argument("-m", "--mode", type=str, choices=DETECTION_MODES, required=False, help="Whether to test new detections, specific detections, or all detections", default="new")
-    parser.add_argument("-tfl","--test_files_list", type=str, required=False, help="The names of files that you want to test, separated by commas.")
-    parser.add_argument("-tff","--test_files_file", type=argparse.FileType('r'), required=False, help="A file containing a list of detections to run, one per line")
-    parser.add_argument("-e","--escu_package", type=argparse.FileType('rb'), required=False, help="The ESCU file to use - will not generate a new ESCU package")
-
-    parser.add_argument("-t", "--types", type=str, required=False, help="Detection types to test. Can be one of more of %s"%(str(DETECTION_TYPES)), default=','.join(DETECTION_TYPES))
-    parser.add_argument("-ct", "--container_tag", type=str, required=False, help="The tag of the Splunk Container to use.  Tags are located at https://hub.docker.com/r/splunk/splunk/tags",default=DEFAULT_CONTAINER_TAG)
-    parser.add_argument("-p", "--persist_security_content", required=False, default=False, action="store_true", help="Assumes security_content directory already exists.  Don't check it out and overwrite it again.  Saves "\
-                                                                                                                     "time and allows you to test a detection that you've updated.  Runs generate again in case you have "\
-                                                                                                                     "updated macros or anything else.  Especially useful for quick, local, iterative testing.")
-    
-
-    parser.add_argument("-split","--split_detections_then_stop", required=False, default=False, action='store_true', help="Should existing images be re-used, or should they be redownloaded?")
-
-    
-    
-    
-    args = parser.parse_args()
-    '''
-    '''
-    branch = args.branch
-    uuid_test = args.uuid
-    pr_number = args.pr_number
-    num_containers = args.num_containers
-    #reuse_containers = args.reuse_containers
-    reuse_image = args.reuse_image
-    success_file = args.success_file
-    splunkbase_username = args.splunkbase_username
-    splunkbase_password = args.splunkbase_password
-    full_docker_hub_container_name = "splunk/splunk:%s"%args.container_tag
-    #full_docker_hub_container_name = "customimage"
-    interactive_failure = args.interactive_failure
-    show_password = args.show_password
-    splunk_password = args.container_password
-    pregenerated_escu_package = args.escu_package
-    '''
 
     FULL_DOCKER_HUB_CONTAINER_NAME = "splunk/splunk:%s" % settings['container_tag']
 
@@ -272,7 +289,13 @@ def main(args: list[str]):
 
     # Check out security content if required
     github_service = ensure_security_content(
-        settings['branch'], settings['pr_number'], settings['persist_security_content'])
+        settings['branch'], settings['commit_hash'], settings['pr_number'], settings['persist_security_content'])
+    settings['commit_hash'] = github_service.commit_hash
+    
+    #Make a backup of this config containing the hash and stripped credentials.
+    #This makes the test perfectly reproducible.
+    validate_args.validate_and_write(settings,output_file=None,strip_credentials=True)
+    
 
     all_test_files = github_service.get_test_files(settings['mode'],
                                                    settings['folders'],
@@ -321,101 +344,7 @@ def main(args: list[str]):
     copy_local_apps_to_directory(settings['local_apps'], local_volume_absolute_path)
     
 
-
-    
-    '''
-    # Now write out the package, whether it was previously generated or
-    # we just generated it
-    try:
-        shutil.copy(source_path, dest_path)
-        #Update apps path for the ESCU package we just built
-        settings['local_apps']['SPLUNK_ES_CONTENT_UPDATE']['local_path'] = dest_path
-
-    except shutil.SameFileError as e:
-        # Same file, not a real error.  The copy just doesn't happen
-        pass
-    except Exception as e:
-        print("Error copying ESCU Package [%s] to [%s]: [%s].\n\tQuitting..." % (
-            source_path, dest_path, str(e)), file=sys.stderr)
-        sys.exit(1)
-
-    print("Wrote ESCU package to volume folder.")
-    '''
-
     if settings['mock']:
-        def finish_mock(settings: dict, detections: list[str], output_file_template: str = "prior_config/config_tests_%d.json"):
-            num_containers = settings['num_containers']
-
-            try:
-                #Remove the prior config directory if it exists.  If not, continue
-                shutil.rmtree("prior_config", ignore_errors=True)
-                
-                #We want to make the prior_config directory and the prior_config/apps directory
-                os.makedirs("prior_config/apps")
-            except FileExistsError as e:
-                print("Directory priorconfig/apps exists, but we just deleted it!\m\tQuitting...",file=sys.stderr)
-                sys.exit(1)
-            except Exception as e:
-                print("Some error occured when trying to make the configs folder: [%s]\n\tQuitting..." % (
-                    str(e)), file=sys.stderr)
-                sys.exit(1)
-
-            
-            #Copy the apps to the appropriate local.  This will also update
-            #the app paths in settings['local_apps']
-            copy_local_apps_to_directory(settings['local_apps'], "prior_config/apps")
-
-            for output_file_index in range(0, num_containers):
-                fname = output_file_template % (output_file_index)
-
-                # Get the n'th detection for this file
-                detection_tests = detections[output_file_index::num_containers]
-                normalized_detection_names = []
-                # Normalize the test filename to the name of the detection instead.
-                # These are what we should write to the file
-                for d in detection_tests:
-                    filename = os.path.basename(d)
-                    filename = filename.replace(".test.yml", ".yml")
-                    leading = os.path.split(d)[0]
-                    leading = leading.replace("tests/", "detections/")
-                    new_name = os.path.join(
-                        "security_content", leading, filename)
-                    normalized_detection_names.append(new_name)
-
-                # Generate an appropriate config file for this test
-                mock_settings = copy.deepcopy(settings)
-                # This may be able to support as many as 2 for GitHub Actions...
-                # we will have to determine in testing.
-                mock_settings['num_containers'] = 1
-
-                # Must be selected since we are passing in a list of detections
-                mock_settings['mode'] = 'selected'
-
-                # Pass in the list of detections to run
-                mock_settings['detections_list'] = normalized_detection_names
-
-                # We want to persist security content and run with the escu package that we created
-                mock_settings['persist_security_content'] = True
-
-                mock_settings['mock'] = False
-
-                # Make sure that it still validates after all of the changes
-
-                try:
-                    with open(fname, 'w') as cfg:
-                        validated_settings, b = validate_and_write(
-                            mock_settings, cfg)
-                        if validated_settings is None:
-                            print(
-                                "There was an error validating the updated mock settings.\n\tQuitting...", file=sys.stderr)
-                            sys.exit(1)
-
-                except Exception as e:
-                    print("Error writing config file %s: [%s]\n\tQuitting..." % (
-                        fname, str(e)), file=sys.stderr)
-                    sys.exit(1)
-
-            sys.exit(0)
         finish_mock(settings, all_test_files)
 
     files_to_copy_to_container = OrderedDict()
@@ -433,6 +362,8 @@ def main(args: list[str]):
                                             settings['num_containers'],
                                             settings['local_apps'],
                                             settings['splunkbase_apps'],
+                                            settings['branch'],
+                                            settings['commit_hash'],
                                             files_to_copy_to_container=files_to_copy_to_container,
                                             web_port_start=8000,
                                             management_port_start=8089,
@@ -444,201 +375,8 @@ def main(args: list[str]):
                                             reuse_image=settings['reuse_image'],
                                             interactive_failure=settings['interactive_failure'])
 
-
     
     cm.run_test()
 
-    '''
-    if args.split_detections_then_stop:
-        for output_file_index in range(0, num_containers):
-            fname = "container_%d_tests.txt" % (output_file_index)
-            print("Writing tests to [%s]..." % (fname), end='')
-            with open(fname, "w") as output_file:
-                detection_tests = test_files[output_file_index::num_containers]
-                normalized_detection_names = []
-                for d in detection_tests:
-                    filename = os.path.basename(d)
-                    filename = filename.replace(".test.yml", ".yml")
-                    leading = os.path.split(d)[0]
-                    leading = leading.replace("tests/", "detections/")
-                    new_name = os.path.join(
-                        "security_content", leading, filename)
-                    normalized_detection_names.append(new_name)
-                output_file.write('\n'.join(normalized_detection_names))
-            print("Done")
-        sys.exit(0)
-
-    # Create threads to manage all of the containers that we will start up
-    splunk_container_manager_threads = []
-
-    results_tracker = SynchronizedResultsTracker(test_files, num_containers)
-
-    #SPLUNK_ADD_ON_FOR_SYSMON_OLD = "https://splunkbase.splunk.com/app/1914/release/10.6.2/download"
-    #SPLUNK_ADD_ON_FOR_SYSMON_NEW = "https://splunkbase.splunk.com/app/5709/release/1.0.1/download"
-    #SYSMON_APP_FOR_SPLUNK = "https://splunkbase.splunk.com/app/3544/release/2.0.0/download"
-    #SPLUNK_ES_CONTENT_UPDATE = "https://splunkbase.splunk.com/app/3449/release/3.29.0/download"
-
-    # Just a hack until we get the new version of system deployed and available from splunkbase
-    CONTAINER_VOLUME_PATH = '/tmp/apps/'
-
-    GENERATED_SPLUNK_ES_CONTENT_UPDATE_CONTAINER_PATH = os.path.join(
-        CONTAINER_VOLUME_PATH, "DA-ESS-ContentUpdate-latest.tar.gz")
-
-    SPLUNKBASE_URL = "https://splunkbase.splunk.com/app/%d/release/%s/download"
-    # Order that we install the apps is actually important
-    APPS_DICT = OrderedDict()
-    APPS_DICT['SPLUNK_ADD_ON_FOR_MICROSOFT_WINDOWS'] = {
-        "app_number": 742, 'app_version': "8.2.0", 'location': 'splunkbase'}
-    APPS_DICT['SPLUNK_SECURITY_ESSENTIALS'] = {
-        "app_number": 3435, 'app_version': "3.3.4", 'location': 'splunkbase'}
-    APPS_DICT['GENERATED_SPLUNK_ES_CONTENT_UPDATE'] = {"app_number": 3449, 'app_version': "Generated at %s" % (
-        datetime.now()), 'location': GENERATED_SPLUNK_ES_CONTENT_UPDATE_CONTAINER_PATH}
-
-    try:
-        BETA_SPLUNK_ADD_ON_FOR_SYSMON_PATH = os.path.expanduser(
-            "~/Downloads/Splunk_TA_microsoft_sysmon-1.0.2-B1.spl")
-        shutil.copyfile(BETA_SPLUNK_ADD_ON_FOR_SYSMON_PATH, os.path.join(
-            local_volume_path, "Splunk_TA_microsoft_sysmon-1.0.2-B1.spl"))
-
-        BETA_SPLUNK_ADD_ON_FOR_SYSMON_CONTAINER_PATH = "/tmp/apps/Splunk_TA_microsoft_sysmon-1.0.2-B1.spl"
-        APPS_DICT['BETA_SPLUNK_ADD_ON_FOR_SYSMON'] = {"app_number": 5709, 'app_version': "Generated at %s" % (
-            datetime.now()), 'location': "local", "container_path": BETA_SPLUNK_ADD_ON_FOR_SYSMON_CONTAINER_PATH}
-    except Exception as e:
-        print("Failed to grab beta sysmon at ~/Downloads/Splunk_TA_microsoft_sysmon-1.0.2-B1.spl. Using the one from Splunkbase")
-        APPS_DICT['SPLUNK_ADD_ON_FOR_SYSMON'] = {
-            "app_number": 5709, 'app_version': "1.0.1", 'location': 'splunkbase'}
-
-    if True:
-        APPS_DICT['SPLUNK_ADD_ON_FOR_AMAZON_WEB_SERVICES'] = {
-            "app_number": 1876, 'app_version': "5.2.0", 'location': 'splunkbase'}
-        APPS_DICT['SPLUNK_ADD_ON_FOR_MICROSOFT_OFFICE_365'] = {
-            "app_number": 4055, 'app_version': "2.2.0", 'location': 'splunkbase'}
-        APPS_DICT['SPLUNK_ADD_ON_FOR_AMAZON_KINESIS_FIREHOSE'] = {
-            "app_number": 3719, 'app_version': "1.3.2", 'location': 'splunkbase'}
-        APPS_DICT['SPLUNK_ANALYTIC_STORY_EXECUTION_APP'] = {
-            "app_number": 4971, 'app_version': "2.0.3", 'location': 'splunkbase'}
-        APPS_DICT['PYTHON_FOR_SCIENTIC_COMPUTING_LINUX_64_BIT'] = {
-            "app_number": 2882, 'app_version': "2.0.2", 'location': 'splunkbase'}
-        APPS_DICT['SPLUNK_MACHINE_LEARNING_TOOLKIT'] = {
-            "app_number": 2890, 'app_version': "5.2.2", 'location': 'splunkbase'}
-        APPS_DICT['SPLUNK_APP_FOR_STREAM'] = {
-            "app_number": 1809, 'app_version': "8.0.1", 'location': 'splunkbase'}
-        APPS_DICT['SPLUNK_ADD_ON_FOR_STREAM_WIRE_DATA'] = {
-            "app_number": 5234, 'app_version': "8.0.1", 'location': 'splunkbase'}
-        APPS_DICT['SPLUNK_ADD_ON_FOR_STREAM_FORWARDERS'] = {
-            "app_number": 5238, 'app_version': "8.0.1", 'location': 'splunkbase'}
-        APPS_DICT['SPLUNK_ADD_ON_FOR_ZEEK_AKA_BRO'] = {
-            "app_number": 1617, 'app_version': "4.0.0", 'location': 'splunkbase'}
-        APPS_DICT['SPLUNK_ADD_ON_FOR_UNIX_AND_LINUX'] = {
-            "app_number": 833, 'app_version': "8.3.1", 'location': 'splunkbase'}
-
-    # CIM is last here for a reason!  Because we copy a file to a directory that does not exist until CIM
-    # has been installed, we use it to prevent the testing from beginning until the copy has succeeded.
-    # KEEP THIS APP LAST!
-    APPS_DICT['SPLUNK_COMMON_INFORMATION_MODEL'] = {
-        "app_number": 1621, 'app_version': "4.20.2", 'location': 'splunkbase'}
-
-    SPLUNK_APPS = []
-    for key, value in APPS_DICT.items():
-        if value['location'] == 'splunkbase':
-            # The app is on Splunkbase
-            target = SPLUNKBASE_URL % (
-                value['app_number'], value['app_version'])
-            SPLUNK_APPS.append(target)
-        else:
-            # The app is a file we generated locally
-            SPLUNK_APPS.append(value['location'])
-
-    for container_index in range(num_containers):
-        container_name = LOCAL_BASE_CONTAINER_NAME % container_index
-
-        web_port = BASE_CONTAINER_WEB_PORT + container_index
-        management_port = BASE_CONTAINER_MANAGEMENT_PORT + container_index
-
-        environment = {"SPLUNK_START_ARGS": "--accept-license",
-                       "SPLUNK_PASSWORD": splunk_password,
-                       "SPLUNK_APPS_URL": ','.join(SPLUNK_APPS),
-                       "SPLUNKBASE_USERNAME": splunkbase_username,
-                       "SPLUNKBASE_PASSWORD": splunkbase_password
-                       }
-        ports = {"8000/tcp": web_port,
-                 "8089/tcp": management_port
-                 }
-
-        mounts = [docker.types.Mount(
-            target=CONTAINER_VOLUME_PATH, source=local_volume_path, type='bind', read_only=True)]
-
-        print("Creating CONTAINER: [%s]" % (container_name))
-        base_container = client.containers.create(
-            full_docker_hub_container_name, ports=ports, environment=environment, name=container_name, mounts=mounts, detach=True)
-        print("Created CONTAINER : [%s]" % (container_name))
-
-        t = threading.Thread(target=splunk_container_manager,
-                             args=(results_tracker,
-                                   container_name,
-                                   "127.0.0.1",
-                                   splunk_password,
-                                   web_port,
-                                   management_port,
-                                   uuid_test,
-                                   interactive_failure
-                                   ))
-
-        splunk_container_manager_threads.append(t)
-
-    # add the queue status thread - there can be some error in one of the test threads, so this
-    # thread doesn't need to complete for the program to finish execution
-    status_thread = threading.Thread(target=queue_status_thread,
-                                     args=(results_tracker,),
-                                     daemon=True)
-    # Start this thread immediately
-    status_thread.start()
-
-    print("Start the testing threads")
-    for t in splunk_container_manager_threads:
-        t.start()
-        # we need to start containers slowly.  Would be great it we could do all the setup and
-        # app install once, but it looks like the container is unlikely to support that.
-        # We don't really want to fundamentally change this container, either, and will
-        # keep it as close to production as possible
-        time.sleep(5)
-
-    # Wait for all of the testing threads to complete
-    for t in splunk_container_manager_threads:
-        t.join()  # blocks on waiting to join
-        print("Testing thread completed execution")
-
-    print("All testing threads have completed execution")
-    # read all the results out from the output queue
-    strtime = str(int(time.time()))
-
-    print("Wait for the status thread to finish executing...")
-    status_thread.join()
-    print("Status thread finished executing.")
-
-    # Remove the attack data and
-    # generate all of the output information
-    stop_time = timer()
-    stop_datetime = datetime.now()
-    baseline = OrderedDict()
-    baseline['SPLUNK_VERSION'] = full_docker_hub_container_name
-    baseline['SPLUNK_APPS'] = APPS_DICT
-    baseline['TEST_START_TIME'] = str(start_datetime)
-    baseline['TEST_FINISH_TIME'] = str(stop_datetime)
-
-    results_tracker.finish(baseline)
-
-    # now we are done!
-
-    print("Total Execution Time: [%s]" % (
-        timedelta(seconds=stop_time - start_time, microseconds=0)))
-
-    # detection testing service has already been prepared, no need to do it here!
-    #testing_service.prepare_detection_testing(ssh_key_name, private_key, splunk_ip, splunk_password)
-
-    #testing_service.test_detections(ssh_key_name, private_key, splunk_ip, splunk_password, test_files, uuid_test)
-    '''
-
-
 if __name__ == "__main__":
     main(sys.argv[1:])
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/container_manager.py b/automated_detection_testing/ci/detection_testing_batch/modules/container_manager.py
index 4e45a9010d..a09dc2ae5a 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/container_manager.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/container_manager.py
@@ -27,6 +27,8 @@ class ContainerManager:
         num_containers: int,
         local_apps: OrderedDict,
         splunkbase_apps:OrderedDict,
+        branch:str,
+        commit_hash:str,
         files_to_copy_to_container: OrderedDict = OrderedDict(),
         web_port_start: int = 8000,
         management_port_start: int = 8089,
@@ -74,6 +76,8 @@ class ContainerManager:
         #Get a datetime and add it as the first entry in the baseline
         self.start_time = datetime.datetime.now()
         self.baseline['SPLUNK_VERSION'] = full_docker_hub_name
+        self.baseline["branch"] = branch
+        self.baseline["commit_hash"] = commit_hash
         #Added here first to preserve ordering for OrderedDict
         self.baseline['TEST_START_TIME'] = "TO BE UPDATED"
         self.baseline['TEST_FINISH_TIME'] = "TO BE UPDATED"
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/github_service.py b/automated_detection_testing/ci/detection_testing_batch/modules/github_service.py
index f1e88a3dae..f14957a642 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/github_service.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/github_service.py
@@ -22,7 +22,7 @@ SECURITY_CONTENT_URL = "https://github.com/splunk/security_content"
 
 class GithubService:
 
-    def __init__(self, security_content_branch: str, PR_number: int = None, existing_directory: bool = False):
+    def __init__(self, security_content_branch: str, commit_hash:str, PR_number: int = None, existing_directory: bool = False):
 
         self.security_content_branch = security_content_branch
         if existing_directory:
@@ -31,11 +31,27 @@ class GithubService:
         self.security_content_repo_obj = self.clone_project(
             SECURITY_CONTENT_URL, f"security_content", f"develop")
 
-        if PR_number:
+        if commit_hash is not None and PR_number is not None:
+            print("Error - both the PR number [%d] and the commit hash [%s] were provided.  "\
+                  "Only 0 or 1 can be passed.\n\tQuitting..."%(PR_number, commit_hash))
+            sys.exit()
+
+        elif PR_number:
             subprocess.call(["git", "-C", "security_content/", "fetch", "origin",
                             "refs/pull/%d/head:%s" % (PR_number, security_content_branch)])
 
-        self.security_content_repo_obj.git.checkout(security_content_branch)
+        # No checking to see if the hash is to a commit inside of the branch - the user
+        # has to do that by hand
+        if commit_hash is not None:
+            print("Checking out commit hash: [%s]"%(commit_hash))
+            self.security_content_repo_obj.git.checkout(commit_hash)
+        else:
+            print("Checking out branch: [%s]..."%(security_content_branch),end='')
+            self.security_content_repo_obj.git.checkout(security_content_branch)
+            commit_hash = self.security_content_repo_obj.head.object.hexsha
+            print("commit_hash %s"%(commit_hash))
+        
+        self.commit_hash = commit_hash
 
     def clone_project(self, url, project, branch):
         LOGGER.info(f"Clone Security Content Project")
@@ -184,7 +200,10 @@ class GithubService:
         changed_test_files = []
         changed_detection_files = []
         if branch1 != 'develop':
-            differ = g.diff('--name-status', branch2 + '...' + branch1)
+            if self.commit_hash is None:
+                differ = g.diff('--name-status', branch2 + '...' + branch1)
+            else:
+                differ = g.diff('--name-status', branch2 + '...' + self.commit_hash)
             changed_files = differ.splitlines()
 
             for file_path in changed_files:
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/new_arguments2.py b/automated_detection_testing/ci/detection_testing_batch/modules/new_arguments2.py
index ab125b4b67..e0548ea868 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/new_arguments2.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/new_arguments2.py
@@ -89,16 +89,7 @@ def update_config_with_cli_arguments(args_dict:dict)->tuple[str, dict]:
         print("Failure while processing updated settings from command line.\n\tQuitting...", file=sys.stderr)
         sys.exit(1)
     
-    now = datetime.datetime.now()
-    configname = now.strftime('%Y-%m-%dT%H:%M:%S.%f%z') + '-test-run.json' 
-    with open(configname,'w') as test_config:
-        settings_with_creds_stripped = copy.deepcopy(settings)
-        #strip out credentials
-        settings_with_creds_stripped['splunkbase_password'] = None
-        settings_with_creds_stripped['splunkbase_username'] = None
-        settings_with_creds_stripped['container_password'] = None
-        validate_args.validate_and_write(settings_with_creds_stripped, test_config)
-
+    
     return ("run", settings)
 
     
@@ -166,6 +157,13 @@ def parse(args)->tuple[str,dict]:
     run_parser.add_argument('-b', '--branch', required=False, type=str,
                             help="The branch to run the tests on.")
     
+    run_parser.add_argument('-hash', '--commit_hash', required=False, type=str,
+                            help="The hash to run the tests on.")
+    
+    run_parser.add_argument('-pr', '--pr_number', required=False, type=int,
+                            help="The Pull request to run the tests on.")
+
+
     run_parser.add_argument('-m', '--mode', required=False, type=str,
                             help="The mode all, changes, or selected for the testing.")
 
@@ -206,9 +204,9 @@ def parse(args)->tuple[str,dict]:
         #file value with None - keep the config file value
         keys = list(args.__dict__.keys())
         for key in keys:
-            if args.__dict__[key] is None and key in ["splunkbase_username","branch","mode",
-                                                      "splunkbase_password","splunk_app_password",
-                                                      "mock","num_containers"]:
+            if args.__dict__[key] is None and key in ["splunkbase_username","branch", "commit_hash", 
+                                                      "pr_number", "mode", "splunkbase_password",
+                                                      "splunk_app_password", "mock","num_containers"]:
                 del args.__dict__[key]
 
         action, settings = args.func(args)
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py b/automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py
index 625fed0390..3f45737f96 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py
@@ -18,6 +18,10 @@ setup_schema = {
             "type": "string",
             "default": "develop"
         },
+        "commit_hash": {
+            "type": ["string","null"],
+            "default": None
+        },
 
         "container_tag": {
             "type": "string",
@@ -266,7 +270,21 @@ def check_dependencies(settings: dict) -> bool:
     return error_free
 
 
-def validate_and_write(configuration: dict, output_file: io.TextIOWrapper) -> tuple[Union[dict, None], dict]:
+def validate_and_write(configuration: dict, output_file: Union[io.TextIOWrapper,None]=None, strip_credentials:bool=False) -> tuple[Union[dict, None], dict]:
+    closeFile = False
+    if output_file is None:
+        import datetime
+        now = datetime.datetime.now()
+        configname = now.strftime('%Y-%m-%dT%H:%M:%S.%f%z') + '-test-run.json' 
+        output_file = open(configname, "w")
+        closeFile = True
+
+    if strip_credentials:    
+        configuration = copy.deepcopy(configuration)
+        configuration['splunkbase_password'] = None
+        configuration['splunkbase_username'] = None
+        configuration['container_password'] = None
+
     validated_json, setup_schema = validate(configuration)
     if validated_json == None:
         print("Error in the new settings! No output file written")
@@ -279,7 +297,9 @@ def validate_and_write(configuration: dict, output_file: io.TextIOWrapper) -> tu
         except Exception as e:
             print("Error writing settings to %s: [%s]" % (
                 output_file.name, str(e)), file=sys.stderr)
-            return None, setup_schema
+            sys.exit(1)
+    if closeFile is True:
+        output_file.close()
 
     return validated_json, setup_schema
 
@@ -311,58 +331,4 @@ def validate(configuration: dict) -> tuple[Union[dict, None], dict]:
         print("There was an error validation the configuration: [%s]"%(str(e)), file=sys.stderr)
         return None, setup_schema
 
-    """
-    try:
-        v.validate({"action":"doot", "branch":"15"}  )
-    except jsonschema.exceptions.ValidationError as e:
-        print("Error validating the json", file=sys.stderr)
-        print(e)
-        return False
-
-    except jsonschema.exceptions.SchemaError as e:
-        print("Error validating the schema", file=sys.stderr)
-    """
-
-
-if __name__ == "__main__":
-    c = v({"action": "test", "branch": "wow"})
-    print(c)
-    if c is False:
-        print("whoops")
-    else:
-        print(c.keys())
-"""
-def load(json_settings: io.TextIOWrapper) -> dict:
-    default_settings = json.load(json_settings)
-    return default_settings
-
-
-def load_and_validate(json_settings: io.TextIOWrapper) -> dict:
-    settings = load(json_settings)
-    validate(settings)
-    return settings
-
-
-def validate(args: dict) -> bool:
-    validate_common_arguments()
-
-    validate_mode()
-
-    return True
-
-
-def validate_mode(args: dict) -> bool:
-    return True
-
-
-def validate_mode_selected(args: dict) -> bool:
-    return True
-
-
-def validate_mode_changes(args: dict) -> -bool:
-    return True
-
-
-def validate_mode_all(args: dict) -> bool:
-    return True
-"""
+ 
\ No newline at end of file
diff --git a/automated_detection_testing/ci/detection_testing_batch/summarize_json.py b/automated_detection_testing/ci/detection_testing_batch/summarize_json.py
index f648d444f5..30d8ef281b 100644
--- a/automated_detection_testing/ci/detection_testing_batch/summarize_json.py
+++ b/automated_detection_testing/ci/detection_testing_batch/summarize_json.py
@@ -3,6 +3,9 @@ from collections import OrderedDict
 import argparse
 import sys
 import json
+from modules import validate_args
+import os.path
+
 def outputResultsJSON(output_filename:str, data:list[dict], baseline:OrderedDict)->bool:
     success = True
     try:
@@ -41,6 +44,18 @@ def outputResultsJSON(output_filename:str, data:list[dict], baseline:OrderedDict
 
         with open(output_filename, "w") as jsonFile:
             json.dump({'summary':summary, 'baseline': baseline, 'results':data}, jsonFile, indent="    ")
+        
+        
+        #Generate a failure that the user can download to reproduce and test ONLY the failures locally.
+        #This makes it easy to test and debug ONLY those that failed.  No need to test the ones
+        #that succeeded!
+        
+        fail_list = [os.path.join("security_content/detections",x['detection_file'] ) for x in data if x['success'] == False]
+        failures_test_override = {"detection_list": fail_list, "interactive_failure":True, 
+                                  "num_containers":1, "branch": baseline["branch"], "commit_hash":baseline["commit_hash"], 
+                                  "mode":"selected"}
+        with open("detection_failure_manifest.json","w") as failures:
+            validate_args.validate_and_write(failures_test_override, failures)
     except Exception as e:
         print("There was an error generating [%s]: [%s]"%(output_filename, str(e)),file=sys.stderr)
         raise(e)
@@ -58,14 +73,12 @@ args = parser.parse_args()
 
 all_data = OrderedDict()
 try:
-    print("We will summarize the files: %s"%(str(args.files)))
+    print("We will summarize the files: %s"%(str([f.name for f in args.files])))
     for f in args.files:
         if not f.name.endswith('.json'):
             print("Error: passed in file must end in .json - you passed in [%s].\n\tQuitting..."%(f.name))
             sys.exit(1)
         data = json.loads(f.read())
-        print(f.name)
-        print(data)
         if 'baseline' in all_data:
             #everything has the same baseline, only need to do it once
             pass

From 9517171204260cd8fa148b51b9889a795e02edaa Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Wed, 24 Nov 2021 14:20:46 -0800
Subject: [PATCH 095/166] Small change to the output format of the test-config
 file that is generated at runtime to accurately describe the current test.
 Also added install of requirements.txt to final summary step.

---
 .github/workflows/docker-detection-testing.yml            | 8 ++++++++
 .../ci/detection_testing_batch/modules/validate_args.py   | 2 +-
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/docker-detection-testing.yml b/.github/workflows/docker-detection-testing.yml
index 9473fdd9bb..505820b37f 100644
--- a/.github/workflows/docker-detection-testing.yml
+++ b/.github/workflows/docker-detection-testing.yml
@@ -239,6 +239,14 @@ jobs:
           with:
             python-version: '3.9' #Available versions here - https://github.com/actions/python-versions/releases  easy to change/make a matrix/use pypy
             architecture: 'x64' # optional x64 or x86. Defaults to x64 if not specified    
+        
+        - name: Install Python Dependencies
+          run: |
+            cd automated_detection_testing/ci/detection_testing_batch
+            python3 -m venv .venv
+            source .venv/bin/activate
+            python3 -m pip install wheel
+            python3 -m pip install -r requirements.txt
 
         - name: Merge Detections into single File
           run: |
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py b/automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py
index 3f45737f96..66a45f68d0 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py
@@ -275,7 +275,7 @@ def validate_and_write(configuration: dict, output_file: Union[io.TextIOWrapper,
     if output_file is None:
         import datetime
         now = datetime.datetime.now()
-        configname = now.strftime('%Y-%m-%dT%H:%M:%S.%f%z') + '-test-run.json' 
+        configname = now.strftime('%Y-%m-%dT%H:%M:%S%z') + '-test-run.json' 
         output_file = open(configname, "w")
         closeFile = True
 

From aa985d5077224a39da75297aad35e05034966f24 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Wed, 24 Nov 2021 14:42:18 -0800
Subject: [PATCH 096/166] Made some small changes to printed strings. Fixed
 summarize_json missing python library error by sourcing virtualenv set up in
 previous step.

---
 .github/workflows/docker-detection-testing.yml            | 8 +-------
 .../detection_testing_execution.py                        | 2 +-
 .../detection_testing_batch/modules/splunk_container.py   | 2 +-
 3 files changed, 3 insertions(+), 9 deletions(-)

diff --git a/.github/workflows/docker-detection-testing.yml b/.github/workflows/docker-detection-testing.yml
index 505820b37f..ad818fd7f8 100644
--- a/.github/workflows/docker-detection-testing.yml
+++ b/.github/workflows/docker-detection-testing.yml
@@ -52,13 +52,6 @@ jobs:
             run: |
               sudo apt update -qq
               
-
-              #python2.7 needed for slim, for now
-              sudo apt install python2
-              sudo apt install virtualenv 
-              curl https://bootstrap.pypa.io/pip/2.7/get-pip.py --output get-pip.py
-              sudo python2.7 get-pip.py
-
           - uses: actions/setup-python@v2
             with:
               python-version: '3.9' #Available versions here - https://github.com/actions/python-versions/releases  easy to change/make a matrix/use pypy
@@ -251,6 +244,7 @@ jobs:
         - name: Merge Detections into single File
           run: |
             cd automated_detection_testing/ci/detection_testing_batch
+            source .venv/bin/activate
             python summarize_json.py --files results_*/combined.json --output_filename summary_test_results.json
             
             
diff --git a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
index fc3529e564..f2c0d74603 100644
--- a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
+++ b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
@@ -74,7 +74,7 @@ def ensure_security_content(branch: str, commit_hash:str, pr_number: Union[int,
     else:
         if persist_security_content is True and not os.path.exists("security_content"):
             print("Error - you chose --persist_security_content but the security_content directory does not exist!"
-                  "  We will check it out for you.\n\tQuitting...")
+                  "  We will check it out for you.")
 
         elif os.path.exists("security_content/"):
             print("Deleting the security_content directory")
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/splunk_container.py b/automated_detection_testing/ci/detection_testing_batch/modules/splunk_container.py
index 1c9f3ff0f0..7a2d245d1e 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/splunk_container.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/splunk_container.py
@@ -247,7 +247,7 @@ class SplunkContainer:
                 datetime.timedelta(
                     microseconds=timedelta_per_test.microseconds)
             
-            testing_time_string = "%s per test (%d tests)"%(timedelta_per_test_rounded, self.num_tests_completed)
+            testing_time_string = "%s (%d tests @ %s per test)"%(testing_seconds_rounded, self.num_tests_completed, timedelta_per_test_rounded)
 
         summary_str = "Summary\n\t"\
                       "Total Time          : [%s]\n\t"\

From ae3e8f1b757c8dd221d046717bb1d7f319a698b9 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Wed, 24 Nov 2021 14:56:38 -0800
Subject: [PATCH 097/166] slim failed to install, causing escu app install to
 fail.

---
 .github/workflows/docker-detection-testing.yml | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/docker-detection-testing.yml b/.github/workflows/docker-detection-testing.yml
index ad818fd7f8..d23da9fb1c 100644
--- a/.github/workflows/docker-detection-testing.yml
+++ b/.github/workflows/docker-detection-testing.yml
@@ -48,10 +48,15 @@ jobs:
           - name: Checkout Repo
             uses: actions/checkout@v2
           
-          - name: Install Docker 
+          - name: Install requirements for installing slim during execution 
             run: |
               sudo apt update -qq
-              
+              #python2.7 needed for slim, for now
+              sudo apt install python2
+              sudo apt install virtualenv 
+              curl https://bootstrap.pypa.io/pip/2.7/get-pip.py --output get-pip.py
+              sudo python2.7 get-pip.py
+
           - uses: actions/setup-python@v2
             with:
               python-version: '3.9' #Available versions here - https://github.com/actions/python-versions/releases  easy to change/make a matrix/use pypy

From eb01127d035c9a0ac98a72793ab88d92a4fc1ce6 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Wed, 24 Nov 2021 15:25:24 -0800
Subject: [PATCH 098/166] Fixed key error in summarize.json.  Had called
 detection_list detections_list, so it was failing jsonschema validation.

---
 .../ci/detection_testing_batch/summarize_json.py    | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/automated_detection_testing/ci/detection_testing_batch/summarize_json.py b/automated_detection_testing/ci/detection_testing_batch/summarize_json.py
index 30d8ef281b..2f90b367bb 100644
--- a/automated_detection_testing/ci/detection_testing_batch/summarize_json.py
+++ b/automated_detection_testing/ci/detection_testing_batch/summarize_json.py
@@ -50,12 +50,15 @@ def outputResultsJSON(output_filename:str, data:list[dict], baseline:OrderedDict
         #This makes it easy to test and debug ONLY those that failed.  No need to test the ones
         #that succeeded!
         
+        
         fail_list = [os.path.join("security_content/detections",x['detection_file'] ) for x in data if x['success'] == False]
-        failures_test_override = {"detection_list": fail_list, "interactive_failure":True, 
-                                  "num_containers":1, "branch": baseline["branch"], "commit_hash":baseline["commit_hash"], 
-                                  "mode":"selected"}
-        with open("detection_failure_manifest.json","w") as failures:
-            validate_args.validate_and_write(failures_test_override, failures)
+
+        if len(fail_list) > 0:
+            failures_test_override = {"detections_list": fail_list, "interactive_failure":True, 
+                                    "num_containers":1, "branch": baseline["branch"], "commit_hash":baseline["commit_hash"], 
+                                    "mode":"selected"}
+            with open("detection_failure_manifest.json","w") as failures:
+                validate_args.validate_and_write(failures_test_override, failures)
     except Exception as e:
         print("There was an error generating [%s]: [%s]"%(output_filename, str(e)),file=sys.stderr)
         raise(e)

From abd568f7d7e4330508cec11e97d99c5c55fc4e28 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Wed, 24 Nov 2021 15:57:13 -0800
Subject: [PATCH 099/166] Small update to summarize_json.py to set
 show_splunk_app_password to True when generatingf the
 detection_failure_manifest.json file.  This way, you can run that file
 directly as the configuration without making any other changes or passing
 command line values and get right into debugging everything

---
 .../ci/detection_testing_batch/summarize_json.py                | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/automated_detection_testing/ci/detection_testing_batch/summarize_json.py b/automated_detection_testing/ci/detection_testing_batch/summarize_json.py
index 2f90b367bb..a1084a4a44 100644
--- a/automated_detection_testing/ci/detection_testing_batch/summarize_json.py
+++ b/automated_detection_testing/ci/detection_testing_batch/summarize_json.py
@@ -56,7 +56,7 @@ def outputResultsJSON(output_filename:str, data:list[dict], baseline:OrderedDict
         if len(fail_list) > 0:
             failures_test_override = {"detections_list": fail_list, "interactive_failure":True, 
                                     "num_containers":1, "branch": baseline["branch"], "commit_hash":baseline["commit_hash"], 
-                                    "mode":"selected"}
+                                    "mode":"selected", "show_splunk_app_password": True}
             with open("detection_failure_manifest.json","w") as failures:
                 validate_args.validate_and_write(failures_test_override, failures)
     except Exception as e:

From 827cc97cdc3e6e27dc47d4273f7e55400af48ed1 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Wed, 24 Nov 2021 17:06:07 -0800
Subject: [PATCH 100/166] Better support for arg validation and command line
 arguments.  Print out the password for the container, if the flag exists to
 print it out, to enable interactive testing.  A few other format updates.

---
 .../detection_testing_execution.py            |  57 +++--
 .../modules/container_manager.py              |   9 +-
 .../modules/github_service.py                 |  72 ++++---
 .../modules/new_arguments2.py                 | 204 +++++-------------
 .../modules/validate_args.py                  |   2 +-
 5 files changed, 133 insertions(+), 211 deletions(-)

diff --git a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
index f2c0d74603..59149f8bdc 100644
--- a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
+++ b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
@@ -44,18 +44,19 @@ datamodel_file_container_path = os.path.join(
 
 MAX_RECOMMENDED_CONTAINERS_BEFORE_WARNING = 2
 
-def copy_local_apps_to_directory(apps: dict[str,dict], target_directory)->None:
+
+def copy_local_apps_to_directory(apps: dict[str, dict], target_directory) -> None:
     for key, item in apps.items():
         source_path = os.path.abspath(os.path.expanduser(item['local_path']))
         base_name = os.path.basename(source_path)
         dest_path = os.path.join(target_directory, base_name)
-        
+
         try:
             shutil.copy(source_path, dest_path)
             item['local_path'] = dest_path
         except shutil.SameFileError as e:
             # Same file, not a real error.  The copy just doesn't happen
-            print("err:%s"%(str(e)))
+            print("err:%s" % (str(e)))
             pass
         except Exception as e:
             print("Error copying ESCU Package [%s] to [%s]: [%s].\n\tQuitting..." % (
@@ -63,7 +64,7 @@ def copy_local_apps_to_directory(apps: dict[str,dict], target_directory)->None:
             sys.exit(1)
 
 
-def ensure_security_content(branch: str, commit_hash:str, pr_number: Union[int, None], persist_security_content: bool) -> GithubService:
+def ensure_security_content(branch: str, commit_hash: str, pr_number: Union[int, None], persist_security_content: bool) -> GithubService:
     if persist_security_content is True and os.path.exists("security_content"):
         print("****** You chose --persist_security_content and the security_content directory exists. "
               "We will not check out the repo again. Please be aware, this could cause issues if you're "
@@ -186,26 +187,26 @@ def generate_escu_app(persist_security_content: bool = False) -> str:
 
     return output_file_path_from_root
 
+
 def finish_mock(settings: dict, detections: list[str], output_file_template: str = "prior_config/config_tests_%d.json"):
     num_containers = settings['num_containers']
 
     try:
-        #Remove the prior config directory if it exists.  If not, continue
+        # Remove the prior config directory if it exists.  If not, continue
         shutil.rmtree("prior_config", ignore_errors=True)
-        
-        #We want to make the prior_config directory and the prior_config/apps directory
+
+        # We want to make the prior_config directory and the prior_config/apps directory
         os.makedirs("prior_config/apps")
     except FileExistsError as e:
-        print("Directory priorconfig/apps exists, but we just deleted it!\m\tQuitting...",file=sys.stderr)
+        print("Directory priorconfig/apps exists, but we just deleted it!\m\tQuitting...", file=sys.stderr)
         sys.exit(1)
     except Exception as e:
         print("Some error occured when trying to make the configs folder: [%s]\n\tQuitting..." % (
             str(e)), file=sys.stderr)
         sys.exit(1)
 
-    
-    #Copy the apps to the appropriate local.  This will also update
-    #the app paths in settings['local_apps']
+    # Copy the apps to the appropriate local.  This will also update
+    # the app paths in settings['local_apps']
     copy_local_apps_to_directory(settings['local_apps'], "prior_config/apps")
 
     for output_file_index in range(0, num_containers):
@@ -265,7 +266,7 @@ def main(args: list[str]):
     try:
         docker.client.from_env()
     except Exception as e:
-        print("Error, failed to get docker client.  Is Docker Running?\n\t%s"%(str(e)))
+        print("Error, failed to get docker client.  Is Docker Running?\n\t%s" % (str(e)))
 
     requests.packages.urllib3.disable_warnings()
 
@@ -278,8 +279,8 @@ def main(args: list[str]):
     elif action != "run":
         print("Unsupported action: [%s]" % (action), file=sys.stderr)
         sys.exit(1)
-    
 
+    
     FULL_DOCKER_HUB_CONTAINER_NAME = "splunk/splunk:%s" % settings['container_tag']
 
     if settings['num_containers'] > MAX_RECOMMENDED_CONTAINERS_BEFORE_WARNING:
@@ -291,11 +292,11 @@ def main(args: list[str]):
     github_service = ensure_security_content(
         settings['branch'], settings['commit_hash'], settings['pr_number'], settings['persist_security_content'])
     settings['commit_hash'] = github_service.commit_hash
-    
-    #Make a backup of this config containing the hash and stripped credentials.
-    #This makes the test perfectly reproducible.
-    validate_args.validate_and_write(settings,output_file=None,strip_credentials=True)
-    
+
+    # Make a backup of this config containing the hash and stripped credentials.
+    # This makes the test perfectly reproducible.
+    validate_args.validate_and_write(
+        settings, output_file=None, strip_credentials=True)
 
     all_test_files = github_service.get_test_files(settings['mode'],
                                                    settings['folders'],
@@ -303,13 +304,11 @@ def main(args: list[str]):
                                                    settings['detections_list'],
                                                    settings['detections_file'])
 
-    
-    
     local_volume_absolute_path = os.path.abspath(
         os.path.join(os.getcwd(), "apps"))
     try:
-        #remove the directory first
-        shutil.rmtree(local_volume_absolute_path,ignore_errors=True)
+        # remove the directory first
+        shutil.rmtree(local_volume_absolute_path, ignore_errors=True)
         os.mkdir(local_volume_absolute_path)
     except FileExistsError as e:
         # Directory already exists, do nothing
@@ -325,7 +324,7 @@ def main(args: list[str]):
         pass
         #file_path = os.path.expanduser(settings['local_apps']['SPLUNK_ES_CONTENT_UPDATE']['local_path'])
         #source_path = file_path
-        #dest_path = os.path.join(
+        # dest_path = os.path.join(
         #    local_volume_absolute_path, os.path.basename(file_path))
 
     elif 'SPLUNK_ES_CONTENT_UPDATE' not in settings['local_apps']:
@@ -336,13 +335,11 @@ def main(args: list[str]):
         # Need to generate that package
         source_path = generate_escu_app(settings['persist_security_content'])
         settings['local_apps']['SPLUNK_ES_CONTENT_UPDATE']['local_path'] = source_path
-        #dest_path = os.path.join(
+        # dest_path = os.path.join(
         #    local_volume_absolute_path, os.path.basename(source_path))
-    
-    
 
-    copy_local_apps_to_directory(settings['local_apps'], local_volume_absolute_path)
-    
+    copy_local_apps_to_directory(
+        settings['local_apps'], local_volume_absolute_path)
 
     if settings['mock']:
         finish_mock(settings, all_test_files)
@@ -355,7 +352,7 @@ def main(args: list[str]):
 
     mounts = [{"local_path": local_volume_absolute_path,
                "container_path": "/tmp/apps", "type": "bind", "read_only": True}]
-    
+
     cm = container_manager.ContainerManager(all_test_files,
                                             FULL_DOCKER_HUB_CONTAINER_NAME,
                                             settings['local_base_container_name'],
@@ -375,8 +372,8 @@ def main(args: list[str]):
                                             reuse_image=settings['reuse_image'],
                                             interactive_failure=settings['interactive_failure'])
 
-    
     cm.run_test()
 
+
 if __name__ == "__main__":
     main(sys.argv[1:])
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/container_manager.py b/automated_detection_testing/ci/detection_testing_batch/modules/container_manager.py
index a09dc2ae5a..5e0cd67cc5 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/container_manager.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/container_manager.py
@@ -53,8 +53,15 @@ class ContainerManager:
         else:
             self.container_password = container_password
         
+        print("\n\n***********************")
+        print("Log into your Splunk Container(s) after they boot at at http://127.0.0.1:[%d-%d]"%(web_port_start, web_port_start + num_containers - 1))
+        print("\tSplunk App Username: [%s]"%("admin"))
+        print("\tSplunk App Password: ", end='')
         if show_container_password:
-            print("Splunk App Password: [%s]"%(self.container_password))
+            print("[%s]"%(self.container_password))
+        else:
+            print(" --show_splunk_app_password set to False - password not printed")
+        print("***********************\n\n")
         
 
         self.containers = self.create_containers(
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/github_service.py b/automated_detection_testing/ci/detection_testing_batch/modules/github_service.py
index f14957a642..eea00800d4 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/github_service.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/github_service.py
@@ -22,7 +22,7 @@ SECURITY_CONTENT_URL = "https://github.com/splunk/security_content"
 
 class GithubService:
 
-    def __init__(self, security_content_branch: str, commit_hash:str, PR_number: int = None, existing_directory: bool = False):
+    def __init__(self, security_content_branch: str, commit_hash: str, PR_number: int = None, existing_directory: bool = False):
 
         self.security_content_branch = security_content_branch
         if existing_directory:
@@ -32,8 +32,8 @@ class GithubService:
             SECURITY_CONTENT_URL, f"security_content", f"develop")
 
         if commit_hash is not None and PR_number is not None:
-            print("Error - both the PR number [%d] and the commit hash [%s] were provided.  "\
-                  "Only 0 or 1 can be passed.\n\tQuitting..."%(PR_number, commit_hash))
+            print("Error - both the PR number [%d] and the commit hash [%s] were provided.  "
+                  "Only 0 or 1 can be passed.\n\tQuitting..." % (PR_number, commit_hash))
             sys.exit()
 
         elif PR_number:
@@ -43,14 +43,16 @@ class GithubService:
         # No checking to see if the hash is to a commit inside of the branch - the user
         # has to do that by hand
         if commit_hash is not None:
-            print("Checking out commit hash: [%s]"%(commit_hash))
+            print("Checking out commit hash: [%s]" % (commit_hash))
             self.security_content_repo_obj.git.checkout(commit_hash)
         else:
-            print("Checking out branch: [%s]..."%(security_content_branch),end='')
-            self.security_content_repo_obj.git.checkout(security_content_branch)
+            print("Checking out branch: [%s]..." %
+                  (security_content_branch), end='')
+            self.security_content_repo_obj.git.checkout(
+                security_content_branch)
             commit_hash = self.security_content_repo_obj.head.object.hexsha
-            print("commit_hash %s"%(commit_hash))
-        
+            print("commit_hash %s" % (commit_hash))
+
         self.commit_hash = commit_hash
 
     def clone_project(self, url, project, branch):
@@ -60,13 +62,13 @@ class GithubService:
 
     def prune_detections(self,
                          detections_to_prune: list[str],
-                           types_to_test: list[str],
+                         types_to_test: list[str],
                          previously_successful_tests: list[str],
                          exclude_ssa: bool = True,
                          summary_file: str = None) -> list[str]:
         pruned_tests = []
         csvlines = []
-        
+
         for detection in detections_to_prune:
             if os.path.basename(detection).startswith("ssa") and exclude_ssa:
                 continue
@@ -79,7 +81,7 @@ class GithubService:
                     pathlib.Path(*pathlib.Path(test_filepath).parts[1:]))
                 # If no   types are provided, then we will get everything
                 if 'type' in description and (description['type'] in types_to_test or len(types_to_test) == 0):
-                    
+
                     if not os.path.exists(test_filepath):
                         print("Detection [%s] references [%s], but it does not exist" % (
                             detection, test_filepath))
@@ -122,54 +124,59 @@ class GithubService:
 
         return pruned_tests
 
-    def get_test_files(self, mode: str, folders:list[str],   types:list[str],
-                        detections_list: Union[list[str], None], 
+    def get_test_files(self, mode: str, folders: list[str],   types: list[str],
+                       detections_list: Union[list[str], None],
                        detections_file=Union[str, None]) -> list[str]:
         if mode == "changes":
             tests = self.get_changed_test_files(folders, types)
         elif mode == "selected":
             if detections_list is None and detections_file is None:
-                #It's actually valid to supply an EMPTY list of files and the test should pass.
-                #This can occur when we try to test, for example, 1 detection but start 2 containers.
-                #We still want this to pass testing, so we shouldn't fail there!
-                print("Trying to test a list of files, but None were provided", file=sys.stderr)
+                # It's actually valid to supply an EMPTY list of files and the test should pass.
+                # This can occur when we try to test, for example, 1 detection but start 2 containers.
+                # We still want this to pass testing, so we shouldn't fail there!
+                print(
+                    "Trying to test a list of files, but None were provided", file=sys.stderr)
                 sys.exit(1)
 
             elif detections_list is not None and detections_file is not None:
-                print("Both detections_list [%s] and detections_file [%s] were provided.  "\
-                      "Because these confilect, we cannot test.\n\tQuitting..."%
+                print("Both detections_list [%s] and detections_file [%s] were provided.  "
+                      "Because these confilect, we cannot test.\n\tQuitting..." %
                       (detections_list, detections_file), file=sys.stderr)
                 sys.exit(1)
             elif detections_list is not None:
                 tests = self.get_selected_test_files(detections_list, types)
             elif detections_file is not None:
                 try:
-                    with open(detections_file,'r') as f:
+                    with open(detections_file, 'r') as f:
                         data = f.readlines()
-                    #Strip all whitespace from lines and exclude lines that are just whitespace
-                    files_to_test = [line.strip() for line in data if len(line.strip()) > 0]
+                    # Strip all whitespace from lines and exclude lines that are just whitespace
+                    files_to_test = [line.strip()
+                                     for line in data if len(line.strip()) > 0]
                 except Exception as e:
-                    print("There was an error reading the input file [%s]: [%s].\n\t"\
-                          "Quitting..."%(detections_file, str(e)))
+                    print("There was an error reading the input file [%s]: [%s].\n\t"
+                          "Quitting..." % (detections_file, str(e)))
                     sys.exit(1)
-                
-                tests = self.get_selected_test_files(files_to_test, folders,   types)
+
+                tests = self.get_selected_test_files(
+                    files_to_test, folders,   types)
             else:
-                #impossible to get here
-                print("Impossible to get here.  Just kept to make the if/elif more self describing",file=sys.stderr)
+                # impossible to get here
+                print(
+                    "Impossible to get here.  Just kept to make the if/elif more self describing", file=sys.stderr)
                 sys.exit(1)
 
         elif mode == "all":
             tests = self.get_all_tests_and_detections(folders,  types)
         else:
-            print("Error, unsupported mode [%s].  Mode must be one of %s", file=sys.stderr)
+            print(
+                "Error, unsupported mode [%s].  Mode must be one of %s", file=sys.stderr)
             sys.exit(1)
 
         return tests
 
     def get_selected_test_files(self,
                                 detection_file_list: list[str],
-                                  types_to_test: list[str] = [
+                                types_to_test: list[str] = [
                                     "Anomaly", "Hunting", "TTP"],
                                 previously_successful_tests: list[str] = []) -> list[str]:
 
@@ -178,7 +185,7 @@ class GithubService:
     def get_all_tests_and_detections(self,
                                      folders: list[str] = [
                                          'endpoint', 'cloud', 'network'],
-                                       types_to_test: list[str] = [
+                                     types_to_test: list[str] = [
                                          "Anomaly", "Hunting", "TTP"],
                                      previously_successful_tests: list[str] = []) -> list[str]:
         detections = []
@@ -203,7 +210,8 @@ class GithubService:
             if self.commit_hash is None:
                 differ = g.diff('--name-status', branch2 + '...' + branch1)
             else:
-                differ = g.diff('--name-status', branch2 + '...' + self.commit_hash)
+                differ = g.diff('--name-status', branch2 +
+                                '...' + self.commit_hash)
             changed_files = differ.splitlines()
 
             for file_path in changed_files:
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/new_arguments2.py b/automated_detection_testing/ci/detection_testing_batch/modules/new_arguments2.py
index e0548ea868..838276ce0f 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/new_arguments2.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/new_arguments2.py
@@ -1,14 +1,12 @@
 import argparse
-import copy
-import datetime
 import json
 from typing import OrderedDict, Union
-#import modules.validate_args as validate_args
 from modules import validate_args
 import sys
 
 DEFAULT_CONFIG_FILE = "test_config.json"
 
+
 def configure_action(args) -> tuple[str, dict]:
     settings = OrderedDict()
     if args.input_config_file is None:
@@ -24,9 +22,10 @@ def configure_action(args) -> tuple[str, dict]:
     for arg in settings:
         default = settings[arg]
         default_string = str(default).replace("'", '"')
-        
+
         if 'enum' in schema['properties'][arg]:
-            choice = input("%s [default: %s | choices: {%s}]: " % (arg, default_string,','.join(schema['properties'][arg]['enum'])))
+            choice = input("%s [default: %s | choices: {%s}]: " % (
+                arg, default_string, ','.join(schema['properties'][arg]['enum'])))
         else:
             choice = input("%s [default: %s]: " % (arg, default_string))
         choice = choice.strip()
@@ -48,7 +47,7 @@ def configure_action(args) -> tuple[str, dict]:
                         choice.count("'")))
                     choice = choice.replace("'", '"')
                 elif '"' in choice:
-                    #Do nothing
+                    # Do nothing
                     pass
                 elif choice.isdigit():
                     pass
@@ -57,52 +56,50 @@ def configure_action(args) -> tuple[str, dict]:
 
                 new_config[arg] = json.loads(choice)
                 formatted_print = choice
-        #We print out choice instead of new_config[arg] because the json.loads() messes up the quotation marks again
+        # We print out choice instead of new_config[arg] because the json.loads() messes up the quotation marks again
         print("\t{0}\n".format(formatted_print))
 
     # Now parse the new config and make sure it's good
-    validated_new_settings, schema = validate_args.validate_and_write(new_config, args.output_config_file)
-    if validated_new_settings == None:        
+    validated_new_settings, schema = validate_args.validate_and_write(
+        new_config, args.output_config_file)
+    if validated_new_settings == None:
         print("Could not update settings.\n\tQuitting...", file=sys.stderr)
         sys.exit(1)
-    
 
     return ("configure", validated_new_settings)
 
 
-def update_config_with_cli_arguments(args_dict:dict)->tuple[str, dict]:
-    #First load the config file
+def update_config_with_cli_arguments(args_dict: dict) -> tuple[str, dict]:
+    # First load the config file
 
-    settings,_ = validate_args.validate_file(args_dict['config_file'])
+    settings, _ = validate_args.validate_file(args_dict['config_file'])
     if settings is None:
-        print("Failure while processing settings in [%s].\n\tQuitting..."%(args_dict['config_file'].name), file=sys.stderr)
+        print("Failure while processing settings in [%s].\n\tQuitting..." % (
+            args_dict['config_file'].name), file=sys.stderr)
         sys.exit(1)
-    
-    #Then update it with the values that were passed as command line arguments
+
+    # Then update it with the values that were passed as command line arguments
     for key, value in args_dict.items():
         if key in settings:
             settings[key] = value
-    
-    #Validate again to make sure we didn't break anything
-    settings,_ = validate_args.validate(settings)
+
+    # Validate again to make sure we didn't break anything
+    settings, _ = validate_args.validate(settings)
     if settings is None:
         print("Failure while processing updated settings from command line.\n\tQuitting...", file=sys.stderr)
         sys.exit(1)
-    
-    
+
     return ("run", settings)
 
-    
 
-def run_action(args) -> tuple[str,dict]:
+def run_action(args) -> tuple[str, dict]:
 
     config = update_config_with_cli_arguments(args.__dict__)
 
-
     return config
 
 
-def parse(args)->tuple[str,dict]:
+def parse(args) -> tuple[str, dict]:
     '''
     try:
         with open(DEFAULT_CONFIG_FILE, 'r') as settings_file:
@@ -113,12 +110,12 @@ def parse(args)->tuple[str,dict]:
     '''
 
     import os
-    #if there is no default config file, then generate one
+    # if there is no default config file, then generate one
     if not os.path.exists(DEFAULT_CONFIG_FILE):
-        print("No default configuration file [%s] found.  Creating one..."%(DEFAULT_CONFIG_FILE))
-        with open(DEFAULT_CONFIG_FILE,'w') as cfg:
-            validate_args.validate_and_write({},cfg)
-
+        print("No default configuration file [%s] found.  Creating one..." % (
+            DEFAULT_CONFIG_FILE))
+        with open(DEFAULT_CONFIG_FILE, 'w') as cfg:
+            validate_args.validate_and_write({}, cfg)
 
     parser = argparse.ArgumentParser(
         description="Use 'SOME_PROGRAM_NAME_STRING --help' to get help with the arguments")
@@ -132,7 +129,7 @@ def parse(args)->tuple[str,dict]:
     configure_parser.set_defaults(func=configure_action)
     configure_parser.add_argument('-i', '--input_config_file', required=False,
                                   type=argparse.FileType('r'), help="The config file to base the configuration off of.")
-    configure_parser.add_argument('-o', '--output_config_file', required=False, default=DEFAULT_CONFIG_FILE, 
+    configure_parser.add_argument('-o', '--output_config_file', required=False, default=DEFAULT_CONFIG_FILE,
                                   type=argparse.FileType('w'), help="The config file to write the configuration off of.")
 
     # Run parser
@@ -141,13 +138,11 @@ def parse(args)->tuple[str,dict]:
     run_parser.set_defaults(func=run_action)
     run_parser.add_argument('-c', '--config_file', required=False,
                             type=argparse.FileType('r'),
-                            default = DEFAULT_CONFIG_FILE,
-                            help="The config file for the test.  Note that this file "\
-                            "cannot be changed (except for credentials that can be "\
+                            default=DEFAULT_CONFIG_FILE,
+                            help="The config file for the test.  Note that this file "
+                            "cannot be changed (except for credentials that can be "
                             "entered on the command line).")
 
-    
-
     run_parser.add_argument('-user', '--splunkbase_username', required=False, type=str,
                             help="Username for login to splunkbase.  This is required "
                             "if downloading packages from Splunkbase.  While this can "
@@ -156,14 +151,13 @@ def parse(args)->tuple[str,dict]:
 
     run_parser.add_argument('-b', '--branch', required=False, type=str,
                             help="The branch to run the tests on.")
-    
+
     run_parser.add_argument('-hash', '--commit_hash', required=False, type=str,
                             help="The hash to run the tests on.")
-    
+
     run_parser.add_argument('-pr', '--pr_number', required=False, type=int,
                             help="The Pull request to run the tests on.")
 
-
     run_parser.add_argument('-m', '--mode', required=False, type=str,
                             help="The mode all, changes, or selected for the testing.")
 
@@ -180,133 +174,49 @@ def parse(args)->tuple[str,dict]:
 
     run_parser.add_argument("-show_pass", "--show_splunk_app_password", required=False,
                             action="store_true",
-                            help="The password to login to the Splunk Server.  If the config "\
-                            "file is set to true, it will override the default False for this.  True "\
+                            help="The password to login to the Splunk Server.  If the config "
+                            "file is set to true, it will override the default False for this.  True "
                             "will override the default value in the config file.")
-    
-    run_parser.add_argument("-mock", "--mock", required=False, 
+
+    run_parser.add_argument("-mock", "--mock", required=False,
                             action="store_true",
-                            help="Split into multiple configs, don't actually run the tests. If the config "\
-                            "file is set to true, it will override the default False for this.  True "\
+                            help="Split into multiple configs, don't actually run the tests. If the config "
+                            "file is set to true, it will override the default False for this.  True "
                             "will override the default value in the config file.")
 
     run_parser.add_argument("-n", "--num_containers", required=False, type=int,
                             help="The number of Splunk containers to run or mock")
-    
-    
+
     args = parser.parse_args()
-    
-    
-    
+
+
     # Run the appropriate parser
     try:
-        #If one of these arguments is not passed on the command line, don't overwrite its config 
-        #file value with None - keep the config file value
+        # If one of these arguments is not passed on the command line, don't overwrite its config
+        # file value with None - keep the config file value
         keys = list(args.__dict__.keys())
         for key in keys:
-            if args.__dict__[key] is None and key in ["splunkbase_username","branch", "commit_hash", 
-                                                      "pr_number", "mode", "splunkbase_password",
-                                                      "splunk_app_password", "mock","num_containers"]:
+
+            # We have to do the check separately because booleans using the --store_true
+            # action have an implict default=False value, even if we don't set it. We cannot
+            # set their value to something else, like None
+
+            # Don't overwite booleans
+            if args.__dict__[key] is False and key in ["show_splunk_app_password", "mock"]:
+                del args.__dict__[key]
+            # Don't overwrite other values
+            elif args.__dict__[key] is None and key in ["splunkbase_username", "branch", "commit_hash",
+                                                        "pr_number", "mode", "splunkbase_password",
+                                                        "num_containers"]:
                 del args.__dict__[key]
 
         action, settings = args.func(args)
+
         
-        '''
-        default_settings,_ = validate_args.validate({})
-        if default_settings is None:
-            print("Somehow default settings were None.\n\tQuitting...",file=sys.stderr)
-            sys.exit(1)
-        #Fix up the show_app_password and mock arguments, as shown in the documentation
-        #for those args
-        settings['show_splunk_app_password'] |= default_settings['show_splunk_app_password']
-        settings['mock'] |= default_settings['mock']
-        '''
         return action, settings
     except Exception as e:
         print("Unknown Error - [%s]" % (str(e)))
         sys.exit(1)
 
-    '''
-
-    configure_parser.add_argument(
-        '-o', '--output_config', required=True, help="Name of config file to generate")
-
-    test_parser = actions_parser.add_parser("test", help="run a test")
-    test_parser.add_argument('-b', '--branch', required=True,
-                             help="The branch whose detections you would like to test.  "\
-                                  "In order to calculate new/changed detections, the detections "\
-                                  "in this branch will be diffed against those in the 'develop' branch")
-    test_parser.add_argument(
-        '-pr', '--pull_request_number', required=False, help="Pull request number.")
-
-    VALID_DETECTION_TYPES = ['endpoint', 'cloud', 'network']
-
-    #Common Test Arguments
-    test_parser.add_argument('-t', '--types', type=str, action="append", 
-                             help="Detection types to test. Can be one or more of %s"%(VALID_DETECTION_TYPES))
-    
-    
-    test_parser.add_argument('-e', '--escu_package', type=argparse.FileType('rb'), required=False, 
-                                        help="A previously generated ESCU PAcklage to use.  If you pass this "\
-                                             "argument, a new ESCU package will not be generated.  Note that this "\
-                                             "may cause newly-written detections to fail (for example, if they "\
-                                             "leverage macros that have been added or modified).")
-   
-    test_parser.add_argument('-p','--persist_security_content', required=False, action="store_true",
-                             help="Assumes security_content directory already exists.  Don't check it out and overwrite it again.  Saves "\
-                             "time and allows you to test a detection that you've updated.  Runs generate again in case you have "\
-                             "updated macros or anything else.  Especially useful for quick, local, iterative testing.")
-
-
-    test_parser.add_argument('-tag', '--container_tag', required=False, default = default_args['container_tag'], 
-                             help="The tag of the Splunk Container to use.  Tags are located "\
-                                  "at https://hub.docker.com/r/splunk/splunk/tags")
-
-    test_parser.add_argument("-show", "--show_password", required=False, default=False, action='store_true', 
-                             help="Show the generated password to use to login to splunk.  For a CI/CD run, "\
-                            "you probably don't want this.")
-
-    test_parser.add_argument('-r','--reuse_image', required=False, default=True, action='store_true', 
-                             help="Should existing images be re-used, or should they be redownloaded?")
-
-    test_parser.add_argument('-i', '--interactive_failure', required=False, default=False, action='store_true',
-                            help="If a test fails, should we pause before removing data so that the search can be debugged?")
-    
-
-    
-    #Mode settings
-    mode_parser = test_parser.add_subparsers(title="Test Modes", required=True)
-    #NEW
-    new_parser = mode_parser.add_parser("changes", 
-                                        help="Test only the new or changed detections")
-
-    #SELECTED
-
-    selected_parser = mode_parser.add_parser("selected", help="Test only the detections from the target branch that "\
-                                                              " are passed on the command line.  These can be given as "\
-                                                              "a list of files or as a file containing a list of files.")
-    selected_group = selected_parser.add_mutually_exclusive_group(required=True)
-    selected_group.add_argument('-df', '--detections_file', type=argparse.FileType('r'), 
-                                required=False, help="A file containing a list of detections to run, one per line")
-    selected_group.add_argument('-dl', '--detections_list', 
-                                required=False, help="The names of files that you want to test, separated by commas.  "\
-                                                     "Do not include spaces between the detections!")
-    
-    #ALL
-    all_parser = mode_parser.add_parser("all", 
-                                        help="Test all of the detections in the target branch.  "\
-                                             "Note that this could take a very long time.")
-
-
-    args = parser.parse_args()
-    try:
-        validate_args.validate(args.__dict__)
-
-    except Exception as e:
-        print("Error validating command line arguments: [%s]"%(str(e)))
-        sys.exit(1)
-    '''
-
-
 if __name__ == "__main__":
     parse(sys.argv[1:])
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py b/automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py
index 66a45f68d0..0602ca92d5 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py
@@ -107,7 +107,7 @@ setup_schema = {
 
         "show_splunk_app_password": {
             "type": "boolean",
-            "default": True
+            "default": False
 
         },
 

From b94ef641ecc4ed0c2aaf5f085ba1778af7c8d8da Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Wed, 24 Nov 2021 17:13:41 -0800
Subject: [PATCH 101/166] Changed the detection testing to run on all, develop
 instead of changes to a different branch. This will collect some data for us
 over the next few days.

---
 .github/workflows/docker-detection-testing.yml | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/docker-detection-testing.yml b/.github/workflows/docker-detection-testing.yml
index d23da9fb1c..d4afad7469 100644
--- a/.github/workflows/docker-detection-testing.yml
+++ b/.github/workflows/docker-detection-testing.yml
@@ -1,6 +1,9 @@
 name: docker-detection-testing
 on:
-  push:
+  #push:
+  #run nightly at 0500 UTC
+  schedule:
+    - cron: "0 5 * * *"
 jobs:
 
   validate-tag-if-present:
@@ -75,7 +78,7 @@ jobs:
               cd automated_detection_testing/ci/detection_testing_batch
               source .venv/bin/activate
               
-              python3 detection_testing_execution.py run --branch RegistryDetectionFixes  --mode changes --num_containers 10 --mock 
+              python3 detection_testing_execution.py run --branch develop  --mode all --num_containers 10 --mock 
           
           - name: Upload Test Results Files
             uses: actions/upload-artifact@v2

From afb07b671494912d7b395d9ccc8c6aa2157d5a60 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Wed, 24 Nov 2021 17:14:33 -0800
Subject: [PATCH 102/166] Run it on push too for an immediate test

---
 .github/workflows/docker-detection-testing.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/docker-detection-testing.yml b/.github/workflows/docker-detection-testing.yml
index d4afad7469..68ecde80f7 100644
--- a/.github/workflows/docker-detection-testing.yml
+++ b/.github/workflows/docker-detection-testing.yml
@@ -1,6 +1,6 @@
 name: docker-detection-testing
 on:
-  #push:
+  push:
   #run nightly at 0500 UTC
   schedule:
     - cron: "0 5 * * *"

From 38ad36c296bf859dee4caec8ed89c0ec368b7219 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Mon, 29 Nov 2021 10:49:54 -0800
Subject: [PATCH 103/166] Run at 55th minute each hour for testing.

---
 .github/workflows/docker-detection-testing.yml | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/.github/workflows/docker-detection-testing.yml b/.github/workflows/docker-detection-testing.yml
index 68ecde80f7..baae897de2 100644
--- a/.github/workflows/docker-detection-testing.yml
+++ b/.github/workflows/docker-detection-testing.yml
@@ -1,9 +1,7 @@
 name: docker-detection-testing
 on:
-  push:
-  #run nightly at 0500 UTC
   schedule:
-    - cron: "0 5 * * *"
+    - cron: "55 * * * *"
 jobs:
 
   validate-tag-if-present:

From e8fb9578ea81b2b197c2d5956b9d1d1db5f19971 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Mon, 29 Nov 2021 10:51:44 -0800
Subject: [PATCH 104/166] Change types of quotes around schedule

---
 .github/workflows/docker-detection-testing.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/docker-detection-testing.yml b/.github/workflows/docker-detection-testing.yml
index baae897de2..3c7ea6985d 100644
--- a/.github/workflows/docker-detection-testing.yml
+++ b/.github/workflows/docker-detection-testing.yml
@@ -1,7 +1,7 @@
 name: docker-detection-testing
 on:
   schedule:
-    - cron: "55 * * * *"
+    - cron: '55 * * * *''
 jobs:
 
   validate-tag-if-present:

From 95e7169e05c75601f01cdbff705cf293858ced2f Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Mon, 29 Nov 2021 10:53:29 -0800
Subject: [PATCH 105/166] fix quote

---
 .github/workflows/docker-detection-testing.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/docker-detection-testing.yml b/.github/workflows/docker-detection-testing.yml
index 3c7ea6985d..8b6fa0a1f7 100644
--- a/.github/workflows/docker-detection-testing.yml
+++ b/.github/workflows/docker-detection-testing.yml
@@ -1,7 +1,7 @@
 name: docker-detection-testing
 on:
   schedule:
-    - cron: '55 * * * *''
+    - cron: '55 * * * *'
 jobs:
 
   validate-tag-if-present:

From fff797358c6a2b9b43eeee86c22abdbdca59c2fc Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Mon, 29 Nov 2021 11:34:59 -0800
Subject: [PATCH 106/166] Run a handlful of tests on push to produce output
 artifacts.

---
 .github/workflows/docker-detection-testing.yml | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/docker-detection-testing.yml b/.github/workflows/docker-detection-testing.yml
index 8b6fa0a1f7..1098a48362 100644
--- a/.github/workflows/docker-detection-testing.yml
+++ b/.github/workflows/docker-detection-testing.yml
@@ -1,7 +1,11 @@
 name: docker-detection-testing
 on:
+  push:
+  #run nightly at 0500 UTC
+  #Note the scheduled workflows will only run if the Action has 
+  #been committed to the default branch (develop)
   schedule:
-    - cron: '55 * * * *'
+    - cron: "0 5 * * *"
 jobs:
 
   validate-tag-if-present:
@@ -76,7 +80,7 @@ jobs:
               cd automated_detection_testing/ci/detection_testing_batch
               source .venv/bin/activate
               
-              python3 detection_testing_execution.py run --branch develop  --mode all --num_containers 10 --mock 
+              python3 detection_testing_execution.py run --branch RegistryDetectionFixes  --mode changes --num_containers 10 --mock 
           
           - name: Upload Test Results Files
             uses: actions/upload-artifact@v2

From 2b3fd66bb39ad21e56a1ee2a953bd52b324600d1 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Mon, 29 Nov 2021 11:54:26 -0800
Subject: [PATCH 107/166] Run a test on everything, saving the intermediate
 files to help debugging.

---
 .../workflows/docker-detection-testing.yml    | 32 +++++++++----------
 1 file changed, 16 insertions(+), 16 deletions(-)

diff --git a/.github/workflows/docker-detection-testing.yml b/.github/workflows/docker-detection-testing.yml
index 1098a48362..9d1fb93066 100644
--- a/.github/workflows/docker-detection-testing.yml
+++ b/.github/workflows/docker-detection-testing.yml
@@ -80,7 +80,7 @@ jobs:
               cd automated_detection_testing/ci/detection_testing_batch
               source .venv/bin/activate
               
-              python3 detection_testing_execution.py run --branch RegistryDetectionFixes  --mode changes --num_containers 10 --mock 
+              python3 detection_testing_execution.py run --branch develop  --mode all --num_containers 10 --mock 
           
           - name: Upload Test Results Files
             uses: actions/upload-artifact@v2
@@ -275,19 +275,19 @@ jobs:
               automated_detection_testing/ci/detection_testing_batch/detection_failure_manifest.json
             
         
-        - name: Clean up intermediate Files
-          uses: geekyeggo/delete-artifact@v1
-          if: always()
-          with:
-              name: |
-                config_tests_0.json.results
-                config_tests_1.json.results
-                config_tests_2.json.results
-                config_tests_3.json.results
-                config_tests_4.json.results
-                config_tests_5.json.results
-                config_tests_6.json.results
-                config_tests_7.json.results
-                config_tests_8.json.results
-                config_tests_9.json.results
+        #Don't delete the intermediate files, they can be useful for debugging
+        # - name: Clean up intermediate Files
+        #   uses: geekyeggo/delete-artifact@v1
+        #   with:
+        #       name: |
+        #         config_tests_0.json.results
+        #         config_tests_1.json.results
+        #         config_tests_2.json.results
+        #         config_tests_3.json.results
+        #         config_tests_4.json.results
+        #         config_tests_5.json.results
+        #         config_tests_6.json.results
+        #         config_tests_7.json.results
+        #         config_tests_8.json.results
+        #         config_tests_9.json.results
                 
\ No newline at end of file

From 180785c7d495182c9fbc28e0d1eb13fcbb09d950 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Mon, 29 Nov 2021 14:40:17 -0800
Subject: [PATCH 108/166] Certain types of error would have missing fields in
 the results json, causing the summarization code to crash.  Fixed that.  Also
 sorted the output of the summarization to made it easier to look at a glance.
 Errors and failures will be at the top and the rest will be sorted
 alphabetically.

---
 .../detection_testing_execution.py            | 68 ++++++++++++-------
 .../modules/splunk_container.py               | 43 ++++++------
 .../modules/test_driver.py                    |  9 +++
 .../detection_testing_batch/summarize_json.py |  9 ++-
 4 files changed, 83 insertions(+), 46 deletions(-)

diff --git a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
index 59149f8bdc..80563e57c6 100644
--- a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
+++ b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
@@ -188,7 +188,7 @@ def generate_escu_app(persist_security_content: bool = False) -> str:
     return output_file_path_from_root
 
 
-def finish_mock(settings: dict, detections: list[str], output_file_template: str = "prior_config/config_tests_%d.json"):
+def finish_mock(settings: dict, detections: list[str], output_file_template: str = "prior_config/config_tests_%d.json")->bool:
     num_containers = settings['num_containers']
 
     try:
@@ -199,11 +199,11 @@ def finish_mock(settings: dict, detections: list[str], output_file_template: str
         os.makedirs("prior_config/apps")
     except FileExistsError as e:
         print("Directory priorconfig/apps exists, but we just deleted it!\m\tQuitting...", file=sys.stderr)
-        sys.exit(1)
+        return False
     except Exception as e:
         print("Some error occured when trying to make the configs folder: [%s]\n\tQuitting..." % (
             str(e)), file=sys.stderr)
-        sys.exit(1)
+        return False
 
     # Copy the apps to the appropriate local.  This will also update
     # the app paths in settings['local_apps']
@@ -252,22 +252,20 @@ def finish_mock(settings: dict, detections: list[str], output_file_template: str
                 if validated_settings is None:
                     print(
                         "There was an error validating the updated mock settings.\n\tQuitting...", file=sys.stderr)
-                    sys.exit(1)
+                    return False
 
         except Exception as e:
             print("Error writing config file %s: [%s]\n\tQuitting..." % (
                 fname, str(e)), file=sys.stderr)
-            sys.exit(1)
+            return False
 
-    sys.exit(0)
+    return True
 
 
 def main(args: list[str]):
-    try:
-        docker.client.from_env()
-    except Exception as e:
-        print("Error, failed to get docker client.  Is Docker Running?\n\t%s" % (str(e)))
-
+    #Disable insecure warnings.  We make a number of HTTPS requests to Splunk
+    #docker containers that we've set up.  Without this line, we get an 
+    #insecure warning every time due to invalid cert.
     requests.packages.urllib3.disable_warnings()
 
     start_datetime = datetime.now()
@@ -280,6 +278,16 @@ def main(args: list[str]):
         print("Unsupported action: [%s]" % (action), file=sys.stderr)
         sys.exit(1)
 
+    if settings['mock'] is False:
+        # If this is a real run, then make sure Docker is installed and running and usable
+        # If this is a mock, then that is not required.  By only checking on a non-mock
+        # run, we save ourselves the need to install docker in the CI for the manifest
+        # generation step.
+        try:
+            docker.client.from_env()
+        except Exception as e:
+            print("Error, failed to get docker client.  Is Docker Installed and Running?\n\t%s" % (str(e)))
+
     
     FULL_DOCKER_HUB_CONTAINER_NAME = "splunk/splunk:%s" % settings['container_tag']
 
@@ -304,6 +312,8 @@ def main(args: list[str]):
                                                    settings['detections_list'],
                                                    settings['detections_file'])
 
+
+    #Set up the directory that will be used to store the local apps/apps we build
     local_volume_absolute_path = os.path.abspath(
         os.path.join(os.getcwd(), "apps"))
     try:
@@ -317,41 +327,51 @@ def main(args: list[str]):
         print("Error creating the apps folder [%s]: [%s]\n\tQuitting..."
               % (local_volume_absolute_path, str(e)), file=sys.stderr)
         sys.exit(1)
+    #Add the info about the mount
+    mounts = [{"local_path": local_volume_absolute_path,
+               "container_path": "/tmp/apps", "type": "bind", "read_only": True}]
+
 
     # Check to see if we want to install ESCU and whether it was preeviously generated and we should use that file
     if 'SPLUNK_ES_CONTENT_UPDATE' in settings['local_apps'] and settings['local_apps']['SPLUNK_ES_CONTENT_UPDATE']['local_path'] is not None:
-        # Using a pregenerated ESCU, copy it to apps (unless it)
+        # Using a pregenerated ESCU, no need to build it
         pass
-        #file_path = os.path.expanduser(settings['local_apps']['SPLUNK_ES_CONTENT_UPDATE']['local_path'])
-        #source_path = file_path
-        # dest_path = os.path.join(
-        #    local_volume_absolute_path, os.path.basename(file_path))
-
     elif 'SPLUNK_ES_CONTENT_UPDATE' not in settings['local_apps']:
         print("%s was not found in %s.  We assume this is an error and shut down.\n\t"
               "Quitting..." % ('SPLUNK_ES_CONTENT_UPDATE', "settings['local_apps']"), file=sys.stderr)
         sys.exit(1)
     else:
-        # Need to generate that package
+        # Generate the ESCU package from this branch.
         source_path = generate_escu_app(settings['persist_security_content'])
         settings['local_apps']['SPLUNK_ES_CONTENT_UPDATE']['local_path'] = source_path
-        # dest_path = os.path.join(
-        #    local_volume_absolute_path, os.path.basename(source_path))
+        
 
+    # Copy all the apps, to include ESCU (whether pregenerated or just generated)
     copy_local_apps_to_directory(
         settings['local_apps'], local_volume_absolute_path)
 
-    if settings['mock']:
-        finish_mock(settings, all_test_files)
 
+    # If this is a mock run, finish it now
+    if settings['mock']:
+        #The function below 
+        if finish_mock(settings, all_test_files):
+            # mock was successful!
+            print("Mock successful!  Manifests generated!")
+            sys.exit(0)
+        else:
+            print("There was an unrecoverage error during the mock.\n\tQuitting...",file=sys.stderr)
+            sys.exit(1)
+
+
+
+    #Add some files that always need to be copied to to container to set up indexes and datamodels.
     files_to_copy_to_container = OrderedDict()
     files_to_copy_to_container["INDEXES"] = {
         "local_file_path": index_file_local_path, "container_file_path": index_file_container_path}
     files_to_copy_to_container["DATAMODELS"] = {
         "local_file_path": datamodel_file_local_path, "container_file_path": datamodel_file_container_path}
 
-    mounts = [{"local_path": local_volume_absolute_path,
-               "container_path": "/tmp/apps", "type": "bind", "read_only": True}]
+    
 
     cm = container_manager.ContainerManager(all_test_files,
                                             FULL_DOCKER_HUB_CONTAINER_NAME,
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/splunk_container.py b/automated_detection_testing/ci/detection_testing_batch/modules/splunk_container.py
index 7a2d245d1e..f6d43ee6ce 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/splunk_container.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/splunk_container.py
@@ -11,7 +11,7 @@ import requests
 import shutil
 from modules import splunk_sdk
 from modules import testing_service
-from modules import test_driver 
+from modules import test_driver
 import time
 import timeit
 from typing import Union
@@ -46,7 +46,7 @@ class SplunkContainer:
         self.container_password = container_password
         self.local_apps = local_apps
         self.splunkbase_apps = splunkbase_apps
-         
+
         self.files_to_copy_to_container = files_to_copy_to_container
         self.splunk_ip = splunk_ip
         self.container_name = container_name
@@ -74,15 +74,15 @@ class SplunkContainer:
     ) -> tuple[str, bool]:
         apps_to_install = []
         require_credentials = False
-        
+
         for app_name, app_info in self.local_apps.items():
-            
+
             app_file_name = os.path.basename(app_info['local_path'])
             app_file_container_path = os.path.join("/tmp/apps", app_file_name)
             apps_to_install.append(app_file_container_path)
 
         for app_name, app_info in self.splunkbase_apps.items():
-            
+
             if splunkbase_username is None or splunkbase_password is None:
                 raise Exception(
                     "Error: Requested app from Splunkbase but Splunkbase username and/or password were not supplied."
@@ -91,12 +91,10 @@ class SplunkContainer:
                 app_info["app_number"], app_info["app_version"])
             apps_to_install.append(target)
             require_credentials = True
-            #elif app["location"] == "local":
+            # elif app["location"] == "local":
             #    apps_to_install.append(app["container_path"])
         return ",".join(apps_to_install), require_credentials
 
-    
-
     def make_environment(
         self,
         local_apps: OrderedDict,
@@ -221,24 +219,24 @@ class SplunkContainer:
         if self.container_start_time == -1:
             total_time_string = "NOT STARTED"
         else:
-            total_time_rounded = datetime.timedelta( seconds =
-                round(current_time - self.container_start_time))
+            total_time_rounded = datetime.timedelta(
+                seconds=round(current_time - self.container_start_time))
             total_time_string = str(total_time_rounded)
 
         # Time that the container setup took
         if self.test_start_time == -1 or self.container_start_time == -1:
             setup_time_string = "NOT SET UP"
         else:
-            setup_secounds_rounded = datetime.timedelta(seconds =
-                round(self.test_start_time - self.container_start_time))
+            setup_secounds_rounded = datetime.timedelta(
+                seconds=round(self.test_start_time - self.container_start_time))
             setup_time_string = str(setup_secounds_rounded)
 
         # Time that the tests have been running
         if self.test_start_time == -1 or self.num_tests_completed == 0:
             testing_time_string = "NO TESTS COMPLETED"
         else:
-            testing_seconds_rounded = datetime.timedelta( seconds =
-                round(current_time - self.test_start_time))
+            testing_seconds_rounded = datetime.timedelta(
+                seconds=round(current_time - self.test_start_time))
 
             # Get the approximate time per test.  This is a clunky way to get rid of decimal
             # seconds.... but it works
@@ -246,14 +244,16 @@ class SplunkContainer:
             timedelta_per_test_rounded = timedelta_per_test - \
                 datetime.timedelta(
                     microseconds=timedelta_per_test.microseconds)
-            
-            testing_time_string = "%s (%d tests @ %s per test)"%(testing_seconds_rounded, self.num_tests_completed, timedelta_per_test_rounded)
+
+            testing_time_string = "%s (%d tests @ %s per test)" % (
+                testing_seconds_rounded, self.num_tests_completed, timedelta_per_test_rounded)
 
         summary_str = "Summary\n\t"\
                       "Total Time          : [%s]\n\t"\
                       "Container Start Time: [%s]\n\t"\
-                      "Test Execution Time : [%s]" %(total_time_string, setup_time_string, testing_time_string)
-        
+                      "Test Execution Time : [%s]" % (
+                          total_time_string, setup_time_string, testing_time_string)
+
         return summary_str
 
     def wait_for_splunk_ready(
@@ -364,12 +364,15 @@ class SplunkContainer:
                     "Warning - uncaught error in detection test for [%s] - this should not happen: [%s]"
                     % (detection_to_test, str(e))
                 )
+                # Fill in all the "Empty" fields with default values. Otherwise, we will not be able to 
+                # process the result correctly.  
                 self.synchronization_object.addError(
                     {"detection_file": detection_to_test,
                         "detection_error": str(e)}
+
+
                 )
-            self.num_tests_completed+=1
+            self.num_tests_completed += 1
 
             # Sleep for a small random time so that containers drift apart and don't synchronize their testing
             time.sleep(random.randint(1, 30))
-
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/test_driver.py b/automated_detection_testing/ci/detection_testing_batch/modules/test_driver.py
index b21d2ed960..171a524598 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/test_driver.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/test_driver.py
@@ -58,6 +58,15 @@ class TestDriver:
             self.lock.release()
 
     def addError(self, detection:dict)->None:
+        #Make sure that even errors have all of the required fields.
+        for required_field in ['search_string', 'diskUsage','runDuration', 'detection_name', 'scanCount']:
+            if required_field not in detection:
+                detection[required_field] = ""
+        if  'error' not in detection:
+            detection['error'] = True
+        if  'success' not in detection:
+            detection['success'] = False
+
         self.lock.acquire()
         try:
             self.errors.append(detection)
diff --git a/automated_detection_testing/ci/detection_testing_batch/summarize_json.py b/automated_detection_testing/ci/detection_testing_batch/summarize_json.py
index a1084a4a44..0e80a3782d 100644
--- a/automated_detection_testing/ci/detection_testing_batch/summarize_json.py
+++ b/automated_detection_testing/ci/detection_testing_batch/summarize_json.py
@@ -5,14 +5,17 @@ import sys
 import json
 from modules import validate_args
 import os.path
+from operator import itemgetter
 
 def outputResultsJSON(output_filename:str, data:list[dict], baseline:OrderedDict)->bool:
     success = True
+    
     try:
         test_count = len(data)
         #Passed
         pass_count = len([x for x in data if x['success'] == True])
         
+        
         #A failure or an error
         fail_only_count = len([x for x in data if x['success'] == False])
         
@@ -42,8 +45,9 @@ def outputResultsJSON(output_filename:str, data:list[dict], baseline:OrderedDict
                  "TOTAL_FAILURES": fail_only_count, "FAIL_ONLY": fail_without_error_count, 
                  "FAIL_AND_ERROR":fail_and_error_count }
 
+        data_sorted = sorted(data, key = lambda k: (-k['error'], k['success'], k['detection_file']))
         with open(output_filename, "w") as jsonFile:
-            json.dump({'summary':summary, 'baseline': baseline, 'results':data}, jsonFile, indent="    ")
+            json.dump({'summary':summary, 'baseline': baseline, 'results':data_sorted}, jsonFile, indent="    ")
         
         
         #Generate a failure that the user can download to reproduce and test ONLY the failures locally.
@@ -51,7 +55,7 @@ def outputResultsJSON(output_filename:str, data:list[dict], baseline:OrderedDict
         #that succeeded!
         
         
-        fail_list = [os.path.join("security_content/detections",x['detection_file'] ) for x in data if x['success'] == False]
+        fail_list = [os.path.join("security_content/detections",x['detection_file'] ) for x in data_sorted if x['success'] == False]
 
         if len(fail_list) > 0:
             failures_test_override = {"detections_list": fail_list, "interactive_failure":True, 
@@ -61,6 +65,7 @@ def outputResultsJSON(output_filename:str, data:list[dict], baseline:OrderedDict
                 validate_args.validate_and_write(failures_test_override, failures)
     except Exception as e:
         print("There was an error generating [%s]: [%s]"%(output_filename, str(e)),file=sys.stderr)
+        print(data)
         raise(e)
         #success = False
         #return success, False

From 868c98082b9dbd381e4d14b343c0d7b671c575c1 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Mon, 29 Nov 2021 16:49:06 -0800
Subject: [PATCH 109/166] Small change to test changes on a specific branch
 instead of develop.

---
 .github/workflows/docker-detection-testing.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/docker-detection-testing.yml b/.github/workflows/docker-detection-testing.yml
index 9d1fb93066..65a8ed89cc 100644
--- a/.github/workflows/docker-detection-testing.yml
+++ b/.github/workflows/docker-detection-testing.yml
@@ -80,7 +80,7 @@ jobs:
               cd automated_detection_testing/ci/detection_testing_batch
               source .venv/bin/activate
               
-              python3 detection_testing_execution.py run --branch develop  --mode all --num_containers 10 --mock 
+              python3 detection_testing_execution.py run --branch DetectionBaselineFixes  --mode changes --num_containers 10 --mock 
           
           - name: Upload Test Results Files
             uses: actions/upload-artifact@v2

From eb6378e805ab27b8c65355535664b9409945661e Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Tue, 30 Nov 2021 09:52:36 -0800
Subject: [PATCH 110/166] Made some changes to properly supporting testing
 changed test file.  Had some trouble before locating the appropriate
 detection file.  Also throw an error if we try to test a file that was
 changed and is in the experimental path.  Set up to run a test of the
 DetectionBaselineFixes branch to test these changes in GitHub Actions.

---
 .../workflows/docker-detection-testing.yml    | 30 ++++++-------
 .../detection_testing_execution.py            | 16 ++++---
 .../modules/github_service.py                 | 44 ++++++++++++++++---
 3 files changed, 63 insertions(+), 27 deletions(-)

diff --git a/.github/workflows/docker-detection-testing.yml b/.github/workflows/docker-detection-testing.yml
index 65a8ed89cc..aac8c1f8c9 100644
--- a/.github/workflows/docker-detection-testing.yml
+++ b/.github/workflows/docker-detection-testing.yml
@@ -275,19 +275,19 @@ jobs:
               automated_detection_testing/ci/detection_testing_batch/detection_failure_manifest.json
             
         
-        #Don't delete the intermediate files, they can be useful for debugging
-        # - name: Clean up intermediate Files
-        #   uses: geekyeggo/delete-artifact@v1
-        #   with:
-        #       name: |
-        #         config_tests_0.json.results
-        #         config_tests_1.json.results
-        #         config_tests_2.json.results
-        #         config_tests_3.json.results
-        #         config_tests_4.json.results
-        #         config_tests_5.json.results
-        #         config_tests_6.json.results
-        #         config_tests_7.json.results
-        #         config_tests_8.json.results
-        #         config_tests_9.json.results
+        
+        - name: Clean up intermediate Files
+          uses: geekyeggo/delete-artifact@v1
+          with:
+              name: |
+                config_tests_0.json.results
+                config_tests_1.json.results
+                config_tests_2.json.results
+                config_tests_3.json.results
+                config_tests_4.json.results
+                config_tests_5.json.results
+                config_tests_6.json.results
+                config_tests_7.json.results
+                config_tests_8.json.results
+                config_tests_9.json.results
                 
\ No newline at end of file
diff --git a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
index 80563e57c6..7207981984 100644
--- a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
+++ b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
@@ -306,12 +306,16 @@ def main(args: list[str]):
     validate_args.validate_and_write(
         settings, output_file=None, strip_credentials=True)
 
-    all_test_files = github_service.get_test_files(settings['mode'],
-                                                   settings['folders'],
-                                                   settings['types'],
-                                                   settings['detections_list'],
-                                                   settings['detections_file'])
-
+    try:
+        all_test_files = github_service.get_test_files(settings['mode'],
+                                                    settings['folders'],
+                                                    settings['types'],
+                                                    settings['detections_list'],
+                                                    settings['detections_file'])
+    except Exception as e:
+        print("Error getting test files:\n%s"%(str(e)), file=sys.stderr)
+        print("\tQuitting...", file=sys.stderr)
+        sys.exit(1)
 
     #Set up the directory that will be used to store the local apps/apps we build
     local_volume_absolute_path = os.path.abspath(
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/github_service.py b/automated_detection_testing/ci/detection_testing_batch/modules/github_service.py
index eea00800d4..1a083527bb 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/github_service.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/github_service.py
@@ -68,7 +68,7 @@ class GithubService:
                          summary_file: str = None) -> list[str]:
         pruned_tests = []
         csvlines = []
-
+        
         for detection in detections_to_prune:
             if os.path.basename(detection).startswith("ssa") and exclude_ssa:
                 continue
@@ -228,22 +228,54 @@ class GithubService:
                 branch1, branch2), file=sys.stderr)
             return []
 
+        
         # all files have the format A\tFILENAME or M\tFILENAME.  Get rid of those leading characters
         changed_test_files = [os.path.join("security_content", name.split(
             '\t')[1]) for name in changed_test_files if len(name.split('\t')) == 2]
         changed_detection_files = [os.path.join("security_content", name.split(
             '\t')[1]) for name in changed_detection_files if len(name.split('\t')) == 2]
 
-        # convert the test files to the detection file equivalent
+        
+        # Convert the test files to the detection file equivalent. 
+        # Note that some of these tests may be baselines and their associated 
+        # detection could be in experimental or not in the experimental folder
         converted_test_files = []
-        for test_filepath in changed_test_files:
-            detection_filename = str(pathlib.Path(
-                *pathlib.Path(test_filepath).parts[-2:])).replace("tests", "detections", 1)
-            converted_test_files.append(detection_filename)
+        #for test_filepath in changed_test_files:
+        #    detection_filename = str(pathlib.Path(
+        #        *pathlib.Path(test_filepath).parts[-2:])).replace("tests", "detections", 1)
+        #    converted_test_files.append(detection_filename)
+        
+        #check and throw an error if we somehow got experimental tests
+        experimental_tests = [x for x in changed_test_files if 'experimental' in x]
+        if len(experimental_tests) > 0:
+            raise(Exception("Error - expected no experimental detections, but found:\n\t%s]"%("\n\t".join(experimental_tests))))
 
+        #Get the appropriate detection file paths for a modified test file
+        for test_filepath in changed_test_files:
+            folder_and_filename =  str(pathlib.Path(*pathlib.Path(test_filepath).parts[-2:]))
+            folder_and_filename_fixed_suffix = folder_and_filename.replace(".test.yml",".yml")
+            result = None
+            for f in glob.glob("security_content/detections/**/" + folder_and_filename_fixed_suffix,recursive=True):
+                if result != None:
+                    #found a duplicate filename that matches
+                    raise(Exception("Error - Found at least two detection files to match for test file [%s]: [%s] and [%s]"%(test_filepath, result, f)))
+                else:
+                    result = f
+            if result is None:
+                raise(Exception("Error - Failed to find detection file for test file [%s]"%(test_filepath)))
+            else:
+                converted_test_files.append(result)
+        
+        
+        
         for name in converted_test_files:
             if name not in changed_detection_files:
                 changed_detection_files.append(name)
+        
+        #check and throw an error if we somehow got experimental detections
+        experimental_detections = [x for x in changed_detection_files if 'experimental' in x]
+        if len(experimental_detections) > 0:
+            raise(Exception("Error - expected no experimental detections, but found:\n\t%s"%('\n\t'.join(experimental_detections))))
 
         return self.prune_detections(changed_detection_files,   types_to_test, previously_successful_tests)
 

From e759f01e0c8a151b62acc01ddf4c0785fa5245cc Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Tue, 30 Nov 2021 10:16:05 -0800
Subject: [PATCH 111/166] Better command line summary of pass and fail for
 summarize_json.py

---
 .../ci/detection_testing_batch/summarize_json.py     | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/automated_detection_testing/ci/detection_testing_batch/summarize_json.py b/automated_detection_testing/ci/detection_testing_batch/summarize_json.py
index 0e80a3782d..2aa1a7a277 100644
--- a/automated_detection_testing/ci/detection_testing_batch/summarize_json.py
+++ b/automated_detection_testing/ci/detection_testing_batch/summarize_json.py
@@ -7,7 +7,8 @@ from modules import validate_args
 import os.path
 from operator import itemgetter
 
-def outputResultsJSON(output_filename:str, data:list[dict], baseline:OrderedDict)->bool:
+
+def outputResultsJSON(output_filename:str, data:list[dict], baseline:OrderedDict)->tuple[bool,int,int,int]:
     success = True
     
     try:
@@ -70,7 +71,7 @@ def outputResultsJSON(output_filename:str, data:list[dict], baseline:OrderedDict
         #success = False
         #return success, False
 
-    return success
+    return success, test_count, pass_count, fail_and_error_count
 
 
 
@@ -98,8 +99,11 @@ try:
         else:
             all_data['results'] = data['results']
 
-    test_pass = outputResultsJSON(args.output_filename, all_data['results'], all_data['baseline'])
-    print("Successfully summarized [%d] detections"%(len(all_data['results'])))
+    test_pass, test_count, pass_count, fail_and_error_count = outputResultsJSON(args.output_filename, all_data['results'], all_data['baseline'])
+    print("Summary:"\
+          "\n\tTotal Tests: %d"\
+          "\n\tTotal Pass : %d"\
+          "\n\tTotal Fail : %d"%(test_count, pass_count, fail_and_error_count))
     if not test_pass:
         print("Result: FAIL")
         sys.exit(1)

From f1a10f0d65ada68b6dac638d35a6d0ef15ad3a27 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Tue, 30 Nov 2021 10:36:12 -0800
Subject: [PATCH 112/166] Upload the updated test file so we can replicate the
 test in the future.

---
 .github/workflows/docker-detection-testing.yml | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/docker-detection-testing.yml b/.github/workflows/docker-detection-testing.yml
index aac8c1f8c9..a565b48527 100644
--- a/.github/workflows/docker-detection-testing.yml
+++ b/.github/workflows/docker-detection-testing.yml
@@ -81,7 +81,7 @@ jobs:
               source .venv/bin/activate
               
               python3 detection_testing_execution.py run --branch DetectionBaselineFixes  --mode changes --num_containers 10 --mock 
-          
+              mv *-test-run.json replicate_test.json
           - name: Upload Test Results Files
             uses: actions/upload-artifact@v2
             with:
@@ -99,6 +99,13 @@ jobs:
                 automated_detection_testing/ci/detection_testing_batch/prior_config/config_tests_8.json
                 automated_detection_testing/ci/detection_testing_batch/prior_config/config_tests_9.json
 
+          - name: Upload File to Enable Replication of the Test at a Different Time or Place
+            uses: actions/upload-artifact@v2
+            with:
+              name: replicate_test
+              path: |
+                automated_detection_testing/ci/detection_testing_batch/replicate_test.json
+
   docker-detection-testing-execution:
     runs-on: ubuntu-latest
     needs: [validate-tag-if-present, quit-for-dependabot, docker-detection-testing-setup]
@@ -275,9 +282,10 @@ jobs:
               automated_detection_testing/ci/detection_testing_batch/detection_failure_manifest.json
             
         
-        
+        #Always clean these up, they make the output messy
         - name: Clean up intermediate Files
           uses: geekyeggo/delete-artifact@v1
+          if: always()
           with:
               name: |
                 config_tests_0.json.results

From 9700571e0c8889ff9da322b678bf83d47b79462f Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Tue, 30 Nov 2021 13:43:25 -0800
Subject: [PATCH 113/166] Set required version of jsonschema.  Added an
 --interactive_failure command line option. Fixed a typo and removed non
 required jsonschema library

---
 .../detection_testing_execution.py             |  2 +-
 .../modules/new_arguments2.py                  |  8 +++++++-
 .../modules/validate_args.py                   |  2 --
 .../detection_testing_batch/requirements.txt   |  2 +-
 .../detection_testing_batch/summarize_json.py  | 18 +++++++++---------
 5 files changed, 18 insertions(+), 14 deletions(-)

diff --git a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
index 7207981984..efff34381f 100644
--- a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
+++ b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
@@ -198,7 +198,7 @@ def finish_mock(settings: dict, detections: list[str], output_file_template: str
         # We want to make the prior_config directory and the prior_config/apps directory
         os.makedirs("prior_config/apps")
     except FileExistsError as e:
-        print("Directory priorconfig/apps exists, but we just deleted it!\m\tQuitting...", file=sys.stderr)
+        print("Directory priorconfig/apps exists, but we just deleted it!\n\tQuitting...", file=sys.stderr)
         return False
     except Exception as e:
         print("Some error occured when trying to make the configs folder: [%s]\n\tQuitting..." % (
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/new_arguments2.py b/automated_detection_testing/ci/detection_testing_batch/modules/new_arguments2.py
index 838276ce0f..e93a49e807 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/new_arguments2.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/new_arguments2.py
@@ -187,6 +187,12 @@ def parse(args) -> tuple[str, dict]:
     run_parser.add_argument("-n", "--num_containers", required=False, type=int,
                             help="The number of Splunk containers to run or mock")
 
+    run_parser.add_argument("-i", "--interactive_failure", required=False,
+                            action="store_true",
+                            help="After a detection fails, pause and allow the user to log into "\
+                            "the Splunk server to interactively debug the failure.  Wait for them "\
+                            "to hit enter before removing the test data and moving on to the next test.")
+
     args = parser.parse_args()
 
 
@@ -202,7 +208,7 @@ def parse(args) -> tuple[str, dict]:
             # set their value to something else, like None
 
             # Don't overwite booleans
-            if args.__dict__[key] is False and key in ["show_splunk_app_password", "mock"]:
+            if args.__dict__[key] is False and key in ["show_splunk_app_password", "mock", "interactive_failure"]:
                 del args.__dict__[key]
             # Don't overwrite other values
             elif args.__dict__[key] is None and key in ["splunkbase_username", "branch", "commit_hash",
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py b/automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py
index 0602ca92d5..86a4de37ee 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py
@@ -2,8 +2,6 @@ import argparse
 import copy
 import io
 import json
-import jsonschema
-import jsonschema.exceptions
 import modules.jsonschema_errorprinter as jsonschema_errorprinter
 import sys
 from typing import Union
diff --git a/automated_detection_testing/ci/detection_testing_batch/requirements.txt b/automated_detection_testing/ci/detection_testing_batch/requirements.txt
index defe6be649..e134bc79f9 100644
--- a/automated_detection_testing/ci/detection_testing_batch/requirements.txt
+++ b/automated_detection_testing/ci/detection_testing_batch/requirements.txt
@@ -14,4 +14,4 @@ splunk-packaging-toolkit==1.0.1
 docker==5.0.3
 
 #For help getting and parsing the configuration
-jsonschema
+jsonschema==4.2.1
diff --git a/automated_detection_testing/ci/detection_testing_batch/summarize_json.py b/automated_detection_testing/ci/detection_testing_batch/summarize_json.py
index 2aa1a7a277..d55b7c46e3 100644
--- a/automated_detection_testing/ci/detection_testing_batch/summarize_json.py
+++ b/automated_detection_testing/ci/detection_testing_batch/summarize_json.py
@@ -18,7 +18,7 @@ def outputResultsJSON(output_filename:str, data:list[dict], baseline:OrderedDict
         
         
         #A failure or an error
-        fail_only_count = len([x for x in data if x['success'] == False])
+        fail_count = len([x for x in data if x['success'] == False])
         
         #An error (every error is also a failure)
         fail_and_error_count = len([x for x in data if x['error'] == True])
@@ -32,18 +32,18 @@ def outputResultsJSON(output_filename:str, data:list[dict], baseline:OrderedDict
             print("Error - a test was successful, but also included an error. This should be impossible.",file=sys.stderr)
             success = False
             
-        if test_count != (pass_count + fail_only_count):
-            print("Error - the total tests [%d] does not equal the pass[%d]/fails[%d]"%(test_count, pass_count,fail_only_count))
+        if test_count != (pass_count + fail_count):
+            print("Error - the total tests [%d] does not equal the pass[%d]/fails[%d]"%(test_count, pass_count,fail_count))
             success=False
 
-        if fail_only_count > 0:
-            result = "FAIL for %d detections"%(fail_only_count)
+        if fail_count > 0:
+            result = "FAIL for %d detections"%(fail_count)
             success = False
         else:
             result = "PASS for all %d detections"%(pass_count)
 
         summary={"TOTAL_TESTS": test_count, "TESTS_PASSED": pass_count, 
-                 "TOTAL_FAILURES": fail_only_count, "FAIL_ONLY": fail_without_error_count, 
+                 "TOTAL_FAILURES": fail_count, "FAIL_ONLY": fail_without_error_count, 
                  "FAIL_AND_ERROR":fail_and_error_count }
 
         data_sorted = sorted(data, key = lambda k: (-k['error'], k['success'], k['detection_file']))
@@ -71,7 +71,7 @@ def outputResultsJSON(output_filename:str, data:list[dict], baseline:OrderedDict
         #success = False
         #return success, False
 
-    return success, test_count, pass_count, fail_and_error_count
+    return success, test_count, pass_count, fail_count
 
 
 
@@ -99,11 +99,11 @@ try:
         else:
             all_data['results'] = data['results']
 
-    test_pass, test_count, pass_count, fail_and_error_count = outputResultsJSON(args.output_filename, all_data['results'], all_data['baseline'])
+    test_pass, test_count, pass_count, fail_count = outputResultsJSON(args.output_filename, all_data['results'], all_data['baseline'])
     print("Summary:"\
           "\n\tTotal Tests: %d"\
           "\n\tTotal Pass : %d"\
-          "\n\tTotal Fail : %d"%(test_count, pass_count, fail_and_error_count))
+          "\n\tTotal Fail : %d"%(test_count, pass_count, fail_count))
     if not test_pass:
         print("Result: FAIL")
         sys.exit(1)

From fee83d2cfd05d4e9e3a0d7d2ec80f39e4a6c41a8 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Tue, 30 Nov 2021 14:22:34 -0800
Subject: [PATCH 114/166] Check for splunkbase_username and splunkbase_password
 immediately if Splunkbase apps are to be installed and fail with a
 descriptive error if they are not provided.

---
 .../detection_testing_execution.py            | 20 +++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
index efff34381f..c990b066ba 100644
--- a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
+++ b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
@@ -287,6 +287,26 @@ def main(args: list[str]):
             docker.client.from_env()
         except Exception as e:
             print("Error, failed to get docker client.  Is Docker Installed and Running?\n\t%s" % (str(e)))
+        
+        credentials_needed = False
+        credential_error = False
+        if len(settings['splunkbase_apps']) > 0:
+            credentials_needed = True
+        
+        
+        if settings['splunkbase_username'] == None and credentials_needed:
+            print("Error - you have listed apps to download from Splunkbase but have "\
+                  "not provided --splunkbase_username via the command line or config file.",file=sys.stderr)
+            credential_error = True
+
+        if settings['splunkbase_password'] == None and credentials_needed:
+            print("Error - you have listed apps to download from Splunkbase but have "\
+                    "not provided --splunkbase_password via the command line or config file.",file=sys.stderr)
+            credential_error = True
+        
+        if credential_error:
+            print("Please supply the required credentials to continue.\n\tQuitting...",file=sys.stderr)
+            sys.exit(1)
 
     
     FULL_DOCKER_HUB_CONTAINER_NAME = "splunk/splunk:%s" % settings['container_tag']

From c9c2f7c3e37d89f2e1c119c5e3f5fd384e1e5f50 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Tue, 30 Nov 2021 19:32:52 -0800
Subject: [PATCH 115/166] Initial support for testing if a container crashes or
 fails to start during setup. In that case, the whole test will finish, the
 other containers will be stopped, and we will fail.

---
 .../detection_testing_execution.py            | 11 +++-
 .../modules/container_manager.py              | 15 ++++-
 .../modules/splunk_container.py               | 58 ++++++++++++-----
 .../modules/splunk_sdk.py                     | 22 +++----
 .../modules/test_driver.py                    | 63 +++++++++++++++++--
 5 files changed, 133 insertions(+), 36 deletions(-)

diff --git a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
index c990b066ba..4e41da9a87 100644
--- a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
+++ b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
@@ -416,7 +416,16 @@ def main(args: list[str]):
                                             reuse_image=settings['reuse_image'],
                                             interactive_failure=settings['interactive_failure'])
 
-    cm.run_test()
+    result = cm.run_test()
+    
+    #Return code indicates whether testing succeeded and all tests were run.
+    #It does NOT indicate that all tests passed!
+    if result is True:
+        print("Test Execution Successful")
+        sys.exit(0)
+    else:
+        print("Test Execution Failed - review the logs for more details")
+        sys.exit(1)
 
 
 if __name__ == "__main__":
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/container_manager.py b/automated_detection_testing/ci/detection_testing_batch/modules/container_manager.py
index 5e0cd67cc5..1794f1f6fb 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/container_manager.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/container_manager.py
@@ -97,7 +97,7 @@ class ContainerManager:
             self.baseline[key] = self.splunkbase_apps[key]
 
 
-    def run_test(self):
+    def run_test(self)->bool:
         self.run_containers()
         self.run_status_thread()
         for container in self.containers:
@@ -116,7 +116,7 @@ class ContainerManager:
         duration = stop_time - self.start_time
         self.baseline['TEST_DURATION'] = str(duration - datetime.timedelta(microseconds=duration.microseconds))
 
-        self.synchronization_object.finish(self.baseline)
+        return self.synchronization_object.finish(self.baseline)
 
 
 
@@ -211,9 +211,18 @@ class ContainerManager:
 
     def queue_status_thread(self)->None:
         #This will run fo
-        while True:      
+        while True:
+            print("running in queue status thread")
+            if self.synchronization_object.checkContainerFailure():
+                print("One of the containers has shut down prematurely and generated an exception. Shut down the rest of the containers.")
+                for container in self.containers:
+                    container.stopContainer()
+                print("All containers stopped")
+                print("return from queue status thread")
+                return None
             if self.synchronization_object.summarize() == False:
                 #There are no more tests to run, so we can return from this thread
+                print("return from queue status thread")
                 return None
             time.sleep(10)
 
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/splunk_container.py b/automated_detection_testing/ci/detection_testing_batch/modules/splunk_container.py
index f6d43ee6ce..329876f800 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/splunk_container.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/splunk_container.py
@@ -60,9 +60,10 @@ class SplunkContainer:
         self.container = self.make_container()
 
         self.thread = threading.Thread(target=self.run_container, )
+        
 
-        self.container_start_time = 0
-        self.test_start_time = 0
+        self.container_start_time = -1
+        self.test_start_time = -1
         self.num_tests_completed = 0
 
     def prepare_apps_path(
@@ -188,6 +189,21 @@ class SplunkContainer:
         )
         return successful_copy
 
+    def stopContainer(self,timeout=10) -> bool:
+        try:        
+            container = self.client.containers.get(self.container_name)
+            #Note that stopping does not remove any of the volumes or logs,
+            #so stopping can be useful if we want to debug any container failure 
+            container.stop(timeout=10)
+            self.synchronization_object.containerFailure()
+            return True
+
+        except Exception as e:
+            # Container does not exist, or we could not get it. Throw and error
+            print("Error stopping docker container [%s]"%(self.container_name))
+            return False
+        
+
     def removeContainer(
         self, removeVolumes: bool = True, forceRemove: bool = True
     ) -> bool:
@@ -211,9 +227,6 @@ class SplunkContainer:
 
     def get_container_summary(self) -> str:
         current_time = timeit.default_timer()
-        # Get rid of the decimal (microseconds) so that we have whole seconds
-        if self.container_start_time is None or self.test_start_time is None:
-            print(self.container_start_time)
 
         # Total time the container has been running
         if self.container_start_time == -1:
@@ -248,11 +261,11 @@ class SplunkContainer:
             testing_time_string = "%s (%d tests @ %s per test)" % (
                 testing_seconds_rounded, self.num_tests_completed, timedelta_per_test_rounded)
 
-        summary_str = "Summary\n\t"\
+        summary_str = "Summary for %s\n\t"\
                       "Total Time          : [%s]\n\t"\
                       "Container Start Time: [%s]\n\t"\
                       "Test Execution Time : [%s]" % (
-                          total_time_string, setup_time_string, testing_time_string)
+                          self.container_name, total_time_string, setup_time_string, testing_time_string)
 
         return summary_str
 
@@ -261,6 +274,7 @@ class SplunkContainer:
         max_seconds: int = 300,
         seconds_between_attempts: int = 5,
     ) -> bool:
+        
         # The smarter version of this will try to hit one of the pages,
         # probably the login page, and when that is available it means that
         # splunk is fully started and ready to go.  Until then, we just
@@ -271,18 +285,21 @@ class SplunkContainer:
             try:
                 # Splunk container will not have proper ssl certificate
                 response = requests.get(
-                    splunk_ready_url, timeout=5, verify=False)
+                    splunk_ready_url, timeout=seconds_between_attempts, verify=False)
                 response.raise_for_status()
                 return True
             except Exception as e:
                 elapsed = timeit.default_timer() - start
+                
                 if elapsed > max_seconds:
+                    self.stopContainer()
                     raise (
                         Exception(
-                            "Container [%s] took longer than maximum start time of [%d].\n\tQuitting..."
+                            "Container [%s] took longer than maximum start time of [%d].\n\tStopping container..."
                             % (self.container_name, max_seconds)
                         )
                     )
+                    
             time.sleep(seconds_between_attempts)
 
     def run_container(self) -> None:
@@ -299,9 +316,7 @@ class SplunkContainer:
         print("Finished copying files to [%s]" % (self.container_name))
 
         try:
-            while not splunk_sdk.enable_delete_for_admin(
-                self.splunk_ip, self.management_port, self.container_password
-            ):
+            while not splunk_sdk.enable_delete_for_admin(self.splunk_ip, self.management_port, self.container_password):
                 time.sleep(10)
         except Exception as e:
             print(
@@ -316,12 +331,25 @@ class SplunkContainer:
         )
 
         self.synchronization_object.start_barrier.wait()
-        self.wait_for_splunk_ready()
+        
+        try:
+            self.wait_for_splunk_ready()
+        except Exception as e:
+            print("Error starting docker container: [%s]"%(str(e)))
+            return None
+        
 
         # Sleep for a small random time so that containers drift apart and don't synchronize their testing
         time.sleep(random.randint(1, 30))
+        
         self.test_start_time = timeit.default_timer()
         while True:
+            print("%s is getting dertection"%(self.container_name))
+            if self.synchronization_object.checkContainerFailure():
+                self.container.stop()
+                print("Container [%s] successfully stopped early due to failure" % (self.container_name))
+                return None
+
             # Try to get something from the queue
             detection_to_test = self.synchronization_object.getTest()
             if detection_to_test is None:
@@ -330,9 +358,7 @@ class SplunkContainer:
                         "Container [%s] has finished running detections, time to stop the container."
                         % (self.container_name)
                     )
-                    self.container.stop()
-                    print("Container [%s] successfully stopped" %
-                          (self.container_name))
+                    
                     # remove the container
                     self.removeContainer()
                 except Exception as e:
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/splunk_sdk.py b/automated_detection_testing/ci/detection_testing_batch/modules/splunk_sdk.py
index 69218ed9be..f51231ce5f 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/splunk_sdk.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/splunk_sdk.py
@@ -15,8 +15,8 @@ def enable_delete_for_admin(splunk_host, splunk_port, splunk_password):
             password=splunk_password
         )
     except Exception as e:
-        print("Unable to connect to Splunk instance: " + str(e))
-        return 1, {}
+        raise(Exception("Unable to connect to Splunk instance: " + str(e)))
+        
 
     # search and replace \\ with \\\
     # search = search.replace('\\','\\\\')
@@ -30,7 +30,7 @@ def enable_delete_for_admin(splunk_host, splunk_port, splunk_password):
 
 
 
-def test_baseline_search(splunk_host, splunk_port, splunk_password, search, pass_condition, baseline_name, baseline_file, earliest_time, latest_time):
+def test_baseline_search(splunk_host, splunk_port, splunk_password, search, pass_condition, baseline_name, baseline_file, earliest_time, latest_time)->dict:
     try:
         service = client.connect(
             host=splunk_host,
@@ -39,8 +39,8 @@ def test_baseline_search(splunk_host, splunk_port, splunk_password, search, pass
             password=splunk_password
         )
     except Exception as e:
-        print("Unable to connect to Splunk instance: " + str(e))
-        return 1, {}
+        raise(Exception("Unable to connect to Splunk instance: " + str(e)))
+        
 
     # search and replace \\ with \\\
     # search = search.replace('\\','\\\\')
@@ -59,8 +59,8 @@ def test_baseline_search(splunk_host, splunk_port, splunk_password, search, pass
     try:
         job = service.jobs.create(splunk_search, **kwargs)
     except Exception as e:
-        print("Unable to execute baseline: " + str(e))
-        return 1, {}
+        raise(Exception("Unable to execute baseline: " + str(e)))
+        
 
     test_results = dict()
     test_results['diskUsage'] = job['diskUsage']
@@ -104,7 +104,7 @@ def test_detection_search(splunk_host:str, splunk_port:int, splunk_password:str,
             password=splunk_password
         )
     except Exception as e:
-        print("Unable to connect to Splunk instance: " + str(e))
+        print("Unable to connect to Splunk instance: " + str(e),file=sys.stderr)
         test_results['error'] = True
         return test_results
 
@@ -152,8 +152,7 @@ def delete_attack_data(splunk_host:str, splunk_password:str, splunk_port:int, wa
             password=splunk_password
         )
     except Exception as e:
-        print("Unable to connect to Splunk instance: " + str(e))
-        return False
+        raise(Exception("Unable to connect to Splunk instance: " + str(e)))
 
     #splunk_search = 'search index=test* | delete'
     if wait_on_delete:
@@ -169,7 +168,6 @@ def delete_attack_data(splunk_host:str, splunk_password:str, splunk_port:int, wa
     try:
         job = service.jobs.create(splunk_search, **kwargs)
     except Exception as e:
-        print("Unable to execute search: " + str(e))
-        return False
+        raise(Exception("Unable to execute search: " + str(e)))
     
     return True
\ No newline at end of file
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/test_driver.py b/automated_detection_testing/ci/detection_testing_batch/modules/test_driver.py
index 171a524598..69ce72efd6 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/test_driver.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/test_driver.py
@@ -10,7 +10,7 @@ import threading
 import time
 import timeit
 from typing import Union
-
+import sys
 
 class TestDriver:
     def __init__(self, tests:list[str], num_containers:int):
@@ -28,12 +28,47 @@ class TestDriver:
         self.errors = []
         self.container_ready_time = None
         
+        #No containers have failed
+        self.container_failure = False
+
         #Just make a random folder to store attack data that we donwload
         self.attack_data_root_folder = tempfile.mkdtemp(prefix="attack_data_", dir=os.getcwd())
         print("Attack data for this run will be stored at: [%s]"%(self.attack_data_root_folder))
         self.start_barrier = threading.Barrier(num_containers)
 
+
+    def checkContainerFailure(self)->bool:
+        
+        self.lock.acquire()
+        
+        try:
+            result = self.container_failure
+        finally:
+            self.lock.release()
+        
+        
+        
+        return result
+        
+
+    def containerFailure(self)->None:
+        self.lock.acquire()
+        try:
+            self.container_failure = True
+        finally:
+            self.lock.release()
+        
+
     def getTest(self)-> Union[str,None]:
+        
+        failure = self.checkContainerFailure()
+        
+        
+
+        if failure:
+            #Just return None, don't continue testing if a container crashed
+            return None
+
         try:
             return self.testing_queue.get(block=False)
         except Exception as e:
@@ -142,6 +177,13 @@ class TestDriver:
     def finish(self, baseline:OrderedDict):
         self.cleanup()
         self.outputResultsFiles(baseline)
+
+        if self.checkContainerFailure():
+            print("One or more containers crashed, so testing did not complete successfully. We wrote out the results we have")
+            return False
+        else:
+            return True
+
         
 
     def cleanup(self):
@@ -154,15 +196,25 @@ class TestDriver:
             self.lock.release()
     def summarize(self)->bool:
         
+        if self.checkContainerFailure() == True:
+            print("Error running containers... shutting down", file=sys.stderr)
+            return False
+        
+        
         self.lock.acquire()
         
         try:
+        
             current_time = timeit.default_timer()
+            
+            
+
             if self.testing_queue.qsize() == self.total_number_of_tests:
                 #Testing has not started yet. We are setting up containers
                 print("***********PROGRESS UPDATE***********\n"\
-                      "\tWaiting for container setup: %s"%(datetime.timedelta(seconds=current_time - self.start_time)))
+                      "\tWaiting for container setup: %s\n"%(datetime.timedelta(seconds=current_time - self.start_time)))
             else:
+                
                 if self.container_ready_time is None:
                     #This is the first status update since container setup has completed.  Get the current time.
                     #This makes our remaining time estimates better since that estimate should not involve
@@ -210,10 +262,13 @@ class TestDriver:
             print("Error in printing execution summary: [%s]"%(str(e)))
         finally:
             self.lock.release()
+            
         
         #Return true while there are tests remaining
-        return (self.total_number_of_tests - 
-                (len(self.successes) + len(self.failures) + len(self.errors)) > 0)
+        completed_tests = len(self.successes) + len(self.failures) + len(self.errors)
+        remaining_tests = self.total_number_of_tests - completed_tests
+        return remaining_tests > 0
+                
         
         
     def addResult(self, result:dict)->None:

From c085327a721f3ca074628f7275f11336b2cf005f Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Tue, 30 Nov 2021 19:33:46 -0800
Subject: [PATCH 116/166] Change the detection testing mode to all for a test
 of everything.

---
 .github/workflows/docker-detection-testing.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/docker-detection-testing.yml b/.github/workflows/docker-detection-testing.yml
index a565b48527..12922767be 100644
--- a/.github/workflows/docker-detection-testing.yml
+++ b/.github/workflows/docker-detection-testing.yml
@@ -80,7 +80,7 @@ jobs:
               cd automated_detection_testing/ci/detection_testing_batch
               source .venv/bin/activate
               
-              python3 detection_testing_execution.py run --branch DetectionBaselineFixes  --mode changes --num_containers 10 --mock 
+              python3 detection_testing_execution.py run --branch develop  --mode all --num_containers 10 --mock 
               mv *-test-run.json replicate_test.json
           - name: Upload Test Results Files
             uses: actions/upload-artifact@v2

From 859bd9c6da7e773202ed6f9ecb92b71f09600171 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Wed, 1 Dec 2021 11:00:04 -0800
Subject: [PATCH 117/166] Some small changes dealing with errors during a test.

---
 .../modules/testing_service.py                  | 17 +++++++----------
 1 file changed, 7 insertions(+), 10 deletions(-)

diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/testing_service.py b/automated_detection_testing/ci/detection_testing_batch/modules/testing_service.py
index 4134a87c78..b8b0866d5f 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/testing_service.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/testing_service.py
@@ -39,15 +39,13 @@ def test_detection_wrapper(container_name:str, splunk_ip:str, splunk_password:st
 
 
 def test_detection(splunk_ip:str, splunk_port:int, container_name:str, splunk_password:str, test_file:str, uuid_var, attack_data_root_folder)->Union[dict,None]:
-    try:
-        test_file_obj = load_file(os.path.join("security_content/", test_file))
-    except Exception as e:
-        print('Error: ' + str(e))
-        return None
+    
+    test_file_obj = load_file(os.path.join("security_content/", test_file))
+    
     
     if not test_file_obj:
         print("Not test_file_obj!")
-        return None
+        raise(Exception("No test file object found for [%s]"%(test_file)))
     #print(test_file_obj)
 
     # write entry dynamodb
@@ -75,7 +73,7 @@ def test_detection(splunk_ip:str, splunk_port:int, container_name:str, splunk_pa
                 data_manipulation.manipulate_timestamp(target_file, attack_data['sourcetype'], attack_data['source'])
         replay_attack_dataset(container_name, splunk_password, folder_name, "main", attack_data['sourcetype'], attack_data['source'], attack_data['file_name'])
     
-    time.sleep(30)
+    #time.sleep(30)
     
     result_test = {}
     test = test_file_obj['tests'][0]
@@ -115,15 +113,14 @@ def test_detection(splunk_ip:str, splunk_port:int, container_name:str, splunk_pa
 
 def load_file(file_path):
     try:
-        #print("Opening file path [%s]"%(file_path))
         
         with open(file_path, 'r', encoding="utf-8") as stream:
             try:
                 file = list(yaml.safe_load_all(stream))[0]
             except yaml.YAMLError as exc:
-                raise(Exception("ERROR: reading {0}:[{1}]".format(file_path, str(exc))))
+                raise(Exception("ERROR: parsing YAML for {0}:[{1}]".format(file_path, str(exc))))
     except Exception as e:
-        raise(Exception("ERROR: reading {0}:[{1}]".format(file_path, str(e))))
+        raise(Exception("ERROR: opening {0}:[{1}]".format(file_path, str(e))))
     return file
 
 

From 531d550b3be39dfb97b7ced6f7350f57a4fb55b6 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Wed, 1 Dec 2021 14:44:51 -0800
Subject: [PATCH 118/166] Just change the workflow to explicitly test the
 RegistryDetectionFixes branch.

---
 .github/workflows/docker-detection-testing.yml                 | 2 +-
 .../ci/detection_testing_batch/modules/container_manager.py    | 1 -
 .../ci/detection_testing_batch/modules/splunk_container.py     | 1 -
 .../ci/detection_testing_batch/modules/testing_service.py      | 3 ++-
 4 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/docker-detection-testing.yml b/.github/workflows/docker-detection-testing.yml
index 12922767be..351ba245f9 100644
--- a/.github/workflows/docker-detection-testing.yml
+++ b/.github/workflows/docker-detection-testing.yml
@@ -80,7 +80,7 @@ jobs:
               cd automated_detection_testing/ci/detection_testing_batch
               source .venv/bin/activate
               
-              python3 detection_testing_execution.py run --branch develop  --mode all --num_containers 10 --mock 
+              python3 detection_testing_execution.py run --branch RegistryDetectionFixes  --mode changes --num_containers 10 --mock 
               mv *-test-run.json replicate_test.json
           - name: Upload Test Results Files
             uses: actions/upload-artifact@v2
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/container_manager.py b/automated_detection_testing/ci/detection_testing_batch/modules/container_manager.py
index 1794f1f6fb..15d82b7842 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/container_manager.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/container_manager.py
@@ -212,7 +212,6 @@ class ContainerManager:
     def queue_status_thread(self)->None:
         #This will run fo
         while True:
-            print("running in queue status thread")
             if self.synchronization_object.checkContainerFailure():
                 print("One of the containers has shut down prematurely and generated an exception. Shut down the rest of the containers.")
                 for container in self.containers:
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/splunk_container.py b/automated_detection_testing/ci/detection_testing_batch/modules/splunk_container.py
index 329876f800..e0462d2b23 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/splunk_container.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/splunk_container.py
@@ -344,7 +344,6 @@ class SplunkContainer:
         
         self.test_start_time = timeit.default_timer()
         while True:
-            print("%s is getting dertection"%(self.container_name))
             if self.synchronization_object.checkContainerFailure():
                 self.container.stop()
                 print("Container [%s] successfully stopped early due to failure" % (self.container_name))
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/testing_service.py b/automated_detection_testing/ci/detection_testing_batch/modules/testing_service.py
index b8b0866d5f..5ab9271bfa 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/testing_service.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/testing_service.py
@@ -73,7 +73,8 @@ def test_detection(splunk_ip:str, splunk_port:int, container_name:str, splunk_pa
                 data_manipulation.manipulate_timestamp(target_file, attack_data['sourcetype'], attack_data['source'])
         replay_attack_dataset(container_name, splunk_password, folder_name, "main", attack_data['sourcetype'], attack_data['source'], attack_data['file_name'])
     
-    #time.sleep(30)
+    #Allow some time for the data to be ingested and processed
+    time.sleep(60)
     
     result_test = {}
     test = test_file_obj['tests'][0]

From 6457b505db7a1f1c3a70d4f3c7b02d6c819a0e16 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Wed, 1 Dec 2021 15:58:31 -0800
Subject: [PATCH 119/166] When there is an error checking out a git PR.  If the
 PR number is bad, we will exit.  Also updated the CI/CD to test all of
 develop again. We will look at the results after we have increased the wait
 time after loading data a bit.

---
 .github/workflows/docker-detection-testing.yml         |  2 +-
 .../detection_testing_execution.py                     | 10 +++++++---
 .../detection_testing_batch/modules/github_service.py  |  4 +++-
 .../ci/detection_testing_batch/modules/splunk_sdk.py   |  1 -
 4 files changed, 11 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/docker-detection-testing.yml b/.github/workflows/docker-detection-testing.yml
index 351ba245f9..12922767be 100644
--- a/.github/workflows/docker-detection-testing.yml
+++ b/.github/workflows/docker-detection-testing.yml
@@ -80,7 +80,7 @@ jobs:
               cd automated_detection_testing/ci/detection_testing_batch
               source .venv/bin/activate
               
-              python3 detection_testing_execution.py run --branch RegistryDetectionFixes  --mode changes --num_containers 10 --mock 
+              python3 detection_testing_execution.py run --branch develop  --mode all --num_containers 10 --mock 
               mv *-test-run.json replicate_test.json
           - name: Upload Test Results Files
             uses: actions/upload-artifact@v2
diff --git a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
index 4e41da9a87..7efbdc1675 100644
--- a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
+++ b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
@@ -317,9 +317,13 @@ def main(args: list[str]):
               "[%d].  We will do what you asked, but be warned!" % (settings['num_containers'], MAX_RECOMMENDED_CONTAINERS_BEFORE_WARNING))
 
     # Check out security content if required
-    github_service = ensure_security_content(
-        settings['branch'], settings['commit_hash'], settings['pr_number'], settings['persist_security_content'])
-    settings['commit_hash'] = github_service.commit_hash
+    try:
+        github_service = ensure_security_content(
+            settings['branch'], settings['commit_hash'], settings['pr_number'], settings['persist_security_content'])
+        settings['commit_hash'] = github_service.commit_hash
+    except Exception as e:
+        print("Failure checking out git repository: %s\n\tQuitting",file=sys.stderr)
+        sys.exit(1)
 
     # Make a backup of this config containing the hash and stripped credentials.
     # This makes the test perfectly reproducible.
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/github_service.py b/automated_detection_testing/ci/detection_testing_batch/modules/github_service.py
index 1a083527bb..15f5e5acf7 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/github_service.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/github_service.py
@@ -37,8 +37,10 @@ class GithubService:
             sys.exit()
 
         elif PR_number:
-            subprocess.call(["git", "-C", "security_content/", "fetch", "origin",
+            ret = subprocess.call(["git", "-C", "security_content/", "fetch", "origin",
                             "refs/pull/%d/head:%s" % (PR_number, security_content_branch)])
+            if ret != 0:
+                raise(Exception("Error checking out repository"))
 
         # No checking to see if the hash is to a commit inside of the branch - the user
         # has to do that by hand
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/splunk_sdk.py b/automated_detection_testing/ci/detection_testing_batch/modules/splunk_sdk.py
index f51231ce5f..2294b27501 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/splunk_sdk.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/splunk_sdk.py
@@ -122,7 +122,6 @@ def test_detection_search(splunk_host:str, splunk_port:int, splunk_password:str,
         test_results['error'] = True
         return test_results
 
-    
     test_results['diskUsage'] = job['diskUsage']
     test_results['runDuration'] = job['runDuration']
     test_results['detection_name'] = detection_name

From 17726fcdff84449bb950330f4649fb824cf2bd45 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Thu, 2 Dec 2021 10:00:01 -0800
Subject: [PATCH 120/166] Running a test with the old sysmon to compare the
 results.

---
 .../ci/detection_testing_batch/modules/validate_args.py     | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py b/automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py
index 86a4de37ee..b8dc120639 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py
@@ -177,9 +177,9 @@ setup_schema = {
                     "app_number": 833,
                     "app_version": "8.3.1"
                 },
-                "SPLUNK_ADD_ON_FOR_SYSMON": {
-                    "app_number": 5709,
-                    "app_version": "1.0.1"
+                "SPLUNK_ADD_ON_FOR_SYSMON_OLD": {
+                    "app_number": 1914,
+                    "app_version": "10.6.2"
                 },
                 "SPLUNK_COMMON_INFORMATION_MODEL": {
                     "app_number": 1621,

From 9be4d1a4eb04bafd026b00ac9acb2391a1330194 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Mon, 6 Dec 2021 11:41:35 -0800
Subject: [PATCH 121/166] Large changes to json validation for local_apps. json
 validation was wrong. also changed CI to run on push and on PR using the new
 CI job!  Note that it will still require lots of testing.

---
 .github/workflows/build-and-validate.yml      | 282 +++++++++++++++++-
 .../workflows/docker-detection-testing.yml    |  62 ++--
 .../modules/splunk_container.py               |   6 +-
 .../modules/testing_service.py                |   2 +-
 .../modules/validate_args.py                  | 130 ++++----
 .../test_config_github_actions.json           |  91 ++++++
 6 files changed, 494 insertions(+), 79 deletions(-)
 create mode 100644 automated_detection_testing/ci/detection_testing_batch/test_config_github_actions.json

diff --git a/.github/workflows/build-and-validate.yml b/.github/workflows/build-and-validate.yml
index 9943233b53..a3946e7b33 100644
--- a/.github/workflows/build-and-validate.yml
+++ b/.github/workflows/build-and-validate.yml
@@ -293,7 +293,287 @@ jobs:
             build/DA-ESS_AmazonWebServices_Content-latest.tar.gz
             build/dev_sec_ops_analytics-latest.tar.gz
             build/checksum.txt
-      
+
+
+  docker-detection-testing-setup:
+    runs-on: ubuntu-latest
+    needs: [build-package]
+    steps:  
+        - name: Get branch and PR required for detection testing main.py
+          id: vars
+          run: |
+            echo "::set-output name=branch::${GITHUB_REF#refs/heads/}"
+
+        - name: Checkout Repo
+          uses: actions/checkout@v2
+        
+        #- name: Install requirements for installing slim during execution 
+        #  run: |
+        #    sudo apt update -qq
+        #    #python2.7 needed for slim, for now
+        #    sudo apt install python2
+        #    sudo apt install virtualenv 
+        #    curl https://bootstrap.pypa.io/pip/2.7/get-pip.py --output get-pip.py
+        #    sudo python2.7 get-pip.py
+
+        # Get the previously built ESCU
+        - name: Get ESCU
+          uses: actions/download-artifact@v2
+          with:
+            name: content-latest
+            path: automated_detection_testing/ci/detection_testing_batch/prior_config/apps
+
+        - uses: actions/setup-python@v2
+          with:
+            python-version: '3.9' #Available versions here - https://github.com/actions/python-versions/releases  easy to change/make a matrix/use pypy
+            architecture: 'x64' # optional x64 or x86. Defaults to x64 if not specified          
+        
+        - name: Install Python Dependencies
+          run: |
+            cd automated_detection_testing/ci/detection_testing_batch
+            python3 -m venv .venv
+            source .venv/bin/activate
+            python3 -m pip install wheel
+            python3 -m pip install -r requirements.txt
+        
+        - name: Run the CI
+          run: |
+            cd automated_detection_testing/ci/detection_testing_batch
+            source .venv/bin/activate
+            echo "github.event.issue.pull_request    : [${{ github.event.issue.pull_request }}]"
+            echo "github.event.pull_request.number   : [${{ github.event.pull_request.number }}]"
+            echo "steps.vars.outputs.branch          : [${{ steps.vars.outputs.branch }}]"
+            echo "github.event.pull_request.head.ref: [${{ github.event.pull_request.head.ref }}]"
+
+            ls -lahr prior_config
+            if [[ ! -z "${{  github.event.pull_request.head.ref }}" &&  ! -z "${{ github.event.pull_request.number }}" ]]; then
+              echo "Pull request from source branch [${{ github.event.pull_request.head.ref }}] for PR number [${{ github.event.issue.number }}]"
+              python detection_testing_execution.py run --branch ${{ github.event.pull_request.head.ref }} --pr_number ${{ github.event.pull_request.number }}  --mode changes --mock --config_fig test_config_github_actions.json  
+            else
+              echo "Push from branch [${{ steps.vars.outputs.branch }}]"
+              python detection_testing_execution.py run --branch  [${{ steps.vars.outputs.branch }}  --mode changes --mock --config_file test_config_github_actions.json
+            fi
+
+            mv *-test-run.json replicate_test.json
+        - name: Upload Test Results Files
+          uses: actions/upload-artifact@v2
+          with:
+            name: testing-results-config
+            path: |
+              automated_detection_testing/ci/detection_testing_batch/prior_config/apps/DA-ESS-ContentUpdate-latest.tar.gz
+              automated_detection_testing/ci/detection_testing_batch/prior_config/config_tests_0.json
+              automated_detection_testing/ci/detection_testing_batch/prior_config/config_tests_1.json
+              automated_detection_testing/ci/detection_testing_batch/prior_config/config_tests_2.json
+              automated_detection_testing/ci/detection_testing_batch/prior_config/config_tests_3.json
+              automated_detection_testing/ci/detection_testing_batch/prior_config/config_tests_4.json
+              automated_detection_testing/ci/detection_testing_batch/prior_config/config_tests_5.json
+              automated_detection_testing/ci/detection_testing_batch/prior_config/config_tests_6.json
+              automated_detection_testing/ci/detection_testing_batch/prior_config/config_tests_7.json
+              automated_detection_testing/ci/detection_testing_batch/prior_config/config_tests_8.json
+              automated_detection_testing/ci/detection_testing_batch/prior_config/config_tests_9.json
+
+        - name: Upload File to Enable Replication of the Test at a Different Time or Place
+          uses: actions/upload-artifact@v2
+          with:
+            name: replicate_test
+            path: |
+              automated_detection_testing/ci/detection_testing_batch/replicate_test.json
+
+  docker-detection-testing-execution:
+    runs-on: ubuntu-latest
+    needs: [docker-detection-testing-setup]
+    strategy:
+      matrix:
+        manifest_filename: ["config_tests_0.json", 
+                        "config_tests_1.json",
+                        "config_tests_2.json",
+                        "config_tests_3.json",
+                        "config_tests_4.json",
+                        "config_tests_5.json",
+                        "config_tests_6.json",
+                        "config_tests_7.json",
+                        "config_tests_8.json",
+                        "config_tests_9.json"]
+    steps:  
+        - name: Get branch and PR required for detection testing main.py
+          id: vars
+          run: |
+            echo "::set-output name=branch::${GITHUB_REF#refs/heads/}"
+
+        - name: Checkout Repo
+          uses: actions/checkout@v2
+        
+        - name: Download artifacts
+          uses: actions/download-artifact@v2
+          with:
+            name: testing-results-config
+            path: automated_detection_testing/ci/detection_testing_batch/prior_config
+        # - name: Install Docker 
+        #   run: |
+        #     sudo apt update -qq
+            
+
+        #     #python2.7 needed for slim, for now
+        #     sudo apt install python2
+        #     sudo apt install virtualenv 
+        #     curl https://bootstrap.pypa.io/pip/2.7/get-pip.py --output get-pip.py
+        #     sudo python2.7 get-pip.py
+
+        - uses: actions/setup-python@v2
+          with:
+            python-version: '3.9' #Available versions here - https://github.com/actions/python-versions/releases  easy to change/make a matrix/use pypy
+            architecture: 'x64' # optional x64 or x86. Defaults to x64 if not specified          
+        
+        - name: Install Python Dependencies
+          run: |
+            cd automated_detection_testing/ci/detection_testing_batch
+            python3 -m venv .venv
+            source .venv/bin/activate
+            python3 -m pip install wheel
+            python3 -m pip install -r requirements.txt
+        
+        - name: Run the CI
+          run: |
+            cd automated_detection_testing/ci/detection_testing_batch
+            source .venv/bin/activate
+            
+            python3 detection_testing_execution.py run -c prior_config/${{ matrix.manifest_filename}} --splunkbase_username ${{ secrets.SPLUNKBASE_TESTING_USERNAME }} --splunkbase_password ${{ secrets.SPLUNKBASE_TESTING_KEY }} 
+                    
+        
+        - name: Upload Test Results Files
+          uses: actions/upload-artifact@v2
+          with:
+            name: ${{ matrix.manifest_filename}}.results
+            path: |
+              automated_detection_testing/ci/detection_testing_batch/success.csv
+              automated_detection_testing/ci/detection_testing_batch/error.csv
+              automated_detection_testing/ci/detection_testing_batch/failure.csv
+              automated_detection_testing/ci/detection_testing_batch/combined.csv
+              automated_detection_testing/ci/detection_testing_batch/success.json
+              automated_detection_testing/ci/detection_testing_batch/error.json
+              automated_detection_testing/ci/detection_testing_batch/failure.json
+              automated_detection_testing/ci/detection_testing_batch/combined.json
+          
+  docker-detection-testing-execution-merge-results:
+    runs-on: ubuntu-latest
+    needs: [docker-detection-testing-setup, docker-detection-testing-execution]
+ 
+    steps:  
+        - name: Get branch and PR required for detection testing main.py
+          id: vars
+          run: |
+            echo "::set-output name=branch::${GITHUB_REF#refs/heads/}"
+
+        - name: Checkout Repo
+          uses: actions/checkout@v2
+        
+        - name: Download artifacts
+          uses: actions/download-artifact@v2
+          with:
+            name: config_tests_0.json.results
+            path: automated_detection_testing/ci/detection_testing_batch/results_0
+        - name: Download artifacts
+          uses: actions/download-artifact@v2
+          with:
+            name: config_tests_1.json.results
+            path: automated_detection_testing/ci/detection_testing_batch/results_1
+        - name: Download artifacts
+          uses: actions/download-artifact@v2
+          with:
+            name: config_tests_2.json.results
+            path: automated_detection_testing/ci/detection_testing_batch/results_2
+        - name: Download artifacts
+          uses: actions/download-artifact@v2
+          with:
+            name: config_tests_3.json.results
+            path: automated_detection_testing/ci/detection_testing_batch/results_3
+        - name: Download artifacts
+          uses: actions/download-artifact@v2
+          with:
+            name: config_tests_4.json.results
+            path: automated_detection_testing/ci/detection_testing_batch/results_4
+        - name: Download artifacts
+          uses: actions/download-artifact@v2
+          with:
+            name: config_tests_5.json.results
+            path: automated_detection_testing/ci/detection_testing_batch/results_5
+        - name: Download artifacts
+          uses: actions/download-artifact@v2
+          with:
+            name: config_tests_6.json.results
+            path: automated_detection_testing/ci/detection_testing_batch/results_6
+        - name: Download artifacts
+          uses: actions/download-artifact@v2
+          with:
+            name: config_tests_7.json.results
+            path: automated_detection_testing/ci/detection_testing_batch/results_7
+        - name: Download artifacts
+          uses: actions/download-artifact@v2
+          with:
+            name: config_tests_8.json.results
+            path: automated_detection_testing/ci/detection_testing_batch/results_8
+        - name: Download artifacts
+          uses: actions/download-artifact@v2
+          with:
+            name: config_tests_9.json.results
+            path: automated_detection_testing/ci/detection_testing_batch/results_9
+        
+        - uses: actions/setup-python@v2
+          with:
+            python-version: '3.9' #Available versions here - https://github.com/actions/python-versions/releases  easy to change/make a matrix/use pypy
+            architecture: 'x64' # optional x64 or x86. Defaults to x64 if not specified    
+        
+        - name: Install Python Dependencies
+          run: |
+            cd automated_detection_testing/ci/detection_testing_batch
+            python3 -m venv .venv
+            source .venv/bin/activate
+            python3 -m pip install wheel
+            python3 -m pip install -r requirements.txt
+
+        - name: Merge Detections into single File
+          run: |
+            cd automated_detection_testing/ci/detection_testing_batch
+            source .venv/bin/activate
+            python summarize_json.py --files results_*/combined.json --output_filename summary_test_results.json
+            
+            
+        - name: Upload Summary Test Results JSON
+          uses: actions/upload-artifact@v2
+          if: always()
+          with:
+            name: SummaryTestResults
+            path: |
+              automated_detection_testing/ci/detection_testing_batch/summary_test_results.json
+
+        - name: Upload Failures Manifest on Failure
+          uses: actions/upload-artifact@v2
+          if: failure()
+          with:
+            name: DetectionFailureManifest
+            path: |
+              automated_detection_testing/ci/detection_testing_batch/detection_failure_manifest.json
+            
+        
+        #Always clean these up, they make the output messy
+        - name: Clean up intermediate Files
+          uses: geekyeggo/delete-artifact@v1
+          if: always()
+          with:
+              name: |
+                config_tests_0.json.results
+                config_tests_1.json.results
+                config_tests_2.json.results
+                config_tests_3.json.results
+                config_tests_4.json.results
+                config_tests_5.json.results
+                config_tests_6.json.results
+                config_tests_7.json.results
+                config_tests_8.json.results
+                config_tests_9.json.results
+
+
+
 
   #Everything below this line should ONLY run on a tag and nothing else
   #We still want all of the above checks to run and pass before running these
diff --git a/.github/workflows/docker-detection-testing.yml b/.github/workflows/docker-detection-testing.yml
index 12922767be..d898d65180 100644
--- a/.github/workflows/docker-detection-testing.yml
+++ b/.github/workflows/docker-detection-testing.yml
@@ -1,11 +1,12 @@
 name: docker-detection-testing
 on:
-  push:
+  #push:
+  #pull_request:
   #run nightly at 0500 UTC
   #Note the scheduled workflows will only run if the Action has 
   #been committed to the default branch (develop)
-  schedule:
-    - cron: "0 5 * * *"
+  #schedule:
+  #  - cron: "0 5 * * *"
 jobs:
 
   validate-tag-if-present:
@@ -53,14 +54,21 @@ jobs:
           - name: Checkout Repo
             uses: actions/checkout@v2
           
-          - name: Install requirements for installing slim during execution 
-            run: |
-              sudo apt update -qq
-              #python2.7 needed for slim, for now
-              sudo apt install python2
-              sudo apt install virtualenv 
-              curl https://bootstrap.pypa.io/pip/2.7/get-pip.py --output get-pip.py
-              sudo python2.7 get-pip.py
+          #- name: Install requirements for installing slim during execution 
+          #  run: |
+          #    sudo apt update -qq
+          #    #python2.7 needed for slim, for now
+          #    sudo apt install python2
+          #    sudo apt install virtualenv 
+          #    curl https://bootstrap.pypa.io/pip/2.7/get-pip.py --output get-pip.py
+          #    sudo python2.7 get-pip.py
+
+          # Get the previously built ESCU
+          - name: Get ESCU
+            uses: actions/download-artifact@v2
+            with:
+              name: content-latest
+              path: automated_detection_testing/ci/detection_testing_batch/prior_config/apps
 
           - uses: actions/setup-python@v2
             with:
@@ -79,8 +87,20 @@ jobs:
             run: |
               cd automated_detection_testing/ci/detection_testing_batch
               source .venv/bin/activate
-              
-              python3 detection_testing_execution.py run --branch develop  --mode all --num_containers 10 --mock 
+              echo "github.event.issue.pull_request    : [${{ github.event.issue.pull_request }}]"
+              echo "github.event.pull_request.number   : [${{ github.event.pull_request.number }}]"
+              echo "steps.vars.outputs.branch          : [${{ steps.vars.outputs.branch }}]"
+              echo "github.event.pull_request.head.ref: [${{ github.event.pull_request.head.ref }}]"
+
+              ls -lahr prior_config
+              if [[ ! -z "${{  github.event.pull_request.head.ref }}" &&  ! -z "${{ github.event.pull_request.number }}" ]]; then
+                echo "Pull request from source branch [${{ github.event.pull_request.head.ref }}] for PR number [${{ github.event.issue.number }}]"
+                python detection_testing_execution.py run --branch ${{ github.event.pull_request.head.ref }} --pr_number ${{ github.event.pull_request.number }}  --mode changes --mock --config_fig test_config_github_actions.json  
+              else
+                echo "Push from branch [${{ steps.vars.outputs.branch }}]"
+                python detection_testing_execution.py run --branch  [${{ steps.vars.outputs.branch }}  --mode changes --mock --config_file test_config_github_actions.json
+              fi
+
               mv *-test-run.json replicate_test.json
           - name: Upload Test Results Files
             uses: actions/upload-artifact@v2
@@ -135,16 +155,16 @@ jobs:
           with:
             name: testing-results-config
             path: automated_detection_testing/ci/detection_testing_batch/prior_config
-        - name: Install Docker 
-          run: |
-            sudo apt update -qq
+        # - name: Install Docker 
+        #   run: |
+        #     sudo apt update -qq
             
 
-            #python2.7 needed for slim, for now
-            sudo apt install python2
-            sudo apt install virtualenv 
-            curl https://bootstrap.pypa.io/pip/2.7/get-pip.py --output get-pip.py
-            sudo python2.7 get-pip.py
+        #     #python2.7 needed for slim, for now
+        #     sudo apt install python2
+        #     sudo apt install virtualenv 
+        #     curl https://bootstrap.pypa.io/pip/2.7/get-pip.py --output get-pip.py
+        #     sudo python2.7 get-pip.py
 
         - uses: actions/setup-python@v2
           with:
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/splunk_container.py b/automated_detection_testing/ci/detection_testing_batch/modules/splunk_container.py
index e0462d2b23..2b00b0ad0c 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/splunk_container.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/splunk_container.py
@@ -77,7 +77,6 @@ class SplunkContainer:
         require_credentials = False
 
         for app_name, app_info in self.local_apps.items():
-
             app_file_name = os.path.basename(app_info['local_path'])
             app_file_container_path = os.path.join("/tmp/apps", app_file_name)
             apps_to_install.append(app_file_container_path)
@@ -94,6 +93,8 @@ class SplunkContainer:
             require_credentials = True
             # elif app["location"] == "local":
             #    apps_to_install.append(app["container_path"])
+        
+        print(apps_to_install)
         return ",".join(apps_to_install), require_credentials
 
     def make_environment(
@@ -337,7 +338,8 @@ class SplunkContainer:
         except Exception as e:
             print("Error starting docker container: [%s]"%(str(e)))
             return None
-        
+        input("CONTAINTER WANTS TO START.... WAIT FOR INPUT FROM USER")
+
 
         # Sleep for a small random time so that containers drift apart and don't synchronize their testing
         time.sleep(random.randint(1, 30))
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/testing_service.py b/automated_detection_testing/ci/detection_testing_batch/modules/testing_service.py
index 5ab9271bfa..f45e649fe7 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/testing_service.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/testing_service.py
@@ -74,7 +74,7 @@ def test_detection(splunk_ip:str, splunk_port:int, container_name:str, splunk_pa
         replay_attack_dataset(container_name, splunk_password, folder_name, "main", attack_data['sourcetype'], attack_data['source'], attack_data['file_name'])
     
     #Allow some time for the data to be ingested and processed
-    time.sleep(60)
+    time.sleep(30)
     
     result_test = {}
     test = test_file_obj['tests'][0]
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py b/automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py
index b8dc120639..28ac4a9105 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py
@@ -17,7 +17,7 @@ setup_schema = {
             "default": "develop"
         },
         "commit_hash": {
-            "type": ["string","null"],
+            "type": ["string", "null"],
             "default": None
         },
 
@@ -44,38 +44,58 @@ setup_schema = {
             "default": None
         },
 
+
         "local_apps": {
             "type": "object",
-            "properties": {
-                "value": {
-                    "type": "string",
+            "additionalProperties": False,
+            "patternProperties": {
+                "^.*$": {
+                    "type": "object",
+                    "additionalProperties": False,
                     "properties": {
-                         "app_name": {
-                        "type": "string"
+                        "app_number": {
+                            "type": [
+                                "integer",
+                                "null"
+                            ]
+                        },
+                        "app_version": {
+                            "type": [
+                                "string",
+                                "null"
+                            ]
+                        },
+                        "local_path": {
+                            "type": [
+                                "string",
+                                "null"
+                            ]
+                        },
+                        "http_path": {
+                            "type": [
+                                "string"
+                            ]
+                        }
                     },
-                    "app_number": {
-                        "type": "integer"
-                    },
-                    "app_version": {
-                        "type": ["string", "null"]
-                    },
-                    "local_path": {
-                        "type": ["string", "null"],
-                        "default": None
-                    },
-                }
+                    "oneOf": [
+                        {"required": ["local_path"]},
+                        {"required": ["http_path"]}
+                    ]
                 }
             },
             "default": {
-               "SPLUNK_ES_CONTENT_UPDATE": {
-                   "app_number": 3449,
-                   "app_version": None,
-                   'local_path': None
-               }
+                "SPLUNK_ES_CONTENT_UPDATE": {
+                    "app_number": 3449,
+                    "app_version": None,
+                    "local_path": None
+                }
             }
         },
 
 
+
+
+
         "mode": {
             "type": "string",
             "enum": ["changes", "selected", "all"],
@@ -109,24 +129,26 @@ setup_schema = {
 
         },
 
+
+
         "splunkbase_apps": {
             "type": "object",
-            "properies": {
-               "value":
-                {
-                   "type": "string",
-                    "properties": {
-                        "app_number": {
-                         "type": "integer"
-                     },
-                     "app_version": {
-                         "type": "string"
+            "patternProperties": {
+                    "^.*$": {
+                        "type": "object",
+                        "additionalProperties": False,
+                        "properties": {
+                            "app_number": {
+                                "type": "integer"
+                            },
+                            "app_version": {
+                                "type": "string"
+                            }
+                        }
                     }
-                    }
-                }
             },
             "default": {
-                "SPLUNK_ADD_ON_FOR_AMAZON_WEB_SERVICES":{
+                "SPLUNK_ADD_ON_FOR_AMAZON_WEB_SERVICES": {
                     "app_number": 1876,
                     "app_version": "5.2.0"
                 },
@@ -177,10 +199,11 @@ setup_schema = {
                     "app_number": 833,
                     "app_version": "8.3.1"
                 },
-                "SPLUNK_ADD_ON_FOR_SYSMON_OLD": {
-                    "app_number": 1914,
-                    "app_version": "10.6.2"
+                "SPLUNK_ADD_ON_FOR_SYSMON": {
+                    "app_number": 5709,
+                    "app_version": "1.0.1"
                 },
+                # According to https://docs.splunk.com/Documentation/ES/6.6.2/Install/Datamodels, these are included in ES. Don't install separately.
                 "SPLUNK_COMMON_INFORMATION_MODEL": {
                     "app_number": 1621,
                     "app_version": "4.20.2"
@@ -246,11 +269,10 @@ def validate_file(file: io.TextIOWrapper) -> tuple[Union[dict, None], dict]:
 def check_dependencies(settings: dict) -> bool:
     # Check complex mode dependencies
     error_free = True
-    
-    
+
     if settings['mode'] == 'selected':
         # Make sure that exactly one of the following fields is populated
-    
+
         if settings['detections_file'] == None and settings['detections_list'] == None:
             print("Error - mode was 'selected' but no detections_list or detections_file were supplied.", file=sys.stderr)
             error_free = False
@@ -263,21 +285,21 @@ def check_dependencies(settings: dict) -> bool:
     elif settings['mode'] != 'selected' and settings['detections_list'] != None:
         print("Error - mode was not 'selected' but detections_list was supplied.", file=sys.stderr)
         error_free = False
-    
+
     # Returns true if there are not errors
     return error_free
 
 
-def validate_and_write(configuration: dict, output_file: Union[io.TextIOWrapper,None]=None, strip_credentials:bool=False) -> tuple[Union[dict, None], dict]:
+def validate_and_write(configuration: dict, output_file: Union[io.TextIOWrapper, None] = None, strip_credentials: bool = False) -> tuple[Union[dict, None], dict]:
     closeFile = False
     if output_file is None:
         import datetime
         now = datetime.datetime.now()
-        configname = now.strftime('%Y-%m-%dT%H:%M:%S%z') + '-test-run.json' 
+        configname = now.strftime('%Y-%m-%dT%H:%M:%S%z') + '-test-run.json'
         output_file = open(configname, "w")
         closeFile = True
 
-    if strip_credentials:    
+    if strip_credentials:
         configuration = copy.deepcopy(configuration)
         configuration['splunkbase_password'] = None
         configuration['splunkbase_username'] = None
@@ -306,17 +328,18 @@ def validate(configuration: dict) -> tuple[Union[dict, None], dict]:
     # v = jsonschema.Draft201909Validator(argument_schema)
 
     try:
-        
+
         validation_errors, validated_json = jsonschema_errorprinter.check_json(
             configuration, setup_schema)
-        
+
         if len(validation_errors) == 0:
-            #check to make sure there were no complex errors
+            # check to make sure there were no complex errors
             no_complex_errors = check_dependencies(validated_json)
             if no_complex_errors:
                 return validated_json, setup_schema
             else:
-                print("Validation failed due to error(s) listed above.", file=sys.stderr)
+                print("Validation failed due to error(s) listed above.",
+                      file=sys.stderr)
                 return None, setup_schema
         else:
             print("[%d] failures detected during validation of the configuration!" % (
@@ -324,9 +347,8 @@ def validate(configuration: dict) -> tuple[Union[dict, None], dict]:
             for error in validation_errors:
                 print(error, end="\n\n", file=sys.stderr)
             return None, setup_schema
-            
-    except Exception as e:
-        print("There was an error validation the configuration: [%s]"%(str(e)), file=sys.stderr)
-        return None, setup_schema
 
- 
\ No newline at end of file
+    except Exception as e:
+        print("There was an error validation the configuration: [%s]" % (
+            str(e)), file=sys.stderr)
+        return None, setup_schema
diff --git a/automated_detection_testing/ci/detection_testing_batch/test_config_github_actions.json b/automated_detection_testing/ci/detection_testing_batch/test_config_github_actions.json
new file mode 100644
index 0000000000..c14dc8fe03
--- /dev/null
+++ b/automated_detection_testing/ci/detection_testing_batch/test_config_github_actions.json
@@ -0,0 +1,91 @@
+{
+    "branch": "BRANCH_DOES_NOT_EXIST_USE_CLI_ARGUMENT",
+    "commit_hash": null,
+    "container_tag": "latest",
+    "detections_file": null,
+    "detections_list": null,
+    "folders": [
+        "endpoint",
+        "cloud",
+        "network"
+    ],
+    "interactive_failure": false,
+    "local_apps": {
+        "SPLUNK_ES_CONTENT_UPDATE": {
+            "app_number": 3449,
+            "app_version": null,
+            "local_path": "prior_config/apps/DA-ESS-ContentUpdate-latest.tar.gz"
+        }
+    },
+    "local_base_container_name": "splunk_test_%d",
+    "mock": false,
+    "mode": "changes",
+    "num_containers": 10,
+    "persist_security_content": false,
+    "pr_number": null,
+    "reuse_image": true,
+    "show_splunk_app_password": false,
+    "splunk_app_password": null,
+    "splunk_container_apps_directory": "/opt/splunk/etc/apps",
+    "splunkbase_apps": {
+        "PYTHON_FOR_SCIENTIC_COMPUTING_LINUX_64_BIT": {
+            "app_number": 2882,
+            "app_version": "2.0.2"
+        },
+        "SPLUNK_ADD_ON_FOR_AMAZON_KINESIS_FIREHOSE": {
+            "app_number": 3719,
+            "app_version": "1.3.2"
+        },
+        "SPLUNK_ADD_ON_FOR_AMAZON_WEB_SERVICES": {
+            "app_number": 1876,
+            "app_version": "5.2.0"
+        },
+        "SPLUNK_ADD_ON_FOR_MICROSOFT_OFFICE_365": {
+            "app_number": 4055,
+            "app_version": "2.2.0"
+        },
+        "SPLUNK_ADD_ON_FOR_STREAM_FORWARDERS": {
+            "app_number": 5238,
+            "app_version": "8.0.1"
+        },
+        "SPLUNK_ADD_ON_FOR_STREAM_WIRE_DATA": {
+            "app_number": 5234,
+            "app_version": "8.0.1"
+        },
+        "SPLUNK_ADD_ON_FOR_SYSMON": {
+            "app_number": 5709,
+            "app_version": "1.0.1"
+        },
+        "SPLUNK_ADD_ON_FOR_UNIX_AND_LINUX": {
+            "app_number": 833,
+            "app_version": "8.3.1"
+        },
+        "SPLUNK_ADD_ON_FOR_ZEEK_AKA_BRO": {
+            "app_number": 1617,
+            "app_version": "4.0.0"
+        },
+        "SPLUNK_ANALYTIC_STORY_EXECUTION_APP": {
+            "app_number": 4971,
+            "app_version": "2.0.3"
+        },
+        "SPLUNK_APP_FOR_STREAM": {
+            "app_number": 1809,
+            "app_version": "8.0.1"
+        },
+        "SPLUNK_COMMON_INFORMATION_MODEL": {
+            "app_number": 1621,
+            "app_version": "4.20.2"
+        },
+        "SPLUNK_MACHINE_LEARNING_TOOLKIT": {
+            "app_number": 2890,
+            "app_version": "5.2.2"
+        }
+    },
+    "splunkbase_password": null,
+    "splunkbase_username": null,
+    "types": [
+        "Anomaly",
+        "Hunting",
+        "TTP"
+    ]
+}

From 652355a1245673ae901e9847df59992866cfbb1c Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Mon, 6 Dec 2021 11:51:14 -0800
Subject: [PATCH 122/166] Typo in branch name caused an error during checking.

---
 .github/workflows/build-and-validate.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/build-and-validate.yml b/.github/workflows/build-and-validate.yml
index a3946e7b33..c1dd26f5ab 100644
--- a/.github/workflows/build-and-validate.yml
+++ b/.github/workflows/build-and-validate.yml
@@ -343,7 +343,7 @@ jobs:
             echo "github.event.issue.pull_request    : [${{ github.event.issue.pull_request }}]"
             echo "github.event.pull_request.number   : [${{ github.event.pull_request.number }}]"
             echo "steps.vars.outputs.branch          : [${{ steps.vars.outputs.branch }}]"
-            echo "github.event.pull_request.head.ref: [${{ github.event.pull_request.head.ref }}]"
+            echo "github.event.pull_request.head.ref : [${{ github.event.pull_request.head.ref }}]"
 
             ls -lahr prior_config
             if [[ ! -z "${{  github.event.pull_request.head.ref }}" &&  ! -z "${{ github.event.pull_request.number }}" ]]; then
@@ -351,7 +351,7 @@ jobs:
               python detection_testing_execution.py run --branch ${{ github.event.pull_request.head.ref }} --pr_number ${{ github.event.pull_request.number }}  --mode changes --mock --config_fig test_config_github_actions.json  
             else
               echo "Push from branch [${{ steps.vars.outputs.branch }}]"
-              python detection_testing_execution.py run --branch  [${{ steps.vars.outputs.branch }}  --mode changes --mock --config_file test_config_github_actions.json
+              python detection_testing_execution.py run --branch  ${{ steps.vars.outputs.branch }}  --mode changes --mock --config_file test_config_github_actions.json
             fi
 
             mv *-test-run.json replicate_test.json

From e453ffefb8a857f2f149be309dac986d67eaa454 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Mon, 6 Dec 2021 11:55:08 -0800
Subject: [PATCH 123/166] More verbose error message if the Git checkout fails.

---
 .../ci/detection_testing_batch/detection_testing_execution.py  | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
index 7efbdc1675..1c35e6f0bd 100644
--- a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
+++ b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
@@ -322,7 +322,8 @@ def main(args: list[str]):
             settings['branch'], settings['commit_hash'], settings['pr_number'], settings['persist_security_content'])
         settings['commit_hash'] = github_service.commit_hash
     except Exception as e:
-        print("Failure checking out git repository: %s\n\tQuitting",file=sys.stderr)
+        print("Failure checking out git repository:\n\t\n\tHash: [%s]\n\tBranch: [%s]\n\tPR: [%s]\n\tQuitting"%
+              (settings['commit_hash'],settings['branch'],settings['pr_number']),file=sys.stderr)
         sys.exit(1)
 
     # Make a backup of this config containing the hash and stripped credentials.

From f19c7c08f21d9eef2f9eb9590e1b30cc3a2bbebd Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Mon, 6 Dec 2021 12:29:04 -0800
Subject: [PATCH 124/166] touched a detection and a test to see if the ci runs
 correctly.

---
 detections/endpoint/7zip_commandline_to_smb_share_path.yml    | 2 +-
 tests/endpoint/access_lsass_memory_for_dump_creation.test.yml | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/detections/endpoint/7zip_commandline_to_smb_share_path.yml b/detections/endpoint/7zip_commandline_to_smb_share_path.yml
index 7a6633a5b3..1138658414 100644
--- a/detections/endpoint/7zip_commandline_to_smb_share_path.yml
+++ b/detections/endpoint/7zip_commandline_to_smb_share_path.yml
@@ -2,7 +2,7 @@ name: 7zip CommandLine To SMB Share Path
 id: 01d29b48-ff6f-11eb-b81e-acde48001122
 version: 1
 date: '2021-08-17'
-author: Teoderick Contreras, Splunk
+author: Teoderick Contreras, Splunk TESTCHANGES
 type: Hunting
 datamodel:
 - Endpoint
diff --git a/tests/endpoint/access_lsass_memory_for_dump_creation.test.yml b/tests/endpoint/access_lsass_memory_for_dump_creation.test.yml
index 32e5fb0e48..b8f4ee4ec8 100644
--- a/tests/endpoint/access_lsass_memory_for_dump_creation.test.yml
+++ b/tests/endpoint/access_lsass_memory_for_dump_creation.test.yml
@@ -1,4 +1,4 @@
-name: Access LSASS Memory for Dump Creation Unit Test
+name: Access LSASS Memory for Dump Creation Unit Test TESTCHANGES
 tests:
 - name: Access LSASS Memory for Dump Creation
   file: endpoint/access_lsass_memory_for_dump_creation.yml

From 20bc63c34d2f25159e3e31d640efab5a58518f3d Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Mon, 6 Dec 2021 14:06:11 -0800
Subject: [PATCH 125/166] Forgot to remove a wait for user input. Removed and
 running again.

---
 .../ci/detection_testing_batch/detection_testing_execution.py  | 3 ++-
 .../ci/detection_testing_batch/modules/splunk_container.py     | 2 +-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
index 1c35e6f0bd..35a8bed3d2 100644
--- a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
+++ b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
@@ -238,7 +238,8 @@ def finish_mock(settings: dict, detections: list[str], output_file_template: str
         # Pass in the list of detections to run
         mock_settings['detections_list'] = normalized_detection_names
 
-        # We want to persist security content and run with the escu package that we created
+        # We want to persist security content and run with the escu package that we created.
+        #Note that if we haven't checked this out yet, we will check it out for you.
         mock_settings['persist_security_content'] = True
 
         mock_settings['mock'] = False
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/splunk_container.py b/automated_detection_testing/ci/detection_testing_batch/modules/splunk_container.py
index 2b00b0ad0c..a27dd9b509 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/splunk_container.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/splunk_container.py
@@ -338,7 +338,7 @@ class SplunkContainer:
         except Exception as e:
             print("Error starting docker container: [%s]"%(str(e)))
             return None
-        input("CONTAINTER WANTS TO START.... WAIT FOR INPUT FROM USER")
+        #input("CONTAINTER WANTS TO START.... WAIT FOR INPUT FROM USER")
 
 
         # Sleep for a small random time so that containers drift apart and don't synchronize their testing

From 2981a9c592635928d662a585de96c81624ef0ac2 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Mon, 6 Dec 2021 14:37:24 -0800
Subject: [PATCH 126/166] added a check to see if we are running on develop,
 meaning that it is a nightly job.  Also, added a cron job to fun of 04:44
 UTC. GitHub recommends against the top of the hour because load is higher at
 that time.

---
 .github/workflows/build-and-validate.yml               | 10 ++++++++--
 .../endpoint/7zip_commandline_to_smb_share_path.yml    |  2 +-
 .../access_lsass_memory_for_dump_creation.test.yml     |  2 +-
 3 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/build-and-validate.yml b/.github/workflows/build-and-validate.yml
index c1dd26f5ab..fa86376384 100644
--- a/.github/workflows/build-and-validate.yml
+++ b/.github/workflows/build-and-validate.yml
@@ -17,6 +17,8 @@ on:
   push:
   pull_request:
     types: [opened, reopened]
+  schedule:
+    - cron: "44 4 * * *"
 jobs:
   validate-tag-if-present:
     runs-on: ubuntu-latest
@@ -346,9 +348,13 @@ jobs:
             echo "github.event.pull_request.head.ref : [${{ github.event.pull_request.head.ref }}]"
 
             ls -lahr prior_config
-            if [[ ! -z "${{  github.event.pull_request.head.ref }}" &&  ! -z "${{ github.event.pull_request.number }}" ]]; then
+            
+            if [[ ${{ steps.vars.outputs.branch == develop }} ]]
+              echo "Running a nightly test on all detections"
+              python detection_testing_execution.py run --branch ${{ github.event.pull_request.head.ref }} --mode all --mock --config_file test_config_github_actions.json  
+            elif [[ ! -z "${{  github.event.pull_request.head.ref }}" &&  ! -z "${{ github.event.pull_request.number }}" ]]; then
               echo "Pull request from source branch [${{ github.event.pull_request.head.ref }}] for PR number [${{ github.event.issue.number }}]"
-              python detection_testing_execution.py run --branch ${{ github.event.pull_request.head.ref }} --pr_number ${{ github.event.pull_request.number }}  --mode changes --mock --config_fig test_config_github_actions.json  
+              python detection_testing_execution.py run --branch ${{ github.event.pull_request.head.ref }} --pr_number ${{ github.event.pull_request.number }}  --mode changes --mock --config_file test_config_github_actions.json  
             else
               echo "Push from branch [${{ steps.vars.outputs.branch }}]"
               python detection_testing_execution.py run --branch  ${{ steps.vars.outputs.branch }}  --mode changes --mock --config_file test_config_github_actions.json
diff --git a/detections/endpoint/7zip_commandline_to_smb_share_path.yml b/detections/endpoint/7zip_commandline_to_smb_share_path.yml
index 1138658414..7a6633a5b3 100644
--- a/detections/endpoint/7zip_commandline_to_smb_share_path.yml
+++ b/detections/endpoint/7zip_commandline_to_smb_share_path.yml
@@ -2,7 +2,7 @@ name: 7zip CommandLine To SMB Share Path
 id: 01d29b48-ff6f-11eb-b81e-acde48001122
 version: 1
 date: '2021-08-17'
-author: Teoderick Contreras, Splunk TESTCHANGES
+author: Teoderick Contreras, Splunk
 type: Hunting
 datamodel:
 - Endpoint
diff --git a/tests/endpoint/access_lsass_memory_for_dump_creation.test.yml b/tests/endpoint/access_lsass_memory_for_dump_creation.test.yml
index b8f4ee4ec8..32e5fb0e48 100644
--- a/tests/endpoint/access_lsass_memory_for_dump_creation.test.yml
+++ b/tests/endpoint/access_lsass_memory_for_dump_creation.test.yml
@@ -1,4 +1,4 @@
-name: Access LSASS Memory for Dump Creation Unit Test TESTCHANGES
+name: Access LSASS Memory for Dump Creation Unit Test
 tests:
 - name: Access LSASS Memory for Dump Creation
   file: endpoint/access_lsass_memory_for_dump_creation.yml

From 3de3ffeadadbf141ee45792a6c2f6dd30dce8373 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Mon, 6 Dec 2021 14:44:27 -0800
Subject: [PATCH 127/166] Brace in wrong location - around non environment
 variable

---
 .github/workflows/build-and-validate.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build-and-validate.yml b/.github/workflows/build-and-validate.yml
index fa86376384..ba91447608 100644
--- a/.github/workflows/build-and-validate.yml
+++ b/.github/workflows/build-and-validate.yml
@@ -349,7 +349,7 @@ jobs:
 
             ls -lahr prior_config
             
-            if [[ ${{ steps.vars.outputs.branch == develop }} ]]
+            if [[ ${{ steps.vars.outputs.branch }} == develop ]]
               echo "Running a nightly test on all detections"
               python detection_testing_execution.py run --branch ${{ github.event.pull_request.head.ref }} --mode all --mock --config_file test_config_github_actions.json  
             elif [[ ! -z "${{  github.event.pull_request.head.ref }}" &&  ! -z "${{ github.event.pull_request.number }}" ]]; then

From 14e548c65a044025d3dcee3c8187a06aec06716e Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Mon, 6 Dec 2021 14:54:58 -0800
Subject: [PATCH 128/166] More changes to CI.  Fixed a bash typo and removed
 the docker-detection-testing.yml since it has been integrated into
 build-and-validate.

---
 .github/workflows/build-and-validate.yml      |   2 +-
 .../workflows/docker-detection-testing.yml    | 321 ------------------
 .../ci/detection_testing_batch/Dockerfile     |  16 -
 3 files changed, 1 insertion(+), 338 deletions(-)
 delete mode 100644 .github/workflows/docker-detection-testing.yml
 delete mode 100644 automated_detection_testing/ci/detection_testing_batch/Dockerfile

diff --git a/.github/workflows/build-and-validate.yml b/.github/workflows/build-and-validate.yml
index ba91447608..43fb214ce3 100644
--- a/.github/workflows/build-and-validate.yml
+++ b/.github/workflows/build-and-validate.yml
@@ -349,7 +349,7 @@ jobs:
 
             ls -lahr prior_config
             
-            if [[ ${{ steps.vars.outputs.branch }} == develop ]]
+            if [[ ${{ steps.vars.outputs.branch }} == develop ]]; then
               echo "Running a nightly test on all detections"
               python detection_testing_execution.py run --branch ${{ github.event.pull_request.head.ref }} --mode all --mock --config_file test_config_github_actions.json  
             elif [[ ! -z "${{  github.event.pull_request.head.ref }}" &&  ! -z "${{ github.event.pull_request.number }}" ]]; then
diff --git a/.github/workflows/docker-detection-testing.yml b/.github/workflows/docker-detection-testing.yml
deleted file mode 100644
index d898d65180..0000000000
--- a/.github/workflows/docker-detection-testing.yml
+++ /dev/null
@@ -1,321 +0,0 @@
-name: docker-detection-testing
-on:
-  #push:
-  #pull_request:
-  #run nightly at 0500 UTC
-  #Note the scheduled workflows will only run if the Action has 
-  #been committed to the default branch (develop)
-  #schedule:
-  #  - cron: "0 5 * * *"
-jobs:
-
-  validate-tag-if-present:
-    runs-on: ubuntu-latest
-
-    steps:
-      - name: TAGGED, Validate that the tag is in the correct format
-        
-        run: |
-          echo "The GITHUB_REF: $GITHUB_REF"
-          #First check to see if the release is a tag
-          if [[ $GITHUB_REF =~ refs/tags/* ]]; then
-            #Yes, this is a tag, so we need to test to make sure that the tag
-            #is in the correct format (like v1.10.20)
-            if [[ $GITHUB_REF =~ refs/tags/v[0-9]+.[0-9]+.[0-9]+ ]]; then
-              echo "PASS: Tagged release with good format"
-              exit 0
-            else
-              echo "FAIL: Tagged release with bad format"
-              exit 1
-            fi
-          else
-          echo "PASS: Not a tagged release"
-          exit 0
-          fi                    
-  
-  quit-for-dependabot:
-    runs-on: ubuntu-latest
-    if: github.actor != 'dependabot[bot]' && github.actor != 'dependabot-preview[bot]'
-    steps:
-      - name: "Placeholder"
-        run: |
-          echo "No, this is not a dependabot run!"
-
-
-  docker-detection-testing-setup:
-      runs-on: ubuntu-latest
-      needs: [validate-tag-if-present, quit-for-dependabot]
-      steps:  
-          - name: Get branch and PR required for detection testing main.py
-            id: vars
-            run: |
-              echo "::set-output name=branch::${GITHUB_REF#refs/heads/}"
-
-          - name: Checkout Repo
-            uses: actions/checkout@v2
-          
-          #- name: Install requirements for installing slim during execution 
-          #  run: |
-          #    sudo apt update -qq
-          #    #python2.7 needed for slim, for now
-          #    sudo apt install python2
-          #    sudo apt install virtualenv 
-          #    curl https://bootstrap.pypa.io/pip/2.7/get-pip.py --output get-pip.py
-          #    sudo python2.7 get-pip.py
-
-          # Get the previously built ESCU
-          - name: Get ESCU
-            uses: actions/download-artifact@v2
-            with:
-              name: content-latest
-              path: automated_detection_testing/ci/detection_testing_batch/prior_config/apps
-
-          - uses: actions/setup-python@v2
-            with:
-              python-version: '3.9' #Available versions here - https://github.com/actions/python-versions/releases  easy to change/make a matrix/use pypy
-              architecture: 'x64' # optional x64 or x86. Defaults to x64 if not specified          
-          
-          - name: Install Python Dependencies
-            run: |
-              cd automated_detection_testing/ci/detection_testing_batch
-              python3 -m venv .venv
-              source .venv/bin/activate
-              python3 -m pip install wheel
-              python3 -m pip install -r requirements.txt
-          
-          - name: Run the CI
-            run: |
-              cd automated_detection_testing/ci/detection_testing_batch
-              source .venv/bin/activate
-              echo "github.event.issue.pull_request    : [${{ github.event.issue.pull_request }}]"
-              echo "github.event.pull_request.number   : [${{ github.event.pull_request.number }}]"
-              echo "steps.vars.outputs.branch          : [${{ steps.vars.outputs.branch }}]"
-              echo "github.event.pull_request.head.ref: [${{ github.event.pull_request.head.ref }}]"
-
-              ls -lahr prior_config
-              if [[ ! -z "${{  github.event.pull_request.head.ref }}" &&  ! -z "${{ github.event.pull_request.number }}" ]]; then
-                echo "Pull request from source branch [${{ github.event.pull_request.head.ref }}] for PR number [${{ github.event.issue.number }}]"
-                python detection_testing_execution.py run --branch ${{ github.event.pull_request.head.ref }} --pr_number ${{ github.event.pull_request.number }}  --mode changes --mock --config_fig test_config_github_actions.json  
-              else
-                echo "Push from branch [${{ steps.vars.outputs.branch }}]"
-                python detection_testing_execution.py run --branch  [${{ steps.vars.outputs.branch }}  --mode changes --mock --config_file test_config_github_actions.json
-              fi
-
-              mv *-test-run.json replicate_test.json
-          - name: Upload Test Results Files
-            uses: actions/upload-artifact@v2
-            with:
-              name: testing-results-config
-              path: |
-                automated_detection_testing/ci/detection_testing_batch/prior_config/apps/DA-ESS-ContentUpdate-latest.tar.gz
-                automated_detection_testing/ci/detection_testing_batch/prior_config/config_tests_0.json
-                automated_detection_testing/ci/detection_testing_batch/prior_config/config_tests_1.json
-                automated_detection_testing/ci/detection_testing_batch/prior_config/config_tests_2.json
-                automated_detection_testing/ci/detection_testing_batch/prior_config/config_tests_3.json
-                automated_detection_testing/ci/detection_testing_batch/prior_config/config_tests_4.json
-                automated_detection_testing/ci/detection_testing_batch/prior_config/config_tests_5.json
-                automated_detection_testing/ci/detection_testing_batch/prior_config/config_tests_6.json
-                automated_detection_testing/ci/detection_testing_batch/prior_config/config_tests_7.json
-                automated_detection_testing/ci/detection_testing_batch/prior_config/config_tests_8.json
-                automated_detection_testing/ci/detection_testing_batch/prior_config/config_tests_9.json
-
-          - name: Upload File to Enable Replication of the Test at a Different Time or Place
-            uses: actions/upload-artifact@v2
-            with:
-              name: replicate_test
-              path: |
-                automated_detection_testing/ci/detection_testing_batch/replicate_test.json
-
-  docker-detection-testing-execution:
-    runs-on: ubuntu-latest
-    needs: [validate-tag-if-present, quit-for-dependabot, docker-detection-testing-setup]
-    strategy:
-      matrix:
-        manifest_filename: ["config_tests_0.json", 
-                        "config_tests_1.json",
-                        "config_tests_2.json",
-                        "config_tests_3.json",
-                        "config_tests_4.json",
-                        "config_tests_5.json",
-                        "config_tests_6.json",
-                        "config_tests_7.json",
-                        "config_tests_8.json",
-                        "config_tests_9.json"]
-    steps:  
-        - name: Get branch and PR required for detection testing main.py
-          id: vars
-          run: |
-            echo "::set-output name=branch::${GITHUB_REF#refs/heads/}"
-
-        - name: Checkout Repo
-          uses: actions/checkout@v2
-        
-        - name: Download artifacts
-          uses: actions/download-artifact@v2
-          with:
-            name: testing-results-config
-            path: automated_detection_testing/ci/detection_testing_batch/prior_config
-        # - name: Install Docker 
-        #   run: |
-        #     sudo apt update -qq
-            
-
-        #     #python2.7 needed for slim, for now
-        #     sudo apt install python2
-        #     sudo apt install virtualenv 
-        #     curl https://bootstrap.pypa.io/pip/2.7/get-pip.py --output get-pip.py
-        #     sudo python2.7 get-pip.py
-
-        - uses: actions/setup-python@v2
-          with:
-            python-version: '3.9' #Available versions here - https://github.com/actions/python-versions/releases  easy to change/make a matrix/use pypy
-            architecture: 'x64' # optional x64 or x86. Defaults to x64 if not specified          
-        
-        - name: Install Python Dependencies
-          run: |
-            cd automated_detection_testing/ci/detection_testing_batch
-            python3 -m venv .venv
-            source .venv/bin/activate
-            python3 -m pip install wheel
-            python3 -m pip install -r requirements.txt
-        
-        - name: Run the CI
-          run: |
-            cd automated_detection_testing/ci/detection_testing_batch
-            source .venv/bin/activate
-            
-            python3 detection_testing_execution.py run -c prior_config/${{ matrix.manifest_filename}} --splunkbase_username ${{ secrets.SPLUNKBASE_TESTING_USERNAME }} --splunkbase_password ${{ secrets.SPLUNKBASE_TESTING_KEY }} 
-                    
-        
-        - name: Upload Test Results Files
-          uses: actions/upload-artifact@v2
-          with:
-            name: ${{ matrix.manifest_filename}}.results
-            path: |
-              automated_detection_testing/ci/detection_testing_batch/success.csv
-              automated_detection_testing/ci/detection_testing_batch/error.csv
-              automated_detection_testing/ci/detection_testing_batch/failure.csv
-              automated_detection_testing/ci/detection_testing_batch/combined.csv
-              automated_detection_testing/ci/detection_testing_batch/success.json
-              automated_detection_testing/ci/detection_testing_batch/error.json
-              automated_detection_testing/ci/detection_testing_batch/failure.json
-              automated_detection_testing/ci/detection_testing_batch/combined.json
-          
-  docker-detection-testing-execution-merge-results:
-    runs-on: ubuntu-latest
-    needs: [validate-tag-if-present, quit-for-dependabot, docker-detection-testing-setup, docker-detection-testing-execution]
- 
-    steps:  
-        - name: Get branch and PR required for detection testing main.py
-          id: vars
-          run: |
-            echo "::set-output name=branch::${GITHUB_REF#refs/heads/}"
-
-        - name: Checkout Repo
-          uses: actions/checkout@v2
-        
-        - name: Download artifacts
-          uses: actions/download-artifact@v2
-          with:
-            name: config_tests_0.json.results
-            path: automated_detection_testing/ci/detection_testing_batch/results_0
-        - name: Download artifacts
-          uses: actions/download-artifact@v2
-          with:
-            name: config_tests_1.json.results
-            path: automated_detection_testing/ci/detection_testing_batch/results_1
-        - name: Download artifacts
-          uses: actions/download-artifact@v2
-          with:
-            name: config_tests_2.json.results
-            path: automated_detection_testing/ci/detection_testing_batch/results_2
-        - name: Download artifacts
-          uses: actions/download-artifact@v2
-          with:
-            name: config_tests_3.json.results
-            path: automated_detection_testing/ci/detection_testing_batch/results_3
-        - name: Download artifacts
-          uses: actions/download-artifact@v2
-          with:
-            name: config_tests_4.json.results
-            path: automated_detection_testing/ci/detection_testing_batch/results_4
-        - name: Download artifacts
-          uses: actions/download-artifact@v2
-          with:
-            name: config_tests_5.json.results
-            path: automated_detection_testing/ci/detection_testing_batch/results_5
-        - name: Download artifacts
-          uses: actions/download-artifact@v2
-          with:
-            name: config_tests_6.json.results
-            path: automated_detection_testing/ci/detection_testing_batch/results_6
-        - name: Download artifacts
-          uses: actions/download-artifact@v2
-          with:
-            name: config_tests_7.json.results
-            path: automated_detection_testing/ci/detection_testing_batch/results_7
-        - name: Download artifacts
-          uses: actions/download-artifact@v2
-          with:
-            name: config_tests_8.json.results
-            path: automated_detection_testing/ci/detection_testing_batch/results_8
-        - name: Download artifacts
-          uses: actions/download-artifact@v2
-          with:
-            name: config_tests_9.json.results
-            path: automated_detection_testing/ci/detection_testing_batch/results_9
-        
-        - uses: actions/setup-python@v2
-          with:
-            python-version: '3.9' #Available versions here - https://github.com/actions/python-versions/releases  easy to change/make a matrix/use pypy
-            architecture: 'x64' # optional x64 or x86. Defaults to x64 if not specified    
-        
-        - name: Install Python Dependencies
-          run: |
-            cd automated_detection_testing/ci/detection_testing_batch
-            python3 -m venv .venv
-            source .venv/bin/activate
-            python3 -m pip install wheel
-            python3 -m pip install -r requirements.txt
-
-        - name: Merge Detections into single File
-          run: |
-            cd automated_detection_testing/ci/detection_testing_batch
-            source .venv/bin/activate
-            python summarize_json.py --files results_*/combined.json --output_filename summary_test_results.json
-            
-            
-        - name: Upload Summary Test Results JSON
-          uses: actions/upload-artifact@v2
-          if: always()
-          with:
-            name: SummaryTestResults
-            path: |
-              automated_detection_testing/ci/detection_testing_batch/summary_test_results.json
-
-        - name: Upload Failures Manifest on Failure
-          uses: actions/upload-artifact@v2
-          if: failure()
-          with:
-            name: DetectionFailureManifest
-            path: |
-              automated_detection_testing/ci/detection_testing_batch/detection_failure_manifest.json
-            
-        
-        #Always clean these up, they make the output messy
-        - name: Clean up intermediate Files
-          uses: geekyeggo/delete-artifact@v1
-          if: always()
-          with:
-              name: |
-                config_tests_0.json.results
-                config_tests_1.json.results
-                config_tests_2.json.results
-                config_tests_3.json.results
-                config_tests_4.json.results
-                config_tests_5.json.results
-                config_tests_6.json.results
-                config_tests_7.json.results
-                config_tests_8.json.results
-                config_tests_9.json.results
-                
\ No newline at end of file
diff --git a/automated_detection_testing/ci/detection_testing_batch/Dockerfile b/automated_detection_testing/ci/detection_testing_batch/Dockerfile
deleted file mode 100644
index 1ed762dde5..0000000000
--- a/automated_detection_testing/ci/detection_testing_batch/Dockerfile
+++ /dev/null
@@ -1,16 +0,0 @@
-FROM ubuntu:18.04
-
-RUN apt-get update
-RUN DEBIAN_FRONTEND="noninteractive" apt-get -y install tzdata
-RUN apt-get install -y python3-dev git python-dev unzip python3-pip awscli
-RUN apt-get install -y python-gitdb
-RUN apt-get install -y wget unzip
-RUN apt-get install -y git
-
-ADD . /app
-
-WORKDIR /app
-RUN pip3 install -r requirements.txt
-
-ENTRYPOINT ["python3", "detection_testing_execution.py"]
-CMD ["-b", "automated_detections_testing_2"]

From b82873bbde48f262a82551cc7dc819ca86fadcc9 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Mon, 6 Dec 2021 15:56:07 -0800
Subject: [PATCH 129/166] Removed unused code from testing_service.

---
 .../modules/testing_service.py                | 132 ------------------
 1 file changed, 132 deletions(-)

diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/testing_service.py b/automated_detection_testing/ci/detection_testing_batch/modules/testing_service.py
index f45e649fe7..d060419c36 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/testing_service.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/testing_service.py
@@ -142,135 +142,3 @@ def update_ESCU_app(container_name, splunk_password):
     print("Successfully updated the ESCU App!")
 
 
-def configure_splunk_vm(container_name, splunk_password):
-    ansible_vars = {}
-    ansible_vars['splunk_password'] = splunk_password
-    ansible_vars['ansible_user'] = 'ansible'
-
-    #BEGIN BLOCK VARS FROM ATTACK_RANGE_LOCAL.CONF
-    splunk_url = "https://download.splunk.com/products/splunk/releases/8.0.2/linux/splunk-8.0.2-a7f645ddaf91-Linux-x86_64.tgz"
-    # Specify the download URL of Splunk Enterprise
-
-    splunk_binary = "splunk-8.0.2-a7f645ddaf91-Linux-x86_64.tgz"
-    # Specify the name of the Splunk Enterprise executable
-
-    s3_bucket_url = "https://attack-range-appbinaries.s3-us-west-2.amazonaws.com"
-    # Specify the S3 bucket url from which you want to download the Splunk Apps
-
-    splunk_windows_ta = "splunk-add-on-for-microsoft-windows_800.tgz"
-    # Specify the Splunk Windows TA
-
-    splunk_sysmon_ta = "splunk-add-on-for-microsoft-sysmon_1062.tgz"
-    # Specify the Splunk Sysmon TA
-
-    splunk_cim_app = "splunk-common-information-model-cim_4180.tgz"
-    # Specify the Splunk CIM App
-
-    splunk_escu_app = "DA-ESS-ContentUpdate-latest.tar.gz"
-    # Specify the Splunk ESCU App
-
-    splunk_asx_app = "Splunk_ASX-latest.tar.gz"
-    # Specify the Splunk ASX App
-
-    splunk_python_app = "python-for-scientific-computing-for-linux-64-bit_200.tgz"
-    # Specify the Splunk python for scientific computing dependency that is needed by the MLTK app
-
-    splunk_mltk_app = "splunk-machine-learning-toolkit_510.tgz"
-    # Specify the Splunk MLTK App
-
-    splunk_stream_app = "splunk-stream_720.tgz"
-    # Specify the Splunk Stream App
-
-    splunk_security_essentials_app = "splunk-security-essentials_310.tgz"
-    # Specify the Splunk SSE App
-
-    punchard_custom_visualization = "punchcard-custom-visualization_140.tgz"
-    # Specify the Punchard Custom Visualization App
-
-    status_indicator_custom_visualization = "status-indicator-custom-visualization_140.tgz"
-    # Specify the Status Indicator Custom Visualization App
-
-    splunk_attack_range_dashboard = "splunk_attack_range_reporting-1.0.5.tar.gz"
-    # Specify the Attack Range Dashboard App
-
-    timeline_custom_visualization = "timeline-custom-visualization_140.tgz"
-    # Specify the Timeline Custom Visualization App
-
-    splunk_aws_app = "splunk-add-on-for-amazon-web-services_500.tgz"
-    # Specify the Splunk AWS App
-    # Will be only installed when cloud_attack_range=1
-    #END BLOCK VARS FROM ATTACK_RANGE_LOCAL.CONF
-
-
-
-    splunk_es_app = 'splunk-enterprise-security_640.spl'
-    splunk_es_app_version = re.findall(r'\d+', splunk_es_app)[0]
-
-    #ansible_vars['ansible_python_interpreter'] = "/usr/bin/python3",
-    ansible_vars['splunk_admin_password'] = splunk_password
-    ansible_vars['splunk_url'] = splunk_url
-    ansible_vars['splunk_binary'] = splunk_binary
-    ansible_vars['s3_bucket_url'] = s3_bucket_url
-    ansible_vars['splunk_escu_app'] = splunk_escu_app
-    ansible_vars['splunk_asx_app'] = splunk_asx_app
-    ansible_vars['splunk_windows_ta'] = splunk_windows_ta
-    ansible_vars['splunk_cim_app'] = splunk_cim_app
-    ansible_vars['splunk_sysmon_ta'] = splunk_sysmon_ta
-    #ansible_vars['caldera_password'] = '{{ caldera_password }}'
-    ansible_vars['splunk_mltk_app'] = splunk_mltk_app
-    #ansible_vars['splunk_bots_dataset'] = '{{ splunk_bots_dataset }}'
-    ansible_vars['splunk_stream_app'] = splunk_stream_app
-    ansible_vars['splunk_python_app'] = splunk_python_app
-    #ansible_vars['phantom_app'] = '{{ phantom_app }}'
-    #ansible_vars['phantom_server'] = '{{ phantom_server }}'
-    #ansible_vars['phantom_server_private_ip'] = '{{ phantom_server_private_ip }}'
-    #ansible_vars['phantom_admin_password'] = '{{ phantom_admin_password }}'
-    ansible_vars['splunk_security_essentials_app'] = splunk_security_essentials_app
-    ansible_vars['punchard_custom_visualization'] = punchard_custom_visualization
-    ansible_vars['status_indicator_custom_visualization'] = status_indicator_custom_visualization
-    ansible_vars['splunk_attack_range_dashboard'] = splunk_attack_range_dashboard
-    ansible_vars['timeline_custom_visualization'] = timeline_custom_visualization
-    #ansible_vars['install_mission_control'] = install_mission_control
-    #ansible_vars['mission_control_app'] = mission_control_app
-    #ansible_vars['install_dsp'] = install_dsp
-    #ansible_vars['dsp_client_cert_path'] = dsp_client_cert_path
-    #ansible_vars['dsp_node'] = dsp_node
-    ansible_vars['splunk_server_private_ip'] = "127.0.0.1"
-    ansible_vars['cloud_attack_range'] = '0'
-
-    ansible_vars['install_es'] = '1'
-    ansible_vars['install_mltk'] = '0'
-    ansible_vars['install_mission_control'] = '0'
-    ansible_vars['install_dsp'] = '0'
-
-    ansible_vars['splunk_es_app'] = splunk_es_app
-    ansible_vars['splunk_es_app_version'] = splunk_es_app_version
-    print(ansible_vars['splunk_es_app_version'])
-
-
-
-
-    cmdline = "--connection docker -i %s, -u %s" % (container_name, ansible_vars['ansible_user'])
-    runner = ansible_runner.run(private_data_dir=os.path.join(os.path.dirname(__file__), '../'),
-                                cmdline=cmdline,
-                                roles_path=os.path.join(os.path.dirname(__file__), '../ansible/roles'),
-                                playbook=os.path.join(os.path.dirname(__file__), '../ansible/splunk_server.yml'),
-                                extravars=ansible_vars)
-
-def replay_attack_dataset(container_name, splunk_password, folder_name, index, sourcetype, source, out):
-    ansible_vars = {}
-    ansible_vars['folder_name'] = folder_name
-    ansible_vars['ansible_user'] = 'ansible'
-    ansible_vars['splunk_password'] = splunk_password
-    ansible_vars['out'] = out
-    ansible_vars['sourcetype'] = sourcetype
-    ansible_vars['source'] = source
-    ansible_vars['index'] = index
-
-    cmdline = "--connection docker -i %s, -u %s" % (container_name, ansible_vars['ansible_user'])
-    runner = ansible_runner.run(private_data_dir=os.path.join(os.path.dirname(__file__), '../'),
-                                cmdline=cmdline,
-                                roles_path=os.path.join(os.path.dirname(__file__), '../ansible/roles'),
-                                playbook=os.path.join(os.path.dirname(__file__), '../ansible/attack_replay.yml'),
-                                extravars=ansible_vars)
-

From 9176e24d4468eef9e8dc9c83108a982cf9826e3b Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Tue, 7 Dec 2021 12:11:29 -0800
Subject: [PATCH 130/166] A number of small usability updates.  Added and
 tested a --interactive mode flag and changed --interactive_failure to
 --no_interactive_failure.  Interactive_failure mode is now the default mode.
 Improved spacing and messages on some error strings.

---
 .../detection_testing_execution.py            | 16 +++++++----
 .../modules/container_manager.py              | 16 +++++++----
 .../modules/github_service.py                 | 15 ++++++----
 .../modules/new_arguments2.py                 | 12 ++++++--
 .../modules/splunk_container.py               |  5 +++-
 .../modules/splunk_sdk.py                     |  6 ++--
 .../modules/testing_service.py                | 28 +++++++++++++++++--
 .../modules/validate_args.py                  |  7 ++++-
 8 files changed, 77 insertions(+), 28 deletions(-)

diff --git a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
index 35a8bed3d2..427858ab25 100644
--- a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
+++ b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
@@ -70,7 +70,7 @@ def ensure_security_content(branch: str, commit_hash: str, pr_number: Union[int,
               "We will not check out the repo again. Please be aware, this could cause issues if you're "
               "out of date. ******")
         github_service = GithubService(
-            branch, commit_hash, existing_directory=persist_security_content)
+            branch, commit_hash, persist_security_content=persist_security_content)
 
     else:
         if persist_security_content is True and not os.path.exists("security_content"):
@@ -160,11 +160,11 @@ def generate_escu_app(persist_security_content: bool = False) -> str:
             print("Error downloading the Splunk Packaging Toolkit: [%s].\n\tQuitting..." %
                   (str(e)), file=sys.stderr)
             sys.exit(1)
-
+        print(os.getcwd())
         commands = ["rm -rf slim_packaging/slim_latest",
                     "mkdir slim_packaging/slim_latest",
                     "cd slim_packaging",
-                    "tar -zxf splunk-packaging-toolkit-latest.tar.gz -C slim_latest --strip-components=1",
+                    "tar -zxf ../splunk-packaging-toolkit-latest.tar.gz -C slim_latest --strip-components=1",
                     "cd slim_latest",
                     "virtualenv --python=/usr/bin/python2.7 --clear .venv",
                     ". ./.venv/bin/activate",
@@ -323,7 +323,10 @@ def main(args: list[str]):
             settings['branch'], settings['commit_hash'], settings['pr_number'], settings['persist_security_content'])
         settings['commit_hash'] = github_service.commit_hash
     except Exception as e:
-        print("Failure checking out git repository:\n\t\n\tHash: [%s]\n\tBranch: [%s]\n\tPR: [%s]\n\tQuitting"%
+        print("\nFailure checking out git repository:"\
+              "\n\tHash:   [%s]"\
+              "\n\tBranch: [%s]"\
+              "\n\tPR:     [%s]\n\tQuitting"%
               (settings['commit_hash'],settings['branch'],settings['pr_number']),file=sys.stderr)
         sys.exit(1)
 
@@ -343,6 +346,8 @@ def main(args: list[str]):
         print("\tQuitting...", file=sys.stderr)
         sys.exit(1)
 
+    print("***This run will test [%d] detections!***"%(len(all_test_files)))
+
     #Set up the directory that will be used to store the local apps/apps we build
     local_volume_absolute_path = os.path.abspath(
         os.path.join(os.getcwd(), "apps"))
@@ -420,7 +425,8 @@ def main(args: list[str]):
                                             splunkbase_username=settings['splunkbase_username'],
                                             splunkbase_password=settings['splunkbase_password'],
                                             reuse_image=settings['reuse_image'],
-                                            interactive_failure=settings['interactive_failure'])
+                                            interactive_failure=not settings['no_interactive_failure'],
+                                            interactive=settings['interactive'])
 
     result = cm.run_test()
     
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/container_manager.py b/automated_detection_testing/ci/detection_testing_batch/modules/container_manager.py
index 15d82b7842..30d0c34ad1 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/container_manager.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/container_manager.py
@@ -38,7 +38,8 @@ class ContainerManager:
         splunkbase_username: Union[str, None] = None,
         splunkbase_password: Union[str, None] = None,
         reuse_image:bool = True,
-        interactive_failure:bool=False
+        interactive_failure:bool=False,
+        interactive:bool=False
 
     ):
         self.synchronization_object = test_driver.TestDriver(
@@ -74,7 +75,8 @@ class ContainerManager:
             splunkbase_password,
             files_to_copy_to_container,
             reuse_image,
-            interactive_failure
+            interactive_failure,
+            interactive
         )
         self.summary_thread = threading.Thread(target=self.queue_status_thread,args=())
 
@@ -142,7 +144,8 @@ class ContainerManager:
         splunkbase_password: Union[str, None] = None,
         files_to_copy_to_container: OrderedDict = OrderedDict(),
         reuse_image:bool = True,
-        interactive_failure:bool = True
+        interactive_failure:bool = False,
+        interactive:bool = False
     ) -> list[splunk_container.SplunkContainer]:
         #First make sure that the image exists and has been downloaded
         self.setup_image(reuse_image, full_docker_hub_name)
@@ -170,7 +173,8 @@ class ContainerManager:
                     self.mounts,
                     splunkbase_username,
                     splunkbase_password,
-                    interactive_failure=interactive_failure
+                    interactive_failure=interactive_failure,
+                    interactive=interactive
                 )
             )
 
@@ -209,7 +213,7 @@ class ContainerManager:
         password = "".join(password_list)
         return password
 
-    def queue_status_thread(self)->None:
+    def queue_status_thread(self, status_interval:int=60)->None:
         #This will run fo
         while True:
             if self.synchronization_object.checkContainerFailure():
@@ -223,7 +227,7 @@ class ContainerManager:
                 #There are no more tests to run, so we can return from this thread
                 print("return from queue status thread")
                 return None
-            time.sleep(10)
+            time.sleep(status_interval)
 
     def setup_image(self, reuse_images: bool, container_name: str) -> None:
         client = docker.client.from_env()
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/github_service.py b/automated_detection_testing/ci/detection_testing_batch/modules/github_service.py
index 15f5e5acf7..a62dbaed4a 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/github_service.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/github_service.py
@@ -22,14 +22,16 @@ SECURITY_CONTENT_URL = "https://github.com/splunk/security_content"
 
 class GithubService:
 
-    def __init__(self, security_content_branch: str, commit_hash: str, PR_number: int = None, existing_directory: bool = False):
+    def __init__(self, security_content_branch: str, commit_hash: str, PR_number: int = None, persist_security_content: bool = False):
 
         self.security_content_branch = security_content_branch
-        if existing_directory:
-            return
-        print("Checking out security_content!")
-        self.security_content_repo_obj = self.clone_project(
-            SECURITY_CONTENT_URL, f"security_content", f"develop")
+        if persist_security_content:
+            print("Getting handle on existing security_content repo!")
+            self.security_content_repo_obj = git.Repo("security_content")
+        else:
+            print("Checking out security_content repo!")            
+            self.security_content_repo_obj = self.clone_project(
+                SECURITY_CONTENT_URL, f"security_content", f"develop")
 
         if commit_hash is not None and PR_number is not None:
             print("Error - both the PR number [%d] and the commit hash [%s] were provided.  "
@@ -50,6 +52,7 @@ class GithubService:
         else:
             print("Checking out branch: [%s]..." %
                   (security_content_branch), end='')
+            sys.stdout.flush()
             self.security_content_repo_obj.git.checkout(
                 security_content_branch)
             commit_hash = self.security_content_repo_obj.head.object.hexsha
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/new_arguments2.py b/automated_detection_testing/ci/detection_testing_batch/modules/new_arguments2.py
index e93a49e807..21603a6e34 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/new_arguments2.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/new_arguments2.py
@@ -187,10 +187,16 @@ def parse(args) -> tuple[str, dict]:
     run_parser.add_argument("-n", "--num_containers", required=False, type=int,
                             help="The number of Splunk containers to run or mock")
 
-    run_parser.add_argument("-i", "--interactive_failure", required=False,
+    run_parser.add_argument("-nif", "--no_interactive_failure", required=False,
                             action="store_true",
                             help="After a detection fails, pause and allow the user to log into "\
-                            "the Splunk server to interactively debug the failure.  Wait for them "\
+                            "the Splunk server to interactively debug the failure.  Wait for the user "\
+                            "to hit enter before removing the test data and moving on to the next test.")
+
+    run_parser.add_argument("-i", "--interactive", required=False,
+                            action="store_true",
+                            help="After a detection runs, pause and allow the user to log into "\
+                            "the Splunk server to debug the detection.  Wait for the user "\
                             "to hit enter before removing the test data and moving on to the next test.")
 
     args = parser.parse_args()
@@ -208,7 +214,7 @@ def parse(args) -> tuple[str, dict]:
             # set their value to something else, like None
 
             # Don't overwite booleans
-            if args.__dict__[key] is False and key in ["show_splunk_app_password", "mock", "interactive_failure"]:
+            if args.__dict__[key] is False and key in ["show_splunk_app_password", "mock", "no_interactive_failure", "interactive"]:
                 del args.__dict__[key]
             # Don't overwrite other values
             elif args.__dict__[key] is None and key in ["splunkbase_username", "branch", "commit_hash",
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/splunk_container.py b/automated_detection_testing/ci/detection_testing_batch/modules/splunk_container.py
index a27dd9b509..251a0fab14 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/splunk_container.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/splunk_container.py
@@ -38,8 +38,10 @@ class SplunkContainer:
         splunkbase_password: Union[str, None] = None,
         splunk_ip: str = "127.0.0.1",
         interactive_failure: bool = False,
+        interactive:bool = False
     ):
         self.interactive_failure = interactive_failure
+        self.interactive = interactive
         self.synchronization_object = synchronization_object
         self.client = docker.client.from_env()
         self.full_docker_hub_path = full_docker_hub_path
@@ -94,7 +96,7 @@ class SplunkContainer:
             # elif app["location"] == "local":
             #    apps_to_install.append(app["container_path"])
         
-        print(apps_to_install)
+        
         return ",".join(apps_to_install), require_credentials
 
     def make_environment(
@@ -380,6 +382,7 @@ class SplunkContainer:
                     detection_to_test,
                     self.synchronization_object.attack_data_root_folder,
                     wait_on_failure=self.interactive_failure,
+                    wait_on_completion = self.interactive
                 )
                 self.synchronization_object.addResult(result)
 
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/splunk_sdk.py b/automated_detection_testing/ci/detection_testing_batch/modules/splunk_sdk.py
index 2294b27501..d0099f1a7c 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/splunk_sdk.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/splunk_sdk.py
@@ -141,7 +141,7 @@ def test_detection_search(splunk_host:str, splunk_port:int, splunk_password:str,
         return test_results
 
 
-def delete_attack_data(splunk_host:str, splunk_password:str, splunk_port:int, wait_on_delete:bool, search_string:str, detection_filename:str)->bool:
+def delete_attack_data(splunk_host:str, splunk_password:str, splunk_port:int, wait_on_delete:Union[dict,None], search_string:str, detection_filename:str)->bool:
     
     try:
         service = client.connect(
@@ -155,10 +155,10 @@ def delete_attack_data(splunk_host:str, splunk_password:str, splunk_port:int, wa
 
     #splunk_search = 'search index=test* | delete'
     if wait_on_delete:
-        print("\n\n\n****SEARCH FAILURE: Allowing time to debug search****")
+        print(wait_on_delete['message'])
         print("FILENAME : [%s]"%(detection_filename))
         print("SEARCH   :\n%s"%(search_string))
-        _ = input("****************Press ENTER to DELETE****************\n\n\n")
+        _ = input("****************Press ENTER to Complete Test and DELETE data****************\n\n\n")
     splunk_search = 'search index=main | delete'
     kwargs = {"exec_mode": "blocking",
               "dispatch.earliest_time": "-1d",
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/testing_service.py b/automated_detection_testing/ci/detection_testing_batch/modules/testing_service.py
index d060419c36..bddca333c1 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/testing_service.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/testing_service.py
@@ -14,7 +14,8 @@ from os.path import relpath
 from tempfile import mkdtemp
 
 
-def test_detection_wrapper(container_name:str, splunk_ip:str, splunk_password:str, splunk_port:int, test_file:str, attack_data_root_folder, wait_on_failure:bool=False)->dict:
+def test_detection_wrapper(container_name:str, splunk_ip:str, splunk_password:str, splunk_port:int, 
+                           test_file:str, attack_data_root_folder, wait_on_failure:bool=False, wait_on_completion:bool=False)->dict:
     
     uuid_var = str(uuid.uuid4())
     result_test = test_detection(splunk_ip, splunk_port, container_name, splunk_password, test_file, uuid_var, attack_data_root_folder)
@@ -28,9 +29,11 @@ def test_detection_wrapper(container_name:str, splunk_ip:str, splunk_password:st
     
     #search failed if there was an error or the detection failed to produce the expected result
     if wait_on_failure and (result_test['detection_result']['error'] or not result_test['detection_result']['success']):
-        wait_on_delete = True
+        wait_on_delete = {'message':"\n\n\n****SEARCH FAILURE: Allowing time to debug search/data****"}
+    elif wait_on_completion:
+        wait_on_delete = {'message':"\n\n\n****SEARCH SUCCESS: Allowing time to examine search/data****"}
     else:
-        wait_on_delete = False
+        wait_on_delete = None
     
     splunk_sdk.delete_attack_data(splunk_ip, splunk_password, splunk_port, wait_on_delete, search_string, test_file)
    
@@ -142,3 +145,22 @@ def update_ESCU_app(container_name, splunk_password):
     print("Successfully updated the ESCU App!")
 
 
+def replay_attack_dataset(container_name, splunk_password, folder_name, index, sourcetype, source, out):
+    ansible_vars = {}
+    ansible_vars['folder_name'] = folder_name
+    ansible_vars['ansible_user'] = 'ansible'
+    ansible_vars['splunk_password'] = splunk_password
+    ansible_vars['out'] = out
+    ansible_vars['sourcetype'] = sourcetype
+    ansible_vars['source'] = source
+    ansible_vars['index'] = index
+    envvars = {"ANSIBLE_STDOUT_CALLBACK": "community.general.null"}
+    cmdline = "--connection docker -i %s, -u %s" % (container_name, ansible_vars['ansible_user'])
+    
+    print("ABOUT TO RUN PLAYBOOK WITH SUPPRESSED OUTPUT")    
+    runner = ansible_runner.run(private_data_dir=os.path.join(os.path.dirname(__file__), '../'),
+                                cmdline=cmdline,
+                                roles_path=os.path.join(os.path.dirname(__file__), '../ansible/roles'),
+                                playbook=os.path.join(os.path.dirname(__file__), '../ansible/attack_replay.yml'),
+                                extravars=ansible_vars, envvars=envvars)
+    
\ No newline at end of file
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py b/automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py
index 28ac4a9105..3cb526aa5b 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py
@@ -26,7 +26,12 @@ setup_schema = {
             "default": "latest"
         },
 
-        "interactive_failure": {
+        "no_interactive_failure": {
+            "type": "boolean",
+            "default": False
+        },
+
+        "interactive": {
             "type": "boolean",
             "default": False
         },

From 0f87a6011c4071e13564640eda87753bf89fe48a Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Tue, 7 Dec 2021 13:27:38 -0800
Subject: [PATCH 131/166] Changes to support more of interactive mode and
 interactive_failure_mode as well as better logic around whether or not the
 password should be printed to the command line. In short, it will be printed
 if you are in interactive mode and have not included it in the config.  If
 you are interactive and the password is in the config file, you will be
 warned but it will not be printed.  If you are not interactive, it will not
 be printed unless you ask it to print.

---
 .../detection_testing_execution.py            |  1 +
 .../modules/new_arguments2.py                 |  8 ++---
 .../modules/validate_args.py                  | 29 +++++++++++++++----
 .../test_config_github_actions.json           |  3 +-
 4 files changed, 31 insertions(+), 10 deletions(-)

diff --git a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
index 427858ab25..512ca30813 100644
--- a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
+++ b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
@@ -270,6 +270,7 @@ def main(args: list[str]):
     requests.packages.urllib3.disable_warnings()
 
     start_datetime = datetime.now()
+    
     action, settings = modules.new_arguments2.parse(args)
     if action == "configure":
         # Done, nothing else to do
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/new_arguments2.py b/automated_detection_testing/ci/detection_testing_batch/modules/new_arguments2.py
index 21603a6e34..c47c02362a 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/new_arguments2.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/new_arguments2.py
@@ -61,7 +61,7 @@ def configure_action(args) -> tuple[str, dict]:
 
     # Now parse the new config and make sure it's good
     validated_new_settings, schema = validate_args.validate_and_write(
-        new_config, args.output_config_file)
+        new_config, args.output_config_file, skip_password_accessibility_check=False)
     if validated_new_settings == None:
         print("Could not update settings.\n\tQuitting...", file=sys.stderr)
         sys.exit(1)
@@ -84,7 +84,7 @@ def update_config_with_cli_arguments(args_dict: dict) -> tuple[str, dict]:
             settings[key] = value
 
     # Validate again to make sure we didn't break anything
-    settings, _ = validate_args.validate(settings)
+    settings, _ = validate_args.validate(settings,skip_password_accessibility_check=False)
     if settings is None:
         print("Failure while processing updated settings from command line.\n\tQuitting...", file=sys.stderr)
         sys.exit(1)
@@ -115,7 +115,7 @@ def parse(args) -> tuple[str, dict]:
         print("No default configuration file [%s] found.  Creating one..." % (
             DEFAULT_CONFIG_FILE))
         with open(DEFAULT_CONFIG_FILE, 'w') as cfg:
-            validate_args.validate_and_write({}, cfg)
+            validate_args.validate_and_write({}, cfg, skip_password_accessibility_check=True)
 
     parser = argparse.ArgumentParser(
         description="Use 'SOME_PROGRAM_NAME_STRING --help' to get help with the arguments")
@@ -227,7 +227,7 @@ def parse(args) -> tuple[str, dict]:
         
         return action, settings
     except Exception as e:
-        print("Unknown Error - [%s]" % (str(e)))
+        print("Unknown Error Validating Json Configuration - [%s]" % (str(e)))
         sys.exit(1)
 
 if __name__ == "__main__":
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py b/automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py
index 3cb526aa5b..856f6c0f4c 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py
@@ -271,10 +271,11 @@ def validate_file(file: io.TextIOWrapper) -> tuple[Union[dict, None], dict]:
         raise(e)
 
 
-def check_dependencies(settings: dict) -> bool:
+def check_dependencies(settings: dict, skip_password_accessibility_check:bool=True) -> bool:
     # Check complex mode dependencies
     error_free = True
 
+    # Make sure that all the mode arguments are sane
     if settings['mode'] == 'selected':
         # Make sure that exactly one of the following fields is populated
 
@@ -291,11 +292,28 @@ def check_dependencies(settings: dict) -> bool:
         print("Error - mode was not 'selected' but detections_list was supplied.", file=sys.stderr)
         error_free = False
 
+
+    # Make sure that if we will be in an interactive mode, that either the user has provided the password or the password will be printed
+    if skip_password_accessibility_check:
+        pass
+    elif (settings['interactive'] or not settings['no_interactive_failure']) and settings['show_splunk_app_password'] is False:
+        print("\n\n******************************************************\n\n")
+        if settings['splunk_app_password'] is not None:
+            print("Warning: You have chosen an interactive mode, set show_splunk_app_password False,\n"\
+                  "and provided a password in the config file.  We will NOT print this password to\n"\
+                  "stdout.  Look in the config file for this password.",file=sys.stderr)
+        else:
+            print("Warning: You have chosen an interactive mode, set show_splunk_app_password False,\n"\
+                  "and DID NOT provide a password in the config file.  We have updated show_splunk_app_password\n"\
+                  "to True for you.  Otherwise, interactive mode login would be impossible.",file=sys.stderr)
+            settings['show_splunk_app_password'] = True
+        print("\n\n******************************************************\n\n")
+
     # Returns true if there are not errors
     return error_free
 
 
-def validate_and_write(configuration: dict, output_file: Union[io.TextIOWrapper, None] = None, strip_credentials: bool = False) -> tuple[Union[dict, None], dict]:
+def validate_and_write(configuration: dict, output_file: Union[io.TextIOWrapper, None] = None, strip_credentials: bool = False, skip_password_accessibility_check:bool=True) -> tuple[Union[dict, None], dict]:
     closeFile = False
     if output_file is None:
         import datetime
@@ -309,8 +327,9 @@ def validate_and_write(configuration: dict, output_file: Union[io.TextIOWrapper,
         configuration['splunkbase_password'] = None
         configuration['splunkbase_username'] = None
         configuration['container_password'] = None
+        configuration['show_splunk_app_password'] = True
 
-    validated_json, setup_schema = validate(configuration)
+    validated_json, setup_schema = validate(configuration,skip_password_accessibility_check)
     if validated_json == None:
         print("Error in the new settings! No output file written")
     else:
@@ -329,7 +348,7 @@ def validate_and_write(configuration: dict, output_file: Union[io.TextIOWrapper,
     return validated_json, setup_schema
 
 
-def validate(configuration: dict) -> tuple[Union[dict, None], dict]:
+def validate(configuration: dict, skip_password_accessibility_check:bool=True) -> tuple[Union[dict, None], dict]:
     # v = jsonschema.Draft201909Validator(argument_schema)
 
     try:
@@ -339,7 +358,7 @@ def validate(configuration: dict) -> tuple[Union[dict, None], dict]:
 
         if len(validation_errors) == 0:
             # check to make sure there were no complex errors
-            no_complex_errors = check_dependencies(validated_json)
+            no_complex_errors = check_dependencies(validated_json,skip_password_accessibility_check)
             if no_complex_errors:
                 return validated_json, setup_schema
             else:
diff --git a/automated_detection_testing/ci/detection_testing_batch/test_config_github_actions.json b/automated_detection_testing/ci/detection_testing_batch/test_config_github_actions.json
index c14dc8fe03..81fa25f89d 100644
--- a/automated_detection_testing/ci/detection_testing_batch/test_config_github_actions.json
+++ b/automated_detection_testing/ci/detection_testing_batch/test_config_github_actions.json
@@ -9,7 +9,7 @@
         "cloud",
         "network"
     ],
-    "interactive_failure": false,
+    "interactive": false,
     "local_apps": {
         "SPLUNK_ES_CONTENT_UPDATE": {
             "app_number": 3449,
@@ -20,6 +20,7 @@
     "local_base_container_name": "splunk_test_%d",
     "mock": false,
     "mode": "changes",
+    "no_interactive_failure": true,
     "num_containers": 10,
     "persist_security_content": false,
     "pr_number": null,

From 934a3727f20e8d8d84092c79892233568b1fbf5e Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Tue, 7 Dec 2021 17:08:11 -0800
Subject: [PATCH 132/166] A number of changes.  First we quit immediately if
 docker is not running on the machine... we were missing the quit before but
 not the check and error message.  Next, we fix a persistent problem with
 storing the output of a test that encounters an error. We also remove some
 un-needed print statements

---
 .../detection_testing_execution.py              |  1 +
 .../modules/container_manager.py                |  4 ++--
 .../modules/splunk_sdk.py                       | 17 ++++++++++++-----
 .../modules/test_driver.py                      |  1 +
 .../modules/testing_service.py                  |  6 ++----
 .../modules/validate_args.py                    |  4 ++++
 6 files changed, 22 insertions(+), 11 deletions(-)

diff --git a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
index 512ca30813..1a4dbe8915 100644
--- a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
+++ b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
@@ -289,6 +289,7 @@ def main(args: list[str]):
             docker.client.from_env()
         except Exception as e:
             print("Error, failed to get docker client.  Is Docker Installed and Running?\n\t%s" % (str(e)))
+            sys.exit(1)
         
         credentials_needed = False
         credential_error = False
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/container_manager.py b/automated_detection_testing/ci/detection_testing_batch/modules/container_manager.py
index 30d0c34ad1..b11f1669dd 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/container_manager.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/container_manager.py
@@ -221,11 +221,11 @@ class ContainerManager:
                 for container in self.containers:
                     container.stopContainer()
                 print("All containers stopped")
-                print("return from queue status thread")
+                
                 return None
             if self.synchronization_object.summarize() == False:
                 #There are no more tests to run, so we can return from this thread
-                print("return from queue status thread")
+                
                 return None
             time.sleep(status_interval)
 
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/splunk_sdk.py b/automated_detection_testing/ci/detection_testing_batch/modules/splunk_sdk.py
index d0099f1a7c..ea782d72e1 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/splunk_sdk.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/splunk_sdk.py
@@ -92,8 +92,14 @@ def test_detection_search(splunk_host:str, splunk_port:int, splunk_password:str,
     splunk_search = search + ' ' + pass_condition
     test_results = dict()
     
+    #These will always be present. By default, we will say that the
+    #test has failed AND there was an error (until they are set otherwise)
     test_results['search_string'] = splunk_search
+    test_results['detection_name'] = detection_name
+    test_results['detection_file'] = detection_file
     
+    test_results['success'] = False
+    test_results['error'] = True
 
     
     try:
@@ -118,19 +124,20 @@ def test_detection_search(splunk_host:str, splunk_port:int, splunk_password:str,
     try:
         job = service.jobs.create(splunk_search, **kwargs)
     except Exception as e:
-        print("Unable to execute detection: " + str(e))
+        print("Unable to execute detection:\n%s"%(str(e)))
         test_results['error'] = True
         return test_results
 
     test_results['diskUsage'] = job['diskUsage']
     test_results['runDuration'] = job['runDuration']
-    test_results['detection_name'] = detection_name
-    test_results['detection_file'] = detection_file
     test_results['scanCount'] = job['scanCount']
     
-    
-    test_results['error'] = False #The search may have FAILED, but there was no error in the search
+    #If we get this far, then there was not an error
+    #The search may have FAILED, but there was no error in the search
+    test_results['error'] = False 
 
+
+    #Should this be 1 for a pass, or should it be greater than 0?
     if int(job['resultCount']) != 1:
         #print("Test failed for detection: " + detection_name)
         test_results['success'] = False
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/test_driver.py b/automated_detection_testing/ci/detection_testing_batch/modules/test_driver.py
index 69ce72efd6..170c3fb19d 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/test_driver.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/test_driver.py
@@ -280,5 +280,6 @@ class TestDriver:
                 self.addSuccess(result['detection_result'])
         except Exception as e:
             #Neither a success or a failure, so add the object to the failures queue
+            print('"There was an error adding the result: [%s]'%(str(e)))
             self.addError({'detection_file':"Unknown File", "detection_error":str(result)})
 
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/testing_service.py b/automated_detection_testing/ci/detection_testing_batch/modules/testing_service.py
index bddca333c1..189e141dfa 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/testing_service.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/testing_service.py
@@ -154,13 +154,11 @@ def replay_attack_dataset(container_name, splunk_password, folder_name, index, s
     ansible_vars['sourcetype'] = sourcetype
     ansible_vars['source'] = source
     ansible_vars['index'] = index
-    envvars = {"ANSIBLE_STDOUT_CALLBACK": "community.general.null"}
-    cmdline = "--connection docker -i %s, -u %s" % (container_name, ansible_vars['ansible_user'])
     
-    print("ABOUT TO RUN PLAYBOOK WITH SUPPRESSED OUTPUT")    
+    cmdline = "--connection docker -i %s, -u %s" % (container_name, ansible_vars['ansible_user'])
     runner = ansible_runner.run(private_data_dir=os.path.join(os.path.dirname(__file__), '../'),
                                 cmdline=cmdline,
                                 roles_path=os.path.join(os.path.dirname(__file__), '../ansible/roles'),
                                 playbook=os.path.join(os.path.dirname(__file__), '../ansible/attack_replay.yml'),
-                                extravars=ansible_vars, envvars=envvars)
+                                extravars=ansible_vars)
     
\ No newline at end of file
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py b/automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py
index 856f6c0f4c..86c8f81a3f 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py
@@ -153,6 +153,10 @@ setup_schema = {
                     }
             },
             "default": {
+                "ADD-ON_FOR_LINUX_SYSMON": {
+                    "app_number": 6176,
+                    "app_version": "1.0.3"
+                },
                 "SPLUNK_ADD_ON_FOR_AMAZON_WEB_SERVICES": {
                     "app_number": 1876,
                     "app_version": "5.2.0"

From 8f55eb5848aa421060f338f464a1512f44909220 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Wed, 8 Dec 2021 12:00:51 -0800
Subject: [PATCH 133/166] Proper summary.json files are now always generated
 after a run of the CI.  It does not require a separate run of
 summarize_json.py.  Becuase the CI runs different jobs, though, it is still
 required to run this explicitly in the CI.  Also fixed an issue where
 searches that resulted in an error had incorrect json output due to improper
 python error handling.

---
 .github/workflows/build-and-validate.yml      | 20 +++--
 .../detection_testing_execution.py            |  7 +-
 .../modules/test_driver.py                    | 42 +++++++--
 .../detection_testing_batch/summarize_json.py | 89 +++++++++++--------
 4 files changed, 101 insertions(+), 57 deletions(-)

diff --git a/.github/workflows/build-and-validate.yml b/.github/workflows/build-and-validate.yml
index 43fb214ce3..04d06b57f0 100644
--- a/.github/workflows/build-and-validate.yml
+++ b/.github/workflows/build-and-validate.yml
@@ -451,14 +451,16 @@ jobs:
           with:
             name: ${{ matrix.manifest_filename}}.results
             path: |
-              automated_detection_testing/ci/detection_testing_batch/success.csv
-              automated_detection_testing/ci/detection_testing_batch/error.csv
-              automated_detection_testing/ci/detection_testing_batch/failure.csv
-              automated_detection_testing/ci/detection_testing_batch/combined.csv
-              automated_detection_testing/ci/detection_testing_batch/success.json
-              automated_detection_testing/ci/detection_testing_batch/error.json
-              automated_detection_testing/ci/detection_testing_batch/failure.json
-              automated_detection_testing/ci/detection_testing_batch/combined.json
+              automated_detection_testing/ci/detection_testing_batch/test_results/success.csv
+              automated_detection_testing/ci/detection_testing_batch/test_results/error.csv
+              automated_detection_testing/ci/detection_testing_batch/test_results/failure.csv
+              automated_detection_testing/ci/detection_testing_batch/test_results/combined.csv
+              automated_detection_testing/ci/detection_testing_batch/test_results/success.json
+              automated_detection_testing/ci/detection_testing_batch/test_results/error.json
+              automated_detection_testing/ci/detection_testing_batch/test_results/failure.json
+              automated_detection_testing/ci/detection_testing_batch/test_results/combined.json
+
+              automated_detection_testing/ci/detection_testing_batch/test_results/summary.json
           
   docker-detection-testing-execution-merge-results:
     runs-on: ubuntu-latest
@@ -541,7 +543,7 @@ jobs:
           run: |
             cd automated_detection_testing/ci/detection_testing_batch
             source .venv/bin/activate
-            python summarize_json.py --files results_*/combined.json --output_filename summary_test_results.json
+            python summarize_json.py --files results_*/summary.json --output_filename summary_test_results.json
             
             
         - name: Upload Summary Test Results JSON
diff --git a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
index 1a4dbe8915..e9b42b6526 100644
--- a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
+++ b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
@@ -67,8 +67,11 @@ def copy_local_apps_to_directory(apps: dict[str, dict], target_directory) -> Non
 def ensure_security_content(branch: str, commit_hash: str, pr_number: Union[int, None], persist_security_content: bool) -> GithubService:
     if persist_security_content is True and os.path.exists("security_content"):
         print("****** You chose --persist_security_content and the security_content directory exists. "
-              "We will not check out the repo again. Please be aware, this could cause issues if you're "
-              "out of date. ******")
+              "We will not check out the repo again. Please be aware, this could cause issues if your "
+              "repo is out of date or if a previous build failed to download all required tools and "\
+              "libraries.  If this occurs, it is suggested to change the "\
+              "persist_security_content setting to false. ******")
+              
         github_service = GithubService(
             branch, commit_hash, persist_security_content=persist_security_content)
 
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/test_driver.py b/automated_detection_testing/ci/detection_testing_batch/modules/test_driver.py
index 170c3fb19d..47d78472ff 100644
--- a/automated_detection_testing/ci/detection_testing_batch/modules/test_driver.py
+++ b/automated_detection_testing/ci/detection_testing_batch/modules/test_driver.py
@@ -5,6 +5,7 @@ import json
 import os
 import queue
 import shutil
+import summarize_json
 import tempfile
 import threading
 import time
@@ -168,21 +169,44 @@ class TestDriver:
         
 
     def outputResultsFiles(self, baseline:OrderedDict, fields:list[str]=['detection_name', 'detection_file','runDuration','diskUsage', 'search_string', 'error', 'success', 'scanCount', 'detection_error'])->bool:
-        res = self.outputResultsFile(fields,"success", self.successes, baseline)
-        res |= self.outputResultsFile(fields, "failure", self.failures, baseline)
-        res |= self.outputResultsFile(fields, "error", self.errors, baseline)
-        res |= self.outputResultsFile(fields, "combined", self.successes + self.failures + self.errors, baseline)
+        results_directory = "test_results"
+        try:
+            shutil.rmtree(results_directory,ignore_errors=True)
+            os.mkdir(results_directory)
+        except Exception as e:
+            print("There was an error removing the results directory [%s]: [%s].\n\t We will try to continue output anyway."%(results_directory, str(e)))
+
+
+        res = self.outputResultsFile(fields,os.path.join(results_directory, "success"), self.successes, baseline)
+        res |= self.outputResultsFile(fields, os.path.join(results_directory, "failure"), self.failures, baseline)
+        res |= self.outputResultsFile(fields, os.path.join(results_directory, "error"), self.errors, baseline)
+        combined_data = self.successes + self.failures + self.errors
+        res |= self.outputResultsFile(fields, os.path.join(results_directory, "combined"), combined_data, baseline)
+        
+        try:
+            success, test_count,pass_count,fail_count,error_count = summarize_json.outputResultsJSON("summary.json", combined_data, baseline, output_folder=results_directory)
+            summarize_json.print_summary(test_count, pass_count, fail_count, error_count)
+            res |= success
+        except Exception as e:
+            print("Failure writing the summary file: [%s]",file=sys.stderr)
+            res = False
+
         return res
 
     def finish(self, baseline:OrderedDict):
         self.cleanup()
-        self.outputResultsFiles(baseline)
+        success = True
+        if self.outputResultsFiles(baseline) == False:
+            print("There was an error generating one or more of the output files. "\
+                  "Check the logs for details.",file=sys.stderr)
+            success = False
+        
 
         if self.checkContainerFailure():
-            print("One or more containers crashed, so testing did not complete successfully. We wrote out the results we have")
+            print("One or more containers crashed, so testing did not complete successfully. We wrote out all the results that we could")
             return False
         else:
-            return True
+            return success
 
         
 
@@ -273,7 +297,9 @@ class TestDriver:
         
     def addResult(self, result:dict)->None:
         try:
-            if result['detection_result']['success'] is False:
+            if result['detection_result']['error'] is True:
+                self.addError(result['detection_result'])
+            elif result['detection_result']['success'] is False:
                 #This is actually a failure of the detection, not an error. Naming is confusiong
                 self.addFailure(result['detection_result'])
             elif result['detection_result']['success'] is True:
diff --git a/automated_detection_testing/ci/detection_testing_batch/summarize_json.py b/automated_detection_testing/ci/detection_testing_batch/summarize_json.py
index d55b7c46e3..eddfe456ab 100644
--- a/automated_detection_testing/ci/detection_testing_batch/summarize_json.py
+++ b/automated_detection_testing/ci/detection_testing_batch/summarize_json.py
@@ -8,7 +8,9 @@ import os.path
 from operator import itemgetter
 
 
-def outputResultsJSON(output_filename:str, data:list[dict], baseline:OrderedDict)->tuple[bool,int,int,int]:
+def outputResultsJSON(output_filename:str, data:list[dict], baseline:OrderedDict, 
+                      failure_manifest_filename = "detection_failure_manifest.json", 
+                      output_folder:str="")->tuple[bool,int,int,int,int]:
     success = True
     
     try:
@@ -47,7 +49,7 @@ def outputResultsJSON(output_filename:str, data:list[dict], baseline:OrderedDict
                  "FAIL_AND_ERROR":fail_and_error_count }
 
         data_sorted = sorted(data, key = lambda k: (-k['error'], k['success'], k['detection_file']))
-        with open(output_filename, "w") as jsonFile:
+        with open(os.path.join(output_folder,output_filename), "w") as jsonFile:
             json.dump({'summary':summary, 'baseline': baseline, 'results':data_sorted}, jsonFile, indent="    ")
         
         
@@ -62,7 +64,7 @@ def outputResultsJSON(output_filename:str, data:list[dict], baseline:OrderedDict
             failures_test_override = {"detections_list": fail_list, "interactive_failure":True, 
                                     "num_containers":1, "branch": baseline["branch"], "commit_hash":baseline["commit_hash"], 
                                     "mode":"selected", "show_splunk_app_password": True}
-            with open("detection_failure_manifest.json","w") as failures:
+            with open(os.path.join(output_folder,failure_manifest_filename),"w") as failures:
                 validate_args.validate_and_write(failures_test_override, failures)
     except Exception as e:
         print("There was an error generating [%s]: [%s]"%(output_filename, str(e)),file=sys.stderr)
@@ -71,39 +73,16 @@ def outputResultsJSON(output_filename:str, data:list[dict], baseline:OrderedDict
         #success = False
         #return success, False
 
-    return success, test_count, pass_count, fail_count
+    #note that total failures is fail_count, fail_and_error count is JUST errors (and every error is also a failure)
+    return success, test_count, pass_count, fail_count, fail_and_error_count
 
-
-
-parser = argparse.ArgumentParser(description="Results Merger")
-parser.add_argument('-f', '--files', type=argparse.FileType('r'), required=True, nargs='+', help="The json files you would like to combine into a single file")
-parser.add_argument('-o', '--output_filename', type=str, required=True, help="The name of the output file")
-args = parser.parse_args()
-
-all_data = OrderedDict()
-try:
-    print("We will summarize the files: %s"%(str([f.name for f in args.files])))
-    for f in args.files:
-        if not f.name.endswith('.json'):
-            print("Error: passed in file must end in .json - you passed in [%s].\n\tQuitting..."%(f.name))
-            sys.exit(1)
-        data = json.loads(f.read())
-        if 'baseline' in all_data:
-            #everything has the same baseline, only need to do it once
-            pass
-        else:
-            all_data['baseline'] = data['baseline']
-        if 'results' in all_data:
-            #this is a list of dictionaries, so add to it
-            all_data['results'].extend(data['results'])
-        else:
-            all_data['results'] = data['results']
-
-    test_pass, test_count, pass_count, fail_count = outputResultsJSON(args.output_filename, all_data['results'], all_data['baseline'])
+def print_summary(test_count: int, pass_count:int, fail_count:int, error_count:int)->None:
     print("Summary:"\
-          "\n\tTotal Tests: %d"\
-          "\n\tTotal Pass : %d"\
-          "\n\tTotal Fail : %d"%(test_count, pass_count, fail_count))
+            "\n\tTotal Tests: %d"\
+            "\n\tTotal Pass : %d"\
+            "\n\tTotal Fail : %d (%d of these were ERRORS)"%(test_count, pass_count, fail_count, error_count))
+
+def exit_with_status(test_pass:bool, test_count: int, pass_count:int, fail_count:int, error_count:int)->None:
     if not test_pass:
         print("Result: FAIL")
         sys.exit(1)
@@ -111,11 +90,45 @@ try:
         print("Result: PASS!")
         sys.exit(0)
 
-    
-except Exception as e:
-    print("Error writing the summary file: [%s].\n\tQuitting..."%(str(e)))
-    sys.exit(1)
 
+def finish(test_pass:bool, test_count: int, pass_count:int, fail_count:int, error_count:int)->None:
+    print_summary(test_count, pass_count, fail_count,error_count)
+    exit_with_status(test_pass, test_count, pass_count, fail_count,error_count)
+
+def main():
+    parser = argparse.ArgumentParser(description="Results Merger")
+    parser.add_argument('-f', '--files', type=argparse.FileType('r'), required=True, nargs='+', help="The json files you would like to combine into a single file")
+    parser.add_argument('-o', '--output_filename', type=str, required=True, help="The name of the output file")
+    args = parser.parse_args()
+
+    all_data = OrderedDict()
+    try:
+        print("We will summarize the files: %s"%(str([f.name for f in args.files])))
+        for f in args.files:
+            if not f.name.endswith('.json'):
+                print("Error: passed in file must end in .json - you passed in [%s].\n\tQuitting..."%(f.name))
+                sys.exit(1)
+            data = json.loads(f.read())
+            if 'baseline' in all_data:
+                #everything has the same baseline, only need to do it once
+                pass
+            else:
+                all_data['baseline'] = data['baseline']
+            if 'results' in all_data:
+                #this is a list of dictionaries, so add to it
+                all_data['results'].extend(data['results'])
+            else:
+                all_data['results'] = data['results']
+
+        test_pass, test_count, pass_count, fail_count, error_count = outputResultsJSON(args.output_filename, all_data['results'], all_data['baseline'])
+        finish(test_pass, test_count, pass_count, fail_count, error_count)
+        
+    except Exception as e:
+        print("Error generating the summary file: [%s].\n\tQuitting..."%(str(e)))
+        sys.exit(1)
+
+if __name__=="__main__":
+    main()
 
 
     

From cc4f91d8c2487b47ada51956c87bf416fb71d9a6 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Wed, 8 Dec 2021 12:46:06 -0800
Subject: [PATCH 134/166] Moved everything into the bin directory.

---
 .../automated_detection_testing}/.dockerignore      |   0
 .../automated_detection_testing}/Dockerfile         |   0
 .../automated_detection_testing}/README.md          |   0
 .../architecture_automated_detection_testing.png    | Bin
 .../ci/attack_range_for_testing/Dockerfile          |   0
 .../attack_range_for_testing/detection_service.py   |   0
 .../helpers/attack_range_controller.py              |   0
 .../attack_range_for_testing/helpers/aws_service.py |   0
 .../helpers/github_service.py                       |   0
 .../ci/attack_range_for_testing/requirements.txt    |   0
 .../ansible/attack_replay.yml                       |   0
 .../ansible/roles/attack_replay/tasks/main.yml      |   0
 .../ansible/roles/update_escu/tasks/main.yml        |   0
 .../detection_testing_batch/ansible/update_escu.yml |   0
 .../ci/detection_testing_batch/datamodels.conf.tar  | Bin
 .../detection_testing_execution.py                  |   0
 .../ci/detection_testing_batch/indexes.conf.tar     | Bin
 .../modules/DataManipulation.py                     |   0
 .../ci/detection_testing_batch/modules/__init__.py  |   0
 .../detection_testing_batch/modules/aws_service.py  |   0
 .../modules/container_manager.py                    |   0
 .../modules/github_service.py                       |   0
 .../modules/jsonschema_errorprinter.py              |   0
 .../modules/new_arguments2.py                       |   0
 .../modules/splunk_container.py                     |   0
 .../detection_testing_batch/modules/splunk_sdk.py   |   0
 .../detection_testing_batch/modules/test_driver.py  |   0
 .../modules/testing_service.py                      |   0
 .../modules/tmp/tstats_endpoint_processes           |   0
 .../modules/validate_args.py                        |   0
 .../ci/detection_testing_batch/new_arguments.py     |   0
 .../ci/detection_testing_batch/requirements.txt     |   0
 .../ci/detection_testing_batch/summarize_json.py    |   0
 .../test_config_github_actions.json                 |   0
 .../ci/labeled_data/Dockerfile                      |   0
 .../ci/labeled_data/ansible/attack_replay.yml       |   0
 .../ansible/roles/attack_replay/tasks/main.yml      |   0
 .../ansible/roles/update_escu/tasks/main.yml        |   0
 .../ci/labeled_data/ansible/update_escu.yml         |   0
 .../ci/labeled_data/labeled_data.py                 |   0
 .../ci/labeled_data/modules/DataManipulation.py     |   0
 .../ci/labeled_data/modules/aws_service.py          |   0
 .../ci/labeled_data/modules/github_service.py       |   0
 .../ci/labeled_data/modules/splunk_sdk.py           |   0
 .../ci/labeled_data/modules/testing_service.py      |   0
 .../ci/labeled_data/requirements.txt                |   0
 .../ci/python_ci_code/main.py                       |   0
 .../ci/python_ci_code/requirements.txt              |   0
 .../automated_detection_testing}/config             |   0
 .../detection_service.py                            |   0
 .../automated_detection_testing}/requirements.txt   |   0
 .../templates/PR_template.j2                        |   0
 .../templates/PR_template_attack_data.j2            |   0
 53 files changed, 0 insertions(+), 0 deletions(-)
 rename {automated_detection_testing => bin/automated_detection_testing}/.dockerignore (100%)
 rename {automated_detection_testing => bin/automated_detection_testing}/Dockerfile (100%)
 rename {automated_detection_testing => bin/automated_detection_testing}/README.md (100%)
 rename {automated_detection_testing => bin/automated_detection_testing}/architecture_automated_detection_testing.png (100%)
 rename {automated_detection_testing => bin/automated_detection_testing}/ci/attack_range_for_testing/Dockerfile (100%)
 rename {automated_detection_testing => bin/automated_detection_testing}/ci/attack_range_for_testing/detection_service.py (100%)
 rename {automated_detection_testing => bin/automated_detection_testing}/ci/attack_range_for_testing/helpers/attack_range_controller.py (100%)
 rename {automated_detection_testing => bin/automated_detection_testing}/ci/attack_range_for_testing/helpers/aws_service.py (100%)
 rename {automated_detection_testing => bin/automated_detection_testing}/ci/attack_range_for_testing/helpers/github_service.py (100%)
 rename {automated_detection_testing => bin/automated_detection_testing}/ci/attack_range_for_testing/requirements.txt (100%)
 rename {automated_detection_testing => bin/automated_detection_testing}/ci/detection_testing_batch/ansible/attack_replay.yml (100%)
 rename {automated_detection_testing => bin/automated_detection_testing}/ci/detection_testing_batch/ansible/roles/attack_replay/tasks/main.yml (100%)
 rename {automated_detection_testing => bin/automated_detection_testing}/ci/detection_testing_batch/ansible/roles/update_escu/tasks/main.yml (100%)
 rename {automated_detection_testing => bin/automated_detection_testing}/ci/detection_testing_batch/ansible/update_escu.yml (100%)
 rename {automated_detection_testing => bin/automated_detection_testing}/ci/detection_testing_batch/datamodels.conf.tar (100%)
 rename {automated_detection_testing => bin/automated_detection_testing}/ci/detection_testing_batch/detection_testing_execution.py (100%)
 rename {automated_detection_testing => bin/automated_detection_testing}/ci/detection_testing_batch/indexes.conf.tar (100%)
 rename {automated_detection_testing => bin/automated_detection_testing}/ci/detection_testing_batch/modules/DataManipulation.py (100%)
 rename {automated_detection_testing => bin/automated_detection_testing}/ci/detection_testing_batch/modules/__init__.py (100%)
 rename {automated_detection_testing => bin/automated_detection_testing}/ci/detection_testing_batch/modules/aws_service.py (100%)
 rename {automated_detection_testing => bin/automated_detection_testing}/ci/detection_testing_batch/modules/container_manager.py (100%)
 rename {automated_detection_testing => bin/automated_detection_testing}/ci/detection_testing_batch/modules/github_service.py (100%)
 rename {automated_detection_testing => bin/automated_detection_testing}/ci/detection_testing_batch/modules/jsonschema_errorprinter.py (100%)
 rename {automated_detection_testing => bin/automated_detection_testing}/ci/detection_testing_batch/modules/new_arguments2.py (100%)
 rename {automated_detection_testing => bin/automated_detection_testing}/ci/detection_testing_batch/modules/splunk_container.py (100%)
 rename {automated_detection_testing => bin/automated_detection_testing}/ci/detection_testing_batch/modules/splunk_sdk.py (100%)
 rename {automated_detection_testing => bin/automated_detection_testing}/ci/detection_testing_batch/modules/test_driver.py (100%)
 rename {automated_detection_testing => bin/automated_detection_testing}/ci/detection_testing_batch/modules/testing_service.py (100%)
 rename {automated_detection_testing => bin/automated_detection_testing}/ci/detection_testing_batch/modules/tmp/tstats_endpoint_processes (100%)
 rename {automated_detection_testing => bin/automated_detection_testing}/ci/detection_testing_batch/modules/validate_args.py (100%)
 rename {automated_detection_testing => bin/automated_detection_testing}/ci/detection_testing_batch/new_arguments.py (100%)
 rename {automated_detection_testing => bin/automated_detection_testing}/ci/detection_testing_batch/requirements.txt (100%)
 rename {automated_detection_testing => bin/automated_detection_testing}/ci/detection_testing_batch/summarize_json.py (100%)
 rename {automated_detection_testing => bin/automated_detection_testing}/ci/detection_testing_batch/test_config_github_actions.json (100%)
 rename {automated_detection_testing => bin/automated_detection_testing}/ci/labeled_data/Dockerfile (100%)
 rename {automated_detection_testing => bin/automated_detection_testing}/ci/labeled_data/ansible/attack_replay.yml (100%)
 rename {automated_detection_testing => bin/automated_detection_testing}/ci/labeled_data/ansible/roles/attack_replay/tasks/main.yml (100%)
 rename {automated_detection_testing => bin/automated_detection_testing}/ci/labeled_data/ansible/roles/update_escu/tasks/main.yml (100%)
 rename {automated_detection_testing => bin/automated_detection_testing}/ci/labeled_data/ansible/update_escu.yml (100%)
 rename {automated_detection_testing => bin/automated_detection_testing}/ci/labeled_data/labeled_data.py (100%)
 rename {automated_detection_testing => bin/automated_detection_testing}/ci/labeled_data/modules/DataManipulation.py (100%)
 rename {automated_detection_testing => bin/automated_detection_testing}/ci/labeled_data/modules/aws_service.py (100%)
 rename {automated_detection_testing => bin/automated_detection_testing}/ci/labeled_data/modules/github_service.py (100%)
 rename {automated_detection_testing => bin/automated_detection_testing}/ci/labeled_data/modules/splunk_sdk.py (100%)
 rename {automated_detection_testing => bin/automated_detection_testing}/ci/labeled_data/modules/testing_service.py (100%)
 rename {automated_detection_testing => bin/automated_detection_testing}/ci/labeled_data/requirements.txt (100%)
 rename {automated_detection_testing => bin/automated_detection_testing}/ci/python_ci_code/main.py (100%)
 rename {automated_detection_testing => bin/automated_detection_testing}/ci/python_ci_code/requirements.txt (100%)
 rename {automated_detection_testing => bin/automated_detection_testing}/config (100%)
 rename {automated_detection_testing => bin/automated_detection_testing}/detection_service.py (100%)
 rename {automated_detection_testing => bin/automated_detection_testing}/requirements.txt (100%)
 rename {automated_detection_testing => bin/automated_detection_testing}/templates/PR_template.j2 (100%)
 rename {automated_detection_testing => bin/automated_detection_testing}/templates/PR_template_attack_data.j2 (100%)

diff --git a/automated_detection_testing/.dockerignore b/bin/automated_detection_testing/.dockerignore
similarity index 100%
rename from automated_detection_testing/.dockerignore
rename to bin/automated_detection_testing/.dockerignore
diff --git a/automated_detection_testing/Dockerfile b/bin/automated_detection_testing/Dockerfile
similarity index 100%
rename from automated_detection_testing/Dockerfile
rename to bin/automated_detection_testing/Dockerfile
diff --git a/automated_detection_testing/README.md b/bin/automated_detection_testing/README.md
similarity index 100%
rename from automated_detection_testing/README.md
rename to bin/automated_detection_testing/README.md
diff --git a/automated_detection_testing/architecture_automated_detection_testing.png b/bin/automated_detection_testing/architecture_automated_detection_testing.png
similarity index 100%
rename from automated_detection_testing/architecture_automated_detection_testing.png
rename to bin/automated_detection_testing/architecture_automated_detection_testing.png
diff --git a/automated_detection_testing/ci/attack_range_for_testing/Dockerfile b/bin/automated_detection_testing/ci/attack_range_for_testing/Dockerfile
similarity index 100%
rename from automated_detection_testing/ci/attack_range_for_testing/Dockerfile
rename to bin/automated_detection_testing/ci/attack_range_for_testing/Dockerfile
diff --git a/automated_detection_testing/ci/attack_range_for_testing/detection_service.py b/bin/automated_detection_testing/ci/attack_range_for_testing/detection_service.py
similarity index 100%
rename from automated_detection_testing/ci/attack_range_for_testing/detection_service.py
rename to bin/automated_detection_testing/ci/attack_range_for_testing/detection_service.py
diff --git a/automated_detection_testing/ci/attack_range_for_testing/helpers/attack_range_controller.py b/bin/automated_detection_testing/ci/attack_range_for_testing/helpers/attack_range_controller.py
similarity index 100%
rename from automated_detection_testing/ci/attack_range_for_testing/helpers/attack_range_controller.py
rename to bin/automated_detection_testing/ci/attack_range_for_testing/helpers/attack_range_controller.py
diff --git a/automated_detection_testing/ci/attack_range_for_testing/helpers/aws_service.py b/bin/automated_detection_testing/ci/attack_range_for_testing/helpers/aws_service.py
similarity index 100%
rename from automated_detection_testing/ci/attack_range_for_testing/helpers/aws_service.py
rename to bin/automated_detection_testing/ci/attack_range_for_testing/helpers/aws_service.py
diff --git a/automated_detection_testing/ci/attack_range_for_testing/helpers/github_service.py b/bin/automated_detection_testing/ci/attack_range_for_testing/helpers/github_service.py
similarity index 100%
rename from automated_detection_testing/ci/attack_range_for_testing/helpers/github_service.py
rename to bin/automated_detection_testing/ci/attack_range_for_testing/helpers/github_service.py
diff --git a/automated_detection_testing/ci/attack_range_for_testing/requirements.txt b/bin/automated_detection_testing/ci/attack_range_for_testing/requirements.txt
similarity index 100%
rename from automated_detection_testing/ci/attack_range_for_testing/requirements.txt
rename to bin/automated_detection_testing/ci/attack_range_for_testing/requirements.txt
diff --git a/automated_detection_testing/ci/detection_testing_batch/ansible/attack_replay.yml b/bin/automated_detection_testing/ci/detection_testing_batch/ansible/attack_replay.yml
similarity index 100%
rename from automated_detection_testing/ci/detection_testing_batch/ansible/attack_replay.yml
rename to bin/automated_detection_testing/ci/detection_testing_batch/ansible/attack_replay.yml
diff --git a/automated_detection_testing/ci/detection_testing_batch/ansible/roles/attack_replay/tasks/main.yml b/bin/automated_detection_testing/ci/detection_testing_batch/ansible/roles/attack_replay/tasks/main.yml
similarity index 100%
rename from automated_detection_testing/ci/detection_testing_batch/ansible/roles/attack_replay/tasks/main.yml
rename to bin/automated_detection_testing/ci/detection_testing_batch/ansible/roles/attack_replay/tasks/main.yml
diff --git a/automated_detection_testing/ci/detection_testing_batch/ansible/roles/update_escu/tasks/main.yml b/bin/automated_detection_testing/ci/detection_testing_batch/ansible/roles/update_escu/tasks/main.yml
similarity index 100%
rename from automated_detection_testing/ci/detection_testing_batch/ansible/roles/update_escu/tasks/main.yml
rename to bin/automated_detection_testing/ci/detection_testing_batch/ansible/roles/update_escu/tasks/main.yml
diff --git a/automated_detection_testing/ci/detection_testing_batch/ansible/update_escu.yml b/bin/automated_detection_testing/ci/detection_testing_batch/ansible/update_escu.yml
similarity index 100%
rename from automated_detection_testing/ci/detection_testing_batch/ansible/update_escu.yml
rename to bin/automated_detection_testing/ci/detection_testing_batch/ansible/update_escu.yml
diff --git a/automated_detection_testing/ci/detection_testing_batch/datamodels.conf.tar b/bin/automated_detection_testing/ci/detection_testing_batch/datamodels.conf.tar
similarity index 100%
rename from automated_detection_testing/ci/detection_testing_batch/datamodels.conf.tar
rename to bin/automated_detection_testing/ci/detection_testing_batch/datamodels.conf.tar
diff --git a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py b/bin/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
similarity index 100%
rename from automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
rename to bin/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
diff --git a/automated_detection_testing/ci/detection_testing_batch/indexes.conf.tar b/bin/automated_detection_testing/ci/detection_testing_batch/indexes.conf.tar
similarity index 100%
rename from automated_detection_testing/ci/detection_testing_batch/indexes.conf.tar
rename to bin/automated_detection_testing/ci/detection_testing_batch/indexes.conf.tar
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/DataManipulation.py b/bin/automated_detection_testing/ci/detection_testing_batch/modules/DataManipulation.py
similarity index 100%
rename from automated_detection_testing/ci/detection_testing_batch/modules/DataManipulation.py
rename to bin/automated_detection_testing/ci/detection_testing_batch/modules/DataManipulation.py
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/__init__.py b/bin/automated_detection_testing/ci/detection_testing_batch/modules/__init__.py
similarity index 100%
rename from automated_detection_testing/ci/detection_testing_batch/modules/__init__.py
rename to bin/automated_detection_testing/ci/detection_testing_batch/modules/__init__.py
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/aws_service.py b/bin/automated_detection_testing/ci/detection_testing_batch/modules/aws_service.py
similarity index 100%
rename from automated_detection_testing/ci/detection_testing_batch/modules/aws_service.py
rename to bin/automated_detection_testing/ci/detection_testing_batch/modules/aws_service.py
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/container_manager.py b/bin/automated_detection_testing/ci/detection_testing_batch/modules/container_manager.py
similarity index 100%
rename from automated_detection_testing/ci/detection_testing_batch/modules/container_manager.py
rename to bin/automated_detection_testing/ci/detection_testing_batch/modules/container_manager.py
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/github_service.py b/bin/automated_detection_testing/ci/detection_testing_batch/modules/github_service.py
similarity index 100%
rename from automated_detection_testing/ci/detection_testing_batch/modules/github_service.py
rename to bin/automated_detection_testing/ci/detection_testing_batch/modules/github_service.py
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/jsonschema_errorprinter.py b/bin/automated_detection_testing/ci/detection_testing_batch/modules/jsonschema_errorprinter.py
similarity index 100%
rename from automated_detection_testing/ci/detection_testing_batch/modules/jsonschema_errorprinter.py
rename to bin/automated_detection_testing/ci/detection_testing_batch/modules/jsonschema_errorprinter.py
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/new_arguments2.py b/bin/automated_detection_testing/ci/detection_testing_batch/modules/new_arguments2.py
similarity index 100%
rename from automated_detection_testing/ci/detection_testing_batch/modules/new_arguments2.py
rename to bin/automated_detection_testing/ci/detection_testing_batch/modules/new_arguments2.py
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/splunk_container.py b/bin/automated_detection_testing/ci/detection_testing_batch/modules/splunk_container.py
similarity index 100%
rename from automated_detection_testing/ci/detection_testing_batch/modules/splunk_container.py
rename to bin/automated_detection_testing/ci/detection_testing_batch/modules/splunk_container.py
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/splunk_sdk.py b/bin/automated_detection_testing/ci/detection_testing_batch/modules/splunk_sdk.py
similarity index 100%
rename from automated_detection_testing/ci/detection_testing_batch/modules/splunk_sdk.py
rename to bin/automated_detection_testing/ci/detection_testing_batch/modules/splunk_sdk.py
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/test_driver.py b/bin/automated_detection_testing/ci/detection_testing_batch/modules/test_driver.py
similarity index 100%
rename from automated_detection_testing/ci/detection_testing_batch/modules/test_driver.py
rename to bin/automated_detection_testing/ci/detection_testing_batch/modules/test_driver.py
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/testing_service.py b/bin/automated_detection_testing/ci/detection_testing_batch/modules/testing_service.py
similarity index 100%
rename from automated_detection_testing/ci/detection_testing_batch/modules/testing_service.py
rename to bin/automated_detection_testing/ci/detection_testing_batch/modules/testing_service.py
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/tmp/tstats_endpoint_processes b/bin/automated_detection_testing/ci/detection_testing_batch/modules/tmp/tstats_endpoint_processes
similarity index 100%
rename from automated_detection_testing/ci/detection_testing_batch/modules/tmp/tstats_endpoint_processes
rename to bin/automated_detection_testing/ci/detection_testing_batch/modules/tmp/tstats_endpoint_processes
diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py b/bin/automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py
similarity index 100%
rename from automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py
rename to bin/automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py
diff --git a/automated_detection_testing/ci/detection_testing_batch/new_arguments.py b/bin/automated_detection_testing/ci/detection_testing_batch/new_arguments.py
similarity index 100%
rename from automated_detection_testing/ci/detection_testing_batch/new_arguments.py
rename to bin/automated_detection_testing/ci/detection_testing_batch/new_arguments.py
diff --git a/automated_detection_testing/ci/detection_testing_batch/requirements.txt b/bin/automated_detection_testing/ci/detection_testing_batch/requirements.txt
similarity index 100%
rename from automated_detection_testing/ci/detection_testing_batch/requirements.txt
rename to bin/automated_detection_testing/ci/detection_testing_batch/requirements.txt
diff --git a/automated_detection_testing/ci/detection_testing_batch/summarize_json.py b/bin/automated_detection_testing/ci/detection_testing_batch/summarize_json.py
similarity index 100%
rename from automated_detection_testing/ci/detection_testing_batch/summarize_json.py
rename to bin/automated_detection_testing/ci/detection_testing_batch/summarize_json.py
diff --git a/automated_detection_testing/ci/detection_testing_batch/test_config_github_actions.json b/bin/automated_detection_testing/ci/detection_testing_batch/test_config_github_actions.json
similarity index 100%
rename from automated_detection_testing/ci/detection_testing_batch/test_config_github_actions.json
rename to bin/automated_detection_testing/ci/detection_testing_batch/test_config_github_actions.json
diff --git a/automated_detection_testing/ci/labeled_data/Dockerfile b/bin/automated_detection_testing/ci/labeled_data/Dockerfile
similarity index 100%
rename from automated_detection_testing/ci/labeled_data/Dockerfile
rename to bin/automated_detection_testing/ci/labeled_data/Dockerfile
diff --git a/automated_detection_testing/ci/labeled_data/ansible/attack_replay.yml b/bin/automated_detection_testing/ci/labeled_data/ansible/attack_replay.yml
similarity index 100%
rename from automated_detection_testing/ci/labeled_data/ansible/attack_replay.yml
rename to bin/automated_detection_testing/ci/labeled_data/ansible/attack_replay.yml
diff --git a/automated_detection_testing/ci/labeled_data/ansible/roles/attack_replay/tasks/main.yml b/bin/automated_detection_testing/ci/labeled_data/ansible/roles/attack_replay/tasks/main.yml
similarity index 100%
rename from automated_detection_testing/ci/labeled_data/ansible/roles/attack_replay/tasks/main.yml
rename to bin/automated_detection_testing/ci/labeled_data/ansible/roles/attack_replay/tasks/main.yml
diff --git a/automated_detection_testing/ci/labeled_data/ansible/roles/update_escu/tasks/main.yml b/bin/automated_detection_testing/ci/labeled_data/ansible/roles/update_escu/tasks/main.yml
similarity index 100%
rename from automated_detection_testing/ci/labeled_data/ansible/roles/update_escu/tasks/main.yml
rename to bin/automated_detection_testing/ci/labeled_data/ansible/roles/update_escu/tasks/main.yml
diff --git a/automated_detection_testing/ci/labeled_data/ansible/update_escu.yml b/bin/automated_detection_testing/ci/labeled_data/ansible/update_escu.yml
similarity index 100%
rename from automated_detection_testing/ci/labeled_data/ansible/update_escu.yml
rename to bin/automated_detection_testing/ci/labeled_data/ansible/update_escu.yml
diff --git a/automated_detection_testing/ci/labeled_data/labeled_data.py b/bin/automated_detection_testing/ci/labeled_data/labeled_data.py
similarity index 100%
rename from automated_detection_testing/ci/labeled_data/labeled_data.py
rename to bin/automated_detection_testing/ci/labeled_data/labeled_data.py
diff --git a/automated_detection_testing/ci/labeled_data/modules/DataManipulation.py b/bin/automated_detection_testing/ci/labeled_data/modules/DataManipulation.py
similarity index 100%
rename from automated_detection_testing/ci/labeled_data/modules/DataManipulation.py
rename to bin/automated_detection_testing/ci/labeled_data/modules/DataManipulation.py
diff --git a/automated_detection_testing/ci/labeled_data/modules/aws_service.py b/bin/automated_detection_testing/ci/labeled_data/modules/aws_service.py
similarity index 100%
rename from automated_detection_testing/ci/labeled_data/modules/aws_service.py
rename to bin/automated_detection_testing/ci/labeled_data/modules/aws_service.py
diff --git a/automated_detection_testing/ci/labeled_data/modules/github_service.py b/bin/automated_detection_testing/ci/labeled_data/modules/github_service.py
similarity index 100%
rename from automated_detection_testing/ci/labeled_data/modules/github_service.py
rename to bin/automated_detection_testing/ci/labeled_data/modules/github_service.py
diff --git a/automated_detection_testing/ci/labeled_data/modules/splunk_sdk.py b/bin/automated_detection_testing/ci/labeled_data/modules/splunk_sdk.py
similarity index 100%
rename from automated_detection_testing/ci/labeled_data/modules/splunk_sdk.py
rename to bin/automated_detection_testing/ci/labeled_data/modules/splunk_sdk.py
diff --git a/automated_detection_testing/ci/labeled_data/modules/testing_service.py b/bin/automated_detection_testing/ci/labeled_data/modules/testing_service.py
similarity index 100%
rename from automated_detection_testing/ci/labeled_data/modules/testing_service.py
rename to bin/automated_detection_testing/ci/labeled_data/modules/testing_service.py
diff --git a/automated_detection_testing/ci/labeled_data/requirements.txt b/bin/automated_detection_testing/ci/labeled_data/requirements.txt
similarity index 100%
rename from automated_detection_testing/ci/labeled_data/requirements.txt
rename to bin/automated_detection_testing/ci/labeled_data/requirements.txt
diff --git a/automated_detection_testing/ci/python_ci_code/main.py b/bin/automated_detection_testing/ci/python_ci_code/main.py
similarity index 100%
rename from automated_detection_testing/ci/python_ci_code/main.py
rename to bin/automated_detection_testing/ci/python_ci_code/main.py
diff --git a/automated_detection_testing/ci/python_ci_code/requirements.txt b/bin/automated_detection_testing/ci/python_ci_code/requirements.txt
similarity index 100%
rename from automated_detection_testing/ci/python_ci_code/requirements.txt
rename to bin/automated_detection_testing/ci/python_ci_code/requirements.txt
diff --git a/automated_detection_testing/config b/bin/automated_detection_testing/config
similarity index 100%
rename from automated_detection_testing/config
rename to bin/automated_detection_testing/config
diff --git a/automated_detection_testing/detection_service.py b/bin/automated_detection_testing/detection_service.py
similarity index 100%
rename from automated_detection_testing/detection_service.py
rename to bin/automated_detection_testing/detection_service.py
diff --git a/automated_detection_testing/requirements.txt b/bin/automated_detection_testing/requirements.txt
similarity index 100%
rename from automated_detection_testing/requirements.txt
rename to bin/automated_detection_testing/requirements.txt
diff --git a/automated_detection_testing/templates/PR_template.j2 b/bin/automated_detection_testing/templates/PR_template.j2
similarity index 100%
rename from automated_detection_testing/templates/PR_template.j2
rename to bin/automated_detection_testing/templates/PR_template.j2
diff --git a/automated_detection_testing/templates/PR_template_attack_data.j2 b/bin/automated_detection_testing/templates/PR_template_attack_data.j2
similarity index 100%
rename from automated_detection_testing/templates/PR_template_attack_data.j2
rename to bin/automated_detection_testing/templates/PR_template_attack_data.j2

From 5a554f3b5b59b019dd9746645d46e5d4bf137a9f Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Wed, 8 Dec 2021 14:09:22 -0800
Subject: [PATCH 135/166] Updated paths in workflows that changed due to
 relocating the automated_detection_testing folder in bin/

---
 .github/workflows/build-and-validate.yml | 82 ++++++++++++------------
 1 file changed, 41 insertions(+), 41 deletions(-)

diff --git a/.github/workflows/build-and-validate.yml b/.github/workflows/build-and-validate.yml
index fce542b2ee..c6ed4e4db5 100644
--- a/.github/workflows/build-and-validate.yml
+++ b/.github/workflows/build-and-validate.yml
@@ -324,7 +324,7 @@ jobs:
           uses: actions/download-artifact@v2
           with:
             name: content-latest
-            path: automated_detection_testing/ci/detection_testing_batch/prior_config/apps
+            path: bin/automated_detection_testing/ci/detection_testing_batch/prior_config/apps
 
         - uses: actions/setup-python@v2
           with:
@@ -333,7 +333,7 @@ jobs:
         
         - name: Install Python Dependencies
           run: |
-            cd automated_detection_testing/ci/detection_testing_batch
+            cd bin/automated_detection_testing/ci/detection_testing_batch
             python3 -m venv .venv
             source .venv/bin/activate
             python3 -m pip install wheel
@@ -341,7 +341,7 @@ jobs:
         
         - name: Run the CI
           run: |
-            cd automated_detection_testing/ci/detection_testing_batch
+            cd bin/automated_detection_testing/ci/detection_testing_batch
             source .venv/bin/activate
             echo "github.event.issue.pull_request    : [${{ github.event.issue.pull_request }}]"
             echo "github.event.pull_request.number   : [${{ github.event.pull_request.number }}]"
@@ -367,24 +367,24 @@ jobs:
           with:
             name: testing-results-config
             path: |
-              automated_detection_testing/ci/detection_testing_batch/prior_config/apps/DA-ESS-ContentUpdate-latest.tar.gz
-              automated_detection_testing/ci/detection_testing_batch/prior_config/config_tests_0.json
-              automated_detection_testing/ci/detection_testing_batch/prior_config/config_tests_1.json
-              automated_detection_testing/ci/detection_testing_batch/prior_config/config_tests_2.json
-              automated_detection_testing/ci/detection_testing_batch/prior_config/config_tests_3.json
-              automated_detection_testing/ci/detection_testing_batch/prior_config/config_tests_4.json
-              automated_detection_testing/ci/detection_testing_batch/prior_config/config_tests_5.json
-              automated_detection_testing/ci/detection_testing_batch/prior_config/config_tests_6.json
-              automated_detection_testing/ci/detection_testing_batch/prior_config/config_tests_7.json
-              automated_detection_testing/ci/detection_testing_batch/prior_config/config_tests_8.json
-              automated_detection_testing/ci/detection_testing_batch/prior_config/config_tests_9.json
+              bin/automated_detection_testing/ci/detection_testing_batch/prior_config/apps/DA-ESS-ContentUpdate-latest.tar.gz
+              bin/automated_detection_testing/ci/detection_testing_batch/prior_config/config_tests_0.json
+              bin/automated_detection_testing/ci/detection_testing_batch/prior_config/config_tests_1.json
+              bin/automated_detection_testing/ci/detection_testing_batch/prior_config/config_tests_2.json
+              bin/automated_detection_testing/ci/detection_testing_batch/prior_config/config_tests_3.json
+              bin/automated_detection_testing/ci/detection_testing_batch/prior_config/config_tests_4.json
+              bin/automated_detection_testing/ci/detection_testing_batch/prior_config/config_tests_5.json
+              bin/automated_detection_testing/ci/detection_testing_batch/prior_config/config_tests_6.json
+              bin/automated_detection_testing/ci/detection_testing_batch/prior_config/config_tests_7.json
+              bin/automated_detection_testing/ci/detection_testing_batch/prior_config/config_tests_8.json
+              bin/automated_detection_testing/ci/detection_testing_batch/prior_config/config_tests_9.json
 
         - name: Upload File to Enable Replication of the Test at a Different Time or Place
           uses: actions/upload-artifact@v2
           with:
             name: replicate_test
             path: |
-              automated_detection_testing/ci/detection_testing_batch/replicate_test.json
+              bin/automated_detection_testing/ci/detection_testing_batch/replicate_test.json
 
   docker-detection-testing-execution:
     runs-on: ubuntu-latest
@@ -414,7 +414,7 @@ jobs:
           uses: actions/download-artifact@v2
           with:
             name: testing-results-config
-            path: automated_detection_testing/ci/detection_testing_batch/prior_config
+            path: bin/automated_detection_testing/ci/detection_testing_batch/prior_config
         # - name: Install Docker 
         #   run: |
         #     sudo apt update -qq
@@ -433,7 +433,7 @@ jobs:
         
         - name: Install Python Dependencies
           run: |
-            cd automated_detection_testing/ci/detection_testing_batch
+            cd bin/automated_detection_testing/ci/detection_testing_batch
             python3 -m venv .venv
             source .venv/bin/activate
             python3 -m pip install wheel
@@ -441,7 +441,7 @@ jobs:
         
         - name: Run the CI
           run: |
-            cd automated_detection_testing/ci/detection_testing_batch
+            cd bin/automated_detection_testing/ci/detection_testing_batch
             source .venv/bin/activate
             
             python3 detection_testing_execution.py run -c prior_config/${{ matrix.manifest_filename}} --splunkbase_username ${{ secrets.SPLUNKBASE_TESTING_USERNAME }} --splunkbase_password ${{ secrets.SPLUNKBASE_TESTING_KEY }} 
@@ -452,16 +452,16 @@ jobs:
           with:
             name: ${{ matrix.manifest_filename}}.results
             path: |
-              automated_detection_testing/ci/detection_testing_batch/test_results/success.csv
-              automated_detection_testing/ci/detection_testing_batch/test_results/error.csv
-              automated_detection_testing/ci/detection_testing_batch/test_results/failure.csv
-              automated_detection_testing/ci/detection_testing_batch/test_results/combined.csv
-              automated_detection_testing/ci/detection_testing_batch/test_results/success.json
-              automated_detection_testing/ci/detection_testing_batch/test_results/error.json
-              automated_detection_testing/ci/detection_testing_batch/test_results/failure.json
-              automated_detection_testing/ci/detection_testing_batch/test_results/combined.json
+              bin/automated_detection_testing/ci/detection_testing_batch/test_results/success.csv
+              bin/automated_detection_testing/ci/detection_testing_batch/test_results/error.csv
+              bin/automated_detection_testing/ci/detection_testing_batch/test_results/failure.csv
+              bin/automated_detection_testing/ci/detection_testing_batch/test_results/combined.csv
+              bin/automated_detection_testing/ci/detection_testing_batch/test_results/success.json
+              bin/automated_detection_testing/ci/detection_testing_batch/test_results/error.json
+              bin/automated_detection_testing/ci/detection_testing_batch/test_results/failure.json
+              bin/automated_detection_testing/ci/detection_testing_batch/test_results/combined.json
 
-              automated_detection_testing/ci/detection_testing_batch/test_results/summary.json
+              bin/automated_detection_testing/ci/detection_testing_batch/test_results/summary.json
           
   docker-detection-testing-execution-merge-results:
     runs-on: ubuntu-latest
@@ -480,52 +480,52 @@ jobs:
           uses: actions/download-artifact@v2
           with:
             name: config_tests_0.json.results
-            path: automated_detection_testing/ci/detection_testing_batch/results_0
+            path: bin/automated_detection_testing/ci/detection_testing_batch/results_0
         - name: Download artifacts
           uses: actions/download-artifact@v2
           with:
             name: config_tests_1.json.results
-            path: automated_detection_testing/ci/detection_testing_batch/results_1
+            path: bin/automated_detection_testing/ci/detection_testing_batch/results_1
         - name: Download artifacts
           uses: actions/download-artifact@v2
           with:
             name: config_tests_2.json.results
-            path: automated_detection_testing/ci/detection_testing_batch/results_2
+            path: bin/automated_detection_testing/ci/detection_testing_batch/results_2
         - name: Download artifacts
           uses: actions/download-artifact@v2
           with:
             name: config_tests_3.json.results
-            path: automated_detection_testing/ci/detection_testing_batch/results_3
+            path: bin/automated_detection_testing/ci/detection_testing_batch/results_3
         - name: Download artifacts
           uses: actions/download-artifact@v2
           with:
             name: config_tests_4.json.results
-            path: automated_detection_testing/ci/detection_testing_batch/results_4
+            path: bin/automated_detection_testing/ci/detection_testing_batch/results_4
         - name: Download artifacts
           uses: actions/download-artifact@v2
           with:
             name: config_tests_5.json.results
-            path: automated_detection_testing/ci/detection_testing_batch/results_5
+            path: bin/automated_detection_testing/ci/detection_testing_batch/results_5
         - name: Download artifacts
           uses: actions/download-artifact@v2
           with:
             name: config_tests_6.json.results
-            path: automated_detection_testing/ci/detection_testing_batch/results_6
+            path: bin/automated_detection_testing/ci/detection_testing_batch/results_6
         - name: Download artifacts
           uses: actions/download-artifact@v2
           with:
             name: config_tests_7.json.results
-            path: automated_detection_testing/ci/detection_testing_batch/results_7
+            path: bin/automated_detection_testing/ci/detection_testing_batch/results_7
         - name: Download artifacts
           uses: actions/download-artifact@v2
           with:
             name: config_tests_8.json.results
-            path: automated_detection_testing/ci/detection_testing_batch/results_8
+            path: bin/automated_detection_testing/ci/detection_testing_batch/results_8
         - name: Download artifacts
           uses: actions/download-artifact@v2
           with:
             name: config_tests_9.json.results
-            path: automated_detection_testing/ci/detection_testing_batch/results_9
+            path: bin/automated_detection_testing/ci/detection_testing_batch/results_9
         
         - uses: actions/setup-python@v2
           with:
@@ -534,7 +534,7 @@ jobs:
         
         - name: Install Python Dependencies
           run: |
-            cd automated_detection_testing/ci/detection_testing_batch
+            cd bin/automated_detection_testing/ci/detection_testing_batch
             python3 -m venv .venv
             source .venv/bin/activate
             python3 -m pip install wheel
@@ -542,7 +542,7 @@ jobs:
 
         - name: Merge Detections into single File
           run: |
-            cd automated_detection_testing/ci/detection_testing_batch
+            cd bin/automated_detection_testing/ci/detection_testing_batch
             source .venv/bin/activate
             python summarize_json.py --files results_*/summary.json --output_filename summary_test_results.json
             
@@ -553,7 +553,7 @@ jobs:
           with:
             name: SummaryTestResults
             path: |
-              automated_detection_testing/ci/detection_testing_batch/summary_test_results.json
+              bin/automated_detection_testing/ci/detection_testing_batch/summary_test_results.json
 
         - name: Upload Failures Manifest on Failure
           uses: actions/upload-artifact@v2
@@ -561,7 +561,7 @@ jobs:
           with:
             name: DetectionFailureManifest
             path: |
-              automated_detection_testing/ci/detection_testing_batch/detection_failure_manifest.json
+              bin/automated_detection_testing/ci/detection_testing_batch/detection_failure_manifest.json
             
         
         #Always clean these up, they make the output messy

From 49695ed2c5882d54f131d4c15f89f81028644de9 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Tue, 14 Dec 2021 12:59:17 -0800
Subject: [PATCH 136/166] Preparing fixes for properly allowing testing to
 include or exclude folders.  For example, by default we will not include and
 test detections or tests in experimental.

---
 .../modules/github_service.py                 | 86 ++++++++++++++++---
 .../modules/splunk_container.py               |  7 +-
 2 files changed, 81 insertions(+), 12 deletions(-)

diff --git a/bin/automated_detection_testing/ci/detection_testing_batch/modules/github_service.py b/bin/automated_detection_testing/ci/detection_testing_batch/modules/github_service.py
index 71c14c3104..c12fa2bf76 100644
--- a/bin/automated_detection_testing/ci/detection_testing_batch/modules/github_service.py
+++ b/bin/automated_detection_testing/ci/detection_testing_batch/modules/github_service.py
@@ -7,11 +7,12 @@ import subprocess
 import sys
 from typing import Union
 from docker import types
-
+import datetime
 import git
 import yaml
 from git.objects import base
-
+from modules import testing_service
+import pathlib
 
 # Logger
 logging.basicConfig(level=os.environ.get("LOGLEVEL", "INFO"))
@@ -62,6 +63,56 @@ class GithubService:
         self.commit_hash = commit_hash
 
 
+    def update_and_commit_passed_tests(self, results:list[dict])->bool:
+        
+        changed_file_paths = []
+        for result in results:
+            detection_obj_path = os.path.join("security_content","detections",result['detection_file'])
+            
+            test_obj_path = detection_obj_path.replace("detections", "tests", 1)
+            test_obj_path = test_obj_path.replace(".yml",".test.yml")
+
+            detection_obj = testing_service.load_file(detection_obj_path)
+            test_obj = testing_service.load_file(test_obj_path)
+            detection_obj['tags']['automated_detection_testing'] = 'passed'
+            #detection_obj['tags']['automated_detection_testing_date'] = datetime.datetime.today().strftime('%Y-%m-%d-%H:%M:%S')
+            
+            for o in test_obj['tests']:
+                if 'attack_data' in o:
+                    datasets = []
+                    for dataset in o['attack_data']:
+                        datasets.append(dataset['data'])
+                    detection_obj['tags']['dataset'] = datasets
+            with open(detection_obj_path, "w") as f:
+                yaml.dump(detection_obj, f, sort_keys=False, allow_unicode=True)
+            
+            changed_file_paths.append(detection_obj_path)
+        
+        relpaths = [pathlib.Path(*pathlib.Path(p).parts[1:]).as_posix() for p in changed_file_paths]
+        newpath = relpaths[0]+'.wow'
+        relpaths.append(newpath)
+        with open('security_content/' + newpath,'w') as d:
+                d.write("fake file")
+        print("status results:")
+        print(self.security_content_repo_obj.index.diff(self.security_content_repo_obj.head.commit))
+        
+        if len(relpaths) > 0:
+            print('there is at least one changed file')
+            print(relpaths)            
+            self.security_content_repo_obj.index.add(relpaths)
+            print("status results after add:")
+            print(self.security_content_repo_obj.index.diff(self.security_content_repo_obj.head.commit))
+
+            commit_message = "The following detections passed detection testing.  Their YAMLs have been updated and their datasets linked:\n - %s"%("\n - ".join(relpaths))
+            self.security_content_repo_obj.index.commit(commit_message)
+            return True
+        else:
+            return False
+                
+        
+
+        
+        return True
 
     def clone_project(self, url, project, branch):
         LOGGER.info(f"Clone Security Content Project")
@@ -214,9 +265,9 @@ class GithubService:
         branch1 = self.security_content_branch
         branch2 = 'develop'
         g = git.Git('security_content')
-        changed_test_files = []
+        all_changed_test_files = []
 
-        changed_detection_files = []
+        all_changed_detection_files = []
         if branch1 != 'develop':
             if self.commit_hash is None:
                 differ = g.diff('--name-status', branch2 + '...' + branch1)
@@ -230,11 +281,11 @@ class GithubService:
                 # added or changed test files
                 if file_path.startswith('A') or file_path.startswith('M'):
                     if 'tests' in file_path and os.path.basename(file_path).endswith('.test.yml'):
-                        changed_test_files.append(file_path)
+                        all_changed_test_files.append(file_path)
 
                     # changed detections
                     if 'detections' in file_path and os.path.basename(file_path).endswith('.yml'):
-                        changed_detection_files.append(file_path)
+                        all_changed_detection_files.append(file_path)
         else:
             print("Looking for changed detections by diffing [%s] against [%s].  They are the same branch, so none were returned." % (
                 branch1, branch2), file=sys.stderr)
@@ -242,11 +293,26 @@ class GithubService:
 
         
         # all files have the format A\tFILENAME or M\tFILENAME.  Get rid of those leading characters
-        changed_test_files = [os.path.join("security_content", name.split(
-            '\t')[1]) for name in changed_test_files if len(name.split('\t')) == 2]
-        changed_detection_files = [os.path.join("security_content", name.split(
-            '\t')[1]) for name in changed_detection_files if len(name.split('\t')) == 2]
+        all_changed_test_files = [os.path.join("security_content", name.split(
+            '\t')[1]) for name in all_changed_test_files if len(name.split('\t')) == 2]
 
+        all_changed_detection_files = [os.path.join("security_content", name.split(
+            '\t')[1]) for name in all_changed_detection_files if len(name.split('\t')) == 2]
+         
+
+        #Trim out any of the tests/detection  that are not in the selected folders, but at least print a notice
+        # to the user.
+        changed_test_files = [x for x in all_changed_test_files if len(pathlib.Path(x).parts) > 3 and 
+                             pathlib.Path(x).parts[2] in folders ]
+        changed_detection_files = [x for x in all_changed_detection_files if 
+                                  (len(pathlib.Path(x).parts) > 3 and pathlib.Path(x).parts[2] in folders) ]
+       
+        #Print out the skipped tests to the user
+        for missing in set(changed_test_files).intersection(all_changed_test_files):
+            print("Ignoring modified test [%s] not in set of selected folders: %s"%(missing,folders)) 
+        
+        for missing in set(changed_detection_files).intersection(all_changed_detection_files):
+            print("Ignoring modified detecton [%s] not in set of selected folders: %s"%(missing,folders))
         
         # Convert the test files to the detection file equivalent. 
         # Note that some of these tests may be baselines and their associated 
diff --git a/bin/automated_detection_testing/ci/detection_testing_batch/modules/splunk_container.py b/bin/automated_detection_testing/ci/detection_testing_batch/modules/splunk_container.py
index 251a0fab14..71ef0f66fc 100644
--- a/bin/automated_detection_testing/ci/detection_testing_batch/modules/splunk_container.py
+++ b/bin/automated_detection_testing/ci/detection_testing_batch/modules/splunk_container.py
@@ -355,6 +355,10 @@ class SplunkContainer:
 
             # Try to get something from the queue
             detection_to_test = self.synchronization_object.getTest()
+
+            # Sleep for a small random time so that containers drift apart and don't synchronize their testing
+            time.sleep(random.randint(1, 30))
+            
             if detection_to_test is None:
                 try:
                     print(
@@ -404,5 +408,4 @@ class SplunkContainer:
                 )
             self.num_tests_completed += 1
 
-            # Sleep for a small random time so that containers drift apart and don't synchronize their testing
-            time.sleep(random.randint(1, 30))
+            

From a475df1cb0b8e3c82d9293753df27019ee586976 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Tue, 14 Dec 2021 13:01:07 -0800
Subject: [PATCH 137/166] Removed throwing error if experimental is in the path
 of a test/detection.

---
 .../ci/detection_testing_batch/modules/github_service.py    | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/bin/automated_detection_testing/ci/detection_testing_batch/modules/github_service.py b/bin/automated_detection_testing/ci/detection_testing_batch/modules/github_service.py
index c12fa2bf76..742ccc2f3c 100644
--- a/bin/automated_detection_testing/ci/detection_testing_batch/modules/github_service.py
+++ b/bin/automated_detection_testing/ci/detection_testing_batch/modules/github_service.py
@@ -324,9 +324,9 @@ class GithubService:
         #    converted_test_files.append(detection_filename)
         
         #check and throw an error if we somehow got experimental tests
-        experimental_tests = [x for x in changed_test_files if 'experimental' in x]
-        if len(experimental_tests) > 0:
-            raise(Exception("Error - expected no experimental detections, but found:\n\t%s]"%("\n\t".join(experimental_tests))))
+        #experimental_tests = [x for x in changed_test_files if 'experimental' in x]
+        #if len(experimental_tests) > 0:
+        #    raise(Exception("Error - expected no experimental detections, but found:\n\t%s]"%("\n\t".join(experimental_tests))))
 
         #Get the appropriate detection file paths for a modified test file
         for test_filepath in changed_test_files:

From 34cfcef5cbea2b9896e729f6fa184b7621d64567 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Tue, 14 Dec 2021 13:14:31 -0800
Subject: [PATCH 138/166] Fixed a bug where we were printing the wrong skipped
 tests due to folder names.  Instead we were printing the tests we did not
 skip.

---
 .../ci/detection_testing_batch/modules/github_service.py      | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/bin/automated_detection_testing/ci/detection_testing_batch/modules/github_service.py b/bin/automated_detection_testing/ci/detection_testing_batch/modules/github_service.py
index 742ccc2f3c..fd0c6b1afb 100644
--- a/bin/automated_detection_testing/ci/detection_testing_batch/modules/github_service.py
+++ b/bin/automated_detection_testing/ci/detection_testing_batch/modules/github_service.py
@@ -308,10 +308,10 @@ class GithubService:
                                   (len(pathlib.Path(x).parts) > 3 and pathlib.Path(x).parts[2] in folders) ]
        
         #Print out the skipped tests to the user
-        for missing in set(changed_test_files).intersection(all_changed_test_files):
+        for missing in set(changed_test_files).symmetric_difference(all_changed_test_files):
             print("Ignoring modified test [%s] not in set of selected folders: %s"%(missing,folders)) 
         
-        for missing in set(changed_detection_files).intersection(all_changed_detection_files):
+        for missing in set(changed_detection_files).symmetric_difference(all_changed_detection_files):
             print("Ignoring modified detecton [%s] not in set of selected folders: %s"%(missing,folders))
         
         # Convert the test files to the detection file equivalent. 

From c8082984f063417a1065ee50ffb387abed127bca Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Fri, 17 Dec 2021 10:13:49 -0800
Subject: [PATCH 139/166] A large number of changes and fixes to make
 everything smoother. The largest differences are improving detection of
 containers that take too long to start (or that crash while) they are
 starting by adding a timeout. That timeout is set to 360 seconds and might
 need to be tuned in the future to a larger number since this was tested on a
 fast machine with fast network.  The other large change is an initial pass at
 updating and committing detections that have passed the test back to the
 repo. This still needs a lot of testing and refinement.

---
 .../detection_testing_execution.py            |  6 ++
 .../modules/github_service.py                 | 12 +--
 .../modules/splunk_container.py               | 86 +++++++++----------
 .../modules/splunk_sdk.py                     | 43 ++++++++--
 .../modules/validate_args.py                  |  2 +-
 .../detection_testing_batch/requirements.txt  |  1 +
 6 files changed, 87 insertions(+), 63 deletions(-)

diff --git a/bin/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py b/bin/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
index 7340dd969e..2bc41acdf8 100644
--- a/bin/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
+++ b/bin/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
@@ -335,6 +335,9 @@ def main(args: list[str]):
               (settings['commit_hash'],settings['branch'],settings['pr_number']),file=sys.stderr)
         sys.exit(1)
 
+    #passes = [{'search_string': '| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) as lastTime from datamodel=Endpoint.Processes where (Processes.process_name ="7z.exe" OR Processes.process_name = "7za.exe" OR Processes.original_file_name = "7z.exe" OR Processes.original_file_name =  "7za.exe") AND (Processes.process="*\\\\C$\\\\*" OR Processes.process="*\\\\Admin$\\\\*" OR Processes.process="*\\\\IPC$\\\\*") by Processes.original_file_name Processes.parent_process_name Processes.parent_process Processes.process_name Processes.process Processes.parent_process_id Processes.process_id  Processes.dest Processes.user | `drop_dm_object_name(Processes)` | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)` | `7zip_commandline_to_smb_share_path_filter` | stats count | where count > 0', 'detection_name': '7zip CommandLine To SMB Share Path', 'detection_file': 'endpoint/7zip_commandline_to_smb_share_path.yml', 'success': True, 'error': False, 'diskUsage': '286720', 'runDuration': '0.922', 'scanCount': '4897'}]
+    #github_service.update_and_commit_passed_tests(passes)
+    #sys.exit(0)
     # Make a backup of this config containing the hash and stripped credentials.
     # This makes the test perfectly reproducible.
     validate_args.validate_and_write(
@@ -435,6 +438,9 @@ def main(args: list[str]):
 
     result = cm.run_test()
     
+    github_service.update_and_commit_passed_tests(cm.synchronization_object.successes)
+    
+
     #Return code indicates whether testing succeeded and all tests were run.
     #It does NOT indicate that all tests passed!
     if result is True:
diff --git a/bin/automated_detection_testing/ci/detection_testing_batch/modules/github_service.py b/bin/automated_detection_testing/ci/detection_testing_batch/modules/github_service.py
index fd0c6b1afb..798ae31914 100644
--- a/bin/automated_detection_testing/ci/detection_testing_batch/modules/github_service.py
+++ b/bin/automated_detection_testing/ci/detection_testing_batch/modules/github_service.py
@@ -313,7 +313,7 @@ class GithubService:
         
         for missing in set(changed_detection_files).symmetric_difference(all_changed_detection_files):
             print("Ignoring modified detecton [%s] not in set of selected folders: %s"%(missing,folders))
-        
+                
         # Convert the test files to the detection file equivalent. 
         # Note that some of these tests may be baselines and their associated 
         # detection could be in experimental or not in the experimental folder
@@ -323,11 +323,7 @@ class GithubService:
         #        *pathlib.Path(test_filepath).parts[-2:])).replace("tests", "detections", 1)
         #    converted_test_files.append(detection_filename)
         
-        #check and throw an error if we somehow got experimental tests
-        #experimental_tests = [x for x in changed_test_files if 'experimental' in x]
-        #if len(experimental_tests) > 0:
-        #    raise(Exception("Error - expected no experimental detections, but found:\n\t%s]"%("\n\t".join(experimental_tests))))
-
+        
         #Get the appropriate detection file paths for a modified test file
         for test_filepath in changed_test_files:
             folder_and_filename =  str(pathlib.Path(*pathlib.Path(test_filepath).parts[-2:]))
@@ -350,10 +346,6 @@ class GithubService:
             if name not in changed_detection_files:
                 changed_detection_files.append(name)
         
-        #check and throw an error if we somehow got experimental detections
-        experimental_detections = [x for x in changed_detection_files if 'experimental' in x]
-        if len(experimental_detections) > 0:
-            raise(Exception("Error - expected no experimental detections, but found:\n\t%s"%('\n\t'.join(experimental_detections))))
 
         return self.prune_detections(changed_detection_files,   types_to_test, previously_successful_tests)
 
diff --git a/bin/automated_detection_testing/ci/detection_testing_batch/modules/splunk_container.py b/bin/automated_detection_testing/ci/detection_testing_batch/modules/splunk_container.py
index 71ef0f66fc..efb423be2e 100644
--- a/bin/automated_detection_testing/ci/detection_testing_batch/modules/splunk_container.py
+++ b/bin/automated_detection_testing/ci/detection_testing_batch/modules/splunk_container.py
@@ -16,11 +16,13 @@ import time
 import timeit
 from typing import Union
 import threading
+import wrapt_timeout_decorator
+import sys
 
 SPLUNKBASE_URL = "https://splunkbase.splunk.com/app/%d/release/%s/download"
 SPLUNK_START_ARGS = "--accept-license"
 
-
+MAX_CONTAINER_START_TIME_SECONDS = 360
 class SplunkContainer:
     def __init__(
         self,
@@ -68,6 +70,7 @@ class SplunkContainer:
         self.test_start_time = -1
         self.num_tests_completed = 0
 
+
     def prepare_apps_path(
         self,
         local_apps: OrderedDict,
@@ -274,42 +277,38 @@ class SplunkContainer:
 
     def wait_for_splunk_ready(
         self,
-        max_seconds: int = 300,
-        seconds_between_attempts: int = 5,
+        seconds_between_attempts: int = 10,
     ) -> bool:
         
         # The smarter version of this will try to hit one of the pages,
         # probably the login page, and when that is available it means that
         # splunk is fully started and ready to go.  Until then, we just
         # use a simple sleep
-        splunk_ready_url = "http://%s:%d" % (self.splunk_ip, self.web_port)
-        start = timeit.default_timer()
+        
+        
         while True:
             try:
-                # Splunk container will not have proper ssl certificate
-                response = requests.get(
-                    splunk_ready_url, timeout=seconds_between_attempts, verify=False)
-                response.raise_for_status()
-                return True
+                service = splunk_sdk.client.connect(host=self.splunk_ip, port=self.management_port, username='admin', password=self.container_password)
+                if service.restart_required:
+                    #The sleep below will wait
+                    pass
+                else:
+                    return True
+              
             except Exception as e:
-                elapsed = timeit.default_timer() - start
-                
-                if elapsed > max_seconds:
-                    self.stopContainer()
-                    raise (
-                        Exception(
-                            "Container [%s] took longer than maximum start time of [%d].\n\tStopping container..."
-                            % (self.container_name, max_seconds)
-                        )
-                    )
+                # There is a good chance the server is restarting, so the SDK connection failed.
+                # Or, we tried to check restart_required while the server was restarting.  In the
+                # calling function, we have a timeout, so it's okay if this function could get 
+                # stuck in an infinite loop (the caller will generate a timeout error)
+                pass
                     
             time.sleep(seconds_between_attempts)
 
-    def run_container(self) -> None:
-        print("Starting the container [%s]" % (self.container_name))
-        self.container_start_time = timeit.default_timer()
-        self.container.start()
+    
+    @wrapt_timeout_decorator.timeout(MAX_CONTAINER_START_TIME_SECONDS, timeout_exception=RuntimeError)
+    def setup_container(self):
 
+        self.container.start()
         # By default, first copy the index file then the datamodel file
         for file_description, file_dict in self.files_to_copy_to_container.items():
             self.extract_tar_file_to_container(
@@ -317,35 +316,34 @@ class SplunkContainer:
             )
 
         print("Finished copying files to [%s]" % (self.container_name))
+        self.wait_for_splunk_ready()
+        
 
-        try:
-            while not splunk_sdk.enable_delete_for_admin(self.splunk_ip, self.management_port, self.container_password):
-                time.sleep(10)
-        except Exception as e:
-            print(
-                "Failure enabling DELETE for container [%s]: [%s].\n\tQuitting..."
-                % (self.container_name, str(e))
-            )
-
-        # Wait for all of the threads to join here
-        print(
-            "Container [%s] setup complete and waiting for other containers to be ready..."
-            % (self.container_name)
-        )
-
-        self.synchronization_object.start_barrier.wait()
+    def run_container(self) -> None:
+        print("Starting the container [%s]" % (self.container_name))
+        self.container_start_time = timeit.default_timer()
+    
+        container_start_time = timeit.default_timer()
         
         try:
-            self.wait_for_splunk_ready()
+            self.setup_container()
         except Exception as e:
-            print("Error starting docker container: [%s]"%(str(e)))
+            print("There was an exception starting the container [%s]: [%s].  Shutting down container"%(self.container_name,str(e)),file=sys.stdout)
+            self.stopContainer()
+            elapsed_rounded = round(timeit.default_timer() - container_start_time)
+            time_string = (datetime.timedelta(seconds=elapsed_rounded))
+            print("Container [%s] FAILED in [%s]"%(self.container_name, time_string))
             return None
-        #input("CONTAINTER WANTS TO START.... WAIT FOR INPUT FROM USER")
+
+
+        #GTive some info about how long the container took to start up
+        elapsed_rounded = round(timeit.default_timer() - container_start_time)
+        time_string = (datetime.timedelta(seconds=elapsed_rounded))
+        print("Container [%s] took [%s] to start"%(self.container_name, time_string))
 
 
         # Sleep for a small random time so that containers drift apart and don't synchronize their testing
         time.sleep(random.randint(1, 30))
-        
         self.test_start_time = timeit.default_timer()
         while True:
             if self.synchronization_object.checkContainerFailure():
diff --git a/bin/automated_detection_testing/ci/detection_testing_batch/modules/splunk_sdk.py b/bin/automated_detection_testing/ci/detection_testing_batch/modules/splunk_sdk.py
index 9c0c567caf..0ed8259077 100644
--- a/bin/automated_detection_testing/ci/detection_testing_batch/modules/splunk_sdk.py
+++ b/bin/automated_detection_testing/ci/detection_testing_batch/modules/splunk_sdk.py
@@ -175,15 +175,42 @@ def delete_attack_data(splunk_host:str, splunk_password:str, splunk_port:int, wa
         print("FILENAME : [%s]"%(detection_filename))
         print("SEARCH   :\n%s"%(search_string))
         _ = input("****************Press ENTER to Complete Test and DELETE data****************\n\n\n")
-    splunk_search = 'search index=main | delete'
+    
+    data_exists = True
+    already_enabled_delete = False
+    while data_exists:
+        splunk_search = 'search index=main | delete'
 
-    kwargs = {"exec_mode": "blocking",
-              "dispatch.earliest_time": "-1d",
-              "dispatch.latest_time": "now"}
+        kwargs = {"dispatch.earliest_time": "-1d",
+                "dispatch.latest_time": "now"}
+        try:
+            
+            job = service.jobs.oneshot(splunk_search, **kwargs)
+            reader = results.ResultsReader(job)
+            error_in_results = False
+            for result in reader:
+                if hasattr(result,"message") and hasattr(result,"type") and ("You have insufficient privileges to delete events" in result.message or result.type == "FATAL"):
+                    print("Delete is not enabled for admin: [%s] - enabling delete and trying to delete again..."%(result.message), file=sys.stderr)
+                    if already_enabled_delete is True:
+                        print("We already enabled delete, but the setting did not take effect.")
+                        raise(Exception("Enabling delete command failed to take effect"))
+                    if enable_delete_for_admin(splunk_host, splunk_port,splunk_password) != True:
+                        raise(Exception("Failure enabling delete for admin. We cannot continue"))
+                    # We enabled delete, so now we will try to delete again
+                    already_enabled_delete = True
+                    break
+                else:
+                    #This is not one of the error messages, do nothing
+                    pass
 
-    try:
-        job = service.jobs.create(splunk_search, **kwargs)
-    except Exception as e:
-        raise(Exception("Unable to execute search: " + str(e)))
+            #No need to issue Delete command again, we will now break out of the loop
+            if error_in_results is False:
+                data_exists = False
+
+            #Otherwise, we will loop again
+
+        except Exception as e:
+            raise(Exception("Unable to delete data from a run: " + str(e)))
+        
     
     return True
diff --git a/bin/automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py b/bin/automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py
index 86c8f81a3f..2b9933fed8 100644
--- a/bin/automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py
+++ b/bin/automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py
@@ -250,7 +250,7 @@ setup_schema = {
             "type": "array",
             "items": {
                 "type": "string",
-                "enum": ["endpoint", "cloud", "network"]
+                "enum": ["endpoint", "cloud", "network","web"]
             },
             "default": ["endpoint", "cloud", "network"]
         },
diff --git a/bin/automated_detection_testing/ci/detection_testing_batch/requirements.txt b/bin/automated_detection_testing/ci/detection_testing_batch/requirements.txt
index c2cd26da5c..d8419b7d63 100644
--- a/bin/automated_detection_testing/ci/detection_testing_batch/requirements.txt
+++ b/bin/automated_detection_testing/ci/detection_testing_batch/requirements.txt
@@ -16,3 +16,4 @@ docker==5.0.3
 
 #For help getting and parsing the configuration
 jsonschema==4.2.1
+wrapt_timeout_decorator==1.3.1

From 76364bf1c0310a26d2b7b24084b16945e966c975 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Fri, 17 Dec 2021 20:06:42 -0800
Subject: [PATCH 140/166] Updated the default configuration to use attack_range
 apps stored on S3.  Pointed at the PATCHED linux_sysmon. Added a slight delay
 in between starts of containers for performance reasons.  Enabled better
 handling and ability to download http_path local apps to a folder instead of
 passing them in as string for the container to download.

---
 .../detection_testing_execution.py            |  86 ++++---
 .../modules/container_manager.py              |   2 +
 .../modules/splunk_container.py               |  18 +-
 .../modules/validate_args.py                  | 217 +++++++++++++-----
 4 files changed, 230 insertions(+), 93 deletions(-)

diff --git a/bin/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py b/bin/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
index 2bc41acdf8..3e5cd98672 100644
--- a/bin/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
+++ b/bin/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
@@ -3,7 +3,6 @@ import copy
 import csv
 import json
 import os
-from posixpath import basename
 import queue
 import random
 import secrets
@@ -15,22 +14,23 @@ import threading
 import time
 from collections import OrderedDict
 from datetime import datetime, timedelta
+from posixpath import basename
 from tempfile import mkdtemp
 from timeit import default_timer as timer
 from typing import Union
+from urllib.parse import urlparse
 
 import docker
+import requests
 import requests.packages.urllib3
 from docker.client import DockerClient
 from requests import get
-from modules import new_arguments2
-from modules.validate_args import validate_and_write
-from modules import container_manager
 
 import modules.new_arguments2
-from modules import aws_service, testing_service, validate_args
+from modules import (aws_service, container_manager, new_arguments2,
+                     testing_service, validate_args)
 from modules.github_service import GithubService
-from modules.validate_args import validate
+from modules.validate_args import validate, validate_and_write
 
 SPLUNK_CONTAINER_APPS_DIR = "/opt/splunk/etc/apps"
 index_file_local_path = "indexes.conf.tar"
@@ -45,23 +45,53 @@ datamodel_file_container_path = os.path.join(
 MAX_RECOMMENDED_CONTAINERS_BEFORE_WARNING = 2
 
 
+def download_file_from_http(url:str, target:str)->None:
+    #Will just overwrite an existing file
+    file_to_download = requests.get(url, stream=True)
+    with open(target, "wb") as output:
+        for piece in file_to_download.iter_content(chunk_size=(1024*1024)):
+            output.write(piece)
+    
+
 def copy_local_apps_to_directory(apps: dict[str, dict], target_directory) -> None:
     for key, item in apps.items():
-        source_path = os.path.abspath(os.path.expanduser(item['local_path']))
-        base_name = os.path.basename(source_path)
-        dest_path = os.path.join(target_directory, base_name)
+        
+        #local apps can either have a local_path or an http_path
+        if 'local_path' in item:
+            source_path = os.path.abspath(os.path.expanduser(item['local_path']))
+            base_name = os.path.basename(source_path)
+            dest_path = os.path.join(target_directory, base_name)
+            try:
+                shutil.copy(source_path, dest_path)
+                item['local_path'] = dest_path
+            except shutil.SameFileError as e:
+                # Same file, not a real error.  The copy just doesn't happen
+                print("err:%s" % (str(e)))
+                pass
+            except Exception as e:
+                print("Error copying ESCU Package [%s] to [%s]: [%s].\n\tQuitting..." % (
+                    source_path, dest_path, str(e)), file=sys.stderr)
+                sys.exit(1)
+
+        # These apps are URLs that will be passed.  The apps will be downloaded and installed by the container
+        # # Get the file from an http source
+        # elif 'http_path' in item:
+        #     http_path = item['http_path']
+        #     try:
+        #         url_parse_obj = urlparse(http_path)
+        #         path_after_host = url_parse_obj[2].rstrip('/') #removes / at the end, if applicable
+        #         base_name = path_after_host.rpartition('/')[-1] #just get the file name
+        #         dest_path = os.path.join(target_directory, base_name) #write the whole path
+        #         download_file_from_http(http_path, dest_path)
+        #         #we need to updat the local path because this is used to copy it into the container later
+        #         item['local_path'] = dest_path
+        #     except Exception as e:
+        #         print("Error trying to download %s @ %s: [%s].  This app is required.\n\tQuitting..."%(key, http_path, str(e)),file=sys.stderr)
+        #         sys.exit(1)
+        # else:
+        #     print("Error - trying to install a local app that does not have 'local_path' or 'http_path'.\n\tQuitting...")
+        #     sys.exit(1)
 
-        try:
-            shutil.copy(source_path, dest_path)
-            item['local_path'] = dest_path
-        except shutil.SameFileError as e:
-            # Same file, not a real error.  The copy just doesn't happen
-            print("err:%s" % (str(e)))
-            pass
-        except Exception as e:
-            print("Error copying ESCU Package [%s] to [%s]: [%s].\n\tQuitting..." % (
-                source_path, dest_path, str(e)), file=sys.stderr)
-            sys.exit(1)
 
 
 def ensure_security_content(branch: str, commit_hash: str, pr_number: Union[int, None], persist_security_content: bool) -> GithubService:
@@ -105,13 +135,13 @@ def generate_escu_app(persist_security_content: bool = False) -> str:
     if persist_security_content is False:
         commands = ["python3 -m venv .venv",
                     ". ./.venv/bin/activate",
-                    "python3 -m pip install wheel",
-                    "python3 -m pip install -r requirements.txt",
-                    "python3 contentctl.py --path . --verbose generate --product ESCU --output dist/escu",
+                    "python -m pip install wheel",
+                    "python -m pip install -r requirements.txt",
+                    "python contentctl.py --path . --verbose generate --product ESCU --output dist/escu",
                     "tar -czf DA-ESS-ContentUpdate.spl -C dist/escu ."]
     else:
         commands = [". ./.venv/bin/activate",
-                    "python3 contentctl.py --path . --verbose generate --product ESCU --output dist/escu",
+                    "python contentctl.py --path . --verbose generate --product ESCU --output dist/escu",
                     "tar -czf DA-ESS-ContentUpdate.spl -C dist/escu ."]
     ret = subprocess.run("; ".join(commands),
                          shell=True, capture_output=True)
@@ -171,10 +201,10 @@ def generate_escu_app(persist_security_content: bool = False) -> str:
                     "cd slim_latest",
                     "virtualenv --python=/usr/bin/python2.7 --clear .venv",
                     ". ./.venv/bin/activate",
-                    "python3 -m pip install --upgrade pip",
-                    "python2 -m pip install wheel",
-                    "python2 -m pip install semantic_version",
-                    "python2 -m pip install .",
+                    "python -m pip install --upgrade pip",
+                    "python -m pip install wheel",
+                    "python -m pip install semantic_version",
+                    "python -m pip install .",
                     "cp -R ../../dist/escu DA-ESS-ContentUpdate",
                     "slim package -o upload DA-ESS-ContentUpdate",
                     "cp upload/DA-ESS-ContentUpdate*.tar.gz %s" % (output_file_path_from_slim_latest)]
diff --git a/bin/automated_detection_testing/ci/detection_testing_batch/modules/container_manager.py b/bin/automated_detection_testing/ci/detection_testing_batch/modules/container_manager.py
index b11f1669dd..2635cdc24c 100644
--- a/bin/automated_detection_testing/ci/detection_testing_batch/modules/container_manager.py
+++ b/bin/automated_detection_testing/ci/detection_testing_batch/modules/container_manager.py
@@ -125,6 +125,8 @@ class ContainerManager:
 
     def run_containers(self) -> None:
         for container in self.containers:
+            #give a little time between container startup
+            time.sleep(15)
             container.thread.start()
         
     
diff --git a/bin/automated_detection_testing/ci/detection_testing_batch/modules/splunk_container.py b/bin/automated_detection_testing/ci/detection_testing_batch/modules/splunk_container.py
index efb423be2e..648a33b488 100644
--- a/bin/automated_detection_testing/ci/detection_testing_batch/modules/splunk_container.py
+++ b/bin/automated_detection_testing/ci/detection_testing_batch/modules/splunk_container.py
@@ -82,9 +82,16 @@ class SplunkContainer:
         require_credentials = False
 
         for app_name, app_info in self.local_apps.items():
-            app_file_name = os.path.basename(app_info['local_path'])
-            app_file_container_path = os.path.join("/tmp/apps", app_file_name)
-            apps_to_install.append(app_file_container_path)
+            
+            if 'local_path' in app_info:
+                app_file_name = os.path.basename(app_info['local_path'])
+                app_file_container_path = os.path.join("/tmp/apps", app_file_name)
+                apps_to_install.append(app_file_container_path)
+            elif 'http_path' in app_info:
+                apps_to_install.append(app_info['http_path'])
+            else:
+                print("Error, the app %s: %s has no http_path or local_path.\n\tQuitting..."%(app_name,app_info), file=sys.stderr)
+                sys.exit(1)
 
         for app_name, app_info in self.splunkbase_apps.items():
 
@@ -99,7 +106,9 @@ class SplunkContainer:
             # elif app["location"] == "local":
             #    apps_to_install.append(app["container_path"])
         
-        
+        #for printing out all the app paths we will install
+        #for num, name in zip(range(len(apps_to_install)),apps_to_install):
+        #    print("%d: %s"%(num,name))
         return ",".join(apps_to_install), require_credentials
 
     def make_environment(
@@ -116,6 +125,7 @@ class SplunkContainer:
         splunk_apps_url, require_credentials = self.prepare_apps_path(
             local_apps, splunkbase_apps, splunkbase_username, splunkbase_password
         )
+        
         if require_credentials:
             env["SPLUNKBASE_USERNAME"] = splunkbase_username
             env["SPLUNKBASE_PASSWORD"] = splunkbase_password
diff --git a/bin/automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py b/bin/automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py
index 2b9933fed8..868c2455c8 100644
--- a/bin/automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py
+++ b/bin/automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py
@@ -93,7 +93,99 @@ setup_schema = {
                     "app_number": 3449,
                     "app_version": None,
                     "local_path": None
-                }
+                },
+                #The default apps below were taken from the attack_range loadout: https://github.com/splunk/attack_range/blob/develop/attack_range.conf.template
+                "SPLUNK_WINDOWS_TA": {
+                    "app_number": 0,
+                    "app_version": None,
+                    "http_path": "https://attack-range-appbinaries.s3-us-west-2.amazonaws.com/splunk-add-on-for-microsoft-windows_812.tgz"
+                },
+                "SPLUNK_ADD_ON_FOR_SYSMON_OLD": {
+                    "app_number": 1,
+                    "app_version": None,
+                    "http_path": "https://attack-range-appbinaries.s3-us-west-2.amazonaws.com/splunk-add-on-for-microsoft-sysmon_1062.tgz"
+                },
+                "SPLUNK_SYSMON_LINUX_TA_PATCHED": {
+                    "app_number": 2,
+                    "app_version": None,
+                    "http_path": "https://attack-range-appbinaries.s3-us-west-2.amazonaws.com/add-on-for-linux-sysmon_103_PATCHED.tgz"
+                },
+                "SPLUNK_CIM_APP": {
+                    "app_number": 3,
+                    "app_version": None,
+                    "http_path": "https://attack-range-appbinaries.s3-us-west-2.amazonaws.com/splunk-common-information-model-cim_4200.tgz"
+                },
+                "SPLUNK_AWS_TA": {
+                    "app_number": 4,
+                    "app_version": None,
+                    "http_path": "https://attack-range-appbinaries.s3-us-west-2.amazonaws.com/splunk-add-on-for-amazon-web-services_510.tgz"
+                },
+                "SPLUNK_PYTHON_APP": {
+                    "app_number": 5,
+                    "app_version": None,
+                    "http_path": "https://attack-range-appbinaries.s3-us-west-2.amazonaws.com/python-for-scientific-computing-for-linux-64-bit_202.tgz"
+                },
+                "SPLUNK_ASX_APP": {
+                    "app_number": 17,
+                    "app_version": None,
+                    "http_path": "https://attack-range-appbinaries.s3-us-west-2.amazonaws.com/Splunk_ASX-latest.tar.gz"
+                },
+                "SPLUNK_MLTK_APP": {
+                    "app_number": 6,
+                    "app_version": None,
+                    "http_path": "https://attack-range-appbinaries.s3-us-west-2.amazonaws.com/splunk-machine-learning-toolkit_521.tgz"
+                },
+                "SPLUNK_STREAM_APP": {
+                    "app_number": 7,
+                    "app_version": None,
+                    "http_path": "https://attack-range-appbinaries.s3-us-west-2.amazonaws.com/splunk-app-for-stream_730.tgz"
+                },
+                "SPLUNK_ADD_ON_FOR_STREAM_WIRE_DATA": {
+                    "app_number": 8,
+                    "app_version": None,
+                    "http_path": "https://attack-range-appbinaries.s3-us-west-2.amazonaws.com/splunk-add-on-for-stream-wire-data_730.tgz"
+                },
+                "SPLUNK_ADD_ON_FOR_STREAM_FORWARDERS": {
+                    "app_number": 9,
+                    "app_version": None,
+                    "http_path": "https://attack-range-appbinaries.s3-us-west-2.amazonaws.com/splunk-add-on-for-stream-forwarders_730.tgz"
+                },
+                "SPLUNK_SECURITY_ESSENTIALS": {
+                    "app_number": 10,
+                    "app_version": None,
+                    "http_path": "https://attack-range-appbinaries.s3-us-west-2.amazonaws.com/splunk-security-essentials_333.tgz"
+                },
+                "SPLUNK_ADD_ON_FOR_ZEEK_AKA_BRO": {
+                    "app_number": 11,
+                    "app_version": None,
+                    "http_path": "https://attack-range-appbinaries.s3-us-west-2.amazonaws.com/splunk-add-on-for-zeek-aka-bro_400.tgz"
+                },
+                "SPLUNK_ADD_ON_FOR_AMAZON_KINESIS_FIREHOSE": {
+                    "app_number": 12,
+                    "app_version": None,
+                    "http_path": "https://attack-range-appbinaries.s3-us-west-2.amazonaws.com/splunk-add-on-for-amazon-kinesis-firehose_131r7d1d093.tgz"
+                },
+                "SPLUNK_ADD_ON_FOR_MICROSOFT_OFFICE_365": {
+                    "app_number": 13,
+                    "app_version": None,
+                    "http_path": "https://attack-range-appbinaries.s3-us-west-2.amazonaws.com/splunk-add-on-for-microsoft-office-365_202.tgz"
+                },
+                "SPLUNK_LINUX_TA": {
+                    "app_number": 14,
+                    "app_version": None,
+                    "http_path": "https://attack-range-appbinaries.s3-us-west-2.amazonaws.com/splunk-add-on-for-unix-and-linux_820.tgz"
+                },
+                "SPLUNK_NGINX_TA": {
+                    "app_number": 15,
+                    "app_version": None,
+                    "http_path": "https://attack-range-appbinaries.s3-us-west-2.amazonaws.com/splunk-add-on-for-nginx_310.tgz"
+                },
+                "SPLUNK_TA_FOR_ZEEK": {
+                    "app_number": 16,
+                    "app_version": None,
+                    "http_path": "https://attack-range-appbinaries.s3-us-west-2.amazonaws.com/ta-for-zeek_105.tgz"
+                },
+                
             }
         },
 
@@ -152,72 +244,75 @@ setup_schema = {
                         }
                     }
             },
-            "default": {
-                "ADD-ON_FOR_LINUX_SYSMON": {
-                    "app_number": 6176,
-                    "app_version": "1.0.3"
-                },
-                "SPLUNK_ADD_ON_FOR_AMAZON_WEB_SERVICES": {
-                    "app_number": 1876,
-                    "app_version": "5.2.0"
-                },
+            
+             "default": {},
+ 
+            # "default": {
+            #     "ADD-ON_FOR_LINUX_SYSMON": {
+            #         "app_number": 6176,
+            #         "app_version": "1.0.3"
+            #     },
+            #     "SPLUNK_ADD_ON_FOR_AMAZON_WEB_SERVICES": {
+            #         "app_number": 1876,
+            #         "app_version": "5.2.0"
+            #     },
 
-                "SPLUNK_ADD_ON_FOR_MICROSOFT_OFFICE_365":
-                {
-                    "app_number": 4055,
-                    "app_version": "2.2.0"
-                },
+            #     "SPLUNK_ADD_ON_FOR_MICROSOFT_OFFICE_365":
+            #     {
+            #         "app_number": 4055,
+            #         "app_version": "2.2.0"
+            #     },
 
-                "SPLUNK_ADD_ON_FOR_AMAZON_KINESIS_FIREHOSE": {
-                    "app_number": 3719,
-                    "app_version": "1.3.2"
-                },
+            #     "SPLUNK_ADD_ON_FOR_AMAZON_KINESIS_FIREHOSE": {
+            #         "app_number": 3719,
+            #         "app_version": "1.3.2"
+            #     },
 
-                "SPLUNK_ANALYTIC_STORY_EXECUTION_APP": {
-                    "app_number": 4971,
-                    "app_version": "2.0.3"
-                },
+            #     "SPLUNK_ANALYTIC_STORY_EXECUTION_APP": {
+            #         "app_number": 4971,
+            #         "app_version": "2.0.3"
+            #     },
 
-                "PYTHON_FOR_SCIENTIC_COMPUTING_LINUX_64_BIT": {
-                    "app_number": 2882,
-                    "app_version": "2.0.2"
-                },
+            #     "PYTHON_FOR_SCIENTIC_COMPUTING_LINUX_64_BIT": {
+            #         "app_number": 2882,
+            #         "app_version": "2.0.2"
+            #     },
 
-                "SPLUNK_MACHINE_LEARNING_TOOLKIT": {
-                    "app_number": 2890,
-                    "app_version": "5.2.2"
-                },
+            #     "SPLUNK_MACHINE_LEARNING_TOOLKIT": {
+            #         "app_number": 2890,
+            #         "app_version": "5.2.2"
+            #     },
 
-                "SPLUNK_APP_FOR_STREAM": {
-                    "app_number": 1809,
-                    "app_version": "8.0.1"
-                },
-                "SPLUNK_ADD_ON_FOR_STREAM_WIRE_DATA": {
-                    "app_number": 5234,
-                    "app_version": "8.0.1"
-                },
-                "SPLUNK_ADD_ON_FOR_STREAM_FORWARDERS": {
-                    "app_number": 5238,
-                    "app_version": "8.0.1"
-                },
-                "SPLUNK_ADD_ON_FOR_ZEEK_AKA_BRO": {
-                    "app_number": 1617,
-                    "app_version": "4.0.0"
-                },
-                "SPLUNK_ADD_ON_FOR_UNIX_AND_LINUX": {
-                    "app_number": 833,
-                    "app_version": "8.3.1"
-                },
-                "SPLUNK_ADD_ON_FOR_SYSMON": {
-                    "app_number": 5709,
-                    "app_version": "1.0.1"
-                },
-                # According to https://docs.splunk.com/Documentation/ES/6.6.2/Install/Datamodels, these are included in ES. Don't install separately.
-                "SPLUNK_COMMON_INFORMATION_MODEL": {
-                    "app_number": 1621,
-                    "app_version": "4.20.2"
-                }
-            }
+            #     "SPLUNK_APP_FOR_STREAM": {
+            #         "app_number": 1809,
+            #         "app_version": "8.0.1"
+            #     },
+            #     "SPLUNK_ADD_ON_FOR_STREAM_WIRE_DATA": {
+            #         "app_number": 5234,
+            #         "app_version": "8.0.1"
+            #     },
+            #     "SPLUNK_ADD_ON_FOR_STREAM_FORWARDERS": {
+            #         "app_number": 5238,
+            #         "app_version": "8.0.1"
+            #     },
+            #     "SPLUNK_ADD_ON_FOR_ZEEK_AKA_BRO": {
+            #         "app_number": 1617,
+            #         "app_version": "4.0.0"
+            #     },
+            #     "SPLUNK_ADD_ON_FOR_UNIX_AND_LINUX": {
+            #         "app_number": 833,
+            #         "app_version": "8.3.1"
+            #     },
+            #     "SPLUNK_ADD_ON_FOR_SYSMON": {
+            #         "app_number": 5709,
+            #         "app_version": "1.0.1"
+            #     },
+            #     # According to https://docs.splunk.com/Documentation/ES/6.6.2/Install/Datamodels, these are included in ES. Don't install separately.
+            #     "SPLUNK_COMMON_INFORMATION_MODEL": {
+            #         "app_number": 1621,
+            #         "app_version": "4.20.2"
+            #     }
+            # }
         },
 
         "splunkbase_username": {

From cc8113ec8680efb8da7aeadbb4d98c8cdab3a3e8 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Mon, 20 Dec 2021 08:34:54 -0800
Subject: [PATCH 141/166] Fixed output of manifest for
 detection_failure_manifest.json. Previously, it contained mostly default
 settings and most notably did not container the proper apps configuration. 
 Also updated a few documentation strings.

---
 .../detection_testing_execution.py                | 10 +++++++---
 .../modules/container_manager.py                  |  5 +++--
 .../modules/test_driver.py                        | 15 +++++++++++----
 .../ci/detection_testing_batch/summarize_json.py  | 10 ++++++----
 4 files changed, 27 insertions(+), 13 deletions(-)

diff --git a/bin/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py b/bin/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
index 3e5cd98672..8977119ec2 100644
--- a/bin/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
+++ b/bin/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
@@ -370,8 +370,11 @@ def main(args: list[str]):
     #sys.exit(0)
     # Make a backup of this config containing the hash and stripped credentials.
     # This makes the test perfectly reproducible.
-    validate_args.validate_and_write(
-        settings, output_file=None, strip_credentials=True)
+    reproduce_test_config, _ = validate_args.validate_and_write(settings, output_file=None, strip_credentials=True)
+    if reproduce_test_config == None:
+        print("Error - there was an error writing out the file to reproduce the test.  This should not happen, as all "\
+              "settings should have been validated by this point.\n\tQuitting...",file=sys.stderr)
+        sys.exit(1)
 
     try:
         all_test_files = github_service.get_test_files(settings['mode'],
@@ -454,6 +457,7 @@ def main(args: list[str]):
                                             settings['splunkbase_apps'],
                                             settings['branch'],
                                             settings['commit_hash'],
+                                            reproduce_test_config,
                                             files_to_copy_to_container=files_to_copy_to_container,
                                             web_port_start=8000,
                                             management_port_start=8089,
@@ -468,7 +472,7 @@ def main(args: list[str]):
 
     result = cm.run_test()
     
-    github_service.update_and_commit_passed_tests(cm.synchronization_object.successes)
+    #github_service.update_and_commit_passed_tests(cm.synchronization_object.successes)
     
 
     #Return code indicates whether testing succeeded and all tests were run.
diff --git a/bin/automated_detection_testing/ci/detection_testing_batch/modules/container_manager.py b/bin/automated_detection_testing/ci/detection_testing_batch/modules/container_manager.py
index 2635cdc24c..6bc53da95a 100644
--- a/bin/automated_detection_testing/ci/detection_testing_batch/modules/container_manager.py
+++ b/bin/automated_detection_testing/ci/detection_testing_batch/modules/container_manager.py
@@ -29,6 +29,7 @@ class ContainerManager:
         splunkbase_apps:OrderedDict,
         branch:str,
         commit_hash:str,
+        summarization_reproduce_failure_config:dict,
         files_to_copy_to_container: OrderedDict = OrderedDict(),
         web_port_start: int = 8000,
         management_port_start: int = 8089,
@@ -43,7 +44,7 @@ class ContainerManager:
 
     ):
         self.synchronization_object = test_driver.TestDriver(
-            test_list, num_containers)
+            test_list, num_containers, summarization_reproduce_failure_config)
 
         self.mounts = self.create_mounts(mounts)
         self.local_apps = local_apps
@@ -55,7 +56,7 @@ class ContainerManager:
             self.container_password = container_password
         
         print("\n\n***********************")
-        print("Log into your Splunk Container(s) after they boot at at http://127.0.0.1:[%d-%d]"%(web_port_start, web_port_start + num_containers - 1))
+        print("Log into your [%d] Splunk Container(s) after they boot at http://127.0.0.1:[%d-%d]"%(num_containers, web_port_start, web_port_start + num_containers - 1))
         print("\tSplunk App Username: [%s]"%("admin"))
         print("\tSplunk App Password: ", end='')
         if show_container_password:
diff --git a/bin/automated_detection_testing/ci/detection_testing_batch/modules/test_driver.py b/bin/automated_detection_testing/ci/detection_testing_batch/modules/test_driver.py
index 47d78472ff..9f7e49e9c9 100644
--- a/bin/automated_detection_testing/ci/detection_testing_batch/modules/test_driver.py
+++ b/bin/automated_detection_testing/ci/detection_testing_batch/modules/test_driver.py
@@ -12,9 +12,9 @@ import time
 import timeit
 from typing import Union
 import sys
-
+import copy
 class TestDriver:
-    def __init__(self, tests:list[str], num_containers:int):
+    def __init__(self, tests:list[str], num_containers:int, summarization_reproduce_failure_config:dict):
         #Create the queue and enque all of the tests
         self.testing_queue = queue.Queue()
         for test in tests:
@@ -35,8 +35,12 @@ class TestDriver:
         #Just make a random folder to store attack data that we donwload
         self.attack_data_root_folder = tempfile.mkdtemp(prefix="attack_data_", dir=os.getcwd())
         print("Attack data for this run will be stored at: [%s]"%(self.attack_data_root_folder))
+        
+        #Not used right now, but we will keep it around for a bit in case we want to use it again
         self.start_barrier = threading.Barrier(num_containers)
 
+        #The config that will be used for writing out the error config reproduction fiel
+        self.summarization_reproduce_failure_config = copy.deepcopy(summarization_reproduce_failure_config)
 
     def checkContainerFailure(self)->bool:
         
@@ -184,11 +188,14 @@ class TestDriver:
         res |= self.outputResultsFile(fields, os.path.join(results_directory, "combined"), combined_data, baseline)
         
         try:
-            success, test_count,pass_count,fail_count,error_count = summarize_json.outputResultsJSON("summary.json", combined_data, baseline, output_folder=results_directory)
+            success, test_count,pass_count,fail_count,error_count = \
+                summarize_json.outputResultsJSON("summary.json", combined_data, 
+                                                 baseline, output_folder=results_directory, 
+                                                 summarization_reproduce_failure_config=self.summarization_reproduce_failure_config)
             summarize_json.print_summary(test_count, pass_count, fail_count, error_count)
             res |= success
         except Exception as e:
-            print("Failure writing the summary file: [%s]",file=sys.stderr)
+            print("Failure writing the summary file: [%s]"%str(e),file=sys.stderr)
             res = False
 
         return res
diff --git a/bin/automated_detection_testing/ci/detection_testing_batch/summarize_json.py b/bin/automated_detection_testing/ci/detection_testing_batch/summarize_json.py
index eddfe456ab..0a868334d8 100644
--- a/bin/automated_detection_testing/ci/detection_testing_batch/summarize_json.py
+++ b/bin/automated_detection_testing/ci/detection_testing_batch/summarize_json.py
@@ -6,11 +6,11 @@ import json
 from modules import validate_args
 import os.path
 from operator import itemgetter
-
+import copy
 
 def outputResultsJSON(output_filename:str, data:list[dict], baseline:OrderedDict, 
                       failure_manifest_filename = "detection_failure_manifest.json", 
-                      output_folder:str="")->tuple[bool,int,int,int,int]:
+                      output_folder:str="", summarization_reproduce_failure_config:dict={})->tuple[bool,int,int,int,int]:
     success = True
     
     try:
@@ -61,9 +61,11 @@ def outputResultsJSON(output_filename:str, data:list[dict], baseline:OrderedDict
         fail_list = [os.path.join("security_content/detections",x['detection_file'] ) for x in data_sorted if x['success'] == False]
 
         if len(fail_list) > 0:
-            failures_test_override = {"detections_list": fail_list, "interactive_failure":True, 
+                                    
+            failures_test_override = copy.deepcopy(summarization_reproduce_failure_config)
+            failures_test_override.update({"detections_list": fail_list, "no_interactive_failure":False, 
                                     "num_containers":1, "branch": baseline["branch"], "commit_hash":baseline["commit_hash"], 
-                                    "mode":"selected", "show_splunk_app_password": True}
+                                    "mode":"selected", "show_splunk_app_password": True})
             with open(os.path.join(output_folder,failure_manifest_filename),"w") as failures:
                 validate_args.validate_and_write(failures_test_override, failures)
     except Exception as e:

From 7230b5c63d62d29c96c75a7631eaf06518806a2d Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Mon, 20 Dec 2021 12:02:11 -0800
Subject: [PATCH 142/166] Print out some system usage information on each
 update. This will help users dianose if their systems are overburdend and
 also helps us figure out what the appropriate number of containers to run on
 cloud infrastructure, like GitHub Actions, may be without logging directly
 into the machine doing the testing.  In some cases, we can't log into those
 machines by design.  If you're running tests at home, you can also just
 listen to the volume of your computer's fans.

---
 .../modules/container_manager.py              |  2 +-
 .../modules/splunk_container.py               |  5 +-
 .../modules/test_driver.py                    | 50 ++++++++++++++++---
 3 files changed, 45 insertions(+), 12 deletions(-)

diff --git a/bin/automated_detection_testing/ci/detection_testing_batch/modules/container_manager.py b/bin/automated_detection_testing/ci/detection_testing_batch/modules/container_manager.py
index 6bc53da95a..5ad47cd70a 100644
--- a/bin/automated_detection_testing/ci/detection_testing_batch/modules/container_manager.py
+++ b/bin/automated_detection_testing/ci/detection_testing_batch/modules/container_manager.py
@@ -217,7 +217,7 @@ class ContainerManager:
         return password
 
     def queue_status_thread(self, status_interval:int=60)->None:
-        #This will run fo
+
         while True:
             if self.synchronization_object.checkContainerFailure():
                 print("One of the containers has shut down prematurely and generated an exception. Shut down the rest of the containers.")
diff --git a/bin/automated_detection_testing/ci/detection_testing_batch/modules/splunk_container.py b/bin/automated_detection_testing/ci/detection_testing_batch/modules/splunk_container.py
index 648a33b488..0845b3fc65 100644
--- a/bin/automated_detection_testing/ci/detection_testing_batch/modules/splunk_container.py
+++ b/bin/automated_detection_testing/ci/detection_testing_batch/modules/splunk_container.py
@@ -199,10 +199,7 @@ class SplunkContainer:
                 # print("Failed copy of [%s] file to CONTAINER:[%s]...we will try again"%(localFilePath, containerName))
                 time.sleep(10)
                 successful_copy = False
-        print(
-            "Successfully copied [%s] to [%s] on [%s]"
-            % (local_file_path, container_file_path, self.container_name)
-        )
+        #print("Successfully copied [%s] to [%s] on [%s]"% (local_file_path, container_file_path, self.container_name))
         return successful_copy
 
     def stopContainer(self,timeout=10) -> bool:
diff --git a/bin/automated_detection_testing/ci/detection_testing_batch/modules/test_driver.py b/bin/automated_detection_testing/ci/detection_testing_batch/modules/test_driver.py
index 9f7e49e9c9..87cb71855f 100644
--- a/bin/automated_detection_testing/ci/detection_testing_batch/modules/test_driver.py
+++ b/bin/automated_detection_testing/ci/detection_testing_batch/modules/test_driver.py
@@ -1,18 +1,22 @@
-from collections import OrderedDict
+import copy
 import csv
 import datetime
 import json
 import os
 import queue
 import shutil
-import summarize_json
+import sys
 import tempfile
 import threading
 import time
 import timeit
+from collections import OrderedDict
 from typing import Union
-import sys
-import copy
+
+import psutil
+import summarize_json
+
+
 class TestDriver:
     def __init__(self, tests:list[str], num_containers:int, summarization_reproduce_failure_config:dict):
         #Create the queue and enque all of the tests
@@ -42,6 +46,13 @@ class TestDriver:
         #The config that will be used for writing out the error config reproduction fiel
         self.summarization_reproduce_failure_config = copy.deepcopy(summarization_reproduce_failure_config)
 
+
+        #According to the docs:
+        # Warning the first time this function is called with interval = 0.0 or None it will return a meaningless 0.0 value which you are supposed to ignore.
+        # We call this exactly once here to prime for future calls and throw away the result
+        cpu_info = psutil.cpu_times_percent(percpu=False)
+            
+
     def checkContainerFailure(self)->bool:
         
         self.lock.acquire()
@@ -225,17 +236,40 @@ class TestDriver:
             print("Successfully removed all attack data")
         finally:
             self.lock.release()
+
+    def get_system_stats(self)->str:
+        
+
+        cpu_info = psutil.cpu_times_percent(percpu=False)
+        print(cpu_info)
+        cpu_info_string = "Total CPU Usage: %d%% (%d CPUs)"%(100 - cpu_info.idle, psutil.cpu_count(logical=False))
+        
+        bytes_per_GB = 1024 * 1024 * 1024
+        memory_info = psutil.virtual_memory()
+        memory_info_string = "Total Memory Usage: %0.1fGB USED / %0.1fGB TOTAL"%(memory_info.used / bytes_per_GB, memory_info.total / bytes_per_GB)
+        
+        disk_usage_info = psutil.disk_usage('/')
+        disk_usage_info_string = "Total Disk Usage: %0.1fGB USED / %0.1fGB TOTAL"%(disk_usage_info.free / bytes_per_GB, disk_usage_info.total / bytes_per_GB)
+
+        return "System Information:\n\t%s\n\t%s\n\t%s"%(cpu_info_string, memory_info_string, disk_usage_info_string)
+
+
     def summarize(self)->bool:
         
         if self.checkContainerFailure() == True:
             print("Error running containers... shutting down", file=sys.stderr)
             return False
         
+
         
+
         self.lock.acquire()
         
         try:
         
+            #Get a summary of some system stats
+            system_stats=self.get_system_stats()
+            
             current_time = timeit.default_timer()
             
             
@@ -243,7 +277,7 @@ class TestDriver:
             if self.testing_queue.qsize() == self.total_number_of_tests:
                 #Testing has not started yet. We are setting up containers
                 print("***********PROGRESS UPDATE***********\n"\
-                      "\tWaiting for container setup: %s\n"%(datetime.timedelta(seconds=current_time - self.start_time)))
+                      "\tWaiting for container setup: %s\n\t%s\n"%(datetime.timedelta(seconds=current_time - self.start_time),system_stats))
             else:
                 
                 if self.container_ready_time is None:
@@ -280,14 +314,16 @@ class TestDriver:
                 "\tTests completed            : %d\n"\
                 "\t\tSuccess : %d\n"\
                 "\t\tFailure : %d\n"\
-                "\t\tError   : %d"%(datetime.timedelta(seconds=total_execution_time_seconds), 
+                "\t\tError   : %d\n"\
+                "\t%s\n"%(datetime.timedelta(seconds=total_execution_time_seconds), 
                                     estimated_completion_time_seconds, 
                                     remaining_tests, 
                                     testsCurrentlyRunning,
                                     numberOfCompletedTests, 
                                     len(self.successes), 
                                     len(self.failures), 
-                                    len(self.errors)))
+                                    len(self.errors),
+                                    system_stats))
     
         except Exception as e:
             print("Error in printing execution summary: [%s]"%(str(e)))

From 0e26dc76ff4053f0065e203f37d82cf9c5f8e93c Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Mon, 20 Dec 2021 13:17:46 -0800
Subject: [PATCH 143/166] Changed so that we no longer start a container if we
 know that we will not have a test for it. For example, if we try to start 4
 containers by have only 2 tests at the beginning we will only start 2
 containers. This saves a lot of startup time and resources. There is a
 descriptive printout for this as well. Also, bumped the maximum startup time
 for 6 minutes to 10 minutes.  This, combined with the system info from the
 previous commit should let us determine if we can bump the number of
 containers per GitHub Actions VM from 1 to 2 or more.

---
 .../modules/container_manager.py              |  4 +-
 .../modules/splunk_container.py               | 56 ++++++++++++-------
 .../modules/test_driver.py                    | 29 +++++++---
 3 files changed, 59 insertions(+), 30 deletions(-)

diff --git a/bin/automated_detection_testing/ci/detection_testing_batch/modules/container_manager.py b/bin/automated_detection_testing/ci/detection_testing_batch/modules/container_manager.py
index 5ad47cd70a..f231a7b915 100644
--- a/bin/automated_detection_testing/ci/detection_testing_batch/modules/container_manager.py
+++ b/bin/automated_detection_testing/ci/detection_testing_batch/modules/container_manager.py
@@ -150,7 +150,9 @@ class ContainerManager:
         interactive_failure:bool = False,
         interactive:bool = False
     ) -> list[splunk_container.SplunkContainer]:
-        #First make sure that the image exists and has been downloaded
+        #First make sure that the image exists and has been downloaded.
+        #Note that this is intentionally not part of the time to start
+        #since it can take a long time on a slow connection!
         self.setup_image(reuse_image, full_docker_hub_name)
 
         new_containers = []
diff --git a/bin/automated_detection_testing/ci/detection_testing_batch/modules/splunk_container.py b/bin/automated_detection_testing/ci/detection_testing_batch/modules/splunk_container.py
index 0845b3fc65..ffe9b78653 100644
--- a/bin/automated_detection_testing/ci/detection_testing_batch/modules/splunk_container.py
+++ b/bin/automated_detection_testing/ci/detection_testing_batch/modules/splunk_container.py
@@ -22,7 +22,8 @@ import sys
 SPLUNKBASE_URL = "https://splunkbase.splunk.com/app/%d/release/%s/download"
 SPLUNK_START_ARGS = "--accept-license"
 
-MAX_CONTAINER_START_TIME_SECONDS = 360
+#Give ten minutes to start - this is probably enough time
+MAX_CONTAINER_START_TIME_SECONDS = 60*10
 class SplunkContainer:
     def __init__(
         self,
@@ -277,7 +278,7 @@ class SplunkContainer:
         summary_str = "Summary for %s\n\t"\
                       "Total Time          : [%s]\n\t"\
                       "Container Start Time: [%s]\n\t"\
-                      "Test Execution Time : [%s]" % (
+                      "Test Execution Time : [%s]\n" % (
                           self.container_name, total_time_string, setup_time_string, testing_time_string)
 
         return summary_str
@@ -325,9 +326,34 @@ class SplunkContainer:
         print("Finished copying files to [%s]" % (self.container_name))
         self.wait_for_splunk_ready()
         
+    def successfully_finish_tests(self)->None:
+        try:
+            if self.num_tests_completed == 0:
+                print("Container [%s] did not find any tests and will not start.\n"\
+                      "This does not mean there was an error!"%(self.container_name))
+            else:
+                print("Container [%s] has finished running [%d] detections, time to stop the container."
+                      % (self.container_name, self.num_tests_completed))
+            
+            
+            # remove the container
+            self.removeContainer()
+        except Exception as e:
+            print(
+                "Error stopping or removing the container: [%s]" % (str(e)))
+
+        return None
+    
 
     def run_container(self) -> None:
         print("Starting the container [%s]" % (self.container_name))
+        
+        # Try to get something from the queue. Check this early on
+        # before launching the container because it can save us a lot of time!
+        detection_to_test = self.synchronization_object.getTest()
+        if detection_to_test is None:
+            return self.successfully_finish_tests()
+
         self.container_start_time = timeit.default_timer()
     
         container_start_time = timeit.default_timer()
@@ -352,33 +378,17 @@ class SplunkContainer:
         # Sleep for a small random time so that containers drift apart and don't synchronize their testing
         time.sleep(random.randint(1, 30))
         self.test_start_time = timeit.default_timer()
-        while True:
+        while detection_to_test is not None:
             if self.synchronization_object.checkContainerFailure():
                 self.container.stop()
                 print("Container [%s] successfully stopped early due to failure" % (self.container_name))
                 return None
 
-            # Try to get something from the queue
-            detection_to_test = self.synchronization_object.getTest()
 
             # Sleep for a small random time so that containers drift apart and don't synchronize their testing
             time.sleep(random.randint(1, 30))
             
-            if detection_to_test is None:
-                try:
-                    print(
-                        "Container [%s] has finished running detections, time to stop the container."
-                        % (self.container_name)
-                    )
-                    
-                    # remove the container
-                    self.removeContainer()
-                except Exception as e:
-                    print(
-                        "Error stopping or removing the container: [%s]" % (str(e)))
-
-                return None
-
+             
             # There is a detection to test
             print("Container [%s]--->[%s]" %
                   (self.container_name, detection_to_test))
@@ -413,4 +423,10 @@ class SplunkContainer:
                 )
             self.num_tests_completed += 1
 
+            # Try to get something from the queue
+            detection_to_test = self.synchronization_object.getTest()
+            
+        #We failed to get a test from the queue, so we must be done gracefully!  Quit
+        return self.successfully_finish_tests()
+
             
diff --git a/bin/automated_detection_testing/ci/detection_testing_batch/modules/test_driver.py b/bin/automated_detection_testing/ci/detection_testing_batch/modules/test_driver.py
index 87cb71855f..d77ee433d1 100644
--- a/bin/automated_detection_testing/ci/detection_testing_batch/modules/test_driver.py
+++ b/bin/automated_detection_testing/ci/detection_testing_batch/modules/test_driver.py
@@ -74,6 +74,21 @@ class TestDriver:
         finally:
             self.lock.release()
         
+    def checkIfTestsRemain(self):
+        failure = self.checkContainerFailure()
+        if failure:
+            #Just return None, don't continue testing if a container crashed
+            #Indicate there are no tests remaining
+            return False
+        
+        try:
+            #This call isn't reliable according to documentation, but can save us some time.
+            #Err on the side of caution
+            return not self.testing_queue.empty()
+        except Exception as e:
+            print("Error determinging if testing queue was empty.  Return False and try to get something.",file=sys.stderr)
+            return True
+        
 
     def getTest(self)-> Union[str,None]:
         
@@ -88,7 +103,6 @@ class TestDriver:
         try:
             return self.testing_queue.get(block=False)
         except Exception as e:
-            print("Testing queue empty!")
             return None
         
     def addSuccess(self, result:dict)->None:
@@ -239,17 +253,14 @@ class TestDriver:
 
     def get_system_stats(self)->str:
         
-
-        cpu_info = psutil.cpu_times_percent(percpu=False)
-        print(cpu_info)
-        cpu_info_string = "Total CPU Usage: %d%% (%d CPUs)"%(100 - cpu_info.idle, psutil.cpu_count(logical=False))
-        
         bytes_per_GB = 1024 * 1024 * 1024
+        cpu_info = psutil.cpu_times_percent(percpu=False)
         memory_info = psutil.virtual_memory()
-        memory_info_string = "Total Memory Usage: %0.1fGB USED / %0.1fGB TOTAL"%(memory_info.used / bytes_per_GB, memory_info.total / bytes_per_GB)
-        
         disk_usage_info = psutil.disk_usage('/')
-        disk_usage_info_string = "Total Disk Usage: %0.1fGB USED / %0.1fGB TOTAL"%(disk_usage_info.free / bytes_per_GB, disk_usage_info.total / bytes_per_GB)
+
+        cpu_info_string =        "Total CPU Usage   : %d%% (%d CPUs)"%(100 - cpu_info.idle, psutil.cpu_count(logical=False))
+        memory_info_string =     "Total Memory Usage: %0.1fGB USED / %0.1fGB TOTAL"%(memory_info.used / bytes_per_GB, memory_info.total / bytes_per_GB)
+        disk_usage_info_string = "Total Disk Usage  : %0.1fGB USED / %0.1fGB TOTAL"%(disk_usage_info.free / bytes_per_GB, disk_usage_info.total / bytes_per_GB)
 
         return "System Information:\n\t%s\n\t%s\n\t%s"%(cpu_info_string, memory_info_string, disk_usage_info_string)
 

From d26032b912993c02255b25646a58a46b823aa7a0 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Mon, 20 Dec 2021 13:38:51 -0800
Subject: [PATCH 144/166] Updated the github_actions config with S3 binary
 paths.Updated the default args to include web and experimental as possible
 folders with Web being a default.

---
 .../modules/validate_args.py                  |   4 +-
 .../test_config_github_actions.json           | 152 +++++++++++-------
 2 files changed, 97 insertions(+), 59 deletions(-)

diff --git a/bin/automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py b/bin/automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py
index 868c2455c8..186a2a5ded 100644
--- a/bin/automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py
+++ b/bin/automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py
@@ -345,9 +345,9 @@ setup_schema = {
             "type": "array",
             "items": {
                 "type": "string",
-                "enum": ["endpoint", "cloud", "network","web"]
+                "enum": ["endpoint", "cloud", "network","web","experimental"]
             },
-            "default": ["endpoint", "cloud", "network"]
+            "default": ["endpoint", "cloud", "network","web"]
         },
 
         "types": {
diff --git a/bin/automated_detection_testing/ci/detection_testing_batch/test_config_github_actions.json b/bin/automated_detection_testing/ci/detection_testing_batch/test_config_github_actions.json
index 81fa25f89d..374f1b5187 100644
--- a/bin/automated_detection_testing/ci/detection_testing_batch/test_config_github_actions.json
+++ b/bin/automated_detection_testing/ci/detection_testing_batch/test_config_github_actions.json
@@ -7,14 +7,105 @@
     "folders": [
         "endpoint",
         "cloud",
-        "network"
+        "network",
+	"web"
     ],
     "interactive": false,
     "local_apps": {
+        "SPLUNK_ADD_ON_FOR_AMAZON_KINESIS_FIREHOSE": {
+            "app_number": 12,
+            "app_version": null,
+            "http_path": "https://attack-range-appbinaries.s3-us-west-2.amazonaws.com/splunk-add-on-for-amazon-kinesis-firehose_131r7d1d093.tgz"
+        },
+        "SPLUNK_ADD_ON_FOR_MICROSOFT_OFFICE_365": {
+            "app_number": 13,
+            "app_version": null,
+            "http_path": "https://attack-range-appbinaries.s3-us-west-2.amazonaws.com/splunk-add-on-for-microsoft-office-365_202.tgz"
+        },
+        "SPLUNK_ADD_ON_FOR_STREAM_FORWARDERS": {
+            "app_number": 9,
+            "app_version": null,
+            "http_path": "https://attack-range-appbinaries.s3-us-west-2.amazonaws.com/splunk-add-on-for-stream-forwarders_730.tgz"
+        },
+        "SPLUNK_ADD_ON_FOR_STREAM_WIRE_DATA": {
+            "app_number": 8,
+            "app_version": null,
+            "http_path": "https://attack-range-appbinaries.s3-us-west-2.amazonaws.com/splunk-add-on-for-stream-wire-data_730.tgz"
+        },
+        "SPLUNK_ADD_ON_FOR_SYSMON_OLD": {
+            "app_number": 1,
+            "app_version": null,
+            "http_path": "https://attack-range-appbinaries.s3-us-west-2.amazonaws.com/splunk-add-on-for-microsoft-sysmon_1062.tgz"
+        },
+        "SPLUNK_ADD_ON_FOR_ZEEK_AKA_BRO": {
+            "app_number": 11,
+            "app_version": null,
+            "http_path": "https://attack-range-appbinaries.s3-us-west-2.amazonaws.com/splunk-add-on-for-zeek-aka-bro_400.tgz"
+        },
+        "SPLUNK_ASX_APP": {
+            "app_number": 17,
+            "app_version": null,
+            "http_path": "https://attack-range-appbinaries.s3-us-west-2.amazonaws.com/Splunk_ASX-latest.tar.gz"
+        },
+        "SPLUNK_AWS_TA": {
+            "app_number": 4,
+            "app_version": null,
+            "http_path": "https://attack-range-appbinaries.s3-us-west-2.amazonaws.com/splunk-add-on-for-amazon-web-services_510.tgz"
+        },
+        "SPLUNK_CIM_APP": {
+            "app_number": 3,
+            "app_version": null,
+            "http_path": "https://attack-range-appbinaries.s3-us-west-2.amazonaws.com/splunk-common-information-model-cim_4200.tgz"
+        },
         "SPLUNK_ES_CONTENT_UPDATE": {
             "app_number": 3449,
             "app_version": null,
-            "local_path": "prior_config/apps/DA-ESS-ContentUpdate-latest.tar.gz"
+            "local_path": null
+        },
+        "SPLUNK_LINUX_TA": {
+            "app_number": 14,
+            "app_version": null,
+            "http_path": "https://attack-range-appbinaries.s3-us-west-2.amazonaws.com/splunk-add-on-for-unix-and-linux_820.tgz"
+        },
+        "SPLUNK_MLTK_APP": {
+            "app_number": 6,
+            "app_version": null,
+            "http_path": "https://attack-range-appbinaries.s3-us-west-2.amazonaws.com/splunk-machine-learning-toolkit_521.tgz"
+        },
+        "SPLUNK_NGINX_TA": {
+            "app_number": 15,
+            "app_version": null,
+            "http_path": "https://attack-range-appbinaries.s3-us-west-2.amazonaws.com/splunk-add-on-for-nginx_310.tgz"
+        },
+        "SPLUNK_PYTHON_APP": {
+            "app_number": 5,
+            "app_version": null,
+            "http_path": "https://attack-range-appbinaries.s3-us-west-2.amazonaws.com/python-for-scientific-computing-for-linux-64-bit_202.tgz"
+        },
+        "SPLUNK_SECURITY_ESSENTIALS": {
+            "app_number": 10,
+            "app_version": null,
+            "http_path": "https://attack-range-appbinaries.s3-us-west-2.amazonaws.com/splunk-security-essentials_333.tgz"
+        },
+        "SPLUNK_STREAM_APP": {
+            "app_number": 7,
+            "app_version": null,
+            "http_path": "https://attack-range-appbinaries.s3-us-west-2.amazonaws.com/splunk-app-for-stream_730.tgz"
+        },
+        "SPLUNK_SYSMON_LINUX_TA_PATCHED": {
+            "app_number": 2,
+            "app_version": null,
+            "http_path": "https://attack-range-appbinaries.s3-us-west-2.amazonaws.com/add-on-for-linux-sysmon_103_PATCHED.tgz"
+        },
+        "SPLUNK_TA_FOR_ZEEK": {
+            "app_number": 16,
+            "app_version": null,
+            "http_path": "https://attack-range-appbinaries.s3-us-west-2.amazonaws.com/ta-for-zeek_105.tgz"
+        },
+        "SPLUNK_WINDOWS_TA": {
+            "app_number": 0,
+            "app_version": null,
+            "http_path": "https://attack-range-appbinaries.s3-us-west-2.amazonaws.com/splunk-add-on-for-microsoft-windows_812.tgz"
         }
     },
     "local_base_container_name": "splunk_test_%d",
@@ -22,66 +113,13 @@
     "mode": "changes",
     "no_interactive_failure": true,
     "num_containers": 10,
-    "persist_security_content": false,
+    "persist_security_content": true,
     "pr_number": null,
     "reuse_image": true,
     "show_splunk_app_password": false,
     "splunk_app_password": null,
     "splunk_container_apps_directory": "/opt/splunk/etc/apps",
-    "splunkbase_apps": {
-        "PYTHON_FOR_SCIENTIC_COMPUTING_LINUX_64_BIT": {
-            "app_number": 2882,
-            "app_version": "2.0.2"
-        },
-        "SPLUNK_ADD_ON_FOR_AMAZON_KINESIS_FIREHOSE": {
-            "app_number": 3719,
-            "app_version": "1.3.2"
-        },
-        "SPLUNK_ADD_ON_FOR_AMAZON_WEB_SERVICES": {
-            "app_number": 1876,
-            "app_version": "5.2.0"
-        },
-        "SPLUNK_ADD_ON_FOR_MICROSOFT_OFFICE_365": {
-            "app_number": 4055,
-            "app_version": "2.2.0"
-        },
-        "SPLUNK_ADD_ON_FOR_STREAM_FORWARDERS": {
-            "app_number": 5238,
-            "app_version": "8.0.1"
-        },
-        "SPLUNK_ADD_ON_FOR_STREAM_WIRE_DATA": {
-            "app_number": 5234,
-            "app_version": "8.0.1"
-        },
-        "SPLUNK_ADD_ON_FOR_SYSMON": {
-            "app_number": 5709,
-            "app_version": "1.0.1"
-        },
-        "SPLUNK_ADD_ON_FOR_UNIX_AND_LINUX": {
-            "app_number": 833,
-            "app_version": "8.3.1"
-        },
-        "SPLUNK_ADD_ON_FOR_ZEEK_AKA_BRO": {
-            "app_number": 1617,
-            "app_version": "4.0.0"
-        },
-        "SPLUNK_ANALYTIC_STORY_EXECUTION_APP": {
-            "app_number": 4971,
-            "app_version": "2.0.3"
-        },
-        "SPLUNK_APP_FOR_STREAM": {
-            "app_number": 1809,
-            "app_version": "8.0.1"
-        },
-        "SPLUNK_COMMON_INFORMATION_MODEL": {
-            "app_number": 1621,
-            "app_version": "4.20.2"
-        },
-        "SPLUNK_MACHINE_LEARNING_TOOLKIT": {
-            "app_number": 2890,
-            "app_version": "5.2.2"
-        }
-    },
+    "splunkbase_apps": {},
     "splunkbase_password": null,
     "splunkbase_username": null,
     "types": [

From 264dbfb7f487da7151afbc88ad775d6b370081ae Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Mon, 20 Dec 2021 13:58:58 -0800
Subject: [PATCH 145/166] Fixed up and error that could occur where the
 environment is not properly set up if the users requested
 PERSIST_SECURITY_CONTENT, but the directory did not exist.  Also fixed the
 default argument for this on GitHub Actions.,

---
 .../detection_testing_execution.py                        | 8 +++++---
 .../test_config_github_actions.json                       | 2 +-
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/bin/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py b/bin/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
index 8977119ec2..c81c4cb86b 100644
--- a/bin/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
+++ b/bin/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
@@ -94,7 +94,7 @@ def copy_local_apps_to_directory(apps: dict[str, dict], target_directory) -> Non
 
 
 
-def ensure_security_content(branch: str, commit_hash: str, pr_number: Union[int, None], persist_security_content: bool) -> GithubService:
+def ensure_security_content(branch: str, commit_hash: str, pr_number: Union[int, None], persist_security_content: bool) -> tuple[GithubService, bool]:
     if persist_security_content is True and os.path.exists("security_content"):
         print("****** You chose --persist_security_content and the security_content directory exists. "
               "We will not check out the repo again. Please be aware, this could cause issues if your "
@@ -109,6 +109,7 @@ def ensure_security_content(branch: str, commit_hash: str, pr_number: Union[int,
         if persist_security_content is True and not os.path.exists("security_content"):
             print("Error - you chose --persist_security_content but the security_content directory does not exist!"
                   "  We will check it out for you.")
+            persist_security_content = False
 
         elif os.path.exists("security_content/"):
             print("Deleting the security_content directory")
@@ -125,7 +126,7 @@ def ensure_security_content(branch: str, commit_hash: str, pr_number: Union[int,
         else:
             github_service = GithubService(branch, commit_hash)
 
-    return github_service
+    return github_service, persist_security_content
 
 
 def generate_escu_app(persist_security_content: bool = False) -> str:
@@ -354,7 +355,8 @@ def main(args: list[str]):
 
     # Check out security content if required
     try:
-        github_service = ensure_security_content(
+        #Make sure we fix up the persist_securiy_content argument if it is passed in error (we say it exists but it doesn't)
+        github_service, settings['persist_security_content'] = ensure_security_content(
             settings['branch'], settings['commit_hash'], settings['pr_number'], settings['persist_security_content'])
         settings['commit_hash'] = github_service.commit_hash
     except Exception as e:
diff --git a/bin/automated_detection_testing/ci/detection_testing_batch/test_config_github_actions.json b/bin/automated_detection_testing/ci/detection_testing_batch/test_config_github_actions.json
index 374f1b5187..5b94c52a15 100644
--- a/bin/automated_detection_testing/ci/detection_testing_batch/test_config_github_actions.json
+++ b/bin/automated_detection_testing/ci/detection_testing_batch/test_config_github_actions.json
@@ -113,7 +113,7 @@
     "mode": "changes",
     "no_interactive_failure": true,
     "num_containers": 10,
-    "persist_security_content": true,
+    "persist_security_content": false,
     "pr_number": null,
     "reuse_image": true,
     "show_splunk_app_password": false,

From d4aae32b3d15179d2af0cd4dfec0198aafbcd717 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Mon, 20 Dec 2021 14:26:57 -0800
Subject: [PATCH 146/166] Removed Python 2 which was previously used for splunk
 packaging toolkit. Replaced with python3

---
 .../ci/detection_testing_batch/detection_testing_execution.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/bin/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py b/bin/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
index c81c4cb86b..ad77b8b5a3 100644
--- a/bin/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
+++ b/bin/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
@@ -194,13 +194,13 @@ def generate_escu_app(persist_security_content: bool = False) -> str:
             print("Error downloading the Splunk Packaging Toolkit: [%s].\n\tQuitting..." %
                   (str(e)), file=sys.stderr)
             sys.exit(1)
-        print(os.getcwd())
+        
         commands = ["rm -rf slim_packaging/slim_latest",
                     "mkdir slim_packaging/slim_latest",
                     "cd slim_packaging",
                     "tar -zxf ../splunk-packaging-toolkit-latest.tar.gz -C slim_latest --strip-components=1",
                     "cd slim_latest",
-                    "virtualenv --python=/usr/bin/python2.7 --clear .venv",
+                    "python3 -m venv .venv",
                     ". ./.venv/bin/activate",
                     "python -m pip install --upgrade pip",
                     "python -m pip install wheel",

From 63555bc53148570f0787966822839deaef7af95e Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Mon, 20 Dec 2021 14:45:01 -0800
Subject: [PATCH 147/166] Added some more robust error handling to the high
 level test runner.  It looks like we were getting errors pulling the image
 from docker hub - was it down...

---
 .../detection_testing_execution.py            | 51 +++++++++++--------
 1 file changed, 29 insertions(+), 22 deletions(-)

diff --git a/bin/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py b/bin/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
index ad77b8b5a3..7502fd64dc 100644
--- a/bin/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
+++ b/bin/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
@@ -450,30 +450,37 @@ def main(args: list[str]):
         "local_file_path": datamodel_file_local_path, "container_file_path": datamodel_file_container_path}
 
     
+    try:
+        cm = container_manager.ContainerManager(all_test_files,
+                                                FULL_DOCKER_HUB_CONTAINER_NAME,
+                                                settings['local_base_container_name'],
+                                                settings['num_containers'],
+                                                settings['local_apps'],
+                                                settings['splunkbase_apps'],
+                                                settings['branch'],
+                                                settings['commit_hash'],
+                                                reproduce_test_config,
+                                                files_to_copy_to_container=files_to_copy_to_container,
+                                                web_port_start=8000,
+                                                management_port_start=8089,
+                                                mounts=mounts,
+                                                show_container_password=settings['show_splunk_app_password'],
+                                                container_password=settings['splunk_app_password'],
+                                                splunkbase_username=settings['splunkbase_username'],
+                                                splunkbase_password=settings['splunkbase_password'],
+                                                reuse_image=settings['reuse_image'],
+                                                interactive_failure=not settings['no_interactive_failure'],
+                                                interactive=settings['interactive'])
+    except Exception as e:
+        print("Error - unrecoverable error trying to set up the containers: [%s].\n\tQuitting..."%(str(e)),file=sys.stderr)
+        sys.exit(1)
 
-    cm = container_manager.ContainerManager(all_test_files,
-                                            FULL_DOCKER_HUB_CONTAINER_NAME,
-                                            settings['local_base_container_name'],
-                                            settings['num_containers'],
-                                            settings['local_apps'],
-                                            settings['splunkbase_apps'],
-                                            settings['branch'],
-                                            settings['commit_hash'],
-                                            reproduce_test_config,
-                                            files_to_copy_to_container=files_to_copy_to_container,
-                                            web_port_start=8000,
-                                            management_port_start=8089,
-                                            mounts=mounts,
-                                            show_container_password=settings['show_splunk_app_password'],
-                                            container_password=settings['splunk_app_password'],
-                                            splunkbase_username=settings['splunkbase_username'],
-                                            splunkbase_password=settings['splunkbase_password'],
-                                            reuse_image=settings['reuse_image'],
-                                            interactive_failure=not settings['no_interactive_failure'],
-                                            interactive=settings['interactive'])
+    try:
+        result = cm.run_test()
+    except Exception as e:
+        print("Error - there was an error running the tests: [%s]\n\tQuitting..."%(str(e)),file=sys.stderr)
+        sys.exit(1)
 
-    result = cm.run_test()
-    
     #github_service.update_and_commit_passed_tests(cm.synchronization_object.successes)
     
 

From 9ba7abd0e83a32f061d097071ff0d713ef31e296 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Tue, 21 Dec 2021 10:44:50 -0800
Subject: [PATCH 148/166] Fixed a small error. If you specified a PR number and
 a branch that did not exist, then it would create a branch and that would be
 bad. Now, when you specify a branch and a PR number, the branch MUST exist.
 If not, we fail and bail.

---
 .../detection_testing_execution.py            | 10 +++---
 .../modules/github_service.py                 | 32 ++++++++++++++-----
 2 files changed, 29 insertions(+), 13 deletions(-)

diff --git a/bin/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py b/bin/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
index 7502fd64dc..235417869d 100644
--- a/bin/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
+++ b/bin/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
@@ -360,11 +360,11 @@ def main(args: list[str]):
             settings['branch'], settings['commit_hash'], settings['pr_number'], settings['persist_security_content'])
         settings['commit_hash'] = github_service.commit_hash
     except Exception as e:
-        print("\nFailure checking out git repository:"\
-              "\n\tHash:   [%s]"\
-              "\n\tBranch: [%s]"\
-              "\n\tPR:     [%s]\n\tQuitting"%
-              (settings['commit_hash'],settings['branch'],settings['pr_number']),file=sys.stderr)
+        print("\nFailure checking out git repository: [%s]"\
+              "\n\tCommit Hash: [%s]"\
+              "\n\tBranch     : [%s]"\
+              "\n\tPR         : [%s]\n\tQuitting..."%
+              (str(e),settings['commit_hash'],settings['branch'],settings['pr_number']),file=sys.stderr)
         sys.exit(1)
 
     #passes = [{'search_string': '| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) as lastTime from datamodel=Endpoint.Processes where (Processes.process_name ="7z.exe" OR Processes.process_name = "7za.exe" OR Processes.original_file_name = "7z.exe" OR Processes.original_file_name =  "7za.exe") AND (Processes.process="*\\\\C$\\\\*" OR Processes.process="*\\\\Admin$\\\\*" OR Processes.process="*\\\\IPC$\\\\*") by Processes.original_file_name Processes.parent_process_name Processes.parent_process Processes.process_name Processes.process Processes.parent_process_id Processes.process_id  Processes.dest Processes.user | `drop_dm_object_name(Processes)` | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)` | `7zip_commandline_to_smb_share_path_filter` | stats count | where count > 0', 'detection_name': '7zip CommandLine To SMB Share Path', 'detection_file': 'endpoint/7zip_commandline_to_smb_share_path.yml', 'success': True, 'error': False, 'diskUsage': '286720', 'runDuration': '0.922', 'scanCount': '4897'}]
diff --git a/bin/automated_detection_testing/ci/detection_testing_batch/modules/github_service.py b/bin/automated_detection_testing/ci/detection_testing_batch/modules/github_service.py
index 798ae31914..a1962a710d 100644
--- a/bin/automated_detection_testing/ci/detection_testing_batch/modules/github_service.py
+++ b/bin/automated_detection_testing/ci/detection_testing_batch/modules/github_service.py
@@ -35,23 +35,39 @@ class GithubService:
             self.security_content_repo_obj = self.clone_project(
                 SECURITY_CONTENT_URL, f"security_content", f"develop")
 
+        #Ensure that the branch name is valid   
+        #Get all the branch names, prefixed with "origin/"
+        branch_names = [branch.name for branch in self.security_content_repo_obj.remote().refs]
+
+        if "origin/%s"%(security_content_branch) not in branch_names:
+            raise(Exception("Branch name [%s] not found in valid branches.  Try running \n"\
+                           "'git branch -a' to examine [%d] branches"%(security_content_branch, len(branch_names))))
+
+
         if commit_hash is not None and PR_number is not None:
-            print("Error - both the PR number [%d] and the commit hash [%s] were provided.  "
-                  "Only 0 or 1 can be passed.\n\tQuitting..." % (PR_number, commit_hash))
-            sys.exit()
+            raise(Exception("Error - both the PR number [%d] and the commit hash [%s] were provided.  "
+                  "Only 0 or 1 can be passed." % (PR_number, commit_hash)))
+            
 
         elif PR_number:
-            ret = subprocess.call(["git", "-C", "security_content/", "fetch", "origin",
-                            "refs/pull/%d/head:%s" % (PR_number, security_content_branch)])
-            if ret != 0:
-                raise(Exception("Error checking out repository"))
+            ret = subprocess.run(["git", "-C", "security_content/", "fetch", "origin",
+                            "refs/pull/%d/head:%s" % (PR_number, security_content_branch)], capture_output=True)
+            #ret = subprocess.call(["git", "-C", "security_content/", "fetch", "origin",
+            #                "refs/pull/%d/head:%s" % (PR_number, security_content_branch)])
+            
+            
+            if ret.returncode != 0:
+                raise(Exception("Error checking out repository: [%s]"%(ret.stdout.decode("utf-8") + "\n" + ret.stderr.decode("utf-8"))))
+            
 
         # No checking to see if the hash is to a commit inside of the branch - the user
-        # has to do that by hand
+        # has to do that by hand.
         if commit_hash is not None:
             print("Checking out commit hash: [%s]" % (commit_hash))
             self.security_content_repo_obj.git.checkout(commit_hash)
         else:
+            #Even if we have fetched a PR, we still MUST check out the branch to
+            # be able to do anything with it. Otherwise we won't have the files
             print("Checking out branch: [%s]..." %
                   (security_content_branch), end='')
             sys.stdout.flush()

From 5f972f734feeef6832d2cff0776cdc491aa90e7c Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Tue, 21 Dec 2021 11:00:11 -0800
Subject: [PATCH 149/166] Re-push to test everything with 2 containers per GH
 Actions machine.

---
 .github/workflows/build-and-validate.yml                        | 2 +-
 .../ci/detection_testing_batch/detection_testing_execution.py   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/build-and-validate.yml b/.github/workflows/build-and-validate.yml
index c6ed4e4db5..5cddd0389e 100644
--- a/.github/workflows/build-and-validate.yml
+++ b/.github/workflows/build-and-validate.yml
@@ -355,7 +355,7 @@ jobs:
               python detection_testing_execution.py run --branch ${{ github.event.pull_request.head.ref }} --mode all --mock --config_file test_config_github_actions.json  
             elif [[ ! -z "${{  github.event.pull_request.head.ref }}" &&  ! -z "${{ github.event.pull_request.number }}" ]]; then
               echo "Pull request from source branch [${{ github.event.pull_request.head.ref }}] for PR number [${{ github.event.issue.number }}]"
-              python detection_testing_execution.py run --branch ${{ github.event.pull_request.head.ref }} --pr_number ${{ github.event.pull_request.number }}  --mode changes --mock --config_file test_config_github_actions.json  
+              python detection_testing_execution.py run --branch ${{ github.event.pull_request.head.ref }} --pr_number ${{ github.event.pull_request.number }}  --mode all --mock --config_file test_config_github_actions.json  
             else
               echo "Push from branch [${{ steps.vars.outputs.branch }}]"
               python detection_testing_execution.py run --branch  ${{ steps.vars.outputs.branch }}  --mode changes --mock --config_file test_config_github_actions.json
diff --git a/bin/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py b/bin/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
index 235417869d..c8f5557eec 100644
--- a/bin/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
+++ b/bin/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
@@ -264,7 +264,7 @@ def finish_mock(settings: dict, detections: list[str], output_file_template: str
         mock_settings = copy.deepcopy(settings)
         # This may be able to support as many as 2 for GitHub Actions...
         # we will have to determine in testing.
-        mock_settings['num_containers'] = 1
+        mock_settings['num_containers'] = 2
 
         # Must be selected since we are passing in a list of detections
         mock_settings['mode'] = 'selected'

From 04c526ba6de64260488a093d4d05f985c8f09837 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Tue, 21 Dec 2021 11:17:09 -0800
Subject: [PATCH 150/166] Changed the wrong line to trigger a test of
 everything. Trying again.

---
 .github/workflows/build-and-validate.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/build-and-validate.yml b/.github/workflows/build-and-validate.yml
index 5cddd0389e..2c7347c1f9 100644
--- a/.github/workflows/build-and-validate.yml
+++ b/.github/workflows/build-and-validate.yml
@@ -355,10 +355,10 @@ jobs:
               python detection_testing_execution.py run --branch ${{ github.event.pull_request.head.ref }} --mode all --mock --config_file test_config_github_actions.json  
             elif [[ ! -z "${{  github.event.pull_request.head.ref }}" &&  ! -z "${{ github.event.pull_request.number }}" ]]; then
               echo "Pull request from source branch [${{ github.event.pull_request.head.ref }}] for PR number [${{ github.event.issue.number }}]"
-              python detection_testing_execution.py run --branch ${{ github.event.pull_request.head.ref }} --pr_number ${{ github.event.pull_request.number }}  --mode all --mock --config_file test_config_github_actions.json  
+              python detection_testing_execution.py run --branch ${{ github.event.pull_request.head.ref }} --pr_number ${{ github.event.pull_request.number }}  --mode changes --mock --config_file test_config_github_actions.json  
             else
               echo "Push from branch [${{ steps.vars.outputs.branch }}]"
-              python detection_testing_execution.py run --branch  ${{ steps.vars.outputs.branch }}  --mode changes --mock --config_file test_config_github_actions.json
+              python detection_testing_execution.py run --branch  ${{ steps.vars.outputs.branch }}  --mode all --mock --config_file test_config_github_actions.json
             fi
 
             mv *-test-run.json replicate_test.json

From 47e155b74724b7b19c8770d90eedd5af54a5305c Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Tue, 21 Dec 2021 14:22:33 -0800
Subject: [PATCH 151/166] Changed back to one container per GH Action Machine
 config with the mock option.  Also, shuffling detections after they are put
 into a list to distribute runtime and load as much as possible.

---
 .../detection_testing_execution.py                    | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/bin/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py b/bin/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
index c8f5557eec..c7cd6c466a 100644
--- a/bin/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
+++ b/bin/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
@@ -264,7 +264,7 @@ def finish_mock(settings: dict, detections: list[str], output_file_template: str
         mock_settings = copy.deepcopy(settings)
         # This may be able to support as many as 2 for GitHub Actions...
         # we will have to determine in testing.
-        mock_settings['num_containers'] = 2
+        mock_settings['num_containers'] = 1
 
         # Must be selected since we are passing in a list of detections
         mock_settings['mode'] = 'selected'
@@ -384,6 +384,15 @@ def main(args: list[str]):
                                                     settings['types'],
                                                     settings['detections_list'],
                                                     settings['detections_file'])
+        
+        #We randomly shuffle this because there are likely patterns in searches.  For example,
+        #cloud/endpoint/network likely have different impacts on the system.  By shuffling,
+        #we spread out this load on a single computer, but also spread it in case
+        #we are running on GitHub Actions against multiple machines.  Hopefully, this
+        #will reduce that chnaces the some machines run and complete quickly while
+        #others take a long time.
+        random.shuffle(all_test_files)        
+
     except Exception as e:
         print("Error getting test files:\n%s"%(str(e)), file=sys.stderr)
         print("\tQuitting...", file=sys.stderr)

From 05821f43037ad1c1892e73c332417e34f49f0bd8 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Tue, 21 Dec 2021 14:46:44 -0800
Subject: [PATCH 152/166] Fixed some parts of the CI that don't give a
 descriptive error message in the output files when a search generates certain
 types of errors. For example, an error where it cannot reach the Splunk
 endpoint server.  These are rare, but good to have.

---
 .../ci/detection_testing_batch/modules/splunk_sdk.py  | 11 ++++++++---
 .../modules/testing_service.py                        |  1 +
 2 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/bin/automated_detection_testing/ci/detection_testing_batch/modules/splunk_sdk.py b/bin/automated_detection_testing/ci/detection_testing_batch/modules/splunk_sdk.py
index 0ed8259077..508da4efe3 100644
--- a/bin/automated_detection_testing/ci/detection_testing_batch/modules/splunk_sdk.py
+++ b/bin/automated_detection_testing/ci/detection_testing_batch/modules/splunk_sdk.py
@@ -1,3 +1,4 @@
+from os import error
 import sys
 from time import sleep
 import splunklib.results as results
@@ -114,8 +115,10 @@ def test_detection_search(splunk_host:str, splunk_port:int, splunk_password:str,
             password=splunk_password
         )
     except Exception as e:
-        print("Unable to connect to Splunk instance: " + str(e),file=sys.stderr)
+        error_message = "Unable to connect to Splunk instance: %s"%(str(e))
+        print(error_message,file=sys.stderr)
         test_results['error'] = True
+        test_results['detection_error'] = error_message
         return test_results
 
 
@@ -130,9 +133,11 @@ def test_detection_search(splunk_host:str, splunk_port:int, splunk_password:str,
     try:
         job = service.jobs.create(splunk_search, **kwargs)
     except Exception as e:
-
-        print("Unable to execute detection:\n%s"%(str(e)))
+        
+        error_message = "Unable to execute detection: %s"%(str(e))
+        print(error_message,file=sys.stderr)
         test_results['error'] = True
+        test_results['detection_error'] = error_message
         return test_results
 
     test_results['diskUsage'] = job['diskUsage']
diff --git a/bin/automated_detection_testing/ci/detection_testing_batch/modules/testing_service.py b/bin/automated_detection_testing/ci/detection_testing_batch/modules/testing_service.py
index 6efda4c375..1329431269 100644
--- a/bin/automated_detection_testing/ci/detection_testing_batch/modules/testing_service.py
+++ b/bin/automated_detection_testing/ci/detection_testing_batch/modules/testing_service.py
@@ -109,6 +109,7 @@ def test_detection(splunk_ip:str, splunk_port:int, container_name:str, splunk_pa
     result_detection = splunk_sdk.test_detection_search(splunk_ip, splunk_port, splunk_password, detection['search'], test['pass_condition'], detection['name'], test['file'], test['earliest_time'], test['latest_time'])
     if result_detection['error']:
         print("There was an error running the search: %s"%(result_detection['search_string']))
+        
 
 
 

From dc3fc8296d3632e4ca95a557b93a240eac2c3c97 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Wed, 22 Dec 2021 13:02:21 -0800
Subject: [PATCH 153/166] Added a config file to run tests against latest
 splunkbase apps. Also, added slightly better error output for certain types
 of test failures.  Finally updated splunkbase app defaults in validate_args,
 but they are commented out for now while we use the S3 versions.

---
 .../modules/splunk_container.py               |   3 +-
 .../modules/test_driver.py                    |   2 +-
 .../modules/validate_args.py                  |  10 +-
 ...test_config_github_actions_splunkbase.json | 111 ++++++++++++++++++
 4 files changed, 119 insertions(+), 7 deletions(-)
 create mode 100644 bin/automated_detection_testing/ci/detection_testing_batch/test_config_github_actions_splunkbase.json

diff --git a/bin/automated_detection_testing/ci/detection_testing_batch/modules/splunk_container.py b/bin/automated_detection_testing/ci/detection_testing_batch/modules/splunk_container.py
index ffe9b78653..71de2a9ed6 100644
--- a/bin/automated_detection_testing/ci/detection_testing_batch/modules/splunk_container.py
+++ b/bin/automated_detection_testing/ci/detection_testing_batch/modules/splunk_container.py
@@ -131,7 +131,8 @@ class SplunkContainer:
             env["SPLUNKBASE_USERNAME"] = splunkbase_username
             env["SPLUNKBASE_PASSWORD"] = splunkbase_password
         env["SPLUNK_APPS_URL"] = splunk_apps_url
-
+        
+        
         return env
 
     def make_ports(self, *ports: tuple[str, int]) -> dict[str, int]:
diff --git a/bin/automated_detection_testing/ci/detection_testing_batch/modules/test_driver.py b/bin/automated_detection_testing/ci/detection_testing_batch/modules/test_driver.py
index d77ee433d1..67c367853b 100644
--- a/bin/automated_detection_testing/ci/detection_testing_batch/modules/test_driver.py
+++ b/bin/automated_detection_testing/ci/detection_testing_batch/modules/test_driver.py
@@ -124,7 +124,7 @@ class TestDriver:
 
     def addError(self, detection:dict)->None:
         #Make sure that even errors have all of the required fields.
-        for required_field in ['search_string', 'diskUsage','runDuration', 'detection_name', 'scanCount']:
+        for required_field in ['search_string', 'diskUsage','runDuration', 'detection_name', 'scanCount', 'detection_error']:
             if required_field not in detection:
                 detection[required_field] = ""
         if  'error' not in detection:
diff --git a/bin/automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py b/bin/automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py
index 186a2a5ded..20704aed77 100644
--- a/bin/automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py
+++ b/bin/automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py
@@ -254,7 +254,7 @@ setup_schema = {
             #     },
             #     "SPLUNK_ADD_ON_FOR_AMAZON_WEB_SERVICES": {
             #         "app_number": 1876,
-            #         "app_version": "5.2.0"
+            #         "app_version": "5.2.1"
             #     },
 
             #     "SPLUNK_ADD_ON_FOR_MICROSOFT_OFFICE_365":
@@ -275,12 +275,12 @@ setup_schema = {
 
             #     "PYTHON_FOR_SCIENTIC_COMPUTING_LINUX_64_BIT": {
             #         "app_number": 2882,
-            #         "app_version": "2.0.2"
+            #         "app_version": "3.0.1"
             #     },
 
             #     "SPLUNK_MACHINE_LEARNING_TOOLKIT": {
             #         "app_number": 2890,
-            #         "app_version": "5.2.2"
+            #         "app_version": "5.3.0"
             #     },
 
             #     "SPLUNK_APP_FOR_STREAM": {
@@ -301,7 +301,7 @@ setup_schema = {
             #     },
             #     "SPLUNK_ADD_ON_FOR_UNIX_AND_LINUX": {
             #         "app_number": 833,
-            #         "app_version": "8.3.1"
+            #         "app_version": "8.4.0"
             #     },
             #     "SPLUNK_ADD_ON_FOR_SYSMON": {
             #         "app_number": 5709,
@@ -310,7 +310,7 @@ setup_schema = {
             #     # According to https://docs.splunk.com/Documentation/ES/6.6.2/Install/Datamodels, these are included in ES. Don't install separately.
             #     "SPLUNK_COMMON_INFORMATION_MODEL": {
             #         "app_number": 1621,
-            #         "app_version": "4.20.2"
+            #         "app_version": "5.0.0"
             #     }
             # }
         },
diff --git a/bin/automated_detection_testing/ci/detection_testing_batch/test_config_github_actions_splunkbase.json b/bin/automated_detection_testing/ci/detection_testing_batch/test_config_github_actions_splunkbase.json
new file mode 100644
index 0000000000..e4e04c2ee4
--- /dev/null
+++ b/bin/automated_detection_testing/ci/detection_testing_batch/test_config_github_actions_splunkbase.json
@@ -0,0 +1,111 @@
+{
+    "branch": "BRANCH_DOES_NOT_EXIST_USE_CLI_ARGUMENT",
+    "commit_hash": null,
+    "container_tag": "latest",
+    "detections_file": null,
+    "detections_list": null,
+    "folders": [
+        "endpoint",
+        "cloud",
+        "network",
+	"web"
+    ],
+    "interactive": false,
+    "local_apps": {
+        "SPLUNK_ES_CONTENT_UPDATE": {
+            "app_number": 3449,
+            "app_version": null,
+            "local_path": null
+        },
+       
+        "SPLUNK_SYSMON_LINUX_TA_PATCHED": {
+            "app_number": 2,
+            "app_version": null,
+            "http_path": "https://attack-range-appbinaries.s3-us-west-2.amazonaws.com/add-on-for-linux-sysmon_103_PATCHED.tgz"
+        }
+  
+    },
+    "local_base_container_name": "splunk_test_%d",
+    "mock": false,
+    "mode": "changes",
+    "no_interactive_failure": true,
+    "num_containers": 10,
+    "persist_security_content": false,
+    "pr_number": null,
+    "reuse_image": true,
+    "show_splunk_app_password": false,
+    "splunk_app_password": null,
+    "splunk_container_apps_directory": "/opt/splunk/etc/apps",
+    "splunkbase_apps": {
+        
+                 
+                 "SPLUNK_ADD_ON_FOR_AMAZON_WEB_SERVICES": {
+                     "app_number": 1876,
+                     "app_version": "5.2.1"
+                 },
+
+                 "SPLUNK_ADD_ON_FOR_MICROSOFT_OFFICE_365":
+                 {
+                     "app_number": 4055,
+                     "app_version": "2.2.0"
+                 },
+
+                 "SPLUNK_ADD_ON_FOR_AMAZON_KINESIS_FIREHOSE": {
+                     "app_number": 3719,
+                     "app_version": "1.3.2"
+                 },
+
+                 "SPLUNK_ANALYTIC_STORY_EXECUTION_APP": {
+                     "app_number": 4971,
+                     "app_version": "2.0.3"
+                 },
+
+                 "PYTHON_FOR_SCIENTIC_COMPUTING_LINUX_64_BIT": {
+                     "app_number": 2882,
+                     "app_version": "3.0.1"
+                 },
+
+                 "SPLUNK_MACHINE_LEARNING_TOOLKIT": {
+                     "app_number": 2890,
+                     "app_version": "5.3.0"
+                 },
+
+                 "SPLUNK_APP_FOR_STREAM": {
+                     "app_number": 1809,
+                     "app_version": "8.0.1"
+                 },
+                 "SPLUNK_ADD_ON_FOR_STREAM_WIRE_DATA": {
+                     "app_number": 5234,
+                     "app_version": "8.0.1"
+                 },
+                 "SPLUNK_ADD_ON_FOR_STREAM_FORWARDERS": {
+                     "app_number": 5238,
+                     "app_version": "8.0.1"
+                 },
+                 "SPLUNK_ADD_ON_FOR_ZEEK_AKA_BRO": {
+                     "app_number": 1617,
+                     "app_version": "4.0.0"
+                 },
+                 "SPLUNK_ADD_ON_FOR_UNIX_AND_LINUX": {
+                     "app_number": 833,
+                     "app_version": "8.4.0"
+                 },
+                 "SPLUNK_ADD_ON_FOR_SYSMON": {
+                     "app_number": 5709,
+                     "app_version": "1.0.1"
+                 },
+                  
+                 "SPLUNK_COMMON_INFORMATION_MODEL": {
+                     "app_number": 1621,
+                     "app_version": "5.0.0"
+                 }
+            
+    },
+    "splunkbase_password": null,
+    "splunkbase_username": null,
+    "types": [
+        "Anomaly",
+        "Hunting",
+        "TTP"
+    ]
+}

From 2574fb2fea26fdfedc09cb11e1b6bef4de3364f2 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Wed, 22 Dec 2021 13:19:05 -0800
Subject: [PATCH 154/166] Hardcoding the branch to develop for CI testing.

---
 .github/workflows/build-and-validate.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/build-and-validate.yml b/.github/workflows/build-and-validate.yml
index 2c7347c1f9..022bc06615 100644
--- a/.github/workflows/build-and-validate.yml
+++ b/.github/workflows/build-and-validate.yml
@@ -358,7 +358,8 @@ jobs:
               python detection_testing_execution.py run --branch ${{ github.event.pull_request.head.ref }} --pr_number ${{ github.event.pull_request.number }}  --mode changes --mock --config_file test_config_github_actions.json  
             else
               echo "Push from branch [${{ steps.vars.outputs.branch }}]"
-              python detection_testing_execution.py run --branch  ${{ steps.vars.outputs.branch }}  --mode all --mock --config_file test_config_github_actions.json
+              #python detection_testing_execution.py run --branch  ${{ steps.vars.outputs.branch }}  --mode all --mock --config_file test_config_github_actions.json
+              python detection_testing_execution.py run --branch  develop  --mode all --mock --config_file test_config_github_actions_splunkbase.json
             fi
 
             mv *-test-run.json replicate_test.json

From eb48e000a84e1d1dc0dc60a760e3eda1a6c19a45 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Wed, 22 Dec 2021 14:05:55 -0800
Subject: [PATCH 155/166] Updated default app loadout for splunkbase. Added
 splunk_ta_for_nginz, splunk_security_essentials, and ta_for_zeek.

---
 .../test_config_github_actions_splunkbase.json      | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/bin/automated_detection_testing/ci/detection_testing_batch/test_config_github_actions_splunkbase.json b/bin/automated_detection_testing/ci/detection_testing_batch/test_config_github_actions_splunkbase.json
index e4e04c2ee4..3e6ae86949 100644
--- a/bin/automated_detection_testing/ci/detection_testing_batch/test_config_github_actions_splunkbase.json
+++ b/bin/automated_detection_testing/ci/detection_testing_batch/test_config_github_actions_splunkbase.json
@@ -98,8 +98,19 @@
                  "SPLUNK_COMMON_INFORMATION_MODEL": {
                      "app_number": 1621,
                      "app_version": "5.0.0"
+                 },
+                 "SPLUNK_ADD_ON_FOR_NGINX" : {
+                    "app_number": 3258,
+                    "app_version": "3.1.0"
+                 },
+                 "SPLUNK_SECURITY_ESSENTIALS": {
+                    "app_number": 3435,
+                    "app_version": "3.4.0"
+                 },
+                 "TA_FOR_ZEEK": {
+                    "app_number": 5466,
+                    "app_version": "1.0.5"
                  }
-            
     },
     "splunkbase_password": null,
     "splunkbase_username": null,

From 64abbe44586ba07f0cd079cfec5a8c23a5ab051f Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Wed, 5 Jan 2022 14:39:13 -0800
Subject: [PATCH 156/166] Significant changes.  Most importantly, instead of a
 hard-coded wait time after uploading data, we poll to make sure that all the
 data has been indexed.  When this is complete, we move ahead with the test. 
 While this costs extra in terms of CPU and Disk access, it usually allows us
 to go much faster since wait time is usually just a few seconds.  Better
 tracking of time per test and time estimation.

---
 .../detection_testing_execution.py            |   8 ++
 .../modules/container_manager.py              |   8 +-
 .../modules/splunk_container.py               |  14 +-
 .../modules/splunk_sdk.py                     |  97 +++++++++++++-
 .../modules/test_driver.py                    |  76 ++++++-----
 .../modules/testing_service.py                | 120 ++++++++++++------
 6 files changed, 235 insertions(+), 88 deletions(-)

diff --git a/bin/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py b/bin/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
index c7cd6c466a..6e63261ef6 100644
--- a/bin/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
+++ b/bin/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
@@ -42,6 +42,10 @@ datamodel_file_container_path = os.path.join(
     SPLUNK_CONTAINER_APPS_DIR, "Splunk_SA_CIM")
 
 
+authorizations_file_local_path = "authorize.conf.tar"
+authorizations_file_container_path = "/opt/splunk/etc/system/local"
+
+
 MAX_RECOMMENDED_CONTAINERS_BEFORE_WARNING = 2
 
 
@@ -399,6 +403,7 @@ def main(args: list[str]):
         sys.exit(1)
 
     print("***This run will test [%d] detections!***"%(len(all_test_files)))
+    
 
     #Set up the directory that will be used to store the local apps/apps we build
     local_volume_absolute_path = os.path.abspath(
@@ -457,6 +462,9 @@ def main(args: list[str]):
         "local_file_path": index_file_local_path, "container_file_path": index_file_container_path}
     files_to_copy_to_container["DATAMODELS"] = {
         "local_file_path": datamodel_file_local_path, "container_file_path": datamodel_file_container_path}
+    files_to_copy_to_container["AUTHORIZATIONS"] = {
+        "local_file_path": authorizations_file_local_path, "container_file_path": authorizations_file_container_path}
+    
 
     
     try:
diff --git a/bin/automated_detection_testing/ci/detection_testing_batch/modules/container_manager.py b/bin/automated_detection_testing/ci/detection_testing_batch/modules/container_manager.py
index f231a7b915..8bd38d9668 100644
--- a/bin/automated_detection_testing/ci/detection_testing_batch/modules/container_manager.py
+++ b/bin/automated_detection_testing/ci/detection_testing_batch/modules/container_manager.py
@@ -228,7 +228,13 @@ class ContainerManager:
                 print("All containers stopped")
                 
                 return None
-            if self.synchronization_object.summarize() == False:
+
+            at_least_one_container_has_started_running_tests = False
+            for container in self.containers:
+                if container.test_start_time != -1:
+                    at_least_one_container_has_started_running_tests = True
+                    break
+            if self.synchronization_object.summarize(testing_currently_active = at_least_one_container_has_started_running_tests) == False:
                 #There are no more tests to run, so we can return from this thread
                 
                 return None
diff --git a/bin/automated_detection_testing/ci/detection_testing_batch/modules/splunk_container.py b/bin/automated_detection_testing/ci/detection_testing_batch/modules/splunk_container.py
index 71de2a9ed6..72986d27f1 100644
--- a/bin/automated_detection_testing/ci/detection_testing_batch/modules/splunk_container.py
+++ b/bin/automated_detection_testing/ci/detection_testing_batch/modules/splunk_container.py
@@ -18,7 +18,7 @@ from typing import Union
 import threading
 import wrapt_timeout_decorator
 import sys
-
+import traceback
 SPLUNKBASE_URL = "https://splunkbase.splunk.com/app/%d/release/%s/download"
 SPLUNK_START_ARGS = "--accept-license"
 
@@ -198,7 +198,7 @@ class SplunkContainer:
                     )
                     successful_copy = True
             except Exception as e:
-                # print("Failed copy of [%s] file to CONTAINER:[%s]...we will try again"%(localFilePath, containerName))
+                #print("Failed copy of [%s] file to [%s] on CONTAINER [%s]: [%s]\n...we will try again"%(local_file_path, container_file_path, self.container_name, str(e)))
                 time.sleep(10)
                 successful_copy = False
         #print("Successfully copied [%s] to [%s] on [%s]"% (local_file_path, container_file_path, self.container_name))
@@ -374,6 +374,7 @@ class SplunkContainer:
         elapsed_rounded = round(timeit.default_timer() - container_start_time)
         time_string = (datetime.timedelta(seconds=elapsed_rounded))
         print("Container [%s] took [%s] to start"%(self.container_name, time_string))
+        self.synchronization_object.start_barrier.wait()
 
 
         # Sleep for a small random time so that containers drift apart and don't synchronize their testing
@@ -387,10 +388,11 @@ class SplunkContainer:
 
 
             # Sleep for a small random time so that containers drift apart and don't synchronize their testing
-            time.sleep(random.randint(1, 30))
+            #time.sleep(random.randint(1, 30))
             
              
             # There is a detection to test
+            
             print("Container [%s]--->[%s]" %
                   (self.container_name, detection_to_test))
             try:
@@ -408,12 +410,16 @@ class SplunkContainer:
 
                 # Remove the data from the test that we just ran.  We MUST do this when running on CI because otherwise, we will download
                 # a massive amount of data over the course of a long path and will run out of space on the relatively small CI runner drive
-                shutil.rmtree(result["attack_data_directory"])
+                shutil.rmtree(result["attack_data_directory"],ignore_errors=True)
             except Exception as e:
                 print(
                     "Warning - uncaught error in detection test for [%s] - this should not happen: [%s]"
                     % (detection_to_test, str(e))
                 )
+                
+                traceback.print_exc()
+                import pdb
+                pdb.set_trace()
                 # Fill in all the "Empty" fields with default values. Otherwise, we will not be able to 
                 # process the result correctly.  
                 self.synchronization_object.addError(
diff --git a/bin/automated_detection_testing/ci/detection_testing_batch/modules/splunk_sdk.py b/bin/automated_detection_testing/ci/detection_testing_batch/modules/splunk_sdk.py
index 508da4efe3..d21f093439 100644
--- a/bin/automated_detection_testing/ci/detection_testing_batch/modules/splunk_sdk.py
+++ b/bin/automated_detection_testing/ci/detection_testing_batch/modules/splunk_sdk.py
@@ -5,10 +5,12 @@ import splunklib.results as results
 import splunklib.client as client
 import splunklib.results as results
 import requests
-
+import time
+import timeit
+import datetime
 from typing import Union
 
-def enable_delete_for_admin(splunk_host, splunk_port, splunk_password):
+def enable_delete_for_admin(splunk_host:str, splunk_port:int, splunk_password:str)->bool:
     try:
         service = client.connect(
             host=splunk_host,
@@ -20,6 +22,29 @@ def enable_delete_for_admin(splunk_host, splunk_port, splunk_password):
         raise(Exception("Unable to connect to Splunk instance: " + str(e)))
         
 
+    #write the following contents to /opt/splunk/etc/system/local/authorize.conf
+    "[role_admin]"\
+    "delete_by_keyword = enabled"\
+    "grantableRoles = admin"\
+    "importRoles = can_delete;user;power_user"\
+    "srchIndexesAllowed = *;_*;main"\
+    "srchIndexesDefault = main"\
+    "srchMaxTime = 8640000"
+
+    #Run the following search, equivalent to running ./splunk reload auth, to get the settings to take effect
+    
+    update_changed_auth_search = "| rest splunk_server=* /services/authentication/providers/services/_reload"
+
+
+    try:
+        job = service.jobs.create(update_changed_auth_search)
+    except Exception as e:
+        error_message = "Unable to enable delete: %s"%(str(e))
+        return False
+    
+    input("Waiting for you to check that delete has been enabled with: %s"%(update_changed_auth_search))
+    return True
+    '''
     # search and replace \\ with \\\
     # search = search.replace('\\','\\\\')
     role = service.roles['admin']
@@ -28,10 +53,64 @@ def enable_delete_for_admin(splunk_host, splunk_port, splunk_password):
     except Exception as e:
         print("Error - failed trying to grant 'can_delete' privs to admin: [%s]"%(str(e)))
         return False
+    '''
     return True
 
 
 
+
+def wait_for_indexing_to_complete(splunk_host, splunk_port, splunk_password, sourcetype:str, index:str, check_interval_seconds:int=5):
+    
+    startTime = timeit.default_timer()
+    previous_count = -1
+    time.sleep(check_interval_seconds)
+    while True:
+        #print("waiting for search...")
+        try:
+            service = client.connect(
+                host=splunk_host,
+                port=splunk_port,
+                username='admin',
+                password=splunk_password
+            )
+        except Exception as e:
+            raise(Exception("Unable to connect to Splunk instance: " + str(e)))
+
+        search = '''search index="%s" sourcetype="%s" | stats count'''%(index,sourcetype)
+        kwargs = {"exec_mode":"blocking"}
+        try:
+            search_result = service.jobs.create(search, **kwargs)
+        except Exception as e:
+            print("Error while waiting for indexing of data to complete: %s"%(str(e)))
+            #return False
+        
+        #This returns the count in string form, not as an int. For example:
+        #OrderedDict([('count', '59630')])
+        try:
+            for result in results.ResultsReader(search_result.results()):
+                count = int(result['count'])
+                #print("count is %d, previous count is %d"%(count,previous_count))
+                if previous_count == -1:
+                    if count == 0:
+                        pass
+                    else:
+                        previous_count = count
+                else:
+                    if count == previous_count:
+                        #After waiting for the check interval, we return the same number of results.  The indexing must be complete 
+                        stopTime = timeit.default_timer()
+                        #print("Indexing completed after: %s "%(datetime.timedelta(seconds=stopTime-startTime)))
+                        return True
+                    else:
+                        previous_count = count
+    
+        except Exception as e:
+            print("Error trying to get the count while waiting for indexing to complete: %s"%(str(e)))
+            #return False
+        time.sleep(check_interval_seconds)
+
+
+
 def test_baseline_search(splunk_host, splunk_port, splunk_password, search, pass_condition, baseline_name, baseline_file, earliest_time, latest_time)->dict:
     try:
         service = client.connect(
@@ -127,7 +206,7 @@ def test_detection_search(splunk_host:str, splunk_port:int, splunk_password:str,
 
    
 
-    print("SEARCH: %s"%(splunk_search))
+    #print("SEARCH: %s"%(splunk_search))
 
 
     try:
@@ -192,6 +271,8 @@ def delete_attack_data(splunk_host:str, splunk_password:str, splunk_port:int, wa
             
             job = service.jobs.oneshot(splunk_search, **kwargs)
             reader = results.ResultsReader(job)
+            data_exists = False
+            '''
             error_in_results = False
             for result in reader:
                 if hasattr(result,"message") and hasattr(result,"type") and ("You have insufficient privileges to delete events" in result.message or result.type == "FATAL"):
@@ -207,15 +288,17 @@ def delete_attack_data(splunk_host:str, splunk_password:str, splunk_port:int, wa
                 else:
                     #This is not one of the error messages, do nothing
                     pass
-
+            '''
             #No need to issue Delete command again, we will now break out of the loop
-            if error_in_results is False:
-                data_exists = False
+            #if error_in_results is False:
+            #    data_exists = False
 
             #Otherwise, we will loop again
 
         except Exception as e:
-            raise(Exception("Unable to delete data from a run: " + str(e)))
+            print("Trouble deleting data from a run.... we will try again")
+            time.sleep(5)
+            #raise(Exception("Unable to delete data from a run: " + str(e)))
         
     
     return True
diff --git a/bin/automated_detection_testing/ci/detection_testing_batch/modules/test_driver.py b/bin/automated_detection_testing/ci/detection_testing_batch/modules/test_driver.py
index 67c367853b..40909ccba7 100644
--- a/bin/automated_detection_testing/ci/detection_testing_batch/modules/test_driver.py
+++ b/bin/automated_detection_testing/ci/detection_testing_batch/modules/test_driver.py
@@ -106,7 +106,7 @@ class TestDriver:
             return None
         
     def addSuccess(self, result:dict)->None:
-        print("Test PASSED for detection: [%s --> %s"%(result['detection_name'], result['detection_file']))
+        print("Test PASSED: [%s --> %s]"%(result['detection_name'], result['detection_file']))
         self.lock.acquire()
         try:
             self.successes.append(result)
@@ -115,26 +115,26 @@ class TestDriver:
         
 
     def addFailure(self, result:dict)->None:
-        print("Test FAILED for detection: [%s --> %s"%(result['detection_name'], result['detection_file']))
+        print("Test FAILED: [%s --> %s"%(result['detection_name'], result['detection_file']))
         self.lock.acquire()
         try:
             self.failures.append(result)
         finally:
             self.lock.release()
 
-    def addError(self, detection:dict)->None:
+    def addError(self, result:dict)->None:
         #Make sure that even errors have all of the required fields.
-        for required_field in ['search_string', 'diskUsage','runDuration', 'detection_name', 'scanCount', 'detection_error']:
-            if required_field not in detection:
-                detection[required_field] = ""
-        if  'error' not in detection:
-            detection['error'] = True
-        if  'success' not in detection:
-            detection['success'] = False
-
+        for required_field in ['search_string', 'diskUsage','runDuration', 'detection_name', 'scanCount', 'detection_error', 'detection_file']:
+            if required_field not in result:
+                result[required_field] = ""
+        if  'error' not in result:
+            result['error'] = True
+        if  'success' not in result:
+            result['success'] = False
+        print("Test ERROR: [%s --> %s"%(result['detection_name'], result['detection_file']))
         self.lock.acquire()
         try:
-            self.errors.append(detection)
+            self.errors.append(result)
         finally:
             self.lock.release()
     
@@ -259,13 +259,13 @@ class TestDriver:
         disk_usage_info = psutil.disk_usage('/')
 
         cpu_info_string =        "Total CPU Usage   : %d%% (%d CPUs)"%(100 - cpu_info.idle, psutil.cpu_count(logical=False))
-        memory_info_string =     "Total Memory Usage: %0.1fGB USED / %0.1fGB TOTAL"%(memory_info.used / bytes_per_GB, memory_info.total / bytes_per_GB)
+        memory_info_string =     "Total Memory Usage: %0.1fGB USED / %0.1fGB TOTAL"%((memory_info.total - memory_info.available) / bytes_per_GB, memory_info.total / bytes_per_GB)
         disk_usage_info_string = "Total Disk Usage  : %0.1fGB USED / %0.1fGB TOTAL"%(disk_usage_info.free / bytes_per_GB, disk_usage_info.total / bytes_per_GB)
-
+        
         return "System Information:\n\t%s\n\t%s\n\t%s"%(cpu_info_string, memory_info_string, disk_usage_info_string)
 
 
-    def summarize(self)->bool:
+    def summarize(self,testing_currently_active:bool=False)->bool:
         
         if self.checkContainerFailure() == True:
             print("Error running containers... shutting down", file=sys.stderr)
@@ -285,7 +285,7 @@ class TestDriver:
             
             
 
-            if self.testing_queue.qsize() == self.total_number_of_tests:
+            if not testing_currently_active:
                 #Testing has not started yet. We are setting up containers
                 print("***********PROGRESS UPDATE***********\n"\
                       "\tWaiting for container setup: %s\n\t%s\n"%(datetime.timedelta(seconds=current_time - self.start_time),system_stats))
@@ -295,47 +295,45 @@ class TestDriver:
                     #This is the first status update since container setup has completed.  Get the current time.
                     #This makes our remaining time estimates better since that estimate should not involve
                     #the container setup time  
+                    print("SETTING THE CONTAINER READY TIME!")
+                    
                     self.container_ready_time = current_time
 
                 numberOfCompletedTests = len(self.successes) + len(self.failures) + len(self.errors)
                 remaining_tests = self.testing_queue.qsize()         
                 testsCurrentlyRunning = self.total_number_of_tests - remaining_tests - numberOfCompletedTests
-                total_execution_time_seconds = current_time - self.start_time
+                total_execution_time_seconds = round(current_time - self.start_time)
 
                 test_execution_time_seconds = current_time - self.container_ready_time
                 
                 
                 if numberOfCompletedTests == 0 or test_execution_time_seconds == 0:
                     estimated_seconds_to_finish_all_tests = "UNKNOWN"
-                    estimated_completion_time_seconds = "UNKNOWN"
+                    estimated_completion_time_string = "UNKNOWN"
+                    average_time_per_test_string = "UNKNOWN"
                 else:
                     average_time_per_test = test_execution_time_seconds / numberOfCompletedTests
+                    average_time_per_test_string = datetime.timedelta(seconds=round(test_execution_time_seconds/numberOfCompletedTests))
                     #divide testsCurrentlyRunning by 2.0 because, on average, each running test will be 50% completed
-                    estimated_seconds_to_finish_all_tests = average_time_per_test * (remaining_tests + testsCurrentlyRunning/2.0)
-                    estimated_completion_time_seconds = datetime.timedelta(seconds=estimated_seconds_to_finish_all_tests)
+                    estimated_seconds_to_finish_all_tests = round(average_time_per_test * (remaining_tests + testsCurrentlyRunning/2.0))
+                    estimated_completion_time_string = datetime.timedelta(seconds=estimated_seconds_to_finish_all_tests)
                     
                 
 
             
-                print("***********PROGRESS UPDATE***********\n"\
-                "\tElapsed Time               : %s\n"\
-                "\tEstimated Remaining Time   : %s\n"\
-                "\tTests to run               : %d\n"\
-                "\tTests currently running    : %d\n"\
-                "\tTests completed            : %d\n"\
-                "\t\tSuccess : %d\n"\
-                "\t\tFailure : %d\n"\
-                "\t\tError   : %d\n"\
-                "\t%s\n"%(datetime.timedelta(seconds=total_execution_time_seconds), 
-                                    estimated_completion_time_seconds, 
-                                    remaining_tests, 
-                                    testsCurrentlyRunning,
-                                    numberOfCompletedTests, 
-                                    len(self.successes), 
-                                    len(self.failures), 
-                                    len(self.errors),
-                                    system_stats))
-    
+                print(f"***********PROGRESS UPDATE***********\n"\
+                f"\tElapsed Time               : {datetime.timedelta(seconds=total_execution_time_seconds)}\n"\
+                f"\tTest Execution Time        : {datetime.timedelta(seconds=round(test_execution_time_seconds))}\n"\
+                f"\tEstimated Remaining Time   : {estimated_completion_time_string}\n"\
+                f"\tTests to run               : {remaining_tests}\n"\
+                f"\tAverage Time Per Test      : {average_time_per_test_string}\n",
+                f"\tTests currently running    : {testsCurrentlyRunning}\n"\
+                f"\tTests completed            : {numberOfCompletedTests}\n"\
+                f"\t\tSuccess : {len(self.successes)}\n"\
+                f"\t\tFailure : {len(self.failures)}\n"\
+                f"\t\tError   : {len(self.errors)}\n"\
+                f"\t{system_stats}\n")
+
         except Exception as e:
             print("Error in printing execution summary: [%s]"%(str(e)))
         finally:
diff --git a/bin/automated_detection_testing/ci/detection_testing_batch/modules/testing_service.py b/bin/automated_detection_testing/ci/detection_testing_batch/modules/testing_service.py
index 1329431269..036ef3d2b2 100644
--- a/bin/automated_detection_testing/ci/detection_testing_batch/modules/testing_service.py
+++ b/bin/automated_detection_testing/ci/detection_testing_batch/modules/testing_service.py
@@ -1,7 +1,7 @@
 
 import re
 
-import ansible_runner
+#import ansible_runner
 import yaml
 import uuid
 import sys
@@ -10,17 +10,20 @@ import time
 import requests
 from modules.DataManipulation import DataManipulation
 from modules import splunk_sdk
-
+import timeit
 from typing import Union
 from os.path import relpath
 from tempfile import mkdtemp
-
+import datetime
 
 def test_detection_wrapper(container_name:str, splunk_ip:str, splunk_password:str, splunk_port:int, 
                            test_file:str, attack_data_root_folder, wait_on_failure:bool=False, wait_on_completion:bool=False)->dict:
     
+    one_test_start = timeit.default_timer()
     uuid_var = str(uuid.uuid4())
     result_test = test_detection(splunk_ip, splunk_port, container_name, splunk_password, test_file, uuid_var, attack_data_root_folder)
+    one_test_stop = timeit.default_timer()
+    
     if result_test is None:
         #We failed so early in the process that we could not produce any meaningful result
         raise(Exception("Test execution Error"))    
@@ -29,11 +32,15 @@ def test_detection_wrapper(container_name:str, splunk_ip:str, splunk_password:st
     # delete test data
     search_string = result_test['detection_result']['search_string']
     
+    #get pretty time info
+    elapsed_search_time_string = str(datetime.timedelta(seconds=round(one_test_stop - one_test_start)))
+
     #search failed if there was an error or the detection failed to produce the expected result
+    #print("Elapsed search time: %s"%(elapsed_search_time_string))
     if wait_on_failure and (result_test['detection_result']['error'] or not result_test['detection_result']['success']):
-        wait_on_delete = {'message':"\n\n\n****SEARCH FAILURE: Allowing time to debug search/data****"}
+        wait_on_delete = {'message':"\n\n\n****SEARCH FAILURE : Allowing time to debug search/data****"}
     elif wait_on_completion:
-        wait_on_delete = {'message':"\n\n\n****SEARCH SUCCESS: Allowing time to examine search/data****"}
+        wait_on_delete = {'message':"\n\n\n****SEARCH SUCCESS : Allowing time to examine search/data****"}
     else:
         wait_on_delete = None
     
@@ -43,6 +50,20 @@ def test_detection_wrapper(container_name:str, splunk_ip:str, splunk_password:st
     return result_test    
 
 
+import splunklib.client as client
+def get_service(splunk_ip:str, splunk_port:int, splunk_password:str):
+
+    try:
+        service = client.connect(
+            host=splunk_ip,
+            port=splunk_port,
+            username='admin',
+            password=splunk_password
+        )
+    except Exception as e:
+        raise(Exception("Unable to connect to Splunk instance: " + str(e)))
+    return service
+
 def test_detection(splunk_ip:str, splunk_port:int, container_name:str, splunk_password:str, test_file:str, uuid_var, attack_data_root_folder)->Union[dict,None]:
     
     test_file_obj = load_file(os.path.join("security_content/", test_file))
@@ -64,6 +85,9 @@ def test_detection(splunk_ip:str, splunk_port:int, container_name:str, splunk_pa
     folder_name = relpath(abs_folder_path, os.getcwd())
 
 
+    
+
+
     for attack_data in test_file_obj['tests'][0]['attack_data']:
         url = attack_data['data']
         r = requests.get(url, allow_redirects=True)
@@ -78,10 +102,32 @@ def test_detection(splunk_ip:str, splunk_port:int, container_name:str, splunk_pa
             if attack_data['update_timestamp'] == True:
                 data_manipulation = DataManipulation()
                 data_manipulation.manipulate_timestamp(target_file, attack_data['sourcetype'], attack_data['source'])
-        replay_attack_dataset(container_name, splunk_password, folder_name, "main", attack_data['sourcetype'], attack_data['source'], attack_data['file_name'])
-    
+        #replay_attack_dataset(container_name, splunk_password, folder_name, "test0", attack_data['sourcetype'], attack_data['source'], attack_data['file_name'])
+        import http.client
+        try:
+            service = get_service(splunk_ip, splunk_port, splunk_password)
+            test_index = service.indexes["main"]
+            
+            with open(target_file, 'rb') as target:
+                test_index.submit(target.read(), sourcetype=attack_data['sourcetype'], source=attack_data['source'])
+        
+        except http.client.HTTPException as e:
+            print("caught the offending exception!")
+            sys.exit(1)
+        except Exception as e:
+            print("did not catch the offending exception")
+            sys.exit(1)
+
+
+        if not splunk_sdk.wait_for_indexing_to_complete(splunk_ip, splunk_port, splunk_password, attack_data['sourcetype'], "main"):
+            raise Exception("There was an error waiting for indexing to complete.")
+        
     #Allow some time for the data to be ingested and processed
-    time.sleep(30)
+    #print("begin sleep 30")
+    #time.sleep(60)
+
+    
+    #print("end sleep 30")
     
     result_test = {}
     test = test_file_obj['tests'][0]
@@ -104,7 +150,7 @@ def test_detection(splunk_ip:str, splunk_port:int, container_name:str, splunk_pa
 
     detection_file_name = test['file']
     detection = load_file(os.path.join(os.path.dirname(__file__), '../security_content/detections', detection_file_name))
-    print("Making test_detection_search request to: [%s:%d]"%(splunk_ip, splunk_port))
+    #print("Making test_detection_search request to: [%s:%d]"%(splunk_ip, splunk_port))
     
     result_detection = splunk_sdk.test_detection_search(splunk_ip, splunk_port, splunk_password, detection['search'], test['pass_condition'], detection['name'], test['file'], test['earliest_time'], test['latest_time'])
     if result_detection['error']:
@@ -136,40 +182,40 @@ def load_file(file_path):
     return file
 
 
-def update_ESCU_app(container_name, splunk_password):
-    print("Update ESCU App. This can take some time")
+# def update_ESCU_app(container_name, splunk_password):
+#     print("Update ESCU App. This can take some time")
 
-    ansible_vars = {}
-    ansible_vars['ansible_user'] = 'ansible_user'
-    ansible_vars['splunk_password'] = splunk_password
-    ansible_vars['security_content_path'] = 'security_content'
+#     ansible_vars = {}
+#     ansible_vars['ansible_user'] = 'ansible_user'
+#     ansible_vars['splunk_password'] = splunk_password
+#     ansible_vars['security_content_path'] = 'security_content'
     
-    cmdline = "--connection docker -i %s, -u %s" % (container_name, ansible_vars['ansible_user'])
+#     cmdline = "--connection docker -i %s, -u %s" % (container_name, ansible_vars['ansible_user'])
 
-    runner = ansible_runner.run(private_data_dir=os.path.join(os.path.dirname(__file__), '../'),
-                                cmdline=cmdline,
-                                roles_path=os.path.join(os.path.dirname(__file__), '../ansible/roles'),
-                                playbook=os.path.join(os.path.dirname(__file__), '../ansible/update_escu.yml'),
-                                extravars=ansible_vars)
-    print("Successfully updated the ESCU App!")
+#     runner = ansible_runner.run(private_data_dir=os.path.join(os.path.dirname(__file__), '../'),
+#                                 cmdline=cmdline,
+#                                 roles_path=os.path.join(os.path.dirname(__file__), '../ansible/roles'),
+#                                 playbook=os.path.join(os.path.dirname(__file__), '../ansible/update_escu.yml'),
+#                                 extravars=ansible_vars)
+#     print("Successfully updated the ESCU App!")
 
 
-def replay_attack_dataset(container_name, splunk_password, folder_name, index, sourcetype, source, out):
-    ansible_vars = {}
-    ansible_vars['folder_name'] = folder_name
-    ansible_vars['ansible_user'] = 'ansible'
+# def replay_attack_dataset(container_name, splunk_password, folder_name, index, sourcetype, source, out):
+#     ansible_vars = {}
+#     ansible_vars['folder_name'] = folder_name
+#     ansible_vars['ansible_user'] = 'ansible'
 
-    ansible_vars['splunk_password'] = splunk_password
-    ansible_vars['out'] = out
-    ansible_vars['sourcetype'] = sourcetype
-    ansible_vars['source'] = source
-    ansible_vars['index'] = index
+#     ansible_vars['splunk_password'] = splunk_password
+#     ansible_vars['out'] = out
+#     ansible_vars['sourcetype'] = sourcetype
+#     ansible_vars['source'] = source
+#     ansible_vars['index'] = index
 
     
-    cmdline = "--connection docker -i %s, -u %s" % (container_name, ansible_vars['ansible_user'])
+#     cmdline = "--connection docker -i %s, -u %s" % (container_name, ansible_vars['ansible_user'])
 
-    runner = ansible_runner.run(private_data_dir=os.path.join(os.path.dirname(__file__), '../'),
-                                cmdline=cmdline,
-                                roles_path=os.path.join(os.path.dirname(__file__), '../ansible/roles'),
-                                playbook=os.path.join(os.path.dirname(__file__), '../ansible/attack_replay.yml'),
-                                extravars=ansible_vars)
+#     runner = ansible_runner.run(private_data_dir=os.path.join(os.path.dirname(__file__), '../'),
+#                                 cmdline=cmdline,
+#                                 roles_path=os.path.join(os.path.dirname(__file__), '../ansible/roles'),
+#                                 playbook=os.path.join(os.path.dirname(__file__), '../ansible/attack_replay.yml'),
+#                                 extravars=ansible_vars)

From 1dea66d75ccf30f5a21678075df81a84902d0b21 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Wed, 5 Jan 2022 15:26:43 -0800
Subject: [PATCH 157/166] Added authorize.conf.tar to make it possible to
 delete data from completed searches properly.

---
 .../detection_testing_batch/authorize.conf.tar   | Bin 0 -> 2048 bytes
 1 file changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 bin/automated_detection_testing/ci/detection_testing_batch/authorize.conf.tar

diff --git a/bin/automated_detection_testing/ci/detection_testing_batch/authorize.conf.tar b/bin/automated_detection_testing/ci/detection_testing_batch/authorize.conf.tar
new file mode 100644
index 0000000000000000000000000000000000000000..3a624a3b27a8fc5451df076403d5db682ca9c90e
GIT binary patch
literal 2048
zcmeH@J!``-5QaVbS7^>q&PJ9CrbD4ihb*C6DMGQ&iKsq;B$K4SzGs`1lA&9m0c*Lt
z_thPDj~de62mOT)t#^HPlLr8TbCyx8g_t6gb4?OJDQZ^mLU2)5RS67=3hE+baFcDW
zh7K{QK&zo`ciK4}ul!3%-}=A!OqalX98kQI_a3}KsXD9ON8X{qgmQb9pLjm{(6J|m
zPHhc#`7S7z#&kmsk&LM)*Y@B;x@fI(avstQ5kqtEM+|aI^BCIwtLyND@!1&C$Kk6+
qt{SVf`)$8quLhII{&Cjo^rkJ6^eFg9`cCLBYc2#90t<ovOW+%nzFY|a

literal 0
HcmV?d00001


From 81df4e0eec017087142882d1e9b66295cfa01add Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Tue, 11 Jan 2022 11:59:26 -0800
Subject: [PATCH 158/166] A bunch of changes to support parallel testing and
 waiting to ensure data has been properly ingested/indexed before running a
 test.  Still double-checking edge cases for this. Also improved some error
 printouts and status printouts.  When a test completes, it now includes its
 runtime in its pass/fail/error print.

---
 .../modules/DataManipulation.py               | 12 ++-
 .../modules/container_manager.py              |  1 +
 .../modules/splunk_container.py               | 14 +--
 .../modules/splunk_sdk.py                     | 92 ++++++++++++++++---
 .../modules/test_driver.py                    | 25 ++---
 5 files changed, 112 insertions(+), 32 deletions(-)

diff --git a/bin/automated_detection_testing/ci/detection_testing_batch/modules/DataManipulation.py b/bin/automated_detection_testing/ci/detection_testing_batch/modules/DataManipulation.py
index a9b516611c..08606e595f 100644
--- a/bin/automated_detection_testing/ci/detection_testing_batch/modules/DataManipulation.py
+++ b/bin/automated_detection_testing/ci/detection_testing_batch/modules/DataManipulation.py
@@ -40,7 +40,11 @@ class DataManipulation:
         difference = now - latest_event
         f.close()
 
-        for line in fileinput.input(path, inplace=True):
+        #Order to make this threadsafe, we need to use fileinput.FileInput.
+        #We cannot use fileinput.fileinput because 
+        #"The instance will be used as global state for the functions of this module, and is also returned to use during iteration."
+        #from: https://docs.python.org/2/library/fileinput.html#fileinput.input
+        for line in fileinput.FileInput(path, inplace=True):
             d = json.loads(line)
             original_time = datetime.strptime(d["CreationTime"],"%Y-%m-%dT%H:%M:%S")
             new_time = (difference + original_time)
@@ -113,7 +117,11 @@ class DataManipulation:
         difference = now - latest_event
         f.close()
 
-        for line in fileinput.input(path, inplace=True):
+        #Order to make this threadsafe, we need to use fileinput.FileInput.
+        #We cannot use fileinput.fileinput because 
+        #"The instance will be used as global state for the functions of this module, and is also returned to use during iteration."
+        #from: https://docs.python.org/2/library/fileinput.html#fileinput.input
+        for line in fileinput.FileInput(path, inplace=True):
             try:
                 d = json.loads(line)
                 original_time = datetime.strptime(d["eventTime"],"%Y-%m-%dT%H:%M:%S.%fZ")
diff --git a/bin/automated_detection_testing/ci/detection_testing_batch/modules/container_manager.py b/bin/automated_detection_testing/ci/detection_testing_batch/modules/container_manager.py
index 8bd38d9668..6b25b19b7f 100644
--- a/bin/automated_detection_testing/ci/detection_testing_batch/modules/container_manager.py
+++ b/bin/automated_detection_testing/ci/detection_testing_batch/modules/container_manager.py
@@ -106,6 +106,7 @@ class ContainerManager:
         for container in self.containers:
             container.thread.join()
             print(container.get_container_summary())
+        print("Waiting for next summary thread printout to finish...")
         self.summary_thread.join()
         print("All containers completed testing!")
         
diff --git a/bin/automated_detection_testing/ci/detection_testing_batch/modules/splunk_container.py b/bin/automated_detection_testing/ci/detection_testing_batch/modules/splunk_container.py
index 72986d27f1..fea6d69bad 100644
--- a/bin/automated_detection_testing/ci/detection_testing_batch/modules/splunk_container.py
+++ b/bin/automated_detection_testing/ci/detection_testing_batch/modules/splunk_container.py
@@ -238,7 +238,7 @@ class SplunkContainer:
         except Exception as e:
             print("Could not remove Docker Container [%s]" % (
                 self.container_name))
-            raise (Exception("CONTAINER REMOVE ERROR"))
+            raise (Exception(f"CONTAINER REMOVE ERROR: {str(e)}"))
 
     def get_container_summary(self) -> str:
         current_time = timeit.default_timer()
@@ -386,7 +386,7 @@ class SplunkContainer:
                 print("Container [%s] successfully stopped early due to failure" % (self.container_name))
                 return None
 
-
+            current_test_start_time = timeit.default_timer()
             # Sleep for a small random time so that containers drift apart and don't synchronize their testing
             #time.sleep(random.randint(1, 30))
             
@@ -406,7 +406,9 @@ class SplunkContainer:
                     wait_on_failure=self.interactive_failure,
                     wait_on_completion = self.interactive
                 )
-                self.synchronization_object.addResult(result)
+                
+                
+                self.synchronization_object.addResult(result, duration_string =  datetime.timedelta(seconds=round(timeit.default_timer() - current_test_start_time)))
 
                 # Remove the data from the test that we just ran.  We MUST do this when running on CI because otherwise, we will download
                 # a massive amount of data over the course of a long path and will run out of space on the relatively small CI runner drive
@@ -418,13 +420,13 @@ class SplunkContainer:
                 )
                 
                 traceback.print_exc()
-                import pdb
-                pdb.set_trace()
+                #import pdb
+                #pdb.set_trace()
                 # Fill in all the "Empty" fields with default values. Otherwise, we will not be able to 
                 # process the result correctly.  
                 self.synchronization_object.addError(
                     {"detection_file": detection_to_test,
-                        "detection_error": str(e)}
+                        "detection_error": str(e)}, duration_string = datetime.timedelta(seconds=round(timeit.default_timer() - current_test_start_time))
 
 
                 )
diff --git a/bin/automated_detection_testing/ci/detection_testing_batch/modules/splunk_sdk.py b/bin/automated_detection_testing/ci/detection_testing_batch/modules/splunk_sdk.py
index d21f093439..7c9a0a0ea9 100644
--- a/bin/automated_detection_testing/ci/detection_testing_batch/modules/splunk_sdk.py
+++ b/bin/automated_detection_testing/ci/detection_testing_batch/modules/splunk_sdk.py
@@ -1,7 +1,6 @@
 from os import error
 import sys
 from time import sleep
-import splunklib.results as results
 import splunklib.client as client
 import splunklib.results as results
 import requests
@@ -59,11 +58,73 @@ def enable_delete_for_admin(splunk_host:str, splunk_port:int, splunk_password:st
 
 
 
-def wait_for_indexing_to_complete(splunk_host, splunk_port, splunk_password, sourcetype:str, index:str, check_interval_seconds:int=5):
+def get_number_of_indexed_events(splunk_host, splunk_port, splunk_password, index:str, sourcetype:Union[str,None]=None )->int:
+
+    try:
+        service = client.connect(
+            host=splunk_host,
+            port=splunk_port,
+            username='admin',
+            password=splunk_password
+        )
+    except Exception as e:
+        raise(Exception("Unable to connect to Splunk instance: " + str(e)))
+
+    if sourcetype is not None:
+        search = '''search index="%s" sourcetype="%s" | stats count'''%(index,sourcetype)
+    else:
+        search = '''search index="%s" | stats count'''%(index)
+    kwargs = {"exec_mode":"blocking"}
+    try:
+        search_result = service.jobs.create(search, **kwargs)
+  
+        #This returns the count in string form, not as an int. For example:
+        #OrderedDict([('count', '59630')])
+        search_results = list(results.ResultsReader(search_result.results()))
+        if len(search_results) != 1:
+            raise Exception(f"Expected the get_number_of_indexed_events search to only return 1 count, but got {len(search_results)} instead.")
+        
+        count = int(search_results[0]['count'])
+        return count    
+
+    except Exception as e:
+        raise Exception("Error trying to get the count while waiting for indexing to complete: %s"%(str(e)))
+        
+    
+
+
+def wait_for_indexing_to_complete(splunk_host, splunk_port, splunk_password, sourcetype:str, index:str, check_interval_seconds:int=5)->bool:
     
     startTime = timeit.default_timer()
     previous_count = -1
     time.sleep(check_interval_seconds)
+    while True:
+        new_count = get_number_of_indexed_events(splunk_host, splunk_port, splunk_password, index=index, sourcetype=sourcetype)
+        #print(f"Previous Count [{previous_count}] New Count [{new_count}]")
+        if previous_count == -1:
+            previous_count = new_count
+        else:
+            if new_count == previous_count:
+                stopTime = timeit.default_timer()
+                return True
+            else:
+                previous_count = new_count
+        
+        #If new_count is really low, then the server is taking some extra time to index the data.
+        # So sleep for longer to make sure that we give time to complete (or at least process more
+        # events so we don't return from this function prematurely) 
+        if new_count < 2:
+            time.sleep(check_interval_seconds*3)
+        else:
+            time.sleep(check_interval_seconds)
+        
+
+'''
+def wait_for_indexing_to_complete(splunk_host, splunk_port, splunk_password, sourcetype:str, index:str, check_interval_seconds:int=10):
+    
+    startTime = timeit.default_timer()
+    previous_count = -1
+    time.sleep(check_interval_seconds/2)
     while True:
         #print("waiting for search...")
         try:
@@ -76,7 +137,7 @@ def wait_for_indexing_to_complete(splunk_host, splunk_port, splunk_password, sou
         except Exception as e:
             raise(Exception("Unable to connect to Splunk instance: " + str(e)))
 
-        search = '''search index="%s" sourcetype="%s" | stats count'''%(index,sourcetype)
+        search = 'search index="%s" sourcetype="%s" | stats count'%(index,sourcetype)
         kwargs = {"exec_mode":"blocking"}
         try:
             search_result = service.jobs.create(search, **kwargs)
@@ -89,7 +150,7 @@ def wait_for_indexing_to_complete(splunk_host, splunk_port, splunk_password, sou
         try:
             for result in results.ResultsReader(search_result.results()):
                 count = int(result['count'])
-                #print("count is %d, previous count is %d"%(count,previous_count))
+                print("count is %d, previous count is %d"%(count,previous_count))
                 if previous_count == -1:
                     if count == 0:
                         pass
@@ -108,7 +169,7 @@ def wait_for_indexing_to_complete(splunk_host, splunk_port, splunk_password, sou
             print("Error trying to get the count while waiting for indexing to complete: %s"%(str(e)))
             #return False
         time.sleep(check_interval_seconds)
-
+'''
 
 
 def test_baseline_search(splunk_host, splunk_port, splunk_password, search, pass_condition, baseline_name, baseline_file, earliest_time, latest_time)->dict:
@@ -239,7 +300,7 @@ def test_detection_search(splunk_host:str, splunk_port:int, splunk_password:str,
         return test_results
 
 
-def delete_attack_data(splunk_host:str, splunk_password:str, splunk_port:int, wait_on_delete:Union[dict,None], search_string:str, detection_filename:str)->bool:
+def delete_attack_data(splunk_host:str, splunk_password:str, splunk_port:int, wait_on_delete:Union[dict,None], search_string:str, detection_filename:str, index:str="main")->bool:
     
     try:
         service = client.connect(
@@ -261,17 +322,22 @@ def delete_attack_data(splunk_host:str, splunk_password:str, splunk_port:int, wa
         _ = input("****************Press ENTER to Complete Test and DELETE data****************\n\n\n")
     
     data_exists = True
-    already_enabled_delete = False
-    while data_exists:
-        splunk_search = 'search index=main | delete'
 
-        kwargs = {"dispatch.earliest_time": "-1d",
+
+
+    while (get_number_of_indexed_events(splunk_host, splunk_port, splunk_password, index=index) != 0) :
+        splunk_search = f'search index={index} | delete'
+
+        kwargs = {
+                "exec_mode": "blocking",
+                "dispatch.earliest_time": "-1d",
                 "dispatch.latest_time": "now"}
         try:
             
-            job = service.jobs.oneshot(splunk_search, **kwargs)
+            job = service.jobs.create(splunk_search, **kwargs)
             reader = results.ResultsReader(job)
-            data_exists = False
+
+            
             '''
             error_in_results = False
             for result in reader:
@@ -296,7 +362,7 @@ def delete_attack_data(splunk_host:str, splunk_password:str, splunk_port:int, wa
             #Otherwise, we will loop again
 
         except Exception as e:
-            print("Trouble deleting data from a run.... we will try again")
+            print(f"Trouble deleting data from a run.... we will try again: {str(e)}")
             time.sleep(5)
             #raise(Exception("Unable to delete data from a run: " + str(e)))
         
diff --git a/bin/automated_detection_testing/ci/detection_testing_batch/modules/test_driver.py b/bin/automated_detection_testing/ci/detection_testing_batch/modules/test_driver.py
index 40909ccba7..a55dc9edb2 100644
--- a/bin/automated_detection_testing/ci/detection_testing_batch/modules/test_driver.py
+++ b/bin/automated_detection_testing/ci/detection_testing_batch/modules/test_driver.py
@@ -105,8 +105,8 @@ class TestDriver:
         except Exception as e:
             return None
         
-    def addSuccess(self, result:dict)->None:
-        print("Test PASSED: [%s --> %s]"%(result['detection_name'], result['detection_file']))
+    def addSuccess(self, result:dict, duration_string:str)->None:
+        print("Test PASSED: [%s --> %s] in %s"%(result['detection_name'], result['detection_file'], duration_string))
         self.lock.acquire()
         try:
             self.successes.append(result)
@@ -114,15 +114,15 @@ class TestDriver:
             self.lock.release()
         
 
-    def addFailure(self, result:dict)->None:
-        print("Test FAILED: [%s --> %s"%(result['detection_name'], result['detection_file']))
+    def addFailure(self, result:dict, duration_string:str)->None:
+        print("Test FAILED: [%s --> %s] in %s"%(result['detection_name'], result['detection_file'], duration_string))
         self.lock.acquire()
         try:
             self.failures.append(result)
         finally:
             self.lock.release()
 
-    def addError(self, result:dict)->None:
+    def addError(self, result:dict, duration_string:str)->None:
         #Make sure that even errors have all of the required fields.
         for required_field in ['search_string', 'diskUsage','runDuration', 'detection_name', 'scanCount', 'detection_error', 'detection_file']:
             if required_field not in result:
@@ -131,7 +131,7 @@ class TestDriver:
             result['error'] = True
         if  'success' not in result:
             result['success'] = False
-        print("Test ERROR: [%s --> %s"%(result['detection_name'], result['detection_file']))
+        print("Test ERROR: [%s --> %s] in %s"%(result['detection_name'], result['detection_file'], duration_string))
         self.lock.acquire()
         try:
             self.errors.append(result)
@@ -258,9 +258,12 @@ class TestDriver:
         memory_info = psutil.virtual_memory()
         disk_usage_info = psutil.disk_usage('/')
 
+        #macOS is really weird about disk usage.... so to get free space we use TOTAL-FREE = USED instead of just USED
+        corrected_used_space = disk_usage_info.total - disk_usage_info.free
+
         cpu_info_string =        "Total CPU Usage   : %d%% (%d CPUs)"%(100 - cpu_info.idle, psutil.cpu_count(logical=False))
         memory_info_string =     "Total Memory Usage: %0.1fGB USED / %0.1fGB TOTAL"%((memory_info.total - memory_info.available) / bytes_per_GB, memory_info.total / bytes_per_GB)
-        disk_usage_info_string = "Total Disk Usage  : %0.1fGB USED / %0.1fGB TOTAL"%(disk_usage_info.free / bytes_per_GB, disk_usage_info.total / bytes_per_GB)
+        disk_usage_info_string = "Total Disk Usage  : %0.1fGB USED / %0.1fGB TOTAL"%(corrected_used_space / bytes_per_GB, disk_usage_info.total / bytes_per_GB)
         
         return "System Information:\n\t%s\n\t%s\n\t%s"%(cpu_info_string, memory_info_string, disk_usage_info_string)
 
@@ -347,15 +350,15 @@ class TestDriver:
                 
         
         
-    def addResult(self, result:dict)->None:
+    def addResult(self, result:dict, duration_string:str)->None:
         try:
             if result['detection_result']['error'] is True:
-                self.addError(result['detection_result'])
+                self.addError(result['detection_result'], duration_string = duration_string)
             elif result['detection_result']['success'] is False:
                 #This is actually a failure of the detection, not an error. Naming is confusiong
-                self.addFailure(result['detection_result'])
+                self.addFailure(result['detection_result'], duration_string = duration_string)
             elif result['detection_result']['success'] is True:
-                self.addSuccess(result['detection_result'])
+                self.addSuccess(result['detection_result'], duration_string = duration_string)
         except Exception as e:
             #Neither a success or a failure, so add the object to the failures queue
             print('"There was an error adding the result: [%s]'%(str(e)))

From 79d13fccef0f9a0098a043751d1d9609c48fe714 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Tue, 11 Jan 2022 14:18:04 -0800
Subject: [PATCH 159/166] Converted DataManipulation from using FileInput to
 with open, readline, etc.  We needed to do this because FileInput with
 inplace=True remaps the print command, which may be called by several
 parallel threads running DataManipulation or to output status info, will
 cause a crash if multiple threads write to it.

---
 .../modules/DataManipulation.py               | 65 ++++++++++++++++++-
 1 file changed, 63 insertions(+), 2 deletions(-)

diff --git a/bin/automated_detection_testing/ci/detection_testing_batch/modules/DataManipulation.py b/bin/automated_detection_testing/ci/detection_testing_batch/modules/DataManipulation.py
index 08606e595f..53b1543a85 100644
--- a/bin/automated_detection_testing/ci/detection_testing_batch/modules/DataManipulation.py
+++ b/bin/automated_detection_testing/ci/detection_testing_batch/modules/DataManipulation.py
@@ -1,7 +1,7 @@
 import json
 from datetime import datetime
 from datetime import timedelta
-import fileinput
+#import fileinput
 import os
 import re
 import io
@@ -40,10 +40,36 @@ class DataManipulation:
         difference = now - latest_event
         f.close()
 
+        #Mimic the behavior of fileinput but in a threadsafe way
+        #Rename the file, which fileinput does for inplace.
+        #Note that path will now be the new file
+        original_backup_file = f"{path}.bak"    
+        os.rename(path, original_backup_file)
+        
+        with open(original_backup_file, "r") as original_file:
+            with open(path, "w") as new_file:
+                for line in original_file:
+                    d = json.loads(line)
+                    original_time = datetime.strptime(d["CreationTime"],"%Y-%m-%dT%H:%M:%S")
+                    new_time = (difference + original_time)
+
+                    original_time = original_time.strftime("%Y-%m-%dT%H:%M:%S")
+                    new_time = new_time.strftime("%Y-%m-%dT%H:%M:%S")
+                    #There is no end character appended, no need for end=''
+                    new_file.write(line.replace(original_time, new_time))
+
+
+        os.remove(original_backup_file)
+
+
+    
+        '''
         #Order to make this threadsafe, we need to use fileinput.FileInput.
         #We cannot use fileinput.fileinput because 
         #"The instance will be used as global state for the functions of this module, and is also returned to use during iteration."
         #from: https://docs.python.org/2/library/fileinput.html#fileinput.input
+        
+        
         for line in fileinput.FileInput(path, inplace=True):
             d = json.loads(line)
             original_time = datetime.strptime(d["CreationTime"],"%Y-%m-%dT%H:%M:%S")
@@ -52,6 +78,7 @@ class DataManipulation:
             original_time = original_time.strftime("%Y-%m-%dT%H:%M:%S")
             new_time = new_time.strftime("%Y-%m-%dT%H:%M:%S")
             print (line.replace(original_time, new_time),end ='')
+        '''
 
 
     def manipulate_timestamp_windows_event_log_raw(self, file_path):
@@ -117,6 +144,39 @@ class DataManipulation:
         difference = now - latest_event
         f.close()
 
+
+
+        #Mimic the behavior of fileinput but in a threadsafe way
+        #Rename the file, which fileinput does for inplace.
+        #Note that path will now be the new file
+        original_backup_file = f"{path}.bak"    
+        os.rename(path, original_backup_file)
+        
+        with open(original_backup_file, "r") as original_file:
+            with open(path, "w") as new_file:
+                for line in original_file:
+                    try:
+                        d = json.loads(line)
+                        original_time = datetime.strptime(d["eventTime"],"%Y-%m-%dT%H:%M:%S.%fZ")
+                        new_time = (difference + original_time)
+
+                        original_time = original_time.strftime("%Y-%m-%dT%H:%M:%S.%fZ")
+                        new_time = new_time.strftime("%Y-%m-%dT%H:%M:%S.%fZ")
+                        new_file.write(line.replace(original_time, new_time))
+                    except ValueError:
+                        d = json.loads(line)
+                        original_time = datetime.strptime(d["eventTime"],"%Y-%m-%dT%H:%M:%SZ")
+                        new_time = (difference + original_time)
+
+                        original_time = original_time.strftime("%Y-%m-%dT%H:%M:%SZ")
+                        new_time = new_time.strftime("%Y-%m-%dT%H:%M:%SZ")
+                        new_file.write(line.replace(original_time, new_time))
+
+
+        os.remove(original_backup_file)
+
+
+        '''
         #Order to make this threadsafe, we need to use fileinput.FileInput.
         #We cannot use fileinput.fileinput because 
         #"The instance will be used as global state for the functions of this module, and is also returned to use during iteration."
@@ -137,4 +197,5 @@ class DataManipulation:
 
                 original_time = original_time.strftime("%Y-%m-%dT%H:%M:%SZ")
                 new_time = new_time.strftime("%Y-%m-%dT%H:%M:%SZ")
-                print (line.replace(original_time, new_time),end ='')
\ No newline at end of file
+                print (line.replace(original_time, new_time),end ='')
+        '''
\ No newline at end of file

From 6a6bd04819d4415e39cdd9b496415888e64b65c7 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Tue, 11 Jan 2022 15:46:38 -0800
Subject: [PATCH 160/166] Removing some dead code from DataManipulation.py

---
 .../test_results/combined.csv                 | 73 +++++++++++++++++++
 1 file changed, 73 insertions(+)
 create mode 100644 bin/automated_detection_testing/ci/detection_testing_batch/test_results/combined.csv

diff --git a/bin/automated_detection_testing/ci/detection_testing_batch/test_results/combined.csv b/bin/automated_detection_testing/ci/detection_testing_batch/test_results/combined.csv
new file mode 100644
index 0000000000..fa91805f76
--- /dev/null
+++ b/bin/automated_detection_testing/ci/detection_testing_batch/test_results/combined.csv
@@ -0,0 +1,73 @@
+"SPLUNK_VERSION","splunk/splunk:latest"
+"branch","UpdateRequiredFields"
+"commit_hash","2670d58f5018345c37bae30e80535a864614133c"
+"TEST_START_TIME","2022-01-11 12:32:39.925019"
+"TEST_FINISH_TIME","2022-01-11 12:59:54.934815"
+"TEST_DURATION","0:27:15"
+"SPLUNK_ADD_ON_FOR_AMAZON_KINESIS_FIREHOSE","{'app_number': 12, 'app_version': None, 'http_path': 'https://attack-range-appbinaries.s3-us-west-2.amazonaws.com/splunk-add-on-for-amazon-kinesis-firehose_131r7d1d093.tgz'}"
+"SPLUNK_ADD_ON_FOR_MICROSOFT_OFFICE_365","{'app_number': 13, 'app_version': None, 'http_path': 'https://attack-range-appbinaries.s3-us-west-2.amazonaws.com/splunk-add-on-for-microsoft-office-365_202.tgz'}"
+"SPLUNK_ADD_ON_FOR_STREAM_FORWARDERS","{'app_number': 9, 'app_version': None, 'http_path': 'https://attack-range-appbinaries.s3-us-west-2.amazonaws.com/splunk-add-on-for-stream-forwarders_730.tgz'}"
+"SPLUNK_ADD_ON_FOR_STREAM_WIRE_DATA","{'app_number': 8, 'app_version': None, 'http_path': 'https://attack-range-appbinaries.s3-us-west-2.amazonaws.com/splunk-add-on-for-stream-wire-data_730.tgz'}"
+"SPLUNK_ADD_ON_FOR_SYSMON_OLD","{'app_number': 1, 'app_version': None, 'http_path': 'https://attack-range-appbinaries.s3-us-west-2.amazonaws.com/splunk-add-on-for-microsoft-sysmon_1062.tgz'}"
+"SPLUNK_ADD_ON_FOR_ZEEK_AKA_BRO","{'app_number': 11, 'app_version': None, 'http_path': 'https://attack-range-appbinaries.s3-us-west-2.amazonaws.com/splunk-add-on-for-zeek-aka-bro_400.tgz'}"
+"SPLUNK_ASX_APP","{'app_number': 17, 'app_version': None, 'http_path': 'https://attack-range-appbinaries.s3-us-west-2.amazonaws.com/Splunk_ASX-latest.tar.gz'}"
+"SPLUNK_AWS_TA","{'app_number': 4, 'app_version': None, 'http_path': 'https://attack-range-appbinaries.s3-us-west-2.amazonaws.com/splunk-add-on-for-amazon-web-services_510.tgz'}"
+"SPLUNK_CIM_APP","{'app_number': 3, 'app_version': None, 'http_path': 'https://attack-range-appbinaries.s3-us-west-2.amazonaws.com/splunk-common-information-model-cim_4200.tgz'}"
+"SPLUNK_ES_CONTENT_UPDATE","{'app_number': 3449, 'app_version': None, 'local_path': '/Users/emcginnis/Documents/GitHub/security_content/bin/automated_detection_testing/ci/detection_testing_batch/apps/DA-ESS-ContentUpdate-latest.tar.gz'}"
+"SPLUNK_LINUX_TA","{'app_number': 14, 'app_version': None, 'http_path': 'https://attack-range-appbinaries.s3-us-west-2.amazonaws.com/splunk-add-on-for-unix-and-linux_820.tgz'}"
+"SPLUNK_MLTK_APP","{'app_number': 6, 'app_version': None, 'http_path': 'https://attack-range-appbinaries.s3-us-west-2.amazonaws.com/splunk-machine-learning-toolkit_521.tgz'}"
+"SPLUNK_NGINX_TA","{'app_number': 15, 'app_version': None, 'http_path': 'https://attack-range-appbinaries.s3-us-west-2.amazonaws.com/splunk-add-on-for-nginx_310.tgz'}"
+"SPLUNK_PYTHON_APP","{'app_number': 5, 'app_version': None, 'http_path': 'https://attack-range-appbinaries.s3-us-west-2.amazonaws.com/python-for-scientific-computing-for-linux-64-bit_202.tgz'}"
+"SPLUNK_SECURITY_ESSENTIALS","{'app_number': 10, 'app_version': None, 'http_path': 'https://attack-range-appbinaries.s3-us-west-2.amazonaws.com/splunk-security-essentials_333.tgz'}"
+"SPLUNK_STREAM_APP","{'app_number': 7, 'app_version': None, 'http_path': 'https://attack-range-appbinaries.s3-us-west-2.amazonaws.com/splunk-app-for-stream_730.tgz'}"
+"SPLUNK_SYSMON_LINUX_TA_PATCHED","{'app_number': 2, 'app_version': None, 'http_path': 'https://attack-range-appbinaries.s3-us-west-2.amazonaws.com/add-on-for-linux-sysmon_103_PATCHED.tgz'}"
+"SPLUNK_TA_FOR_ZEEK","{'app_number': 16, 'app_version': None, 'http_path': 'https://attack-range-appbinaries.s3-us-west-2.amazonaws.com/ta-for-zeek_105.tgz'}"
+"SPLUNK_WINDOWS_TA","{'app_number': 0, 'app_version': None, 'http_path': 'https://attack-range-appbinaries.s3-us-west-2.amazonaws.com/splunk-add-on-for-microsoft-windows_812.tgz'}"
+"SPLUNK_ADD_ON_FOR_SYSMON","{'app_number': 5709, 'app_version': '1.0.1'}"
+"",""
+detection_name,detection_file,runDuration,diskUsage,search_string,error,success,scanCount,detection_error
+CMD Carry Out String Command Parameter,endpoint/cmd_carry_out_string_command_parameter.yml,1.6640000000000001,393216,"| tstats `security_content_summariesonly` min(_time) as firstTime max(_time) as lastTime from datamodel=Endpoint.Processes where `process_cmd` AND Processes.process=""* /c *"" by Processes.dest Processes.user Processes.parent_process Processes.process_name Processes.original_file_name Processes.process Processes.process_id Processes.parent_process_id | `drop_dm_object_name(Processes)` | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)` | `cmd_carry_out_string_command_parameter_filter` | stats count | where count > 0",False,True,2109,
+Suspicious writes to windows Recycle Bin,endpoint/suspicious_writes_to_windows_recycle_bin.yml,1.8760000000000001,589824,"| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) as lastTime values(Filesystem.file_path) as file_path values(Filesystem.file_name) as file_name FROM datamodel=Endpoint.Filesystem where Filesystem.file_path = ""*$Recycle.Bin*"" by Filesystem.process_id Filesystem.dest | `drop_dm_object_name(""Filesystem"")`| search [| tstats `security_content_summariesonly` values(Processes.user) as user values(Processes.process_name) as process_name values(Processes.parent_process_name) as parent_process_name FROM datamodel=Endpoint.Processes where Processes.process_name != ""explorer.exe"" by Processes.process_id Processes.dest| `drop_dm_object_name(""Processes"")` | table process_id dest] | `suspicious_writes_to_windows_recycle_bin_filter` | stats count | where count > 0",False,True,848,
+Process Kill Base On File Path,endpoint/process_kill_base_on_file_path.yml,118.84100000000001,364544,"| tstats `security_content_summariesonly` values(Processes.process) as process values(Processes.process_id) as process_id count min(_time) as firstTime max(_time) as lastTime  from datamodel=Endpoint.Processes where `process_wmic` AND Processes.process=""*process*"" AND Processes.process=""*executablepath*"" AND Processes.process=""*delete*"" by  Processes.parent_process_name Processes.process_name Processes.original_file_name Processes.dest Processes.user | `drop_dm_object_name(Processes)` | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)` | `process_kill_base_on_file_path_filter` | stats count | where count > 0",False,True,271715,
+High Process Termination Frequency,endpoint/high_process_termination_frequency.yml,17.758,155648,search `sysmon` EventCode=5 |bin _time span=3s |stats values(Image) as proc_terminated min(_time) as firstTime max(_time) as lastTime  count by Computer EventCode ProcessID | where count >= 15 | `security_content_ctime(firstTime)`| `security_content_ctime(lastTime)` | `high_process_termination_frequency_filter` | stats count | where count > 0,False,True,46964,
+Detect Rundll32 Application Control Bypass - advpack and ieadvpack,endpoint/detect_rundll32_application_control_bypass___advpack.yml,7.022,360448,| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) as lastTime from datamodel=Endpoint.Processes where `process_rundll32` Processes.process=*advpack* by Processes.dest Processes.user Processes.parent_process_name Processes.original_file_name Processes.process_name Processes.process Processes.process_id Processes.parent_process_id | `drop_dm_object_name(Processes)` | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)` | `detect_rundll32_application_control_bypass___advpack_filter` | stats count | where count > 0,False,True,13364,
+Detect hosts connecting to dynamic domain providers,network/detect_hosts_connecting_to_dynamic_domain_providers.yml,1.052,274432,"| tstats `security_content_summariesonly` count values(DNS.answer) as answer min(_time) as firstTime from datamodel=Network_Resolution.DNS by DNS.query host | `drop_dm_object_name(""DNS"")` | `security_content_ctime(firstTime)` | `dynamic_dns_providers` | `detect_hosts_connecting_to_dynamic_domain_providers_filter` | stats count | where count > 0",False,True,644,
+Process execution via wmi,endpoint/process_execution_via_wmi.yml,3.237,339968,| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) as lastTime from datamodel=Endpoint.Processes where Processes.parent_process_name=WmiPrvSE.exe by Processes.dest Processes.user Processes.parent_process Processes.process_name Processes.process Processes.process_id Processes.parent_process_id | `drop_dm_object_name(Processes)` | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)` | `process_execution_via_wmi_filter`  | stats count | where count = 1,False,False,6572,
+Registry Keys Used For Privilege Escalation,endpoint/registry_keys_used_for_privilege_escalation.yml,2.915,311296,"| tstats `security_content_summariesonly` count  min(_time) as firstTime max(_time) as lastTime FROM datamodel=Endpoint.Registry where (Registry.registry_path=""*Microsoft\\Windows NT\\CurrentVersion\\Image File Execution Options*"") AND (Registry.registry_value_name=GlobalFlag OR Registry.registry_value_name=Debugger) by Registry.dest  Registry.user Registry.registry_path Registry.registry_value_name | `security_content_ctime(lastTime)`  | `security_content_ctime(firstTime)` | `drop_dm_object_name(Registry)` | `registry_keys_used_for_privilege_escalation_filter` | stats count | where count > 0",False,False,6391,
+Disable Defender BlockAtFirstSeen Feature,endpoint/disable_defender_blockatfirstseen_feature.yml,2.364,311296,"| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) as lastTime from datamodel=Endpoint.Registry where Registry.registry_path = ""*\\Microsoft\\Windows Defender\\SpyNet*"" Registry.registry_value_name = DisableBlockAtFirstSeen Registry.registry_value_data = 0x00000001 by Registry.dest Registry.user Registry.registry_path Registry.registry_value_name Registry.registry_value_data | `drop_dm_object_name(Registry)` | `security_content_ctime(lastTime)` | `security_content_ctime(firstTime)` | `disable_defender_blockatfirstseen_feature_filter` | stats count | where count > 0",False,False,4369,
+Allow Operation with Consent Admin,endpoint/allow_operation_with_consent_admin.yml,7.696,311296,"| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) as lastTime FROM datamodel=Endpoint.Registry where Registry.registry_path= ""*\\Microsoft\\Windows\\CurrentVersion\\Policies\\System*"" Registry.registry_value_name = ConsentPromptBehaviorAdmin Registry.registry_value_data = ""0x00000000"" by Registry.registry_path Registry.registry_key_name Registry.registry_value_data Registry.dest | `drop_dm_object_name(Registry)` | `security_content_ctime(firstTime)` |`security_content_ctime(lastTime)` | `allow_operation_with_consent_admin_filter` | stats count | where count > 0",False,False,17459,
+Cloud Provisioning Activity From Previously Unseen IP Address,cloud/cloud_provisioning_from_previously_unseen_ip_address.yml,2.399,512000,"| tstats earliest(_time) as firstTime, latest(_time) as lastTime, values(All_Changes.object_id) as object_id from datamodel=Change.All_Changes where (All_Changes.action=started OR All_Changes.action=created) All_Changes.status=success by All_Changes.src, All_Changes.user, All_Changes.command | `drop_dm_object_name(""All_Changes"")` | lookup previously_seen_cloud_provisioning_activity_sources src as src OUTPUT firstTimeSeen, enough_data | eventstats max(enough_data) as enough_data | where enough_data=1 | eval firstTimeSeenSrc=min(firstTimeSeen) | where isnull(firstTimeSeenSrc) OR firstTimeSeenSrc > relative_time(now(), `previously_unseen_cloud_provisioning_activity_window`) | table firstTime, src, user, object_id, command | `cloud_provisioning_activity_from_previously_unseen_ip_address_filter` | `security_content_ctime(firstTime)` | stats count | where count > 0",False,False,8104,
+Disable Defender Enhanced Notification,endpoint/disable_defender_enhanced_notification.yml,2.622,311296,"| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) as lastTime from datamodel=Endpoint.Registry where Registry.registry_path = ""*Microsoft\\Windows Defender\\Reporting*"" Registry.registry_value_name = DisableEnhancedNotifications Registry.registry_value_data = 0x00000001 by Registry.dest Registry.user Registry.registry_path Registry.registry_value_name Registry.registry_value_data | `drop_dm_object_name(Registry)` | `security_content_ctime(lastTime)` | `security_content_ctime(firstTime)` | `disable_defender_enhanced_notification_filter` | stats count | where count > 0",False,False,4369,
+Disable Defender Submit Samples Consent Feature,endpoint/disable_defender_submit_samples_consent_feature.yml,2.262,311296,"| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) as lastTime from datamodel=Endpoint.Registry where Registry.registry_path = ""*\\Microsoft\\Windows Defender\\SpyNet*"" Registry.registry_value_name = SubmitSamplesConsent Registry.registry_value_data = 0x00000000 by Registry.dest Registry.user Registry.registry_path Registry.registry_value_name Registry.registry_value_data | `drop_dm_object_name(Registry)` | `security_content_ctime(lastTime)` | `security_content_ctime(firstTime)` | `disable_defender_submit_samples_consent_feature_filter` | stats count | where count > 0",False,False,4369,
+Monitor Registry Keys for Print Monitors,endpoint/monitor_registry_keys_for_print_monitors.yml,3.4130000000000003,401408,"| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) as lastTime FROM datamodel=Endpoint.Registry where Registry.action=modified AND Registry.registry_path=""*CurrentControlSet\\Control\\Print\\Monitors*"" by Registry.dest, Registry.registry_key_name Registry.user Registry.registry_path Registry.registry_value_name Registry.action | `drop_dm_object_name(Registry)` | `monitor_registry_keys_for_print_monitors_filter` | stats count | where count > 0",False,False,6309,
+Disabling CMD Application,endpoint/disabling_cmd_application.yml,5.796,307200,"| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) as lastTime from datamodel=Endpoint.Registry where Registry.registry_path= ""*\\SOFTWARE\\Policies\\Microsoft\\Windows\\System\\DisableCMD"" Registry.registry_value_data = ""0x00000001"" by Registry.registry_path Registry.registry_key_name Registry.registry_value_data Registry.dest | `drop_dm_object_name(Registry)` | `security_content_ctime(firstTime)` |`security_content_ctime(lastTime)` | `disabling_cmd_application_filter` | stats count | where count > 0",False,False,11619,
+Disabling FolderOptions Windows Feature,endpoint/disabling_folderoptions_windows_feature.yml,0.869,303104,"| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) as lastTime from datamodel=Endpoint.Registry where Registry.registry_path= ""*\\SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\Policies\\Explorer\\NoFolderOptions"" Registry.registry_value_data = ""0x00000001"" by Registry.registry_path Registry.registry_key_name Registry.registry_value_data Registry.dest | `drop_dm_object_name(Registry)` | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)` | `disabling_folderoptions_windows_feature_filter` | stats count | where count > 0",False,False,11620,
+Disabling NoRun Windows App,endpoint/disabling_norun_windows_app.yml,5.142,307200,"| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) as lastTime from datamodel=Endpoint.Registry where Registry.registry_path= ""*\\SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\Policies\\Explorer\\NoRun"" Registry.registry_value_data = ""0x00000001"" by Registry.registry_path Registry.registry_key_name Registry.registry_value_data Registry.dest | `drop_dm_object_name(Registry)` | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)` | `disabling_norun_windows_app_filter` | stats count | where count > 0",False,False,11620,
+Enable RDP In Other Port Number,endpoint/enable_rdp_in_other_port_number.yml,5.255,315392,"| tstats `security_content_summariesonly` count values(Registry.registry_key_name) as registry_key_name values(Registry.registry_path) as registry_path min(_time) as firstTime max(_time) as lastTime FROM datamodel=Endpoint.Registry where Registry.registry_path=""*HKLM\\SYSTEM\\CurrentControlSet\\Control\\Terminal Server\\WinStations\\RDP-Tcp*"" Registry.registry_value_name = ""PortNumber"" by Registry.dest Registry.user Registry.registry_value_name | `security_content_ctime(lastTime)` | `security_content_ctime(firstTime)` | `drop_dm_object_name(Registry)` | `enable_rdp_in_other_port_number_filter` | stats count | where count > 0",False,False,12982,
+Disable Defender Spynet Reporting,endpoint/disable_defender_spynet_reporting.yml,2.149,311296,"| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) as lastTime from datamodel=Endpoint.Registry where Registry.registry_path = ""*\\Microsoft\\Windows Defender\\SpyNet*"" Registry.registry_value_name = SpynetReporting Registry.registry_value_data = 0x00000000 by Registry.dest Registry.user Registry.registry_path Registry.registry_value_name Registry.registry_value_data | `drop_dm_object_name(Registry)` | `security_content_ctime(lastTime)` | `security_content_ctime(firstTime)` | `disable_defender_spynet_reporting_filter` | stats count | where count > 0",False,False,4367,
+Disable Defender AntiVirus Registry,endpoint/disable_defender_antivirus_registry.yml,2.126,311296,"| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) as lastTime from datamodel=Endpoint.Registry where Registry.registry_path = ""*\\Policies\\Microsoft\\Windows Defender*"" Registry.registry_value_name = DisableAntiVirus Registry.registry_value_data = 0x00000001 by Registry.dest Registry.user Registry.registry_path Registry.registry_value_name Registry.registry_value_data | `drop_dm_object_name(Registry)` | `security_content_ctime(lastTime)` | `security_content_ctime(firstTime)` | `disable_defender_antivirus_registry_filter` | stats count | where count > 0",False,False,4367,
+Disable Show Hidden Files,endpoint/disable_show_hidden_files.yml,4.976,344064,"| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) as lastTime from datamodel=Endpoint.Registry where (Registry.registry_path= ""*\\SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\Explorer\\Advanced\\Hidden"" OR Registry.registry_path= ""*\\SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\Explorer\\Advanced\\HideFileExt"" Registry.registry_value_data = ""0x00000001"") OR (Registry.registry_path= ""*\\SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\Explorer\\Advanced\\ShowSuperHidden"" Registry.registry_value_data = ""0x00000000"") by Registry.registry_path Registry.registry_key_name Registry.registry_value_data Registry.dest | `drop_dm_object_name(Registry)` | `security_content_ctime(firstTime)` |`security_content_ctime(lastTime)` | `disable_show_hidden_files_filter` | stats count | where count > 0",False,False,11618,
+Disabling SystemRestore In Registry,endpoint/disabling_systemrestore_in_registry.yml,4.792,323584,"| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) as lastTime from datamodel=Endpoint.Registry where Registry.registry_path= ""*\\SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion\\SystemRestore\\DisableSR"" OR Registry.registry_path= ""*\\SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion\\SystemRestore\\DisableConfig"" Registry.registry_value_data = ""0x00000001"" by Registry.registry_path Registry.registry_key_name Registry.registry_value_data Registry.dest | `drop_dm_object_name(Registry)` | `security_content_ctime(firstTime)` |`security_content_ctime(lastTime)` | `disabling_systemrestore_in_registry_filter` | stats count | where count > 0",False,False,11619,
+Disable Defender MpEngine Registry,endpoint/disable_defender_mpengine_registry.yml,2.201,311296,"| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) as lastTime from datamodel=Endpoint.Registry where Registry.registry_path = ""*\\Policies\\Microsoft\\Windows Defender\\MpEngine*"" Registry.registry_value_name = MpEnablePus Registry.registry_value_data = 0x00000000 by Registry.dest Registry.user Registry.registry_path Registry.registry_value_name Registry.registry_value_data | `drop_dm_object_name(Registry)` | `security_content_ctime(lastTime)` | `security_content_ctime(firstTime)` | `disable_defender_mpengine_registry_filter` | stats count | where count > 0",False,False,4368,
+Disable AMSI Through Registry,endpoint/disable_amsi_through_registry.yml,2.164,307200,"| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) as lastTime from datamodel=Endpoint.Registry where Registry.registry_path= ""*\\SOFTWARE\\Microsoft\\Windows Script\\Settings\\AmsiEnable"" Registry.registry_value_data = ""0x00000000"" by Registry.registry_path Registry.registry_key_name Registry.registry_value_data Registry.dest | `drop_dm_object_name(Registry)` | `security_content_ctime(firstTime)` |`security_content_ctime(lastTime)` | `disable_amsi_through_registry_filter` | stats count | where count > 0",False,False,4624,
+Circle CI Disable Security Step,cloud/circle_ci_disable_security_step.yml,0.6920000000000001,241664,"search `circleci` | rename workflows.job_id AS job_id | join job_id [ | search `circleci` | stats values(name) as step_names count by job_id job_name ] | stats count by step_names job_id job_name vcs.committer_name vcs.subject vcs.url owners{} | rename vcs.* as * , owners{} as user | lookup mandatory_step_for_job job_name OUTPUTNEW step_name AS mandatory_step | search mandatory_step=* | eval mandatory_step_executed=if(like(step_names, ""%"".mandatory_step.""%""), 1, 0) | where mandatory_step_executed=0 | rex field=url ""(?<repository>[^\/]*\/[^\/]*)$"" | eval phase=""build""  | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)` | `circle_ci_disable_security_step_filter` | stats count | where count > 0",False,False,4,
+Disabling Task Manager,endpoint/disabling_task_manager.yml,4.777,311296,"| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) as lastTime from datamodel=Endpoint.Registry where Registry.registry_path= ""*\\SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\Policies\\System\\DisableTaskMgr"" Registry.registry_value_data = ""0x00000001"" by Registry.registry_path Registry.registry_key_name Registry.registry_value_data Registry.dest | `drop_dm_object_name(Registry)` | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)` | `disabling_task_manager_filter` | stats count | where count > 0",False,False,11618,
+Disable ETW Through Registry,endpoint/disable_etw_through_registry.yml,2.216,307200,"| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) as lastTime from datamodel=Endpoint.Registry where Registry.registry_path= ""*\\SOFTWARE\\Microsoft\\.NETFramework\\ETWEnabled"" Registry.registry_value_data = ""0x00000000"" by Registry.registry_path Registry.registry_key_name Registry.registry_value_data Registry.dest | `drop_dm_object_name(Registry)` | `security_content_ctime(firstTime)` |`security_content_ctime(lastTime)` | `disable_etw_through_registry_filter` | stats count | where count > 0",False,False,4623,
+Cloud Provisioning Activity From Previously Unseen City,cloud/cloud_provisioning_from_previously_unseen_city.yml,2.341,499712,"| tstats earliest(_time) as firstTime, latest(_time) as lastTime from datamodel=Change.All_Changes where (All_Changes.action=started OR All_Changes.action=created) All_Changes.status=success by All_Changes.src, All_Changes.user, All_Changes.object, All_Changes.command | `drop_dm_object_name(""All_Changes"")` | iplocation src | where isnotnull(City) | lookup previously_seen_cloud_provisioning_activity_sources City as City OUTPUT firstTimeSeen, enough_data | eventstats max(enough_data) as enough_data | where enough_data=1 | eval firstTimeSeenCity=min(firstTimeSeen) | where isnull(firstTimeSeenCity) OR firstTimeSeenCity > relative_time(now(), `previously_unseen_cloud_provisioning_activity_window`) | table firstTime, src, City, user, object, command | `cloud_provisioning_activity_from_previously_unseen_city_filter` | `security_content_ctime(firstTime)` | stats count | where count > 0",False,False,8111,
+Excessive Usage of NSLOOKUP App,endpoint/excessive_usage_of_nslookup_app.yml,1.704,315392,"search `sysmon` EventCode = 1 process_name = ""nslookup.exe"" |  bucket _time span=15m | stats count as numNsLookup by Computer, _time |  eventstats avg(numNsLookup) as avgNsLookup, stdev(numNsLookup) as stdNsLookup, count as numSlots by Computer |  eval upperThreshold=(avgNsLookup + stdNsLookup *3) |  eval isOutlier=if(avgNsLookup > 20 and avgNsLookup >= upperThreshold, 1, 0) |  search isOutlier=1 | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)` | `excessive_usage_of_nslookup_app_filter` | stats count | where count > 0",False,False,2950,
+MSHTML Module Load in Office Product,endpoint/mshtml_module_load_in_office_product.yml,0.5680000000000001,544768,"search `sysmon` EventID=7  process_name IN (""winword.exe"",""excel.exe"",""powerpnt.exe"",""mspub.exe"",""visio.exe"",""wordpad.exe"",""wordview.exe"") ImageLoaded IN (""*\\mshtml.dll"", ""*\\Microsoft.mshtml.dll"",""*\\IE.Interop.MSHTML.dll"",""*\\MshtmlDac.dll"",""*\\MshtmlDed.dll"",""*\\MshtmlDer.dll"") | stats count min(_time) as firstTime max(_time) as lastTime by Computer, process_name, ImageLoaded, OriginalFileName, process_id | rename Computer as dest | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)` | `mshtml_module_load_in_office_product_filter` | stats count | where count > 0",False,False,1,
+Cloud Provisioning Activity From Previously Unseen Region,cloud/cloud_provisioning_from_previously_unseen_region.yml,2.455,499712,"| tstats earliest(_time) as firstTime, latest(_time) as lastTime from datamodel=Change.All_Changes where (All_Changes.action=started OR All_Changes.action=created) All_Changes.status=success by All_Changes.src, All_Changes.user, All_Changes.object, All_Changes.command | `drop_dm_object_name(""All_Changes"")` | iplocation src | where isnotnull(Region) | lookup previously_seen_cloud_provisioning_activity_sources Region as Region OUTPUT firstTimeSeen, enough_data | eventstats max(enough_data) as enough_data | where enough_data=1 | eval firstTimeSeenRegion=min(firstTimeSeen) | where isnull(firstTimeSeenRegion) OR firstTimeSeenRegion > relative_time(now(), `previously_unseen_cloud_provisioning_activity_window`) | table firstTime, src, Region, user, object, command | `cloud_provisioning_activity_from_previously_unseen_region_filter` | `security_content_ctime(firstTime)` | stats count | where count > 0",False,False,8109,
+Outbound Network Connection from Java Using Default Ports,endpoint/outbound_network_connection_from_java_using_default_ports.yml,4.581,569344,"| tstats `security_content_summariesonly` count FROM datamodel=Endpoint.Processes where (Processes.process_name=""java.exe"" OR Processes.process_name=javaw.exe OR Processes.process_name=javaw.exe) by _time Processes.process_guid Processes.process_name Processes.dest Processes.process_path Processes.process Processes.parent_process_name |  `drop_dm_object_name(Processes)` | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)` | join  process_guid [| tstats `security_content_summariesonly` count FROM datamodel=Endpoint.Ports where (Ports.dest_port= 389 OR Ports.dest_port= 636 OR Ports.dest_port = 1389 OR Ports.dest_port = 1099 ) by Ports.process_guid Ports.dest Ports.dest_port| `drop_dm_object_name(Ports)` |  rename  dest as connection_to_CNC] | table _time dest parent_process_name process_name process_path process connection_to_CNC dest_port | `outbound_network_connection_from_java_using_default_ports_filter` | stats count | where count > 0",False,False,4459,
+Windows Disable Antispyware Reg,endpoint/windows_disableantispyware_reg.yml,3.4090000000000003,303104,"| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) as lastTime from datamodel=Endpoint.Registry where Registry.registry_value_name=""DisableAntiSpyware"" AND Registry.registry_value_data=""0x00000001"" by Registry.dest Registry.user Registry.registry_path Registry.registry_value_data | `drop_dm_object_name(Registry)` | `security_content_ctime(lastTime)` | `security_content_ctime(firstTime)` | `windows_disableantispyware_registry_filter` | stats count | where count > 0",False,False,7398,
+Disable Registry Tool,endpoint/disable_registry_tool.yml,9.412,319488,"| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) as lastTime from datamodel=Endpoint.Registry where Registry.registry_path= ""*\\SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\Policies\\System\\DisableRegistryTools"" Registry.registry_value_data = ""0x00000001"" by Registry.registry_path Registry.registry_key_name Registry.registry_value_data Registry.dest | `drop_dm_object_name(Registry)` | `security_content_ctime(firstTime)` |`security_content_ctime(lastTime)` | `disable_registry_tool_filter` | stats count | where count > 0",False,False,11618,
+Disabling Defender Services,endpoint/disabling_defender_services.yml,3.1670000000000003,352256,"| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) as lastTime from datamodel=Endpoint.Registry where Registry.registry_path = ""*\\System\\CurrentControlSet\\Services\\*"" AND (Registry.registry_path IN(""*WdBoot*"", ""*WdFilter*"", ""*WdNisDrv*"", ""*WdNisSvc*"", ""*WinDefend*"", ""*SecurityHealthService*""))  AND Registry.registry_value_name = Start Registry.registry_value_data = 0x00000004 by Registry.dest Registry.user Registry.registry_path Registry.registry_value_name Registry.registry_value_data | `drop_dm_object_name(Registry)` | `security_content_ctime(lastTime)` | `security_content_ctime(firstTime)` | `disabling_defender_services_filter` | stats count | where count > 0",False,False,5228,
+Disabling Remote User Account Control,endpoint/disabling_remote_user_account_control.yml,3.205,315392,"| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) as lastTime FROM datamodel=Endpoint.Registry where Registry.registry_path=*HKLM\\SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\Policies\\System\\EnableLUA* Registry.registry_value_data=""0x00000000"" by Registry.dest, Registry.registry_key_name Registry.user Registry.registry_path Registry.registry_value_data Registry.action | `drop_dm_object_name(Registry)` | `disabling_remote_user_account_control_filter` | stats count | where count > 0",False,False,6715,
+Windows InstallUtil Credential Theft,endpoint/windows_installutil_credential_theft.yml,0.366,307200,"search `sysmon` EventCode=7  process_name=installutil.exe ImageLoaded IN (""*\\samlib.dll"", ""*\\vaultcli.dll"") | stats count min(_time) as firstTime max(_time) as lastTime by Computer, process_name, ImageLoaded, OriginalFileName, process_id | rename Computer as dest | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)` | `windows_installutil_credential_theft_filter` | stats count | where count > 0",False,False,4,
+Wermgr Process Connecting To IP Check Web Services,endpoint/wermgr_process_connecting_to_ip_check_web_services.yml,3.145,323584,"search `sysmon` EventCode =22 process_name = wermgr.exe QueryName IN (""*wtfismyip.com"", ""*checkip.amazonaws.com"", ""*ipecho.net"", ""*ipinfo.io"", ""*api.ipify.org"", ""*icanhazip.com"", ""*ip.anysrc.com"",""*api.ip.sb"", ""ident.me"", ""www.myexternalip.com"", ""*zen.spamhaus.org"", ""*cbl.abuseat.org"", ""*b.barracudacentral.org"",""*dnsbl-1.uceprotect.net"", ""*spam.dnsbl.sorbs.net"") |  stats  min(_time) as firstTime max(_time) as lastTime count by  process_path process_name process_id QueryName QueryStatus QueryResults Computer EventCode | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)` | `wermgr_process_connecting_to_ip_check_web_services_filter` | stats count | where count > 0",False,False,6829,
+Gsuite Drive Share In External Email,cloud/gsuite_drive_share_in_external_email.yml,0.343,151552,"search `gsuite_drive` NOT (email IN("""", ""null"")) | rex field=parameters.owner ""[^@]+@(?<src_domain>[^@]+)"" | rex field=email ""[^@]+@(?<dest_domain>[^@]+)"" | where src_domain = ""internal_test_email.com"" and not dest_domain = ""internal_test_email.com"" | eval phase=""plan"" | eval severity=""low"" | stats values(parameters.doc_title) as doc_title, values(parameters.doc_type) as doc_types, values(email) as dst_email_list, values(parameters.visibility) as visibility, values(parameters.doc_id) as doc_id, count min(_time) as firstTime max(_time) as lastTime by parameters.owner ip_address phase severity  | rename parameters.owner as user ip_address as src_ip | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)` | `gsuite_drive_share_in_external_email_filter` | stats count | where count > 0",False,False,1,
+Spoolsv Suspicious Loaded Modules,endpoint/spoolsv_suspicious_loaded_modules.yml,0.364,159744,"search `sysmon` EventCode=7 Image =""*\\spoolsv.exe"" ImageLoaded=""*\\Windows\\System32\\spool\\drivers\\x64\\*"" ImageLoaded = ""*.dll"" | stats dc(ImageLoaded) as countImgloaded values(ImageLoaded) as ImgLoaded count min(_time) as firstTime max(_time) as lastTime by Image Computer process_id EventCode | where countImgloaded >= 3 | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)` | `spoolsv_suspicious_loaded_modules_filter` | stats count | where count > 0",False,False,45,
+Disabling ControlPanel,endpoint/disabling_controlpanel.yml,5.238,315392,"| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) as lastTime from datamodel=Endpoint.Registry where Registry.registry_path= ""*\\SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\Policies\\Explorer\\NoControlPanel"" Registry.registry_value_data = ""0x00000001"" by Registry.registry_path Registry.registry_key_name Registry.registry_value_data Registry.dest | `drop_dm_object_name(Registry)` | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)` | `disabling_controlpanel_filter` | stats count | where count > 0",False,False,11619,
+Print Spooler Failed to Load a Plug-in,endpoint/print_spooler_failed_to_load_a_plug_in.yml,0.375,139264,"search `printservice` ((ErrorCode=""0x45A"" (EventCode=""808"" OR EventCode=""4909"")) OR (""The print spooler failed to load a plug-in module"" OR ""\\drivers\\x64\\"")) | stats count min(_time) as firstTime max(_time) as lastTime by OpCode EventCode ComputerName Message | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)` | `print_spooler_failed_to_load_a_plug_in_filter` | stats count | where count > 0",False,False,0,
+Hide User Account From Sign-In Screen,endpoint/hide_user_account_from_sign_in_screen.yml,1.373,319488,"| tstats `security_content_summariesonly` count values(Registry.registry_key_name) as registry_key_name values(Registry.registry_path) as registry_path min(_time) as firstTime max(_time) as lastTime FROM datamodel=Endpoint.Registry where Registry.registry_path=""*\\Windows NT\\CurrentVersion\\Winlogon\\SpecialAccounts\\Userlist*"" AND Registry.registry_value_data = ""0x00000000"" by Registry.dest Registry.user Registry.registry_value_data | `security_content_ctime(lastTime)` | `security_content_ctime(firstTime)` | `drop_dm_object_name(Registry)` | `hide_user_account_from_sign_in_screen_filter` | stats count | where count > 0",False,False,1978,
+Disable Windows Behavior Monitoring,endpoint/disable_windows_behavior_monitoring.yml,4.997,401408,"| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) as lastTime from datamodel=Endpoint.Registry where Registry.registry_path= ""*\\SOFTWARE\\Policies\\Microsoft\\Windows Defender\\Real-Time Protection\\DisableBehaviorMonitoring"" OR Registry.registry_path= ""*\\SOFTWARE\\Policies\\Microsoft\\Windows Defender\\Real-Time Protection\\DisableOnAccessProtection"" OR Registry.registry_path= ""*\\SOFTWARE\\Policies\\Microsoft\\Windows Defender\\Real-Time Protection\\DisableScanOnRealtimeEnable"" OR Registry.registry_path= ""*\\SOFTWARE\\Microsoft\\Windows Defender\\Real-Time Protection\\DisableRealtimeMonitoring"" OR Registry.registry_path= ""*\\Real-Time Protection\\DisableIntrusionPreventionSystem"" OR Registry.registry_path= ""*\\Real-Time Protection\\DisableIOAVProtection"" OR Registry.registry_path= ""*\\Real-Time Protection\\DisableScriptScanning"" Registry.registry_value_data = ""0x00000001"" by Registry.registry_path Registry.registry_key_name Registry.registry_value_data Registry.dest | `drop_dm_object_name(Registry)` | `security_content_ctime(firstTime)` |`security_content_ctime(lastTime)` | `disable_windows_behavior_monitoring_filter` | stats count | where count > 0",False,False,11619,
+Interactive Session on Remote Endpoint with PowerShell,endpoint/interactive_session_on_remote_endpoint_with_powershell.yml,,,"search powershell` EventCode=4104 (Message=""*Enter-PSSession*"" AND Message=""*-ComputerName*"") | stats count min(_time) as firstTime max(_time) as lastTime by EventCode Message ComputerName User | `security_content_ctime(firstTime)` | `interactive_session_on_remote_endpoint_with_powershell_filter` | stats count | where count > 0",True,False,,"Unable to execute detection: HTTP 400 Bad Request -- Error in 'SearchParser': The name 'EventCode=4104 ' is invalid. Macro and argument names might only include alphanumerics, '_' and '-'."

From e3ba9f0074a4761f456549353acba1e3ae89d09a Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Tue, 11 Jan 2022 16:03:04 -0800
Subject: [PATCH 161/166] Finished removing dead code from DataManipulation.
 Added the new_sysmon as an additional default application. While it is not
 ideal to have the old and new sysmons installed side by side, there are some
 detection that only work with one and not the other.  We will run both until
 we have moved all detections to the new sysmon.

---
 .../modules/DataManipulation.py               | 47 +------------------
 .../modules/validate_args.py                  |  7 ++-
 .../test_config_github_actions.json           | 11 +++--
 3 files changed, 15 insertions(+), 50 deletions(-)

diff --git a/bin/automated_detection_testing/ci/detection_testing_batch/modules/DataManipulation.py b/bin/automated_detection_testing/ci/detection_testing_batch/modules/DataManipulation.py
index 53b1543a85..9c28b4a156 100644
--- a/bin/automated_detection_testing/ci/detection_testing_batch/modules/DataManipulation.py
+++ b/bin/automated_detection_testing/ci/detection_testing_batch/modules/DataManipulation.py
@@ -61,26 +61,6 @@ class DataManipulation:
 
         os.remove(original_backup_file)
 
-
-    
-        '''
-        #Order to make this threadsafe, we need to use fileinput.FileInput.
-        #We cannot use fileinput.fileinput because 
-        #"The instance will be used as global state for the functions of this module, and is also returned to use during iteration."
-        #from: https://docs.python.org/2/library/fileinput.html#fileinput.input
-        
-        
-        for line in fileinput.FileInput(path, inplace=True):
-            d = json.loads(line)
-            original_time = datetime.strptime(d["CreationTime"],"%Y-%m-%dT%H:%M:%S")
-            new_time = (difference + original_time)
-
-            original_time = original_time.strftime("%Y-%m-%dT%H:%M:%S")
-            new_time = new_time.strftime("%Y-%m-%dT%H:%M:%S")
-            print (line.replace(original_time, new_time),end ='')
-        '''
-
-
     def manipulate_timestamp_windows_event_log_raw(self, file_path):
         path =  os.path.join(os.path.dirname(__file__), '../' + file_path)
         path =  path.replace('modules/../','')
@@ -173,29 +153,4 @@ class DataManipulation:
                         new_file.write(line.replace(original_time, new_time))
 
 
-        os.remove(original_backup_file)
-
-
-        '''
-        #Order to make this threadsafe, we need to use fileinput.FileInput.
-        #We cannot use fileinput.fileinput because 
-        #"The instance will be used as global state for the functions of this module, and is also returned to use during iteration."
-        #from: https://docs.python.org/2/library/fileinput.html#fileinput.input
-        for line in fileinput.FileInput(path, inplace=True):
-            try:
-                d = json.loads(line)
-                original_time = datetime.strptime(d["eventTime"],"%Y-%m-%dT%H:%M:%S.%fZ")
-                new_time = (difference + original_time)
-
-                original_time = original_time.strftime("%Y-%m-%dT%H:%M:%S.%fZ")
-                new_time = new_time.strftime("%Y-%m-%dT%H:%M:%S.%fZ")
-                print (line.replace(original_time, new_time),end ='')
-            except ValueError:
-                d = json.loads(line)
-                original_time = datetime.strptime(d["eventTime"],"%Y-%m-%dT%H:%M:%SZ")
-                new_time = (difference + original_time)
-
-                original_time = original_time.strftime("%Y-%m-%dT%H:%M:%SZ")
-                new_time = new_time.strftime("%Y-%m-%dT%H:%M:%SZ")
-                print (line.replace(original_time, new_time),end ='')
-        '''
\ No newline at end of file
+        os.remove(original_backup_file)
\ No newline at end of file
diff --git a/bin/automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py b/bin/automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py
index 20704aed77..a7398dd0d0 100644
--- a/bin/automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py
+++ b/bin/automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py
@@ -245,7 +245,12 @@ setup_schema = {
                     }
             },
             
-             "default": {},
+             "default": {
+                     "SPLUNK_ADD_ON_FOR_SYSMON": {
+                     "app_number": 5709,
+                     "app_version": "1.0.1"
+                }
+             },
  
             # "default": {
             #     "ADD-ON_FOR_LINUX_SYSMON": {
diff --git a/bin/automated_detection_testing/ci/detection_testing_batch/test_config_github_actions.json b/bin/automated_detection_testing/ci/detection_testing_batch/test_config_github_actions.json
index 5b94c52a15..cf84556d02 100644
--- a/bin/automated_detection_testing/ci/detection_testing_batch/test_config_github_actions.json
+++ b/bin/automated_detection_testing/ci/detection_testing_batch/test_config_github_actions.json
@@ -8,7 +8,7 @@
         "endpoint",
         "cloud",
         "network",
-	"web"
+        "web"
     ],
     "interactive": false,
     "local_apps": {
@@ -119,7 +119,12 @@
     "show_splunk_app_password": false,
     "splunk_app_password": null,
     "splunk_container_apps_directory": "/opt/splunk/etc/apps",
-    "splunkbase_apps": {},
+    "splunkbase_apps": {
+        "SPLUNK_ADD_ON_FOR_SYSMON": {
+            "app_number": 5709,
+            "app_version": "1.0.1"
+        }
+    },
     "splunkbase_password": null,
     "splunkbase_username": null,
     "types": [
@@ -127,4 +132,4 @@
         "Hunting",
         "TTP"
     ]
-}
+}
\ No newline at end of file

From 23da2f45d24954cb8728952577e444f1e4be940f Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Tue, 11 Jan 2022 16:06:47 -0800
Subject: [PATCH 162/166] removing file that was accidentally added.

---
 .../test_results/combined.csv                 | 73 -------------------
 1 file changed, 73 deletions(-)
 delete mode 100644 bin/automated_detection_testing/ci/detection_testing_batch/test_results/combined.csv

diff --git a/bin/automated_detection_testing/ci/detection_testing_batch/test_results/combined.csv b/bin/automated_detection_testing/ci/detection_testing_batch/test_results/combined.csv
deleted file mode 100644
index fa91805f76..0000000000
--- a/bin/automated_detection_testing/ci/detection_testing_batch/test_results/combined.csv
+++ /dev/null
@@ -1,73 +0,0 @@
-"SPLUNK_VERSION","splunk/splunk:latest"
-"branch","UpdateRequiredFields"
-"commit_hash","2670d58f5018345c37bae30e80535a864614133c"
-"TEST_START_TIME","2022-01-11 12:32:39.925019"
-"TEST_FINISH_TIME","2022-01-11 12:59:54.934815"
-"TEST_DURATION","0:27:15"
-"SPLUNK_ADD_ON_FOR_AMAZON_KINESIS_FIREHOSE","{'app_number': 12, 'app_version': None, 'http_path': 'https://attack-range-appbinaries.s3-us-west-2.amazonaws.com/splunk-add-on-for-amazon-kinesis-firehose_131r7d1d093.tgz'}"
-"SPLUNK_ADD_ON_FOR_MICROSOFT_OFFICE_365","{'app_number': 13, 'app_version': None, 'http_path': 'https://attack-range-appbinaries.s3-us-west-2.amazonaws.com/splunk-add-on-for-microsoft-office-365_202.tgz'}"
-"SPLUNK_ADD_ON_FOR_STREAM_FORWARDERS","{'app_number': 9, 'app_version': None, 'http_path': 'https://attack-range-appbinaries.s3-us-west-2.amazonaws.com/splunk-add-on-for-stream-forwarders_730.tgz'}"
-"SPLUNK_ADD_ON_FOR_STREAM_WIRE_DATA","{'app_number': 8, 'app_version': None, 'http_path': 'https://attack-range-appbinaries.s3-us-west-2.amazonaws.com/splunk-add-on-for-stream-wire-data_730.tgz'}"
-"SPLUNK_ADD_ON_FOR_SYSMON_OLD","{'app_number': 1, 'app_version': None, 'http_path': 'https://attack-range-appbinaries.s3-us-west-2.amazonaws.com/splunk-add-on-for-microsoft-sysmon_1062.tgz'}"
-"SPLUNK_ADD_ON_FOR_ZEEK_AKA_BRO","{'app_number': 11, 'app_version': None, 'http_path': 'https://attack-range-appbinaries.s3-us-west-2.amazonaws.com/splunk-add-on-for-zeek-aka-bro_400.tgz'}"
-"SPLUNK_ASX_APP","{'app_number': 17, 'app_version': None, 'http_path': 'https://attack-range-appbinaries.s3-us-west-2.amazonaws.com/Splunk_ASX-latest.tar.gz'}"
-"SPLUNK_AWS_TA","{'app_number': 4, 'app_version': None, 'http_path': 'https://attack-range-appbinaries.s3-us-west-2.amazonaws.com/splunk-add-on-for-amazon-web-services_510.tgz'}"
-"SPLUNK_CIM_APP","{'app_number': 3, 'app_version': None, 'http_path': 'https://attack-range-appbinaries.s3-us-west-2.amazonaws.com/splunk-common-information-model-cim_4200.tgz'}"
-"SPLUNK_ES_CONTENT_UPDATE","{'app_number': 3449, 'app_version': None, 'local_path': '/Users/emcginnis/Documents/GitHub/security_content/bin/automated_detection_testing/ci/detection_testing_batch/apps/DA-ESS-ContentUpdate-latest.tar.gz'}"
-"SPLUNK_LINUX_TA","{'app_number': 14, 'app_version': None, 'http_path': 'https://attack-range-appbinaries.s3-us-west-2.amazonaws.com/splunk-add-on-for-unix-and-linux_820.tgz'}"
-"SPLUNK_MLTK_APP","{'app_number': 6, 'app_version': None, 'http_path': 'https://attack-range-appbinaries.s3-us-west-2.amazonaws.com/splunk-machine-learning-toolkit_521.tgz'}"
-"SPLUNK_NGINX_TA","{'app_number': 15, 'app_version': None, 'http_path': 'https://attack-range-appbinaries.s3-us-west-2.amazonaws.com/splunk-add-on-for-nginx_310.tgz'}"
-"SPLUNK_PYTHON_APP","{'app_number': 5, 'app_version': None, 'http_path': 'https://attack-range-appbinaries.s3-us-west-2.amazonaws.com/python-for-scientific-computing-for-linux-64-bit_202.tgz'}"
-"SPLUNK_SECURITY_ESSENTIALS","{'app_number': 10, 'app_version': None, 'http_path': 'https://attack-range-appbinaries.s3-us-west-2.amazonaws.com/splunk-security-essentials_333.tgz'}"
-"SPLUNK_STREAM_APP","{'app_number': 7, 'app_version': None, 'http_path': 'https://attack-range-appbinaries.s3-us-west-2.amazonaws.com/splunk-app-for-stream_730.tgz'}"
-"SPLUNK_SYSMON_LINUX_TA_PATCHED","{'app_number': 2, 'app_version': None, 'http_path': 'https://attack-range-appbinaries.s3-us-west-2.amazonaws.com/add-on-for-linux-sysmon_103_PATCHED.tgz'}"
-"SPLUNK_TA_FOR_ZEEK","{'app_number': 16, 'app_version': None, 'http_path': 'https://attack-range-appbinaries.s3-us-west-2.amazonaws.com/ta-for-zeek_105.tgz'}"
-"SPLUNK_WINDOWS_TA","{'app_number': 0, 'app_version': None, 'http_path': 'https://attack-range-appbinaries.s3-us-west-2.amazonaws.com/splunk-add-on-for-microsoft-windows_812.tgz'}"
-"SPLUNK_ADD_ON_FOR_SYSMON","{'app_number': 5709, 'app_version': '1.0.1'}"
-"",""
-detection_name,detection_file,runDuration,diskUsage,search_string,error,success,scanCount,detection_error
-CMD Carry Out String Command Parameter,endpoint/cmd_carry_out_string_command_parameter.yml,1.6640000000000001,393216,"| tstats `security_content_summariesonly` min(_time) as firstTime max(_time) as lastTime from datamodel=Endpoint.Processes where `process_cmd` AND Processes.process=""* /c *"" by Processes.dest Processes.user Processes.parent_process Processes.process_name Processes.original_file_name Processes.process Processes.process_id Processes.parent_process_id | `drop_dm_object_name(Processes)` | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)` | `cmd_carry_out_string_command_parameter_filter` | stats count | where count > 0",False,True,2109,
-Suspicious writes to windows Recycle Bin,endpoint/suspicious_writes_to_windows_recycle_bin.yml,1.8760000000000001,589824,"| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) as lastTime values(Filesystem.file_path) as file_path values(Filesystem.file_name) as file_name FROM datamodel=Endpoint.Filesystem where Filesystem.file_path = ""*$Recycle.Bin*"" by Filesystem.process_id Filesystem.dest | `drop_dm_object_name(""Filesystem"")`| search [| tstats `security_content_summariesonly` values(Processes.user) as user values(Processes.process_name) as process_name values(Processes.parent_process_name) as parent_process_name FROM datamodel=Endpoint.Processes where Processes.process_name != ""explorer.exe"" by Processes.process_id Processes.dest| `drop_dm_object_name(""Processes"")` | table process_id dest] | `suspicious_writes_to_windows_recycle_bin_filter` | stats count | where count > 0",False,True,848,
-Process Kill Base On File Path,endpoint/process_kill_base_on_file_path.yml,118.84100000000001,364544,"| tstats `security_content_summariesonly` values(Processes.process) as process values(Processes.process_id) as process_id count min(_time) as firstTime max(_time) as lastTime  from datamodel=Endpoint.Processes where `process_wmic` AND Processes.process=""*process*"" AND Processes.process=""*executablepath*"" AND Processes.process=""*delete*"" by  Processes.parent_process_name Processes.process_name Processes.original_file_name Processes.dest Processes.user | `drop_dm_object_name(Processes)` | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)` | `process_kill_base_on_file_path_filter` | stats count | where count > 0",False,True,271715,
-High Process Termination Frequency,endpoint/high_process_termination_frequency.yml,17.758,155648,search `sysmon` EventCode=5 |bin _time span=3s |stats values(Image) as proc_terminated min(_time) as firstTime max(_time) as lastTime  count by Computer EventCode ProcessID | where count >= 15 | `security_content_ctime(firstTime)`| `security_content_ctime(lastTime)` | `high_process_termination_frequency_filter` | stats count | where count > 0,False,True,46964,
-Detect Rundll32 Application Control Bypass - advpack and ieadvpack,endpoint/detect_rundll32_application_control_bypass___advpack.yml,7.022,360448,| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) as lastTime from datamodel=Endpoint.Processes where `process_rundll32` Processes.process=*advpack* by Processes.dest Processes.user Processes.parent_process_name Processes.original_file_name Processes.process_name Processes.process Processes.process_id Processes.parent_process_id | `drop_dm_object_name(Processes)` | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)` | `detect_rundll32_application_control_bypass___advpack_filter` | stats count | where count > 0,False,True,13364,
-Detect hosts connecting to dynamic domain providers,network/detect_hosts_connecting_to_dynamic_domain_providers.yml,1.052,274432,"| tstats `security_content_summariesonly` count values(DNS.answer) as answer min(_time) as firstTime from datamodel=Network_Resolution.DNS by DNS.query host | `drop_dm_object_name(""DNS"")` | `security_content_ctime(firstTime)` | `dynamic_dns_providers` | `detect_hosts_connecting_to_dynamic_domain_providers_filter` | stats count | where count > 0",False,True,644,
-Process execution via wmi,endpoint/process_execution_via_wmi.yml,3.237,339968,| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) as lastTime from datamodel=Endpoint.Processes where Processes.parent_process_name=WmiPrvSE.exe by Processes.dest Processes.user Processes.parent_process Processes.process_name Processes.process Processes.process_id Processes.parent_process_id | `drop_dm_object_name(Processes)` | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)` | `process_execution_via_wmi_filter`  | stats count | where count = 1,False,False,6572,
-Registry Keys Used For Privilege Escalation,endpoint/registry_keys_used_for_privilege_escalation.yml,2.915,311296,"| tstats `security_content_summariesonly` count  min(_time) as firstTime max(_time) as lastTime FROM datamodel=Endpoint.Registry where (Registry.registry_path=""*Microsoft\\Windows NT\\CurrentVersion\\Image File Execution Options*"") AND (Registry.registry_value_name=GlobalFlag OR Registry.registry_value_name=Debugger) by Registry.dest  Registry.user Registry.registry_path Registry.registry_value_name | `security_content_ctime(lastTime)`  | `security_content_ctime(firstTime)` | `drop_dm_object_name(Registry)` | `registry_keys_used_for_privilege_escalation_filter` | stats count | where count > 0",False,False,6391,
-Disable Defender BlockAtFirstSeen Feature,endpoint/disable_defender_blockatfirstseen_feature.yml,2.364,311296,"| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) as lastTime from datamodel=Endpoint.Registry where Registry.registry_path = ""*\\Microsoft\\Windows Defender\\SpyNet*"" Registry.registry_value_name = DisableBlockAtFirstSeen Registry.registry_value_data = 0x00000001 by Registry.dest Registry.user Registry.registry_path Registry.registry_value_name Registry.registry_value_data | `drop_dm_object_name(Registry)` | `security_content_ctime(lastTime)` | `security_content_ctime(firstTime)` | `disable_defender_blockatfirstseen_feature_filter` | stats count | where count > 0",False,False,4369,
-Allow Operation with Consent Admin,endpoint/allow_operation_with_consent_admin.yml,7.696,311296,"| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) as lastTime FROM datamodel=Endpoint.Registry where Registry.registry_path= ""*\\Microsoft\\Windows\\CurrentVersion\\Policies\\System*"" Registry.registry_value_name = ConsentPromptBehaviorAdmin Registry.registry_value_data = ""0x00000000"" by Registry.registry_path Registry.registry_key_name Registry.registry_value_data Registry.dest | `drop_dm_object_name(Registry)` | `security_content_ctime(firstTime)` |`security_content_ctime(lastTime)` | `allow_operation_with_consent_admin_filter` | stats count | where count > 0",False,False,17459,
-Cloud Provisioning Activity From Previously Unseen IP Address,cloud/cloud_provisioning_from_previously_unseen_ip_address.yml,2.399,512000,"| tstats earliest(_time) as firstTime, latest(_time) as lastTime, values(All_Changes.object_id) as object_id from datamodel=Change.All_Changes where (All_Changes.action=started OR All_Changes.action=created) All_Changes.status=success by All_Changes.src, All_Changes.user, All_Changes.command | `drop_dm_object_name(""All_Changes"")` | lookup previously_seen_cloud_provisioning_activity_sources src as src OUTPUT firstTimeSeen, enough_data | eventstats max(enough_data) as enough_data | where enough_data=1 | eval firstTimeSeenSrc=min(firstTimeSeen) | where isnull(firstTimeSeenSrc) OR firstTimeSeenSrc > relative_time(now(), `previously_unseen_cloud_provisioning_activity_window`) | table firstTime, src, user, object_id, command | `cloud_provisioning_activity_from_previously_unseen_ip_address_filter` | `security_content_ctime(firstTime)` | stats count | where count > 0",False,False,8104,
-Disable Defender Enhanced Notification,endpoint/disable_defender_enhanced_notification.yml,2.622,311296,"| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) as lastTime from datamodel=Endpoint.Registry where Registry.registry_path = ""*Microsoft\\Windows Defender\\Reporting*"" Registry.registry_value_name = DisableEnhancedNotifications Registry.registry_value_data = 0x00000001 by Registry.dest Registry.user Registry.registry_path Registry.registry_value_name Registry.registry_value_data | `drop_dm_object_name(Registry)` | `security_content_ctime(lastTime)` | `security_content_ctime(firstTime)` | `disable_defender_enhanced_notification_filter` | stats count | where count > 0",False,False,4369,
-Disable Defender Submit Samples Consent Feature,endpoint/disable_defender_submit_samples_consent_feature.yml,2.262,311296,"| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) as lastTime from datamodel=Endpoint.Registry where Registry.registry_path = ""*\\Microsoft\\Windows Defender\\SpyNet*"" Registry.registry_value_name = SubmitSamplesConsent Registry.registry_value_data = 0x00000000 by Registry.dest Registry.user Registry.registry_path Registry.registry_value_name Registry.registry_value_data | `drop_dm_object_name(Registry)` | `security_content_ctime(lastTime)` | `security_content_ctime(firstTime)` | `disable_defender_submit_samples_consent_feature_filter` | stats count | where count > 0",False,False,4369,
-Monitor Registry Keys for Print Monitors,endpoint/monitor_registry_keys_for_print_monitors.yml,3.4130000000000003,401408,"| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) as lastTime FROM datamodel=Endpoint.Registry where Registry.action=modified AND Registry.registry_path=""*CurrentControlSet\\Control\\Print\\Monitors*"" by Registry.dest, Registry.registry_key_name Registry.user Registry.registry_path Registry.registry_value_name Registry.action | `drop_dm_object_name(Registry)` | `monitor_registry_keys_for_print_monitors_filter` | stats count | where count > 0",False,False,6309,
-Disabling CMD Application,endpoint/disabling_cmd_application.yml,5.796,307200,"| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) as lastTime from datamodel=Endpoint.Registry where Registry.registry_path= ""*\\SOFTWARE\\Policies\\Microsoft\\Windows\\System\\DisableCMD"" Registry.registry_value_data = ""0x00000001"" by Registry.registry_path Registry.registry_key_name Registry.registry_value_data Registry.dest | `drop_dm_object_name(Registry)` | `security_content_ctime(firstTime)` |`security_content_ctime(lastTime)` | `disabling_cmd_application_filter` | stats count | where count > 0",False,False,11619,
-Disabling FolderOptions Windows Feature,endpoint/disabling_folderoptions_windows_feature.yml,0.869,303104,"| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) as lastTime from datamodel=Endpoint.Registry where Registry.registry_path= ""*\\SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\Policies\\Explorer\\NoFolderOptions"" Registry.registry_value_data = ""0x00000001"" by Registry.registry_path Registry.registry_key_name Registry.registry_value_data Registry.dest | `drop_dm_object_name(Registry)` | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)` | `disabling_folderoptions_windows_feature_filter` | stats count | where count > 0",False,False,11620,
-Disabling NoRun Windows App,endpoint/disabling_norun_windows_app.yml,5.142,307200,"| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) as lastTime from datamodel=Endpoint.Registry where Registry.registry_path= ""*\\SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\Policies\\Explorer\\NoRun"" Registry.registry_value_data = ""0x00000001"" by Registry.registry_path Registry.registry_key_name Registry.registry_value_data Registry.dest | `drop_dm_object_name(Registry)` | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)` | `disabling_norun_windows_app_filter` | stats count | where count > 0",False,False,11620,
-Enable RDP In Other Port Number,endpoint/enable_rdp_in_other_port_number.yml,5.255,315392,"| tstats `security_content_summariesonly` count values(Registry.registry_key_name) as registry_key_name values(Registry.registry_path) as registry_path min(_time) as firstTime max(_time) as lastTime FROM datamodel=Endpoint.Registry where Registry.registry_path=""*HKLM\\SYSTEM\\CurrentControlSet\\Control\\Terminal Server\\WinStations\\RDP-Tcp*"" Registry.registry_value_name = ""PortNumber"" by Registry.dest Registry.user Registry.registry_value_name | `security_content_ctime(lastTime)` | `security_content_ctime(firstTime)` | `drop_dm_object_name(Registry)` | `enable_rdp_in_other_port_number_filter` | stats count | where count > 0",False,False,12982,
-Disable Defender Spynet Reporting,endpoint/disable_defender_spynet_reporting.yml,2.149,311296,"| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) as lastTime from datamodel=Endpoint.Registry where Registry.registry_path = ""*\\Microsoft\\Windows Defender\\SpyNet*"" Registry.registry_value_name = SpynetReporting Registry.registry_value_data = 0x00000000 by Registry.dest Registry.user Registry.registry_path Registry.registry_value_name Registry.registry_value_data | `drop_dm_object_name(Registry)` | `security_content_ctime(lastTime)` | `security_content_ctime(firstTime)` | `disable_defender_spynet_reporting_filter` | stats count | where count > 0",False,False,4367,
-Disable Defender AntiVirus Registry,endpoint/disable_defender_antivirus_registry.yml,2.126,311296,"| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) as lastTime from datamodel=Endpoint.Registry where Registry.registry_path = ""*\\Policies\\Microsoft\\Windows Defender*"" Registry.registry_value_name = DisableAntiVirus Registry.registry_value_data = 0x00000001 by Registry.dest Registry.user Registry.registry_path Registry.registry_value_name Registry.registry_value_data | `drop_dm_object_name(Registry)` | `security_content_ctime(lastTime)` | `security_content_ctime(firstTime)` | `disable_defender_antivirus_registry_filter` | stats count | where count > 0",False,False,4367,
-Disable Show Hidden Files,endpoint/disable_show_hidden_files.yml,4.976,344064,"| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) as lastTime from datamodel=Endpoint.Registry where (Registry.registry_path= ""*\\SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\Explorer\\Advanced\\Hidden"" OR Registry.registry_path= ""*\\SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\Explorer\\Advanced\\HideFileExt"" Registry.registry_value_data = ""0x00000001"") OR (Registry.registry_path= ""*\\SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\Explorer\\Advanced\\ShowSuperHidden"" Registry.registry_value_data = ""0x00000000"") by Registry.registry_path Registry.registry_key_name Registry.registry_value_data Registry.dest | `drop_dm_object_name(Registry)` | `security_content_ctime(firstTime)` |`security_content_ctime(lastTime)` | `disable_show_hidden_files_filter` | stats count | where count > 0",False,False,11618,
-Disabling SystemRestore In Registry,endpoint/disabling_systemrestore_in_registry.yml,4.792,323584,"| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) as lastTime from datamodel=Endpoint.Registry where Registry.registry_path= ""*\\SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion\\SystemRestore\\DisableSR"" OR Registry.registry_path= ""*\\SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion\\SystemRestore\\DisableConfig"" Registry.registry_value_data = ""0x00000001"" by Registry.registry_path Registry.registry_key_name Registry.registry_value_data Registry.dest | `drop_dm_object_name(Registry)` | `security_content_ctime(firstTime)` |`security_content_ctime(lastTime)` | `disabling_systemrestore_in_registry_filter` | stats count | where count > 0",False,False,11619,
-Disable Defender MpEngine Registry,endpoint/disable_defender_mpengine_registry.yml,2.201,311296,"| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) as lastTime from datamodel=Endpoint.Registry where Registry.registry_path = ""*\\Policies\\Microsoft\\Windows Defender\\MpEngine*"" Registry.registry_value_name = MpEnablePus Registry.registry_value_data = 0x00000000 by Registry.dest Registry.user Registry.registry_path Registry.registry_value_name Registry.registry_value_data | `drop_dm_object_name(Registry)` | `security_content_ctime(lastTime)` | `security_content_ctime(firstTime)` | `disable_defender_mpengine_registry_filter` | stats count | where count > 0",False,False,4368,
-Disable AMSI Through Registry,endpoint/disable_amsi_through_registry.yml,2.164,307200,"| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) as lastTime from datamodel=Endpoint.Registry where Registry.registry_path= ""*\\SOFTWARE\\Microsoft\\Windows Script\\Settings\\AmsiEnable"" Registry.registry_value_data = ""0x00000000"" by Registry.registry_path Registry.registry_key_name Registry.registry_value_data Registry.dest | `drop_dm_object_name(Registry)` | `security_content_ctime(firstTime)` |`security_content_ctime(lastTime)` | `disable_amsi_through_registry_filter` | stats count | where count > 0",False,False,4624,
-Circle CI Disable Security Step,cloud/circle_ci_disable_security_step.yml,0.6920000000000001,241664,"search `circleci` | rename workflows.job_id AS job_id | join job_id [ | search `circleci` | stats values(name) as step_names count by job_id job_name ] | stats count by step_names job_id job_name vcs.committer_name vcs.subject vcs.url owners{} | rename vcs.* as * , owners{} as user | lookup mandatory_step_for_job job_name OUTPUTNEW step_name AS mandatory_step | search mandatory_step=* | eval mandatory_step_executed=if(like(step_names, ""%"".mandatory_step.""%""), 1, 0) | where mandatory_step_executed=0 | rex field=url ""(?<repository>[^\/]*\/[^\/]*)$"" | eval phase=""build""  | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)` | `circle_ci_disable_security_step_filter` | stats count | where count > 0",False,False,4,
-Disabling Task Manager,endpoint/disabling_task_manager.yml,4.777,311296,"| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) as lastTime from datamodel=Endpoint.Registry where Registry.registry_path= ""*\\SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\Policies\\System\\DisableTaskMgr"" Registry.registry_value_data = ""0x00000001"" by Registry.registry_path Registry.registry_key_name Registry.registry_value_data Registry.dest | `drop_dm_object_name(Registry)` | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)` | `disabling_task_manager_filter` | stats count | where count > 0",False,False,11618,
-Disable ETW Through Registry,endpoint/disable_etw_through_registry.yml,2.216,307200,"| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) as lastTime from datamodel=Endpoint.Registry where Registry.registry_path= ""*\\SOFTWARE\\Microsoft\\.NETFramework\\ETWEnabled"" Registry.registry_value_data = ""0x00000000"" by Registry.registry_path Registry.registry_key_name Registry.registry_value_data Registry.dest | `drop_dm_object_name(Registry)` | `security_content_ctime(firstTime)` |`security_content_ctime(lastTime)` | `disable_etw_through_registry_filter` | stats count | where count > 0",False,False,4623,
-Cloud Provisioning Activity From Previously Unseen City,cloud/cloud_provisioning_from_previously_unseen_city.yml,2.341,499712,"| tstats earliest(_time) as firstTime, latest(_time) as lastTime from datamodel=Change.All_Changes where (All_Changes.action=started OR All_Changes.action=created) All_Changes.status=success by All_Changes.src, All_Changes.user, All_Changes.object, All_Changes.command | `drop_dm_object_name(""All_Changes"")` | iplocation src | where isnotnull(City) | lookup previously_seen_cloud_provisioning_activity_sources City as City OUTPUT firstTimeSeen, enough_data | eventstats max(enough_data) as enough_data | where enough_data=1 | eval firstTimeSeenCity=min(firstTimeSeen) | where isnull(firstTimeSeenCity) OR firstTimeSeenCity > relative_time(now(), `previously_unseen_cloud_provisioning_activity_window`) | table firstTime, src, City, user, object, command | `cloud_provisioning_activity_from_previously_unseen_city_filter` | `security_content_ctime(firstTime)` | stats count | where count > 0",False,False,8111,
-Excessive Usage of NSLOOKUP App,endpoint/excessive_usage_of_nslookup_app.yml,1.704,315392,"search `sysmon` EventCode = 1 process_name = ""nslookup.exe"" |  bucket _time span=15m | stats count as numNsLookup by Computer, _time |  eventstats avg(numNsLookup) as avgNsLookup, stdev(numNsLookup) as stdNsLookup, count as numSlots by Computer |  eval upperThreshold=(avgNsLookup + stdNsLookup *3) |  eval isOutlier=if(avgNsLookup > 20 and avgNsLookup >= upperThreshold, 1, 0) |  search isOutlier=1 | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)` | `excessive_usage_of_nslookup_app_filter` | stats count | where count > 0",False,False,2950,
-MSHTML Module Load in Office Product,endpoint/mshtml_module_load_in_office_product.yml,0.5680000000000001,544768,"search `sysmon` EventID=7  process_name IN (""winword.exe"",""excel.exe"",""powerpnt.exe"",""mspub.exe"",""visio.exe"",""wordpad.exe"",""wordview.exe"") ImageLoaded IN (""*\\mshtml.dll"", ""*\\Microsoft.mshtml.dll"",""*\\IE.Interop.MSHTML.dll"",""*\\MshtmlDac.dll"",""*\\MshtmlDed.dll"",""*\\MshtmlDer.dll"") | stats count min(_time) as firstTime max(_time) as lastTime by Computer, process_name, ImageLoaded, OriginalFileName, process_id | rename Computer as dest | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)` | `mshtml_module_load_in_office_product_filter` | stats count | where count > 0",False,False,1,
-Cloud Provisioning Activity From Previously Unseen Region,cloud/cloud_provisioning_from_previously_unseen_region.yml,2.455,499712,"| tstats earliest(_time) as firstTime, latest(_time) as lastTime from datamodel=Change.All_Changes where (All_Changes.action=started OR All_Changes.action=created) All_Changes.status=success by All_Changes.src, All_Changes.user, All_Changes.object, All_Changes.command | `drop_dm_object_name(""All_Changes"")` | iplocation src | where isnotnull(Region) | lookup previously_seen_cloud_provisioning_activity_sources Region as Region OUTPUT firstTimeSeen, enough_data | eventstats max(enough_data) as enough_data | where enough_data=1 | eval firstTimeSeenRegion=min(firstTimeSeen) | where isnull(firstTimeSeenRegion) OR firstTimeSeenRegion > relative_time(now(), `previously_unseen_cloud_provisioning_activity_window`) | table firstTime, src, Region, user, object, command | `cloud_provisioning_activity_from_previously_unseen_region_filter` | `security_content_ctime(firstTime)` | stats count | where count > 0",False,False,8109,
-Outbound Network Connection from Java Using Default Ports,endpoint/outbound_network_connection_from_java_using_default_ports.yml,4.581,569344,"| tstats `security_content_summariesonly` count FROM datamodel=Endpoint.Processes where (Processes.process_name=""java.exe"" OR Processes.process_name=javaw.exe OR Processes.process_name=javaw.exe) by _time Processes.process_guid Processes.process_name Processes.dest Processes.process_path Processes.process Processes.parent_process_name |  `drop_dm_object_name(Processes)` | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)` | join  process_guid [| tstats `security_content_summariesonly` count FROM datamodel=Endpoint.Ports where (Ports.dest_port= 389 OR Ports.dest_port= 636 OR Ports.dest_port = 1389 OR Ports.dest_port = 1099 ) by Ports.process_guid Ports.dest Ports.dest_port| `drop_dm_object_name(Ports)` |  rename  dest as connection_to_CNC] | table _time dest parent_process_name process_name process_path process connection_to_CNC dest_port | `outbound_network_connection_from_java_using_default_ports_filter` | stats count | where count > 0",False,False,4459,
-Windows Disable Antispyware Reg,endpoint/windows_disableantispyware_reg.yml,3.4090000000000003,303104,"| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) as lastTime from datamodel=Endpoint.Registry where Registry.registry_value_name=""DisableAntiSpyware"" AND Registry.registry_value_data=""0x00000001"" by Registry.dest Registry.user Registry.registry_path Registry.registry_value_data | `drop_dm_object_name(Registry)` | `security_content_ctime(lastTime)` | `security_content_ctime(firstTime)` | `windows_disableantispyware_registry_filter` | stats count | where count > 0",False,False,7398,
-Disable Registry Tool,endpoint/disable_registry_tool.yml,9.412,319488,"| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) as lastTime from datamodel=Endpoint.Registry where Registry.registry_path= ""*\\SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\Policies\\System\\DisableRegistryTools"" Registry.registry_value_data = ""0x00000001"" by Registry.registry_path Registry.registry_key_name Registry.registry_value_data Registry.dest | `drop_dm_object_name(Registry)` | `security_content_ctime(firstTime)` |`security_content_ctime(lastTime)` | `disable_registry_tool_filter` | stats count | where count > 0",False,False,11618,
-Disabling Defender Services,endpoint/disabling_defender_services.yml,3.1670000000000003,352256,"| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) as lastTime from datamodel=Endpoint.Registry where Registry.registry_path = ""*\\System\\CurrentControlSet\\Services\\*"" AND (Registry.registry_path IN(""*WdBoot*"", ""*WdFilter*"", ""*WdNisDrv*"", ""*WdNisSvc*"", ""*WinDefend*"", ""*SecurityHealthService*""))  AND Registry.registry_value_name = Start Registry.registry_value_data = 0x00000004 by Registry.dest Registry.user Registry.registry_path Registry.registry_value_name Registry.registry_value_data | `drop_dm_object_name(Registry)` | `security_content_ctime(lastTime)` | `security_content_ctime(firstTime)` | `disabling_defender_services_filter` | stats count | where count > 0",False,False,5228,
-Disabling Remote User Account Control,endpoint/disabling_remote_user_account_control.yml,3.205,315392,"| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) as lastTime FROM datamodel=Endpoint.Registry where Registry.registry_path=*HKLM\\SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\Policies\\System\\EnableLUA* Registry.registry_value_data=""0x00000000"" by Registry.dest, Registry.registry_key_name Registry.user Registry.registry_path Registry.registry_value_data Registry.action | `drop_dm_object_name(Registry)` | `disabling_remote_user_account_control_filter` | stats count | where count > 0",False,False,6715,
-Windows InstallUtil Credential Theft,endpoint/windows_installutil_credential_theft.yml,0.366,307200,"search `sysmon` EventCode=7  process_name=installutil.exe ImageLoaded IN (""*\\samlib.dll"", ""*\\vaultcli.dll"") | stats count min(_time) as firstTime max(_time) as lastTime by Computer, process_name, ImageLoaded, OriginalFileName, process_id | rename Computer as dest | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)` | `windows_installutil_credential_theft_filter` | stats count | where count > 0",False,False,4,
-Wermgr Process Connecting To IP Check Web Services,endpoint/wermgr_process_connecting_to_ip_check_web_services.yml,3.145,323584,"search `sysmon` EventCode =22 process_name = wermgr.exe QueryName IN (""*wtfismyip.com"", ""*checkip.amazonaws.com"", ""*ipecho.net"", ""*ipinfo.io"", ""*api.ipify.org"", ""*icanhazip.com"", ""*ip.anysrc.com"",""*api.ip.sb"", ""ident.me"", ""www.myexternalip.com"", ""*zen.spamhaus.org"", ""*cbl.abuseat.org"", ""*b.barracudacentral.org"",""*dnsbl-1.uceprotect.net"", ""*spam.dnsbl.sorbs.net"") |  stats  min(_time) as firstTime max(_time) as lastTime count by  process_path process_name process_id QueryName QueryStatus QueryResults Computer EventCode | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)` | `wermgr_process_connecting_to_ip_check_web_services_filter` | stats count | where count > 0",False,False,6829,
-Gsuite Drive Share In External Email,cloud/gsuite_drive_share_in_external_email.yml,0.343,151552,"search `gsuite_drive` NOT (email IN("""", ""null"")) | rex field=parameters.owner ""[^@]+@(?<src_domain>[^@]+)"" | rex field=email ""[^@]+@(?<dest_domain>[^@]+)"" | where src_domain = ""internal_test_email.com"" and not dest_domain = ""internal_test_email.com"" | eval phase=""plan"" | eval severity=""low"" | stats values(parameters.doc_title) as doc_title, values(parameters.doc_type) as doc_types, values(email) as dst_email_list, values(parameters.visibility) as visibility, values(parameters.doc_id) as doc_id, count min(_time) as firstTime max(_time) as lastTime by parameters.owner ip_address phase severity  | rename parameters.owner as user ip_address as src_ip | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)` | `gsuite_drive_share_in_external_email_filter` | stats count | where count > 0",False,False,1,
-Spoolsv Suspicious Loaded Modules,endpoint/spoolsv_suspicious_loaded_modules.yml,0.364,159744,"search `sysmon` EventCode=7 Image =""*\\spoolsv.exe"" ImageLoaded=""*\\Windows\\System32\\spool\\drivers\\x64\\*"" ImageLoaded = ""*.dll"" | stats dc(ImageLoaded) as countImgloaded values(ImageLoaded) as ImgLoaded count min(_time) as firstTime max(_time) as lastTime by Image Computer process_id EventCode | where countImgloaded >= 3 | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)` | `spoolsv_suspicious_loaded_modules_filter` | stats count | where count > 0",False,False,45,
-Disabling ControlPanel,endpoint/disabling_controlpanel.yml,5.238,315392,"| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) as lastTime from datamodel=Endpoint.Registry where Registry.registry_path= ""*\\SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\Policies\\Explorer\\NoControlPanel"" Registry.registry_value_data = ""0x00000001"" by Registry.registry_path Registry.registry_key_name Registry.registry_value_data Registry.dest | `drop_dm_object_name(Registry)` | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)` | `disabling_controlpanel_filter` | stats count | where count > 0",False,False,11619,
-Print Spooler Failed to Load a Plug-in,endpoint/print_spooler_failed_to_load_a_plug_in.yml,0.375,139264,"search `printservice` ((ErrorCode=""0x45A"" (EventCode=""808"" OR EventCode=""4909"")) OR (""The print spooler failed to load a plug-in module"" OR ""\\drivers\\x64\\"")) | stats count min(_time) as firstTime max(_time) as lastTime by OpCode EventCode ComputerName Message | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)` | `print_spooler_failed_to_load_a_plug_in_filter` | stats count | where count > 0",False,False,0,
-Hide User Account From Sign-In Screen,endpoint/hide_user_account_from_sign_in_screen.yml,1.373,319488,"| tstats `security_content_summariesonly` count values(Registry.registry_key_name) as registry_key_name values(Registry.registry_path) as registry_path min(_time) as firstTime max(_time) as lastTime FROM datamodel=Endpoint.Registry where Registry.registry_path=""*\\Windows NT\\CurrentVersion\\Winlogon\\SpecialAccounts\\Userlist*"" AND Registry.registry_value_data = ""0x00000000"" by Registry.dest Registry.user Registry.registry_value_data | `security_content_ctime(lastTime)` | `security_content_ctime(firstTime)` | `drop_dm_object_name(Registry)` | `hide_user_account_from_sign_in_screen_filter` | stats count | where count > 0",False,False,1978,
-Disable Windows Behavior Monitoring,endpoint/disable_windows_behavior_monitoring.yml,4.997,401408,"| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) as lastTime from datamodel=Endpoint.Registry where Registry.registry_path= ""*\\SOFTWARE\\Policies\\Microsoft\\Windows Defender\\Real-Time Protection\\DisableBehaviorMonitoring"" OR Registry.registry_path= ""*\\SOFTWARE\\Policies\\Microsoft\\Windows Defender\\Real-Time Protection\\DisableOnAccessProtection"" OR Registry.registry_path= ""*\\SOFTWARE\\Policies\\Microsoft\\Windows Defender\\Real-Time Protection\\DisableScanOnRealtimeEnable"" OR Registry.registry_path= ""*\\SOFTWARE\\Microsoft\\Windows Defender\\Real-Time Protection\\DisableRealtimeMonitoring"" OR Registry.registry_path= ""*\\Real-Time Protection\\DisableIntrusionPreventionSystem"" OR Registry.registry_path= ""*\\Real-Time Protection\\DisableIOAVProtection"" OR Registry.registry_path= ""*\\Real-Time Protection\\DisableScriptScanning"" Registry.registry_value_data = ""0x00000001"" by Registry.registry_path Registry.registry_key_name Registry.registry_value_data Registry.dest | `drop_dm_object_name(Registry)` | `security_content_ctime(firstTime)` |`security_content_ctime(lastTime)` | `disable_windows_behavior_monitoring_filter` | stats count | where count > 0",False,False,11619,
-Interactive Session on Remote Endpoint with PowerShell,endpoint/interactive_session_on_remote_endpoint_with_powershell.yml,,,"search powershell` EventCode=4104 (Message=""*Enter-PSSession*"" AND Message=""*-ComputerName*"") | stats count min(_time) as firstTime max(_time) as lastTime by EventCode Message ComputerName User | `security_content_ctime(firstTime)` | `interactive_session_on_remote_endpoint_with_powershell_filter` | stats count | where count > 0",True,False,,"Unable to execute detection: HTTP 400 Bad Request -- Error in 'SearchParser': The name 'EventCode=4104 ' is invalid. Macro and argument names might only include alphanumerics, '_' and '-'."

From 7b3029b18743e4408eabff392e8c6460f99ebc74 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Tue, 11 Jan 2022 16:35:19 -0800
Subject: [PATCH 163/166] Removed splunkbase-only test_config file. Pushing to
 run a test on all. Will push again for changes only.

---
 .github/workflows/build-and-validate.yml      |   5 +-
 ...test_config_github_actions_splunkbase.json | 122 ------------------
 2 files changed, 2 insertions(+), 125 deletions(-)
 delete mode 100644 bin/automated_detection_testing/ci/detection_testing_batch/test_config_github_actions_splunkbase.json

diff --git a/.github/workflows/build-and-validate.yml b/.github/workflows/build-and-validate.yml
index 022bc06615..6d8d4aae7b 100644
--- a/.github/workflows/build-and-validate.yml
+++ b/.github/workflows/build-and-validate.yml
@@ -352,14 +352,13 @@ jobs:
             
             if [[ ${{ steps.vars.outputs.branch }} == develop ]]; then
               echo "Running a nightly test on all detections"
-              python detection_testing_execution.py run --branch ${{ github.event.pull_request.head.ref }} --mode all --mock --config_file test_config_github_actions.json  
+              python detection_testing_execution.py run --branch ${{ github.event.pull_request.head.ref }} --mode changes --mock --config_file test_config_github_actions.json  
             elif [[ ! -z "${{  github.event.pull_request.head.ref }}" &&  ! -z "${{ github.event.pull_request.number }}" ]]; then
               echo "Pull request from source branch [${{ github.event.pull_request.head.ref }}] for PR number [${{ github.event.issue.number }}]"
               python detection_testing_execution.py run --branch ${{ github.event.pull_request.head.ref }} --pr_number ${{ github.event.pull_request.number }}  --mode changes --mock --config_file test_config_github_actions.json  
             else
               echo "Push from branch [${{ steps.vars.outputs.branch }}]"
-              #python detection_testing_execution.py run --branch  ${{ steps.vars.outputs.branch }}  --mode all --mock --config_file test_config_github_actions.json
-              python detection_testing_execution.py run --branch  develop  --mode all --mock --config_file test_config_github_actions_splunkbase.json
+              python detection_testing_execution.py run --branch  develop  --mode all --mock --config_file test_config_github_actions.json
             fi
 
             mv *-test-run.json replicate_test.json
diff --git a/bin/automated_detection_testing/ci/detection_testing_batch/test_config_github_actions_splunkbase.json b/bin/automated_detection_testing/ci/detection_testing_batch/test_config_github_actions_splunkbase.json
deleted file mode 100644
index 3e6ae86949..0000000000
--- a/bin/automated_detection_testing/ci/detection_testing_batch/test_config_github_actions_splunkbase.json
+++ /dev/null
@@ -1,122 +0,0 @@
-{
-    "branch": "BRANCH_DOES_NOT_EXIST_USE_CLI_ARGUMENT",
-    "commit_hash": null,
-    "container_tag": "latest",
-    "detections_file": null,
-    "detections_list": null,
-    "folders": [
-        "endpoint",
-        "cloud",
-        "network",
-	"web"
-    ],
-    "interactive": false,
-    "local_apps": {
-        "SPLUNK_ES_CONTENT_UPDATE": {
-            "app_number": 3449,
-            "app_version": null,
-            "local_path": null
-        },
-       
-        "SPLUNK_SYSMON_LINUX_TA_PATCHED": {
-            "app_number": 2,
-            "app_version": null,
-            "http_path": "https://attack-range-appbinaries.s3-us-west-2.amazonaws.com/add-on-for-linux-sysmon_103_PATCHED.tgz"
-        }
-  
-    },
-    "local_base_container_name": "splunk_test_%d",
-    "mock": false,
-    "mode": "changes",
-    "no_interactive_failure": true,
-    "num_containers": 10,
-    "persist_security_content": false,
-    "pr_number": null,
-    "reuse_image": true,
-    "show_splunk_app_password": false,
-    "splunk_app_password": null,
-    "splunk_container_apps_directory": "/opt/splunk/etc/apps",
-    "splunkbase_apps": {
-        
-                 
-                 "SPLUNK_ADD_ON_FOR_AMAZON_WEB_SERVICES": {
-                     "app_number": 1876,
-                     "app_version": "5.2.1"
-                 },
-
-                 "SPLUNK_ADD_ON_FOR_MICROSOFT_OFFICE_365":
-                 {
-                     "app_number": 4055,
-                     "app_version": "2.2.0"
-                 },
-
-                 "SPLUNK_ADD_ON_FOR_AMAZON_KINESIS_FIREHOSE": {
-                     "app_number": 3719,
-                     "app_version": "1.3.2"
-                 },
-
-                 "SPLUNK_ANALYTIC_STORY_EXECUTION_APP": {
-                     "app_number": 4971,
-                     "app_version": "2.0.3"
-                 },
-
-                 "PYTHON_FOR_SCIENTIC_COMPUTING_LINUX_64_BIT": {
-                     "app_number": 2882,
-                     "app_version": "3.0.1"
-                 },
-
-                 "SPLUNK_MACHINE_LEARNING_TOOLKIT": {
-                     "app_number": 2890,
-                     "app_version": "5.3.0"
-                 },
-
-                 "SPLUNK_APP_FOR_STREAM": {
-                     "app_number": 1809,
-                     "app_version": "8.0.1"
-                 },
-                 "SPLUNK_ADD_ON_FOR_STREAM_WIRE_DATA": {
-                     "app_number": 5234,
-                     "app_version": "8.0.1"
-                 },
-                 "SPLUNK_ADD_ON_FOR_STREAM_FORWARDERS": {
-                     "app_number": 5238,
-                     "app_version": "8.0.1"
-                 },
-                 "SPLUNK_ADD_ON_FOR_ZEEK_AKA_BRO": {
-                     "app_number": 1617,
-                     "app_version": "4.0.0"
-                 },
-                 "SPLUNK_ADD_ON_FOR_UNIX_AND_LINUX": {
-                     "app_number": 833,
-                     "app_version": "8.4.0"
-                 },
-                 "SPLUNK_ADD_ON_FOR_SYSMON": {
-                     "app_number": 5709,
-                     "app_version": "1.0.1"
-                 },
-                  
-                 "SPLUNK_COMMON_INFORMATION_MODEL": {
-                     "app_number": 1621,
-                     "app_version": "5.0.0"
-                 },
-                 "SPLUNK_ADD_ON_FOR_NGINX" : {
-                    "app_number": 3258,
-                    "app_version": "3.1.0"
-                 },
-                 "SPLUNK_SECURITY_ESSENTIALS": {
-                    "app_number": 3435,
-                    "app_version": "3.4.0"
-                 },
-                 "TA_FOR_ZEEK": {
-                    "app_number": 5466,
-                    "app_version": "1.0.5"
-                 }
-    },
-    "splunkbase_password": null,
-    "splunkbase_username": null,
-    "types": [
-        "Anomaly",
-        "Hunting",
-        "TTP"
-    ]
-}

From 4d9349084e49796fc5ef801c99fd744acfc537f6 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Tue, 11 Jan 2022 16:37:03 -0800
Subject: [PATCH 164/166] Changed to run nightly on develop and changes on
 other branches for pushes and PRs.

---
 .github/workflows/build-and-validate.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/build-and-validate.yml b/.github/workflows/build-and-validate.yml
index 6d8d4aae7b..ce4e667277 100644
--- a/.github/workflows/build-and-validate.yml
+++ b/.github/workflows/build-and-validate.yml
@@ -352,13 +352,13 @@ jobs:
             
             if [[ ${{ steps.vars.outputs.branch }} == develop ]]; then
               echo "Running a nightly test on all detections"
-              python detection_testing_execution.py run --branch ${{ github.event.pull_request.head.ref }} --mode changes --mock --config_file test_config_github_actions.json  
+              python detection_testing_execution.py run --branch ${{ github.event.pull_request.head.ref }} --mode all --mock --config_file test_config_github_actions.json  
             elif [[ ! -z "${{  github.event.pull_request.head.ref }}" &&  ! -z "${{ github.event.pull_request.number }}" ]]; then
               echo "Pull request from source branch [${{ github.event.pull_request.head.ref }}] for PR number [${{ github.event.issue.number }}]"
               python detection_testing_execution.py run --branch ${{ github.event.pull_request.head.ref }} --pr_number ${{ github.event.pull_request.number }}  --mode changes --mock --config_file test_config_github_actions.json  
             else
               echo "Push from branch [${{ steps.vars.outputs.branch }}]"
-              python detection_testing_execution.py run --branch  develop  --mode all --mock --config_file test_config_github_actions.json
+              python detection_testing_execution.py run --branch  develop  --mode changes --mock --config_file test_config_github_actions.json
             fi
 
             mv *-test-run.json replicate_test.json

From 3afef5d55902f82c4a011f4e1b09fc0c49458d2a Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Wed, 12 Jan 2022 10:38:23 -0800
Subject: [PATCH 165/166] Updated the error for providing both PR and
 commit_hash to a warning.  This was causing an error during PR triggered
 testing in github actions as both of these are provided.  Perhaps we actually
 can include both of these and can get old PRs based on commit hash, but until
 then we will just test using the PR number and ignore the commit_hash if we
 find both.

---
 .../detection_testing_execution.py                  |  4 +++-
 .../modules/github_service.py                       | 13 ++++++++-----
 2 files changed, 11 insertions(+), 6 deletions(-)

diff --git a/bin/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py b/bin/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
index 6e63261ef6..32e2d045bb 100644
--- a/bin/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
+++ b/bin/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py
@@ -98,7 +98,7 @@ def copy_local_apps_to_directory(apps: dict[str, dict], target_directory) -> Non
 
 
 
-def ensure_security_content(branch: str, commit_hash: str, pr_number: Union[int, None], persist_security_content: bool) -> tuple[GithubService, bool]:
+def ensure_security_content(branch: str, commit_hash: Union[str,None], pr_number: Union[int, None], persist_security_content: bool) -> tuple[GithubService, bool]:
     if persist_security_content is True and os.path.exists("security_content"):
         print("****** You chose --persist_security_content and the security_content directory exists. "
               "We will not check out the repo again. Please be aware, this could cause issues if your "
@@ -508,6 +508,8 @@ def main(args: list[str]):
         sys.exit(0)
     else:
         print("Test Execution Failed - review the logs for more details")
+        print("IN THE FUTURE, THIS WILL RETURN NONZERO CAUSING THE WORKFLOW TO FAIL!")
+        sys.exit(0)
         sys.exit(1)
 
 
diff --git a/bin/automated_detection_testing/ci/detection_testing_batch/modules/github_service.py b/bin/automated_detection_testing/ci/detection_testing_batch/modules/github_service.py
index a1962a710d..ffedd07ea2 100644
--- a/bin/automated_detection_testing/ci/detection_testing_batch/modules/github_service.py
+++ b/bin/automated_detection_testing/ci/detection_testing_batch/modules/github_service.py
@@ -24,7 +24,7 @@ SECURITY_CONTENT_URL = "https://github.com/splunk/security_content"
 class GithubService:
 
 
-    def __init__(self, security_content_branch: str, commit_hash: str, PR_number: int = None, persist_security_content: bool = False):
+    def __init__(self, security_content_branch: str, commit_hash: Union[str,None], PR_number: int = None, persist_security_content: bool = False):
 
         self.security_content_branch = security_content_branch
         if persist_security_content:
@@ -44,12 +44,15 @@ class GithubService:
                            "'git branch -a' to examine [%d] branches"%(security_content_branch, len(branch_names))))
 
 
-        if commit_hash is not None and PR_number is not None:
-            raise(Exception("Error - both the PR number [%d] and the commit hash [%s] were provided.  "
-                  "Only 0 or 1 can be passed." % (PR_number, commit_hash)))
+        if commit_hash is not None and PR_number is not None: 
+            print(f"\n************\nWARNING - both the PR_number {PR_number} and the commit_hash {commit_hash} were provided. "
+                  f"You should only pass neither or one of these.  We will ASSUME you want to use the PR_number, not the commit_hash. " 
+                  f"Removing the commit_hash...\n************\n")
+            commit_hash = None
+            
             
 
-        elif PR_number:
+        if PR_number:
             ret = subprocess.run(["git", "-C", "security_content/", "fetch", "origin",
                             "refs/pull/%d/head:%s" % (PR_number, security_content_branch)], capture_output=True)
             #ret = subprocess.call(["git", "-C", "security_content/", "fetch", "origin",

From b12ccf0cd23eed28f9e629698d344251c89da8ea Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Wed, 12 Jan 2022 10:54:45 -0800
Subject: [PATCH 166/166] Added a readme.md. Need to check that the markdown
 renders appropriately

---
 .../ci/detection_testing_batch/README.md      | 97 +++++++++++++++++++
 1 file changed, 97 insertions(+)
 create mode 100644 bin/automated_detection_testing/ci/detection_testing_batch/README.md

diff --git a/bin/automated_detection_testing/ci/detection_testing_batch/README.md b/bin/automated_detection_testing/ci/detection_testing_batch/README.md
new file mode 100644
index 0000000000..bfd203c515
--- /dev/null
+++ b/bin/automated_detection_testing/ci/detection_testing_batch/README.md
@@ -0,0 +1,97 @@
+# Batch Detection Testing
+
+The Splunk Threat Research Team produces the [Enterprise Security Content Update Splunk App](https://splunkbase.splunk.com/app/3449/), a power app that includes hundreds of curated, tested detections that you can run on your own Splunk Enterprise Server today.  The splunk/security_content repo gives users an insight into our work and allows them to replicate our workflow and even author their own detections!
+
+A core component of ESCU is that all of detections must be tested and validated against datasets to ensure they work correctly.  In order to achieve that, the Security Content Detection Testing System was built with a few goals in mind:
+ - How can we quickly test a small number of detections, and how can I easily debug those detections?
+ - How can we reliably test new or modified detections every time they are committed to our repo (or every time that a PR is created) inside of GitHub Actions?
+ - At the time of this writing, ESCU contains over 550 detections!  How can we quickly test a large number of detections?
+ - How can we run these tests on a wide variety of architectures, from develops' own machines to GitHub Actions to other Cloud Instances?
+
+This architecture diagram gives insight into the workflow:
+In summary, the tool:
+
+ 1. Downloads the latest version of the security_content Repo
+ 2. Lints/Sanity Checks all the Detections
+ 3. Builds an ESCU Splunk App
+ 4. Starts Docker containers (available from Docker Hub as splunk/splunk:latest), installing required Splunkbase Apps and ESCU
+ 5. Distributes Detection Tests Across those Containers
+ 6. Summarizes the Results of those Detections
+
+
+# Running a Basic Test
+
+## Running the Tool
+The easiest way to run a tests with default, which is suitable for most use cases, is to run:
+
+    python detection_testing_exectuion.py run --branch BRANCH_TO_TEST [--splunkbase_username YOUR_USERNAME --splunkbase_password YOUR_PASSWORD]
+    
+ While the test is running, you'll see helpful information printed, letting you know what step of the process is taking place and what detection is being tested.
+ When you start your Splunk Server, the credentials will be printed out on the command line.  You may want to use these credentials to log into the Splunk server, hosted locally, during testing for debugging or other exploration:
+
+    ***********************
+    Log into your [1] Splunk Container(s) after they boot at http://127.0.0.1:[8000-8000]
+    Splunk App Username: [admin]
+    Splunk App Password: [PBlZEeGvQrOF57zmUXFPOP]
+    ***********************
+
+While you're running, you'll receive helpful progress updates each minute.  They give you information about how long your test has been running, your approximate time remaining, and your approximate system load.  Please note that this is total system load, not JUST load used by the detection testing:
+
+    ***********PROGRESS UPDATE***********
+    Elapsed Time               : 0:27:53.358628
+    Estimated Remaining Time   : 0:49:18.896474
+    Tests to run               : 36
+    Tests currently running    : 1
+    Tests completed            : 20
+        Success : 15
+        Failure : 5
+        Error   : 0
+    System Information:
+        Total CPU Usage   : 44% (2 CPUs)
+        Total Memory Usage: 2.1GB USED / 6.8GB TOTAL
+        Total Disk Usage  : 21.4GB USED / 83.2GB TOTAL
+
+ Since you're probably running locally to test and debug your searches, there is a feature (enabled by default) called interactive_failure.  If one of your detections fails, the test will pause and the offending detection will print out a message like this:
+
+
+
+ This allows you to login to your Splunk server and debug the search.  All of the uploaded data for this search remains on the server.  To continue, delete the data for this search, and move on to the next search, simply hit "Enter" in the command prompt. 
+ 
+When the test run is completed, you'll see some cleanup and summarization information.  Finally, asimple output summarizes the test run, such as:
+
+    All containers completed testing!
+    Removing all attack data that was downloaded during this test at: [/home/runner/work/security_content/security_content/bin/automated_detection_testing/ci/detection_testing_batch/attack_data_ogcm5da9]
+    Successfully removed all attack data
+    Generating test_results/success.csv...Done with [48] detections
+    Generating test_results/failure.csv...Done with [9] detections
+    Generating test_results/error.csv...Done with [0] detections
+    Generating test_results/combined.csv...Done with [57] detections
+    Settings updated.  Writing results to: test_results/detection_failure_manifest.json
+    Summary:
+        Total Tests: 57
+        Total Pass : 48
+        Total Fail : 9 (0 of these were ERRORS)
+    Test Execution Successful
+
+ Note that execution of this test will be successful if all tests complete, **even if 1 or more of the tests fail or contain errors!**
+
+## Viewing Detailed Results
+A number of helpful files are generated when the tool runs and written to the `test_results/` directory.  The most important files are:
+
+ - summary.json - A file which contains a summary of the test :
+		 - Successes, failures, and errors 
+		 - The Splunk Apps (and their versions) that were installed 
+		 - Specific Information about the Branch and Commit Hash the test was run against 
+		 - Detailed Information about each individual test, including success/failure/error information.
+		 
+ - detection_failure_manifest.json - A file which allows you to replicate your test, testing ONLY the detections that have failed.  This gives the user the chance to interactively debug these failures.  This is especially useful because it is also generated by the GitHub Actions CI Pipeline - allowing you to pull a single file and debug failed tests locally in minutes!  Because it contains specific application versions and the commit hash, this also lets you reproduce this test, exactly, at any point in the future.
+
+
+## Advanced Usage
+
+### Command Line Arguments
+There are a large number of configurable parameters for advanced users.  To view the most common parameters, simply run
+
+    python detection_testing_batch.py --help
+
+These commands will be described in more detail at a later time.
\ No newline at end of file