From 06a9c0f15ef82989fa00a1c82708a27bdd385fd6 Mon Sep 17 00:00:00 2001 From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com> Date: Mon, 11 Oct 2021 10:23:42 -0700 Subject: [PATCH] Lots more changes to how we run. Now based off of the splunk/splunk container on docker hub as much as possible. We use ENV arguments to install required apps. Still only at 50% pass rate, so we need more troubleshooting to figure out why. --- .../detection_testing_execution.py | 350 +++++++++++------- .../detection_testing_batch/indexes.conf.tar | Bin 0 -> 2560 bytes .../modules/splunk_sdk.py | 9 +- .../modules/testing_service.py | 135 ++++++- 4 files changed, 355 insertions(+), 139 deletions(-) create mode 100644 automated_detection_testing/ci/detection_testing_batch/indexes.conf.tar diff --git a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py index 7e6e71316a..07acac2a62 100644 --- a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py +++ b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py @@ -11,16 +11,20 @@ import queue from modules.github_service import GithubService from modules import aws_service, testing_service import time - - +import subprocess +from datetime import datetime +index_file_container_path = "/opt/splunk/etc/apps/search/" +index_file_local_path = "indexes.conf.tar" PASSWORD_LENGTH=20 MAX_RECOMMENDED_CONTAINERS_BEFORE_WARNING=2 DOCKER_HUB_CONTAINER_PATH="splunk/splunk:latest" BASE_CONTAINER_NAME="splunk" -DOCKER_COMMIT_NAME = "splunk_configured" -RUNNER_BASE_NAME = "splunk_runner" + + +#DOCKER_COMMIT_NAME = "splunk_configured" +#RUNNER_BASE_NAME = "splunk_runner" BASE_CONTAINER_WEB_PORT=8000 @@ -76,7 +80,7 @@ def stop_container(docker_client, container_name, force=True): def main(args): - + start_time = datetime.now() parser = argparse.ArgumentParser(description="CI Detection Testing") parser.add_argument("-b", "--branch", type=str, required=True, help="security content branch") parser.add_argument("-u", "--uuid", type=str, required=True, help="uuid for detection test") @@ -101,8 +105,8 @@ def main(args): sys.exit(1) elif num_containers > MAX_RECOMMENDED_CONTAINERS_BEFORE_WARNING: print("You requested to run with [%d] containers which may use a very large amount of resources " - "as they all run in parallel. The maximum suggested number of parallel. The maximum " - "suggested number of containers is [%d]. We will do what you asked, but be warned!"%(num_containers, MAX_RECOMMENDED_CONTAINERS_BEFORE_WARNING)) + "as they all run in parallel. The maximum suggested number of parallel containers is " + "[%d]. We will do what you asked, but be warned!"%(num_containers, MAX_RECOMMENDED_CONTAINERS_BEFORE_WARNING)) if pr_number: @@ -114,7 +118,30 @@ def main(args): print("No new detections to test.") #aws_service.dynamo_db_nothing_to_test(REGION, uuid_test, str(int(time.time()))) sys.exit(0) + print("The files to test: %s", str(test_files)) + + #Go into the security content directory + print("****GENERATE NEW CONTENT****") + os.chdir("security_content") + commands = ["python3 -m venv .venv", "source .venv/bin/activate", "python3 -m pip install -r requirements.txt", "python3 contentctl.py --path . --verbose generate --product ESCU --output dist/escu"] + ret = subprocess.run("; ".join(commands), shell=True, capture_output=False) + if ret.returncode != 0: + print("Error generating new content. Exiting...") + sys.exit(1) + print("New content generated successfully") + os.chdir("..") + client = docker.client.from_env() + splunk_password = '123456qwertyQWERTY' + + + + + + + #print("run it now!") + + #time.sleep(180) #dt_ar = aws_service.get_ar_information_from_dynamo_db(REGION, DT_ATTACK_RANGE_STATE) #splunk_instance = aws_service.get_splunk_instance(REGION, dt_ar['ssh_key_name']) @@ -129,170 +156,202 @@ def main(args): #splunk_password = secrets.token_urlsafe(PASSWORD_LENGTH) #Only accessible on local host, it's okay to expose the password for debugging - splunk_password = "123456qwerty!@#$%^QWERTY" + #splunk_password = "123456qwerty!@#$%^QWERTY" + #splunk_password = '123456qwerty%^QWERTY' splunk_container_manager_threads = [] - print("***Files to test: %d"%(len(test_files))) + # print("***Files to test: %d"%(len(test_files))) test_file_queue = queue.Queue() for filename in test_files: test_file_queue.put(filename) - print("***Test files enqueued") + # print("***Test files enqueued") - print("Getting docker client") + testing_service.test_detection_wrapper(None, None, splunk_password, None, test_file_queue.get(), 0%1, uuid_test) + + # print("Getting docker client") client = docker.client.from_env() + + # try: + # print("Removing any existing containers called [%s]."%(BASE_CONTAINER_NAME)) - try: - print("Removing any existing containers called [%s]."%(BASE_CONTAINER_NAME)) - - c = client.containers.get(BASE_CONTAINER_NAME) - except: - print("Container [%s] did not exist. No need to remove it. It will; be created for you."%(BASE_CONTAINER_NAME)) - c = None + # c = client.containers.get(BASE_CONTAINER_NAME) + # except: + # print("Container [%s] did not exist. No need to remove it. It will; be created for you."%(BASE_CONTAINER_NAME)) + # c = None - if (c and reuse_containers): - print("Found a container called [%s]. NOT removing it because you have specified --reuse_containers [%s]. " - "However, we must stop the container. Stopping it now..."%(BASE_CONTAINER_NAME, reuse_containers)) - stop_container(client, BASE_CONTAINER_NAME) + # if (c and reuse_containers): + # print("Found a container called [%s]. NOT removing it because you have specified --reuse_containers [%s]. " + # "However, we must stop the container. Stopping it now..."%(BASE_CONTAINER_NAME, reuse_containers)) + # stop_container(client, BASE_CONTAINER_NAME) - elif c: - print("Found a container called [%s]. Removing it because you have specified --reuse_containers [%s]"%(BASE_CONTAINER_NAME, reuse_containers)) - remove_container(client, BASE_CONTAINER_NAME) + # elif c: + # print("Found a container called [%s]. Removing it because you have specified --reuse_containers [%s]"%(BASE_CONTAINER_NAME, reuse_containers)) + # remove_container(client, BASE_CONTAINER_NAME) - download_image = False - try: - client.images.get(DOCKER_HUB_CONTAINER_PATH) - if reuse_images: - print("You already have an image named [%s]."%(DOCKER_HUB_CONTAINER_PATH)) - download_image = False - else: - print("You already have an image named [%s]., " - "but have speicified --reuse_images %s"%(DOCKER_HUB_CONTAINER_PATH, reuse_images)) - download_image = True + # download_image = False + # try: + # client.images.get(DOCKER_HUB_CONTAINER_PATH) + # if reuse_images: + # print("You already have an image named [%s]."%(DOCKER_HUB_CONTAINER_PATH)) + # download_image = False + # else: + # print("You already have an image named [%s]., " + # "but have speicified --reuse_images %s"%(DOCKER_HUB_CONTAINER_PATH, reuse_images)) + # download_image = True - except: - print("You did not have an image named [%s]."%(DOCKER_HUB_CONTAINER_PATH)) - download_image = True + # except: + # print("You did not have an image named [%s]."%(DOCKER_HUB_CONTAINER_PATH)) + # download_image = True - if download_image: - try: - print("Downloading image [%s]. Please note " - "that this could take a long time depending on your " - "connection. It's around 2GB."%(DOCKER_HUB_CONTAINER_PATH)) - client.images.pull(DOCKER_HUB_CONTAINER_PATH) - print("Finished downloading the image [%s]"%(DOCKER_HUB_CONTAINER_PATH)) - except Exception as e: - print("Unrecoverable error downloading image [%s]:[%s]"%(DOCKER_HUB_CONTAINER_PATH, str(e))) - sys.exit(1) + # if download_image: + # try: + # print("Downloading image [%s]. Please note " + # "that this could take a long time depending on your " + # "connection. It's around 2GB."%(DOCKER_HUB_CONTAINER_PATH)) + # client.images.pull(DOCKER_HUB_CONTAINER_PATH) + # print("Finished downloading the image [%s]"%(DOCKER_HUB_CONTAINER_PATH)) + # except Exception as e: + # print("Unrecoverable error downloading image [%s]:[%s]"%(DOCKER_HUB_CONTAINER_PATH, str(e))) + # sys.exit(1) - remove_tag = False - try: - image = client.images.get(DOCKER_COMMIT_NAME) - print("Found an image called [%s]"%(DOCKER_COMMIT_NAME)) - if reuse_images == False: - print("We will remove the image [%s] because you have specificed --reuse_images %s"%(DOCKER_COMMIT_NAME, reuse_images)) - remove_tag = True - build_Tag = True - else: - print("We will use the preexisting image for [%s]"%(DOCKER_COMMIT_NAME)) - build_tag = False - except: - print("No image found named [%s]"%(DOCKER_COMMIT_NAME)) - build_tag = True + # remove_tag = False + # try: + # image = client.images.get(DOCKER_COMMIT_NAME) + # print("Found an image called [%s]"%(DOCKER_COMMIT_NAME)) + # if reuse_images == False: + # print("We will remove the image [%s] because you have specificed --reuse_images %s"%(DOCKER_COMMIT_NAME, reuse_images)) + # remove_tag = True + # build_Tag = True + # else: + # print("We will use the preexisting image for [%s]"%(DOCKER_COMMIT_NAME)) + # build_tag = False + # except: + # print("No image found named [%s]"%(DOCKER_COMMIT_NAME)) + # build_tag = True - if remove_tag: - try: - #Stop it if it's running, remove associated volumes too - client.images.remove(image=DOCKER_COMMIT_NAME, force=True) + # if remove_tag: + # try: + # #Stop it if it's running, remove associated volumes too + # client.images.remove(image=DOCKER_COMMIT_NAME, force=True) - except Exception as e: - print("Unrecoverable error removing [%s]: [%s]"%(DOCKER_COMMIT_NAME, str(e))) - sys.exit(1) + # except Exception as e: + # print("Unrecoverable error removing [%s]: [%s]"%(DOCKER_COMMIT_NAME, str(e))) + # sys.exit(1) + # for ind in range(num_containers): + # ind = str(ind) + # if not reuse_containers: + # try: + # print("Creating a new container called [%s]"%(BASE_CONTAINER_NAME+ind)) + - if not reuse_containers: - try: - print("Creating a new container called [%s]"%(BASE_CONTAINER_NAME)) + # environment = {"SPLUNK_START_ARGS": "--accept-license", + # "SPLUNK_PASSWORD" : splunk_password } + # ports= {"8000/tcp": BASE_CONTAINER_WEB_PORT - 1 + int(ind) + 1, + # "8089/tcp": BASE_CONTAINER_MANAGEMENT_PORT - 1 + int(ind) + 1 + # } + # base_container = client.containers.create("splunk/splunk:latest", ports=ports, environment=environment, name=BASE_CONTAINER_NAME+ind, detach=True) + # print("Running the new container called [%s]"%(BASE_CONTAINER_NAME+ind)) + # base_container.start() + # print("Container is running [%s]"%(BASE_CONTAINER_NAME+ind)) + # print("Sleep for 60 seconds to allow the container to fully start up...") + # wait_for_splunk_ready(max_seconds=60) + # print("The container has fully started!") + + # print("Do the ESCU installation on this container. That way we don't have to " + # "do it on every container that we then spin up.") + + # testing_service.prepare_detection_testing(BASE_CONTAINER_NAME+ind, splunk_password) + # print("Waiting for a few seconds for the splunk app to come up.") + # wait_for_splunk_ready(max_seconds=30) + # print("Install the apps and enable accelerate") + # wait_for_splunk_ready(max_seconds=180) + # print("Stopping the running container [%s]"%(BASE_CONTAINER_NAME+ind)) + # base_container.stop() + # #I am almost positive that I'm doing this wrong but it works for now... + + # #print("Committing the configured container: [%s]--->[%s]"%(BASE_CONTAINER_NAME, DOCKER_COMMIT_NAME)) + # #base_container.commit(repository=DOCKER_COMMIT_NAME) - environment = {"SPLUNK_START_ARGS": "--accept-license", - "SPLUNK_PASSWORD" : splunk_password } - ports= {"8000/tcp": BASE_CONTAINER_WEB_PORT - 1, - "8089/tcp": BASE_CONTAINER_MANAGEMENT_PORT - 1 - } - base_container = client.containers.create("splunk/splunk:latest", ports=ports, environment=environment, name=BASE_CONTAINER_NAME, detach=True) - print("Running the new container called [%s]"%(BASE_CONTAINER_NAME)) - base_container.start() - print("Container is running [%s]"%(BASE_CONTAINER_NAME)) - print("Sleep for 60 seconds to allow the container to fully start up...") - wait_for_splunk_ready(max_seconds=60) - print("The container has fully started!") - - print("Do the ESCU installation on this container. That way we don't have to " - "do it on every container that we then spin up.") - - testing_service.prepare_detection_testing(BASE_CONTAINER_NAME, splunk_password) - print("Waiting for a few seconds for the splunk app to come up.") - wait_for_splunk_ready(max_seconds=30) - - print("Stopping the running container [%s]"%(BASE_CONTAINER_NAME)) - base_container.stop() - #I am almost positive that I'm doing this wrong but it works for now... - - print("Committing the configured container: [%s]--->[%s]"%(BASE_CONTAINER_NAME, DOCKER_COMMIT_NAME)) - base_container.commit(repository=DOCKER_COMMIT_NAME) - - - except Exception as e: - print("There was an error getting the base container up and running. " - "We cannot recover from this: [%s]\nGoodbye..."%(str(e))) - sys.exit(1) + # except Exception as e: + # print("There was an error getting the base container up and running. " + # "We cannot recover from this: [%s]\nGoodbye..."%(str(e))) + # sys.exit(1) - #The part below does not seem to be working as expected. Will need to look into it - #When I create the new container, it fails to boot with - # The CA file specified (/opt/splunk/etc/auth/cacert.pem) does not exist. Cannot continue. - # SSL certificate generation failed. + # # # #The part below does not seem to be working as expected. Will need to look into it + # # # #When I create the new container, it fails to boot with + # # # # The CA file specified (/opt/splunk/etc/auth/cacert.pem) does not exist. Cannot continue. + # # # # SSL certificate generation failed. - # MSG: + # # # # MSG: - # non-zero return code + # # # # non-zero return code + - - print("Make all the threads...") + print("The number of detections we will test is [%d]"%(test_file_queue.qsize())) + results_queue = queue.Queue() + success_names_queue = queue.Queue() + failure_names_queue = queue.Queue() for container_index in range(num_containers): - container_name = "%s_%d"%(RUNNER_BASE_NAME, container_index) + container_name = "%s_%d"%(BASE_CONTAINER_NAME, container_index) + web_port = BASE_CONTAINER_WEB_PORT + container_index management_port = BASE_CONTAINER_MANAGEMENT_PORT + container_index - print("Creating a new container called [%s]"%(container_name)) + + #docker run -p8089:8089 -p 8000:8000 -e "SPLUNK_START_ARGS=--accept-license" -e "SPLUNK_PASSWORD=123456qwertyQWERTY" -e "SPLUNK_APPS_URL=https://splunkbase.splunk.com/app/3435/release/3.3.4/download,https://splunkbase.splunk.com/app/5709/release/1.0.1/download,https://splunkbase.splunk.com/app/3449/release/3.29.0/download,https://splunkbase.splunk.com/app/1621/release/4.20.2/download" -e "SPLUNKBASE_USERNAME=ericmcginnistwo" -e "SPLUNKBASE_PASSWORD=splunkSecondAccount5@" -name splunktemplate splunk/splunk:latest environment = {"SPLUNK_START_ARGS": "--accept-license", - "SPLUNK_PASSWORD" : splunk_password } + "SPLUNK_PASSWORD" : splunk_password, + "SPLUNK_APPS_URL" : "https://splunkbase.splunk.com/app/3435/release/3.3.4/download,https://splunkbase.splunk.com/app/5709/release/1.0.1/download,https://splunkbase.splunk.com/app/3449/release/3.29.0/download,https://splunkbase.splunk.com/app/1621/release/4.20.2/download", + "SPLUNKBASE_USERNAME" : "emcginnistwo", + "SPLUNKBASE_PASSWORD" : "splunkSecondAccount5@" + } ports= {"8000/tcp": web_port, "8089/tcp": management_port - } + } - test_container = client.containers.create(DOCKER_COMMIT_NAME, ports=ports, environment=environment, name=container_name, detach=True, volumes_from=[BASE_CONTAINER_NAME]) - t = threading.Thread(target=splunk_container_manager, args=(test_file_queue, container_name, "127.0.0.1", splunk_password, management_port, uuid_test, results_queue)) + print("Creating CONTAINER: [%s]"%(container_name)) + base_container = client.containers.create(DOCKER_HUB_CONTAINER_PATH, ports=ports, environment=environment, name=container_name, detach=True) + print("Created CONTAINER : [%s]"%(container_name)) + #print("Creating a new container called [%s]"%(container_name)) + #environment = {"SPLUNK_START_ARGS": "--accept-license", + # "SPLUNK_PASSWORD" : splunk_password } + #ports= {"8000/tcp": web_port, + # "8089/tcp": management_port + # } + + #test_container = client.containers.create(DOCKER_COMMIT_NAME, ports=ports, environment=environment, name=container_name, detach=True, volumes_from=[BASE_CONTAINER_NAME]) + + t = threading.Thread(target=splunk_container_manager, args=(test_file_queue, container_name, "127.0.0.1", splunk_password, management_port, uuid_test, results_queue, success_names_queue, failure_names_queue)) splunk_container_manager_threads.append(t) + #add the queue status thread - there can be some error in one of the test threads, so this + #thread doesn't need to complete for the program to finish execution + status_thread = threading.Thread(target=queue_status_thread, args=(test_file_queue.qsize(), test_file_queue, results_queue, success_names_queue, failure_names_queue), daemon=True) + status_thread.start() print("Start all the threads...") for t in splunk_container_manager_threads: t.start() + #we need to start containers slowly. Would be great it we could do all the setup and + #app install once (with Dockerfile?) + time.sleep(60) #Try to join all the threads for t in splunk_container_manager_threads: @@ -305,38 +364,68 @@ def main(args): while True: o = results_queue.get(block=False) - print("Got from queue:") print(o) except queue.Empty: print("That's all the output!") #now we are done! - + stop_time = datetime.now() + print("Total Execution Time: [%s]"%(stop_time-start_time)) + #detection testing service has already been prepared, no need to do it here! #testing_service.prepare_detection_testing(ssh_key_name, private_key, splunk_ip, splunk_password) #testing_service.test_detections(ssh_key_name, private_key, splunk_ip, splunk_password, test_files, uuid_test) +def queue_status_thread(total_tests_count, testing_queue, results_queue, success_names_queue, failure_names_queue): + while True: + print("***Progress Update:\n"\ + "\tTests to run : %d\n"\ + "\tTests currently running: %d\n"\ + "\tTests completed : %d\n"\ + "\t\tSuccess : %d\n"\ + "\t\tFailure : %d"%(testing_queue.qsize(), total_tests_count - testing_queue.qsize() - results_queue.qsize(), results_queue.qsize(), success_names_queue.qsize(), failure_names_queue.qsize())) + if results_queue.qsize() == total_tests_count: + return + else: + time.sleep(10) -def splunk_container_manager(testing_queue, container_name, splunk_ip, splunk_password, splunk_port, uuid_test, results_queue): +def splunk_container_manager(testing_queue, container_name, splunk_ip, splunk_password, splunk_port, uuid_test, results_queue, success_names_queue, failure_names_queue): print("Starting the container [%s] after a sleep"%(container_name)) #Is this going to be safe to use in different threads client = docker.client.from_env() + #start up the container from the base container #Assume that the base container has already been fully built with #escu etc #sleep for a little bit so that we don't all start at once... - time.sleep(random.randrange(0,120)) + time.sleep(random.randrange(0,60)) container = client.containers.get(container_name) print("Starting the container [%s]"%(container_name)) + + #need to use the low level client to put a file onto a container + apiclient = docker.APIClient() + container.start() - wait_for_splunk_ready(max_seconds=60) + successful_copy = False + while not successful_copy: + try: + with open(index_file_local_path,"rb") as indexData: + #splunk will restart a few times will installation of apps takes place so it will reload its indexes... + time.sleep(10) + apiclient.put_archive(container=container_name, path=index_file_container_path, data=indexData) + successful_copy=True + except Exception as e: + print("Failed copy of index file to CONTAINER:[%s]...we will try again"%(container_name)) + successful_copy=False + + + + wait_for_splunk_ready(max_seconds=120) index=0 - print("Inspect your containers, you have 5 minutes!") - wait_for_splunk_ready(max_seconds=60) try: while True: #Try to get something from the queue @@ -346,9 +435,12 @@ def splunk_container_manager(testing_queue, container_name, splunk_ip, splunk_pa #There is a detection to test print("Container [%s]--->[%s]"%(container_name, detection_to_test)) try: - pass - #result = testing_service.test_detection_wrapper(container_name, splunk_ip, splunk_password, splunk_port, detection_to_test, index, uuid_test) - #results_queue.put(result) + result = testing_service.test_detection_wrapper(container_name, splunk_ip, splunk_password, splunk_port, detection_to_test, index%1, uuid_test) + if result['detection_result']['error']: + failure_names_queue.put(result['detection_result']['detection_name']) + else: + success_names_queue.put(result['detection_result']['detection_name']) + results_queue.put(result) except Exception as e: print("Caught some exception in test detection: [%s]"%(str(e))) #just log the error itself for now so that we can continue @@ -360,7 +452,7 @@ def splunk_container_manager(testing_queue, container_name, splunk_ip, splunk_pa print("Shutting down the container [%s]"%(container_name)) container.stop() - print("Finished shutting down the container [%s]"&(container_name)) + print("Finished shutting down the container [%s]"%(container_name)) if __name__ == "__main__": main(sys.argv[1:]) \ No newline at end of file diff --git a/automated_detection_testing/ci/detection_testing_batch/indexes.conf.tar b/automated_detection_testing/ci/detection_testing_batch/indexes.conf.tar new file mode 100644 index 0000000000000000000000000000000000000000..813a484efefb92eddfc9f2c81965778d79e48461 GIT binary patch literal 2560 zcmeHF%WA_g5cJt!p+nEL{E%|$CC#PuL1@xbDMhh2u|Sr<-cYw+-}QqYN*zcb5U6w+ z(Cp5xW@KEhOcb2;aL#8!u%JUJd*n$rAl-F`#aW#4I31g9l14i86iu+7{44bDcYT=#3{XPS(h5$otRyQ!anyFNB;0 zjGs8~sJ4He|0_a+{4LC#DV7RLcEhe#%ZHbz`}f5>>dC0%icJVsZ44|FsymCYt2KV! zI*YY4)Wk=6-RPoyhGL(icD5%