From 2f06facd23e2f7a52c8a7cf4850fb2c3229ab8ae Mon Sep 17 00:00:00 2001 From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com> Date: Mon, 11 Oct 2021 17:05:28 -0700 Subject: [PATCH] Lots of testing changes.... lots of progress. Need to clean up and test entire set of detections properly with sleeps and diagnose how long sleep should be or if there is a way to block while input data is processed entirely. --- .../datamodels.conf.tar | Bin 0 -> 11776 bytes .../detection_testing_execution.py | 88 ++++++++++++++---- .../modules/github_service.py | 67 +++++++++++-- .../modules/splunk_sdk.py | 36 +++++-- .../modules/testing_service.py | 22 +++-- 5 files changed, 173 insertions(+), 40 deletions(-) create mode 100644 automated_detection_testing/ci/detection_testing_batch/datamodels.conf.tar diff --git a/automated_detection_testing/ci/detection_testing_batch/datamodels.conf.tar b/automated_detection_testing/ci/detection_testing_batch/datamodels.conf.tar new file mode 100644 index 0000000000000000000000000000000000000000..d374ca68cdd4cc6c571541e0809efc5765d8ebf3 GIT binary patch literal 11776 zcmeHNTW{Mo7|pYP1%bS51>M?-ooo+z2%0{0Xog_vx(8zrXzH^?gd#Ow?6|+aA9Zmq zkYqQlqo%VA#IktgNau*(<2=6B+`8)1b@3@N}hNI!}a5xwa zjz?rLJU$wo9+2@)ud4iVZyhsuX^`@{P)gW_f4WQnWW!iEoZrU!rbr9&hZ>T({xNeb z)iFr>nrk)t&R2-koQy_I`X3z~jmD4kKRFr=4#?m;Uq|iN{m=D3tlN5rhvZy>aTfP# zw%()|{kO+B2Lv)Fv?6ue8F9wL1KevSrCvr_#*y`D%8UR@>ToGjp2^6-)C(Eg{Ojv* zmM0K<2~lQ*Ho~nCc}5a3Pr$mKV{;oV6X8G#>uRp?jAUH&dJVT4sK_!YcyW){jyWK| zTtH?34?+BE!+Dc?XGQs0J7$X;t)hyG@;1-Naew^Vn=yG+ydLTR%t++kMNXu!KBGTO zwR#AAP9bPe36xNCnrY)`W;6$Dse$~X1+x$wXt-o*SBfWi+u}1&pJ5P#RvEoZK)F)M zM%f;qUW{g|4i$@G#=LY?C=1*hpj_gk<={4`Cl*4&9AcUxM9dg>+O(F11(BdJ z`rOCt71|u!sI`(hX{iFY)aHhAou*#pB)4jjFg1s!nesLt?S%iM{`mB3qgt0U1k8YU^O*nNs|98fZB}jgNr{fu=p8HdtZABOp56t1hczH~VgWRT6nR-c z+E%g5CfZSpbg_BJ*_hr_x&ULbW~5GQj7uKVI$a3ku-aV znMlVB>zKmw2ivwA`kx168i~l6#1UsHQX|1;j5n4-cRYT$YJGe!K3wgXyZVy#g$4Jf z=QYHOcV(on=eakj#yjEpSzCjY-9~6FV0maTau&_T85usTTC48mTN*ja^Ri@jZN$5j z3E3!U7<@5mc3Bc(&4cLz;}<>^*zY+y6B4sX(`;Htt@nsFWm8BlG4?UE?dJ7*l&RqU zDbf0rPE;E8$G^Xz!gU)pj1TFDTwn!@6#!ZXIqS85dJ~kQz(;HRUWYBp*HqYG%!;`$ zyf&H%L5-e!G(HqV%L09`&k*-x$lVdIKykCm(4V49VX(chcB7Vme7bDa4##(_)$!q6apfz{G)3_m42x`TJ3!V@ao<4#T4jn93h!*$V&G&pzV1gKP~z}A8C_&*Ygs-THbYOfHN01UCAz?xfC&YhkLg==j3SsAVMVBy*mdx=a{2HWJ`@^mM!94?e N2KEf>8Tb|kz5p7CHva$s literal 0 HcmV?d00001 diff --git a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py index 07acac2a62..128f44acef 100644 --- a/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py +++ b/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py @@ -13,8 +13,14 @@ from modules import aws_service, testing_service import time import subprocess from datetime import datetime -index_file_container_path = "/opt/splunk/etc/apps/search/" + +SPLUNK_CONTAINER_APPS_DIR = "/opt/splunk/etc/apps" index_file_local_path = "indexes.conf.tar" +index_file_container_path = os.path.join(SPLUNK_CONTAINER_APPS_DIR, "search") + +datamodel_file_local_path = "datamodels.conf.tar" +datamodel_file_container_path = os.path.join(SPLUNK_CONTAINER_APPS_DIR, "Splunk_SA_CIM") + PASSWORD_LENGTH=20 MAX_RECOMMENDED_CONTAINERS_BEFORE_WARNING=2 @@ -90,6 +96,8 @@ def main(args): parser.add_argument("-i", "--reuse_images", required=False, type=bool, default=False, help="Should existing images be re-used, or should they be redownloaded?") parser.add_argument("-c", "--reuse_containers", required=False, type=bool, default=False, help="Should existing containers be re-used, or should they be rebuilt?") + parser.add_argument("-s", "--success", type=str, required=False, help="File that contains previously successful runs that we don't need to test") + args = parser.parse_args() branch = args.branch uuid_test = args.uuid @@ -97,6 +105,13 @@ def main(args): num_containers = args.num_containers reuse_containers = args.reuse_containers reuse_images = args.reuse_images + success_file = args.success_file + + if success_file is not None: + with open(success_file, "r") as successes: + success_tests = [x.strip() for x in successes.readlines()] + else: + success_tests = [] if num_containers < 1: @@ -119,11 +134,18 @@ def main(args): #aws_service.dynamo_db_nothing_to_test(REGION, uuid_test, str(int(time.time()))) sys.exit(0) print("The files to test: %s", str(test_files)) + new_test_files = [] + for f in test_files: + if f not in success_tests: + new_test_files.append(f) + else: + print("Already found [%s] in success file, not testing it again"%(f)) + test_files = new_test_files #Go into the security content directory print("****GENERATE NEW CONTENT****") os.chdir("security_content") - commands = ["python3 -m venv .venv", "source .venv/bin/activate", "python3 -m pip install -r requirements.txt", "python3 contentctl.py --path . --verbose generate --product ESCU --output dist/escu"] + commands = ["python3 -m venv .venv", "source .venv/bin/activate", "python3 -m pip install wheel", "python3 -m pip install -r requirements.txt", "python3 contentctl.py --path . --verbose generate --product ESCU --output dist/escu"] ret = subprocess.run("; ".join(commands), shell=True, capture_output=False) if ret.returncode != 0: print("Error generating new content. Exiting...") @@ -167,7 +189,6 @@ def main(args): test_file_queue.put(filename) # print("***Test files enqueued") - testing_service.test_detection_wrapper(None, None, splunk_password, None, test_file_queue.get(), 0%1, uuid_test) # print("Getting docker client") client = docker.client.from_env() @@ -310,16 +331,28 @@ def main(args): results_queue = queue.Queue() success_names_queue = queue.Queue() failure_names_queue = queue.Queue() + + + for container_index in range(num_containers): container_name = "%s_%d"%(BASE_CONTAINER_NAME, container_index) web_port = BASE_CONTAINER_WEB_PORT + container_index management_port = BASE_CONTAINER_MANAGEMENT_PORT + container_index + SPLUNK_COMMON_INFORMATION_MODEL = "https://splunkbase.splunk.com/app/1621/release/4.20.2/download" + SPLUNK_SECURITY_ESSENTIALS = "https://splunkbase.splunk.com/app/3435/release/3.3.4/download" + #SPLUNK_ADD_ON_FOR_SYSMON = "https://splunkbase.splunk.com/app/5709/release/1.0.1/download" + SPLUNK_ADD_ON_FOR_SYSMON = "https://splunkbase.splunk.com/app/1914/release/10.6.2/download" + SYSMON_APP_FOR_SPLUNK = "https://splunkbase.splunk.com/app/3544/release/2.0.0/download" + SPLUNK_ES_CONTENT_UPDATE = "https://splunkbase.splunk.com/app/3449/release/3.29.0/download" + + SPLUNK_APPS = [SPLUNK_COMMON_INFORMATION_MODEL, SPLUNK_SECURITY_ESSENTIALS, SPLUNK_ADD_ON_FOR_SYSMON, SYSMON_APP_FOR_SPLUNK, SPLUNK_ES_CONTENT_UPDATE] + #docker run -p8089:8089 -p 8000:8000 -e "SPLUNK_START_ARGS=--accept-license" -e "SPLUNK_PASSWORD=123456qwertyQWERTY" -e "SPLUNK_APPS_URL=https://splunkbase.splunk.com/app/3435/release/3.3.4/download,https://splunkbase.splunk.com/app/5709/release/1.0.1/download,https://splunkbase.splunk.com/app/3449/release/3.29.0/download,https://splunkbase.splunk.com/app/1621/release/4.20.2/download" -e "SPLUNKBASE_USERNAME=ericmcginnistwo" -e "SPLUNKBASE_PASSWORD=splunkSecondAccount5@" -name splunktemplate splunk/splunk:latest environment = {"SPLUNK_START_ARGS": "--accept-license", "SPLUNK_PASSWORD" : splunk_password, - "SPLUNK_APPS_URL" : "https://splunkbase.splunk.com/app/3435/release/3.3.4/download,https://splunkbase.splunk.com/app/5709/release/1.0.1/download,https://splunkbase.splunk.com/app/3449/release/3.29.0/download,https://splunkbase.splunk.com/app/1621/release/4.20.2/download", + "SPLUNK_APPS_URL" : ','.join(SPLUNK_APPS), "SPLUNKBASE_USERNAME" : "emcginnistwo", "SPLUNKBASE_PASSWORD" : "splunkSecondAccount5@" } @@ -360,6 +393,10 @@ def main(args): print("DONE!") #read all the results out from the output queue + strtime = str(int(time.time())) + #write success and failure + success_output = open("success_%s"%(strtime), "w") + failure_output = open("failure_%s"%(strtime), "w") try: while True: @@ -391,6 +428,23 @@ def queue_status_thread(total_tests_count, testing_queue, results_queue, success else: time.sleep(10) +def copy_file_to_container(localFilePath, remoteFilePath, containerName, sleepTimeSeconds=5): + successful_copy = False + #need to use the low level client to put a file onto a container + apiclient = docker.APIClient() + while not successful_copy: + try: + with open(localFilePath,"rb") as fileData: + #splunk will restart a few times will installation of apps takes place so it will reload its indexes... + apiclient.put_archive(container=containerName, path=remoteFilePath, data=fileData) + successful_copy=True + except Exception as e: + print("Failed copy of [%s] file to CONTAINER:[%s]...we will try again"%(localFilePath, containerName)) + time.sleep(10) + successful_copy=False + print("Successfully copied [%s] to [%s] on [%s]"%(localFilePath, remoteFilePath, containerName)) + + def splunk_container_manager(testing_queue, container_name, splunk_ip, splunk_password, splunk_port, uuid_test, results_queue, success_names_queue, failure_names_queue): print("Starting the container [%s] after a sleep"%(container_name)) #Is this going to be safe to use in different threads @@ -405,25 +459,22 @@ def splunk_container_manager(testing_queue, container_name, splunk_ip, splunk_pa container = client.containers.get(container_name) print("Starting the container [%s]"%(container_name)) - #need to use the low level client to put a file onto a container - apiclient = docker.APIClient() + container.start() - successful_copy = False - while not successful_copy: - try: - with open(index_file_local_path,"rb") as indexData: - #splunk will restart a few times will installation of apps takes place so it will reload its indexes... - time.sleep(10) - apiclient.put_archive(container=container_name, path=index_file_container_path, data=indexData) - successful_copy=True - except Exception as e: - print("Failed copy of index file to CONTAINER:[%s]...we will try again"%(container_name)) - successful_copy=False - + print("Start copying files to container") + copy_file_to_container(index_file_local_path, index_file_container_path, container_name) + copy_file_to_container(datamodel_file_local_path, datamodel_file_container_path, container_name) + print("Finished copying files to container!") wait_for_splunk_ready(max_seconds=120) + from modules.splunk_sdk import enable_delete_for_admin + if not enable_delete_for_admin(splunk_ip, splunk_port, splunk_password): + print("COULD NOT ENABLE DELETE FOR [%s].... quitting"%(container_name)) + sys.exit(0) + + print("Successfully enabled DELETE for [%s]"%(container_name)) index=0 try: @@ -431,7 +482,6 @@ def splunk_container_manager(testing_queue, container_name, splunk_ip, splunk_pa #Try to get something from the queue detection_to_test = testing_queue.get(block=False) - #There is a detection to test print("Container [%s]--->[%s]"%(container_name, detection_to_test)) try: diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/github_service.py b/automated_detection_testing/ci/detection_testing_batch/modules/github_service.py index 418f53460d..d113ae6d54 100644 --- a/automated_detection_testing/ci/detection_testing_batch/modules/github_service.py +++ b/automated_detection_testing/ci/detection_testing_batch/modules/github_service.py @@ -4,7 +4,7 @@ import os import logging import glob import subprocess - +import yaml # Logger logging.basicConfig(level=os.environ.get("LOGLEVEL", "INFO")) @@ -33,7 +33,7 @@ class GithubService: branch2 = 'develop' g = git.Git('security_content') changed_test_files = [] - + changed_detection_files = [] if branch1 != 'develop': differ = g.diff('--name-status', branch2 + '...' + branch1) changed_files = differ.splitlines() @@ -49,12 +49,67 @@ class GithubService: # changed detections if 'detections' in file_path: if not os.path.basename(file_path).startswith('ssa') and os.path.basename(file_path).endswith('.yml'): - file_path_base = os.path.splitext(file_path)[0].replace('detections', 'tests') + '.test' - file_path_new = file_path_base + '.yml' - if file_path_new not in changed_test_files: - changed_test_files.append(file_path_new) + changed_detection_files.append(file_path) + #file_path_base = os.path.splitext(file_path)[0].replace('detections', 'tests') + '.test' + #file_path_new = file_path_base + '.yml' + #if file_path_new not in changed_test_files: + # changed_test_files.append(file_path_new) + #all files have the format A\tFILENAME or M\tFILENAME. Get rid of those leading characters + changed_test_files = [name.split('\t')[1] for name in changed_test_files if len(name.split('\t')) == 2] + changed_detection_files = [name.split('\t')[1] for name in changed_detection_files if len(name.split('\t')) == 2] + + + detections_to_test,_,_ = self.filter_test_types(changed_detection_files) + for f in detections_to_test: + file_path_base = os.path.splitext(f)[0].replace('detections', 'tests') + '.test' + file_path_new = file_path_base + '.yml' + if file_path_new not in changed_test_files: + changed_test_files.append(file_path_new) + + + + + + print("Total things to test (test files and detection files changed): [%d]"%(len(changed_test_files))) + #for l in changed_test_files: + # print(l) + #print(len(changed_test_files)) + import time + time.sleep(5) return changed_test_files + def filter_test_types(self, test_files, test_types = ["Anomaly", "Hunting", "TTP"]): + files_to_test = [] + files_not_to_test = [] + error_files = [] + for filename in test_files: + try: + with open(os.path.join("security_content", filename), "r") as fileData: + yaml_dict = list(yaml.safe_load_all(fileData))[0] + if 'type' not in yaml_dict.keys(): + print("Failed to find 'type' in the yaml for: [%s]"%(filename)) + error_files.append(filename) + if yaml_dict['type'] in test_types: + files_to_test.append(filename) + else: + files_not_to_test.append(filename) + except Exception as e: + print("Error on trying to scan [%s]: [%s]"%(filename, str(e))) + error_files.append(filename) + print("***Detection Information***\n"\ + "\tTotal Files : %d" + "\tFiles to test : %d" + "\tFiles not to test : %d" + "\tError files : %d"%(len(test_files), len(files_to_test), len(files_not_to_test), len(error_files))) + import time + time.sleep(5) + return files_to_test, files_not_to_test, error_files + + + + + + diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/splunk_sdk.py b/automated_detection_testing/ci/detection_testing_batch/modules/splunk_sdk.py index adfd175baf..4ea12edd39 100644 --- a/automated_detection_testing/ci/detection_testing_batch/modules/splunk_sdk.py +++ b/automated_detection_testing/ci/detection_testing_batch/modules/splunk_sdk.py @@ -5,6 +5,31 @@ import splunklib.client as client import splunklib.results as results import requests + +def enable_delete_for_admin(splunk_host, splunk_port, splunk_password): + try: + service = client.connect( + host=splunk_host, + port=splunk_port, + username='admin', + password=splunk_password + ) + except Exception as e: + print("Unable to connect to Splunk instance: " + str(e)) + return 1, {} + + # search and replace \\ with \\\ + # search = search.replace('\\','\\\\') + role = service.roles['admin'] + try: + role.grant('delete_by_keyword') + except Exception as e: + print("Error - failed trying to grant 'can_delete' privs to admin: [%s]"%(str(e))) + return False + return True + + + def test_baseline_search(splunk_host, splunk_port, splunk_password, search, pass_condition, baseline_name, baseline_file, earliest_time, latest_time): try: service = client.connect( @@ -82,8 +107,8 @@ def test_detection_search(splunk_host, splunk_port, splunk_password, search, pas splunk_search = search + ' ' + pass_condition print("SEARCH:") print(splunk_search) - print("Sleep for 30 seconds") - sleep(30) + #print("Sleep for 30 seconds") + #sleep(30) try: job = service.jobs.create(splunk_search, **kwargs) except Exception as e: @@ -109,8 +134,7 @@ def test_detection_search(splunk_host, splunk_port, splunk_password, search, pas def delete_attack_data(splunk_host, splunk_password, splunk_port): - #print("DO NOT DELETE ANYTHING!") - #return None + print("Deleting test data!") try: service = client.connect( host=splunk_host, @@ -122,8 +146,8 @@ def delete_attack_data(splunk_host, splunk_password, splunk_port): print("Unable to connect to Splunk instance: " + str(e)) return 1, {} - splunk_search = 'search index=test* | delete' - + #splunk_search = 'search index=test* | delete' + splunk_search = 'search index=main | delete' kwargs = {"exec_mode": "blocking", "dispatch.earliest_time": "-1d", "dispatch.latest_time": "now"} diff --git a/automated_detection_testing/ci/detection_testing_batch/modules/testing_service.py b/automated_detection_testing/ci/detection_testing_batch/modules/testing_service.py index 47dda5892b..6ba2fef846 100644 --- a/automated_detection_testing/ci/detection_testing_batch/modules/testing_service.py +++ b/automated_detection_testing/ci/detection_testing_batch/modules/testing_service.py @@ -28,8 +28,9 @@ def test_detection_wrapper(container_name, splunk_ip, splunk_password, splunk_po result_test = test_detection(splunk_ip, splunk_port, container_name, splunk_password, test_file, test_index, uuid_test, uuid_var) + #enter = input("Run some tests from [%s] on [%s] - we don't delete until you hit enter :)"%(container_name, test_file)) # delete test data - #splunk_sdk.delete_attack_data(splunk_ip, splunk_password, splunk_port) + splunk_sdk.delete_attack_data(splunk_ip, splunk_password, splunk_port) if result_test['detection_result']['error']: @@ -42,7 +43,7 @@ def test_detection_wrapper(container_name, splunk_ip, splunk_password, splunk_po def test_detection(splunk_ip, splunk_port, container_name, splunk_password, test_file, test_index, uuid_test, uuid_var): try: - test_file_obj = load_file("security_content/" + test_file[2:]) + test_file_obj = load_file(os.path.join("security_content/", test_file)) except Exception as e: raise #print('Error: ' + str(e)) @@ -64,18 +65,21 @@ def test_detection(splunk_ip, splunk_port, container_name, splunk_password, test for attack_data in test_file_obj['tests'][0]['attack_data']: url = attack_data['data'] r = requests.get(url, allow_redirects=True) - open(folder_name + '/' + attack_data['file_name'], 'wb').write(r.content) - print(folder_name + '/' + attack_data['file_name']) + target_file = os.path.join(folder_name, attack_data['file_name']) + with open(target_file, 'wb') as target: + target.write(r.content) + print(target_file) # Update timestamps before replay if 'update_timestamp' in attack_data: if attack_data['update_timestamp'] == True: data_manipulation = DataManipulation() - data_manipulation.manipulate_timestamp(folder_name + '/' + attack_data['file_name'], attack_data['sourcetype'], attack_data['source']) - - replay_attack_dataset(container_name, splunk_password, folder_name, 'test' + str(test_index), attack_data['sourcetype'], attack_data['source'], attack_data['file_name']) + data_manipulation.manipulate_timestamp(target_file, attack_data['sourcetype'], attack_data['source']) + INDEX_TO_REPLAY_INTO = 'test' + str(test_index) + INDEX_TO_REPLAY_INTO = 'main' + replay_attack_dataset(container_name, splunk_password, folder_name, INDEX_TO_REPLAY_INTO, attack_data['sourcetype'], attack_data['source'], attack_data['file_name']) print("START SLEEP AFTER REPLAY") - time.sleep(30) + time.sleep(60) print("DONE SLEEP AFTER REPLAY") result_test = {} test = test_file_obj['tests'][0] @@ -120,7 +124,7 @@ def test_detection(splunk_ip, splunk_port, container_name, splunk_password, test def load_file(file_path): try: print("Opening file path [%s]"%(file_path)) - sys.exit(0) + with open(file_path, 'r', encoding="utf-8") as stream: try: file = list(yaml.safe_load_all(stream))[0]