From d657aec86e726a6f9414bfe0ae1e8e03d64d8795 Mon Sep 17 00:00:00 2001 From: P4T12ICK Date: Wed, 7 Apr 2021 16:29:08 +0200 Subject: [PATCH 1/5] ssa testing improvments --- .gitlab-ci.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index eea0eb4d53..a6715d945c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -25,7 +25,6 @@ validate: - python3 bin/testing_coverage.py --type streaming --min-coverage 1.0 only: - /^ssa.*$/ - - develop publish_deployer: stage: publish_deployer @@ -39,7 +38,6 @@ publish_deployer: - docker push ${K8_DEPLOYER_CONTAINER}:${CI_COMMIT_SHORT_SHA} only: - /^ssa.*$/ - - develop publish_smoketest_runner: stage: publish_smoketest_runner @@ -52,7 +50,6 @@ publish_smoketest_runner: - docker push ${SMOKETEST_RUNNER}:${CI_COMMIT_SHORT_SHA} only: - /^ssa.*$/ - - develop smoketest_staging: stage: smoketest_staging @@ -72,4 +69,3 @@ smoketest_staging: SMOKETEST_RUNNER_IMAGE: ${SMOKETEST_RUNNER}:${CI_COMMIT_SHORT_SHA} only: - /^ssa.*$/ - - develop From b597a6f22664e09dfaa9b7e310b501f4e0db1830 Mon Sep 17 00:00:00 2001 From: P4T12ICK Date: Tue, 13 Apr 2021 09:08:34 +0200 Subject: [PATCH 2/5] bug fixes --- .gitlab-ci.yml | 4 + bin/content_analysis.py | 75 +++++++ bin/content_changer.py | 200 +++++++++++++++--- .../remote_registry_key_modifications.yml | 2 +- ...uspicious_changes_to_file_associations.yml | 2 +- 5 files changed, 252 insertions(+), 31 deletions(-) create mode 100644 bin/content_analysis.py diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a6715d945c..eea0eb4d53 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -25,6 +25,7 @@ validate: - python3 bin/testing_coverage.py --type streaming --min-coverage 1.0 only: - /^ssa.*$/ + - develop publish_deployer: stage: publish_deployer @@ -38,6 +39,7 @@ publish_deployer: - docker push ${K8_DEPLOYER_CONTAINER}:${CI_COMMIT_SHORT_SHA} only: - /^ssa.*$/ + - develop publish_smoketest_runner: stage: publish_smoketest_runner @@ -50,6 +52,7 @@ publish_smoketest_runner: - docker push ${SMOKETEST_RUNNER}:${CI_COMMIT_SHORT_SHA} only: - /^ssa.*$/ + - develop smoketest_staging: stage: smoketest_staging @@ -69,3 +72,4 @@ smoketest_staging: SMOKETEST_RUNNER_IMAGE: ${SMOKETEST_RUNNER}:${CI_COMMIT_SHORT_SHA} only: - /^ssa.*$/ + - develop diff --git a/bin/content_analysis.py b/bin/content_analysis.py new file mode 100644 index 0000000000..69e1cddb07 --- /dev/null +++ b/bin/content_analysis.py @@ -0,0 +1,75 @@ +import glob +import yaml +import sys +import re +import argparse + +from os import path + + +def load_objects(file_path, REPO_PATH): + files = [] + manifest_files = path.join(path.expanduser(REPO_PATH), file_path) + for file in sorted(glob.glob(manifest_files)): + files.append(load_file(file)) + return files + + +def load_file(file_path): + with open(file_path, 'r', encoding="utf-8") as stream: + try: + file = list(yaml.safe_load_all(stream))[0] + except yaml.YAMLError as exc: + print(exc) + sys.exit("ERROR: reading {0}".format(file_path)) + return file + + +def load_content(old_project): + stories = load_objects("stories/*.yml", old_project) + macros = load_objects("macros/*.yml", old_project) + lookups = load_objects("lookups/*.yml", old_project) + baselines = load_objects("baselines/*.yml", old_project) + responses = load_objects("responses/*.yml", old_project) + response_tasks = load_objects("response_tasks/*.yml", old_project) + deployments = load_objects("deployments/*.yml", old_project) + + # process all detections + detections = [] + detections = load_objects("detections/*/*.yml", old_project) + detections.extend(load_objects("detections/*/*/*.yml", old_project)) + + #print(len(detections)) + + return detections, stories, macros, lookups, baselines, responses, response_tasks, deployments + + +def add_required_field(detections, new_project): + #for detection in detections: + matches = re.findall(r'(?[^\s]*)=', detections[0]) + for match in matches: + print(match) + + +def main(new_project, old_project, change): + + detections, stories, macros, lookups, baselines, responses, response_tasks, deployments = load_content(old_project) + + if change == "add_required_field": + add_required_field(detections, new_project) + + +if __name__ == "__main__": + + parser = argparse.ArgumentParser(description="applies security content changes to the whole project") + parser.add_argument("-np", "--new_project", required=True, help="the security content project to write the new configs in to") + parser.add_argument("-op", "--old_project", required=True, help="the security content project to read the files from") + parser.add_argument("-c", "--change", required=True, help="the name of your change") + + # parse them + args = parser.parse_args() + new_project = args.new_project + old_project = args.old_project + change = args.change + + main(new_project, old_project, change) diff --git a/bin/content_changer.py b/bin/content_changer.py index 69e1cddb07..492cddc8c5 100644 --- a/bin/content_changer.py +++ b/bin/content_changer.py @@ -3,9 +3,19 @@ import yaml import sys import re import argparse +import requests +import csv +from requests.auth import HTTPBasicAuth +from urllib3.exceptions import InsecureRequestWarning from os import path +BASE_URL = f"https://18.198.253.138:8089" +SEARCH_PARSER_ENDPOINT = f"/services/search/parser" +USER = f"admin" +PASSWORD = f"Pl3ase-k1Ll-me:p1" +parsed_fields = dict() + def load_objects(file_path, REPO_PATH): files = [] @@ -26,13 +36,6 @@ def load_file(file_path): def load_content(old_project): - stories = load_objects("stories/*.yml", old_project) - macros = load_objects("macros/*.yml", old_project) - lookups = load_objects("lookups/*.yml", old_project) - baselines = load_objects("baselines/*.yml", old_project) - responses = load_objects("responses/*.yml", old_project) - response_tasks = load_objects("response_tasks/*.yml", old_project) - deployments = load_objects("deployments/*.yml", old_project) # process all detections detections = [] @@ -41,35 +44,174 @@ def load_content(old_project): #print(len(detections)) - return detections, stories, macros, lookups, baselines, responses, response_tasks, deployments + return detections -def add_required_field(detections, new_project): - #for detection in detections: - matches = re.findall(r'(?[^\s]*)=', detections[0]) - for match in matches: - print(match) +def analysis_detection(detections): + + for detection in detections:# + if detection['type'] != 'streaming': + #if detection['name'] == 'Attempted Credential Dump From Registry via Reg exe': + print('Analysis Detection: ' + detection['name']) + call_splunk_parser_api(detection) + + # sort parsed fields by occurence + sorted_dict = {k: v for k, v in sorted(parsed_fields.items(), key=lambda item: item[1], reverse=True)} + + with open('output_fields_ordered_by_usage.csv', mode='w') as csv_file: + writer = csv.writer(csv_file, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL) + + writer.writerow(['field_name', 'occurence']) + + for field_name in sorted_dict: + writer.writerow([field_name, sorted_dict[field_name]]) + + # sort parsed fields by name + sorted_dict_2 = sorted(parsed_fields.items()) + + with open('output_fields_ordered_by_keys.csv', mode='w') as csv_file: + writer = csv.writer(csv_file, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL) + + writer.writerow(['field_name', 'occurence']) + + for field_name in sorted_dict_2: + writer.writerow([field_name[0], field_name[1]]) -def main(new_project, old_project, change): +def call_splunk_parser_api(detection): + requests.packages.urllib3.disable_warnings(category=InsecureRequestWarning) + spl = '' + if detection['search'].startswith('| tstats'): + spl = detection['search'] + else: + spl = 'search ' + detection['search'] + data = { + "output_mode": "json", + "q": spl, + "parse_only": "true" + } + response = requests.post(BASE_URL + SEARCH_PARSER_ENDPOINT, data=data, auth=(USER, PASSWORD), verify=False, headers={"Content-Type": "application/x-www-form-urlencoded"}) + if response.status_code != 200: + print(response.json()) + print('ERROR: parser endpoint problems') + return + parse_commands(response.json()) - detections, stories, macros, lookups, baselines, responses, response_tasks, deployments = load_content(old_project) - if change == "add_required_field": - add_required_field(detections, new_project) +def parse_commands(api_response): + tmp_parsed_fields = {} + + last_stat_command = '' + rename_command_after_stats_arr = [] + + for command in api_response['commands']: + if command['command'] in ['tstats', 'stats', 'table']: + last_stat_command = command + if (command['command'] == 'rename') and last_stat_command: + rename_command_after_stats_arr.append(command) + + + if not last_stat_command: + print('ERROR: could not find stats table or tasts command') + return + + # last command table + if last_stat_command['command'] == 'table': + matches = re.findall(r'([0-9a-zA-Z_]+)', last_stat_command['rawargs']) + for match in matches: + if match in tmp_parsed_fields: + tmp_parsed_fields[match] = tmp_parsed_fields[match] + 1 + else: + tmp_parsed_fields[match] = 1 + + # last command stats + if last_stat_command['command'] == 'stats': + match = re.match(r'(.*)by', last_stat_command['rawargs']) + if match: + args_one = match.group(1) + matches = re.findall(r'(?:min|max|values)\(([0-9a-zA-Z_]+)\)', args_one) + if matches: + for match in matches: + if match in tmp_parsed_fields: + tmp_parsed_fields[match] = tmp_parsed_fields[match] + 1 + else: + tmp_parsed_fields[match] = 1 + match = re.match(r'.*by(.*)$', last_stat_command['rawargs']) + if match: + args_two = match.group(1) + matches = re.findall(r'([0-9a-zA-Z_]+)', args_two) + if matches: + for match in matches: + if match in tmp_parsed_fields: + tmp_parsed_fields[match] = tmp_parsed_fields[match] + 1 + else: + tmp_parsed_fields[match] = 1 + + # tstats command + if last_stat_command['command'] == 'tstats': + match = re.match(r'(.*)(?:from|FROM)', last_stat_command['rawargs']) + if match: + args_one = match.group(1) + matches = re.findall(r'(?:min|max|values)\(([0-9a-zA-Z_]+)\.([0-9a-zA-Z_]+)\)', args_one) + if matches: + for match in matches: + field = match[0] + '.' + match[1] + if field in tmp_parsed_fields: + tmp_parsed_fields[field] = tmp_parsed_fields[field] + 1 + else: + tmp_parsed_fields[field] = 1 + + match = re.match(r'.*by(.*)$', last_stat_command['rawargs']) + if match: + args_two = match.group(1) + matches = re.findall(r'([0-9_a-zA-Z]+)\.([0-9a-zA-Z_]+)', args_two) + if matches: + for match in matches: + field = match[0] + '.' + match[1] + if field in tmp_parsed_fields: + tmp_parsed_fields[field] = tmp_parsed_fields[field] + 1 + else: + tmp_parsed_fields[field] = 1 + + match = re.match(r'.*where(.*)by.*$', last_stat_command['rawargs']) + if match: + args_three = match.group(1) + matches = re.findall(r'([0-9_a-zA-Z]+)\.([0-9a-zA-Z_]+)=', args_three) + if matches: + for match in matches: + field = match[0] + '.' + match[1] + if field in tmp_parsed_fields: + tmp_parsed_fields[field] = tmp_parsed_fields[field] + 1 + else: + tmp_parsed_fields[field] = 1 + + + # rename occured + for rename_command_after_stats in rename_command_after_stats_arr: + if rename_command_after_stats: + renamed_field = {} + matches = re.findall(r'(?:(([0-9a-zA-Z_]+)\s+as\s+([0-9a-zA-Z_]+)))', rename_command_after_stats['rawargs']) + for match in matches: + renamed_field[match[1]] = match[2] + + for key in renamed_field: + if key in tmp_parsed_fields: + tmp_parsed_fields[renamed_field[key]] = tmp_parsed_fields.pop(key) + + # write to global parsed fields var + for key in tmp_parsed_fields: + if key in parsed_fields: + parsed_fields[key] = parsed_fields[key] + tmp_parsed_fields[key] + else: + parsed_fields[key] = tmp_parsed_fields[key] + + +def main(project): + + detections = load_content(project) + analysis_detection(detections) if __name__ == "__main__": - parser = argparse.ArgumentParser(description="applies security content changes to the whole project") - parser.add_argument("-np", "--new_project", required=True, help="the security content project to write the new configs in to") - parser.add_argument("-op", "--old_project", required=True, help="the security content project to read the files from") - parser.add_argument("-c", "--change", required=True, help="the name of your change") - - # parse them - args = parser.parse_args() - new_project = args.new_project - old_project = args.old_project - change = args.change - - main(new_project, old_project, change) + main("../") diff --git a/detections/deprecated/remote_registry_key_modifications.yml b/detections/deprecated/remote_registry_key_modifications.yml index 323901d3e9..90a7aed987 100644 --- a/detections/deprecated/remote_registry_key_modifications.yml +++ b/detections/deprecated/remote_registry_key_modifications.yml @@ -8,7 +8,7 @@ datamodel: [] description: This search monitors for remote modifications to registry keys. search: '| tstats `security_content_summariesonly` count values(Registry.registry_key_name) as registry_key_name values(Registry.registry_path) as registry_path min(_time) - as firstTime max(_time) as lastTime FROM datamodel=Endpoint.Registry where Registry.registry_path="\\\\*" by + as firstTime max(_time) as lastTime from datamodel=Endpoint.Registry where Registry.registry_path="\\\\*" by Registry.dest , Registry.user | `security_content_ctime(lastTime)` | `security_content_ctime(firstTime)` | `drop_dm_object_name(Registry)` | `remote_registry_key_modifications_filter`' how_to_implement: To successfully implement this search, you must populate the `Endpoint` diff --git a/detections/deprecated/suspicious_changes_to_file_associations.yml b/detections/deprecated/suspicious_changes_to_file_associations.yml index 047d9fb62f..4abd03ab4c 100644 --- a/detections/deprecated/suspicious_changes_to_file_associations.yml +++ b/detections/deprecated/suspicious_changes_to_file_associations.yml @@ -14,7 +14,7 @@ search: '| tstats `security_content_summariesonly` count min(_time) as firstTime AND Processes.process_name!=OpenWith.exe by Processes.process_id Processes.dest | `drop_dm_object_name("Processes")` | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)` | join [| tstats `security_content_summariesonly` values(Registry.registry_path) - as registry_path count FROM datamodel=Endpoint.Registry where Registry.registry_path=*\\Explorer\\FileExts* + as registry_path count from datamodel=Endpoint.Registry where Registry.registry_path=*\\Explorer\\FileExts* by Registry.process_id Registry.dest | `drop_dm_object_name("Registry")` | table process_id dest registry_path]| `suspicious_changes_to_file_associations_filter` ' how_to_implement: To successfully implement this search you need to be ingesting information From c5d9250091a99796edf78efc556830127d08dd7f Mon Sep 17 00:00:00 2001 From: P4T12ICK Date: Tue, 13 Apr 2021 10:00:31 +0200 Subject: [PATCH 3/5] bug fixes --- detections/endpoint/ssa___detect_pass_hash.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/detections/endpoint/ssa___detect_pass_hash.yml b/detections/endpoint/ssa___detect_pass_hash.yml index 62c1db2418..cf39140d4b 100644 --- a/detections/endpoint/ssa___detect_pass_hash.yml +++ b/detections/endpoint/ssa___detect_pass_hash.yml @@ -46,11 +46,12 @@ tags: product: - UEBA for Security Cloud required_fields: - - logon_process - - dest_user_primary_artifact + - signature_id + - authentication_type - _time - - event_code - - dest_ip_primary_artifact - - logon_type + - authentication_method + - origin_device_domain + - dest_user_id + - dest_device_id= risk_severity: low security_domain: endpoint From 4235b117cd8750b47216c0687bf36abf3ee5f686 Mon Sep 17 00:00:00 2001 From: P4T12ICK Date: Tue, 13 Apr 2021 10:05:14 +0200 Subject: [PATCH 4/5] bug fixes --- detections/endpoint/ssa___detect_pass_hash.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/detections/endpoint/ssa___detect_pass_hash.yml b/detections/endpoint/ssa___detect_pass_hash.yml index cf39140d4b..96098bf366 100644 --- a/detections/endpoint/ssa___detect_pass_hash.yml +++ b/detections/endpoint/ssa___detect_pass_hash.yml @@ -52,6 +52,6 @@ tags: - authentication_method - origin_device_domain - dest_user_id - - dest_device_id= + - dest_device_id risk_severity: low security_domain: endpoint From 33638d3b9428fe0e660fa7d31365b64c57212bc7 Mon Sep 17 00:00:00 2001 From: P4T12ICK Date: Tue, 13 Apr 2021 10:20:29 +0200 Subject: [PATCH 5/5] bug fixes --- bin/content_changer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/content_changer.py b/bin/content_changer.py index 492cddc8c5..22d9837fe6 100644 --- a/bin/content_changer.py +++ b/bin/content_changer.py @@ -10,10 +10,10 @@ from urllib3.exceptions import InsecureRequestWarning from os import path -BASE_URL = f"https://18.198.253.138:8089" +BASE_URL = f"https://ip:8089" SEARCH_PARSER_ENDPOINT = f"/services/search/parser" USER = f"admin" -PASSWORD = f"Pl3ase-k1Ll-me:p1" +PASSWORD = f"password" parsed_fields = dict()