diff --git a/bin/content_analysis.py b/bin/content_analysis.py new file mode 100644 index 0000000000..69e1cddb07 --- /dev/null +++ b/bin/content_analysis.py @@ -0,0 +1,75 @@ +import glob +import yaml +import sys +import re +import argparse + +from os import path + + +def load_objects(file_path, REPO_PATH): + files = [] + manifest_files = path.join(path.expanduser(REPO_PATH), file_path) + for file in sorted(glob.glob(manifest_files)): + files.append(load_file(file)) + return files + + +def load_file(file_path): + with open(file_path, 'r', encoding="utf-8") as stream: + try: + file = list(yaml.safe_load_all(stream))[0] + except yaml.YAMLError as exc: + print(exc) + sys.exit("ERROR: reading {0}".format(file_path)) + return file + + +def load_content(old_project): + stories = load_objects("stories/*.yml", old_project) + macros = load_objects("macros/*.yml", old_project) + lookups = load_objects("lookups/*.yml", old_project) + baselines = load_objects("baselines/*.yml", old_project) + responses = load_objects("responses/*.yml", old_project) + response_tasks = load_objects("response_tasks/*.yml", old_project) + deployments = load_objects("deployments/*.yml", old_project) + + # process all detections + detections = [] + detections = load_objects("detections/*/*.yml", old_project) + detections.extend(load_objects("detections/*/*/*.yml", old_project)) + + #print(len(detections)) + + return detections, stories, macros, lookups, baselines, responses, response_tasks, deployments + + +def add_required_field(detections, new_project): + #for detection in detections: + matches = re.findall(r'(?[^\s]*)=', detections[0]) + for match in matches: + print(match) + + +def main(new_project, old_project, change): + + detections, stories, macros, lookups, baselines, responses, response_tasks, deployments = load_content(old_project) + + if change == "add_required_field": + add_required_field(detections, new_project) + + +if __name__ == "__main__": + + parser = argparse.ArgumentParser(description="applies security content changes to the whole project") + parser.add_argument("-np", "--new_project", required=True, help="the security content project to write the new configs in to") + parser.add_argument("-op", "--old_project", required=True, help="the security content project to read the files from") + parser.add_argument("-c", "--change", required=True, help="the name of your change") + + # parse them + args = parser.parse_args() + new_project = args.new_project + old_project = args.old_project + change = args.change + + main(new_project, old_project, change) diff --git a/bin/content_changer.py b/bin/content_changer.py index 69e1cddb07..22d9837fe6 100644 --- a/bin/content_changer.py +++ b/bin/content_changer.py @@ -3,9 +3,19 @@ import yaml import sys import re import argparse +import requests +import csv +from requests.auth import HTTPBasicAuth +from urllib3.exceptions import InsecureRequestWarning from os import path +BASE_URL = f"https://ip:8089" +SEARCH_PARSER_ENDPOINT = f"/services/search/parser" +USER = f"admin" +PASSWORD = f"password" +parsed_fields = dict() + def load_objects(file_path, REPO_PATH): files = [] @@ -26,13 +36,6 @@ def load_file(file_path): def load_content(old_project): - stories = load_objects("stories/*.yml", old_project) - macros = load_objects("macros/*.yml", old_project) - lookups = load_objects("lookups/*.yml", old_project) - baselines = load_objects("baselines/*.yml", old_project) - responses = load_objects("responses/*.yml", old_project) - response_tasks = load_objects("response_tasks/*.yml", old_project) - deployments = load_objects("deployments/*.yml", old_project) # process all detections detections = [] @@ -41,35 +44,174 @@ def load_content(old_project): #print(len(detections)) - return detections, stories, macros, lookups, baselines, responses, response_tasks, deployments + return detections -def add_required_field(detections, new_project): - #for detection in detections: - matches = re.findall(r'(?[^\s]*)=', detections[0]) - for match in matches: - print(match) +def analysis_detection(detections): + + for detection in detections:# + if detection['type'] != 'streaming': + #if detection['name'] == 'Attempted Credential Dump From Registry via Reg exe': + print('Analysis Detection: ' + detection['name']) + call_splunk_parser_api(detection) + + # sort parsed fields by occurence + sorted_dict = {k: v for k, v in sorted(parsed_fields.items(), key=lambda item: item[1], reverse=True)} + + with open('output_fields_ordered_by_usage.csv', mode='w') as csv_file: + writer = csv.writer(csv_file, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL) + + writer.writerow(['field_name', 'occurence']) + + for field_name in sorted_dict: + writer.writerow([field_name, sorted_dict[field_name]]) + + # sort parsed fields by name + sorted_dict_2 = sorted(parsed_fields.items()) + + with open('output_fields_ordered_by_keys.csv', mode='w') as csv_file: + writer = csv.writer(csv_file, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL) + + writer.writerow(['field_name', 'occurence']) + + for field_name in sorted_dict_2: + writer.writerow([field_name[0], field_name[1]]) -def main(new_project, old_project, change): +def call_splunk_parser_api(detection): + requests.packages.urllib3.disable_warnings(category=InsecureRequestWarning) + spl = '' + if detection['search'].startswith('| tstats'): + spl = detection['search'] + else: + spl = 'search ' + detection['search'] + data = { + "output_mode": "json", + "q": spl, + "parse_only": "true" + } + response = requests.post(BASE_URL + SEARCH_PARSER_ENDPOINT, data=data, auth=(USER, PASSWORD), verify=False, headers={"Content-Type": "application/x-www-form-urlencoded"}) + if response.status_code != 200: + print(response.json()) + print('ERROR: parser endpoint problems') + return + parse_commands(response.json()) - detections, stories, macros, lookups, baselines, responses, response_tasks, deployments = load_content(old_project) - if change == "add_required_field": - add_required_field(detections, new_project) +def parse_commands(api_response): + tmp_parsed_fields = {} + + last_stat_command = '' + rename_command_after_stats_arr = [] + + for command in api_response['commands']: + if command['command'] in ['tstats', 'stats', 'table']: + last_stat_command = command + if (command['command'] == 'rename') and last_stat_command: + rename_command_after_stats_arr.append(command) + + + if not last_stat_command: + print('ERROR: could not find stats table or tasts command') + return + + # last command table + if last_stat_command['command'] == 'table': + matches = re.findall(r'([0-9a-zA-Z_]+)', last_stat_command['rawargs']) + for match in matches: + if match in tmp_parsed_fields: + tmp_parsed_fields[match] = tmp_parsed_fields[match] + 1 + else: + tmp_parsed_fields[match] = 1 + + # last command stats + if last_stat_command['command'] == 'stats': + match = re.match(r'(.*)by', last_stat_command['rawargs']) + if match: + args_one = match.group(1) + matches = re.findall(r'(?:min|max|values)\(([0-9a-zA-Z_]+)\)', args_one) + if matches: + for match in matches: + if match in tmp_parsed_fields: + tmp_parsed_fields[match] = tmp_parsed_fields[match] + 1 + else: + tmp_parsed_fields[match] = 1 + match = re.match(r'.*by(.*)$', last_stat_command['rawargs']) + if match: + args_two = match.group(1) + matches = re.findall(r'([0-9a-zA-Z_]+)', args_two) + if matches: + for match in matches: + if match in tmp_parsed_fields: + tmp_parsed_fields[match] = tmp_parsed_fields[match] + 1 + else: + tmp_parsed_fields[match] = 1 + + # tstats command + if last_stat_command['command'] == 'tstats': + match = re.match(r'(.*)(?:from|FROM)', last_stat_command['rawargs']) + if match: + args_one = match.group(1) + matches = re.findall(r'(?:min|max|values)\(([0-9a-zA-Z_]+)\.([0-9a-zA-Z_]+)\)', args_one) + if matches: + for match in matches: + field = match[0] + '.' + match[1] + if field in tmp_parsed_fields: + tmp_parsed_fields[field] = tmp_parsed_fields[field] + 1 + else: + tmp_parsed_fields[field] = 1 + + match = re.match(r'.*by(.*)$', last_stat_command['rawargs']) + if match: + args_two = match.group(1) + matches = re.findall(r'([0-9_a-zA-Z]+)\.([0-9a-zA-Z_]+)', args_two) + if matches: + for match in matches: + field = match[0] + '.' + match[1] + if field in tmp_parsed_fields: + tmp_parsed_fields[field] = tmp_parsed_fields[field] + 1 + else: + tmp_parsed_fields[field] = 1 + + match = re.match(r'.*where(.*)by.*$', last_stat_command['rawargs']) + if match: + args_three = match.group(1) + matches = re.findall(r'([0-9_a-zA-Z]+)\.([0-9a-zA-Z_]+)=', args_three) + if matches: + for match in matches: + field = match[0] + '.' + match[1] + if field in tmp_parsed_fields: + tmp_parsed_fields[field] = tmp_parsed_fields[field] + 1 + else: + tmp_parsed_fields[field] = 1 + + + # rename occured + for rename_command_after_stats in rename_command_after_stats_arr: + if rename_command_after_stats: + renamed_field = {} + matches = re.findall(r'(?:(([0-9a-zA-Z_]+)\s+as\s+([0-9a-zA-Z_]+)))', rename_command_after_stats['rawargs']) + for match in matches: + renamed_field[match[1]] = match[2] + + for key in renamed_field: + if key in tmp_parsed_fields: + tmp_parsed_fields[renamed_field[key]] = tmp_parsed_fields.pop(key) + + # write to global parsed fields var + for key in tmp_parsed_fields: + if key in parsed_fields: + parsed_fields[key] = parsed_fields[key] + tmp_parsed_fields[key] + else: + parsed_fields[key] = tmp_parsed_fields[key] + + +def main(project): + + detections = load_content(project) + analysis_detection(detections) if __name__ == "__main__": - parser = argparse.ArgumentParser(description="applies security content changes to the whole project") - parser.add_argument("-np", "--new_project", required=True, help="the security content project to write the new configs in to") - parser.add_argument("-op", "--old_project", required=True, help="the security content project to read the files from") - parser.add_argument("-c", "--change", required=True, help="the name of your change") - - # parse them - args = parser.parse_args() - new_project = args.new_project - old_project = args.old_project - change = args.change - - main(new_project, old_project, change) + main("../") diff --git a/detections/deprecated/remote_registry_key_modifications.yml b/detections/deprecated/remote_registry_key_modifications.yml index 323901d3e9..90a7aed987 100644 --- a/detections/deprecated/remote_registry_key_modifications.yml +++ b/detections/deprecated/remote_registry_key_modifications.yml @@ -8,7 +8,7 @@ datamodel: [] description: This search monitors for remote modifications to registry keys. search: '| tstats `security_content_summariesonly` count values(Registry.registry_key_name) as registry_key_name values(Registry.registry_path) as registry_path min(_time) - as firstTime max(_time) as lastTime FROM datamodel=Endpoint.Registry where Registry.registry_path="\\\\*" by + as firstTime max(_time) as lastTime from datamodel=Endpoint.Registry where Registry.registry_path="\\\\*" by Registry.dest , Registry.user | `security_content_ctime(lastTime)` | `security_content_ctime(firstTime)` | `drop_dm_object_name(Registry)` | `remote_registry_key_modifications_filter`' how_to_implement: To successfully implement this search, you must populate the `Endpoint` diff --git a/detections/deprecated/suspicious_changes_to_file_associations.yml b/detections/deprecated/suspicious_changes_to_file_associations.yml index 047d9fb62f..4abd03ab4c 100644 --- a/detections/deprecated/suspicious_changes_to_file_associations.yml +++ b/detections/deprecated/suspicious_changes_to_file_associations.yml @@ -14,7 +14,7 @@ search: '| tstats `security_content_summariesonly` count min(_time) as firstTime AND Processes.process_name!=OpenWith.exe by Processes.process_id Processes.dest | `drop_dm_object_name("Processes")` | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)` | join [| tstats `security_content_summariesonly` values(Registry.registry_path) - as registry_path count FROM datamodel=Endpoint.Registry where Registry.registry_path=*\\Explorer\\FileExts* + as registry_path count from datamodel=Endpoint.Registry where Registry.registry_path=*\\Explorer\\FileExts* by Registry.process_id Registry.dest | `drop_dm_object_name("Registry")` | table process_id dest registry_path]| `suspicious_changes_to_file_associations_filter` ' how_to_implement: To successfully implement this search you need to be ingesting information diff --git a/detections/endpoint/ssa___detect_pass_hash.yml b/detections/endpoint/ssa___detect_pass_hash.yml index 62c1db2418..96098bf366 100644 --- a/detections/endpoint/ssa___detect_pass_hash.yml +++ b/detections/endpoint/ssa___detect_pass_hash.yml @@ -46,11 +46,12 @@ tags: product: - UEBA for Security Cloud required_fields: - - logon_process - - dest_user_primary_artifact + - signature_id + - authentication_type - _time - - event_code - - dest_ip_primary_artifact - - logon_type + - authentication_method + - origin_device_domain + - dest_user_id + - dest_device_id risk_severity: low security_domain: endpoint