Branch was auto-updated.

bin/content_analysis.py

@@ -0,0 +1,75 @@
+import glob
+import yaml
+import sys
+import re
+import argparse
+
+from os import path
+
+
+def load_objects(file_path, REPO_PATH):
+    files = []
+    manifest_files = path.join(path.expanduser(REPO_PATH), file_path)
+    for file in sorted(glob.glob(manifest_files)):
+        files.append(load_file(file))
+    return files
+
+
+def load_file(file_path):
+    with open(file_path, 'r', encoding="utf-8") as stream:
+        try:
+            file = list(yaml.safe_load_all(stream))[0]
+        except yaml.YAMLError as exc:
+            print(exc)
+            sys.exit("ERROR: reading {0}".format(file_path))
+    return file
+
+
+def load_content(old_project):
+    stories = load_objects("stories/*.yml", old_project)
+    macros = load_objects("macros/*.yml", old_project)
+    lookups = load_objects("lookups/*.yml", old_project)
+    baselines = load_objects("baselines/*.yml", old_project)
+    responses = load_objects("responses/*.yml", old_project)
+    response_tasks = load_objects("response_tasks/*.yml", old_project)
+    deployments = load_objects("deployments/*.yml", old_project)
+
+    # process all detections
+    detections = []
+    detections = load_objects("detections/*/*.yml", old_project)
+    detections.extend(load_objects("detections/*/*/*.yml", old_project))
+
+    #print(len(detections))
+
+    return detections, stories, macros, lookups, baselines, responses, response_tasks, deployments
+
+
+def add_required_field(detections, new_project):
+    #for detection in detections:
+    matches = re.findall(r'(?<key>[^\s]*)=', detections[0])
+    for match in matches:
+        print(match)
+
+
+def main(new_project, old_project, change):
+
+    detections, stories, macros, lookups, baselines, responses, response_tasks, deployments = load_content(old_project)
+
+    if change == "add_required_field":
+        add_required_field(detections, new_project)
+
+
+if __name__ == "__main__":
+
+    parser = argparse.ArgumentParser(description="applies security content changes to the whole project")
+    parser.add_argument("-np", "--new_project", required=True, help="the security content project to write the new configs in to")
+    parser.add_argument("-op", "--old_project", required=True, help="the security content project to read the files from")
+    parser.add_argument("-c", "--change", required=True, help="the name of your change")
+
+   # parse them
+    args = parser.parse_args()
+    new_project = args.new_project
+    old_project = args.old_project
+    change = args.change
+
+    main(new_project, old_project, change)

bin/content_changer.py

@@ -3,9 +3,19 @@ import yaml
 import sys
 import re
 import argparse
+import requests
+import csv
+from requests.auth import HTTPBasicAuth
+from urllib3.exceptions import InsecureRequestWarning
 
 from os import path
 
+BASE_URL = f"https://ip:8089"
+SEARCH_PARSER_ENDPOINT = f"/services/search/parser"
+USER = f"admin"
+PASSWORD = f"password"
+parsed_fields = dict()
+
 
 def load_objects(file_path, REPO_PATH):
     files = []
@@ -26,13 +36,6 @@ def load_file(file_path):
 
 
 def load_content(old_project):
-    stories = load_objects("stories/*.yml", old_project)
-    macros = load_objects("macros/*.yml", old_project)
-    lookups = load_objects("lookups/*.yml", old_project)
-    baselines = load_objects("baselines/*.yml", old_project)
-    responses = load_objects("responses/*.yml", old_project)
-    response_tasks = load_objects("response_tasks/*.yml", old_project)
-    deployments = load_objects("deployments/*.yml", old_project)
 
     # process all detections
     detections = []
@@ -41,35 +44,174 @@ def load_content(old_project):
 
     #print(len(detections))
 
-    return detections, stories, macros, lookups, baselines, responses, response_tasks, deployments
+    return detections
 
 
-def add_required_field(detections, new_project):
-    #for detection in detections:
-    matches = re.findall(r'(?<key>[^\s]*)=', detections[0])
-    for match in matches:
-        print(match)
+def analysis_detection(detections):
+
+    for detection in detections:#
+        if detection['type'] != 'streaming':
+            #if detection['name'] == 'Attempted Credential Dump From Registry via Reg exe':
+            print('Analysis Detection: ' + detection['name'])
+            call_splunk_parser_api(detection)
+
+    # sort parsed fields by occurence
+    sorted_dict = {k: v for k, v in sorted(parsed_fields.items(), key=lambda item: item[1], reverse=True)}
+
+    with open('output_fields_ordered_by_usage.csv', mode='w') as csv_file:
+        writer = csv.writer(csv_file, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL)
+
+        writer.writerow(['field_name', 'occurence'])
+
+        for field_name in sorted_dict:
+            writer.writerow([field_name, sorted_dict[field_name]])
+
+    # sort parsed fields by name
+    sorted_dict_2 = sorted(parsed_fields.items())
+
+    with open('output_fields_ordered_by_keys.csv', mode='w') as csv_file:
+        writer = csv.writer(csv_file, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL)
+
+        writer.writerow(['field_name', 'occurence'])
+
+        for field_name in sorted_dict_2:
+            writer.writerow([field_name[0], field_name[1]])
 
 
-def main(new_project, old_project, change):
+def call_splunk_parser_api(detection):
+    requests.packages.urllib3.disable_warnings(category=InsecureRequestWarning)
+    spl = ''
+    if detection['search'].startswith('| tstats'):
+        spl = detection['search']
+    else:
+        spl = 'search ' + detection['search']
+    data = {
+        "output_mode": "json",
+        "q": spl,
+        "parse_only": "true"
+    }
+    response = requests.post(BASE_URL + SEARCH_PARSER_ENDPOINT, data=data, auth=(USER, PASSWORD), verify=False, headers={"Content-Type": "application/x-www-form-urlencoded"})
+    if response.status_code != 200:
+        print(response.json())
+        print('ERROR: parser endpoint problems')
+        return
+    parse_commands(response.json())
 
-    detections, stories, macros, lookups, baselines, responses, response_tasks, deployments = load_content(old_project)
 
-    if change == "add_required_field":
-        add_required_field(detections, new_project)
+def parse_commands(api_response):
+    tmp_parsed_fields = {}
+
+    last_stat_command = ''
+    rename_command_after_stats_arr = []
+
+    for command in api_response['commands']:
+        if command['command'] in ['tstats', 'stats', 'table']:
+            last_stat_command = command
+        if (command['command'] == 'rename') and last_stat_command:
+            rename_command_after_stats_arr.append(command)
+        
+
+    if not last_stat_command:
+        print('ERROR: could not find stats table or tasts command')
+        return
+        
+    # last command table
+    if last_stat_command['command'] == 'table':
+        matches = re.findall(r'([0-9a-zA-Z_]+)', last_stat_command['rawargs'])
+        for match in matches:
+            if match in tmp_parsed_fields:
+                tmp_parsed_fields[match] = tmp_parsed_fields[match] + 1
+            else:
+                tmp_parsed_fields[match] = 1
+
+    # last command stats
+    if last_stat_command['command'] == 'stats':
+        match = re.match(r'(.*)by', last_stat_command['rawargs'])
+        if match:
+            args_one = match.group(1)
+            matches = re.findall(r'(?:min|max|values)\(([0-9a-zA-Z_]+)\)', args_one)
+            if matches:
+                for match in matches:
+                    if match in tmp_parsed_fields:
+                        tmp_parsed_fields[match] = tmp_parsed_fields[match] + 1
+                    else:
+                        tmp_parsed_fields[match] = 1
+        match = re.match(r'.*by(.*)$', last_stat_command['rawargs'])
+        if match:
+            args_two = match.group(1)
+            matches = re.findall(r'([0-9a-zA-Z_]+)', args_two)
+            if matches:
+                for match in matches:
+                    if match in tmp_parsed_fields:
+                        tmp_parsed_fields[match] = tmp_parsed_fields[match] + 1
+                    else:
+                        tmp_parsed_fields[match] = 1    
+
+    # tstats command
+    if last_stat_command['command'] == 'tstats':
+        match = re.match(r'(.*)(?:from|FROM)', last_stat_command['rawargs'])
+        if match:
+            args_one = match.group(1)
+            matches = re.findall(r'(?:min|max|values)\(([0-9a-zA-Z_]+)\.([0-9a-zA-Z_]+)\)', args_one)
+            if matches:
+                for match in matches:
+                    field = match[0] + '.' + match[1]
+                    if field in tmp_parsed_fields:
+                        tmp_parsed_fields[field] = tmp_parsed_fields[field] + 1
+                    else:
+                        tmp_parsed_fields[field] = 1    
+
+        match = re.match(r'.*by(.*)$', last_stat_command['rawargs'])
+        if match:
+            args_two = match.group(1)
+            matches = re.findall(r'([0-9_a-zA-Z]+)\.([0-9a-zA-Z_]+)', args_two)
+            if matches:
+                for match in matches:
+                    field = match[0] + '.' + match[1]
+                    if field in tmp_parsed_fields:
+                        tmp_parsed_fields[field] = tmp_parsed_fields[field] + 1
+                    else:
+                        tmp_parsed_fields[field] = 1    
+
+        match = re.match(r'.*where(.*)by.*$', last_stat_command['rawargs'])
+        if match:
+            args_three = match.group(1)
+            matches = re.findall(r'([0-9_a-zA-Z]+)\.([0-9a-zA-Z_]+)=', args_three)
+            if matches:
+                for match in matches:
+                    field = match[0] + '.' + match[1]
+                    if field in tmp_parsed_fields:
+                        tmp_parsed_fields[field] = tmp_parsed_fields[field] + 1
+                    else:
+                        tmp_parsed_fields[field] = 1    
+
+
+    # rename occured 
+    for rename_command_after_stats in rename_command_after_stats_arr:
+        if rename_command_after_stats:
+            renamed_field = {}
+            matches = re.findall(r'(?:(([0-9a-zA-Z_]+)\s+as\s+([0-9a-zA-Z_]+)))', rename_command_after_stats['rawargs'])
+            for match in matches:
+                renamed_field[match[1]] = match[2]
+
+            for key in renamed_field:
+                if key in tmp_parsed_fields:
+                    tmp_parsed_fields[renamed_field[key]] = tmp_parsed_fields.pop(key)
+
+    # write to global parsed fields var
+    for key in tmp_parsed_fields:
+        if key in parsed_fields:
+            parsed_fields[key] = parsed_fields[key] + tmp_parsed_fields[key]
+        else:
+            parsed_fields[key] = tmp_parsed_fields[key]
+
+
+def main(project):
+
+    detections = load_content(project)
+    analysis_detection(detections)
 
 
 if __name__ == "__main__":
 
-    parser = argparse.ArgumentParser(description="applies security content changes to the whole project")
-    parser.add_argument("-np", "--new_project", required=True, help="the security content project to write the new configs in to")
-    parser.add_argument("-op", "--old_project", required=True, help="the security content project to read the files from")
-    parser.add_argument("-c", "--change", required=True, help="the name of your change")
-
-   # parse them
-    args = parser.parse_args()
-    new_project = args.new_project
-    old_project = args.old_project
-    change = args.change
-
-    main(new_project, old_project, change)
+    main("../")

detections/deprecated/remote_registry_key_modifications.yml

@@ -8,7 +8,7 @@ datamodel: []
 description: This search monitors for remote modifications to registry keys.
 search: '| tstats `security_content_summariesonly` count values(Registry.registry_key_name)
   as registry_key_name values(Registry.registry_path) as registry_path min(_time)
-  as firstTime max(_time) as lastTime FROM datamodel=Endpoint.Registry where  Registry.registry_path="\\\\*"  by
+  as firstTime max(_time) as lastTime from datamodel=Endpoint.Registry where  Registry.registry_path="\\\\*"  by
   Registry.dest , Registry.user | `security_content_ctime(lastTime)` | `security_content_ctime(firstTime)`
   | `drop_dm_object_name(Registry)` | `remote_registry_key_modifications_filter`'
 how_to_implement: To successfully implement this search, you must populate the `Endpoint`

detections/deprecated/suspicious_changes_to_file_associations.yml

@@ -14,7 +14,7 @@ search: '| tstats `security_content_summariesonly` count min(_time) as firstTime
   AND Processes.process_name!=OpenWith.exe by Processes.process_id Processes.dest
   | `drop_dm_object_name("Processes")` | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)`
   | join [| tstats `security_content_summariesonly` values(Registry.registry_path)
-  as registry_path count  FROM datamodel=Endpoint.Registry where Registry.registry_path=*\\Explorer\\FileExts*
+  as registry_path count from datamodel=Endpoint.Registry where Registry.registry_path=*\\Explorer\\FileExts*
   by Registry.process_id Registry.dest | `drop_dm_object_name("Registry")` | table
   process_id dest registry_path]| `suspicious_changes_to_file_associations_filter` '
 how_to_implement: To successfully implement this search you need to be ingesting information

detections/endpoint/ssa___detect_pass_hash.yml

@@ -46,11 +46,12 @@ tags:
   product:
   - UEBA for Security Cloud
   required_fields:
-  - logon_process
-  - dest_user_primary_artifact
+  - signature_id
+  - authentication_type
   - _time
-  - event_code
-  - dest_ip_primary_artifact
-  - logon_type
+  - authentication_method
+  - origin_device_domain
+  - dest_user_id
+  - dest_device_id
   risk_severity: low
   security_domain: endpoint