Merge branch 'develop' into add_gitlab_ci

bin/doc-gen.py

@@ -684,7 +684,8 @@ if __name__ == "__main__":
     gmd = args.gen_markdown_docs
 
     stories = load_objects("stories/*.yml")
-    detections = load_objects("detections/*.yml")
+    detections = []
+    detections = load_objects("detections/*/*.yml")
 
     # complete_stories = generate_stories(REPO_PATH, verbose)
     # complete_detections = generate_detections(REPO_PATH, complete_stories)
@@ -697,7 +698,7 @@ if __name__ == "__main__":
 
     if gmd:
         story_count, path = write_markdown_docs(stories, detections,  OUTPUT_DIR)
-        print("{0} story documents have been successfully written to {1}".format(story_count, path)) 
+        print("{0} story documents have been successfully written to {1}".format(story_count, path))
     else:
         print("--gen_splunk_docs  was set to false, not generating splunk documentation")
 

bin/generate-actors-map.py

@@ -31,7 +31,8 @@ def main(argv):
     counted_techniques, max_count = count_techniques(techniques, all_techniques)
 
     print("load detections techniques")
-    detections = load_objects(path.join(cmdargs.projects_path),'detections/*.yml')
+    detections = []
+    detections = load_objects(path.join(cmdargs.projects_path),'detections/*/*.yml')
 
     print("get matched techniques")
     matched_techniques = get_matched_techniques(counted_techniques, detections)

bin/generate-coverage-map.py

@@ -28,7 +28,8 @@ def main(argv):
     techniques = get_all_techniques(cmdargs.projects_path)
 
     print("load detections")
-    detections = load_objects(path.join(cmdargs.projects_path),'detections/*.yml')
+    detections = []
+    detections = load_objects(path.join(cmdargs.projects_path),'detections/*/*.yml')
 
     print("get matched techniques")
     matched_techniques = get_matched_techniques(techniques, detections)

bin/generate.py

@@ -569,11 +569,14 @@ if __name__ == "__main__":
     macros = load_objects("macros/*.yml", VERBOSE)
     lookups = load_objects("lookups/*.yml", VERBOSE)
     baselines = load_objects("baselines/*.yml", VERBOSE)
-    detections = load_objects("detections/*.yml", VERBOSE)
     responses = load_objects("responses/*.yml", VERBOSE)
     response_tasks = load_objects("response_tasks/*.yml", VERBOSE)
     deployments = load_objects("deployments/*.yml", VERBOSE)
 
+    # process all detections
+    detections = []
+    detections = load_objects("detections/*/*.yml", VERBOSE)
+
     try:
         if VERBOSE:
             print("generating Mitre lookups")
@@ -585,6 +588,10 @@ if __name__ == "__main__":
     lookups_path = generate_collections_conf(lookups)
 
     detections = sorted(detections, key=lambda d: d['name'])
+
+    # only use ESCU detections to the configurations
+    detections = [object for object in detections if object["type"].lower() == "escu"]
+
     response_tasks = sorted(response_tasks, key=lambda i: i['name'])
     baselines = sorted(baselines, key=lambda b: b['name'])
     detection_path = generate_savedsearches_conf(detections, response_tasks, baselines, deployments)

bin/validate.py

@@ -13,7 +13,7 @@ import argparse
 import datetime
 import string
 import re
-from os import path
+from os import path, walk
 
 
 def validate_schema(REPO_PATH, type, objects):
@@ -28,9 +28,13 @@ def validate_schema(REPO_PATH, type, objects):
     except IOError:
         print("ERROR: reading schema file {0}".format(schema_file))
 
-    manifest_files = path.join(path.expanduser(REPO_PATH), type + '/*.yml')
+    manifest_files = []
+    for root, dirs, files in walk(REPO_PATH + "/" + type):
+        for file in files:
+            if file.endswith(".yml"):
+                manifest_files.append((path.join(root, file)))
 
-    for manifest_file in glob.glob(manifest_files):
+    for manifest_file in manifest_files:
         if verbose:
             print("processing manifest {0}".format(manifest_file))
 
@@ -42,7 +46,6 @@ def validate_schema(REPO_PATH, type, objects):
                 print("Error reading {0}".format(manifest_file))
                 error = True
                 continue
-
         try:
             jsonschema.validate(instance=object, schema=schema)
         except jsonschema.exceptions.ValidationError as json_ve: