diff --git a/bin/generate.py b/bin/generate.py index 9c1289b634..9807e8a981 100644 --- a/bin/generate.py +++ b/bin/generate.py @@ -31,12 +31,33 @@ def markdown(x): return markdown -def generate_analytics_story(REPO_PATH, OUTPUT_DIR, verbose): +def generate_savedsearches(REPO_PATH, stories): + # first we process detections - # Create conf files from analytics stories files - story_output_path = OUTPUT_DIR + "/default/analytics_stories.conf" - output_file = open(story_output_path, 'w') + # detections_files = [] + # detections_manifest_files = path.join(path.expanduser(REPO_PATH), "detections/*.json") + for story_name, story in stories.iteritems(): + if 'detections' in story: + + search_manifests = {} + for search_manifest_file in glob.glob(path.join(REPO_PATH, '*/detections/*.json')): + try: + search_manifest = json.loads(open(search_manifest_file, 'r').read()) + except Exception as e: + print "Error reading %s" % search_manifest_file + print e + continue + + search_manifests[search_manifest['search_name']] = search_manifest + + # story['mappings'] = + # complete_stories[story['name']]['mappings'] = {} + # complete_stories[story['name']]['data_models'] = set() + # complete_stories[story['name']]['providing_technologies'] = set() + + +def generate_analytics_story(REPO_PATH, verbose): story_files = [] story_manifest_files = path.join(path.expanduser(REPO_PATH), "stories/*.json") @@ -51,6 +72,7 @@ def generate_analytics_story(REPO_PATH, OUTPUT_DIR, verbose): story_files.append(story) # store an object with all stories and their data + complete_stories = dict() for story in story_files: if verbose: @@ -74,16 +96,58 @@ def generate_analytics_story(REPO_PATH, OUTPUT_DIR, verbose): complete_stories[story['name']]['references'] = story['references'] complete_stories[story['name']]['category'] = story['category'] complete_stories[story['name']]['version'] = story['version'] - complete_stories[story['name']]['mappings'] = {} - complete_stories[story['name']]['data_models'] = set() - complete_stories[story['name']]['providing_technologies'] = set() - complete_stories[story['name']]['tags'] = set() complete_stories[story['name']]['narrative'] = story['narrative'] - complete_stories[story['name']]['detection_searches'] = [] - complete_stories[story['name']]['investigative_searches'] = [] - complete_stories[story['name']]['contextual_searches'] = [] - complete_stories[story['name']]['support_searches'] = [] + # grab searches + if story['spec_version'] == 1: + if 'detection_searches' in story['searches']: + detections = [] + for d in story['searches']['detection_searches']: + detections.append({"type": "splunk", "name": "ESCU - " + d + " - Rule"}) + complete_stories[story['name']]['detections'] = detections + + if 'support_searches' in story['searches']: + baselines = [] + for b in story['searches']['support_searches']: + detections.append({"type": "splunk", "name": "ESCU - " + b}) + complete_stories[story['name']]['baselines'] = baselines + + investigations = [] + if 'contexual_searches' in story['searches']: + for i in story['searches']['contexual_searches']: + investigations.append({"type": "splunk", "name": "ESCU - " + i}) + if 'investigative_searches' in story['searches']: + for i in story['searches']['investigative_searches']: + investigations.append({"type": "splunk", "name": "ESCU - " + i}) + complete_stories[story['name']]['investigations'] = investigations + + if story['spec_version'] == 2: + if 'detections' in story: + detections = [] + for d in story['detections']: + detections.append({"type": d['type'], "name": "ESCU - " + d['name'] + " - Rule"}) + complete_stories[story['name']]['detections'] = detections + + if 'baselines' in story: + baselines = [] + for b in story['baselines']: + detections.append({"type": d['type'], "name": "ESCU - " + b['name']}) + complete_stories[story['name']]['baselines'] = baselines + + if 'investigations' in story: + investigations = [] + for i in story['investigations']: + investigations.append({"type": i['type'], "name": "ESCU - " + i['name']}) + complete_stories[story['name']]['investigations'] = investigations + + return complete_stories + + +def write_story_output(complete_stories, OUTPUT_DIR): + + # Create conf files from analytics stories files + story_output_path = OUTPUT_DIR + "/default/analytics_stories.conf" + output_file = open(story_output_path, 'w') # Finish the story for story_name, story in sorted(complete_stories.iteritems()): @@ -95,6 +159,13 @@ def generate_analytics_story(REPO_PATH, OUTPUT_DIR, verbose): output_file.write("version = %s\n" % story['version']) output_file.write("reference = %s\n" % json.dumps(story['references'])) + if 'detections' in story: + output_file.write("detections = %s\n" % json.dumps(story['detections'])) + if 'investigations' in story: + output_file.write("investigations = %s\n" % json.dumps(story['investigations'])) + if 'baselines' in story: + output_file.write("baselines = %s\n" % json.dumps(story['baselines'])) + # REMOVE THIS FUNCTION MAKE SURE ALL DESCRIPTIONs ARE NATIVELY IN MARKDOWN description = markdown(story['description']) output_file.write("description = %s\n" % description) @@ -107,7 +178,7 @@ def generate_analytics_story(REPO_PATH, OUTPUT_DIR, verbose): # close file, count stories we found and return output_file.close() - story_count = len(story_files) + story_count = len(complete_stories.keys()) return story_count, story_output_path @@ -127,5 +198,7 @@ if __name__ == "__main__": OUTPUT_DIR = args.output verbose = args.verbose - story_count, story_path = generate_analytics_story(REPO_PATH, OUTPUT_DIR, verbose) + complete_stories = generate_analytics_story(REPO_PATH, verbose) + # generate_detections(REPO_PATH, complete_stories) + story_count, story_path = write_story_output(complete_stories, OUTPUT_DIR) print "{0} stories have been successfully to {1}".format(story_count, story_path)