From 8389cce2c0ea760ab116f381702ae341cc6b4ce4 Mon Sep 17 00:00:00 2001 From: Thomas Nicholson Date: Thu, 29 Apr 2021 09:48:06 +0100 Subject: [PATCH 1/6] Rearranged generate Put all objects in objects dictionary. Split up the importing, computing and output logic into different functions --- bin/generate.py | 371 ++++++++++++++++++++++-------------------------- 1 file changed, 170 insertions(+), 201 deletions(-) diff --git a/bin/generate.py b/bin/generate.py index aed5cc18f5..69b4b99c76 100644 --- a/bin/generate.py +++ b/bin/generate.py @@ -67,7 +67,6 @@ def generate_collections_conf(lookups, TEMPLATE_PATH, OUTPUT_PATH): return output_path - def generate_savedsearches_conf(detections, response_tasks, baselines, deployments, TEMPLATE_PATH, OUTPUT_PATH): ''' @param detections: input list of individual YAML detections in detections/ directory @@ -77,74 +76,6 @@ def generate_savedsearches_conf(detections, response_tasks, baselines, deploymen @return: the savedsearches.conf file located in package/default/ ''' - for detection in detections: - # parse out data_models - data_model = parse_data_models_from_search(detection['search']) - if data_model: - detection['data_model'] = data_model - - matched_deployment = get_deployments(detection, deployments) - detection['deployment'] = matched_deployment - nes_fields = get_nes_fields(detection['search'], detection['deployment']) - if len(nes_fields) > 0: - detection['nes_fields'] = nes_fields - - keys = ['mitre_attack', 'kill_chain_phases', 'cis20', 'nist'] - mappings = {} - for key in keys: - if key == 'mitre_attack': - if 'mitre_attack_id' in detection['tags']: - mappings[key] = detection['tags']['mitre_attack_id'] - else: - if key in detection['tags']: - mappings[key] = detection['tags'][key] - detection['mappings'] = mappings - - # used for upstream processing of risk scoring annotations in ECSU - # this is not currently compatible with newer instances of ESCU (6.3.0+) - # we are duplicating the code block above for now and just changing variable names to make future - # changes to this data structure separate from the mappings generation - # @todo expose the JSON data structure for newer risk type - annotation_keys = ['mitre_attack', 'kill_chain_phases', 'cis20', 'nist', 'analytic_story'] - savedsearch_annotations = {} - for key in annotation_keys: - if key == 'mitre_attack': - if 'mitre_attack_id' in detection['tags']: - savedsearch_annotations[key] = detection['tags']['mitre_attack_id'] - else: - if key in detection['tags']: - savedsearch_annotations[key] = detection['tags'][key] - detection['savedsearch_annotations'] = savedsearch_annotations - - if 'risk_object' in detection['tags']: - detection['risk_object'] = detection['tags']['risk_object'] - if 'risk_object_type' in detection['tags']: - detection['risk_object_type'] = detection['tags']['risk_object_type'] - if 'risk_score' in detection['tags']: - detection['risk_score'] = detection['tags']['risk_score'] - if 'product' in detection['tags']: - detection['product'] = detection['tags']['product'] - if (OUTPUT_PATH) == 'dist/saaws': - detection['disabled'] = 'false' - - - - for baseline in baselines: - data_model = parse_data_models_from_search(baseline['search']) - if data_model: - baseline['data_model'] = data_model - if (OUTPUT_PATH) == 'dist/saaws': - baseline['disabled'] = 'false' - - matched_deployment = get_deployments(baseline, deployments) - baseline['deployment'] = matched_deployment - - for response_task in response_tasks: - if 'search' in response_task: - data_model = parse_data_models_from_search(response_task['search']) - if data_model: - response_task['data_model'] = data_model - utc_time = datetime.datetime.utcnow().replace(microsecond=0).isoformat() j2_env = Environment(loader=FileSystemLoader(TEMPLATE_PATH), @@ -159,25 +90,7 @@ def generate_savedsearches_conf(detections, response_tasks, baselines, deploymen return output_path - def generate_analytic_story_conf(stories, detections, response_tasks, baselines, TEMPLATE_PATH, OUTPUT_PATH): - - sto_det = map_detection_to_stories(detections) - - sto_res = map_response_tasks_to_stories(response_tasks) - - sto_bas = map_baselines_to_stories(baselines) - - for story in stories: - if story['name'] in sto_det: - story['detections'] = list(sto_det[story['name']]) - if story['name'] in sto_res: - story['response_tasks'] = list(sto_res[story['name']]) - if story['name'] in sto_bas: - story['baselines'] = list(sto_bas[story['name']]) - - stories = prepare_stories(stories, detections) - utc_time = datetime.datetime.utcnow().replace(microsecond=0).isoformat() j2_env = Environment(loader=FileSystemLoader(TEMPLATE_PATH), @@ -190,38 +103,7 @@ def generate_analytic_story_conf(stories, detections, response_tasks, baselines, return output_path - def generate_use_case_library_conf(stories, detections, response_tasks, baselines, TEMPLATE_PATH, OUTPUT_PATH): - - sto_det = map_detection_to_stories(detections) - - sto_res = map_response_tasks_to_stories(response_tasks) - - for story in stories: - story['author_name'], story['author_company'] = parse_author_company(story) - if story['name'] in sto_det: - story['detections'] = list(sto_det[story['name']]) - - if story['name'] in sto_res: - - story['response_tasks'] = list(sto_res[story['name']]) - story['searches'] = story['detections'] + story['response_tasks'] - else: - story['searches'] = story['detections'] - - for detection in detections: - - keys = ['mitre_attack', 'kill_chain_phases', 'cis20', 'nist'] - mappings = {} - for key in keys: - if key == 'mitre_attack': - if 'mitre_attack_id' in detection['tags']: - mappings[key] = detection['tags']['mitre_attack_id'] - else: - if key in detection['tags']: - mappings[key] = detection['tags'][key] - detection['mappings'] = mappings - utc_time = datetime.datetime.utcnow().replace(microsecond=0).isoformat() j2_env = Environment(loader=FileSystemLoader(TEMPLATE_PATH), @@ -236,7 +118,6 @@ def generate_use_case_library_conf(stories, detections, response_tasks, baseline return output_path - def generate_macros_conf(macros, detections, TEMPLATE_PATH, OUTPUT_PATH): filter_macros = [] for detection in detections: @@ -261,20 +142,7 @@ def generate_macros_conf(macros, detections, TEMPLATE_PATH, OUTPUT_PATH): return output_path - def generate_workbench_panels(response_tasks, stories, TEMPLATE_PATH, OUTPUT_PATH): - - sto_res = map_response_tasks_to_stories(response_tasks) - - for story in stories: - if story['name'] in sto_res: - response_task_names = list(sto_res[story['name']]) - story['workbench_panels'] = [] - for response_task_name in response_task_names: - str = 'panel://workbench_panel_' + response_task_name[7:].replace(' ', '_').replace('-','_').replace('.','_').replace('/','_').lower() - story['workbench_panels'].append(str) - story['lowercase_name'] = story['name'].replace(' ', '_').replace('-','_').replace('.','_').replace('/','_').lower() - workbench_panel_objects = [] for response_task in response_tasks: if 'search' in response_task: @@ -368,23 +236,23 @@ def get_deployments(object, deployments): last_deployment = deployment else: last_deployment = matched_deployments[-1] - last_deployment = replace_vars_in_deployment(last_deployment, object) + # last_deployment = replace_vars_in_deployment(last_deployment, object) # print(last_deployment) return last_deployment -def replace_vars_in_deployment(deployment, object): - if 'alert_action' in deployment: - if 'email' in deployment['alert_action']: - deployment['alert_action']['email']['message']=re.sub(r'%([a-z_]+)%]', lambda x: object[x.group(1)], str(v)) - deployment['alert_action']['email']['subject']=re.sub(r'%([a-z_]+)%]', lambda x: object[x.group(1)], str(v)) +# def replace_vars_in_deployment(deployment, object): +# if 'alert_action' in deployment: +# if 'email' in deployment['alert_action']: +# deployment['alert_action']['email']['message']=re.sub(r'%([a-z_]+)%]', lambda x: object[x.group(1)], str(v)) +# deployment['alert_action']['email']['subject']=re.sub(r'%([a-z_]+)%]', lambda x: object[x.group(1)], str(v)) - if 'notable' in deployment: - deployment['alert_action']['notable']['rule_description']=re.sub(r'%([a-z_]+)%]', lambda x: object[x.group(1)], str(v)) - deployment['alert_action']['notable']['rule_title']=re.sub(r'%([a-z_]+)%]', lambda x: object[x.group(1)], str(v)) +# if 'notable' in deployment: +# deployment['alert_action']['notable']['rule_description']=re.sub(r'%([a-z_]+)%]', lambda x: object[x.group(1)], str(v)) +# deployment['alert_action']['notable']['rule_title']=re.sub(r'%([a-z_]+)%]', lambda x: object[x.group(1)], str(v)) - return deployment +# return deployment def get_nes_fields(search, deployment): @@ -399,21 +267,21 @@ def get_nes_fields(search, deployment): return nes_fields_matches -def map_detection_to_stories(detections): - sto_det = {} - for detection in detections: - if 'analytic_story' in detection['tags']: - for story in detection['tags']['analytic_story']: - if 'type' in detection.keys(): - if detection['type'] == 'batch': - rule_name = str('ESCU - ' + detection['name'] + ' - Rule') - else: - rule_name = str('ESCU - ' + detection['name'] + ' - Rule') - if not (story in sto_det): - sto_det[story] = {rule_name} - else: - sto_det[story].add(rule_name) - return sto_det +# def map_detection_to_stories(detections): +# sto_det = {} +# for detection in detections: +# if 'analytic_story' in detection['tags']: +# for story in detection['tags']['analytic_story']: +# if 'type' in detection.keys(): +# if detection['type'] == 'batch': +# rule_name = str('ESCU - ' + detection['name'] + ' - Rule') +# else: +# rule_name = str('ESCU - ' + detection['name'] + ' - Rule') +# if not (story in sto_det): +# sto_det[story] = {rule_name} +# else: +# sto_det[story].add(rule_name) +# return sto_det def map_response_tasks_to_stories(response_tasks): @@ -463,8 +331,82 @@ def custom_jinja2_enrichment_filter(string, object): return customized_string -def prepare_stories(stories, detections): +def prepare_detections(detections, deployments): + for detection in detections: + # parse out data_models + data_model = parse_data_models_from_search(detection['search']) + if data_model: + detection['data_model'] = data_model + matched_deployment = get_deployments(detection, deployments) + detection['deployment'] = matched_deployment + nes_fields = get_nes_fields(detection['search'], detection['deployment']) + if len(nes_fields) > 0: + detection['nes_fields'] = nes_fields + + keys = ['mitre_attack', 'kill_chain_phases', 'cis20', 'nist'] + mappings = {} + for key in keys: + if key == 'mitre_attack': + if 'mitre_attack_id' in detection['tags']: + mappings[key] = detection['tags']['mitre_attack_id'] + else: + if key in detection['tags']: + mappings[key] = detection['tags'][key] + detection['mappings'] = mappings + + # used for upstream processing of risk scoring annotations in ECSU + # this is not currently compatible with newer instances of ESCU (6.3.0+) + # we are duplicating the code block above for now and just changing variable names to make future + # changes to this data structure separate from the mappings generation + # @todo expose the JSON data structure for newer risk type + annotation_keys = ['mitre_attack', 'kill_chain_phases', 'cis20', 'nist', 'analytic_story'] + savedsearch_annotations = {} + for key in annotation_keys: + if key == 'mitre_attack': + if 'mitre_attack_id' in detection['tags']: + savedsearch_annotations[key] = detection['tags']['mitre_attack_id'] + else: + if key in detection['tags']: + savedsearch_annotations[key] = detection['tags'][key] + detection['savedsearch_annotations'] = savedsearch_annotations + + if 'risk_object' in detection['tags']: + detection['risk_object'] = detection['tags']['risk_object'] + if 'risk_object_type' in detection['tags']: + detection['risk_object_type'] = detection['tags']['risk_object_type'] + if 'risk_score' in detection['tags']: + detection['risk_score'] = detection['tags']['risk_score'] + if 'product' in detection['tags']: + detection['product'] = detection['tags']['product'] + if (OUTPUT_PATH) == 'dist/saaws': + detection['disabled'] = 'false' + + return detections + +def prepare_baselines(baselines, deployments): + for baseline in baselines: + data_model = parse_data_models_from_search(baseline['search']) + if data_model: + baseline['data_model'] = data_model + if (OUTPUT_PATH) == 'dist/saaws': + baseline['disabled'] = 'false' + + matched_deployment = get_deployments(baseline, deployments) + baseline['deployment'] = matched_deployment + + return baselines + +def prepare_response_tasks(response_tasks): + for response_task in response_tasks: + if 'search' in response_task: + data_model = parse_data_models_from_search(response_task['search']) + if data_model: + response_task['data_model'] = data_model + + return response_tasks + +def prepare_stories(stories, detections, response_tasks, baselines): # enrich stories with information from detections: data_models, mitre_ids, kill_chain_phases, nists sto_to_data_models = {} sto_to_mitre_attack_ids = {} @@ -521,7 +463,12 @@ def prepare_stories(stories, detections): else: sto_to_nists[story] = set(detection['tags']['nist']) + sto_res = map_response_tasks_to_stories(response_tasks) + sto_bas = map_baselines_to_stories(baselines) + for story in stories: + story['author_name'], story['author_company'] = parse_author_company(story) + story['lowercase_name'] = story['name'].replace(' ', '_').replace('-','_').replace('.','_').replace('/','_').lower() story['detections'] = sorted(sto_to_det[story['name']]) if story['name'] in sto_to_data_models: story['data_models'] = sorted(sto_to_data_models[story['name']]) @@ -533,6 +480,16 @@ def prepare_stories(stories, detections): story['cis20'] = sorted(sto_to_ciss[story['name']]) if story['name'] in sto_to_nists: story['nist'] = sorted(sto_to_nists[story['name']]) + if story['name'] in sto_res: + story['response_tasks'] = list(sto_res[story['name']]) + story['searches'] = story['detections'] + story['response_tasks'] + story['workbench_panels'] = [] + for response_task_name in story['response_tasks']: + str = 'panel://workbench_panel_' + response_task_name[7:].replace(' ', '_').replace('-','_').replace('.','_').replace('/','_').lower() + story['workbench_panels'].append(str) + if story['name'] in sto_bas: + story['baselines'] = list(sto_bas[story['name']]) + keys = ['mitre_attack', 'kill_chain_phases', 'cis20', 'nist'] mappings = {} @@ -572,29 +529,54 @@ def generate_mitre_lookup(OUTPUT_PATH): writer.writerows(csv_mitre_rows) +def import_objects(VERBOSE, REPO_PATH): + objects = { + "stories": load_objects("stories/*.yml", VERBOSE, REPO_PATH), + "macros": load_objects("macros/*.yml", VERBOSE, REPO_PATH), + "lookups": load_objects("lookups/*.yml", VERBOSE, REPO_PATH), + "baselines": load_objects("baselines/*.yml", VERBOSE, REPO_PATH), + "responses": load_objects("responses/*.yml", VERBOSE, REPO_PATH), + "response_tasks": load_objects("response_tasks/*.yml", VERBOSE, REPO_PATH), + "deployments": load_objects("deployments/*.yml", VERBOSE, REPO_PATH), + "detections": load_objects("detections/*/*.yml", VERBOSE, REPO_PATH), + } + objects["detections"].extend(load_objects("detections/*/*/*.yml", VERBOSE, REPO_PATH)) + + return objects + +def compute_objects(objects, PRODUCT): + if PRODUCT == "SAAWS": + objects["detections"] = [object for object in objects["detections"] if 'Splunk Security Analytics for AWS' in object['tags']['product']] + objects["stories"] = [object for object in objects["stories"] if 'Splunk Security Analytics for AWS' in object['tags']['product']] + objects["baselines"] = [object for object in objects["baselines"] if 'Splunk Security Analytics for AWS' in object['tags']['product']] + objects["response_tasks"] = [object for object in objects["response_tasks"] if 'Splunk Security Analytics for AWS' in object['tags']['product']] + + # only use ESCU detections to the configurations + objects["detections"] = sorted(filter(lambda d: d['type'].lower() == 'batch', objects["detections"]), key=lambda d: d['name']) + # only use ESCU stories to the configuration + objects["stories"] = sorted(filter(lambda s: s['type'].lower() == 'batch', objects["stories"]), key=lambda s: s['name']) + + objects["response_tasks"] = sorted(objects["response_tasks"], key=lambda i: i['name']) + objects["baselines"] = sorted(objects["baselines"], key=lambda b: b['name']) + objects["macros"] = sorted(objects["macros"], key=lambda m: m['name']) + + objects["detections"] = prepare_detections(objects["detections"], objects["deployments"]) + objects["baselines"] = prepare_baselines(objects["baselines"], objects["deployments"]) + objects["response_tasks"] = prepare_response_tasks(objects["response_tasks"]) + objects["stories"] = prepare_stories(objects["stories"], objects["detections"], objects["response_tasks"], objects["baselines"]) + + return objects + +def get_objects(REPO_PATH, OUTPUT_PATH, PRODUCT, VERBOSE): + objects = import_objects(VERBOSE, REPO_PATH) + objects = compute_objects(objects, PRODUCT) + return objects def main(REPO_PATH, OUTPUT_PATH, PRODUCT, VERBOSE): TEMPLATE_PATH = path.join(REPO_PATH, 'bin/jinja2_templates') - stories = load_objects("stories/*.yml", VERBOSE, REPO_PATH) - macros = load_objects("macros/*.yml", VERBOSE, REPO_PATH) - lookups = load_objects("lookups/*.yml", VERBOSE, REPO_PATH) - baselines = load_objects("baselines/*.yml", VERBOSE, REPO_PATH) - responses = load_objects("responses/*.yml", VERBOSE, REPO_PATH) - response_tasks = load_objects("response_tasks/*.yml", VERBOSE, REPO_PATH) - deployments = load_objects("deployments/*.yml", VERBOSE, REPO_PATH) - - # process all detections - detections = [] - detections = load_objects("detections/*/*.yml", VERBOSE, REPO_PATH) - detections.extend(load_objects("detections/*/*/*.yml", VERBOSE, REPO_PATH)) - - if PRODUCT == "SAAWS": - detections = [object for object in detections if 'Splunk Security Analytics for AWS' in object['tags']['product']] - stories = [object for object in stories if 'Splunk Security Analytics for AWS' in object['tags']['product']] - baselines = [object for object in baselines if 'Splunk Security Analytics for AWS' in object['tags']['product']] - response_tasks = [object for object in response_tasks if 'Splunk Security Analytics for AWS' in object['tags']['product']] + objects = get_objects(REPO_PATH, OUTPUT_PATH, PRODUCT, VERBOSE) try: if VERBOSE: @@ -604,39 +586,26 @@ def main(REPO_PATH, OUTPUT_PATH, PRODUCT, VERBOSE): print('Error: ' + str(e)) print("WARNING: Generation of Mitre lookup failed.") - lookups_path = generate_transforms_conf(lookups, TEMPLATE_PATH, OUTPUT_PATH) - lookups_path = generate_collections_conf(lookups, TEMPLATE_PATH, OUTPUT_PATH) + lookups_path = generate_transforms_conf(objects["lookups"], TEMPLATE_PATH, OUTPUT_PATH) + lookups_path = generate_collections_conf(objects["lookups"], TEMPLATE_PATH, OUTPUT_PATH) - detections = sorted(detections, key=lambda d: d['name']) + detection_path = generate_savedsearches_conf(objects["detections"], objects["response_tasks"], objects["baselines"], objects["deployments"], TEMPLATE_PATH, OUTPUT_PATH) - # only use ESCU detections to the configurations - detections = [object for object in detections if object["type"].lower() == "batch"] + story_path = generate_analytic_story_conf(objects["stories"], objects["detections"], objects["response_tasks"], objects["baselines"], TEMPLATE_PATH, OUTPUT_PATH) - response_tasks = sorted(response_tasks, key=lambda i: i['name']) - baselines = sorted(baselines, key=lambda b: b['name']) - detection_path = generate_savedsearches_conf(detections, response_tasks, baselines, deployments, TEMPLATE_PATH, OUTPUT_PATH) + use_case_lib_path = generate_use_case_library_conf(objects["stories"], objects["detections"], objects["response_tasks"], objects["baselines"], TEMPLATE_PATH, OUTPUT_PATH) - # only use ESCU stories to the configuration - stories = sorted(filter(lambda s: s['type'].lower() == 'batch', stories), key=lambda s: s['name']) + macros_path = generate_macros_conf(objects["macros"], objects["detections"], TEMPLATE_PATH, OUTPUT_PATH) - - - story_path = generate_analytic_story_conf(stories, detections, response_tasks, baselines, TEMPLATE_PATH, OUTPUT_PATH) - - use_case_lib_path = generate_use_case_library_conf(stories, detections, response_tasks, baselines, TEMPLATE_PATH, OUTPUT_PATH) - - macros = sorted(macros, key=lambda m: m['name']) - macros_path = generate_macros_conf(macros, detections, TEMPLATE_PATH, OUTPUT_PATH) - - workbench_panels_objects = generate_workbench_panels(response_tasks, stories, TEMPLATE_PATH, OUTPUT_PATH) + workbench_panels_objects = generate_workbench_panels(objects["response_tasks"], objects["stories"], TEMPLATE_PATH, OUTPUT_PATH) if VERBOSE: - print("{0} stories have been successfully written to {1}".format(len(stories), story_path)) - print("{0} detections have been successfully written to {1}".format(len(detections), detection_path)) - print("{0} response tasks have been successfully written to {1}".format(len(response_tasks), detection_path)) - print("{0} baselines have been successfully written to {1}".format(len(baselines), detection_path)) - print("{0} macros have been successfully written to {1}".format(len(macros), macros_path)) + print("{0} stories have been successfully written to {1}".format(len(objects["stories"]), story_path)) + print("{0} detections have been successfully written to {1}".format(len(objects["detections"]), detection_path)) + print("{0} response tasks have been successfully written to {1}".format(len(objects["response_tasks"]), detection_path)) + print("{0} baselines have been successfully written to {1}".format(len(objects["baselines"]), detection_path)) + print("{0} macros have been successfully written to {1}".format(len(objects["macros"]), macros_path)) print("{0} workbench panels have been successfully written to {1}, {2} and {3}".format(len(workbench_panels_objects), OUTPUT_PATH + "/default/es_investigations.conf", OUTPUT_PATH + "/default/workflow_actions.conf", OUTPUT_PATH + "/default/data/ui/panels/*")) print("security content generation completed..") From d7cece2f537cdacb0243afa190aec75561b23235 Mon Sep 17 00:00:00 2001 From: Thomas Nicholson Date: Thu, 29 Apr 2021 10:48:01 +0100 Subject: [PATCH 2/6] Update generate.py --- bin/generate.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/bin/generate.py b/bin/generate.py index 69b4b99c76..87b9bff9ea 100644 --- a/bin/generate.py +++ b/bin/generate.py @@ -331,7 +331,7 @@ def custom_jinja2_enrichment_filter(string, object): return customized_string -def prepare_detections(detections, deployments): +def prepare_detections(detections, deployments, OUTPUT_PATH): for detection in detections: # parse out data_models data_model = parse_data_models_from_search(detection['search']) @@ -384,7 +384,7 @@ def prepare_detections(detections, deployments): return detections -def prepare_baselines(baselines, deployments): +def prepare_baselines(baselines, deployments, OUTPUT_PATH): for baseline in baselines: data_model = parse_data_models_from_search(baseline['search']) if data_model: @@ -544,7 +544,7 @@ def import_objects(VERBOSE, REPO_PATH): return objects -def compute_objects(objects, PRODUCT): +def compute_objects(objects, PRODUCT, OUTPUT_PATH): if PRODUCT == "SAAWS": objects["detections"] = [object for object in objects["detections"] if 'Splunk Security Analytics for AWS' in object['tags']['product']] objects["stories"] = [object for object in objects["stories"] if 'Splunk Security Analytics for AWS' in object['tags']['product']] @@ -560,8 +560,8 @@ def compute_objects(objects, PRODUCT): objects["baselines"] = sorted(objects["baselines"], key=lambda b: b['name']) objects["macros"] = sorted(objects["macros"], key=lambda m: m['name']) - objects["detections"] = prepare_detections(objects["detections"], objects["deployments"]) - objects["baselines"] = prepare_baselines(objects["baselines"], objects["deployments"]) + objects["detections"] = prepare_detections(objects["detections"], objects["deployments"], OUTPUT_PATH) + objects["baselines"] = prepare_baselines(objects["baselines"], objects["deployments"], OUTPUT_PATH) objects["response_tasks"] = prepare_response_tasks(objects["response_tasks"]) objects["stories"] = prepare_stories(objects["stories"], objects["detections"], objects["response_tasks"], objects["baselines"]) @@ -569,7 +569,7 @@ def compute_objects(objects, PRODUCT): def get_objects(REPO_PATH, OUTPUT_PATH, PRODUCT, VERBOSE): objects = import_objects(VERBOSE, REPO_PATH) - objects = compute_objects(objects, PRODUCT) + objects = compute_objects(objects, PRODUCT, OUTPUT_PATH) return objects def main(REPO_PATH, OUTPUT_PATH, PRODUCT, VERBOSE): From e9e1be755f273a602227789d72831c1f51d0e12d Mon Sep 17 00:00:00 2001 From: Thomas Nicholson Date: Thu, 29 Apr 2021 11:44:25 +0100 Subject: [PATCH 3/6] Update generate.py --- bin/generate.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/generate.py b/bin/generate.py index 87b9bff9ea..2b39b0b242 100644 --- a/bin/generate.py +++ b/bin/generate.py @@ -485,8 +485,8 @@ def prepare_stories(stories, detections, response_tasks, baselines): story['searches'] = story['detections'] + story['response_tasks'] story['workbench_panels'] = [] for response_task_name in story['response_tasks']: - str = 'panel://workbench_panel_' + response_task_name[7:].replace(' ', '_').replace('-','_').replace('.','_').replace('/','_').lower() - story['workbench_panels'].append(str) + s = 'panel://workbench_panel_' + response_task_name[7:].replace(' ', '_').replace('-','_').replace('.','_').replace('/','_').lower() + story['workbench_panels'].append(s) if story['name'] in sto_bas: story['baselines'] = list(sto_bas[story['name']]) From 581bd1b3a7ce0c625f37d077b9c3f0fed3fe3e09 Mon Sep 17 00:00:00 2001 From: Thomas Nicholson Date: Thu, 29 Apr 2021 11:59:38 +0100 Subject: [PATCH 4/6] Update generate.py --- bin/generate.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bin/generate.py b/bin/generate.py index 2b39b0b242..a9c1a9a401 100644 --- a/bin/generate.py +++ b/bin/generate.py @@ -470,6 +470,7 @@ def prepare_stories(stories, detections, response_tasks, baselines): story['author_name'], story['author_company'] = parse_author_company(story) story['lowercase_name'] = story['name'].replace(' ', '_').replace('-','_').replace('.','_').replace('/','_').lower() story['detections'] = sorted(sto_to_det[story['name']]) + story['searches'] = story['detections'] if story['name'] in sto_to_data_models: story['data_models'] = sorted(sto_to_data_models[story['name']]) if story['name'] in sto_to_mitre_attack_ids: @@ -482,7 +483,7 @@ def prepare_stories(stories, detections, response_tasks, baselines): story['nist'] = sorted(sto_to_nists[story['name']]) if story['name'] in sto_res: story['response_tasks'] = list(sto_res[story['name']]) - story['searches'] = story['detections'] + story['response_tasks'] + story['searches'] = story['searches'] + story['response_tasks'] story['workbench_panels'] = [] for response_task_name in story['response_tasks']: s = 'panel://workbench_panel_' + response_task_name[7:].replace(' ', '_').replace('-','_').replace('.','_').replace('/','_').lower() From fbf3edaa11a7afc47788f2c23284120740b59b16 Mon Sep 17 00:00:00 2001 From: Thomas Nicholson Date: Thu, 29 Apr 2021 12:39:56 +0100 Subject: [PATCH 5/6] Update generate.py --- bin/generate.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/generate.py b/bin/generate.py index a9c1a9a401..41c83f5e50 100644 --- a/bin/generate.py +++ b/bin/generate.py @@ -482,14 +482,14 @@ def prepare_stories(stories, detections, response_tasks, baselines): if story['name'] in sto_to_nists: story['nist'] = sorted(sto_to_nists[story['name']]) if story['name'] in sto_res: - story['response_tasks'] = list(sto_res[story['name']]) + story['response_tasks'] = sorted(list(sto_res[story['name']])) story['searches'] = story['searches'] + story['response_tasks'] story['workbench_panels'] = [] for response_task_name in story['response_tasks']: s = 'panel://workbench_panel_' + response_task_name[7:].replace(' ', '_').replace('-','_').replace('.','_').replace('/','_').lower() story['workbench_panels'].append(s) if story['name'] in sto_bas: - story['baselines'] = list(sto_bas[story['name']]) + story['baselines'] = sorted(list(sto_bas[story['name']])) keys = ['mitre_attack', 'kill_chain_phases', 'cis20', 'nist'] From 8d22e9934f8ec883b9ecda1e2aefa5e7e9fa95d5 Mon Sep 17 00:00:00 2001 From: Thomas Nicholson Date: Thu, 29 Apr 2021 12:48:12 +0100 Subject: [PATCH 6/6] Update generate.py --- bin/generate.py | 35 +---------------------------------- 1 file changed, 1 insertion(+), 34 deletions(-) diff --git a/bin/generate.py b/bin/generate.py index 41c83f5e50..81cb23e938 100644 --- a/bin/generate.py +++ b/bin/generate.py @@ -187,7 +187,6 @@ def parse_data_models_from_search(search): return match.group(1) return False - def parse_author_company(story): match_author = re.search(r'^([^,]+)', story['author']) if match_author is None: @@ -236,25 +235,11 @@ def get_deployments(object, deployments): last_deployment = deployment else: last_deployment = matched_deployments[-1] - # last_deployment = replace_vars_in_deployment(last_deployment, object) + # last_deployment = replace_vars_in_deployment(last_deployment, object) # Not needed because of custom_jinja2_enrichment_filter # print(last_deployment) return last_deployment - -# def replace_vars_in_deployment(deployment, object): -# if 'alert_action' in deployment: -# if 'email' in deployment['alert_action']: -# deployment['alert_action']['email']['message']=re.sub(r'%([a-z_]+)%]', lambda x: object[x.group(1)], str(v)) -# deployment['alert_action']['email']['subject']=re.sub(r'%([a-z_]+)%]', lambda x: object[x.group(1)], str(v)) - -# if 'notable' in deployment: -# deployment['alert_action']['notable']['rule_description']=re.sub(r'%([a-z_]+)%]', lambda x: object[x.group(1)], str(v)) -# deployment['alert_action']['notable']['rule_title']=re.sub(r'%([a-z_]+)%]', lambda x: object[x.group(1)], str(v)) - -# return deployment - - def get_nes_fields(search, deployment): nes_fields_matches = [] if 'alert_action' in deployment: @@ -267,23 +252,6 @@ def get_nes_fields(search, deployment): return nes_fields_matches -# def map_detection_to_stories(detections): -# sto_det = {} -# for detection in detections: -# if 'analytic_story' in detection['tags']: -# for story in detection['tags']['analytic_story']: -# if 'type' in detection.keys(): -# if detection['type'] == 'batch': -# rule_name = str('ESCU - ' + detection['name'] + ' - Rule') -# else: -# rule_name = str('ESCU - ' + detection['name'] + ' - Rule') -# if not (story in sto_det): -# sto_det[story] = {rule_name} -# else: -# sto_det[story].add(rule_name) -# return sto_det - - def map_response_tasks_to_stories(response_tasks): sto_res = {} for response_task in response_tasks: @@ -301,7 +269,6 @@ def map_response_tasks_to_stories(response_tasks): sto_res[story].add(task_name) return sto_res - def map_baselines_to_stories(baselines): sto_bas = {} for baseline in baselines: