From 702a0bc932b40b92c33038f3c7f29efd6031d56d Mon Sep 17 00:00:00 2001 From: divious1 Date: Mon, 15 Feb 2021 08:25:52 -0500 Subject: [PATCH 01/14] skeleton --- bin/generate.py | 16 ++----- bin/validate.py | 18 ++------ contentctl.py | 120 ++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 127 insertions(+), 27 deletions(-) create mode 100644 contentctl.py diff --git a/bin/generate.py b/bin/generate.py index 7c59709d12..24669f333c 100644 --- a/bin/generate.py +++ b/bin/generate.py @@ -602,21 +602,11 @@ def generate_mitre_lookup(OUTPUT_PATH): -def main(args): +def new(security_content_path, output, VERBOSE): - parser = argparse.ArgumentParser(description="generates splunk conf files out of security_content manifests", epilog=""" - This tool converts manifests to the source files to be used by products like Splunk Enterprise. - It generates the savesearches.conf, analytics_stories.conf files for ES.""") - parser.add_argument("-p", "--path", required=True, help="path to security_content repo") - parser.add_argument("-o", "--output", required=True, help="path to the output directory") - parser.add_argument("-v", "--verbose", required=False, default=False, action='store_true', help="prints verbose output") - - # parse them - args = parser.parse_args() - REPO_PATH = args.path - OUTPUT_PATH = args.output + REPO_PATH = security_content_path + OUTPUT_PATH = output TEMPLATE_PATH = path.join(REPO_PATH, 'bin/jinja2_templates') - VERBOSE = args.verbose stories = load_objects("stories/*.yml", VERBOSE, REPO_PATH) macros = load_objects("macros/*.yml", VERBOSE, REPO_PATH) lookups = load_objects("lookups/*.yml", VERBOSE, REPO_PATH) diff --git a/bin/validate.py b/bin/validate.py index 2ecc721ef7..b578b28428 100644 --- a/bin/validate.py +++ b/bin/validate.py @@ -16,7 +16,7 @@ import re from os import path, walk -def validate_schema(REPO_PATH, type, objects): +def validate_schema(REPO_PATH, type, objects, verbose): error = False errors = [] @@ -222,18 +222,8 @@ def validate_lookups_content(REPO_PATH, lookup_path, lookup): return errors - -if __name__ == "__main__": - # grab arguments - parser = argparse.ArgumentParser(description="validates security content manifest files", epilog=""" - Validates security manifest for correctness, adhering to spec and other common items. - VALIDATE DOES NOT PROCESS RESPONSES SPEC for the moment.""") - parser.add_argument("-p", "--path", required=True, help="path to security-security content repo") - parser.add_argument("-v", "--verbose", required=False, action='store_true', help="prints verbose output") - # parse them - args = parser.parse_args() - REPO_PATH = args.path - verbose = args.verbose +def new(security_content_path, verbose): + REPO_PATH = security_content_path validation_objects = ['macros','lookups','stories','detections','baselines','response_tasks','responses','deployments'] @@ -242,7 +232,7 @@ if __name__ == "__main__": schema_errors = [] for validation_object in validation_objects: - objects, error, errors = validate_schema(REPO_PATH, validation_object, objects) + objects, error, errors = validate_schema(REPO_PATH, validation_object, objects, verbose) schema_error = schema_error or error if len(errors) > 0: schema_errors = schema_errors + errors diff --git a/contentctl.py b/contentctl.py new file mode 100644 index 0000000000..6014eeaa4e --- /dev/null +++ b/contentctl.py @@ -0,0 +1,120 @@ +import os +import sys +import argparse +from bin import validate as validator +from bin import generate as generator +from pathlib import Path + +VERSION = 1 + + +def init(args): + path = args.path + print(""" +Running Splunk Security Content Control Tool (contentctl) v{0} +starting program loaded for TIE Fighter... + _ _ + T T T T + | | | | + | | | | + | | | | + | | | | + | | | | + | | | | + | | ____ | | + | | ___.r-"`--'"-r.____ | | + | |.-._,.,---~"_/_/ .----. \_\_"~---,.,_,-.| | + | ]|.[_]_ T~T[_.-Y / \ / \ Y-._]T~T _[_].|| | + [|-+[ ___]| [__ |-=[--()--]=-| __] |[___ ]+-|] + | ]|"[_] l_j[_"-l \ / \ / !-"_]l_j [_]~|| | + | |`-' "~"---.,_\"\ "o--o" /"/_,.---"~" `-'| | + | | ~~"^-.____.-^"~~ | | + | | | | + | | | | + | | | | + | | | | + | | | | + | | | | + | | | | + l_i l_j -Row + + """.format(VERSION)) + + # parse config + security_content_path = Path(path).resolve() + if security_content_path.is_dir(): + print("contentctl is reading from path {0}".format( + security_content_path)) + else: + print("ERROR: contentctl failed to find security_content project") + sys.exit(1) + return str(security_content_path) + + +def new(args): + security_content_path = init(args) + +def validate(args): + security_content_path = init(args) + # hard setting verbosity for now + VERBOSE = True + print("contentctl is validating all content under {0}".format(security_content_path)) + validator.new(security_content_path, VERBOSE) + + +def generate(args): + security_content_path = init(args) + # hard setting verbosity for now + VERBOSE = True + + output = Path(args.output).resolve() + if output.is_dir(): + print("contentctl is using folder {0} to write deployment".format( + output)) + else: + print("ERROR: contentctl failed to find folder for deployment {0}".format(output)) + sys.exit(1) + + print("contentctl is generating a new splunk_app under ".format(output)) + generator.new(security_content_path, args.output, VERBOSE) + + +def main(args): + # grab arguments + parser = argparse.ArgumentParser( + description="Use `contentctl.py action -h` to get help with any Splunk Security Content action") + parser.add_argument("-p", "--path", required=False, default=".", + help="path to the Splunk Security Content. Defaults to `.`") + parser.add_argument("-v", "--version", default=False, action="version", version="version: {0}".format(VERSION), + help="shows current contentctl version") + parser.set_defaults(func=lambda _: parser.print_help()) + + actions_parser = parser.add_subparsers(title="Splunk Security Content actions", dest="action") + new_parser = actions_parser.add_parser("new", help="Create new content (detections, stories, workbooks)") + validate_parser = actions_parser.add_parser("validate", help="Validates written content") + generate_parser = actions_parser.add_parser("generate", help="Generates a deployment package for different platforms (splunk_app)") + + # new arguments + new_parser.add_argument("-t", "--type", required=False, type=str, default="detection", + help="Type of new content to create, please chose between detection, workbook, or story") + new_parser.set_defaults(func=new) + + # validate arguments + validate_parser.set_defaults(func=validate, epilog=""" + Validates security manifest for correctness, adhering to spec and other common items. + VALIDATE DOES NOT PROCESS RESPONSES SPEC for the moment.""") + + # generate arguments + generate_parser.add_argument("-f", "--format", required=False, type=str, default="splunk_app", + help="Format of our deployment package, defaults to `splunk_app`.\n The deployment `splunk_app` runs on product Splunk Enterprise Security and Splunk Enterprise.") + generate_parser.add_argument("-o", "--output", required=False, type=str, default="package", + help="Path where to store the deployment package, defaults to `package`") + generate_parser.set_defaults(func=generate) + + # # parse them + args = parser.parse_args() + return args.func(args) + + +if __name__ == "__main__": + main(sys.argv[1:]) From b82c5bcc9b57057a3447b29f367de2db526f5bef Mon Sep 17 00:00:00 2001 From: divious1 Date: Mon, 15 Feb 2021 22:25:18 -0500 Subject: [PATCH 02/14] added skeleton for wizar, also example creation for detections --- .gitignore | 4 +++ bin/jinja2_templates/detection.j2 | 33 +++++++++++++++++++++ bin/newcontent.py | 49 +++++++++++++++++++++++++++++++ contentctl.py | 15 ++++++++-- requirements.txt | 6 ++++ 5 files changed, 105 insertions(+), 2 deletions(-) create mode 100644 bin/jinja2_templates/detection.j2 create mode 100644 bin/newcontent.py diff --git a/.gitignore b/.gitignore index 512c76d162..a63d0fcb80 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,7 @@ +# Ignore example files from contentctl tool + +detections/*/*_example.yml + # usual mac files .DS_Store #vim files diff --git a/bin/jinja2_templates/detection.j2 b/bin/jinja2_templates/detection.j2 new file mode 100644 index 0000000000..b4b366a8c9 --- /dev/null +++ b/bin/jinja2_templates/detection.j2 @@ -0,0 +1,33 @@ +name: {{name}} +id: {{uuid}} +version: 1 +date: '{{date}}' +author: {{author}} +type: {{type}} +datamodel: +{% for datamodel in datamodels -%} + - {{datamodel}} +{% endfor -%} +description: {{description}} +search: '{{search}}' +how_to_implement: {{how_to_implement}} +known_false_positives: {{known_false_positives}} +references: +{% for reference in references -%} + - {{reference}} +{% endfor -%} +tags: + analytic_story: + - {{analytic_story_name}} + dataset: + - {{dataset_url}} + kill_chain_phases: + {% for kill_chain_phase in kill_chain_phases -%} + - {{kill_chain_phase}} + {% endfor -%} + mitre_attack_id: + - {{mitre_attack_id}} + product: + {% for product in products -%} + - {{product}} + {% endfor -%} diff --git a/bin/newcontent.py b/bin/newcontent.py new file mode 100644 index 0000000000..0d9bdd9988 --- /dev/null +++ b/bin/newcontent.py @@ -0,0 +1,49 @@ +#!/usr/bin/python + +''' +Helps you create new Splunk Security Content. +''' + +from pathlib import Path +from PyInquirer import prompt, Separator +import os +import getpass +from jinja2 import Environment, FileSystemLoader +import uuid +from datetime import date +from os import path + + +def create_example(security_content_path,type, TEMPLATE_PATH): + getpass.getuser() + + if type == 'detection': + j2_env = Environment(loader=FileSystemLoader(TEMPLATE_PATH), + trim_blocks=True) + template = j2_env.get_template('detection.j2') + example_name = getpass.getuser() + '_' + type + '_example.yml' + output_path = path.join(security_content_path, 'detections/endpoint/' + example_name) + output = template.render(uuid=uuid.uuid1(), date=date.today().strftime('%Y-%m-%d'), + author='Robert Johansson', name=getpass.getuser().capitalize() + ' ' + type.capitalize() + ' Example', + description='Describe your detection the best way possible, if you need inspiration just look over others.', + how_to_implement='How would a user implement this detection, describe any TAs, or specific configuration they might require', + known_false_positives='Although unlikely, some legitimate applications may exhibit this behavior, triggering a false positive.', + references=['https://wearebob.fandom.com/wiki/Bob','https://en.wikipedia.org/wiki/Dennis_E._Taylor'], + datamodels=['Endpoint'], search='SPLUNKSPLv1GOESHERE', type='batch', analytic_story_name='Story Name Goes Here', + mitre_attack_id = 'T0000.00', kill_chain_phases=['Exploitation'], dataset_url='https://github.com/splunk/attack_data/', + products=['Splunk Enterprise','Splunk Enterprise Security','Splunk Cloud']) + with open(output_path, 'w', encoding="utf-8") as f: + f.write(output) + print("contentctl wrote a example detection to: {0}".format(output_path)) + +def new(security_content_path, VERBOSE, type, example_only): + + valid_content_objects = ['detection','workbook','story'] + if type not in valid_content_objects: + print("ERROR: content type: {0} is not valid, please use: {1}".format(type, str(valid_content_objects))) + sys.exit(1) + + TEMPLATE_PATH = path.join(security_content_path, 'bin/jinja2_templates') + + if example_only: + create_example(security_content_path,type, TEMPLATE_PATH) diff --git a/contentctl.py b/contentctl.py index 6014eeaa4e..d755c3d484 100644 --- a/contentctl.py +++ b/contentctl.py @@ -3,6 +3,7 @@ import sys import argparse from bin import validate as validator from bin import generate as generator +from bin import newcontent as content from pathlib import Path VERSION = 1 @@ -53,11 +54,17 @@ starting program loaded for TIE Fighter... def new(args): security_content_path = init(args) + VERBOSE = not args.silence + + # hard setting verbosity for now + print("contentctl is creating a new {0} under {0}".format(args.type, security_content_path)) + content.new(security_content_path, VERBOSE, args.type, args.example_only) def validate(args): security_content_path = init(args) + VERBOSE = not args.silence + # hard setting verbosity for now - VERBOSE = True print("contentctl is validating all content under {0}".format(security_content_path)) validator.new(security_content_path, VERBOSE) @@ -65,7 +72,7 @@ def validate(args): def generate(args): security_content_path = init(args) # hard setting verbosity for now - VERBOSE = True + VERBOSE = not args.silence output = Path(args.output).resolve() if output.is_dir(): @@ -87,6 +94,8 @@ def main(args): help="path to the Splunk Security Content. Defaults to `.`") parser.add_argument("-v", "--version", default=False, action="version", version="version: {0}".format(VERSION), help="shows current contentctl version") + parser.add_argument("-s", "--silence", required=False, action='store_true', + help="silences all verbose output, defaults to False") parser.set_defaults(func=lambda _: parser.print_help()) actions_parser = parser.add_subparsers(title="Splunk Security Content actions", dest="action") @@ -97,6 +106,8 @@ def main(args): # new arguments new_parser.add_argument("-t", "--type", required=False, type=str, default="detection", help="Type of new content to create, please chose between detection, workbook, or story") + new_parser.add_argument("-x", "--example_only", required=False, action='store_true', + help="Generates an example content with UPDATETHIS where a value is required. Use `git status` to see what specific files are added. Skips new content wizard prompts.") new_parser.set_defaults(func=new) # validate arguments diff --git a/requirements.txt b/requirements.txt index 8919522ce6..b62e8bb320 100644 --- a/requirements.txt +++ b/requirements.txt @@ -24,10 +24,14 @@ more-itertools==8.6.0 nodeenv==1.5.0 pathlib2==2.3.5 pre-commit==2.9.3 +prompt-toolkit==1.0.14 +Pygments==2.8.0 +PyInquirer==1.0.3 pyrsistent==0.17.3 python-dateutil==2.8.1 pytz==2021.1 PyYAML==5.4.1 +regex==2020.11.13 requests==2.25.1 scandir==1.10.0 semantic-version==2.8.5 @@ -36,9 +40,11 @@ six==1.15.0 sly==0.4 smmap==3.0.5 stix2==2.1.0 +stix2-patterns==1.2.1 taxii2-client==2.2.2 toml==0.10.2 typing==3.7.4.3 urllib3==1.26.3 virtualenv==20.4.2 +wcwidth==0.2.5 zipp==3.4.0 From 7faeae2949be58ee1de879af81e209bc6c81073f Mon Sep 17 00:00:00 2001 From: divious1 Date: Mon, 15 Feb 2021 22:32:26 -0500 Subject: [PATCH 03/14] added filter --- bin/newcontent.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/bin/newcontent.py b/bin/newcontent.py index 0d9bdd9988..4e5272d5f0 100644 --- a/bin/newcontent.py +++ b/bin/newcontent.py @@ -29,8 +29,9 @@ def create_example(security_content_path,type, TEMPLATE_PATH): how_to_implement='How would a user implement this detection, describe any TAs, or specific configuration they might require', known_false_positives='Although unlikely, some legitimate applications may exhibit this behavior, triggering a false positive.', references=['https://wearebob.fandom.com/wiki/Bob','https://en.wikipedia.org/wiki/Dennis_E._Taylor'], - datamodels=['Endpoint'], search='SPLUNKSPLv1GOESHERE', type='batch', analytic_story_name='Story Name Goes Here', - mitre_attack_id = 'T0000.00', kill_chain_phases=['Exploitation'], dataset_url='https://github.com/splunk/attack_data/', + datamodels=['Endpoint'], search='SPLUNKSPLv1GOESHERE | `' + getpass.getuser() + '_' + type + '_example_filter`', + type='batch', analytic_story_name='Story Name Goes Here', mitre_attack_id = 'T0000.00', + kill_chain_phases=['Exploitation'], dataset_url='https://github.com/splunk/attack_data/', products=['Splunk Enterprise','Splunk Enterprise Security','Splunk Cloud']) with open(output_path, 'w', encoding="utf-8") as f: f.write(output) From 6cc4bdddb69971c792216797900f3216a1acf460 Mon Sep 17 00:00:00 2001 From: divious1 Date: Tue, 16 Feb 2021 16:18:06 -0500 Subject: [PATCH 04/14] missing import --- bin/newcontent.py | 1 + 1 file changed, 1 insertion(+) diff --git a/bin/newcontent.py b/bin/newcontent.py index 4e5272d5f0..d77acdaa33 100644 --- a/bin/newcontent.py +++ b/bin/newcontent.py @@ -12,6 +12,7 @@ from jinja2 import Environment, FileSystemLoader import uuid from datetime import date from os import path +import sys def create_example(security_content_path,type, TEMPLATE_PATH): From 8d03d90493b4dcba24d1724e46022493598d4176 Mon Sep 17 00:00:00 2001 From: divious1 Date: Tue, 16 Feb 2021 22:17:18 -0500 Subject: [PATCH 05/14] added test file for generation example --- bin/jinja2_templates/test.j2 | 12 ++++++++++++ bin/newcontent.py | 27 +++++++++++++++++++++------ 2 files changed, 33 insertions(+), 6 deletions(-) create mode 100644 bin/jinja2_templates/test.j2 diff --git a/bin/jinja2_templates/test.j2 b/bin/jinja2_templates/test.j2 new file mode 100644 index 0000000000..a57e99ba53 --- /dev/null +++ b/bin/jinja2_templates/test.j2 @@ -0,0 +1,12 @@ +name: {{name}} +tests: +- name: {{detection_name}} + file: {{detection_path}} + pass_condition: '{{pass_condition}}' + earliest_time: '{{earliest_time}}' + latest_time: '{{latest_time}}' + attack_data: + - file_name: {{file_name}} + data: {{dataset_url}} + source: {{splunk_source}} + sourcetype: {{splunk_sourcetype}} diff --git a/bin/newcontent.py b/bin/newcontent.py index d77acdaa33..42d0acc670 100644 --- a/bin/newcontent.py +++ b/bin/newcontent.py @@ -12,7 +12,6 @@ from jinja2 import Environment, FileSystemLoader import uuid from datetime import date from os import path -import sys def create_example(security_content_path,type, TEMPLATE_PATH): @@ -21,23 +20,39 @@ def create_example(security_content_path,type, TEMPLATE_PATH): if type == 'detection': j2_env = Environment(loader=FileSystemLoader(TEMPLATE_PATH), trim_blocks=True) + + # write a detection example template = j2_env.get_template('detection.j2') - example_name = getpass.getuser() + '_' + type + '_example.yml' - output_path = path.join(security_content_path, 'detections/endpoint/' + example_name) + detection_name = getpass.getuser() + '_' + type + '_example.yml' + output_path = path.join(security_content_path, 'detections/endpoint/' + detection_name) output = template.render(uuid=uuid.uuid1(), date=date.today().strftime('%Y-%m-%d'), author='Robert Johansson', name=getpass.getuser().capitalize() + ' ' + type.capitalize() + ' Example', description='Describe your detection the best way possible, if you need inspiration just look over others.', how_to_implement='How would a user implement this detection, describe any TAs, or specific configuration they might require', known_false_positives='Although unlikely, some legitimate applications may exhibit this behavior, triggering a false positive.', references=['https://wearebob.fandom.com/wiki/Bob','https://en.wikipedia.org/wiki/Dennis_E._Taylor'], - datamodels=['Endpoint'], search='SPLUNKSPLv1GOESHERE | `' + getpass.getuser() + '_' + type + '_example_filter`', - type='batch', analytic_story_name='Story Name Goes Here', mitre_attack_id = 'T0000.00', - kill_chain_phases=['Exploitation'], dataset_url='https://github.com/splunk/attack_data/', + datamodels=['Endpoint'], search='SPLUNKSPLGOESHERE | `' + getpass.getuser() + '_' + type + '_example_filter`', + type='batch', analytic_story_name='STORY NAME GOES HERE', mitre_attack_id = 'T1003.01', + kill_chain_phases=['Exploitation'], dataset_url='https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1003.001/atomic_red_team/windows-sysmon.log', products=['Splunk Enterprise','Splunk Enterprise Security','Splunk Cloud']) with open(output_path, 'w', encoding="utf-8") as f: f.write(output) print("contentctl wrote a example detection to: {0}".format(output_path)) + # and a corresponding test files + template = j2_env.get_template('test.j2') + test_name = getpass.getuser() + '_' + type + '_example.test.yml' + output_path = path.join(security_content_path, 'tests/endpoint/' + test_name) + output = template.render(name=getpass.getuser().capitalize() + ' ' + type.capitalize() + ' Example Unit Test', + detection_name=getpass.getuser().capitalize() + ' ' + type.capitalize() + ' Example', + detection_path='detections/endpoint/' + detection_name, pass_condition='| stats count | where count > 0', + earliest_time='-24h', latest_time='now', file_name='windows-sysmon.log', splunk_source='XmlWinEventLog:Microsoft-Windows-Sysmon/Operational', + splunk_sourcetype='xmlwineventlog',dataset_url='https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1003.001/atomic_red_team/windows-sysmon.log') + with open(output_path, 'w', encoding="utf-8") as f: + f.write(output) + print("contentctl wrote a example test for this detection to: {0}".format(output_path)) + + def new(security_content_path, VERBOSE, type, example_only): valid_content_objects = ['detection','workbook','story'] From 8d7784ea377cc550db1457e1b8b852cc90fe1a51 Mon Sep 17 00:00:00 2001 From: divious1 Date: Tue, 16 Feb 2021 22:56:36 -0500 Subject: [PATCH 06/14] added example for story --- .gitignore | 2 ++ bin/jinja2_templates/story.j2 | 24 ++++++++++++++++++++++++ bin/newcontent.py | 22 +++++++++++++++++++--- contentctl.py | 2 +- 4 files changed, 46 insertions(+), 4 deletions(-) create mode 100644 bin/jinja2_templates/story.j2 diff --git a/.gitignore b/.gitignore index a63d0fcb80..5f5e0d4c99 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,8 @@ # Ignore example files from contentctl tool detections/*/*_example.yml +stories/*_example.yml +tests/*/*_example.yml # usual mac files .DS_Store diff --git a/bin/jinja2_templates/story.j2 b/bin/jinja2_templates/story.j2 new file mode 100644 index 0000000000..be09c2f78f --- /dev/null +++ b/bin/jinja2_templates/story.j2 @@ -0,0 +1,24 @@ +name: {{name}} +id: {{uuid}} +version: 1 +date: '{{date}}' +author: {{author}} +type: {{type}} +description: {{description}} +narrative: {{narrative}} +references: +{% for reference in references -%} + - {{reference}} +{% endfor -%} +tags: + analytic_story: + - {{analytic_story_name}} + category: + {% for category in categories -%} + - {{category}} + {% endfor -%} + product: + {% for product in products -%} + - {{product}} + {% endfor -%} + usecase: {{usecase}} diff --git a/bin/newcontent.py b/bin/newcontent.py index 42d0acc670..6d3368e957 100644 --- a/bin/newcontent.py +++ b/bin/newcontent.py @@ -16,10 +16,11 @@ from os import path def create_example(security_content_path,type, TEMPLATE_PATH): getpass.getuser() + j2_env = Environment(loader=FileSystemLoader(TEMPLATE_PATH), + trim_blocks=True) if type == 'detection': - j2_env = Environment(loader=FileSystemLoader(TEMPLATE_PATH), - trim_blocks=True) + # write a detection example template = j2_env.get_template('detection.j2') @@ -52,10 +53,25 @@ def create_example(security_content_path,type, TEMPLATE_PATH): f.write(output) print("contentctl wrote a example test for this detection to: {0}".format(output_path)) + elif type == 'story': + # write a detection example + template = j2_env.get_template('story.j2') + story_name = getpass.getuser() + '_' + type + '_example.yml' + output_path = path.join(security_content_path, 'stories/' + story_name) + output = template.render(uuid=uuid.uuid1(), date=date.today().strftime('%Y-%m-%d'), + author='Robert Johansson', name=getpass.getuser().capitalize() + ' ' + type.capitalize() + ' Example', + description='Describe your story the best way possible, if you need inspiration just look over others.', + narrative='Explain why should a SOC manager or Director care about this use case, if you need inspiration just look over others.', + references=['https://wearebob.fandom.com/wiki/Bob','https://en.wikipedia.org/wiki/Dennis_E._Taylor'], + type='batch', analytic_story_name=getpass.getuser().capitalize() + ' ' + type.capitalize() + ' Example', + category=['Adversary Tactics'], usecase='Advanced Threat Detection', products=['Splunk Enterprise','Splunk Enterprise Security','Splunk Cloud']) + with open(output_path, 'w', encoding="utf-8") as f: + f.write(output) + print("contentctl wrote a example story to: {0}".format(output_path)) def new(security_content_path, VERBOSE, type, example_only): - valid_content_objects = ['detection','workbook','story'] + valid_content_objects = ['detection','story'] if type not in valid_content_objects: print("ERROR: content type: {0} is not valid, please use: {1}".format(type, str(valid_content_objects))) sys.exit(1) diff --git a/contentctl.py b/contentctl.py index d755c3d484..1521bd13d4 100644 --- a/contentctl.py +++ b/contentctl.py @@ -105,7 +105,7 @@ def main(args): # new arguments new_parser.add_argument("-t", "--type", required=False, type=str, default="detection", - help="Type of new content to create, please chose between detection, workbook, or story") + help="Type of new content to create, please chose between detection or story") new_parser.add_argument("-x", "--example_only", required=False, action='store_true', help="Generates an example content with UPDATETHIS where a value is required. Use `git status` to see what specific files are added. Skips new content wizard prompts.") new_parser.set_defaults(func=new) From 85d657e97713d0f28c0587ee995262e02d54f290 Mon Sep 17 00:00:00 2001 From: divious1 Date: Tue, 16 Feb 2021 23:39:30 -0500 Subject: [PATCH 07/14] skeleton of wizar --- bin/newcontent.py | 126 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 126 insertions(+) diff --git a/bin/newcontent.py b/bin/newcontent.py index 6d3368e957..b78a8a0a91 100644 --- a/bin/newcontent.py +++ b/bin/newcontent.py @@ -80,3 +80,129 @@ def new(security_content_path, VERBOSE, type, example_only): if example_only: create_example(security_content_path,type, TEMPLATE_PATH) + + + if type == 'detection': + questions = [ + { + # get api_key + 'type': 'input', + 'message': 'enter detection name', + 'name': 'detection_name', + 'default': 'Suspicious Mshta Spawn', + }, + { + # get api_key + 'type': 'input', + 'message': 'enter detection description, Markdown is `supported`', + 'name': 'detection_description', + }, + { + # get provider + 'type': 'list', + 'message': 'select a detection type', + 'name': 'detection_type', + 'choices': [ + { + 'name': 'batch' + }, + { + 'name': 'streaming' + }, + ], + 'default': 'batch' + }, + { + # get provider + 'type': 'checkbox', + 'message': 'select a datamodels for the detection', + 'name': 'detection_datamodels', + 'choices': [ + { + 'name': 'Endpoint', + 'checked': True + }, + { + 'name': 'Network_Traffic' + }, + { + 'name': 'Authentication' + }, + { + 'name': 'Change' + }, + { + 'name': 'Change_Analysis' + }, + { + 'name': 'Email' + }, + { + 'name': 'Network_Resolution' + }, + { + 'name': 'Network_Traffic' + }, + { + 'name': 'Network_Sessions' + }, + { + 'name': 'Updates' + }, + { + 'name': 'Vulnerabilities' + }, + { + 'name': 'Web' + }, + ], + }, + { + # get api_key + 'type': 'input', + 'message': 'enter search (spl)', + 'name': 'detect', + }, + { + # get api_key + 'type': 'input', + 'message': 'enter author name', + 'name': 'detection_author', + }, + ] + answers = prompt(questions) + + j2_env = Environment(loader=FileSystemLoader(TEMPLATE_PATH), + trim_blocks=True) + + # write a detection example + template = j2_env.get_template('detection.j2') + detection_name = answers['detection_name'] + detection_file_name = detection_name.replace(' ', '_').replace('-','_').replace('.','_').replace('/','_').lower() + output_path = path.join(security_content_path, 'detections/endpoint/' + detection_file_name + '.yml') + output = template.render(uuid=uuid.uuid1(), date=date.today().strftime('%Y-%m-%d'), + author='Robert Johansson', name=getpass.getuser().capitalize() + ' ' + type.capitalize() + ' Example', + description='Describe your detection the best way possible, if you need inspiration just look over others.', + how_to_implement='How would a user implement this detection, describe any TAs, or specific configuration they might require', + known_false_positives='Although unlikely, some legitimate applications may exhibit this behavior, triggering a false positive.', + references=['https://wearebob.fandom.com/wiki/Bob','https://en.wikipedia.org/wiki/Dennis_E._Taylor'], + datamodels=['Endpoint'], search='SPLUNKSPLGOESHERE | `' + getpass.getuser() + '_' + type + '_example_filter`', + type='batch', analytic_story_name='STORY NAME GOES HERE', mitre_attack_id = 'T1003.01', + kill_chain_phases=['Exploitation'], dataset_url='https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1003.001/atomic_red_team/windows-sysmon.log', + products=['Splunk Enterprise','Splunk Enterprise Security','Splunk Cloud']) + with open(output_path, 'w', encoding="utf-8") as f: + f.write(output) + print("contentctl wrote a example detection to: {0}".format(output_path)) + + # and a corresponding test files + template = j2_env.get_template('test.j2') + test_name = getpass.getuser() + '_' + type + '_example.test.yml' + output_path = path.join(security_content_path, 'tests/endpoint/' + test_name) + output = template.render(name=getpass.getuser().capitalize() + ' ' + type.capitalize() + ' Example Unit Test', + detection_name=getpass.getuser().capitalize() + ' ' + type.capitalize() + ' Example', + detection_path='detections/endpoint/' + detection_name, pass_condition='| stats count | where count > 0', + earliest_time='-24h', latest_time='now', file_name='windows-sysmon.log', splunk_source='XmlWinEventLog:Microsoft-Windows-Sysmon/Operational', + splunk_sourcetype='xmlwineventlog',dataset_url='https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1003.001/atomic_red_team/windows-sysmon.log') + with open(output_path, 'w', encoding="utf-8") as f: + f.write(output) + print("contentctl wrote a example test for this detection to: {0}".format(output_path)) From 6634909d8e6cd36e74b9282eacbf5bd3c6acb401 Mon Sep 17 00:00:00 2001 From: divious1 Date: Wed, 17 Feb 2021 23:18:18 -0500 Subject: [PATCH 08/14] added wizard for detections and tests --- bin/newcontent.py | 238 ++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 208 insertions(+), 30 deletions(-) diff --git a/bin/newcontent.py b/bin/newcontent.py index b78a8a0a91..ea4fe27efc 100644 --- a/bin/newcontent.py +++ b/bin/newcontent.py @@ -84,6 +84,34 @@ def new(security_content_path, VERBOSE, type, example_only): if type == 'detection': questions = [ + { + # get provider + 'type': 'list', + 'message': 'what kind of detection is this', + 'name': 'detection_kind', + 'choices': [ + { + 'name': 'endpoint' + }, + { + 'name': 'cloud' + }, + { + 'name': 'application' + }, + { + 'name': 'network' + }, + { + 'name': 'web' + }, + { + 'name': 'experimental' + }, + + ], + 'default': 'endpoint' + }, { # get api_key 'type': 'input', @@ -91,6 +119,12 @@ def new(security_content_path, VERBOSE, type, example_only): 'name': 'detection_name', 'default': 'Suspicious Mshta Spawn', }, + { + # get api_key + 'type': 'input', + 'message': 'enter author name', + 'name': 'detection_author', + }, { # get api_key 'type': 'input', @@ -115,8 +149,8 @@ def new(security_content_path, VERBOSE, type, example_only): { # get provider 'type': 'checkbox', - 'message': 'select a datamodels for the detection', - 'name': 'detection_datamodels', + 'message': 'select the datamodels used in the detection', + 'name': 'datamodels', 'choices': [ { 'name': 'Endpoint', @@ -161,48 +195,192 @@ def new(security_content_path, VERBOSE, type, example_only): # get api_key 'type': 'input', 'message': 'enter search (spl)', - 'name': 'detect', + 'name': 'detection_search', }, { # get api_key 'type': 'input', - 'message': 'enter author name', - 'name': 'detection_author', + 'message': 'enter a steps how to implement the detection', + 'name': 'how_to_implement', }, - ] - answers = prompt(questions) + { + # get api_key + 'type': 'input', + 'message': 'enter any known false positives', + 'name': 'know_false_positives', + }, + { + # get api_key + 'type': 'input', + 'message': 'enter references (urls) the give context to the detection, comma delimited for multiple', + 'name': 'references', + }, + { + # get api_key + 'type': 'input', + 'message': 'enter associated Splunk Analytic Story, comma delimited for multiple', + 'name': 'detection_stories', + }, + { + # get api_key + 'type': 'input', + 'message': 'enter MITRE ATT&CK Technique related to the detection, comma delimited for multiple', + 'name': 'mitre_attack_ids', + }, + { + # get provider + 'type': 'checkbox', + 'message': 'select kill chain phases related to the detection', + 'name': 'kill_chain_phases', + 'choices': [ + { + 'name': 'Reconnaissance' + }, + { + 'name': 'Intrusion' + }, + { + 'name': 'Exploitation', + 'checked': True + }, + { + 'name': 'Privilege Escalation' + }, + { + 'name': 'Lateral Movement' + }, + { + 'name': 'Obfuscation' + }, + { + 'name': 'Denial of Service' + }, + { + 'name': 'Exfiltration' + }, + ], + }, + + { + # get api_key + 'type': 'input', + 'message': 'enter attack_data dataset url used for detection testing.', + 'name': 'dataset_url', + }, + + + + ] + + answers = prompt(questions) + mitre_attack_id = answers['mitre_attack_ids'].split(',') j2_env = Environment(loader=FileSystemLoader(TEMPLATE_PATH), trim_blocks=True) + if answers['detection_type'] == 'batch': + answers['products'] = ['Splunk Enterprise','Splunk Enterprise Security','Splunk Cloud'] + elif answers['detection_type'] == 'streaming': + answers['products'] = ['UEBA for Security Cloud'] + + # grab some vars for the test + detection_dataset_url = answers['dataset_url'] + detection_kind = answers['detection_kind'] + + # write a detection example template = j2_env.get_template('detection.j2') detection_name = answers['detection_name'] detection_file_name = detection_name.replace(' ', '_').replace('-','_').replace('.','_').replace('/','_').lower() - output_path = path.join(security_content_path, 'detections/endpoint/' + detection_file_name + '.yml') + output_path = path.join(security_content_path, 'detections/' + detection_kind + '/' + detection_file_name + '.yml') output = template.render(uuid=uuid.uuid1(), date=date.today().strftime('%Y-%m-%d'), - author='Robert Johansson', name=getpass.getuser().capitalize() + ' ' + type.capitalize() + ' Example', - description='Describe your detection the best way possible, if you need inspiration just look over others.', - how_to_implement='How would a user implement this detection, describe any TAs, or specific configuration they might require', - known_false_positives='Although unlikely, some legitimate applications may exhibit this behavior, triggering a false positive.', - references=['https://wearebob.fandom.com/wiki/Bob','https://en.wikipedia.org/wiki/Dennis_E._Taylor'], - datamodels=['Endpoint'], search='SPLUNKSPLGOESHERE | `' + getpass.getuser() + '_' + type + '_example_filter`', - type='batch', analytic_story_name='STORY NAME GOES HERE', mitre_attack_id = 'T1003.01', - kill_chain_phases=['Exploitation'], dataset_url='https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1003.001/atomic_red_team/windows-sysmon.log', - products=['Splunk Enterprise','Splunk Enterprise Security','Splunk Cloud']) + author=answers['detection_author'], name=answers['detection_name'], + description=answers['detection_description'], how_to_implement=answers['how_to_implement'], known_false_positives=answers['know_false_positives'], + references=answers['references'].split(","),datamodels=answers['datamodels'], + search= answers['detection_search'] + ' | ' + detection_file_name + '_filter', + type=answers['detection_type'], analytic_story_name=answers['detection_stories'].split(','), mitre_attack_id = answers['mitre_attack_ids'].split(','), + kill_chain_phases=answers['kill_chain_phases'], dataset_url=detection_dataset_url, + products=answers['products']) with open(output_path, 'w', encoding="utf-8") as f: f.write(output) - print("contentctl wrote a example detection to: {0}".format(output_path)) - # and a corresponding test files - template = j2_env.get_template('test.j2') - test_name = getpass.getuser() + '_' + type + '_example.test.yml' - output_path = path.join(security_content_path, 'tests/endpoint/' + test_name) - output = template.render(name=getpass.getuser().capitalize() + ' ' + type.capitalize() + ' Example Unit Test', - detection_name=getpass.getuser().capitalize() + ' ' + type.capitalize() + ' Example', - detection_path='detections/endpoint/' + detection_name, pass_condition='| stats count | where count > 0', - earliest_time='-24h', latest_time='now', file_name='windows-sysmon.log', splunk_source='XmlWinEventLog:Microsoft-Windows-Sysmon/Operational', - splunk_sourcetype='xmlwineventlog',dataset_url='https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1003.001/atomic_red_team/windows-sysmon.log') - with open(output_path, 'w', encoding="utf-8") as f: - f.write(output) - print("contentctl wrote a example test for this detection to: {0}".format(output_path)) + print("\n> contentctl wrote the detection to: {0}\n".format(output_path)) + + questions = [ + { + 'type': 'confirm', + 'message': 'would you like to configure the test file for detection: {0}'.format(answers['detection_name']), + 'name': 'continue', + 'default': True, + }, + { + 'type': 'input', + 'message': 'enter pass condition for the test of detection: {0}'.format(answers['detection_name']), + 'name': 'pass_condition', + 'default': '| stats count | where count > 0', + 'when': lambda answers: answers['continue'], + }, + { + 'type': 'input', + 'message': 'enter earliest_time for the test of detection: {0}'.format(answers['detection_name']), + 'name': 'earliest_time', + 'default': '-24h', + 'when': lambda answers: answers['continue'], + }, + { + 'type': 'input', + 'message': 'enter latest_time for the test of detection: {0}'.format(answers['detection_name']), + 'name': 'latest_time', + 'default': 'now', + 'when': lambda answers: answers['continue'], + }, + { + 'type': 'input', + 'message': 'enter the file_name of attack_data dataset file', + 'name': 'file_name', + 'default': 'windows-sysmon.log', + 'when': lambda answers: answers['continue'], + }, + { + 'type': 'input', + 'message': 'enter the Splunk source used in the dataset file', + 'name': 'splunk_source', + 'default': 'XmlWinEventLog:Microsoft-Windows-Sysmon/Operational', + 'when': lambda answers: answers['continue'], + }, + { + 'type': 'input', + 'message': 'enter the Splunk sourcetype used in the dataset file', + 'name': 'splunk_sourcetype', + 'default': 'xmlwineventlog', + 'when': lambda answers: answers['continue'], + }, + ] + + + answers = prompt(questions) + if answers['continue']: + # and a corresponding test files + template = j2_env.get_template('test.j2') + test_name = detection_file_name + '.test.yml' + output_path = path.join(security_content_path, 'tests/' + detection_kind + '/' + test_name) + output = template.render(name=detection_name + ' Unit Test', + detection_name=detection_name, + detection_path='detections/' + detection_kind + '/' + detection_file_name + '.yml', pass_condition=answers['pass_condition'], + earliest_time=answers['earliest_time'], latest_time=answers['latest_time'], file_name=answers['file_name'], + splunk_source=answers['splunk_source'],splunk_sourcetype=answers['splunk_sourcetype'],dataset_url=detection_dataset_url) + with open(output_path, 'w', encoding="utf-8") as f: + f.write(output) + else: + # and a corresponding test files + template = j2_env.get_template('test.j2') + test_name = detection_file_name + '.test.yml' + output_path = path.join(security_content_path, 'tests/' + detection_kind + '/' + test_name) + output = template.render(name=detection_name + ' Unit Test', + detection_name=detection_name, + detection_path='detections/' + detection_kind + '/' + detection_file_name + '.yml', pass_condition='| stats count | where count > 0', + earliest_time='-24h', latest_time='now', file_name='windows-sysmon.log', + splunk_source='XmlWinEventLog:Microsoft-Windows-Sysmon/Operational',splunk_sourcetype='xmlwineventlog',dataset_url=detection_dataset_url) + with open(output_path, 'w', encoding="utf-8") as f: + f.write(output) + print("\n> contentctl wrote the test for this detection to: {0}\n".format(output_path)) From b81bb8c53d7860005ce87f1ff711bbe639386a86 Mon Sep 17 00:00:00 2001 From: divious1 Date: Thu, 18 Feb 2021 10:29:19 -0500 Subject: [PATCH 09/14] fixed minor bug --- bin/newcontent.py | 2 ++ contentctl.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/bin/newcontent.py b/bin/newcontent.py index ea4fe27efc..0f195e206f 100644 --- a/bin/newcontent.py +++ b/bin/newcontent.py @@ -12,6 +12,7 @@ from jinja2 import Environment, FileSystemLoader import uuid from datetime import date from os import path +import sys def create_example(security_content_path,type, TEMPLATE_PATH): @@ -80,6 +81,7 @@ def new(security_content_path, VERBOSE, type, example_only): if example_only: create_example(security_content_path,type, TEMPLATE_PATH) + sys.exit(0) if type == 'detection': diff --git a/contentctl.py b/contentctl.py index 1521bd13d4..5ea245bf18 100644 --- a/contentctl.py +++ b/contentctl.py @@ -57,7 +57,7 @@ def new(args): VERBOSE = not args.silence # hard setting verbosity for now - print("contentctl is creating a new {0} under {0}".format(args.type, security_content_path)) + print("contentctl is creating a new {0}".format(args.type)) content.new(security_content_path, VERBOSE, args.type, args.example_only) def validate(args): From 8a73c3237be53d0b37e57b3a5a8e223bbf2dc2e1 Mon Sep 17 00:00:00 2001 From: divious1 Date: Thu, 18 Feb 2021 21:32:59 -0500 Subject: [PATCH 10/14] first set of feedback --- bin/newcontent.py | 6 ++---- contentctl.py | 4 ++-- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/bin/newcontent.py b/bin/newcontent.py index 0f195e206f..c488f9ce3f 100644 --- a/bin/newcontent.py +++ b/bin/newcontent.py @@ -115,11 +115,9 @@ def new(security_content_path, VERBOSE, type, example_only): 'default': 'endpoint' }, { - # get api_key 'type': 'input', - 'message': 'enter detection name', + 'message': 'enter detection name (Suspicious MSHTA)', 'name': 'detection_name', - 'default': 'Suspicious Mshta Spawn', }, { # get api_key @@ -136,7 +134,7 @@ def new(security_content_path, VERBOSE, type, example_only): { # get provider 'type': 'list', - 'message': 'select a detection type', + 'message': 'select a detection type (see type details here: https://wiki)', 'name': 'detection_type', 'choices': [ { diff --git a/contentctl.py b/contentctl.py index 5ea245bf18..40ccd77ad7 100644 --- a/contentctl.py +++ b/contentctl.py @@ -28,7 +28,7 @@ starting program loaded for TIE Fighter... | ]|.[_]_ T~T[_.-Y / \ / \ Y-._]T~T _[_].|| | [|-+[ ___]| [__ |-=[--()--]=-| __] |[___ ]+-|] | ]|"[_] l_j[_"-l \ / \ / !-"_]l_j [_]~|| | - | |`-' "~"---.,_\"\ "o--o" /"/_,.---"~" `-'| | + | |`-' "~"---.,_\\"\ "o--o" /"/_,.---"~" `-'| | | | ~~"^-.____.-^"~~ | | | | | | | | | | @@ -99,7 +99,7 @@ def main(args): parser.set_defaults(func=lambda _: parser.print_help()) actions_parser = parser.add_subparsers(title="Splunk Security Content actions", dest="action") - new_parser = actions_parser.add_parser("new", help="Create new content (detections, stories, workbooks)") + new_parser = actions_parser.add_parser("new", help="Create new content (detection, story, baseline)") validate_parser = actions_parser.add_parser("validate", help="Validates written content") generate_parser = actions_parser.add_parser("generate", help="Generates a deployment package for different platforms (splunk_app)") From f5fe07171fa6e34fa11621712d6e68ba9298bb6b Mon Sep 17 00:00:00 2001 From: divious1 Date: Fri, 19 Feb 2021 22:59:09 -0500 Subject: [PATCH 11/14] updated given feedback from the team --- .gitignore | 6 +- bin/jinja2_templates/baseline.j2 | 27 ++ bin/newcontent.py | 721 +++++++++++++++++-------------- contentctl.py | 23 +- docs/detections.spec.json | 166 ------- docs/detections.spec.md | 390 ----------------- 6 files changed, 436 insertions(+), 897 deletions(-) create mode 100644 bin/jinja2_templates/baseline.j2 delete mode 100644 docs/detections.spec.json delete mode 100644 docs/detections.spec.md diff --git a/.gitignore b/.gitignore index 5f5e0d4c99..558ce4523f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,8 @@ # Ignore example files from contentctl tool -detections/*/*_example.yml -stories/*_example.yml -tests/*/*_example.yml +detections/*/.yml.example +stories/*.yml.example +tests/*/*.yml.example # usual mac files .DS_Store diff --git a/bin/jinja2_templates/baseline.j2 b/bin/jinja2_templates/baseline.j2 new file mode 100644 index 0000000000..3c48684ff2 --- /dev/null +++ b/bin/jinja2_templates/baseline.j2 @@ -0,0 +1,27 @@ +name: {{name}} +id: {{uuid}} +version: 1 +date: '{{date}}' +author: {{author}} +type: {{type}} +datamodel: +{% for datamodel in datamodels -%} + - {{datamodel}} +{% endfor -%} +description: {{description}} +search: '{{search}}' +how_to_implement: {{how_to_implement}} +known_false_positives: {{known_false_positives}} +references: +{% for reference in references -%} + - {{reference}} +{% endfor -%} +tags: + analytic_story: + - {{analytic_story_name}} + detections: + - {{detection_name}} + product: + {% for product in products -%} + - {{product}} + {% endfor -%} diff --git a/bin/newcontent.py b/bin/newcontent.py index c488f9ce3f..d9f67e775a 100644 --- a/bin/newcontent.py +++ b/bin/newcontent.py @@ -15,6 +15,358 @@ from os import path import sys +def detection_wizard(security_content_path,type,TEMPLATE_PATH): + questions = [ + { + # get provider + 'type': 'list', + 'message': 'what kind of detection is this', + 'name': 'detection_kind', + 'choices': [ + { + 'name': 'endpoint' + }, + { + 'name': 'cloud' + }, + { + 'name': 'application' + }, + { + 'name': 'network' + }, + { + 'name': 'web' + }, + { + 'name': 'experimental' + }, + + ], + 'default': 'endpoint' + }, + { + 'type': 'input', + 'message': 'enter detection name', + 'name': 'detection_name', + 'default': 'Powershell Encoded Command', + }, + { + 'type': 'input', + 'message': 'enter author name', + 'name': 'detection_author', + }, + { + # get provider + 'type': 'list', + 'message': 'select a detection type', + 'name': 'detection_type', + 'choices': [ + { + 'name': 'batch' + }, + { + 'name': 'streaming' + }, + ], + 'default': 'batch' + }, + { + # get provider + 'type': 'checkbox', + 'message': 'select the datamodels used in the detection', + 'name': 'datamodels', + 'choices': [ + { + 'name': 'Endpoint', + 'checked': True + }, + { + 'name': 'Network_Traffic' + }, + { + 'name': 'Authentication' + }, + { + 'name': 'Change' + }, + { + 'name': 'Change_Analysis' + }, + { + 'name': 'Email' + }, + { + 'name': 'Network_Resolution' + }, + { + 'name': 'Network_Traffic' + }, + { + 'name': 'Network_Sessions' + }, + { + 'name': 'Updates' + }, + { + 'name': 'Vulnerabilities' + }, + { + 'name': 'Web' + }, + ], + }, + { + # get api_key + 'type': 'input', + 'message': 'enter search (spl)', + 'name': 'detection_search', + 'default': '| UPDATE_SPL' + }, + { + # get api_key + 'type': 'input', + 'message': 'enter MITRE ATT&CK Technique IDs related to the detection, comma delimited for multiple', + 'name': 'mitre_attack_ids', + 'default': 'T1003.002' + }, + { + # get provider + 'type': 'checkbox', + 'message': 'select kill chain phases related to the detection', + 'name': 'kill_chain_phases', + 'choices': [ + + { + 'name': 'Reconnaissance' + }, + { + 'name': 'Intrusion' + }, + { + 'name': 'Exploitation', + 'checked': True + }, + { + 'name': 'Privilege Escalation' + }, + { + 'name': 'Lateral Movement' + }, + { + 'name': 'Obfuscation' + }, + { + 'name': 'Denial of Service' + }, + { + 'name': 'Exfiltration' + }, + ], + }, + ] + + answers = prompt(questions) + mitre_attack_id = answers['mitre_attack_ids'].split(',') + j2_env = Environment(loader=FileSystemLoader(TEMPLATE_PATH), + trim_blocks=True) + + if answers['detection_type'] == 'batch': + answers['products'] = ['Splunk Enterprise','Splunk Enterprise Security','Splunk Cloud'] + elif answers['detection_type'] == 'streaming': + answers['products'] = ['UEBA for Security Cloud'] + + # grab some vars for the test + detection_kind = answers['detection_kind'] + + + # write a detection example + template = j2_env.get_template('detection.j2') + detection_name = answers['detection_name'] + detection_file_name = detection_name.replace(' ', '_').replace('-','_').replace('.','_').replace('/','_').lower() + output_path = path.join(security_content_path, 'detections/' + detection_kind + '/' + detection_file_name + '.yml') + output = template.render(uuid=uuid.uuid1(), date=date.today().strftime('%Y-%m-%d'), + author=answers['detection_author'], name=answers['detection_name'], + description='UPDATE_DESCRIPTION', how_to_implement='UPDATE_HOW_TO_IMPLEMENT', known_false_positives='UPDATE_KNOWN_FALSE_POSITIVES', + references='',datamodels=answers['datamodels'], + search= answers['detection_search'] + ' | `' + detection_file_name + '_filter`', + type=answers['detection_type'], analytic_story_name='UPDATE_STORY_NAME', mitre_attack_id = answers['mitre_attack_ids'], + kill_chain_phases=answers['kill_chain_phases'], dataset_url='UPDATE_DATASET_URL', + products=answers['products']) + with open(output_path, 'w', encoding="utf-8") as f: + f.write(output) + + print("\n> contentctl wrote the detection to: {0}\n".format(output_path)) + + questions = [ + { + 'type': 'confirm', + 'message': 'would you like to configure the test file for detection: {0}'.format(answers['detection_name']), + 'name': 'continue', + 'default': True, + }, + { + 'type': 'input', + 'message': 'enter pass condition for the test of detection: {0}'.format(answers['detection_name']), + 'name': 'pass_condition', + 'default': '| stats count | where count > 0', + 'when': lambda answers: answers['continue'], + }, + { + 'type': 'input', + 'message': 'enter earliest_time for the test of detection: {0}'.format(answers['detection_name']), + 'name': 'earliest_time', + 'default': '-24h', + 'when': lambda answers: answers['continue'], + }, + { + 'type': 'input', + 'message': 'enter latest_time for the test of detection: {0}'.format(answers['detection_name']), + 'name': 'latest_time', + 'default': 'now', + 'when': lambda answers: answers['continue'], + }, + ] + + + answers = prompt(questions) + if answers['continue']: + # and a corresponding test files + template = j2_env.get_template('test.j2') + test_name = detection_file_name + '.test.yml' + output_path = path.join(security_content_path, 'tests/' + detection_kind + '/' + test_name) + output = template.render(name=detection_name + ' Unit Test', + detection_name=detection_name, + detection_path='detections/' + detection_kind + '/' + detection_file_name + '.yml', pass_condition=answers['pass_condition'], + earliest_time=answers['earliest_time'], latest_time=answers['latest_time'], file_name='UPDATE_FILE_NAME', + splunk_source='UPDATE_SPLUNK_SOURCE',splunk_sourcetype='UPDATE_SPLUNK_SOURCETYPE',dataset_url='UPDATE_DATASET_URL') + with open(output_path, 'w', encoding="utf-8") as f: + f.write(output) + else: + # and a corresponding test files + template = j2_env.get_template('test.j2') + test_name = detection_file_name + '.test.yml' + output_path = path.join(security_content_path, 'tests/' + detection_kind + '/' + test_name) + output = template.render(name=detection_name + ' Unit Test', + detection_name=detection_name, + detection_path='detections/' + detection_kind + '/' + detection_file_name + '.yml', pass_condition='| stats count | where count > 0', + earliest_time='-24h', latest_time='now',file_name='UPDATE_FILE_NAME', splunk_source='UPDATE_SPLUNK_SOURCE', + splunk_sourcetype='UPDATE_SPLUNK_SOURCETYPE', dataset_url='UPDATE_DATASET_URL' ) + with open(output_path, 'w', encoding="utf-8") as f: + f.write(output) + print("\n> contentctl wrote the test for this detection to: {0}\n".format(output_path)) + +def story_wizard(security_content_path,type, TEMPLATE_PATH): + questions = [ + { + 'type': 'input', + 'message': 'enter story name', + 'name': 'story_name', + 'default': 'Suspicious Powershell Behavior', + }, + { + 'type': 'input', + 'message': 'enter author name', + 'name': 'story_author', + }, + { + 'type': 'list', + 'message': 'select a story type', + 'name': 'story_type', + 'choices': [ + { + 'name': 'batch' + }, + { + 'name': 'streaming' + }, + ], + 'default': 'batch' + }, + { + 'type': 'checkbox', + 'message': 'select a category', + 'name': 'category', + 'choices': [ + { + 'name': 'Adversary Tactics', + 'checked': True + }, + { + 'name': 'Account Compromise' + }, + { + 'name': 'Unauthorized Software' + }, + { + 'name': 'Best Practices' + }, + { + 'name': 'Cloud Security' + }, + { + 'name': 'Command and Control' + }, + { + 'name': 'Lateral Movement' + }, + { + 'name': 'Ransomware' + }, + { + 'name': 'Privilege Escalation' + }, + ], + }, + { + # get provider + 'type': 'list', + 'message': 'select a use case', + 'name': 'usecase', + 'choices': [ + { + 'name': 'Advanced Threat Detection', + 'checked': True + }, + { + 'name': 'Security Monitoring' + }, + { + 'name': 'Compliance' + }, + { + 'name': 'Insider Threat' + }, + { + 'name': 'Application Security' + }, + { + 'name': 'Other' + }, + ], + }, + ] + answers = prompt(questions) + j2_env = Environment(loader=FileSystemLoader(TEMPLATE_PATH), + trim_blocks=True) + if answers['story_type'] == 'batch': + answers['products'] = ['Splunk Enterprise','Splunk Enterprise Security','Splunk Cloud'] + elif answers['story_type'] == 'streaming': + answers['products'] = ['UEBA for Security Cloud'] + + template = j2_env.get_template('story.j2') + story_name = answers['story_name'] + story_file_name = story_name.replace(' ', '_').replace('-','_').replace('.','_').replace('/','_').lower() + output_path = path.join(security_content_path, 'stories/' + story_file_name + '.yml') + output = template.render(uuid=uuid.uuid1(), date=date.today().strftime('%Y-%m-%d'), + author=answers['story_author'], name=answers['story_name'], description='UPDATE_DESCRIPTION', + narrative='UPDATE_NARRATIVE', references=['https://www.destroyallsoftware.com/talks/wat'], + type=answers['story_type'], analytic_story_name=answers['story_name'], + categories=answers['category'], usecase=answers['usecase'], products=answers['products']) + with open(output_path, 'w', encoding="utf-8") as f: + f.write(output) + print("contentctl wrote a example story to: {0}".format(output_path)) + def create_example(security_content_path,type, TEMPLATE_PATH): getpass.getuser() j2_env = Environment(loader=FileSystemLoader(TEMPLATE_PATH), @@ -25,17 +377,17 @@ def create_example(security_content_path,type, TEMPLATE_PATH): # write a detection example template = j2_env.get_template('detection.j2') - detection_name = getpass.getuser() + '_' + type + '_example.yml' + detection_name = getpass.getuser() + '_' + type + '.yml.example' output_path = path.join(security_content_path, 'detections/endpoint/' + detection_name) output = template.render(uuid=uuid.uuid1(), date=date.today().strftime('%Y-%m-%d'), - author='Robert Johansson', name=getpass.getuser().capitalize() + ' ' + type.capitalize() + ' Example', - description='Describe your detection the best way possible, if you need inspiration just look over others.', - how_to_implement='How would a user implement this detection, describe any TAs, or specific configuration they might require', - known_false_positives='Although unlikely, some legitimate applications may exhibit this behavior, triggering a false positive.', - references=['https://wearebob.fandom.com/wiki/Bob','https://en.wikipedia.org/wiki/Dennis_E._Taylor'], - datamodels=['Endpoint'], search='SPLUNKSPLGOESHERE | `' + getpass.getuser() + '_' + type + '_example_filter`', - type='batch', analytic_story_name='STORY NAME GOES HERE', mitre_attack_id = 'T1003.01', - kill_chain_phases=['Exploitation'], dataset_url='https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1003.001/atomic_red_team/windows-sysmon.log', + author='UPDATE_AUTHOR', name=getpass.getuser().capitalize() + ' ' + type.capitalize(), + description='UPDATE_DESCRIPTION', + how_to_implement='UPDATE_HOW_TO_IMPLENT', + known_false_positives='UPDATE_KNOWN_FALSE_POSITIVES', + references=['https://html5zombo.com/'], + datamodels=['Endpoint'], search='| UPDATE_SPL | `' + getpass.getuser() + '_' + type + '_filter`', + type='batch', analytic_story_name=' UPDATE_STORY_NAME', mitre_attack_id = 'T1003.01', + kill_chain_phases=['Exploitation'], dataset_url='UPDATE_DATASET_URL', products=['Splunk Enterprise','Splunk Enterprise Security','Splunk Cloud']) with open(output_path, 'w', encoding="utf-8") as f: f.write(output) @@ -43,36 +395,56 @@ def create_example(security_content_path,type, TEMPLATE_PATH): # and a corresponding test files template = j2_env.get_template('test.j2') - test_name = getpass.getuser() + '_' + type + '_example.test.yml' + test_name = getpass.getuser() + '_' + type + '.test.yml.example' output_path = path.join(security_content_path, 'tests/endpoint/' + test_name) - output = template.render(name=getpass.getuser().capitalize() + ' ' + type.capitalize() + ' Example Unit Test', - detection_name=getpass.getuser().capitalize() + ' ' + type.capitalize() + ' Example', + output = template.render(name=getpass.getuser().capitalize() + ' ' + type.capitalize() + ' Unit Test', + detection_name=getpass.getuser().capitalize() + ' ' + type.capitalize(), detection_path='detections/endpoint/' + detection_name, pass_condition='| stats count | where count > 0', - earliest_time='-24h', latest_time='now', file_name='windows-sysmon.log', splunk_source='XmlWinEventLog:Microsoft-Windows-Sysmon/Operational', - splunk_sourcetype='xmlwineventlog',dataset_url='https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1003.001/atomic_red_team/windows-sysmon.log') + earliest_time='-24h', latest_time='now', file_name='UPDATE_FILE_NAME', splunk_source='UPDATE_SPLUNK_SOURCE', + splunk_sourcetype='UPDATE_SPLUNK_SOURCETYPE',dataset_url='UPDATE_DATASET_URL') with open(output_path, 'w', encoding="utf-8") as f: f.write(output) print("contentctl wrote a example test for this detection to: {0}".format(output_path)) elif type == 'story': - # write a detection example + # write a story example template = j2_env.get_template('story.j2') - story_name = getpass.getuser() + '_' + type + '_example.yml' + story_name = getpass.getuser() + '_' + type + '.yml.example' output_path = path.join(security_content_path, 'stories/' + story_name) output = template.render(uuid=uuid.uuid1(), date=date.today().strftime('%Y-%m-%d'), - author='Robert Johansson', name=getpass.getuser().capitalize() + ' ' + type.capitalize() + ' Example', - description='Describe your story the best way possible, if you need inspiration just look over others.', - narrative='Explain why should a SOC manager or Director care about this use case, if you need inspiration just look over others.', - references=['https://wearebob.fandom.com/wiki/Bob','https://en.wikipedia.org/wiki/Dennis_E._Taylor'], - type='batch', analytic_story_name=getpass.getuser().capitalize() + ' ' + type.capitalize() + ' Example', - category=['Adversary Tactics'], usecase='Advanced Threat Detection', products=['Splunk Enterprise','Splunk Enterprise Security','Splunk Cloud']) + author='UPDATE_AUTHOR', name=getpass.getuser().capitalize() + ' ' + type.capitalize(), + description='UPDATE_DESCRIPTION', + narrative='UPDATE_NARRATIVE', + references=['https://www.destroyallsoftware.com/talks/wat'], + type='batch', analytic_story_name=getpass.getuser().capitalize() + ' ' + type.capitalize(), + categories=['Adversary Tactics'], usecase='Advanced Threat Detection', products=['Splunk Enterprise','Splunk Enterprise Security','Splunk Cloud']) with open(output_path, 'w', encoding="utf-8") as f: f.write(output) print("contentctl wrote a example story to: {0}".format(output_path)) + elif type == 'baseline': + # write a baseline example + template = j2_env.get_template('baseline.j2') + baseline_name = getpass.getuser() + '_' + type + '.yml.example' + output_path = path.join(security_content_path, 'baselines/' + baseline_name) + output = template.render(uuid=uuid.uuid1(), date=date.today().strftime('%Y-%m-%d'), + author='UPDATE_AUTHOR', name=getpass.getuser().capitalize() + ' ' + type.capitalize(), + description='UPDATE_DESCRIPTION', + how_to_implement='UPDATE_HOW_TO_IMPLENT', + known_false_positives='UPDATE_KNOWN_FALSE_POSITIVES', + references=['https://html5zombo.com/'], + datamodels=['Endpoint'], search='| UPDATE_SPL', + type='batch', analytic_story_name='UPDATE_STORY_NAME', + detection_name = 'UPDATE_DETECTION_NAME', dataset_url='UPDATE_DATASET_URL', + products=['Splunk Enterprise','Splunk Enterprise Security','Splunk Cloud']) + with open(output_path, 'w', encoding="utf-8") as f: + f.write(output) + print("contentctl wrote a example baseline to: {0}".format(output_path)) + + def new(security_content_path, VERBOSE, type, example_only): - valid_content_objects = ['detection','story'] + valid_content_objects = ['detection','story', 'baseline'] if type not in valid_content_objects: print("ERROR: content type: {0} is not valid, please use: {1}".format(type, str(valid_content_objects))) sys.exit(1) @@ -83,304 +455,9 @@ def new(security_content_path, VERBOSE, type, example_only): create_example(security_content_path,type, TEMPLATE_PATH) sys.exit(0) - if type == 'detection': - questions = [ - { - # get provider - 'type': 'list', - 'message': 'what kind of detection is this', - 'name': 'detection_kind', - 'choices': [ - { - 'name': 'endpoint' - }, - { - 'name': 'cloud' - }, - { - 'name': 'application' - }, - { - 'name': 'network' - }, - { - 'name': 'web' - }, - { - 'name': 'experimental' - }, + detection_wizard(security_content_path, type, TEMPLATE_PATH) + elif type == 'story': + story_wizard(security_content_path, type, TEMPLATE_PATH) - ], - 'default': 'endpoint' - }, - { - 'type': 'input', - 'message': 'enter detection name (Suspicious MSHTA)', - 'name': 'detection_name', - }, - { - # get api_key - 'type': 'input', - 'message': 'enter author name', - 'name': 'detection_author', - }, - { - # get api_key - 'type': 'input', - 'message': 'enter detection description, Markdown is `supported`', - 'name': 'detection_description', - }, - { - # get provider - 'type': 'list', - 'message': 'select a detection type (see type details here: https://wiki)', - 'name': 'detection_type', - 'choices': [ - { - 'name': 'batch' - }, - { - 'name': 'streaming' - }, - ], - 'default': 'batch' - }, - { - # get provider - 'type': 'checkbox', - 'message': 'select the datamodels used in the detection', - 'name': 'datamodels', - 'choices': [ - { - 'name': 'Endpoint', - 'checked': True - }, - { - 'name': 'Network_Traffic' - }, - { - 'name': 'Authentication' - }, - { - 'name': 'Change' - }, - { - 'name': 'Change_Analysis' - }, - { - 'name': 'Email' - }, - { - 'name': 'Network_Resolution' - }, - { - 'name': 'Network_Traffic' - }, - { - 'name': 'Network_Sessions' - }, - { - 'name': 'Updates' - }, - { - 'name': 'Vulnerabilities' - }, - { - 'name': 'Web' - }, - ], - }, - { - # get api_key - 'type': 'input', - 'message': 'enter search (spl)', - 'name': 'detection_search', - }, - { - # get api_key - 'type': 'input', - 'message': 'enter a steps how to implement the detection', - 'name': 'how_to_implement', - }, - { - # get api_key - 'type': 'input', - 'message': 'enter any known false positives', - 'name': 'know_false_positives', - }, - { - # get api_key - 'type': 'input', - 'message': 'enter references (urls) the give context to the detection, comma delimited for multiple', - 'name': 'references', - }, - { - # get api_key - 'type': 'input', - 'message': 'enter associated Splunk Analytic Story, comma delimited for multiple', - 'name': 'detection_stories', - }, - { - # get api_key - 'type': 'input', - 'message': 'enter MITRE ATT&CK Technique related to the detection, comma delimited for multiple', - 'name': 'mitre_attack_ids', - }, - { - # get provider - 'type': 'checkbox', - 'message': 'select kill chain phases related to the detection', - 'name': 'kill_chain_phases', - 'choices': [ - - { - 'name': 'Reconnaissance' - }, - { - 'name': 'Intrusion' - }, - { - 'name': 'Exploitation', - 'checked': True - }, - { - 'name': 'Privilege Escalation' - }, - { - 'name': 'Lateral Movement' - }, - { - 'name': 'Obfuscation' - }, - { - 'name': 'Denial of Service' - }, - { - 'name': 'Exfiltration' - }, - ], - }, - - { - # get api_key - 'type': 'input', - 'message': 'enter attack_data dataset url used for detection testing.', - 'name': 'dataset_url', - }, - - - - ] - - answers = prompt(questions) - mitre_attack_id = answers['mitre_attack_ids'].split(',') - j2_env = Environment(loader=FileSystemLoader(TEMPLATE_PATH), - trim_blocks=True) - - if answers['detection_type'] == 'batch': - answers['products'] = ['Splunk Enterprise','Splunk Enterprise Security','Splunk Cloud'] - elif answers['detection_type'] == 'streaming': - answers['products'] = ['UEBA for Security Cloud'] - - # grab some vars for the test - detection_dataset_url = answers['dataset_url'] - detection_kind = answers['detection_kind'] - - - # write a detection example - template = j2_env.get_template('detection.j2') - detection_name = answers['detection_name'] - detection_file_name = detection_name.replace(' ', '_').replace('-','_').replace('.','_').replace('/','_').lower() - output_path = path.join(security_content_path, 'detections/' + detection_kind + '/' + detection_file_name + '.yml') - output = template.render(uuid=uuid.uuid1(), date=date.today().strftime('%Y-%m-%d'), - author=answers['detection_author'], name=answers['detection_name'], - description=answers['detection_description'], how_to_implement=answers['how_to_implement'], known_false_positives=answers['know_false_positives'], - references=answers['references'].split(","),datamodels=answers['datamodels'], - search= answers['detection_search'] + ' | ' + detection_file_name + '_filter', - type=answers['detection_type'], analytic_story_name=answers['detection_stories'].split(','), mitre_attack_id = answers['mitre_attack_ids'].split(','), - kill_chain_phases=answers['kill_chain_phases'], dataset_url=detection_dataset_url, - products=answers['products']) - with open(output_path, 'w', encoding="utf-8") as f: - f.write(output) - - print("\n> contentctl wrote the detection to: {0}\n".format(output_path)) - - questions = [ - { - 'type': 'confirm', - 'message': 'would you like to configure the test file for detection: {0}'.format(answers['detection_name']), - 'name': 'continue', - 'default': True, - }, - { - 'type': 'input', - 'message': 'enter pass condition for the test of detection: {0}'.format(answers['detection_name']), - 'name': 'pass_condition', - 'default': '| stats count | where count > 0', - 'when': lambda answers: answers['continue'], - }, - { - 'type': 'input', - 'message': 'enter earliest_time for the test of detection: {0}'.format(answers['detection_name']), - 'name': 'earliest_time', - 'default': '-24h', - 'when': lambda answers: answers['continue'], - }, - { - 'type': 'input', - 'message': 'enter latest_time for the test of detection: {0}'.format(answers['detection_name']), - 'name': 'latest_time', - 'default': 'now', - 'when': lambda answers: answers['continue'], - }, - { - 'type': 'input', - 'message': 'enter the file_name of attack_data dataset file', - 'name': 'file_name', - 'default': 'windows-sysmon.log', - 'when': lambda answers: answers['continue'], - }, - { - 'type': 'input', - 'message': 'enter the Splunk source used in the dataset file', - 'name': 'splunk_source', - 'default': 'XmlWinEventLog:Microsoft-Windows-Sysmon/Operational', - 'when': lambda answers: answers['continue'], - }, - { - 'type': 'input', - 'message': 'enter the Splunk sourcetype used in the dataset file', - 'name': 'splunk_sourcetype', - 'default': 'xmlwineventlog', - 'when': lambda answers: answers['continue'], - }, - ] - - - answers = prompt(questions) - if answers['continue']: - # and a corresponding test files - template = j2_env.get_template('test.j2') - test_name = detection_file_name + '.test.yml' - output_path = path.join(security_content_path, 'tests/' + detection_kind + '/' + test_name) - output = template.render(name=detection_name + ' Unit Test', - detection_name=detection_name, - detection_path='detections/' + detection_kind + '/' + detection_file_name + '.yml', pass_condition=answers['pass_condition'], - earliest_time=answers['earliest_time'], latest_time=answers['latest_time'], file_name=answers['file_name'], - splunk_source=answers['splunk_source'],splunk_sourcetype=answers['splunk_sourcetype'],dataset_url=detection_dataset_url) - with open(output_path, 'w', encoding="utf-8") as f: - f.write(output) - else: - # and a corresponding test files - template = j2_env.get_template('test.j2') - test_name = detection_file_name + '.test.yml' - output_path = path.join(security_content_path, 'tests/' + detection_kind + '/' + test_name) - output = template.render(name=detection_name + ' Unit Test', - detection_name=detection_name, - detection_path='detections/' + detection_kind + '/' + detection_file_name + '.yml', pass_condition='| stats count | where count > 0', - earliest_time='-24h', latest_time='now', file_name='windows-sysmon.log', - splunk_source='XmlWinEventLog:Microsoft-Windows-Sysmon/Operational',splunk_sourcetype='xmlwineventlog',dataset_url=detection_dataset_url) - with open(output_path, 'w', encoding="utf-8") as f: - f.write(output) - print("\n> contentctl wrote the test for this detection to: {0}\n".format(output_path)) + print("WARNING do not forget to replace the UPDATE_* values with the correct information on the files!\ncompleted..") diff --git a/contentctl.py b/contentctl.py index 40ccd77ad7..476985a829 100644 --- a/contentctl.py +++ b/contentctl.py @@ -54,26 +54,17 @@ starting program loaded for TIE Fighter... def new(args): security_content_path = init(args) - VERBOSE = not args.silence - - # hard setting verbosity for now print("contentctl is creating a new {0}".format(args.type)) - content.new(security_content_path, VERBOSE, args.type, args.example_only) + content.new(security_content_path, args.verbose, args.type, args.example_only) def validate(args): security_content_path = init(args) - VERBOSE = not args.silence - - # hard setting verbosity for now print("contentctl is validating all content under {0}".format(security_content_path)) - validator.new(security_content_path, VERBOSE) + validator.new(security_content_path, args.verbose) def generate(args): security_content_path = init(args) - # hard setting verbosity for now - VERBOSE = not args.silence - output = Path(args.output).resolve() if output.is_dir(): print("contentctl is using folder {0} to write deployment".format( @@ -83,7 +74,7 @@ def generate(args): sys.exit(1) print("contentctl is generating a new splunk_app under ".format(output)) - generator.new(security_content_path, args.output, VERBOSE) + generator.new(security_content_path, args.output, args.verbose) def main(args): @@ -92,9 +83,9 @@ def main(args): description="Use `contentctl.py action -h` to get help with any Splunk Security Content action") parser.add_argument("-p", "--path", required=False, default=".", help="path to the Splunk Security Content. Defaults to `.`") - parser.add_argument("-v", "--version", default=False, action="version", version="version: {0}".format(VERSION), + parser.add_argument("--Version", default=False, action="version", version="version: {0}".format(VERSION), help="shows current contentctl version") - parser.add_argument("-s", "--silence", required=False, action='store_true', + parser.add_argument("-v", "--verbose", required=False, action='store_true', help="silences all verbose output, defaults to False") parser.set_defaults(func=lambda _: parser.print_help()) @@ -105,9 +96,9 @@ def main(args): # new arguments new_parser.add_argument("-t", "--type", required=False, type=str, default="detection", - help="Type of new content to create, please chose between detection or story") + help="Type of new content to create, please choose between `detection`, `baseline` or `story`. Defaults to `detection`") new_parser.add_argument("-x", "--example_only", required=False, action='store_true', - help="Generates an example content with UPDATETHIS where a value is required. Use `git status` to see what specific files are added. Skips new content wizard prompts.") + help="Generates an example content UPDATE on the fields that need updating. Use `git status` to see what specific files are added. Skips new content wizard prompts.") new_parser.set_defaults(func=new) # validate arguments diff --git a/docs/detections.spec.json b/docs/detections.spec.json deleted file mode 100644 index ba614e8de0..0000000000 --- a/docs/detections.spec.json +++ /dev/null @@ -1,166 +0,0 @@ -{ - "$id": "http://example.com/example.json", - "$schema": "http://json-schema.org/draft-07/schema", - "additionalProperties": true, - "description": "schema for detections", - "properties": { - "author": { - "$id": "#/properties/author", - "default": "", - "description": "Author of the detection", - "examples": [ - "Patrick Bareiss, Splunk" - ], - "type": "string" - }, - "date": { - "$id": "#/properties/date", - "default": "", - "description": "date of creation or modification, format yyyy-mm-dd", - "examples": [ - "2019-12-06" - ], - "type": "string" - }, - "description": { - "$id": "#/properties/description", - "default": "", - "description": "A detailed description of the detection", - "examples": [ - "dbgcore.dll is a specifc DLL for Windows core debugging. It is used to obtain a memory dump of a process. This search detects the usage of this DLL for creating a memory dump of LSASS process. Memory dumps of the LSASS process can be created with tools such as Windows Task Manager or procdump." - ], - "type": "string" - }, - "how_to_implement": { - "$id": "#/properties/how_to_implement", - "default": "", - "description": "information about how to implement. Only needed for non standard implementations.", - "examples": [ - "This search requires Sysmon Logs and a Sysmon configuration, which includes EventCode 10 for lsass.exe." - ], - "type": "string" - }, - "id": { - "$id": "#/properties/id", - "default": "", - "description": "UUID as unique identifier", - "examples": [ - "fb4c31b0-13e8-4155-8aa5-24de4b8d6717" - ], - "type": "string" - }, - "known_false_positives": { - "$id": "#/properties/knwon_false_positives", - "default": "", - "description": "known false postives", - "examples": [ - "Administrators can create memory dumps for debugging purposes, but memory dumps of the LSASS process would be unusual." - ], - "type": "string" - }, - "name": { - "$id": "#/properties/name", - "default": "", - "examples": [ - "Access LSASS Memory for Dump Creation" - ], - "title": "Name of detection", - "type": "string" - }, - "references": { - "$id": "#/properties/references", - "additionalItems": true, - "default": [], - "description": "A list of references for this detection", - "examples": [ - [ - "https://2017.zeronights.org/wp-content/uploads/materials/ZN17_Kheirkhabarov_Hunting_for_Credentials_Dumping_in_Windows_Environment.pdf" - ] - ], - "items": { - "$id": "#/properties/references/items", - "default": "", - "description": "An explanation about the purpose of this instance.", - "examples": [ - "https://2017.zeronights.org/wp-content/uploads/materials/ZN17_Kheirkhabarov_Hunting_for_Credentials_Dumping_in_Windows_Environment.pdf" - ], - "title": "The Items Schema", - "type": "string" - }, - "type": "array" - }, - "search": { - "$id": "#/properties/search", - "default": "", - "description": "The Splunk search for the detection", - "examples": [ - "`sysmon` EventCode=10 TargetImage=*lsass.exe CallTrace=*dbgcore.dll* OR CallTrace=*dbghelp.dll* | stats count min(_time) as firstTime max(_time) as lastTime by Computer, TargetImage, TargetProcessId, SourceImage, SourceProcessId | rename Computer as dest | `security_content_ctime(firstTime)`| `security_content_ctime(lastTime)` | `access_lsass_memory_for_dump_creation_filter`" - ], - "type": "string" - }, - "tags": { - "$id": "#/properties/tags", - "additionalProperties": true, - "default": {}, - "description": "An array of key value pairs for tagging", - "examples": [ - { - "analytics_story": "credential_dumping", - "kill_chain_phases": "Action on Objectives", - "mitre_attack_id": "T1078.004", - "cis20": "CIS 13", - "nist": "DE.DP", - "security domain": "network", - "asset_type": "AWS Instance", - "risk_object": "user", - "risk_object_type": "network_artifacts", - "risk score": "60", - "custom_key": "custom_value" - } - ], - "minItems": 1, - "type": "object", - "uniqueItems": true - }, - "type": { - "$id": "#/properties/type", - "default": "", - "description": "type of detection", - "examples": [ - "ESCU" - ], - "items": { - "enum": [ - "ESCU", - "SSE", - "RBA" - ], - "type": "string" - }, - "type": "string" - }, - "version": { - "$id": "#/properties/version", - "default": 0, - "description": "version of detection, e.g. 1 or 2 ...", - "examples": [ - 2 - ], - "type": "integer" - } - }, - "required": [ - "name", - "id", - "version", - "date", - "description", - "type", - "author", - "search", - "known_false_positives", - "tags" - ], - "title": "Detection Schema", - "type": "object" -} \ No newline at end of file diff --git a/docs/detections.spec.md b/docs/detections.spec.md deleted file mode 100644 index 1ff1d8619b..0000000000 --- a/docs/detections.spec.md +++ /dev/null @@ -1,390 +0,0 @@ - -# Detection Schema Schema - -``` -http://example.com/example.json -``` - -schema for detections - -| Abstract | Extensible | Status | Identifiable | Custom Properties | Additional Properties | Defined In | -|----------|------------|--------|--------------|-------------------|-----------------------|------------| -| Can be instantiated | No | Experimental | No | Forbidden | Permitted | | - -# Detection Schema Properties - -| Property | Type | Required | Nullable | Default | Defined by | -|----------|------|----------|----------|---------|------------| -| [author](#author) | `string` | **Required** | No | `""` | Detection Schema (this schema) | -| [date](#date) | `string` | **Required** | No | `""` | Detection Schema (this schema) | -| [description](#description) | `string` | **Required** | No | `""` | Detection Schema (this schema) | -| [how_to_implement](#how_to_implement) | `string` | Optional | No | `""` | Detection Schema (this schema) | -| [id](#id) | `string` | **Required** | No | `""` | Detection Schema (this schema) | -| [known_false_positives](#known_false_positives) | `string` | **Required** | No | `""` | Detection Schema (this schema) | -| [name](#name) | `string` | **Required** | No | `""` | Detection Schema (this schema) | -| [references](#references) | `string[]` | Optional | No | `[]` | Detection Schema (this schema) | -| [search](#search) | `string` | **Required** | No | `""` | Detection Schema (this schema) | -| [tags](#tags) | `object` | **Required** | No | `{}` | Detection Schema (this schema) | -| [type](#type) | `string` | **Required** | No | `""` | Detection Schema (this schema) | -| [version](#version) | `integer` | **Required** | No | `0` | Detection Schema (this schema) | -| `*` | any | Additional | Yes | this schema *allows* additional properties | - -## author - -Author of the detection - -`author` - -* is **required** -* type: `string` -* default: `""` -* defined in this schema - -### author Type - - -`string` - - - - - - -### author Example - -```json -"Patrick Bareiss, Splunk" -``` - - -## date - -date of creation or modification, format yyyy-mm-dd - -`date` - -* is **required** -* type: `string` -* default: `""` -* defined in this schema - -### date Type - - -`string` - - - - - - -### date Example - -```json -"2019-12-06" -``` - - -## description - -A detailed description of the detection - -`description` - -* is **required** -* type: `string` -* default: `""` -* defined in this schema - -### description Type - - -`string` - - - - - - -### description Example - -```json -"dbgcore.dll is a specifc DLL for Windows core debugging. It is used to obtain a memory dump of a process. This search detects the usage of this DLL for creating a memory dump of LSASS process. Memory dumps of the LSASS process can be created with tools such as Windows Task Manager or procdump." -``` - - -## how_to_implement - -information about how to implement. Only needed for non standard implementations. - -`how_to_implement` - -* is optional -* type: `string` -* default: `""` -* defined in this schema - -### how_to_implement Type - - -`string` - - - - - - -### how_to_implement Example - -```json -"This search requires Sysmon Logs and a Sysmon configuration, which includes EventCode 10 for lsass.exe." -``` - - -## id - -UUID as unique identifier - -`id` - -* is **required** -* type: `string` -* default: `""` -* defined in this schema - -### id Type - - -`string` - - - - - - -### id Example - -```json -"fb4c31b0-13e8-4155-8aa5-24de4b8d6717" -``` - - -## known_false_positives - -known false postives - -`known_false_positives` - -* is **required** -* type: `string` -* default: `""` -* defined in this schema - -### known_false_positives Type - - -`string` - - - - - - -### known_false_positives Example - -```json -"Administrators can create memory dumps for debugging purposes, but memory dumps of the LSASS process would be unusual." -``` - - -## name -### Name of detection - -`name` - -* is **required** -* type: `string` -* default: `""` -* defined in this schema - -### name Type - - -`string` - - - - - - -### name Example - -```json -"Access LSASS Memory for Dump Creation" -``` - - -## references - -A list of references for this detection - -`references` - -* is optional -* type: `string[]` - -* default: `[]` -* defined in this schema - -### references Type - - -Array type: `string[]` - -All items must be of the type: -`string` - - - - - -An explanation about the purpose of this instance. - - - - - -### references Example - -```json -[ - "https://2017.zeronights.org/wp-content/uploads/materials/ZN17_Kheirkhabarov_Hunting_for_Credentials_Dumping_in_Windows_Environment.pdf" -] -``` - - -## search - -The Splunk search for the detection - -`search` - -* is **required** -* type: `string` -* default: `""` -* defined in this schema - -### search Type - - -`string` - - - - - - -### search Example - -```json -"`sysmon` EventCode=10 TargetImage=*lsass.exe CallTrace=*dbgcore.dll* OR CallTrace=*dbghelp.dll* | stats count min(_time) as firstTime max(_time) as lastTime by Computer, TargetImage, TargetProcessId, SourceImage, SourceProcessId | rename Computer as dest | `security_content_ctime(firstTime)`| `security_content_ctime(lastTime)` | `access_lsass_memory_for_dump_creation_filter`" -``` - - -## tags - -An array of key value pairs for tagging - -`tags` - -* is **required** -* type: `object` -* default: `{}` -* defined in this schema - -### tags Type - - -`object` with following properties: - - -| Property | Type | Required | -|----------|------|----------| - - - - -### tags Example - -```json -{ - "analytics_story": "credential_dumping", - "kill_chain_phases": "Action on Objectives", - "mitre_attack_id": "T1078.004", - "cis20": "CIS 13", - "nist": "DE.DP", - "security domain": "network", - "asset_type": "AWS Instance", - "risk_object": "user", - "risk_object_type": "network_artifacts", - "risk score": "60", - "custom_key": "custom_value" -} -``` - - -## type - -type of detection - -`type` - -* is **required** -* type: `string` -* default: `""` -* defined in this schema - -### type Type - - -`string` - - - - - - -### type Example - -```json -"ESCU" -``` - - -## version - -version of detection, e.g. 1 or 2 ... - -`version` - -* is **required** -* type: `integer` -* default: `0` -* defined in this schema - -### version Type - - -`integer` - - - - - - -### version Example - -```json -2 -``` - From 3ef02c2213744203f9880a864ee3c3853ce3690e Mon Sep 17 00:00:00 2001 From: divious1 Date: Sat, 20 Feb 2021 20:38:57 -0500 Subject: [PATCH 12/14] adjusting ci jobs --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 711c24dc80..91eaff61cd 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -80,7 +80,7 @@ jobs: command: | cd security-content source venv/bin/activate - python bin/validate.py --path . --verbose + python contentctl.py --path . --verbose validate - run: name: run doc-gen command: | @@ -118,7 +118,7 @@ jobs: command: | cd security-content source venv/bin/activate - python bin/generate.py --path . --output package -v + python contentctl.py --path . --verbose generate --output package # make a copy of use_case_lib in order to have ES work :-( cp package/default/use_case_library.conf package/default/analyticstories.conf - run: From 85905e2901000e4f09c80e7576068785ec5885a4 Mon Sep 17 00:00:00 2001 From: divious1 Date: Sat, 20 Feb 2021 21:09:42 -0500 Subject: [PATCH 13/14] updating README --- README.md | 68 +++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 43 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index d207e71a1d..c1aa21e220 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@

- + # Splunk Security Content ![security_content](docs/static/logo.png) ===== @@ -22,11 +22,14 @@ Welcome to the Splunk Security Content This project gives you access to our repository of Analytic Stories that are security guides the provide background on TTPs, mapped to the MITRE framework, the Lockheed Martin Kill Chain, and CIS controls. They include Splunk searches, machine-learning algorithms, and Splunk Phantom playbooks (where available)—all designed to work together to detect, investigate, and respond to threats. -# Usage🛡 -The Splunk Security Content can be used via: +# Get Content🛡 +The latest Splunk Security Content can be obtained via: -#### [Splunk App](https://github.com/splunk/security_content/releases) -Grab the latest release of DA-ESS-ContentUpdate and install it on a Splunk Enterprise instance. Alternatively, you can download it from [splunkbase](https://splunkbase.splunk.com/app/3449/), it is currently a Splunk Supported App. +#### [SSE App](https://github.com/splunk/security_content/releases) +Grab the latest release of Splunk Security Essentials App and install it on a Splunk instance. You can download it from [splunkbase](https://splunkbase.splunk.com/app/3449/), it is a Splunk Supported App. SSE Splunk app today supports push updates for security content release, this is the **preferred way** to get content! + +#### [ESCU App](https://github.com/splunk/security_content/releases) +Grab the latest release of DA-ESS-ContentUpdate.spl and install it on a Splunk instance. Alternatively, you can download it from [splunkbase](https://splunkbase.splunk.com/app/3449/), it is currently a Splunk Supported App. #### [API](https://docs.splunkresearch.com/?version=latest) ``` @@ -36,35 +39,54 @@ curl -s https://content.splunkresearch.com | jq } ``` -#### [GitHub Workflow](https://github.com/splunk/security_content/wiki/Installation-and-Usage) -Create your customized version of Security Content by forking this project and following this [guide](https://github.com/splunk/security_content/wiki/Installation-and-Usage#github-workflow). +# Usage 🧰 +### contentctl.py +The Content Control tool allows you to manipulate Splunk Security Content via the following actions: -# MITRE ATT&CK +1. **new** - Creates new content (detection, story, baseline) +2. **validate** - Validates written content +3. **generate** - Generates a deployment package for different platforms (splunk_app) + +### pre-requisites + +``` +git clone git@github.com:splunk/security_content.git +cd security_content +pip install virtualenv +virtualenv venv +source venv/bin/activate +pip install -r requirements.txt +``` +### create a new detection +`python contentctl.py new` + +for a more indepth write up on how to write content see our [guide](https://github.com/splunk/security_content/wiki/Developing-Content). + +### create a new analytic story +`python contentctl.py new -t story` + +### validate written content a new analytic story +`python contentctl.py --verbose validate` + +### generate a splunk app from current content +`python contentctl.py --path . --verbose generate --output package` + +# MITRE ATT&CK ⚔️ ### Detection Coverage To view an up-to-date detection coverage map for all the content tagged with MITRE techniques visit: [https://mitremap.splunkresearch.com/](https://mitremap.splunkresearch.com/) under the **Detection Coverage** layer. Below is a snapshot in time of what technique we currently have some detection coverage for. The darker the shade of blue the more detections we have for this particular technique. This map is automatically updated on every release and generated from the [generate-coverage-map.py](https://github.com/splunk/security_content/blob/develop/bin/generate-coverage-map.py). ![](docs/mitre-map/coverage.png) ### Detection Priority by Threat Actors -If curious about how the Threat Research team prioritizes what content to build refer to our **Detection Priority by Threat Actors** layer. Using the actor data from [MITRE CTI](https://github.com/mitre/cti) we add a point for every threat actor that uses a particular technique, and then subtract a point of every detection we have mapped to that technique. The resulting map below is how we prioritize what techniques and detections to focus on next. This map is automatically updated on every release and is generated by the [generate-actors-map.py](https://github.com/splunk/security_content/blob/develop/bin/generate-actors-map.py) script. +If curious about how the Threat Research team prioritizes what content to build refer to our **Detection Priority by Threat Actors** layer in [https://mitremap.splunkresearch.com/](https://mitremap.splunkresearch.com/). Using the actor data from [MITRE CTI](https://github.com/mitre/cti) we add a point for every threat actor that uses a particular technique, and then subtract a point of every detection we have mapped to that technique. The resulting map below is how we prioritize what techniques and detections to focus on next. This map is automatically updated on every release and is generated by the [generate-actors-map.py](https://github.com/splunk/security_content/blob/develop/bin/generate-actors-map.py) script. ![](docs/mitre-map/priority.png) # Customize to your Environment 🏗 Customize your content to change how [often detections run](https://github.com/splunk/security_content/wiki/Customize-to-Your-Environment#customizing-scheduling-and-alert-actions-with-deployments), or what the right source type for [sysmon](https://github.com/splunk/security_content/wiki/Customize-to-Your-Environment#customizing-source-types-with-macros) in your environment is please follow this [guide](https://github.com/splunk/security_content/wiki/Customize-to-Your-Environment). -# Writing Content 📓 -Please see the Developing Content [guide](https://github.com/splunk/security_content/wiki/Developing-Content) for instructions. - -# What's in an Analytic Story? -A complete use case, specifically built to detect, investigate, and respond to a specific threat like [Credential Dumping](https://github.com/splunk/security_content/blob/develop/stories/credential_dumping.yml) or [Ransomware](https://github.com/splunk/security_content/blob/develop/stories/ransomware.yml). A group of detections and a response make up an analytic story, they are associated with the tag `analytics_story: `. - -# Execute an Analytic Story 🏃‍♀️ -Download and install the latest version of [Splunk Analytic Story Execution](https://github.com/splunk/analytic_story_execution/releases). This Splunk application will help the user do the following: - -1. Execute an analytic story in an ad-hoc mode and view the results. -2. Schedule all the detection searches in an analytic story. -3. Update security_content via an API +# What's in an Analytic Story? 🗺 +A complete use case, specifically built to detect, investigate, and respond to a specific threat like [Credential Dumping](https://github.com/splunk/security_content/blob/develop/stories/credential_dumping.yml) or [Ransomware](https://github.com/splunk/security_content/blob/develop/stories/ransomware.yml). A group of detections and a response make up an analytic story, they are associated with the tag `analytic_story: `. # Content Parts 🧩 @@ -78,10 +100,6 @@ Download and install the latest version of [Splunk Analytic Story Execution](htt * [macros/](macros/): Implements Splunk’s search macros, shortcuts to commonly used search patterns like sysmon source type. More on how macros are used to customize content below. * [lookups/](lookups/): Implements Splunk’s lookup, usually to provide a list of static values like commonly used ransomware extensions. -#### Supporting Parts -* [package/](package/): Splunk content app-source files, including lookups, binaries, and default config files -* [bin/](bin/): All binaries required to produce and test content - # Contribution 🥰 We welcome feedback and contributions from the community! Please see our [contributing to the project](https://github.com/splunk/security_content/wiki/Contributing-to-the-Project) for more information on how to get involved. From be9035064168966a43d9f8f793bd6fb4c6ba56b3 Mon Sep 17 00:00:00 2001 From: divious1 Date: Sat, 20 Feb 2021 21:26:45 -0500 Subject: [PATCH 14/14] adding correct links --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index c1aa21e220..b660ff3a49 100644 --- a/README.md +++ b/README.md @@ -25,8 +25,8 @@ This project gives you access to our repository of Analytic Stories that are sec # Get Content🛡 The latest Splunk Security Content can be obtained via: -#### [SSE App](https://github.com/splunk/security_content/releases) -Grab the latest release of Splunk Security Essentials App and install it on a Splunk instance. You can download it from [splunkbase](https://splunkbase.splunk.com/app/3449/), it is a Splunk Supported App. SSE Splunk app today supports push updates for security content release, this is the **preferred way** to get content! +#### [SSE App](https://splunkbase.splunk.com/app/3435/) +Grab the latest release of Splunk Security Essentials App and install it on a Splunk instance. You can download it from [splunkbase](https://splunkbase.splunk.com/app/3435/), it is a Splunk Supported App. SSE Splunk app today supports push updates for security content release, this is the **preferred way** to get content! #### [ESCU App](https://github.com/splunk/security_content/releases) Grab the latest release of DA-ESS-ContentUpdate.spl and install it on a Splunk instance. Alternatively, you can download it from [splunkbase](https://splunkbase.splunk.com/app/3449/), it is currently a Splunk Supported App.