From c55e114dac5d7f485ce70dc03c218197405fcee9 Mon Sep 17 00:00:00 2001 From: akhunti Date: Thu, 7 Apr 2022 12:11:15 +0530 Subject: [PATCH 1/3] Removed enrich detection script and Github Action stage --- .github/workflows/build-and-validate.yml | 10 - .../enrich_detections.py | 328 ------------------ security_content_automation/requirements.txt | 3 - 3 files changed, 341 deletions(-) delete mode 100644 security_content_automation/enrich_detections.py delete mode 100644 security_content_automation/requirements.txt diff --git a/.github/workflows/build-and-validate.yml b/.github/workflows/build-and-validate.yml index 99a0da2ab7..674bed1a6a 100644 --- a/.github/workflows/build-and-validate.yml +++ b/.github/workflows/build-and-validate.yml @@ -426,16 +426,6 @@ jobs: source venv/bin/activate python3 contentctl.py -p . docgen -o docs - # this step is creating some TOKEN issues while tagging a build, commenting this out for now - - # - name: Enrich detections with TAs - # env: - # GITHUB_ACCESS_TOKEN: ${{ secrets.GITHUB_TOKEN }} - # run: | - # source venv/bin/activate - # python3 -m pip install -r security_content_automation/requirements.txt - # python3 security_content_automation/enrich_detections.py - - name: Update github with new docs and package bits run: | rm -rf dist diff --git a/security_content_automation/enrich_detections.py b/security_content_automation/enrich_detections.py deleted file mode 100644 index e6d2af4d59..0000000000 --- a/security_content_automation/enrich_detections.py +++ /dev/null @@ -1,328 +0,0 @@ -import base64 -import csv -import io -import json -import logging -import os -import re -import shutil -import sys -import time -from pathlib import Path - -import git -import yaml -from github import Github - - -TIMESTAMP_FORMAT = '%(asctime)s %(levelname)s - %(message)s' - -def fetch_ta_cim_mapping_report(file_name): - try: - with open(file_name) as file_content: - cim_field_report = json.load(file_content) - return cim_field_report - except Exception as error: - error_message = f"Unexpected error occurred while reading file. {error}" - logging.error(error_message) - - -def load_file(file_path): - - try: - with open(file_path, "r", encoding="utf-8") as stream: - file = list(yaml.safe_load_all(stream))[0] - return file - except yaml.YAMLError as exc: - sys.exit("ERROR: reading {0}".format(file_path)) - - -def map_required_fields(cim_summary, datamodel, required_fields): - datasets_fields = {} - add_addon = False - flag = 0 - for item in required_fields: - # Only required field with valid format will be mapped - if re.match("^[A-Za-z0-9_.]*$", item): - if item == "_time" or item == "_times": - continue - else: - dataset_field = item.split(".") - length = len(dataset_field) - if length == 1: - dataset = datamodel[0] - field = dataset_field[0] - else: - dataset = dataset_field[length - 2] - field = dataset_field[length - 1] - if dataset not in datasets_fields: - datasets_fields[dataset] = [] - datasets_fields[dataset].append(field) - - for dataset in datasets_fields: - add_addon = False - mapping_set = datamodel[0] + ":" + dataset - for item in cim_summary: - if mapping_set in item: - for eventtype in cim_summary[item].values(): - for e_type in eventtype: - cim_fields = e_type.get("fields", []) - if set(datasets_fields[dataset]).issubset(set(cim_fields)): - add_addon = True - if add_addon == False: - return add_addon - - return add_addon - - -def is_valid_detection_file(filepath): - - detection_analytic_type = ["ttp", "anomaly"] - detection_with_valid_analytic_type = False - detection_with_valid_datamodel = False - detection_file_path = load_file(filepath) - - if detection_file_path.get("type", "").lower() in detection_analytic_type: - detection_with_valid_analytic_type = True - - if detection_file_path.get("datamodel", []): - detection_with_valid_datamodel = True - - return detection_with_valid_analytic_type & detection_with_valid_datamodel - - -def enrich_detection_file(file, ta_list, keyname): - detection_obj = load_file(file) - detection_obj["tags"][keyname] = ta_list - - with open(file, "w") as f: - yaml.dump(detection_obj, f, sort_keys=False, allow_unicode=True) - - -def main(): - - security_content_repo = "splunk/security_content" - security_content_branch = "develop" - - ta_cim_field_reports_repo = "splunk/ta-cim-field-reports" - ta_cim_field_reports_branch = "main" - - # Decodin GITHUB_ACCESS_TOKEN from base64 - git_token_base64_bytes = os.environ.get("GITHUB_ACCESS_TOKEN").encode('ascii') - git_token_bytes = base64.b64decode(git_token_base64_bytes) - github_token = git_token_bytes.decode('ascii') - - git_token = Github(github_token) - detection_types = ["cloud", "endpoint", "network"] - cim_report_path = ( - "ta_cim_mapping_reports/ta_cim_mapping/cim_mapping_reports/latest/" - ) - detection_ta_mapping = {} - - - try: - # clone security content repository - security_content_repo_obj = git.Repo.clone_from( - "https://" - + github_token - + ":x-oauth-basic@github.com/" - + security_content_repo, - "security_content", - branch=security_content_branch, - ) - message = "Successfully cloned security_content." - logging.info(message) - except Exception as error: - error_message = f"Unexpected error occurred while Cloning security_content, {error}" - logging.error(error_message) - - try: - # clone ta cim field reports repository - ta_cim_field_reports_obj = git.Repo.clone_from( - "https://" - + github_token - + ":x-oauth-basic@github.com/" - + ta_cim_field_reports_repo, - "ta_cim_mapping_reports", - branch=ta_cim_field_reports_branch, - ) - message = "Successfully cloned ta_cim_mapping_reports." - logging.info(message) - except Exception as error: - error_message = f"Unexpected error occurred while Cloning ta-cim-field-reports repo, {error}" - logging.error(error_message) - - - # iterate for every detection types - - - for detection_type in detection_types: - - for subdir, _, files in os.walk(f"security_content/tests/{detection_type}"): - - for file in files: - filepath = subdir + os.sep + file - tas_with_cim_mapping_list = [] - supported_tas_list = [] - detection_obj = load_file(filepath) - source_types = [] - for data in detection_obj.get("tests")[0].get("attack_data"): - source_types.append(data.get("sourcetype")) - - detection_file_name_path = ( - detection_obj.get("tests")[0] - .get("file") - .rsplit("/", 1)[1] - ) - detection_file_name = Path(detection_file_name_path).stem - filepath = "security_content/detections/" + detection_obj.get("tests")[ - 0 - ].get("file") - if not os.path.isfile(filepath): - continue - - if is_valid_detection_file(filepath): - for ta_cim_mapping_file in os.listdir(cim_report_path): - ta_cim_map = fetch_ta_cim_mapping_report( - cim_report_path + ta_cim_mapping_file - ) - - detection_obj = load_file(filepath) - required_fields = detection_obj.get("tags", {}).get( - "required_fields" - ) - datamodel = detection_obj.get("datamodel", []) - result = map_required_fields( - ta_cim_map["cimsummary"], datamodel, required_fields - ) - cim_version = ta_cim_map["cim_version"] - - if result: - tas_with_cim_mapping_list.append( - ta_cim_map.get("ta_name").get("name") - ) - ta_sourcetype = ta_cim_map["sourcetypes"] - for source_type in source_types: - - if ( - source_type in ta_sourcetype - and ta_cim_map.get("ta_name").get("name") - not in supported_tas_list - ): - supported_tas_list.append( - ta_cim_map.get("ta_name").get("name") - ) - detection_ta_mapping[detection_file_name] = {} - - if tas_with_cim_mapping_list: - keyname = "tas_with_cim_mapping" - detection_ta_mapping[detection_file_name][ - "cim_version" - ] = cim_version - detection_ta_mapping[detection_file_name][ - keyname - ] = tas_with_cim_mapping_list - - if supported_tas_list: - keyname = "supported_tas" - enrich_detection_file(filepath, supported_tas_list, keyname) - detection_ta_mapping[detection_file_name][ - keyname - ] = supported_tas_list - - logging.info(f"Enriched {detection_file_name} with supported TAs : {supported_tas_list}") - - security_content_repo_obj.index.add( - [filepath.strip("security_content/")] - ) - - # Generating detection_ta_mapping yml file - try: - with io.open( - r"./security_content/security_content_automation/detection_ta_mapping.yml", - "w", - encoding="utf8", - ) as outfile: - yaml.safe_dump( - detection_ta_mapping, outfile, default_flow_style=False, allow_unicode=True - ) - - security_content_repo_obj.index.add( - ["security_content_automation/detection_ta_mapping.yml"] - ) - message = "Created detection_ta_mapping.yml file" - logging.info(message) - - except Exception as error: - error_message = f"Unexpected error occurred while generating detection_ta_mapping.yml file, {error}" - logging.error(error_message) - - - # Generating detection_ta_mapping CSV report - try: - with open(r"./security_content/security_content_automation/detection_ta_mapping.csv", 'w+', newline='') as csv_file: - fieldnames = ['detection_name', 'cim_version', 'supported_tas', 'tas_with_cim_mapping'] - writer = csv.DictWriter(csv_file, fieldnames=fieldnames) - writer.writeheader() - for detection_name, detection_content in detection_ta_mapping.items(): - detection_content.update({ - 'tas_with_cim_mapping': ', '.join(detection_content["tas_with_cim_mapping"]) if detection_content.get( - 'tas_with_cim_mapping') else '', - 'supported_tas': ', '.join(detection_content["supported_tas"]) if detection_content.get( - 'supported_tas') else '', - 'detection_name': detection_name - }) - writer.writerow(detection_content) - security_content_repo_obj.index.add( - ["security_content_automation/detection_ta_mapping.csv"] - ) - message = "Created detection_ta_mapping.csv file" - logging.info(message) - - except Exception as error: - error_message = f"Unexpected error occurred while generating detection_ta_mapping CSV report, {error}" - logging.error(error_message) - - - - - - try: - security_content_repo_obj.index.commit( - "Updated detection files with recommended TA list." - ) - - epoch_time = str(int(time.time())) - branch_name = "security_content_automation_" + epoch_time - security_content_repo_obj.git.checkout("-b", branch_name) - security_content_repo_obj.git.push("--set-upstream", "origin", branch_name) - repo = git_token.get_repo("splunk/security_content") - - pr = repo.create_pull( - title="Enrich Detection PR " + branch_name, - body="Enriched the detections with supported TAs", - head=branch_name, - base="develop", - ) - message = "Created pull request" - logging.info(message) - except Exception as error: - error_message = f"Unexpected error occurred while creating pull request, {error}" - logging.error(error_message) - - - try: - shutil.rmtree("./security_content") - shutil.rmtree("./ta_cim_mapping_reports") - message = "Cleaned up the environment" - logging.info(message) - except OSError as e: - error_message = f"Unexpected error occurred while deleting files, {error}" - logging.error(error_message) - - -if __name__ == "__main__": - log_level=logging.INFO - handlers = [logging.StreamHandler()] - logging.basicConfig(level=log_level, format=TIMESTAMP_FORMAT, handlers=handlers) - main() diff --git a/security_content_automation/requirements.txt b/security_content_automation/requirements.txt deleted file mode 100644 index 3cfcdd61da..0000000000 --- a/security_content_automation/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -GitPython==3.1.24 -PyYAML==6.0 -PyGithub==1.55 \ No newline at end of file From 305a20bc54b76ca89fa055009affc76ccf98d126 Mon Sep 17 00:00:00 2001 From: mhaag-spl <5632822+MHaggis@users.noreply.github.com> Date: Thu, 7 Apr 2022 14:39:56 -0600 Subject: [PATCH 2/3] Hunting Changed to hunting, added renamed logic to not use the process_macro Issue #2010 --- detections/deprecated/suspicious_rundll32_rename.yml | 6 +++--- detections/endpoint/detect_html_help_renamed.yml | 6 +++--- detections/endpoint/detect_mshta_renamed.yml | 6 +++--- detections/endpoint/detect_renamed_psexec.yml | 6 +++--- .../suspicious_microsoft_workflow_compiler_rename.yml | 6 +++--- detections/endpoint/suspicious_msbuild_rename.yml | 8 ++++---- 6 files changed, 19 insertions(+), 19 deletions(-) diff --git a/detections/deprecated/suspicious_rundll32_rename.yml b/detections/deprecated/suspicious_rundll32_rename.yml index 410cf8256e..f772449975 100644 --- a/detections/deprecated/suspicious_rundll32_rename.yml +++ b/detections/deprecated/suspicious_rundll32_rename.yml @@ -1,7 +1,7 @@ name: Suspicious Rundll32 Rename id: 7360137f-abad-473e-8189-acbdaa34d114 -version: 4 -date: '2022-02-01' +version: 5 +date: '2022-04-07' author: Michael Haag, Splunk type: Hunting datamodel: @@ -13,7 +13,7 @@ description: The following hunting analytic identifies renamed instances of rund name from the PE meta data. Expand the query as needed by looking for specific command line arguments outlined in other analytics. search: '| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) - as lastTime from datamodel=Endpoint.Processes where `process_rundll32` by Processes.dest + as lastTime from datamodel=Endpoint.Processes where Processes.original_file_name=RUNDLL32.exe AND Processes.process_name!=rundll32.exe by Processes.dest Processes.user Processes.parent_process_name Processes.process_name Processes.process Processes.process_id Processes.parent_process_id Processes.original_file_name | `drop_dm_object_name(Processes)` | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)` diff --git a/detections/endpoint/detect_html_help_renamed.yml b/detections/endpoint/detect_html_help_renamed.yml index 0a9704fd49..ba7bbab21e 100644 --- a/detections/endpoint/detect_html_help_renamed.yml +++ b/detections/endpoint/detect_html_help_renamed.yml @@ -1,7 +1,7 @@ name: Detect HTML Help Renamed id: 62fed254-513b-460e-953d-79771493a9f3 -version: 3 -date: '2021-09-16' +version: 4 +date: '2022-04-07' author: Michael Haag, Splunk type: Hunting datamodel: @@ -18,7 +18,7 @@ description: The following analytic identifies a renamed instance of hh.exe (HTM it is the legitimate version of hh.exe by reviewing the PE metadata. hh.exe is natively found in C:\Windows\system32 and C:\Windows\syswow64. search: '| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) - as lastTime from datamodel=Endpoint.Processes where `process_hh` by Processes.dest + as lastTime from datamodel=Endpoint.Processes where Processes.process_name!=hh.exe AND Processes.original_file_name=HH.EXE by Processes.dest Processes.user Processes.parent_process_name Processes.original_file_name Processes.process_name Processes.process Processes.process_id Processes.parent_process_id | `drop_dm_object_name(Processes)` | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)` | `detect_html_help_renamed_filter`' diff --git a/detections/endpoint/detect_mshta_renamed.yml b/detections/endpoint/detect_mshta_renamed.yml index f302aeee8a..3bc653c8f0 100644 --- a/detections/endpoint/detect_mshta_renamed.yml +++ b/detections/endpoint/detect_mshta_renamed.yml @@ -1,7 +1,7 @@ name: Detect mshta renamed id: 8f45fcf0-5b68-11eb-ae93-0242ac130002 -version: 2 -date: '2021-09-16' +version: 3 +date: '2022-04-07' author: Michael Haag, Splunk type: Hunting datamodel: @@ -12,7 +12,7 @@ description: The following analytic identifies renamed instances of mshta.exe ex binary. Further analysis should be performed to review the executed content and validation it is the real mshta. search: '| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) - as lastTime from datamodel=Endpoint.Processes where `process_mshta` by Processes.dest + as lastTime from datamodel=Endpoint.Processes where Processes.process_name!=mshta.exe AND Processes.original_file_name=MSHTA.EXE by Processes.dest Processes.user Processes.parent_process_name Processes.process_name Processes.process Processes.process_id Processes.parent_process_id Processes.original_file_name | `drop_dm_object_name(Processes)` | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)` diff --git a/detections/endpoint/detect_renamed_psexec.yml b/detections/endpoint/detect_renamed_psexec.yml index 08c657f411..cba97a20e6 100644 --- a/detections/endpoint/detect_renamed_psexec.yml +++ b/detections/endpoint/detect_renamed_psexec.yml @@ -1,7 +1,7 @@ name: Detect Renamed PSExec id: 683e6196-b8e8-11eb-9a79-acde48001122 -version: 3 -date: '2021-09-16' +version: 4 +date: '2022-04-07' author: Michael Haag, Splunk type: Hunting datamodel: @@ -12,7 +12,7 @@ description: The following analytic identifies renamed instances of `PsExec.exe` During triage, validate this is the legitimate version of `PsExec` by reviewing the PE metadata. In addition, review parallel processes for further suspicious behavior. search: '| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) - as lastTime from datamodel=Endpoint.Processes where `process_psexec` by Processes.dest + as lastTime from datamodel=Endpoint.Processes where (Processes.process_name!=psexec.exe OR Processes.process_name!=psexec64.exe) AND Processes.original_file_name=psexec.c by Processes.dest Processes.user Processes.parent_process_name Processes.process_name Processes.process Processes.process_id Processes.parent_process_id Processes.original_file_name | `drop_dm_object_name(Processes)` | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)` diff --git a/detections/endpoint/suspicious_microsoft_workflow_compiler_rename.yml b/detections/endpoint/suspicious_microsoft_workflow_compiler_rename.yml index 9fa6a8b73e..777e166849 100644 --- a/detections/endpoint/suspicious_microsoft_workflow_compiler_rename.yml +++ b/detections/endpoint/suspicious_microsoft_workflow_compiler_rename.yml @@ -1,7 +1,7 @@ name: Suspicious microsoft workflow compiler rename id: f0db4464-55d9-11eb-ae93-0242ac130002 -version: 3 -date: '2021-09-20' +version: 4 +date: '2022-04-07' author: Michael Haag, Splunk type: Hunting datamodel: @@ -13,7 +13,7 @@ description: The following analytic identifies a renamed instance of microsoft.w In any instance, microsoft.workflow.compiler.exe spawning from an Office product or any living off the land binary is highly suspect. search: '| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) - as lastTime from datamodel=Endpoint.Processes where `process_microsoftworkflowcompiler` + as lastTime from datamodel=Endpoint.Processes where Processes.process_name!=microsoft.workflow.compiler.exe AND Processes.original_file_name=Microsoft.Workflow.Compiler.exe by Processes.dest Processes.user Processes.parent_process_name Processes.process_name Processes.process Processes.process_id Processes.parent_process_id Processes.original_file_name | `drop_dm_object_name(Processes)` | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)` diff --git a/detections/endpoint/suspicious_msbuild_rename.yml b/detections/endpoint/suspicious_msbuild_rename.yml index 7f283e8b1f..b9f39f58d4 100644 --- a/detections/endpoint/suspicious_msbuild_rename.yml +++ b/detections/endpoint/suspicious_msbuild_rename.yml @@ -1,9 +1,9 @@ name: Suspicious MSBuild Rename id: 4006adac-5937-11eb-ae93-0242ac130002 -version: 2 -date: '2021-01-12' +version: 3 +date: '2022-04-07' author: Michael Haag, Splunk -type: TTP +type: Hunting datamodel: - Endpoint description: The following analytic identifies renamed instances of msbuild.exe executing. @@ -11,7 +11,7 @@ description: The following analytic identifies renamed instances of msbuild.exe C:\Windows\Microsoft.NET\Framework64\v4.0.30319. During investigation, identify the code executed and what is executing a renamed instance of MSBuild. search: '| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) - as lastTime from datamodel=Endpoint.Processes where `process_msbuild` by Processes.dest + as lastTime from datamodel=Endpoint.Processes where Processes.process_name!=msbuild.exe AND Processes.original_file_name=MSBuild.exe by Processes.dest Processes.user Processes.parent_process_name Processes.process_name Processes.process Processes.process_id Processes.parent_process_id Processes.original_file_name | `drop_dm_object_name(Processes)` | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)` From d3e26be675d5f4fb371e02e795f8708867c84ae3 Mon Sep 17 00:00:00 2001 From: mhaag-spl <5632822+MHaggis@users.noreply.github.com> Date: Thu, 7 Apr 2022 15:06:13 -0600 Subject: [PATCH 3/3] T# add --- detections/endpoint/any_powershell_downloadfile.yml | 5 +++-- detections/endpoint/any_powershell_downloadstring.yml | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/detections/endpoint/any_powershell_downloadfile.yml b/detections/endpoint/any_powershell_downloadfile.yml index cf07696d64..43730b9334 100644 --- a/detections/endpoint/any_powershell_downloadfile.yml +++ b/detections/endpoint/any_powershell_downloadfile.yml @@ -1,7 +1,7 @@ name: Any Powershell DownloadFile id: 1a93b7ea-7af7-11eb-adb5-acde48001122 -version: 2 -date: '2021-03-01' +version: 3 +date: '2022-04-07' author: Michael Haag, Splunk type: TTP datamodel: @@ -52,6 +52,7 @@ tags: mitre_attack_id: - T1059 - T1059.001 + - T1105 observable: - name: user type: User diff --git a/detections/endpoint/any_powershell_downloadstring.yml b/detections/endpoint/any_powershell_downloadstring.yml index 4638611219..7f22b096e4 100644 --- a/detections/endpoint/any_powershell_downloadstring.yml +++ b/detections/endpoint/any_powershell_downloadstring.yml @@ -1,7 +1,7 @@ name: Any Powershell DownloadString id: 4d015ef2-7adf-11eb-95da-acde48001122 -version: 2 -date: '2021-03-01' +version: 3 +date: '2022-04-07' author: Michael Haag, Splunk type: TTP datamodel: @@ -49,6 +49,7 @@ tags: mitre_attack_id: - T1059 - T1059.001 + - T1105 observable: - name: user type: User