From f793a802ecc3cedd6cbc20c4967d4bb7cf222582 Mon Sep 17 00:00:00 2001 From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com> Date: Thu, 3 Feb 2022 17:15:45 -0800 Subject: [PATCH] Changes and updates to support better, more reliable checkouts of PRs from remote forks and better ESCU app building. Still in testing phases. --- .github/workflows/detection-testing.yml | 4 +- .../detection_testing_execution.py | 102 ++++--- .../modules/github_service.py | 248 +++++++++++++++--- .../modules/validate_args.py | 6 +- 4 files changed, 265 insertions(+), 95 deletions(-) diff --git a/.github/workflows/detection-testing.yml b/.github/workflows/detection-testing.yml index 0146f06aed..804f4abf04 100644 --- a/.github/workflows/detection-testing.yml +++ b/.github/workflows/detection-testing.yml @@ -81,8 +81,8 @@ jobs: # Note that scheduled actions ONLY run on the default branch, so it won't run on all other branches! echo "Running a nightly test on all detections OR a commit was made directly to develop" python detection_testing_execution.py run --branch develop --mode all --mock --config_file test_config_github_actions.json - elif [[ ! -z "${{ github.event.pull_request.head.ref }}" && ! -z "${{ github.event.pull_request.number }}" ]]; then - echo "Pull request from source branch [${{ github.event.pull_request.head.ref }}] for PR number [${{ github.event.issue.number }}]" + elif [[ ${{ github.event_name }} == pull_request && ! -z "${{ github.event.pull_request.number }}" ]]; then + echo "Pull request from source branch [${{ github.event.pull_request.head.ref }}] for PR number [${{ github.event.pull_request.number }}]" python detection_testing_execution.py run --branch ${{ github.event.pull_request.head.ref }} --pr_number ${{ github.event.pull_request.number }} --mode changes --mock --config_file test_config_github_actions.json else echo "Push from branch [${{ steps.vars.outputs.branch }}]" diff --git a/bin/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py b/bin/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py index ae4ae4ef2b..1a8cb61ae5 100644 --- a/bin/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py +++ b/bin/automated_detection_testing/ci/detection_testing_batch/detection_testing_execution.py @@ -41,6 +41,10 @@ datamodel_file_local_path = "datamodels.conf.tar" datamodel_file_container_path = os.path.join( SPLUNK_CONTAINER_APPS_DIR, "Splunk_SA_CIM") +#es_datamodels_local_path = "es_datamodels.conf.tar" +#es_datamodels_container_path = os.path.join( +# SPLUNK_CONTAINER_APPS_DIR, "Splunk_SA_CIM","default","data","models") + authorizations_file_local_path = "authorize.conf.tar" authorizations_file_container_path = "/opt/splunk/etc/system/local" @@ -98,62 +102,42 @@ def copy_local_apps_to_directory(apps: dict[str, dict], target_directory) -> Non -def ensure_security_content(branch: str, commit_hash: Union[str,None], pr_number: Union[int, None], persist_security_content: bool) -> tuple[GithubService, bool]: - if persist_security_content is True and os.path.exists("security_content"): - print("****** You chose --persist_security_content and the security_content directory exists. " - "We will not check out the repo again. Please be aware, this could cause issues if your " - "repo is out of date or if a previous build failed to download all required tools and "\ - "libraries. If this occurs, it is suggested to change the "\ - "persist_security_content setting to false. ******") - - github_service = GithubService( - branch, commit_hash, persist_security_content=persist_security_content) - - else: - if persist_security_content is True and not os.path.exists("security_content"): - print("Error - you chose --persist_security_content but the security_content directory does not exist!" - " We will check it out for you.") - persist_security_content = False - - elif os.path.exists("security_content/"): - print("Deleting the security_content directory") - try: - shutil.rmtree("security_content/", ignore_errors=True) - print("Successfully removed security_content directory") - except Exception as e: - print( - "Error - could not remove the security_content directory: [%s].\n\tQuitting..." % (str(e))) - sys.exit(1) - - if pr_number: - github_service = GithubService(branch, commit_hash, pr_number) - else: - github_service = GithubService(branch, commit_hash) - - return github_service, persist_security_content -def generate_escu_app(persist_security_content: bool = False) -> str: +def generate_escu_app() -> str: # Go into the security content directory print("****GENERATING ESCU APP****") - os.chdir("security_content") - if persist_security_content is False: - commands = ["python3 -m venv .venv", - ". ./.venv/bin/activate", - "python -m pip install wheel", - "python -m pip install -r requirements.txt", - "python contentctl.py --path . --verbose generate --product ESCU --output dist/escu", - "tar -czf DA-ESS-ContentUpdate.spl -C dist/escu ."] + + #Check if pre-reqs were already installed + if os.path.exists("security_content/slim_packaging/slim_latest") is True: + pre_reqs_installed = True + print("\tESCU Generation Prereqs found!") + escu_prereq_commands = [". ./.venv/bin/activate"] else: - commands = [". ./.venv/bin/activate", - "python contentctl.py --path . --verbose generate --product ESCU --output dist/escu", - "tar -czf DA-ESS-ContentUpdate.spl -C dist/escu ."] + pre_reqs_installed = False + print("\tESCU Generation Prereqs not found - we will install them") + escu_prereq_commands = ["python3 -m venv .venv", + ". ./.venv/bin/activate", + "python -m pip install wheel"] + + os.chdir("security_content") + print("\tRunning contentcl...",end='') + sys.stdout.flush() + + + + commands = escu_prereq_commands + ["python -m pip install -r requirements.txt", + "python contentctl.py --path . --verbose generate --product ESCU --output dist/escu", + "tar -czf DA-ESS-ContentUpdate.spl -C dist/escu ."] + ret = subprocess.run("; ".join(commands), shell=True, capture_output=True) if ret.returncode != 0: - print("Error generating new content.\n\tQuitting and dumping error...\n[%s]" % ( + print("Error generating new content.\n\tQuitting and dumping error...Try removing the security_content directory and running again\n[%s]" % ( ret.stderr)) sys.exit(1) + print("done") + output_file_name = "DA-ESS-ContentUpdate-latest.tar.gz" output_file_path_from_slim_latest = os.path.join( @@ -163,7 +147,7 @@ def generate_escu_app(persist_security_content: bool = False) -> str: output_file_path_from_root = os.path.join( "security_content", output_file_path_from_security_content) - if persist_security_content is True: + if os.path.exists("security_content/slim_packaging/slim_latest") is True: try: os.remove(output_file_path_from_security_content) except FileNotFoundError: @@ -174,7 +158,7 @@ def generate_escu_app(persist_security_content: bool = False) -> str: str(e)), file=sys.stderr) sys.exit(1) - # There remove the latest file if it exists + commands = ["cd slim_packaging/slim_latest", ". ./.venv/bin/activate", "cp -R ../../dist/escu DA-ESS-ContentUpdate", @@ -185,15 +169,17 @@ def generate_escu_app(persist_security_content: bool = False) -> str: os.mkdir("slim_packaging") os.mkdir("apps") try: - SPLUNK_PACKAGING_TOOLKIT_URL = "https://download.splunk.com/misc/packaging-toolkit/splunk-packaging-toolkit-0.9.0.tar.gz" + SPLUNK_PACKAGING_TOOLKIT_URL = "https://download.splunk.com/misc/packaging-toolkit/splunk-packaging-toolkit-1.0.1.tar.gz" + SPLUNK_PACKAGING_TOOLKIT_FILENAME = 'splunk-packaging-toolkit-latest.tar.gz' - print("Downloading the Splunk Packaging Toolkit from %s..." % + print("\tDownloading the Splunk Packaging Toolkit from %s..." % (SPLUNK_PACKAGING_TOOLKIT_URL), end='') + sys.stdout.flush() response = get(SPLUNK_PACKAGING_TOOLKIT_URL) response.raise_for_status() with open(SPLUNK_PACKAGING_TOOLKIT_FILENAME, 'wb') as slim_file: slim_file.write(response.content) - print("Done") + print("done") except Exception as e: print("Error downloading the Splunk Packaging Toolkit: [%s].\n\tQuitting..." % (str(e)), file=sys.stderr) @@ -223,6 +209,7 @@ def generate_escu_app(persist_security_content: bool = False) -> str: sys.exit(1) os.chdir("../") + print(f"ESCU App Generated at: {output_file_path_from_root}") return output_file_path_from_root @@ -360,8 +347,7 @@ def main(args: list[str]): # Check out security content if required try: #Make sure we fix up the persist_securiy_content argument if it is passed in error (we say it exists but it doesn't) - github_service, settings['persist_security_content'] = ensure_security_content( - settings['branch'], settings['commit_hash'], settings['pr_number'], settings['persist_security_content']) + github_service = GithubService(settings['branch'], settings['commit_hash'], settings['pr_number'], settings['persist_security_content']) settings['commit_hash'] = github_service.commit_hash except Exception as e: print("\nFailure checking out git repository: [%s]"\ @@ -403,7 +389,7 @@ def main(args: list[str]): sys.exit(1) print("***This run will test [%d] detections!***"%(len(all_test_files))) - + #Set up the directory that will be used to store the local apps/apps we build local_volume_absolute_path = os.path.abspath( @@ -434,7 +420,8 @@ def main(args: list[str]): sys.exit(1) else: # Generate the ESCU package from this branch. - source_path = generate_escu_app(settings['persist_security_content']) + source_path = generate_escu_app() + settings['local_apps']['SPLUNK_ES_CONTENT_UPDATE']['local_path'] = source_path @@ -464,7 +451,10 @@ def main(args: list[str]): "local_file_path": datamodel_file_local_path, "container_file_path": datamodel_file_container_path} files_to_copy_to_container["AUTHORIZATIONS"] = { "local_file_path": authorizations_file_local_path, "container_file_path": authorizations_file_container_path} - + + #files_to_copy_to_container["ES_DATAMODELS"] = { + # "local_file_path": es_datamodels_local_path, "container_file_path": es_datamodels_container_path} + try: diff --git a/bin/automated_detection_testing/ci/detection_testing_batch/modules/github_service.py b/bin/automated_detection_testing/ci/detection_testing_batch/modules/github_service.py index ffedd07ea2..aa225a925a 100644 --- a/bin/automated_detection_testing/ci/detection_testing_batch/modules/github_service.py +++ b/bin/automated_detection_testing/ci/detection_testing_batch/modules/github_service.py @@ -1,4 +1,6 @@ +from cgi import test import csv +from curses import pair_number import glob import logging import os @@ -13,35 +15,195 @@ import yaml from git.objects import base from modules import testing_service import pathlib - +import shutil +import re # Logger logging.basicConfig(level=os.environ.get("LOGLEVEL", "INFO")) LOGGER = logging.getLogger(__name__) SECURITY_CONTENT_URL = "https://github.com/splunk/security_content" +LOCAL_PR_BRANCH = "LOCAL_PR_%d_TESTING_BRANCH" +SECURITY_CONTENT_MAIN_BRANCH = "develop" +SECURITY_CONTENT_DIRECTORY_NAME = "security_content/" + class GithubService: - def __init__(self, security_content_branch: str, commit_hash: Union[str,None], PR_number: int = None, persist_security_content: bool = False): + def __init__(self, security_content_branch: Union[str,None], commit_hash: Union[str,None], PR_number: Union[int,None], persist_security_content:bool = False): + + if PR_number is not None and security_content_branch is not None: + raise(Exception(f"Error - branch {security_content_branch} and PR_number {PR_number} were provided. You may only provide branch OR PR_number, not both")) + if PR_number is None and security_content_branch is None and commit_hash is None: + raise(Exception("PR number, branch, and commit hash were all none. There is nothing we can check out.")) + + + if PR_number is not None : + if commit_hash is not None: + print(f"PR number {PR_number} and commit hash {commit_hash} provided. We will ignore the commit hash and check out latest most updated version of the PR.") + commit_hash = None + if security_content_branch is not None: + print(f"PR number {PR_number} and branch {security_content_branch} provided. We will ignore the branch and check out latest most updated version of the PR.") + commit_hash = None + + + + self.PR_number = PR_number + persist_security_content=False + if persist_security_content is False and os.path.exists(SECURITY_CONTENT_DIRECTORY_NAME): + print("Deleting the security_content directory...",end='') + sys.stdout.flush() + try: + shutil.rmtree("security_content/", ignore_errors=True) + except Exception as e: + print( + "Error - could not remove the security_content directory: [%s].\n\tQuitting..." % (str(e))) + sys.exit(1) + print("done") + + if persist_security_content is True and os.path.exists(SECURITY_CONTENT_DIRECTORY_NAME): + print(f"************************ \n" + "Warning: the {SECURITY_CONTENT_DIRECTORY_NAME} directory exists and you have set " + "persist_security_content to true. We will intentionally not overwrite/update/check " + "out the repo - doing so can clobber local changes. Only use this option if you are " + "using a very low bandwidth connection or have made local changes you wish to test " + "without pushing them to the repo.\n************************") + + #Now get the security_content repo if it doesn't exist: + #It didn't exist in the first place + #It was removed bycause persist_security_content was false + if not os.path.exists(SECURITY_CONTENT_DIRECTORY_NAME): + print(f"Checking out the repo [{SECURITY_CONTENT_URL}]...",end='') + sys.stdout.flush() + self.security_content_repo_obj = self.clone_project(SECURITY_CONTENT_URL, SECURITY_CONTENT_DIRECTORY_NAME, SECURITY_CONTENT_MAIN_BRANCH) + print("done") + + + #For a Fork PR, that branch may not exist in security_content. Or, worse, it may + #exist but have different content. Make sure to account for this. + if self.PR_number is not None: + self.security_content_remote_branch = security_content_branch + self.security_content_local_branch = LOCAL_PR_BRANCH%self.PR_number + + self.security_content_repo_obj.git.fetch("origin", f"pull/{self.PR_number}/head:{self.security_content_local_branch}") + self.security_content_repo_obj.git.checkout(self.security_content_local_branch) + + elif commit_hash is not None: + if security_content_branch is None: + security_content_branch = f"UNKNOWN_BRANCH_{commit_hash}" + self.security_content_remote_branch = security_content_branch + self.security_content_local_branch = security_content_branch + + print(f"Checking out by commit hash {commit_hash}...",end='') + self.security_content_repo_obj.git.checkout(commit_hash) + sys.stdout.flush() + print("done") + #Not checking to see that the hash is in the branch it claims to be in. This is tricky because the + #commit could be in several branches... + + #Ensure that the commit hash we just checked out is in the branch we think it is + #output = self.security_content_repo_obj.git.branch("-a", "--contains", commit_hash) + #matches = re.search(f"\*.*{self.security_content_local_branch}" ,str(output)) + #if matches == 1: + # raise(Exception(f"The commit hash {commit_hash} was found, but not found in the " + # "specified branch {self.security_content_local_branch}")) + + elif security_content_branch is not None: + self.security_content_remote_branch = security_content_branch + self.security_content_local_branch = security_content_branch + branch_names = [branch.name for branch in self.security_content_repo_obj.remote().refs] + #Ensure that the branch name is valid (it is a real branch in our repo) + if "origin/%s"%(self.security_content_remote_branch) not in branch_names: + raise(Exception("Branch name [%s] not found in valid branches. Try running \n"\ + "'git branch -a' to examine [%d] branches"%(self.security_content_remote_branch, len(branch_names)))) + print(f"Checking out branch: [{security_content_branch}]...", end='') + sys.stdout.flush() + self.security_content_repo_obj.git.checkout(security_content_branch) + print("done") + - self.security_content_branch = security_content_branch - if persist_security_content: - print("Getting handle on existing security_content repo!") - self.security_content_repo_obj = git.Repo("security_content") else: - print("Checking out security_content repo!") - self.security_content_repo_obj = self.clone_project( - SECURITY_CONTENT_URL, f"security_content", f"develop") + raise(Exception("Could not check out by PR number, commit hash, or branch")) + + + + #Branch should not be none here + self.security_content_remote_branch = security_content_branch + self.security_content_local_branch = security_content_branch + #Ensure that the branch name is valid + #Get all the branch names, prefixed with "origin/" + branch_names = [branch.name for branch in self.security_content_repo_obj.remote().refs] + + #Ensure that the branch name is valid (it is a real branch in our repo) + if "origin/%s"%(self.security_content_remote_branch) not in branch_names: + raise(Exception("Branch name [%s] not found in valid branches. Try running \n"\ + "'git branch -a' to examine [%d] branches"%(self.security_content_remote_branch, len(branch_names)))) + + + + else: + print(f"Checking out branch: [{security_content_branch}]...", end='') + sys.stdout.flush() + self.security_content_repo_obj.git.checkout(security_content_branch) + print("done") + + if (commit_hash != None) and (commit_hash != self.security_content_repo_obj.head.object.hexsha): + print("Warning: commit hash in configuration file [{commit_hash}] did NOT match the commit " + "hash of the PR [{self.security_content_repo_obj.head.object.hexsha}]. We just warn " + "about this and update the commit hash in the new test file.") + + self.commit_hash = self.security_content_repo_obj.head.object.hexsha + + + + + + ''' + #If the directory exists, we do a checkout on develop and pull to make sure we have the latest content for develop + try: + if os.path.exists(SECURITY_CONTENT_DIRECTORY_NAME): + print(f"SECURITY_CONTENT_DIRECTORY_NAME exists. Getting the most updated version of the main branch ({SECURITY_CONTENT_MAIN_BRANCH})...",end='') + sys.stdout.flush() + + self.security_content_repo_obj = git.Repo(SECURITY_CONTENT_DIRECTORY_NAME) + + #Make sure we're on the right branch (develop) + self.security_content_repo_obj.git.checkout(SECURITY_CONTENT_MAIN_BRANCH) + + #Make sure that branch is up to date + self.security_content_repo_obj.git.pull() + + + + + else: + #the folder did not exist, so clone the repo + print(f"{SECURITY_CONTENT_DIRECTORY_NAME} does not exist. Downloading it and switching to the main branch ({SECURITY_CONTENT_MAIN_BRANCH})...",end='') + sys.stdout.flush() + + self.security_content_repo_obj = self.clone_project(SECURITY_CONTENT_URL, f"security_content", f"develop") + except Exception as e: + raise(Exception(f"Failure updating {SECURITY_CONTENT_DIRECTORY_NAME} folder - is it a valid repo? - [{str(e)}]")) + + print("done") + + if self.PR_number is not None: + self.security_content_remote_branch = security_content_branch + self.security_content_local_branch = LOCAL_PR_BRANCH%self.PR_number + else: + self.security_content_remote_branch = security_content_branch + self.security_content_local_branch = security_content_branch + #Ensure that the branch name is valid #Get all the branch names, prefixed with "origin/" branch_names = [branch.name for branch in self.security_content_repo_obj.remote().refs] - if "origin/%s"%(security_content_branch) not in branch_names: + #Validate that the branch name is valid. For a PR, this should be the branch that the PR resides in + if "origin/%s"%(self.security_content_remote_branch) not in branch_names: raise(Exception("Branch name [%s] not found in valid branches. Try running \n"\ - "'git branch -a' to examine [%d] branches"%(security_content_branch, len(branch_names)))) + "'git branch -a' to examine [%d] branches"%(self.security_content_remote_branch, len(branch_names)))) if commit_hash is not None and PR_number is not None: @@ -53,33 +215,51 @@ class GithubService: if PR_number: - ret = subprocess.run(["git", "-C", "security_content/", "fetch", "origin", - "refs/pull/%d/head:%s" % (PR_number, security_content_branch)], capture_output=True) - #ret = subprocess.call(["git", "-C", "security_content/", "fetch", "origin", - # "refs/pull/%d/head:%s" % (PR_number, security_content_branch)]) + print(f"Fetching PR {self.PR_number} into local branch {self.security_content_local_branch}...",end='') + self.security_content_repo_obj.git.fetch("origin", f"pull/{self.PR_number}/head:{self.security_content_local_branch}") + self.security_content_repo_obj.git.checkout(self.security_content_local_branch) - - if ret.returncode != 0: - raise(Exception("Error checking out repository: [%s]"%(ret.stdout.decode("utf-8") + "\n" + ret.stderr.decode("utf-8")))) - - - # No checking to see if the hash is to a commit inside of the branch - the user - # has to do that by hand. - if commit_hash is not None: - print("Checking out commit hash: [%s]" % (commit_hash)) + elif commit_hash is not None: + # No checking to see if the hash is to a commit inside of the branch - the user + # has to do that by hand. + print("Checking out commit hash: [%s]..." % (commit_hash),end='') self.security_content_repo_obj.git.checkout(commit_hash) + else: - #Even if we have fetched a PR, we still MUST check out the branch to - # be able to do anything with it. Otherwise we won't have the files + #Just checking out and testing a branch print("Checking out branch: [%s]..." % (security_content_branch), end='') sys.stdout.flush() - self.security_content_repo_obj.git.checkout( - security_content_branch) - commit_hash = self.security_content_repo_obj.head.object.hexsha - print("commit_hash %s" % (commit_hash)) - + self.security_content_repo_obj.git.checkout(security_content_branch) + #Pull to make sure we are up to date + + + #Print completion and store the commit hash + + print(f"done") + if (commit_hash != None) and (commit_hash != self.security_content_repo_obj.head.object.hexsha): + print("Warning: commit hash in configuration file [{commit_hash}] did NOT match the commit " + "hash of the PR [{self.security_content_repo_obj.head.object.hexsha}]. We just warn " + "about this and update the commit hash in the new test file.") + + commit_hash = self.security_content_repo_obj.head.object.hexsha self.commit_hash = commit_hash + + if self.PR_number is not None: + #Verify that, if this was a PR, that the PR exists in the branch we believe it does + output = self.security_content_repo_obj.git.branch("-a", "--contains", commit_hash) + print(output) + print("***") + print(f"remotes/origin/{self.security_content_remote_branch}") + if f"remotes/origin/{self.security_content_remote_branch}" not in output: + #raise(Exception(f"PR number {self.PR_number} not found in branch {self.security_content_remote_branch}")) + print(f"PR number {self.PR_number} not found in branch {self.security_content_remote_branch}") + print("maybe this is a remote pr? or maybe you gave the wrong branch?") + + ''' + + + def update_and_commit_passed_tests(self, results:list[dict])->bool: @@ -281,13 +461,13 @@ class GithubService: def get_changed_test_files(self, folders=['endpoint', 'cloud', 'network'], types_to_test=["Anomaly", "Hunting", "TTP"], previously_successful_tests=[]) -> list[str]: - branch1 = self.security_content_branch - branch2 = 'develop' + branch1 = self.security_content_local_branch + branch2 = SECURITY_CONTENT_MAIN_BRANCH g = git.Git('security_content') all_changed_test_files = [] all_changed_detection_files = [] - if branch1 != 'develop': + if branch1 != SECURITY_CONTENT_MAIN_BRANCH: if self.commit_hash is None: differ = g.diff('--name-status', branch2 + '...' + branch1) else: diff --git a/bin/automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py b/bin/automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py index a7398dd0d0..31843fbc2b 100644 --- a/bin/automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py +++ b/bin/automated_detection_testing/ci/detection_testing_batch/modules/validate_args.py @@ -13,7 +13,7 @@ setup_schema = { "type": "object", "properties": { "branch": { - "type": "string", + "type": ["string", "null"], "default": "develop" }, "commit_hash": { @@ -359,9 +359,9 @@ setup_schema = { "type": "array", "items": { "type": "string", - "enum": ["Anomaly", "Hunting", "TTP"] + "enum": ["Anomaly", "Hunting", "TTP", "Correlation"] }, - "default": ["Anomaly", "Hunting", "TTP"] + "default": ["Anomaly", "Hunting", "TTP", "Correlation"] }, } }