diff --git a/.github/workflows/build-and-validate.yml b/.github/workflows/build-and-validate.yml index 086323d8ba..b4f6489170 100644 --- a/.github/workflows/build-and-validate.yml +++ b/.github/workflows/build-and-validate.yml @@ -273,16 +273,16 @@ jobs: run: | #Get the virtualenv set up rm -rf venv - python3 -m venv --clear venv + python -m venv --clear venv source venv/bin/activate - python3 -m pip install --upgrade pip - python3 -m pip install wheel - python3 -m pip install -q -r requirements.txt + python -m pip install --upgrade pip + python -m pip install wheel + python -m pip install -q -r requirements.txt - name: run reporting run: | source venv/bin/activate - python3 contentctl.py -p . reporting + python contentctl.py -p . reporting #Official, Verified Amazon-AWS Github Account Provided Action - uses: aws-actions/configure-aws-credentials@v1 @@ -329,11 +329,11 @@ jobs: run: | #Get the virtualenv set up rm -rf venv - python3 -m venv --clear venv + python -m venv --clear venv source venv/bin/activate - python3 -m pip install --upgrade pip - python3 -m pip install wheel - python3 -m pip install -q -r requirements.txt + python -m pip install --upgrade pip + python -m pip install wheel + python -m pip install -q -r requirements.txt - name: Get branch and PR required for detection testing main.py id: vars @@ -343,7 +343,7 @@ jobs: - name: Run reporting run: | source venv/bin/activate - python3 contentctl.py -p . reporting + python contentctl.py -p . reporting - name: Update github with new docs and package bits run: | @@ -452,16 +452,16 @@ jobs: run: | #Get the virtualenv set up rm -rf venv - python3 -m venv --clear venv + python -m venv --clear venv source venv/bin/activate - python3 -m pip install --upgrade pip - python3 -m pip install wheel - python3 -m pip install -q -r requirements.txt + python -m pip install --upgrade pip + python -m pip install wheel + python -m pip install -q -r requirements.txt - name: Create YML to JSON Folder run: | source venv/bin/activate - python3 contentctl.py --path . generate --product API --output dist/api + python contentctl.py --path . generate --product API --output dist/api - name: Generate content version and timestamp JSON run : | diff --git a/.github/workflows/code-testing.yml b/.github/workflows/code-testing.yml index 70776ee534..2e565fd0c5 100644 --- a/.github/workflows/code-testing.yml +++ b/.github/workflows/code-testing.yml @@ -23,11 +23,11 @@ jobs: run: | #Get the virtualenv set up rm -rf venv - python3 -m venv --clear venv + python -m venv --clear venv source venv/bin/activate - python3 -m pip install --upgrade pip - python3 -m pip install wheel - python3 -m pip install -q -r requirements.txt + python -m pip install --upgrade pip + python -m pip install wheel + python -m pip install -q -r requirements.txt - name: test code with pytest run: | diff --git a/.github/workflows/detection-testing.yml b/.github/workflows/detection-testing.yml index 0bd7a20c76..087821ac77 100644 --- a/.github/workflows/detection-testing.yml +++ b/.github/workflows/detection-testing.yml @@ -58,20 +58,20 @@ jobs: - uses: actions/setup-python@v2 with: python-version: '3.9' #Available versions here - https://github.com/actions/python-versions/releases easy to change/make a matrix/use pypy - architecture: 'x64' # optional x64 or x86. Defaults to x64 if not specified + architecture: 'x64' # optional x64 or x86. Defaults to x64 if not specified + cache: 'pip' - name: Install Python Dependencies run: | - cd bin/docker_detection_tester - python3 -m venv .venv + python -m venv .venv source .venv/bin/activate - python3 -m pip install wheel - python3 -m pip install -r requirements.txt + python -m pip install wheel + python -m pip install -r requirements.txt - name: Run the CI run: | - cd bin/docker_detection_tester source .venv/bin/activate + cd bin/docker_detection_tester echo "github.event.issue.pull_request : [${{ github.event.issue.pull_request }}]" echo "github.event.pull_request.number : [${{ github.event.pull_request.number }}]" echo "steps.vars.outputs.branch : [${{ steps.vars.outputs.branch }}]" @@ -152,22 +152,23 @@ jobs: - uses: actions/setup-python@v2 with: python-version: '3.9' #Available versions here - https://github.com/actions/python-versions/releases easy to change/make a matrix/use pypy - architecture: 'x64' # optional x64 or x86. Defaults to x64 if not specified + architecture: 'x64' # optional x64 or x86. Defaults to x64 if not specified + cache: 'pip' - name: Install Python Dependencies run: | - cd bin/docker_detection_tester - python3 -m venv .venv + python -m venv .venv source .venv/bin/activate - python3 -m pip install wheel - python3 -m pip install -r requirements.txt + python -m pip install wheel + python -m pip install -r requirements.txt - name: Run the CI run: | - cd bin/docker_detection_tester source .venv/bin/activate + cd bin/docker_detection_tester - python3 detection_testing_execution.py run -c prior_config/${{ matrix.manifest_filename}} + + python detection_testing_execution.py run -c prior_config/${{ matrix.manifest_filename}} - name: Upload Test Results Files @@ -255,20 +256,20 @@ jobs: - uses: actions/setup-python@v2 with: python-version: '3.9' #Available versions here - https://github.com/actions/python-versions/releases easy to change/make a matrix/use pypy - architecture: 'x64' # optional x64 or x86. Defaults to x64 if not specified + architecture: 'x64' # optional x64 or x86. Defaults to x64 if not specified + cache: 'pip' - name: Install Python Dependencies run: | - cd bin/docker_detection_tester - python3 -m venv .venv + python -m venv .venv source .venv/bin/activate - python3 -m pip install wheel - python3 -m pip install -r requirements.txt + python -m pip install wheel + python -m pip install -r requirements.txt - name: Merge Detections into single File run: | - cd bin/docker_detection_tester source .venv/bin/activate + cd bin/docker_detection_tester python summarize_json.py --files results_*/summary.json --output_filename summary_test_results.json @@ -340,4 +341,4 @@ jobs: - \ No newline at end of file + diff --git a/bin/docker_detection_tester/Dockerfile b/bin/docker_detection_tester/Dockerfile deleted file mode 100644 index 1ed762dde5..0000000000 --- a/bin/docker_detection_tester/Dockerfile +++ /dev/null @@ -1,16 +0,0 @@ -FROM ubuntu:18.04 - -RUN apt-get update -RUN DEBIAN_FRONTEND="noninteractive" apt-get -y install tzdata -RUN apt-get install -y python3-dev git python-dev unzip python3-pip awscli -RUN apt-get install -y python-gitdb -RUN apt-get install -y wget unzip -RUN apt-get install -y git - -ADD . /app - -WORKDIR /app -RUN pip3 install -r requirements.txt - -ENTRYPOINT ["python3", "detection_testing_execution.py"] -CMD ["-b", "automated_detections_testing_2"] diff --git a/bin/docker_detection_tester/detection_testing_execution.py b/bin/docker_detection_tester/detection_testing_execution.py index 01a457ae11..4df19e5037 100644 --- a/bin/docker_detection_tester/detection_testing_execution.py +++ b/bin/docker_detection_tester/detection_testing_execution.py @@ -173,14 +173,9 @@ def generate_escu_app(persist_security_content: bool = False) -> str: print("****GENERATING ESCU APP****") os.chdir("security_content") if persist_security_content is False: - commands = ["python3 -m venv .venv", - ". ./.venv/bin/activate", - "python -m pip install wheel", - "python -m pip install -r requirements.txt", - "python3 ../../../contentctl.py --path . --skip_enrichment generate --product ESCU --output dist/escu"] + commands = ["python ../../../contentctl.py --path . --skip_enrichment generate --product ESCU --output dist/escu"] else: - commands = [". ./.venv/bin/activate", - "python3 ../../../contentctl.py --path . --skip_enrichment generate --product ESCU --output dist/escu"] + commands = ["python ../../../contentctl.py --path . --skip_enrichment generate --product ESCU --output dist/escu"] ret = subprocess.run("; ".join(commands), shell=True, capture_output=True) if ret.returncode != 0: @@ -199,7 +194,7 @@ def generate_escu_app(persist_security_content: bool = False) -> str: output_file_path_from_slim_latest = os.path.join( "upload", output_file_name) output_file_path_from_security_content = os.path.join( - "slim_packaging", "slim_latest", output_file_path_from_slim_latest) + "slim_packaging", output_file_path_from_slim_latest) output_file_path_from_root = os.path.join( "security_content", output_file_path_from_security_content) @@ -215,42 +210,17 @@ def generate_escu_app(persist_security_content: bool = False) -> str: sys.exit(1) # There remove the latest file if it exists - commands = ["cd slim_packaging/slim_latest", - ". ./.venv/bin/activate", - "cp -R ../../dist/escu DA-ESS-ContentUpdate", + commands = ["cd slim_packaging", + "cp -R ../dist/escu DA-ESS-ContentUpdate", "slim package -o upload DA-ESS-ContentUpdate", "cp upload/DA-ESS-ContentUpdate*.tar.gz %s" % (output_file_path_from_slim_latest)] else: os.mkdir("slim_packaging") - - try: - SPLUNK_PACKAGING_TOOLKIT_URL = "https://download.splunk.com/misc/packaging-toolkit/splunk-packaging-toolkit-0.9.0.tar.gz" - SPLUNK_PACKAGING_TOOLKIT_FILENAME = 'splunk-packaging-toolkit-latest.tar.gz' - print("Downloading the Splunk Packaging Toolkit from %s..." % - (SPLUNK_PACKAGING_TOOLKIT_URL), end='') - response = get(SPLUNK_PACKAGING_TOOLKIT_URL) - response.raise_for_status() - with open(SPLUNK_PACKAGING_TOOLKIT_FILENAME, 'wb') as slim_file: - slim_file.write(response.content) - print("Done") - except Exception as e: - print("Error downloading the Splunk Packaging Toolkit: [%s].\n\tQuitting..." % - (str(e)), file=sys.stderr) - sys.exit(1) - commands = ["rm -rf slim_packaging/slim_latest", - "mkdir slim_packaging/slim_latest", + "mkdir slim_packaging", "cd slim_packaging", - "tar -zxf ../splunk-packaging-toolkit-latest.tar.gz -C slim_latest --strip-components=1", - "cd slim_latest", - "python3 -m venv .venv", - ". ./.venv/bin/activate", - "python -m pip install --upgrade pip", - "python -m pip install wheel", - "python -m pip install semantic_version", - "python -m pip install .", - "cp -R ../../dist/escu DA-ESS-ContentUpdate", + "cp -R ../dist/escu DA-ESS-ContentUpdate", "slim package -o upload DA-ESS-ContentUpdate", "cp upload/DA-ESS-ContentUpdate*.tar.gz %s" % (output_file_path_from_slim_latest)] diff --git a/bin/docker_detection_tester/modules/splunk_sdk.py b/bin/docker_detection_tester/modules/splunk_sdk.py index f930e7531b..cdca4f0995 100644 --- a/bin/docker_detection_tester/modules/splunk_sdk.py +++ b/bin/docker_detection_tester/modules/splunk_sdk.py @@ -80,15 +80,18 @@ def get_number_of_indexed_events(splunk_host, splunk_port, splunk_password, inde search = f'''search index="{index}" host="{event_host}" | stats count''' kwargs = {"exec_mode":"blocking"} try: - search_result = service.jobs.create(search, **kwargs) + job = service.jobs.create(search, **kwargs) #This returns the count in string form, not as an int. For example: #OrderedDict([('count', '59630')]) - search_results = list(results.ResultsReader(search_result.results())) - if len(search_results) != 1: + results_stream = job.results(output_mode='json') + count = None + for res in results.JSONResultsReader(results_stream): + if 'count' in res: + count = int(res['count'],10) + if count is None: raise Exception(f"Expected the get_number_of_indexed_events search to only return 1 count, but got {len(search_results)} instead.") - count = int(search_results[0]['count']) return count except Exception as e: @@ -122,59 +125,6 @@ def wait_for_indexing_to_complete(splunk_host, splunk_port, splunk_password, sou time.sleep(check_interval_seconds) -''' -def wait_for_indexing_to_complete(splunk_host, splunk_port, splunk_password, sourcetype:str, index:str, check_interval_seconds:int=10): - - startTime = timeit.default_timer() - previous_count = -1 - time.sleep(check_interval_seconds/2) - while True: - #print("waiting for search...") - try: - service = client.connect( - host=splunk_host, - port=splunk_port, - username='admin', - password=splunk_password - ) - except Exception as e: - raise(Exception("Unable to connect to Splunk instance: " + str(e))) - - search = 'search index="%s" sourcetype="%s" | stats count'%(index,sourcetype) - kwargs = {"exec_mode":"blocking"} - try: - search_result = service.jobs.create(search, **kwargs) - except Exception as e: - print("Error while waiting for indexing of data to complete: %s"%(str(e))) - #return False - - #This returns the count in string form, not as an int. For example: - #OrderedDict([('count', '59630')]) - try: - for result in results.ResultsReader(search_result.results()): - count = int(result['count']) - print("count is %d, previous count is %d"%(count,previous_count)) - if previous_count == -1: - if count == 0: - pass - else: - previous_count = count - else: - if count == previous_count: - #After waiting for the check interval, we return the same number of results. The indexing must be complete - stopTime = timeit.default_timer() - #print("Indexing completed after: %s "%(datetime.timedelta(seconds=stopTime-startTime))) - return True - else: - previous_count = count - - except Exception as e: - print("Error trying to get the count while waiting for indexing to complete: %s"%(str(e))) - #return False - time.sleep(check_interval_seconds) -''' - - def test_baseline_search(splunk_host, splunk_port, splunk_password, search, pass_condition, baseline_name, baseline_file, earliest_time, latest_time)->dict: try: service = client.connect( @@ -280,6 +230,8 @@ def test_detection_search(splunk_host:str, splunk_port:int, splunk_password:str, try: job = service.jobs.create(splunk_search, **kwargs) + results_stream = job.results(output_mode='json') + except Exception as e: error_message = "Unable to execute detection: %s"%(str(e)) @@ -301,7 +253,7 @@ def test_detection_search(splunk_host:str, splunk_port:int, splunk_password:str, if int(job['resultCount']) != 1: #print("Test failed for detection: " + detection_name) if attempts_remaining > 0: - print(f"Execution of test failed for [{detection_name}]. Sleeping for [{failure_sleep_interval_seconds} seconds] and trying again...") + print(f"Execution of test failed for [{detection_name}]. Sleeping for [{failure_sleep_interval_seconds} seconds] and trying up to {attempts_remaining} more times...") time.sleep(failure_sleep_interval_seconds) return test_detection_search(splunk_host, splunk_port, splunk_password, search, pass_condition, detection_name, detection_file, earliest_time, latest_time, attempts_remaining=attempts_remaining, @@ -347,7 +299,8 @@ def delete_attack_data(splunk_host:str, splunk_password:str, splunk_port:int, wa try: job = service.jobs.create(splunk_search, **kwargs) - reader = results.ResultsReader(job) + results_stream = job.results(output_mode='json') + reader = results.JSONResultsReader(results_stream) except Exception as e: diff --git a/bin/docker_detection_tester/requirements.txt b/bin/docker_detection_tester/requirements.txt deleted file mode 100644 index 6986ae3ee8..0000000000 --- a/bin/docker_detection_tester/requirements.txt +++ /dev/null @@ -1,17 +0,0 @@ -psutil==5.8.0 -attackcti==0.3.4.3 -GitPython==3.1.14 -Jinja2==3.0.0 -PyYAML==5.4 -requests==2.25.1 -six==1.16.0 - -splunk-sdk==1.6.12 -splunk-packaging-toolkit==1.0.1 -#newest version of docker for managing the splunk containers -#we will freeze at a specific version later -docker==5.0.3 - -#For help getting and parsing the configuration -jsonschema==4.2.1 -wrapt_timeout_decorator==1.3.1 diff --git a/requirements.txt b/requirements.txt index dae96ec7ae..6703252ca2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,12 +1,20 @@ attackcti +docker +gitpython jinja2 +jsonschema mock +psutil pycvesearch pydantic pytest -PyYAML +pyyaml questionary requests +six +splunk-packaging-toolkit +splunk-sdk +wrapt_timeout_decorator xmltodict splunk-sdk https://download.splunk.com/misc/packaging-toolkit/splunk-packaging-toolkit-1.0.1.tar.gz