From 63aad3ee438db44d585af29dd0f908870967e83c Mon Sep 17 00:00:00 2001 From: divious1 Date: Fri, 19 Mar 2021 13:58:48 -0400 Subject: [PATCH 01/15] deprecating detection --- .../ssa___rare_parent_process_relationship_lolbas.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename detections/{endpoint => deprecated}/ssa___rare_parent_process_relationship_lolbas.yml (98%) diff --git a/detections/endpoint/ssa___rare_parent_process_relationship_lolbas.yml b/detections/deprecated/ssa___rare_parent_process_relationship_lolbas.yml similarity index 98% rename from detections/endpoint/ssa___rare_parent_process_relationship_lolbas.yml rename to detections/deprecated/ssa___rare_parent_process_relationship_lolbas.yml index 016e7518ad..1fa1d784df 100644 --- a/detections/endpoint/ssa___rare_parent_process_relationship_lolbas.yml +++ b/detections/deprecated/ssa___rare_parent_process_relationship_lolbas.yml @@ -47,7 +47,7 @@ search: '| from read_ssa_enriched_events() | eval timestamp=parse_long(ucast(map OR process_name="pcalua.exe" OR process_name="cmdkey.exe" OR process_name="msconfig.exe") | eval start_time = timestamp, end_time = timestamp, entities = mvappend(dest_device_id, - dest_user_id), body = "TBD" | into write_ssa_detected_events();' + dest_user_id), body = "TBD"' how_to_implement: Collect endpoint data such as sysmon or 4688 events. known_false_positives: 'Some custom tools used by admins could be used rarely to launch remotely applications. This might trigger false positives at the beginning when From 066b81e6ef797f73970d20d19bc1fc6787f71ee3 Mon Sep 17 00:00:00 2001 From: divious1 Date: Fri, 19 Mar 2021 13:59:56 -0400 Subject: [PATCH 02/15] moved out of deprecated --- .../ssa___rare_parent_process_relationship_lolbas.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename detections/{deprecated => endpoint}/ssa___rare_parent_process_relationship_lolbas.yml (100%) diff --git a/detections/deprecated/ssa___rare_parent_process_relationship_lolbas.yml b/detections/endpoint/ssa___rare_parent_process_relationship_lolbas.yml similarity index 100% rename from detections/deprecated/ssa___rare_parent_process_relationship_lolbas.yml rename to detections/endpoint/ssa___rare_parent_process_relationship_lolbas.yml From 6bf8b82877eb2bcbb3812f671dc6272d4ef8a355 Mon Sep 17 00:00:00 2001 From: divious1 Date: Fri, 19 Mar 2021 14:08:26 -0400 Subject: [PATCH 03/15] added write to null --- .../endpoint/ssa___rare_parent_process_relationship_lolbas.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/detections/endpoint/ssa___rare_parent_process_relationship_lolbas.yml b/detections/endpoint/ssa___rare_parent_process_relationship_lolbas.yml index 1fa1d784df..2168b8e1ef 100644 --- a/detections/endpoint/ssa___rare_parent_process_relationship_lolbas.yml +++ b/detections/endpoint/ssa___rare_parent_process_relationship_lolbas.yml @@ -47,7 +47,7 @@ search: '| from read_ssa_enriched_events() | eval timestamp=parse_long(ucast(map OR process_name="pcalua.exe" OR process_name="cmdkey.exe" OR process_name="msconfig.exe") | eval start_time = timestamp, end_time = timestamp, entities = mvappend(dest_device_id, - dest_user_id), body = "TBD"' + dest_user_id), body = "TBD" | into write_null();' how_to_implement: Collect endpoint data such as sysmon or 4688 events. known_false_positives: 'Some custom tools used by admins could be used rarely to launch remotely applications. This might trigger false positives at the beginning when From 0ec2001838689af50d00ae0a82586d59cb02ca15 Mon Sep 17 00:00:00 2001 From: Ignacio Bermudez Corrales Date: Fri, 19 Mar 2021 13:52:34 -0700 Subject: [PATCH 04/15] Renaming test .yaml -> .yml | No events should come out of the detection --- ...l => ssa___rare_parent_process_relationship_lolbas.test.yml} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename tests/endpoint/{ssa___rare_parent_process_relationship_lolbas.test.yaml => ssa___rare_parent_process_relationship_lolbas.test.yml} (93%) diff --git a/tests/endpoint/ssa___rare_parent_process_relationship_lolbas.test.yaml b/tests/endpoint/ssa___rare_parent_process_relationship_lolbas.test.yml similarity index 93% rename from tests/endpoint/ssa___rare_parent_process_relationship_lolbas.test.yaml rename to tests/endpoint/ssa___rare_parent_process_relationship_lolbas.test.yml index a44cdc41f0..7cc3da7342 100644 --- a/tests/endpoint/ssa___rare_parent_process_relationship_lolbas.test.yaml +++ b/tests/endpoint/ssa___rare_parent_process_relationship_lolbas.test.yml @@ -2,7 +2,7 @@ name: Rare Parent/Child Process Relationship - SSA Unit Test tests: - name: Access LSASS Memory for Dump Creation file: endpoint/ssa___rare_parent_process_relationship_lolbas.yml - pass_condition: '@count_gt(0)' + pass_condition: '@count_eq(0)' description: Test detection looking for LOLBAS processes spawned by other processes that are rarely seen together attack_data: - file_name: T1059.all.labeled.lolbas-test.json From 593ae74fdf3d654fe6513ff124c37aa571dcb4b1 Mon Sep 17 00:00:00 2001 From: Ignacio Bermudez Corrales Date: Fri, 19 Mar 2021 14:51:53 -0700 Subject: [PATCH 05/15] Refactoring template replacement --- bin/ssa-end-to-end-testing/modules/utils.py | 24 +++++++++++---------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/bin/ssa-end-to-end-testing/modules/utils.py b/bin/ssa-end-to-end-testing/modules/utils.py index b6c5cc7ede..8548d0be47 100644 --- a/bin/ssa-end-to-end-testing/modules/utils.py +++ b/bin/ssa-end-to-end-testing/modules/utils.py @@ -76,13 +76,19 @@ def request_headers(header_token): def manipulate_spl(env, spl, results_index): - spl = replace_ssa_macros(env, spl) + # Obtain the SSA source + pulsar_source_connection_id, pulsar_source_topic = return_macros(env) + source = READ_SSA_ENRICHED_EVENTS_EXPANDED\ + .replace("__PULSAR_SOURCE_CONNECTION_ID__", pulsar_source_connection_id)\ + .replace("__PULSAR_SOURCE_TOPIC__", pulsar_source_topic) + # Obtain the test sink + sink = ";" if results_index is not None: - # When an index is defined for a test, it writes the output of this pipeline to this index. - # original_pipeline; => original_pipeline | into index("module", "index"); module = results_index["module"] index = results_index["name"] - spl = spl[:spl.rindex(";")] + f" | into index(\"{module}\", \"{index}\");" + sink = f"| into index(\"{module}\", \"{index}\");" + # Replace spl template with its `source` and `sink` + spl = replace_ssa_macros(source, sink, spl) LOGGER.info(f"spl: {spl}") return spl @@ -93,13 +99,9 @@ def read_spl(file_path, file_name): return spl -def replace_ssa_macros(env, spl): - pulsar_source_connection_id, pulsar_source_topic = return_macros(env) - macro_expanded = READ_SSA_ENRICHED_EVENTS_EXPANDED.replace("__PULSAR_SOURCE_CONNECTION_ID__", pulsar_source_connection_id) - macro_expanded = macro_expanded.replace("__PULSAR_SOURCE_TOPIC__", pulsar_source_topic) - spl = spl.replace(READ_SSA_ENRICHED_EVENTS, macro_expanded) - spl = spl.replace(WRITE_SSA_DETECTED_EVENTS, ";") - #spl = spl.replace("\n", " ") +def replace_ssa_macros(source, sink, spl): + spl = spl.replace(READ_SSA_ENRICHED_EVENTS, source) + spl = spl.replace(WRITE_SSA_DETECTED_EVENTS, sink) return spl From 548f768ff34b7484d757878dbb1e3ecce4cfcd1c Mon Sep 17 00:00:00 2001 From: Ignacio Bermudez Corrales Date: Fri, 19 Mar 2021 15:10:06 -0700 Subject: [PATCH 06/15] refactoring base tests --- bin/ssa-end-to-end-testing/modules/spl/detection.spl | 3 ++- bin/ssa-end-to-end-testing/modules/spl/detection2.spl | 3 ++- bin/ssa-end-to-end-testing/modules/spl/firehose.spl | 2 +- bin/ssa-end-to-end-testing/modules/spl/troubleshoot.spl | 2 +- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/bin/ssa-end-to-end-testing/modules/spl/detection.spl b/bin/ssa-end-to-end-testing/modules/spl/detection.spl index c4cf977814..1e2c938227 100644 --- a/bin/ssa-end-to-end-testing/modules/spl/detection.spl +++ b/bin/ssa-end-to-end-testing/modules/spl/detection.spl @@ -17,4 +17,5 @@ | eval start_time = timestamp, end_time = timestamp, entities = mvappend(dest_device_id, dest_user_id), -body = "TBD"; \ No newline at end of file +body = "TBD" +| into write_ssa_detected_events(); \ No newline at end of file diff --git a/bin/ssa-end-to-end-testing/modules/spl/detection2.spl b/bin/ssa-end-to-end-testing/modules/spl/detection2.spl index d035fc7f36..73cabaaafc 100644 --- a/bin/ssa-end-to-end-testing/modules/spl/detection2.spl +++ b/bin/ssa-end-to-end-testing/modules/spl/detection2.spl @@ -17,4 +17,5 @@ | eval start_time = timestamp, end_time = timestamp, entities = mvappend(dest_device_id, dest_user_id), -body = "TBD"; +body = "TBD" +| into write_ssa_detected_events(); diff --git a/bin/ssa-end-to-end-testing/modules/spl/firehose.spl b/bin/ssa-end-to-end-testing/modules/spl/firehose.spl index be0790a827..c8bdf3a602 100644 --- a/bin/ssa-end-to-end-testing/modules/spl/firehose.spl +++ b/bin/ssa-end-to-end-testing/modules/spl/firehose.spl @@ -1 +1 @@ -| from read_splunk_firehose(); \ No newline at end of file + | from read_splunk_firehose(); \ No newline at end of file diff --git a/bin/ssa-end-to-end-testing/modules/spl/troubleshoot.spl b/bin/ssa-end-to-end-testing/modules/spl/troubleshoot.spl index c093070090..d374be63ef 100644 --- a/bin/ssa-end-to-end-testing/modules/spl/troubleshoot.spl +++ b/bin/ssa-end-to-end-testing/modules/spl/troubleshoot.spl @@ -1 +1 @@ -| from read_ssa_enriched_events(); \ No newline at end of file +| from read_ssa_enriched_events() | into write_ssa_detected_events(); \ No newline at end of file From 1eb44354e82d00693d10943caa55eaa6d4c90110 Mon Sep 17 00:00:00 2001 From: Ignacio Bermudez Corrales Date: Fri, 19 Mar 2021 15:54:47 -0700 Subject: [PATCH 07/15] renaming back test for ssa detection --- ...ml => ssa___rare_parent_process_relationship_lolbas.test.yaml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/endpoint/{ssa___rare_parent_process_relationship_lolbas.test.yml => ssa___rare_parent_process_relationship_lolbas.test.yaml} (100%) diff --git a/tests/endpoint/ssa___rare_parent_process_relationship_lolbas.test.yml b/tests/endpoint/ssa___rare_parent_process_relationship_lolbas.test.yaml similarity index 100% rename from tests/endpoint/ssa___rare_parent_process_relationship_lolbas.test.yml rename to tests/endpoint/ssa___rare_parent_process_relationship_lolbas.test.yaml From d47e5c395e5f1177f91a53d05e6a8a410caa3576 Mon Sep 17 00:00:00 2001 From: Ignacio Bermudez Corrales Date: Fri, 19 Mar 2021 16:45:05 -0700 Subject: [PATCH 08/15] Dealing with yaml and yml for testing --- bin/ssa-end-to-end-testing/modules/github_service.py | 7 ++++--- bin/ssa-end-to-end-testing/run_ssa_smoketest.py | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/bin/ssa-end-to-end-testing/modules/github_service.py b/bin/ssa-end-to-end-testing/modules/github_service.py index de159e476c..82ed3b8c15 100644 --- a/bin/ssa-end-to-end-testing/modules/github_service.py +++ b/bin/ssa-end-to-end-testing/modules/github_service.py @@ -35,8 +35,6 @@ class GithubService: changed_ssa_test_files = [] - #tests = self.read_security_content_test_files() - for file_path in changed_files: # added or changed test files if file_path.startswith('tests'): @@ -47,7 +45,10 @@ class GithubService: # changed detections if file_path.startswith('detections'): if os.path.basename(file_path).startswith('ssa'): - file_path_new = os.path.splitext(file_path)[0].replace('detections', 'tests') + '.test.yml' + file_path_base = os.path.splitext(file_path)[0].replace('detections', 'tests') + '.test' + file_path_new = file_path_base + '.yml' + if not os.path.exists(file_path_new): + file_path_new = file_path_base + '.yaml' if file_path_new not in changed_ssa_test_files: changed_ssa_test_files.append(file_path_new) diff --git a/bin/ssa-end-to-end-testing/run_ssa_smoketest.py b/bin/ssa-end-to-end-testing/run_ssa_smoketest.py index aeb006f38b..6e69820fba 100644 --- a/bin/ssa-end-to-end-testing/run_ssa_smoketest.py +++ b/bin/ssa-end-to-end-testing/run_ssa_smoketest.py @@ -38,7 +38,7 @@ def main(args): for test_file in test_files_ssa: LOGGER.info(test_file) - if len(test_files_ssa)==0: + if len(test_files_ssa) == 0: LOGGER.info('Nothing to test for SSA smoke test.') sys.exit(0) From 13ab17f2d18f42f55eca800ad209e15994f60c3a Mon Sep 17 00:00:00 2001 From: Ignacio Bermudez Corrales Date: Mon, 22 Mar 2021 11:30:53 -0700 Subject: [PATCH 09/15] refactor test workflow with assertions and catch. Teardown as well --- .../modules/github_service.py | 3 - .../modules/test_ssa_detections.py | 116 +++++++++--------- bin/ssa-end-to-end-testing/modules/utils.py | 6 + 3 files changed, 64 insertions(+), 61 deletions(-) diff --git a/bin/ssa-end-to-end-testing/modules/github_service.py b/bin/ssa-end-to-end-testing/modules/github_service.py index 82ed3b8c15..09e13de335 100644 --- a/bin/ssa-end-to-end-testing/modules/github_service.py +++ b/bin/ssa-end-to-end-testing/modules/github_service.py @@ -2,8 +2,6 @@ import git import os import logging -from os import path -import sys # Logger @@ -25,7 +23,6 @@ class GithubService: repo_obj = git.Repo.clone_from(url, project, branch=branch) return repo_obj - def get_changed_test_files_ssa(self): branch1 = self.security_content_branch branch2 = 'develop' diff --git a/bin/ssa-end-to-end-testing/modules/test_ssa_detections.py b/bin/ssa-end-to-end-testing/modules/test_ssa_detections.py index 126045831a..3b506247bb 100644 --- a/bin/ssa-end-to-end-testing/modules/test_ssa_detections.py +++ b/bin/ssa-end-to-end-testing/modules/test_ssa_detections.py @@ -1,4 +1,3 @@ - import logging import os import time @@ -6,8 +5,7 @@ import sys from http import HTTPStatus from modules.streams_service_api_helper import DSPApi -from modules.utils import manipulate_spl, read_spl, read_data - +from modules.utils import check_source_sink, manipulate_spl, read_spl, read_data # Logger logging.basicConfig(level=os.environ.get("LOGLEVEL", "INFO")) @@ -22,6 +20,7 @@ MAX_EXECUTION_TIME_LIMIT = 600 # per detection test TEST_DATASET = 'windows-security_small.txt' + class SSADetectionTesting: def __init__(self, env, tenant, header_token): @@ -45,7 +44,7 @@ class SSADetectionTesting: ] test_results = [] - for i in range(0,len(test_spls)): + for i in range(0, len(test_spls)): self.max_execution_time = MAX_EXECUTION_TIME_LIMIT test_result = self.ssa_detection_test(read_spl(file_path_spl, test_spls[i]), file_path_data, test_names[i]) test_results.append(test_result.copy()) @@ -62,17 +61,16 @@ class SSADetectionTesting: return passed - def test_ssa_detections(self, test_obj): LOGGER.info('Test SSA Detection: ' + test_obj["detection_obj"]["name"]) self.max_execution_time = MAX_EXECUTION_TIME_LIMIT file_path_attack_data = os.path.join(os.path.dirname(__file__), "../", test_obj["attack_data_file_path"]) - test_results = self.ssa_detection_test(test_obj["detection_obj"]["search"], file_path_attack_data, "SSA Smoke Test " + test_obj["test_obj"]["name"]) + test_results = self.ssa_detection_test(test_obj["detection_obj"]["search"], file_path_attack_data, + "SSA Smoke Test " + test_obj["test_obj"]["name"]) return test_results - ## Helper Functions ## def update_execution_time(self, time_frame): @@ -86,30 +84,12 @@ class SSADetectionTesting: time.sleep(time_in_s) return self.update_execution_time(time_in_s) - def check_result(self, condition, error_message): - try: - assert condition - except: - self.execution_passed = False - LOGGER.error(error_message) - - def write_test_results(self, test_name): - if not self.execution_passed: - msg = f"Detection test failed for {test_name}" - LOGGER.error(msg) - self.test_results["msg"] = msg - self.test_results["result"] = False - else: - msg = f"Detection test successful for {test_name}" - LOGGER.info(msg) - self.test_results["msg"] = msg - - def ssa_detection_test_init(self): self.test_results["result"] = True self.test_results["msg"] = "" - self.results_index = self.api.create_temp_index("mc") - + self.result_indexes = [self.api.create_temp_index("mc")] + self.created_pipelines = [] + self.activated_pipelines = [] def ssa_detection_test_main(self, spl, source, test_name): self.execution_passed = True @@ -117,27 +97,31 @@ class SSADetectionTesting: self.wait_time(SLEEP_TIME_CREATE_INDEX) spl = manipulate_spl(self.api.env, spl, self.results_index) - self.check_result(spl is not None, "fail to manipulate spl file") + assert spl is not None, "fail to manipulate spl file" pipeline_id = self.api.create_pipeline_from_spl(spl) - self.check_result(pipeline_id is not None, "failed to create a pipeline") + assert pipeline_id is not None, "failed to create a pipeline" _pipeline_status = self.api.pipeline_status(pipeline_id) - self.check_result(_pipeline_status=="CREATED", f"Current status of pipeline {pipeline_id} should be CREATED") + assert _pipeline_status == "CREATED", f"Current status of pipeline {pipeline_id} should be CREATED" + self.created_pipelines.append(pipeline_id) response_body = self.api.activate_pipeline(pipeline_id) - self.check_result(response_body.get("activated")==pipeline_id, f"pipeline {pipeline_id} should be successfully activate.") + assert response_body.get("activated") == pipeline_id, f"pipeline {pipeline_id} should be successfully activate." + self.activated_pipelines.append(pipeline_id) self.wait_time(SLEEP_TIME_ACTIVATE_PIPELINE) + if not check_source_sink(spl): + msg = f"Detection test successful for {test_name}" + LOGGER.warning(f"Test not completed. Detection seems deprecated, and will not send messages to SSA") + self.test_results["msg"] = msg + return self.test_results + data = read_data(source) LOGGER.info("Sending (%d) events" % (len(data))) - if len(data) == 0: - LOGGER.warning("No events to send, skip to next test.") - self.execution_passed = False - self.write_test_results(test_name) - return self.test_results + assert len(data) > 0, "No events to send, skip to next test." for d in data: response_body = self.api.ingest_data(d) @@ -151,8 +135,8 @@ class SSADetectionTesting: max_execution_time_reached = self.wait_time(WAIT_CYCLE) query = f"from indexes('{self.results_index['name']}') | search source!=\"Search Catalog\" " sid = self.api.submit_search_job(self.results_index['module'], query) - self.check_result(sid is not None, f"Failed to create a Search Job") - + assert sid is not None, f"Failed to create a Search Job" + job_finished = False while not job_finished: self.wait_time(WAIT_CYCLE) @@ -162,33 +146,49 @@ class SSADetectionTesting: results = self.api.get_search_job_results(sid) search_results = (len(results) > 0) if not search_results: - LOGGER.info(f"Search didn't return any results. Retrying in {WAIT_CYCLE}s, max execution time left {self.max_execution_time}s") - - self.check_result(len(results) > 0, "Search job didn't return any results") + LOGGER.info( + f"Search didn't return any results. Retrying in {WAIT_CYCLE}s, max execution time left {self.max_execution_time}s") - response, response_body = self.api.deactivate_pipeline(pipeline_id) - self.check_result(response.status_code == HTTPStatus.OK, f"The pipeline {pipeline_id} fails to deactivated.") + assert len(results) > 0, "Search job didn't return any results" - response = self.api.delete_pipeline(pipeline_id) - self.check_result(response.status_code == HTTPStatus.NO_CONTENT, f"Fail to delete pipeline {pipeline_id}.") - - self.write_test_results(test_name) + msg = f"Detection test successful for {test_name}" + LOGGER.info(msg) + self.test_results["msg"] = msg return self.test_results - def ssa_detection_test_teardown(self): - pass - self.api.delete_temp_index(self.results_index["id"]) - + """ + Deactivate and deletes pipelines, deletes results indexes, + and when it fails it shows pipelines and result indexes that were not removed. + :return: + None + """ + deactivate_pipeline = lambda p: self.api.deactivate_pipeline(p).status_code == HTTPStatus.OK + delete_pipeline = lambda p: self.api.delete_pipeline(p).status_code == HTTPStatus.NO_CONTENT + delete_index = lambda p: self.api.delete_temp_index(self.results_index["id"]) == HTTPStatus.NO_CONTENT + self.activated_pipelines = [p for p in self.activated_pipelines if not deactivate_pipeline(p)] + self.created_pipelines = [p for p in self.created_pipelines if not delete_pipeline(p)] + self.result_indexes = [p for p in self.result_indexes if not delete_index] + if len(self.activated_pipelines) > 0 or len(self.created_pipelines) > 0 or len(self.result_indexes) > 0: + LOGGER.warning("Not all SCS resources fred up") + LOGGER.info(f"Created Pipelines: {','.join(self.created_pipelines)}") + LOGGER.info(f"Active Pipelines: {','.join(self.activated_pipelines)}") + LOGGER.info(f"Result Indexes: {','.join(self.result_indexes)}") + else: + LOGGER.info("Testing successfully cleaned up") def ssa_detection_test(self, spl, source, test_name): self.ssa_detection_test_init() - test_result = self.ssa_detection_test_main(spl, source, test_name) - self.ssa_detection_test_teardown() - return test_result - - + try: + test_result = self.ssa_detection_test_main(spl, source, test_name) + self.ssa_detection_test_teardown() + return test_result + except AssertionError as e: + self.ssa_detection_test_teardown() + LOGGER.error(e.args[0]) + return {"result": False, + "msg": f"Detection test failure for {test_name}"} # only for troubleshooting # def ssa_detection_in_dsp_with_preview_session(self, spl, source, test_name): @@ -213,4 +213,4 @@ class SSADetectionTesting: # response = self.api.stop_preview_session(preview_id) - # self.write_test_results(test_name) \ No newline at end of file + # self.write_test_results(test_name) diff --git a/bin/ssa-end-to-end-testing/modules/utils.py b/bin/ssa-end-to-end-testing/modules/utils.py index 8548d0be47..7830901892 100644 --- a/bin/ssa-end-to-end-testing/modules/utils.py +++ b/bin/ssa-end-to-end-testing/modules/utils.py @@ -75,6 +75,12 @@ def request_headers(header_token): return headers +def check_source_sink(spl): + match_source = re.match(r"^\s*\|\s+from\s+read_ssa_enriched_events\(\s*\)", spl) + match_sink = re.match(r".*into\s+write_ssa_detected_events\(\s*\)\s*;", spl) + return match_source and match_sink + + def manipulate_spl(env, spl, results_index): # Obtain the SSA source pulsar_source_connection_id, pulsar_source_topic = return_macros(env) From d6d02eb6fcfdbfab1988fff8e1bc88683c621d08 Mon Sep 17 00:00:00 2001 From: Ignacio Bermudez Corrales Date: Mon, 22 Mar 2021 11:56:08 -0700 Subject: [PATCH 10/15] removing index fixed --- .../modules/test_ssa_detections.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/bin/ssa-end-to-end-testing/modules/test_ssa_detections.py b/bin/ssa-end-to-end-testing/modules/test_ssa_detections.py index 3b506247bb..fa6270c46f 100644 --- a/bin/ssa-end-to-end-testing/modules/test_ssa_detections.py +++ b/bin/ssa-end-to-end-testing/modules/test_ssa_detections.py @@ -87,7 +87,7 @@ class SSADetectionTesting: def ssa_detection_test_init(self): self.test_results["result"] = True self.test_results["msg"] = "" - self.result_indexes = [self.api.create_temp_index("mc")] + self.results_index = self.api.create_temp_index("mc") self.created_pipelines = [] self.activated_pipelines = [] @@ -166,15 +166,14 @@ class SSADetectionTesting: """ deactivate_pipeline = lambda p: self.api.deactivate_pipeline(p).status_code == HTTPStatus.OK delete_pipeline = lambda p: self.api.delete_pipeline(p).status_code == HTTPStatus.NO_CONTENT - delete_index = lambda p: self.api.delete_temp_index(self.results_index["id"]) == HTTPStatus.NO_CONTENT + delete_index = lambda p: self.api.delete_temp_index(p["id"]) == HTTPStatus.NO_CONTENT self.activated_pipelines = [p for p in self.activated_pipelines if not deactivate_pipeline(p)] self.created_pipelines = [p for p in self.created_pipelines if not delete_pipeline(p)] - self.result_indexes = [p for p in self.result_indexes if not delete_index] - if len(self.activated_pipelines) > 0 or len(self.created_pipelines) > 0 or len(self.result_indexes) > 0: + if len(self.activated_pipelines) > 0 or len(self.created_pipelines) > 0 or not delete_index(self.results_index): LOGGER.warning("Not all SCS resources fred up") LOGGER.info(f"Created Pipelines: {','.join(self.created_pipelines)}") LOGGER.info(f"Active Pipelines: {','.join(self.activated_pipelines)}") - LOGGER.info(f"Result Indexes: {','.join(self.result_indexes)}") + LOGGER.info(f"Result Indexes: {self.results_index}") else: LOGGER.info("Testing successfully cleaned up") From 61d2dbb0e84d63b6f643982372bfa92401189571 Mon Sep 17 00:00:00 2001 From: Ignacio Bermudez Corrales Date: Mon, 22 Mar 2021 12:44:49 -0700 Subject: [PATCH 11/15] api queries return tuples --- bin/ssa-end-to-end-testing/modules/test_ssa_detections.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bin/ssa-end-to-end-testing/modules/test_ssa_detections.py b/bin/ssa-end-to-end-testing/modules/test_ssa_detections.py index fa6270c46f..61cfdce563 100644 --- a/bin/ssa-end-to-end-testing/modules/test_ssa_detections.py +++ b/bin/ssa-end-to-end-testing/modules/test_ssa_detections.py @@ -164,9 +164,9 @@ class SSADetectionTesting: :return: None """ - deactivate_pipeline = lambda p: self.api.deactivate_pipeline(p).status_code == HTTPStatus.OK - delete_pipeline = lambda p: self.api.delete_pipeline(p).status_code == HTTPStatus.NO_CONTENT - delete_index = lambda p: self.api.delete_temp_index(p["id"]) == HTTPStatus.NO_CONTENT + deactivate_pipeline = lambda p: self.api.deactivate_pipeline(p)[0].status_code == HTTPStatus.OK + delete_pipeline = lambda p: self.api.delete_pipeline(p)[0].status_code == HTTPStatus.NO_CONTENT + delete_index = lambda p: self.api.delete_temp_index(p["id"])[0] == HTTPStatus.NO_CONTENT self.activated_pipelines = [p for p in self.activated_pipelines if not deactivate_pipeline(p)] self.created_pipelines = [p for p in self.created_pipelines if not delete_pipeline(p)] if len(self.activated_pipelines) > 0 or len(self.created_pipelines) > 0 or not delete_index(self.results_index): From 641fa24fb33d7e570844d4f17fbf2854d5d0c01c Mon Sep 17 00:00:00 2001 From: Ignacio Bermudez Corrales Date: Mon, 22 Mar 2021 12:56:35 -0700 Subject: [PATCH 12/15] matching api outputs --- bin/ssa-end-to-end-testing/modules/test_ssa_detections.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/ssa-end-to-end-testing/modules/test_ssa_detections.py b/bin/ssa-end-to-end-testing/modules/test_ssa_detections.py index 61cfdce563..84bc48b59c 100644 --- a/bin/ssa-end-to-end-testing/modules/test_ssa_detections.py +++ b/bin/ssa-end-to-end-testing/modules/test_ssa_detections.py @@ -165,8 +165,8 @@ class SSADetectionTesting: None """ deactivate_pipeline = lambda p: self.api.deactivate_pipeline(p)[0].status_code == HTTPStatus.OK - delete_pipeline = lambda p: self.api.delete_pipeline(p)[0].status_code == HTTPStatus.NO_CONTENT - delete_index = lambda p: self.api.delete_temp_index(p["id"])[0] == HTTPStatus.NO_CONTENT + delete_pipeline = lambda p: self.api.delete_pipeline(p).status_code == HTTPStatus.NO_CONTENT + delete_index = lambda p: self.api.delete_temp_index(p["id"]) == HTTPStatus.NO_CONTENT self.activated_pipelines = [p for p in self.activated_pipelines if not deactivate_pipeline(p)] self.created_pipelines = [p for p in self.created_pipelines if not delete_pipeline(p)] if len(self.activated_pipelines) > 0 or len(self.created_pipelines) > 0 or not delete_index(self.results_index): From ec2ba3450ab29230d35a99bbffd51f1587163d9a Mon Sep 17 00:00:00 2001 From: Ignacio Bermudez Corrales Date: Mon, 22 Mar 2021 14:16:21 -0700 Subject: [PATCH 13/15] fix check source/sink --- bin/ssa-end-to-end-testing/modules/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/ssa-end-to-end-testing/modules/utils.py b/bin/ssa-end-to-end-testing/modules/utils.py index 7830901892..13424416f6 100644 --- a/bin/ssa-end-to-end-testing/modules/utils.py +++ b/bin/ssa-end-to-end-testing/modules/utils.py @@ -77,7 +77,7 @@ def request_headers(header_token): def check_source_sink(spl): match_source = re.match(r"^\s*\|\s+from\s+read_ssa_enriched_events\(\s*\)", spl) - match_sink = re.match(r".*into\s+write_ssa_detected_events\(\s*\)\s*;", spl) + match_sink = re.search(r"\|\s*into\s+write_ssa_detected_events\(\s*\)\s*;", spl) return match_source and match_sink From 1b76734dd381121104ecd0bbda1c5ac9ec5aa1d6 Mon Sep 17 00:00:00 2001 From: Ignacio Bermudez Corrales Date: Mon, 22 Mar 2021 14:42:16 -0700 Subject: [PATCH 14/15] check before modifications --- bin/ssa-end-to-end-testing/modules/test_ssa_detections.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bin/ssa-end-to-end-testing/modules/test_ssa_detections.py b/bin/ssa-end-to-end-testing/modules/test_ssa_detections.py index 84bc48b59c..0747c8856b 100644 --- a/bin/ssa-end-to-end-testing/modules/test_ssa_detections.py +++ b/bin/ssa-end-to-end-testing/modules/test_ssa_detections.py @@ -96,6 +96,7 @@ class SSADetectionTesting: self.wait_time(SLEEP_TIME_CREATE_INDEX) + check_ssa_spl = check_source_sink(spl) spl = manipulate_spl(self.api.env, spl, self.results_index) assert spl is not None, "fail to manipulate spl file" @@ -112,7 +113,7 @@ class SSADetectionTesting: self.wait_time(SLEEP_TIME_ACTIVATE_PIPELINE) - if not check_source_sink(spl): + if not check_ssa_spl: msg = f"Detection test successful for {test_name}" LOGGER.warning(f"Test not completed. Detection seems deprecated, and will not send messages to SSA") self.test_results["msg"] = msg From adc4aa9908dda5695c8774ac86b45dc318188846 Mon Sep 17 00:00:00 2001 From: Ignacio Bermudez Corrales Date: Mon, 22 Mar 2021 16:09:20 -0700 Subject: [PATCH 15/15] tear down on any sort of error (not just assertions) --- bin/ssa-end-to-end-testing/modules/test_ssa_detections.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/bin/ssa-end-to-end-testing/modules/test_ssa_detections.py b/bin/ssa-end-to-end-testing/modules/test_ssa_detections.py index 0747c8856b..d9c323c189 100644 --- a/bin/ssa-end-to-end-testing/modules/test_ssa_detections.py +++ b/bin/ssa-end-to-end-testing/modules/test_ssa_detections.py @@ -189,6 +189,11 @@ class SSADetectionTesting: LOGGER.error(e.args[0]) return {"result": False, "msg": f"Detection test failure for {test_name}"} + except Exception as e: + self.ssa_detection_test_teardown() + LOGGER.error(e) + return {"result": False, + "msg": f"Detection test failure for {test_name} (perhaps SCS problems)"} # only for troubleshooting # def ssa_detection_in_dsp_with_preview_session(self, spl, source, test_name):