From 34e69271f336160694c83d3011760bae600869c7 Mon Sep 17 00:00:00 2001 From: mhaag-spl <5632822+MHaggis@users.noreply.github.com> Date: Thu, 18 Mar 2021 21:00:09 -0600 Subject: [PATCH 01/22] Updated Detect Exchange Web Shell Updated Detect Exchange Web Shell based on data --- .../endpoint/detect_exchange_web_shell.yml | 54 +++++++++++++++++++ .../detect_exchange_web_shell.test.yml | 12 +++++ 2 files changed, 66 insertions(+) create mode 100644 detections/endpoint/detect_exchange_web_shell.yml create mode 100644 tests/endpoint/detect_exchange_web_shell.test.yml diff --git a/detections/endpoint/detect_exchange_web_shell.yml b/detections/endpoint/detect_exchange_web_shell.yml new file mode 100644 index 0000000000..2fc9219582 --- /dev/null +++ b/detections/endpoint/detect_exchange_web_shell.yml @@ -0,0 +1,54 @@ +name: Detect Exchange Web Shell +id: 44465d80-810b-11eb-aa30-acde48001122 +version: 1 +date: '2021-03-09' +author: Michael Haag, Splunk +type: batch +datamodel: +- Endpoint +description: 'The following query identifies suspicious .aspx created in 3 paths identified + by Microsoft as known drop locations for Exchange exploitation related to HAFNIUM + group. Paths include: `\HttpProxy\owa\auth\`, `\inetpub\wwwroot\aspnet_client\`, + and `\HttpProxy\OAB\`. Upon triage, the suspicious .aspx file will have a randomized + name of 8 characters long. Review the file for suspect commands. Identify additional + log sources, IIS included, to review source and other potential exploitation.' +search: '| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) +as lastTime FROM datamodel=Endpoint.Filesystem where Filesystem.file_path IN ("*\\HttpProxy\\owa\\auth\\*", +"*\\inetpub\\wwwroot\\aspnet_client\\*", "*\\HttpProxy\\OAB\\*") Filesystem.file_name="*.aspx" +by _time span=1h Filesystem.file_create_time Filesystem.file_name Filesystem.file_path +| `drop_dm_object_name(Filesystem)` | join process_id, _time [| tstats `security_content_summariesonly` + count FROM datamodel=Endpoint.Processes where Processes.process_name=System by + _time span=1h Processes.process_id Processes.process_name + Processes.dest Processes.process_path Processes.process | `drop_dm_object_name(Processes)` + | fields _time process_guid dest process_name process_path process] +| table file_create_time, file_name, file_path, process_name +| `detect_exchange_web_shell_filter`' +how_to_implement: To successfully implement this search you need to be ingesting information + on process that include the name of the process responsible for the changes from + your endpoints into the `Endpoint` datamodel in the `Processes` node and `Filesystem` + node. +known_false_positives: The query is structured in a way that `action` (read, create) + is not defined. Review the results of this query, filter, and tune as necessary. + It may be necessary to generate this query specific to your endpoint product. +references: +- https://raw.githubusercontent.com/Azure/Azure-Sentinel/master/Sample%20Data/Feeds/MSTICIoCs-ExchangeServerVulnerabilitiesDisclosedMarch2021.csv +tags: + analytic_story: + - HAFNIUM Group + dataset: [] + kill_chain_phases: + - Exploitation + mitre_attack_id: + - T1505.003 + product: + - Splunk Enterprise + - Splunk Enterprise Security + - Splunk Cloud + required_fields: + - _time + - Filesystem.file_path + - Filesystem.process_id + - Filesystem.file_name + - Filesystem.file_hash + - Filesystem.user + security_domain: endpoint diff --git a/tests/endpoint/detect_exchange_web_shell.test.yml b/tests/endpoint/detect_exchange_web_shell.test.yml new file mode 100644 index 0000000000..220a75216b --- /dev/null +++ b/tests/endpoint/detect_exchange_web_shell.test.yml @@ -0,0 +1,12 @@ +name: Detect Exchange Web Shell Unit Test +tests: +- name: Detect Exchange Web Shell, ProxyLogon + file: endpoint/detect_exchange_web_shell.yml + pass_condition: '| stats count | where count > 0' + earliest_time: '-24h' + latest_time: 'now' + attack_data: + - file_name: windows-sysmon.log + data: https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1505.003/windows-sysmon_proxylogon.log + source: XmlWinEventLog:Microsoft-Windows-Sysmon/Operational + sourcetype: xmlwineventlog \ No newline at end of file From 1a15d623cf416c2557e171f7faace046ee2d394d Mon Sep 17 00:00:00 2001 From: mhaag-spl <5632822+MHaggis@users.noreply.github.com> Date: Thu, 18 Mar 2021 21:06:38 -0600 Subject: [PATCH 02/22] Update detect_exchange_web_shell.yml UID --- detections/endpoint/detect_exchange_web_shell.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/detections/endpoint/detect_exchange_web_shell.yml b/detections/endpoint/detect_exchange_web_shell.yml index 2fc9219582..73f1ac1bb2 100644 --- a/detections/endpoint/detect_exchange_web_shell.yml +++ b/detections/endpoint/detect_exchange_web_shell.yml @@ -1,6 +1,6 @@ name: Detect Exchange Web Shell -id: 44465d80-810b-11eb-aa30-acde48001122 -version: 1 +id: 8c14eeee-2af1-4a4b-bda8-228da0f4862a +version: 2 date: '2021-03-09' author: Michael Haag, Splunk type: batch From a912985629e840f4e640859435fa7fe00bf2d49e Mon Sep 17 00:00:00 2001 From: mhaag-spl <5632822+MHaggis@users.noreply.github.com> Date: Fri, 19 Mar 2021 06:27:24 -0600 Subject: [PATCH 03/22] Update detect_exchange_web_shell.yml --- .../endpoint/detect_exchange_web_shell.yml | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/detections/endpoint/detect_exchange_web_shell.yml b/detections/endpoint/detect_exchange_web_shell.yml index 73f1ac1bb2..7f64e0eda0 100644 --- a/detections/endpoint/detect_exchange_web_shell.yml +++ b/detections/endpoint/detect_exchange_web_shell.yml @@ -13,16 +13,14 @@ description: 'The following query identifies suspicious .aspx created in 3 paths name of 8 characters long. Review the file for suspect commands. Identify additional log sources, IIS included, to review source and other potential exploitation.' search: '| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) -as lastTime FROM datamodel=Endpoint.Filesystem where Filesystem.file_path IN ("*\\HttpProxy\\owa\\auth\\*", -"*\\inetpub\\wwwroot\\aspnet_client\\*", "*\\HttpProxy\\OAB\\*") Filesystem.file_name="*.aspx" -by _time span=1h Filesystem.file_create_time Filesystem.file_name Filesystem.file_path -| `drop_dm_object_name(Filesystem)` | join process_id, _time [| tstats `security_content_summariesonly` - count FROM datamodel=Endpoint.Processes where Processes.process_name=System by - _time span=1h Processes.process_id Processes.process_name - Processes.dest Processes.process_path Processes.process | `drop_dm_object_name(Processes)` + as lastTime FROM datamodel=Endpoint.Filesystem where Filesystem.file_path IN ("*\\HttpProxy\\owa\\auth\\*", + "*\\inetpub\\wwwroot\\aspnet_client\\*", "*\\HttpProxy\\OAB\\*") Filesystem.file_name="*.aspx" + by _time span=1h Filesystem.file_create_time Filesystem.file_name Filesystem.file_path + | `drop_dm_object_name(Filesystem)` | join process_id, _time [| tstats `security_content_summariesonly` count FROM datamodel=Endpoint.Processes where Processes.process_name=System by _time span=1h Processes.process_id Processes.process_name Processes.dest Processes.process_path Processes.process + | `drop_dm_object_name(Processes)` | fields _time process_guid dest process_name process_path process] -| table file_create_time, file_name, file_path, process_name -| `detect_exchange_web_shell_filter`' + | table file_create_time, file_name, file_path, process_name + | `detect_exchange_web_shell_filter`' how_to_implement: To successfully implement this search you need to be ingesting information on process that include the name of the process responsible for the changes from your endpoints into the `Endpoint` datamodel in the `Processes` node and `Filesystem` From 00965fbd34dc6e4493c55d1aed5aaccad6790bb7 Mon Sep 17 00:00:00 2001 From: mhaag-spl <5632822+MHaggis@users.noreply.github.com> Date: Fri, 19 Mar 2021 06:41:51 -0600 Subject: [PATCH 04/22] Delete detect_exchange_web_shell.yml --- .../endpoint/detect_exchange_web_shell.yml | 56 ------------------- 1 file changed, 56 deletions(-) delete mode 100644 detections/experimental/endpoint/detect_exchange_web_shell.yml diff --git a/detections/experimental/endpoint/detect_exchange_web_shell.yml b/detections/experimental/endpoint/detect_exchange_web_shell.yml deleted file mode 100644 index 1b0a5db71d..0000000000 --- a/detections/experimental/endpoint/detect_exchange_web_shell.yml +++ /dev/null @@ -1,56 +0,0 @@ -name: Detect Exchange Web Shell -id: 44465d80-810b-11eb-aa30-acde48001122 -version: 1 -date: '2021-03-09' -author: Michael Haag, Splunk -type: batch -datamodel: -- Endpoint -description: 'The following query identifies suspicious .aspx created in 3 paths identified - by Microsoft as known drop locations for Exchange exploitation related to HAFNIUM - group. Paths include: `\HttpProxy\owa\auth\`, `\inetpub\wwwroot\aspnet_client\`, - and `\HttpProxy\OAB\`. Upon triage, the suspicious .aspx file will have a randomized - name of 8 characters long. Review the file for suspect commands. Identify additional - log sources, IIS included, to review source and other potential exploitation.' -search: '| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) - as lastTime FROM datamodel=Endpoint.Filesystem where Filesystem.file_path IN ("*\\HttpProxy\\owa\\auth\\*", - "*\\inetpub\\wwwroot\\aspnet_client\\*", "*\\HttpProxy\\OAB\\*") Filesystem.file_name="*.aspx" - by _time span=1h Filesystem.process_id Filesystem.file_name Filesystem.file_path - Filesystem.file_hash Filesystem.user | `drop_dm_object_name(Filesystem)` | rename - process_id as aspx_pid | join aspx_pid, _time [| tstats `security_content_summariesonly` - count FROM datamodel=Endpoint.Processes where Processes.process_name=w3wp.exe by - _time span=1h Processes.parent_process_id Processes.process_id Processes.process_name - Processes.dest Processes.process_path Processes.process | `drop_dm_object_name(Processes)` - | rename parent_process_id as aspx_pid | fields _time aspx_pid process_id dest process_name - process_path process] | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)` - | table firstTime, lastTime, aspx_pid, process_id, user, dest, file_name, file_path, - process_name, process, process_path, file_hash | `detect_exchange_web_shell_filter`' -how_to_implement: To successfully implement this search you need to be ingesting information - on process that include the name of the process responsible for the changes from - your endpoints into the `Endpoint` datamodel in the `Processes` node and `Filesystem` - node. -known_false_positives: The query is structured in a way that `action` (read, create) - is not defined. Review the results of this query, filter, and tune as necessary. - It may be necessary to generate this query specific to your endpoint product. -references: -- https://raw.githubusercontent.com/Azure/Azure-Sentinel/master/Sample%20Data/Feeds/MSTICIoCs-ExchangeServerVulnerabilitiesDisclosedMarch2021.csv -tags: - analytic_story: - - HAFNIUM Group - dataset: [] - kill_chain_phases: - - Exploitation - mitre_attack_id: - - T1505.003 - product: - - Splunk Enterprise - - Splunk Enterprise Security - - Splunk Cloud - required_fields: - - _time - - Filesystem.file_path - - Filesystem.process_id - - Filesystem.file_name - - Filesystem.file_hash - - Filesystem.user - security_domain: endpoint From 63aad3ee438db44d585af29dd0f908870967e83c Mon Sep 17 00:00:00 2001 From: divious1 Date: Fri, 19 Mar 2021 13:58:48 -0400 Subject: [PATCH 05/22] deprecating detection --- .../ssa___rare_parent_process_relationship_lolbas.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename detections/{endpoint => deprecated}/ssa___rare_parent_process_relationship_lolbas.yml (98%) diff --git a/detections/endpoint/ssa___rare_parent_process_relationship_lolbas.yml b/detections/deprecated/ssa___rare_parent_process_relationship_lolbas.yml similarity index 98% rename from detections/endpoint/ssa___rare_parent_process_relationship_lolbas.yml rename to detections/deprecated/ssa___rare_parent_process_relationship_lolbas.yml index 016e7518ad..1fa1d784df 100644 --- a/detections/endpoint/ssa___rare_parent_process_relationship_lolbas.yml +++ b/detections/deprecated/ssa___rare_parent_process_relationship_lolbas.yml @@ -47,7 +47,7 @@ search: '| from read_ssa_enriched_events() | eval timestamp=parse_long(ucast(map OR process_name="pcalua.exe" OR process_name="cmdkey.exe" OR process_name="msconfig.exe") | eval start_time = timestamp, end_time = timestamp, entities = mvappend(dest_device_id, - dest_user_id), body = "TBD" | into write_ssa_detected_events();' + dest_user_id), body = "TBD"' how_to_implement: Collect endpoint data such as sysmon or 4688 events. known_false_positives: 'Some custom tools used by admins could be used rarely to launch remotely applications. This might trigger false positives at the beginning when From 066b81e6ef797f73970d20d19bc1fc6787f71ee3 Mon Sep 17 00:00:00 2001 From: divious1 Date: Fri, 19 Mar 2021 13:59:56 -0400 Subject: [PATCH 06/22] moved out of deprecated --- .../ssa___rare_parent_process_relationship_lolbas.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename detections/{deprecated => endpoint}/ssa___rare_parent_process_relationship_lolbas.yml (100%) diff --git a/detections/deprecated/ssa___rare_parent_process_relationship_lolbas.yml b/detections/endpoint/ssa___rare_parent_process_relationship_lolbas.yml similarity index 100% rename from detections/deprecated/ssa___rare_parent_process_relationship_lolbas.yml rename to detections/endpoint/ssa___rare_parent_process_relationship_lolbas.yml From 6bf8b82877eb2bcbb3812f671dc6272d4ef8a355 Mon Sep 17 00:00:00 2001 From: divious1 Date: Fri, 19 Mar 2021 14:08:26 -0400 Subject: [PATCH 07/22] added write to null --- .../endpoint/ssa___rare_parent_process_relationship_lolbas.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/detections/endpoint/ssa___rare_parent_process_relationship_lolbas.yml b/detections/endpoint/ssa___rare_parent_process_relationship_lolbas.yml index 1fa1d784df..2168b8e1ef 100644 --- a/detections/endpoint/ssa___rare_parent_process_relationship_lolbas.yml +++ b/detections/endpoint/ssa___rare_parent_process_relationship_lolbas.yml @@ -47,7 +47,7 @@ search: '| from read_ssa_enriched_events() | eval timestamp=parse_long(ucast(map OR process_name="pcalua.exe" OR process_name="cmdkey.exe" OR process_name="msconfig.exe") | eval start_time = timestamp, end_time = timestamp, entities = mvappend(dest_device_id, - dest_user_id), body = "TBD"' + dest_user_id), body = "TBD" | into write_null();' how_to_implement: Collect endpoint data such as sysmon or 4688 events. known_false_positives: 'Some custom tools used by admins could be used rarely to launch remotely applications. This might trigger false positives at the beginning when From 0ec2001838689af50d00ae0a82586d59cb02ca15 Mon Sep 17 00:00:00 2001 From: Ignacio Bermudez Corrales Date: Fri, 19 Mar 2021 13:52:34 -0700 Subject: [PATCH 08/22] Renaming test .yaml -> .yml | No events should come out of the detection --- ...l => ssa___rare_parent_process_relationship_lolbas.test.yml} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename tests/endpoint/{ssa___rare_parent_process_relationship_lolbas.test.yaml => ssa___rare_parent_process_relationship_lolbas.test.yml} (93%) diff --git a/tests/endpoint/ssa___rare_parent_process_relationship_lolbas.test.yaml b/tests/endpoint/ssa___rare_parent_process_relationship_lolbas.test.yml similarity index 93% rename from tests/endpoint/ssa___rare_parent_process_relationship_lolbas.test.yaml rename to tests/endpoint/ssa___rare_parent_process_relationship_lolbas.test.yml index a44cdc41f0..7cc3da7342 100644 --- a/tests/endpoint/ssa___rare_parent_process_relationship_lolbas.test.yaml +++ b/tests/endpoint/ssa___rare_parent_process_relationship_lolbas.test.yml @@ -2,7 +2,7 @@ name: Rare Parent/Child Process Relationship - SSA Unit Test tests: - name: Access LSASS Memory for Dump Creation file: endpoint/ssa___rare_parent_process_relationship_lolbas.yml - pass_condition: '@count_gt(0)' + pass_condition: '@count_eq(0)' description: Test detection looking for LOLBAS processes spawned by other processes that are rarely seen together attack_data: - file_name: T1059.all.labeled.lolbas-test.json From 593ae74fdf3d654fe6513ff124c37aa571dcb4b1 Mon Sep 17 00:00:00 2001 From: Ignacio Bermudez Corrales Date: Fri, 19 Mar 2021 14:51:53 -0700 Subject: [PATCH 09/22] Refactoring template replacement --- bin/ssa-end-to-end-testing/modules/utils.py | 24 +++++++++++---------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/bin/ssa-end-to-end-testing/modules/utils.py b/bin/ssa-end-to-end-testing/modules/utils.py index b6c5cc7ede..8548d0be47 100644 --- a/bin/ssa-end-to-end-testing/modules/utils.py +++ b/bin/ssa-end-to-end-testing/modules/utils.py @@ -76,13 +76,19 @@ def request_headers(header_token): def manipulate_spl(env, spl, results_index): - spl = replace_ssa_macros(env, spl) + # Obtain the SSA source + pulsar_source_connection_id, pulsar_source_topic = return_macros(env) + source = READ_SSA_ENRICHED_EVENTS_EXPANDED\ + .replace("__PULSAR_SOURCE_CONNECTION_ID__", pulsar_source_connection_id)\ + .replace("__PULSAR_SOURCE_TOPIC__", pulsar_source_topic) + # Obtain the test sink + sink = ";" if results_index is not None: - # When an index is defined for a test, it writes the output of this pipeline to this index. - # original_pipeline; => original_pipeline | into index("module", "index"); module = results_index["module"] index = results_index["name"] - spl = spl[:spl.rindex(";")] + f" | into index(\"{module}\", \"{index}\");" + sink = f"| into index(\"{module}\", \"{index}\");" + # Replace spl template with its `source` and `sink` + spl = replace_ssa_macros(source, sink, spl) LOGGER.info(f"spl: {spl}") return spl @@ -93,13 +99,9 @@ def read_spl(file_path, file_name): return spl -def replace_ssa_macros(env, spl): - pulsar_source_connection_id, pulsar_source_topic = return_macros(env) - macro_expanded = READ_SSA_ENRICHED_EVENTS_EXPANDED.replace("__PULSAR_SOURCE_CONNECTION_ID__", pulsar_source_connection_id) - macro_expanded = macro_expanded.replace("__PULSAR_SOURCE_TOPIC__", pulsar_source_topic) - spl = spl.replace(READ_SSA_ENRICHED_EVENTS, macro_expanded) - spl = spl.replace(WRITE_SSA_DETECTED_EVENTS, ";") - #spl = spl.replace("\n", " ") +def replace_ssa_macros(source, sink, spl): + spl = spl.replace(READ_SSA_ENRICHED_EVENTS, source) + spl = spl.replace(WRITE_SSA_DETECTED_EVENTS, sink) return spl From 548f768ff34b7484d757878dbb1e3ecce4cfcd1c Mon Sep 17 00:00:00 2001 From: Ignacio Bermudez Corrales Date: Fri, 19 Mar 2021 15:10:06 -0700 Subject: [PATCH 10/22] refactoring base tests --- bin/ssa-end-to-end-testing/modules/spl/detection.spl | 3 ++- bin/ssa-end-to-end-testing/modules/spl/detection2.spl | 3 ++- bin/ssa-end-to-end-testing/modules/spl/firehose.spl | 2 +- bin/ssa-end-to-end-testing/modules/spl/troubleshoot.spl | 2 +- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/bin/ssa-end-to-end-testing/modules/spl/detection.spl b/bin/ssa-end-to-end-testing/modules/spl/detection.spl index c4cf977814..1e2c938227 100644 --- a/bin/ssa-end-to-end-testing/modules/spl/detection.spl +++ b/bin/ssa-end-to-end-testing/modules/spl/detection.spl @@ -17,4 +17,5 @@ | eval start_time = timestamp, end_time = timestamp, entities = mvappend(dest_device_id, dest_user_id), -body = "TBD"; \ No newline at end of file +body = "TBD" +| into write_ssa_detected_events(); \ No newline at end of file diff --git a/bin/ssa-end-to-end-testing/modules/spl/detection2.spl b/bin/ssa-end-to-end-testing/modules/spl/detection2.spl index d035fc7f36..73cabaaafc 100644 --- a/bin/ssa-end-to-end-testing/modules/spl/detection2.spl +++ b/bin/ssa-end-to-end-testing/modules/spl/detection2.spl @@ -17,4 +17,5 @@ | eval start_time = timestamp, end_time = timestamp, entities = mvappend(dest_device_id, dest_user_id), -body = "TBD"; +body = "TBD" +| into write_ssa_detected_events(); diff --git a/bin/ssa-end-to-end-testing/modules/spl/firehose.spl b/bin/ssa-end-to-end-testing/modules/spl/firehose.spl index be0790a827..c8bdf3a602 100644 --- a/bin/ssa-end-to-end-testing/modules/spl/firehose.spl +++ b/bin/ssa-end-to-end-testing/modules/spl/firehose.spl @@ -1 +1 @@ -| from read_splunk_firehose(); \ No newline at end of file + | from read_splunk_firehose(); \ No newline at end of file diff --git a/bin/ssa-end-to-end-testing/modules/spl/troubleshoot.spl b/bin/ssa-end-to-end-testing/modules/spl/troubleshoot.spl index c093070090..d374be63ef 100644 --- a/bin/ssa-end-to-end-testing/modules/spl/troubleshoot.spl +++ b/bin/ssa-end-to-end-testing/modules/spl/troubleshoot.spl @@ -1 +1 @@ -| from read_ssa_enriched_events(); \ No newline at end of file +| from read_ssa_enriched_events() | into write_ssa_detected_events(); \ No newline at end of file From 1eb44354e82d00693d10943caa55eaa6d4c90110 Mon Sep 17 00:00:00 2001 From: Ignacio Bermudez Corrales Date: Fri, 19 Mar 2021 15:54:47 -0700 Subject: [PATCH 11/22] renaming back test for ssa detection --- ...ml => ssa___rare_parent_process_relationship_lolbas.test.yaml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/endpoint/{ssa___rare_parent_process_relationship_lolbas.test.yml => ssa___rare_parent_process_relationship_lolbas.test.yaml} (100%) diff --git a/tests/endpoint/ssa___rare_parent_process_relationship_lolbas.test.yml b/tests/endpoint/ssa___rare_parent_process_relationship_lolbas.test.yaml similarity index 100% rename from tests/endpoint/ssa___rare_parent_process_relationship_lolbas.test.yml rename to tests/endpoint/ssa___rare_parent_process_relationship_lolbas.test.yaml From d47e5c395e5f1177f91a53d05e6a8a410caa3576 Mon Sep 17 00:00:00 2001 From: Ignacio Bermudez Corrales Date: Fri, 19 Mar 2021 16:45:05 -0700 Subject: [PATCH 12/22] Dealing with yaml and yml for testing --- bin/ssa-end-to-end-testing/modules/github_service.py | 7 ++++--- bin/ssa-end-to-end-testing/run_ssa_smoketest.py | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/bin/ssa-end-to-end-testing/modules/github_service.py b/bin/ssa-end-to-end-testing/modules/github_service.py index de159e476c..82ed3b8c15 100644 --- a/bin/ssa-end-to-end-testing/modules/github_service.py +++ b/bin/ssa-end-to-end-testing/modules/github_service.py @@ -35,8 +35,6 @@ class GithubService: changed_ssa_test_files = [] - #tests = self.read_security_content_test_files() - for file_path in changed_files: # added or changed test files if file_path.startswith('tests'): @@ -47,7 +45,10 @@ class GithubService: # changed detections if file_path.startswith('detections'): if os.path.basename(file_path).startswith('ssa'): - file_path_new = os.path.splitext(file_path)[0].replace('detections', 'tests') + '.test.yml' + file_path_base = os.path.splitext(file_path)[0].replace('detections', 'tests') + '.test' + file_path_new = file_path_base + '.yml' + if not os.path.exists(file_path_new): + file_path_new = file_path_base + '.yaml' if file_path_new not in changed_ssa_test_files: changed_ssa_test_files.append(file_path_new) diff --git a/bin/ssa-end-to-end-testing/run_ssa_smoketest.py b/bin/ssa-end-to-end-testing/run_ssa_smoketest.py index aeb006f38b..6e69820fba 100644 --- a/bin/ssa-end-to-end-testing/run_ssa_smoketest.py +++ b/bin/ssa-end-to-end-testing/run_ssa_smoketest.py @@ -38,7 +38,7 @@ def main(args): for test_file in test_files_ssa: LOGGER.info(test_file) - if len(test_files_ssa)==0: + if len(test_files_ssa) == 0: LOGGER.info('Nothing to test for SSA smoke test.') sys.exit(0) From 5f1b737acec86755ee52e29711cf4278d6e05d63 Mon Sep 17 00:00:00 2001 From: mhaag-spl <5632822+MHaggis@users.noreply.github.com> Date: Mon, 22 Mar 2021 06:27:06 -0600 Subject: [PATCH 13/22] Update detect_exchange_web_shell.yml --- .../endpoint/detect_exchange_web_shell.yml | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/detections/endpoint/detect_exchange_web_shell.yml b/detections/endpoint/detect_exchange_web_shell.yml index 7f64e0eda0..e29b962dfd 100644 --- a/detections/endpoint/detect_exchange_web_shell.yml +++ b/detections/endpoint/detect_exchange_web_shell.yml @@ -9,17 +9,13 @@ datamodel: description: 'The following query identifies suspicious .aspx created in 3 paths identified by Microsoft as known drop locations for Exchange exploitation related to HAFNIUM group. Paths include: `\HttpProxy\owa\auth\`, `\inetpub\wwwroot\aspnet_client\`, - and `\HttpProxy\OAB\`. Upon triage, the suspicious .aspx file will have a randomized - name of 8 characters long. Review the file for suspect commands. Identify additional + and `\HttpProxy\OAB\`. Upon triage, the suspicious .aspx file will likely look obvious on the surface. inspect the contents for script code inside. Identify additional log sources, IIS included, to review source and other potential exploitation.' -search: '| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) - as lastTime FROM datamodel=Endpoint.Filesystem where Filesystem.file_path IN ("*\\HttpProxy\\owa\\auth\\*", - "*\\inetpub\\wwwroot\\aspnet_client\\*", "*\\HttpProxy\\OAB\\*") Filesystem.file_name="*.aspx" - by _time span=1h Filesystem.file_create_time Filesystem.file_name Filesystem.file_path - | `drop_dm_object_name(Filesystem)` | join process_id, _time [| tstats `security_content_summariesonly` count FROM datamodel=Endpoint.Processes where Processes.process_name=System by _time span=1h Processes.process_id Processes.process_name Processes.dest Processes.process_path Processes.process - | `drop_dm_object_name(Processes)` - | fields _time process_guid dest process_name process_path process] - | table file_create_time, file_name, file_path, process_name +search: '| tstats `security_content_summariesonly` count FROM datamodel=Endpoint.Processes where Processes.process_name=System by _time span=1h Processes.process_id Processes.process_name Processes.dest | `drop_dm_object_name(Processes)` | join process_guid, _time [| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) as lastTime FROM datamodel=Endpoint.Filesystem where Filesystem.file_path IN ("*\\HttpProxy\\owa\\auth\\*", + "*\\inetpub\\wwwroot\\aspnet_client\\*", "*\\HttpProxy\\OAB\\*") Filesystem.file_name="*.aspx" by _time span=1h Filesystem.dest Filesystem.file_create_time Filesystem.file_name Filesystem.file_path + | `drop_dm_object_name(Filesystem)` | fields _time dest file_create_time file_name file_path process_name process_path process] + | dedup file_create_time + | table dest file_create_time, file_name, file_path, process_name | `detect_exchange_web_shell_filter`' how_to_implement: To successfully implement this search you need to be ingesting information on process that include the name of the process responsible for the changes from From 3657fc8ea76730b3afaf2e37ece3bce3490da70b Mon Sep 17 00:00:00 2001 From: root Date: Mon, 22 Mar 2021 12:54:27 +0000 Subject: [PATCH 14/22] Added detection testing service results inDetect Exchange Web Shell, ProxyLogon --- .../endpoint/detect_exchange_web_shell.yml | 23 ++++++++++++------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/detections/endpoint/detect_exchange_web_shell.yml b/detections/endpoint/detect_exchange_web_shell.yml index e29b962dfd..82570624ef 100644 --- a/detections/endpoint/detect_exchange_web_shell.yml +++ b/detections/endpoint/detect_exchange_web_shell.yml @@ -9,14 +9,19 @@ datamodel: description: 'The following query identifies suspicious .aspx created in 3 paths identified by Microsoft as known drop locations for Exchange exploitation related to HAFNIUM group. Paths include: `\HttpProxy\owa\auth\`, `\inetpub\wwwroot\aspnet_client\`, - and `\HttpProxy\OAB\`. Upon triage, the suspicious .aspx file will likely look obvious on the surface. inspect the contents for script code inside. Identify additional + and `\HttpProxy\OAB\`. Upon triage, the suspicious .aspx file will likely look obvious + on the surface. inspect the contents for script code inside. Identify additional log sources, IIS included, to review source and other potential exploitation.' -search: '| tstats `security_content_summariesonly` count FROM datamodel=Endpoint.Processes where Processes.process_name=System by _time span=1h Processes.process_id Processes.process_name Processes.dest | `drop_dm_object_name(Processes)` | join process_guid, _time [| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) as lastTime FROM datamodel=Endpoint.Filesystem where Filesystem.file_path IN ("*\\HttpProxy\\owa\\auth\\*", - "*\\inetpub\\wwwroot\\aspnet_client\\*", "*\\HttpProxy\\OAB\\*") Filesystem.file_name="*.aspx" by _time span=1h Filesystem.dest Filesystem.file_create_time Filesystem.file_name Filesystem.file_path - | `drop_dm_object_name(Filesystem)` | fields _time dest file_create_time file_name file_path process_name process_path process] - | dedup file_create_time - | table dest file_create_time, file_name, file_path, process_name - | `detect_exchange_web_shell_filter`' +search: '| tstats `security_content_summariesonly` count FROM datamodel=Endpoint.Processes + where Processes.process_name=System by _time span=1h Processes.process_id Processes.process_name + Processes.dest | `drop_dm_object_name(Processes)` | join process_guid, _time [| + tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) + as lastTime FROM datamodel=Endpoint.Filesystem where Filesystem.file_path IN ("*\\HttpProxy\\owa\\auth\\*", + "*\\inetpub\\wwwroot\\aspnet_client\\*", "*\\HttpProxy\\OAB\\*") Filesystem.file_name="*.aspx" + by _time span=1h Filesystem.dest Filesystem.file_create_time Filesystem.file_name + Filesystem.file_path | `drop_dm_object_name(Filesystem)` | fields _time dest file_create_time + file_name file_path process_name process_path process] | dedup file_create_time + | table dest file_create_time, file_name, file_path, process_name | `detect_exchange_web_shell_filter`' how_to_implement: To successfully implement this search you need to be ingesting information on process that include the name of the process responsible for the changes from your endpoints into the `Endpoint` datamodel in the `Processes` node and `Filesystem` @@ -29,7 +34,8 @@ references: tags: analytic_story: - HAFNIUM Group - dataset: [] + dataset: + - https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1505.003/windows-sysmon_proxylogon.log kill_chain_phases: - Exploitation mitre_attack_id: @@ -46,3 +52,4 @@ tags: - Filesystem.file_hash - Filesystem.user security_domain: endpoint + automated_detection_testing: passed From 17cbdf590cc9863f7f33e15582a4721c60a0375a Mon Sep 17 00:00:00 2001 From: mhaag-spl <5632822+MHaggis@users.noreply.github.com> Date: Mon, 22 Mar 2021 09:27:09 -0600 Subject: [PATCH 15/22] Update detect_exchange_web_shell.yml --- detections/endpoint/detect_exchange_web_shell.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/detections/endpoint/detect_exchange_web_shell.yml b/detections/endpoint/detect_exchange_web_shell.yml index 82570624ef..011bfc3c34 100644 --- a/detections/endpoint/detect_exchange_web_shell.yml +++ b/detections/endpoint/detect_exchange_web_shell.yml @@ -2,7 +2,7 @@ name: Detect Exchange Web Shell id: 8c14eeee-2af1-4a4b-bda8-228da0f4862a version: 2 date: '2021-03-09' -author: Michael Haag, Splunk +author: Michael Haag, Shannon Davis, Splunk type: batch datamodel: - Endpoint From 13ab17f2d18f42f55eca800ad209e15994f60c3a Mon Sep 17 00:00:00 2001 From: Ignacio Bermudez Corrales Date: Mon, 22 Mar 2021 11:30:53 -0700 Subject: [PATCH 16/22] refactor test workflow with assertions and catch. Teardown as well --- .../modules/github_service.py | 3 - .../modules/test_ssa_detections.py | 116 +++++++++--------- bin/ssa-end-to-end-testing/modules/utils.py | 6 + 3 files changed, 64 insertions(+), 61 deletions(-) diff --git a/bin/ssa-end-to-end-testing/modules/github_service.py b/bin/ssa-end-to-end-testing/modules/github_service.py index 82ed3b8c15..09e13de335 100644 --- a/bin/ssa-end-to-end-testing/modules/github_service.py +++ b/bin/ssa-end-to-end-testing/modules/github_service.py @@ -2,8 +2,6 @@ import git import os import logging -from os import path -import sys # Logger @@ -25,7 +23,6 @@ class GithubService: repo_obj = git.Repo.clone_from(url, project, branch=branch) return repo_obj - def get_changed_test_files_ssa(self): branch1 = self.security_content_branch branch2 = 'develop' diff --git a/bin/ssa-end-to-end-testing/modules/test_ssa_detections.py b/bin/ssa-end-to-end-testing/modules/test_ssa_detections.py index 126045831a..3b506247bb 100644 --- a/bin/ssa-end-to-end-testing/modules/test_ssa_detections.py +++ b/bin/ssa-end-to-end-testing/modules/test_ssa_detections.py @@ -1,4 +1,3 @@ - import logging import os import time @@ -6,8 +5,7 @@ import sys from http import HTTPStatus from modules.streams_service_api_helper import DSPApi -from modules.utils import manipulate_spl, read_spl, read_data - +from modules.utils import check_source_sink, manipulate_spl, read_spl, read_data # Logger logging.basicConfig(level=os.environ.get("LOGLEVEL", "INFO")) @@ -22,6 +20,7 @@ MAX_EXECUTION_TIME_LIMIT = 600 # per detection test TEST_DATASET = 'windows-security_small.txt' + class SSADetectionTesting: def __init__(self, env, tenant, header_token): @@ -45,7 +44,7 @@ class SSADetectionTesting: ] test_results = [] - for i in range(0,len(test_spls)): + for i in range(0, len(test_spls)): self.max_execution_time = MAX_EXECUTION_TIME_LIMIT test_result = self.ssa_detection_test(read_spl(file_path_spl, test_spls[i]), file_path_data, test_names[i]) test_results.append(test_result.copy()) @@ -62,17 +61,16 @@ class SSADetectionTesting: return passed - def test_ssa_detections(self, test_obj): LOGGER.info('Test SSA Detection: ' + test_obj["detection_obj"]["name"]) self.max_execution_time = MAX_EXECUTION_TIME_LIMIT file_path_attack_data = os.path.join(os.path.dirname(__file__), "../", test_obj["attack_data_file_path"]) - test_results = self.ssa_detection_test(test_obj["detection_obj"]["search"], file_path_attack_data, "SSA Smoke Test " + test_obj["test_obj"]["name"]) + test_results = self.ssa_detection_test(test_obj["detection_obj"]["search"], file_path_attack_data, + "SSA Smoke Test " + test_obj["test_obj"]["name"]) return test_results - ## Helper Functions ## def update_execution_time(self, time_frame): @@ -86,30 +84,12 @@ class SSADetectionTesting: time.sleep(time_in_s) return self.update_execution_time(time_in_s) - def check_result(self, condition, error_message): - try: - assert condition - except: - self.execution_passed = False - LOGGER.error(error_message) - - def write_test_results(self, test_name): - if not self.execution_passed: - msg = f"Detection test failed for {test_name}" - LOGGER.error(msg) - self.test_results["msg"] = msg - self.test_results["result"] = False - else: - msg = f"Detection test successful for {test_name}" - LOGGER.info(msg) - self.test_results["msg"] = msg - - def ssa_detection_test_init(self): self.test_results["result"] = True self.test_results["msg"] = "" - self.results_index = self.api.create_temp_index("mc") - + self.result_indexes = [self.api.create_temp_index("mc")] + self.created_pipelines = [] + self.activated_pipelines = [] def ssa_detection_test_main(self, spl, source, test_name): self.execution_passed = True @@ -117,27 +97,31 @@ class SSADetectionTesting: self.wait_time(SLEEP_TIME_CREATE_INDEX) spl = manipulate_spl(self.api.env, spl, self.results_index) - self.check_result(spl is not None, "fail to manipulate spl file") + assert spl is not None, "fail to manipulate spl file" pipeline_id = self.api.create_pipeline_from_spl(spl) - self.check_result(pipeline_id is not None, "failed to create a pipeline") + assert pipeline_id is not None, "failed to create a pipeline" _pipeline_status = self.api.pipeline_status(pipeline_id) - self.check_result(_pipeline_status=="CREATED", f"Current status of pipeline {pipeline_id} should be CREATED") + assert _pipeline_status == "CREATED", f"Current status of pipeline {pipeline_id} should be CREATED" + self.created_pipelines.append(pipeline_id) response_body = self.api.activate_pipeline(pipeline_id) - self.check_result(response_body.get("activated")==pipeline_id, f"pipeline {pipeline_id} should be successfully activate.") + assert response_body.get("activated") == pipeline_id, f"pipeline {pipeline_id} should be successfully activate." + self.activated_pipelines.append(pipeline_id) self.wait_time(SLEEP_TIME_ACTIVATE_PIPELINE) + if not check_source_sink(spl): + msg = f"Detection test successful for {test_name}" + LOGGER.warning(f"Test not completed. Detection seems deprecated, and will not send messages to SSA") + self.test_results["msg"] = msg + return self.test_results + data = read_data(source) LOGGER.info("Sending (%d) events" % (len(data))) - if len(data) == 0: - LOGGER.warning("No events to send, skip to next test.") - self.execution_passed = False - self.write_test_results(test_name) - return self.test_results + assert len(data) > 0, "No events to send, skip to next test." for d in data: response_body = self.api.ingest_data(d) @@ -151,8 +135,8 @@ class SSADetectionTesting: max_execution_time_reached = self.wait_time(WAIT_CYCLE) query = f"from indexes('{self.results_index['name']}') | search source!=\"Search Catalog\" " sid = self.api.submit_search_job(self.results_index['module'], query) - self.check_result(sid is not None, f"Failed to create a Search Job") - + assert sid is not None, f"Failed to create a Search Job" + job_finished = False while not job_finished: self.wait_time(WAIT_CYCLE) @@ -162,33 +146,49 @@ class SSADetectionTesting: results = self.api.get_search_job_results(sid) search_results = (len(results) > 0) if not search_results: - LOGGER.info(f"Search didn't return any results. Retrying in {WAIT_CYCLE}s, max execution time left {self.max_execution_time}s") - - self.check_result(len(results) > 0, "Search job didn't return any results") + LOGGER.info( + f"Search didn't return any results. Retrying in {WAIT_CYCLE}s, max execution time left {self.max_execution_time}s") - response, response_body = self.api.deactivate_pipeline(pipeline_id) - self.check_result(response.status_code == HTTPStatus.OK, f"The pipeline {pipeline_id} fails to deactivated.") + assert len(results) > 0, "Search job didn't return any results" - response = self.api.delete_pipeline(pipeline_id) - self.check_result(response.status_code == HTTPStatus.NO_CONTENT, f"Fail to delete pipeline {pipeline_id}.") - - self.write_test_results(test_name) + msg = f"Detection test successful for {test_name}" + LOGGER.info(msg) + self.test_results["msg"] = msg return self.test_results - def ssa_detection_test_teardown(self): - pass - self.api.delete_temp_index(self.results_index["id"]) - + """ + Deactivate and deletes pipelines, deletes results indexes, + and when it fails it shows pipelines and result indexes that were not removed. + :return: + None + """ + deactivate_pipeline = lambda p: self.api.deactivate_pipeline(p).status_code == HTTPStatus.OK + delete_pipeline = lambda p: self.api.delete_pipeline(p).status_code == HTTPStatus.NO_CONTENT + delete_index = lambda p: self.api.delete_temp_index(self.results_index["id"]) == HTTPStatus.NO_CONTENT + self.activated_pipelines = [p for p in self.activated_pipelines if not deactivate_pipeline(p)] + self.created_pipelines = [p for p in self.created_pipelines if not delete_pipeline(p)] + self.result_indexes = [p for p in self.result_indexes if not delete_index] + if len(self.activated_pipelines) > 0 or len(self.created_pipelines) > 0 or len(self.result_indexes) > 0: + LOGGER.warning("Not all SCS resources fred up") + LOGGER.info(f"Created Pipelines: {','.join(self.created_pipelines)}") + LOGGER.info(f"Active Pipelines: {','.join(self.activated_pipelines)}") + LOGGER.info(f"Result Indexes: {','.join(self.result_indexes)}") + else: + LOGGER.info("Testing successfully cleaned up") def ssa_detection_test(self, spl, source, test_name): self.ssa_detection_test_init() - test_result = self.ssa_detection_test_main(spl, source, test_name) - self.ssa_detection_test_teardown() - return test_result - - + try: + test_result = self.ssa_detection_test_main(spl, source, test_name) + self.ssa_detection_test_teardown() + return test_result + except AssertionError as e: + self.ssa_detection_test_teardown() + LOGGER.error(e.args[0]) + return {"result": False, + "msg": f"Detection test failure for {test_name}"} # only for troubleshooting # def ssa_detection_in_dsp_with_preview_session(self, spl, source, test_name): @@ -213,4 +213,4 @@ class SSADetectionTesting: # response = self.api.stop_preview_session(preview_id) - # self.write_test_results(test_name) \ No newline at end of file + # self.write_test_results(test_name) diff --git a/bin/ssa-end-to-end-testing/modules/utils.py b/bin/ssa-end-to-end-testing/modules/utils.py index 8548d0be47..7830901892 100644 --- a/bin/ssa-end-to-end-testing/modules/utils.py +++ b/bin/ssa-end-to-end-testing/modules/utils.py @@ -75,6 +75,12 @@ def request_headers(header_token): return headers +def check_source_sink(spl): + match_source = re.match(r"^\s*\|\s+from\s+read_ssa_enriched_events\(\s*\)", spl) + match_sink = re.match(r".*into\s+write_ssa_detected_events\(\s*\)\s*;", spl) + return match_source and match_sink + + def manipulate_spl(env, spl, results_index): # Obtain the SSA source pulsar_source_connection_id, pulsar_source_topic = return_macros(env) From d6d02eb6fcfdbfab1988fff8e1bc88683c621d08 Mon Sep 17 00:00:00 2001 From: Ignacio Bermudez Corrales Date: Mon, 22 Mar 2021 11:56:08 -0700 Subject: [PATCH 17/22] removing index fixed --- .../modules/test_ssa_detections.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/bin/ssa-end-to-end-testing/modules/test_ssa_detections.py b/bin/ssa-end-to-end-testing/modules/test_ssa_detections.py index 3b506247bb..fa6270c46f 100644 --- a/bin/ssa-end-to-end-testing/modules/test_ssa_detections.py +++ b/bin/ssa-end-to-end-testing/modules/test_ssa_detections.py @@ -87,7 +87,7 @@ class SSADetectionTesting: def ssa_detection_test_init(self): self.test_results["result"] = True self.test_results["msg"] = "" - self.result_indexes = [self.api.create_temp_index("mc")] + self.results_index = self.api.create_temp_index("mc") self.created_pipelines = [] self.activated_pipelines = [] @@ -166,15 +166,14 @@ class SSADetectionTesting: """ deactivate_pipeline = lambda p: self.api.deactivate_pipeline(p).status_code == HTTPStatus.OK delete_pipeline = lambda p: self.api.delete_pipeline(p).status_code == HTTPStatus.NO_CONTENT - delete_index = lambda p: self.api.delete_temp_index(self.results_index["id"]) == HTTPStatus.NO_CONTENT + delete_index = lambda p: self.api.delete_temp_index(p["id"]) == HTTPStatus.NO_CONTENT self.activated_pipelines = [p for p in self.activated_pipelines if not deactivate_pipeline(p)] self.created_pipelines = [p for p in self.created_pipelines if not delete_pipeline(p)] - self.result_indexes = [p for p in self.result_indexes if not delete_index] - if len(self.activated_pipelines) > 0 or len(self.created_pipelines) > 0 or len(self.result_indexes) > 0: + if len(self.activated_pipelines) > 0 or len(self.created_pipelines) > 0 or not delete_index(self.results_index): LOGGER.warning("Not all SCS resources fred up") LOGGER.info(f"Created Pipelines: {','.join(self.created_pipelines)}") LOGGER.info(f"Active Pipelines: {','.join(self.activated_pipelines)}") - LOGGER.info(f"Result Indexes: {','.join(self.result_indexes)}") + LOGGER.info(f"Result Indexes: {self.results_index}") else: LOGGER.info("Testing successfully cleaned up") From 61d2dbb0e84d63b6f643982372bfa92401189571 Mon Sep 17 00:00:00 2001 From: Ignacio Bermudez Corrales Date: Mon, 22 Mar 2021 12:44:49 -0700 Subject: [PATCH 18/22] api queries return tuples --- bin/ssa-end-to-end-testing/modules/test_ssa_detections.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bin/ssa-end-to-end-testing/modules/test_ssa_detections.py b/bin/ssa-end-to-end-testing/modules/test_ssa_detections.py index fa6270c46f..61cfdce563 100644 --- a/bin/ssa-end-to-end-testing/modules/test_ssa_detections.py +++ b/bin/ssa-end-to-end-testing/modules/test_ssa_detections.py @@ -164,9 +164,9 @@ class SSADetectionTesting: :return: None """ - deactivate_pipeline = lambda p: self.api.deactivate_pipeline(p).status_code == HTTPStatus.OK - delete_pipeline = lambda p: self.api.delete_pipeline(p).status_code == HTTPStatus.NO_CONTENT - delete_index = lambda p: self.api.delete_temp_index(p["id"]) == HTTPStatus.NO_CONTENT + deactivate_pipeline = lambda p: self.api.deactivate_pipeline(p)[0].status_code == HTTPStatus.OK + delete_pipeline = lambda p: self.api.delete_pipeline(p)[0].status_code == HTTPStatus.NO_CONTENT + delete_index = lambda p: self.api.delete_temp_index(p["id"])[0] == HTTPStatus.NO_CONTENT self.activated_pipelines = [p for p in self.activated_pipelines if not deactivate_pipeline(p)] self.created_pipelines = [p for p in self.created_pipelines if not delete_pipeline(p)] if len(self.activated_pipelines) > 0 or len(self.created_pipelines) > 0 or not delete_index(self.results_index): From 641fa24fb33d7e570844d4f17fbf2854d5d0c01c Mon Sep 17 00:00:00 2001 From: Ignacio Bermudez Corrales Date: Mon, 22 Mar 2021 12:56:35 -0700 Subject: [PATCH 19/22] matching api outputs --- bin/ssa-end-to-end-testing/modules/test_ssa_detections.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/ssa-end-to-end-testing/modules/test_ssa_detections.py b/bin/ssa-end-to-end-testing/modules/test_ssa_detections.py index 61cfdce563..84bc48b59c 100644 --- a/bin/ssa-end-to-end-testing/modules/test_ssa_detections.py +++ b/bin/ssa-end-to-end-testing/modules/test_ssa_detections.py @@ -165,8 +165,8 @@ class SSADetectionTesting: None """ deactivate_pipeline = lambda p: self.api.deactivate_pipeline(p)[0].status_code == HTTPStatus.OK - delete_pipeline = lambda p: self.api.delete_pipeline(p)[0].status_code == HTTPStatus.NO_CONTENT - delete_index = lambda p: self.api.delete_temp_index(p["id"])[0] == HTTPStatus.NO_CONTENT + delete_pipeline = lambda p: self.api.delete_pipeline(p).status_code == HTTPStatus.NO_CONTENT + delete_index = lambda p: self.api.delete_temp_index(p["id"]) == HTTPStatus.NO_CONTENT self.activated_pipelines = [p for p in self.activated_pipelines if not deactivate_pipeline(p)] self.created_pipelines = [p for p in self.created_pipelines if not delete_pipeline(p)] if len(self.activated_pipelines) > 0 or len(self.created_pipelines) > 0 or not delete_index(self.results_index): From ec2ba3450ab29230d35a99bbffd51f1587163d9a Mon Sep 17 00:00:00 2001 From: Ignacio Bermudez Corrales Date: Mon, 22 Mar 2021 14:16:21 -0700 Subject: [PATCH 20/22] fix check source/sink --- bin/ssa-end-to-end-testing/modules/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/ssa-end-to-end-testing/modules/utils.py b/bin/ssa-end-to-end-testing/modules/utils.py index 7830901892..13424416f6 100644 --- a/bin/ssa-end-to-end-testing/modules/utils.py +++ b/bin/ssa-end-to-end-testing/modules/utils.py @@ -77,7 +77,7 @@ def request_headers(header_token): def check_source_sink(spl): match_source = re.match(r"^\s*\|\s+from\s+read_ssa_enriched_events\(\s*\)", spl) - match_sink = re.match(r".*into\s+write_ssa_detected_events\(\s*\)\s*;", spl) + match_sink = re.search(r"\|\s*into\s+write_ssa_detected_events\(\s*\)\s*;", spl) return match_source and match_sink From 1b76734dd381121104ecd0bbda1c5ac9ec5aa1d6 Mon Sep 17 00:00:00 2001 From: Ignacio Bermudez Corrales Date: Mon, 22 Mar 2021 14:42:16 -0700 Subject: [PATCH 21/22] check before modifications --- bin/ssa-end-to-end-testing/modules/test_ssa_detections.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bin/ssa-end-to-end-testing/modules/test_ssa_detections.py b/bin/ssa-end-to-end-testing/modules/test_ssa_detections.py index 84bc48b59c..0747c8856b 100644 --- a/bin/ssa-end-to-end-testing/modules/test_ssa_detections.py +++ b/bin/ssa-end-to-end-testing/modules/test_ssa_detections.py @@ -96,6 +96,7 @@ class SSADetectionTesting: self.wait_time(SLEEP_TIME_CREATE_INDEX) + check_ssa_spl = check_source_sink(spl) spl = manipulate_spl(self.api.env, spl, self.results_index) assert spl is not None, "fail to manipulate spl file" @@ -112,7 +113,7 @@ class SSADetectionTesting: self.wait_time(SLEEP_TIME_ACTIVATE_PIPELINE) - if not check_source_sink(spl): + if not check_ssa_spl: msg = f"Detection test successful for {test_name}" LOGGER.warning(f"Test not completed. Detection seems deprecated, and will not send messages to SSA") self.test_results["msg"] = msg From adc4aa9908dda5695c8774ac86b45dc318188846 Mon Sep 17 00:00:00 2001 From: Ignacio Bermudez Corrales Date: Mon, 22 Mar 2021 16:09:20 -0700 Subject: [PATCH 22/22] tear down on any sort of error (not just assertions) --- bin/ssa-end-to-end-testing/modules/test_ssa_detections.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/bin/ssa-end-to-end-testing/modules/test_ssa_detections.py b/bin/ssa-end-to-end-testing/modules/test_ssa_detections.py index 0747c8856b..d9c323c189 100644 --- a/bin/ssa-end-to-end-testing/modules/test_ssa_detections.py +++ b/bin/ssa-end-to-end-testing/modules/test_ssa_detections.py @@ -189,6 +189,11 @@ class SSADetectionTesting: LOGGER.error(e.args[0]) return {"result": False, "msg": f"Detection test failure for {test_name}"} + except Exception as e: + self.ssa_detection_test_teardown() + LOGGER.error(e) + return {"result": False, + "msg": f"Detection test failure for {test_name} (perhaps SCS problems)"} # only for troubleshooting # def ssa_detection_in_dsp_with_preview_session(self, spl, source, test_name):