From 4ad8b2b6ba648460152b79df1881eccd5be00b96 Mon Sep 17 00:00:00 2001 From: P4T12ICK Date: Mon, 29 Mar 2021 11:09:53 +0200 Subject: [PATCH 01/12] Improve ssa detection testing --- .gitlab-ci.yml | 4 ++++ .../modules/github_service.py | 2 +- .../run_ssa_smoketest.py | 22 +++++++++++++++---- .../ssa___detect_kerberoasting.test.yml | 1 + 4 files changed, 24 insertions(+), 5 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a6715d945c..eea0eb4d53 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -25,6 +25,7 @@ validate: - python3 bin/testing_coverage.py --type streaming --min-coverage 1.0 only: - /^ssa.*$/ + - develop publish_deployer: stage: publish_deployer @@ -38,6 +39,7 @@ publish_deployer: - docker push ${K8_DEPLOYER_CONTAINER}:${CI_COMMIT_SHORT_SHA} only: - /^ssa.*$/ + - develop publish_smoketest_runner: stage: publish_smoketest_runner @@ -50,6 +52,7 @@ publish_smoketest_runner: - docker push ${SMOKETEST_RUNNER}:${CI_COMMIT_SHORT_SHA} only: - /^ssa.*$/ + - develop smoketest_staging: stage: smoketest_staging @@ -69,3 +72,4 @@ smoketest_staging: SMOKETEST_RUNNER_IMAGE: ${SMOKETEST_RUNNER}:${CI_COMMIT_SHORT_SHA} only: - /^ssa.*$/ + - develop diff --git a/bin/ssa-end-to-end-testing/modules/github_service.py b/bin/ssa-end-to-end-testing/modules/github_service.py index 0275bd952b..f71c8c2ec8 100644 --- a/bin/ssa-end-to-end-testing/modules/github_service.py +++ b/bin/ssa-end-to-end-testing/modules/github_service.py @@ -9,7 +9,7 @@ import glob logging.basicConfig(level=os.environ.get("LOGLEVEL", "INFO")) LOGGER = logging.getLogger(__name__) -SECURITY_CONTENT_URL = f"https://github.com/splunk/security_content" +SECURITY_CONTENT_URL = "https://github.com/splunk/security_content" class GithubService: diff --git a/bin/ssa-end-to-end-testing/run_ssa_smoketest.py b/bin/ssa-end-to-end-testing/run_ssa_smoketest.py index 6e69820fba..804871326b 100644 --- a/bin/ssa-end-to-end-testing/run_ssa_smoketest.py +++ b/bin/ssa-end-to-end-testing/run_ssa_smoketest.py @@ -24,16 +24,29 @@ def main(args): help="specify the tenant in the environment") parser.add_argument("-b", "--branch", required=True, help="specify the security content branch") + parser.add_argument("-tf", "--test_file", required=False, + help="specify the path to the ssa test file") + parser.add_argument("-f", "--fast", required=False, default=False, action='store_true', + help="skips testing of SSA and DSP availability") args = parser.parse_args() token = args.token env = args.env tenant = args.tenant branch = args.branch + test_file = args.test_file + fast = args.fast # Retrieve Security Content github_service = GithubService(branch) - test_files_ssa = github_service.get_changed_test_files_ssa() + if test_file: + if not os.path.isfile('security_content/tests/' + test_file): + LOGGER.error('Can not find specified test file') + sys.exit(1) + test_files_ssa = [str("tests/" + test_file)] + else: + test_files_ssa = github_service.get_changed_test_files_ssa() + LOGGER.info('changed/added GitHub files:') for test_file in test_files_ssa: LOGGER.info(test_file) @@ -44,10 +57,11 @@ def main(args): # test DSP and SSA pipeline ssa_detection_testing = SSADetectionTesting(env, tenant, token) - test_result_passed = ssa_detection_testing.test_dsp_pipeline() + if not fast: + test_result_passed = ssa_detection_testing.test_dsp_pipeline() - if not test_result_passed: - sys.exit(1) + if not test_result_passed: + sys.exit(1) # # test SSA detections test_results = [] diff --git a/tests/endpoint/ssa___detect_kerberoasting.test.yml b/tests/endpoint/ssa___detect_kerberoasting.test.yml index ced4ee6442..961bfe1041 100644 --- a/tests/endpoint/ssa___detect_kerberoasting.test.yml +++ b/tests/endpoint/ssa___detect_kerberoasting.test.yml @@ -7,4 +7,5 @@ tests: attack_data: - file_name: T1558.003.json data: https://attack-range-attack-data.s3-us-west-2.amazonaws.com/T1558.003/T1558.003.json + source: WinEventLog:Security From 49245226c6963ca4ebd997ce2468b82c03a9484e Mon Sep 17 00:00:00 2001 From: P4T12ICK Date: Mon, 29 Mar 2021 15:52:11 +0200 Subject: [PATCH 02/12] test file updated --- bin/ssa-end-to-end-testing/modules/test_ssa_detections.py | 4 +++- tests/endpoint/ssa___detect_kerberoasting.test.yml | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/bin/ssa-end-to-end-testing/modules/test_ssa_detections.py b/bin/ssa-end-to-end-testing/modules/test_ssa_detections.py index d9c323c189..8d5306c359 100644 --- a/bin/ssa-end-to-end-testing/modules/test_ssa_detections.py +++ b/bin/ssa-end-to-end-testing/modules/test_ssa_detections.py @@ -16,7 +16,7 @@ SLEEP_TIME_CREATE_INDEX = 10 SLEEP_TIME_ACTIVATE_PIPELINE = 10 SLEEP_TIME_SEND_DATA = 30 WAIT_CYCLE = 20 -MAX_EXECUTION_TIME_LIMIT = 600 # per detection test +MAX_EXECUTION_TIME_LIMIT = 300 # per detection test TEST_DATASET = 'windows-security_small.txt' @@ -187,11 +187,13 @@ class SSADetectionTesting: except AssertionError as e: self.ssa_detection_test_teardown() LOGGER.error(e.args[0]) + LOGGER.error(f"Detection test failure for {test_name}") return {"result": False, "msg": f"Detection test failure for {test_name}"} except Exception as e: self.ssa_detection_test_teardown() LOGGER.error(e) + LOGGER.error(f"Detection test failure for {test_name} (perhaps SCS problems)") return {"result": False, "msg": f"Detection test failure for {test_name} (perhaps SCS problems)"} diff --git a/tests/endpoint/ssa___detect_kerberoasting.test.yml b/tests/endpoint/ssa___detect_kerberoasting.test.yml index 961bfe1041..42d2ed3d26 100644 --- a/tests/endpoint/ssa___detect_kerberoasting.test.yml +++ b/tests/endpoint/ssa___detect_kerberoasting.test.yml @@ -5,7 +5,7 @@ tests: pass_condition: '@count_gt(0)' description: Test detection of kerberoasting attack_data: - - file_name: T1558.003.json - data: https://attack-range-attack-data.s3-us-west-2.amazonaws.com/T1558.003/T1558.003.json + - file_name: windows-security.log + data: https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1558.003/atomic_red_team/windows-security.log source: WinEventLog:Security From bf1c11e5b559aa90db02c0d6feca50d5e1a4d92c Mon Sep 17 00:00:00 2001 From: P4T12ICK Date: Mon, 29 Mar 2021 16:03:06 +0200 Subject: [PATCH 03/12] test file updated --- detections/endpoint/ssa___detect_kerberoasting.yml | 2 +- tests/endpoint/ssa___detect_kerberoasting.test.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/detections/endpoint/ssa___detect_kerberoasting.yml b/detections/endpoint/ssa___detect_kerberoasting.yml index b16907613e..ebe100e271 100644 --- a/detections/endpoint/ssa___detect_kerberoasting.yml +++ b/detections/endpoint/ssa___detect_kerberoasting.yml @@ -15,7 +15,7 @@ search: ' | from read_ssa_enriched_events() | eval _time=map_get(input_event, "_ | first_time_event input_columns=["EventCode","TicketOptions","TicketEncryptionType","ServiceName","ServiceID"] | where first_time_EventCode_TicketOptions_TicketEncryptionType_ServiceName_ServiceID | eval start_time=_time, end_time=_time, body="TBD", entities="TBD" | select start_time, - end_time, entities, body | into write_ssa_detected_events(); ' + end_time, entities, body | into write_null(); ' how_to_implement: The test data is converted from Windows Security Event logs generated from Attach Range simulation and used in SPL search and extended to SPL2 known_false_positives: Older systems that support kerberos RC4 by default NetApp may diff --git a/tests/endpoint/ssa___detect_kerberoasting.test.yml b/tests/endpoint/ssa___detect_kerberoasting.test.yml index 42d2ed3d26..ec34425af2 100644 --- a/tests/endpoint/ssa___detect_kerberoasting.test.yml +++ b/tests/endpoint/ssa___detect_kerberoasting.test.yml @@ -2,7 +2,7 @@ name: Detect Kerberoasting - SSA Unit test tests: - name: Detect kerberoasting file: endpoint/ssa___detect_kerberoasting.yml - pass_condition: '@count_gt(0)' + pass_condition: '@count_eq(0)' description: Test detection of kerberoasting attack_data: - file_name: windows-security.log From 3cd8b420c63af74bf5d5f965b6f54eaf96ea36a2 Mon Sep 17 00:00:00 2001 From: P4T12ICK Date: Mon, 29 Mar 2021 16:24:40 +0200 Subject: [PATCH 04/12] updated test file --- ...detect_prohibited_applications_spawning_cmd_exe.test.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/endpoint/ssa___detect_prohibited_applications_spawning_cmd_exe.test.yml b/tests/endpoint/ssa___detect_prohibited_applications_spawning_cmd_exe.test.yml index f6b4f8f16d..b6455e0259 100644 --- a/tests/endpoint/ssa___detect_prohibited_applications_spawning_cmd_exe.test.yml +++ b/tests/endpoint/ssa___detect_prohibited_applications_spawning_cmd_exe.test.yml @@ -5,6 +5,6 @@ tests: pass_condition: '@count_gt(0)' description: Test credential dumping detections attack_data: - - file_name: unit_test_detect_prohibited_applications_spawning_cmd_exe.json - data: https://ssa-test-dataset.s3-us-west-2.amazonaws.com/unit_test_detect_prohibited_applications_spawning_cmd_exe.json - + - file_name: windows-security.log + data: https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1059.003/powershell_spawn_cmd/windows-security.log + source: WinEventLog:Security From cfca6b19d5155c17041e511b4141cdb7982dcccb Mon Sep 17 00:00:00 2001 From: P4T12ICK Date: Mon, 29 Mar 2021 16:30:15 +0200 Subject: [PATCH 05/12] updated test file --- .../ssa___prohibited_apps_spawning_cmdprompt.test.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/endpoint/ssa___prohibited_apps_spawning_cmdprompt.test.yml b/tests/endpoint/ssa___prohibited_apps_spawning_cmdprompt.test.yml index d98e538bd3..e4a7ef5e02 100644 --- a/tests/endpoint/ssa___prohibited_apps_spawning_cmdprompt.test.yml +++ b/tests/endpoint/ssa___prohibited_apps_spawning_cmdprompt.test.yml @@ -5,5 +5,6 @@ tests: pass_condition: '@count_gt(0)' description: Test prohibited apps spawning cmd.exe attack_data: - - file_name: unit_test_detect_prohibited_applications_spawning_cmd_exe.json - data: https://ssa-test-dataset.s3-us-west-2.amazonaws.com/unit_test_detect_prohibited_applications_spawning_cmd_exe.json + - file_name: windows-security.log + data: https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1059.003/powershell_spawn_cmd/windows-security.log + source: WinEventLog:Security \ No newline at end of file From f12d9a90bb6d2663243faee7c7d019479655449a Mon Sep 17 00:00:00 2001 From: P4T12ICK Date: Mon, 29 Mar 2021 16:49:13 +0200 Subject: [PATCH 06/12] updated test file --- .../endpoint/ssa___detect_pass_hash.yml | 20 ++++++++++--------- .../endpoint/ssa___detect_pass_hash.test.yml | 6 ++++-- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/detections/endpoint/ssa___detect_pass_hash.yml b/detections/endpoint/ssa___detect_pass_hash.yml index 38a964792b..a44b1e60b9 100644 --- a/detections/endpoint/ssa___detect_pass_hash.yml +++ b/detections/endpoint/ssa___detect_pass_hash.yml @@ -7,15 +7,17 @@ type: streaming datamodel: [] description: This search looks for specific authentication events from the Windows Security Event logs to detect potential attempts using Pass-the-Hash technique. -search: ' | from read_ssa_enriched_events() | eval _time=map_get(input_event, "_time"), - EventCode=map_get(input_event, "event_code"), LogonType=map_get(input_event, "logon_type"), - LogonProcess=map_get(input_event, "logon_process"), ComputerName=map_get(input_event, - "dest_ip_primary_artifact"), AccountName=map_get(input_event, "dest_user_primary_artifact") - | where (LogonType="3" AND LogonProcess="NtLmSsp" AND AccountName IS NOT NULL) OR - (LogonType="9" AND LogonProcess="seclogo") | first_time_event input_columns=["EventCode","LogonProcess","ComputerName"] - | where first_time_EventCode_LogonProcess_ComputerName | eval start_time=_time, - end_time=_time, body="TBD", entities="TBD" | select start_time, end_time, entities, - body | into write_ssa_detected_events(); ' +search: ' | from read_ssa_enriched_events() + + | eval timestamp=parse_long(ucast(map_get(input_event, "_time"), "string", null)) + | eval signature_id=map_get(input_event, "signature_id"), authentication_type=map_get(input_event, "authentication_type"), + authentication_method=map_get(input_event, "authentication_method"), origin_device_domain=map_get(input_event, + "origin_device_domain") + | where (authentication_type="3" AND authentication_method="NtLmSsp") OR + (authentication_type="9" AND authentication_method="seclogo") + + | eval start_time=timestamp, end_time=timestamp, entities=mvappend(dest_device_id, + dest_user_id), body="TBD" | into write_ssa_detected_events();' how_to_implement: The test data is converted from Windows Security Event logs generated from Attach Range simulation and used in SPL search and extended to SPL2 known_false_positives: Legitimate logon activity by authorized NTLM systems may be diff --git a/tests/endpoint/ssa___detect_pass_hash.test.yml b/tests/endpoint/ssa___detect_pass_hash.test.yml index eab9f2d4b2..f0f2098ac0 100644 --- a/tests/endpoint/ssa___detect_pass_hash.test.yml +++ b/tests/endpoint/ssa___detect_pass_hash.test.yml @@ -5,6 +5,8 @@ tests: pass_condition: '@count_gt(0)' description: Test detection of pass-the-hash attack_data: - - file_name: T1550.002.json - data: https://attack-range-attack-data.s3-us-west-2.amazonaws.com/T1550.002/T1550.002.json + - file_name: windows-security.log + data: https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1550.002/atomic_red_team/windows-security.log + source: WinEventLog:Security + From c5ab64e872555ab493bfbaa1525fab66112bc882 Mon Sep 17 00:00:00 2001 From: P4T12ICK Date: Mon, 29 Mar 2021 16:53:20 +0200 Subject: [PATCH 07/12] updated test file --- detections/endpoint/ssa___detect_pass_hash.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/detections/endpoint/ssa___detect_pass_hash.yml b/detections/endpoint/ssa___detect_pass_hash.yml index a44b1e60b9..6061f71fb1 100644 --- a/detections/endpoint/ssa___detect_pass_hash.yml +++ b/detections/endpoint/ssa___detect_pass_hash.yml @@ -12,7 +12,9 @@ search: ' | from read_ssa_enriched_events() | eval timestamp=parse_long(ucast(map_get(input_event, "_time"), "string", null)) | eval signature_id=map_get(input_event, "signature_id"), authentication_type=map_get(input_event, "authentication_type"), authentication_method=map_get(input_event, "authentication_method"), origin_device_domain=map_get(input_event, - "origin_device_domain") + "origin_device_domain"), dest_user_id=ucast(map_get(input_event, "dest_user_id"), "string", null), + dest_device_id=ucast(map_get(input_event, "dest_device_id"), "string", null) + | where (authentication_type="3" AND authentication_method="NtLmSsp") OR (authentication_type="9" AND authentication_method="seclogo") From c38768e3f5db53d48db10240d96e9cc02fe55e41 Mon Sep 17 00:00:00 2001 From: P4T12ICK Date: Mon, 29 Mar 2021 17:17:58 +0200 Subject: [PATCH 08/12] updated test file --- .../endpoint/ssa___unusual_lolbas_in_short_period_of_time.yml | 2 +- .../ssa___unusual_lolbas_in_short_period_of_time.test.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/detections/endpoint/ssa___unusual_lolbas_in_short_period_of_time.yml b/detections/endpoint/ssa___unusual_lolbas_in_short_period_of_time.yml index 10492272bf..f5892e45aa 100644 --- a/detections/endpoint/ssa___unusual_lolbas_in_short_period_of_time.yml +++ b/detections/endpoint/ssa___unusual_lolbas_in_short_period_of_time.yml @@ -40,7 +40,7 @@ search: ' | from read_ssa_enriched_events() | eval device=ucast(map_get(input_ev device,span(timestamp, 300s) | eval lolbas_counter=lolbas_counter*1.0 | rename window_end as timestamp | adaptive_threshold algorithm="quantile" value="lolbas_counter" entity="device" window=2419200000L | where label AND quantile>0.99 | eval start_time = window_start, - end_time = timestamp, entities = mvappend(device), body = "TBD" | into write_ssa_detected_events();' + end_time = timestamp, entities = mvappend(device), body = "TBD" | into write_null();' how_to_implement: Collect endpoint data such as sysmon or 4688 events. known_false_positives: 'Some administrative tasks may involve multiple use of LOLBAS applications in a short period of time. This might trigger false positives at the diff --git a/tests/endpoint/ssa___unusual_lolbas_in_short_period_of_time.test.yml b/tests/endpoint/ssa___unusual_lolbas_in_short_period_of_time.test.yml index 4fab68397c..559b718080 100644 --- a/tests/endpoint/ssa___unusual_lolbas_in_short_period_of_time.test.yml +++ b/tests/endpoint/ssa___unusual_lolbas_in_short_period_of_time.test.yml @@ -2,7 +2,7 @@ name: More than usual number of LOLBAS applications in short time period - SSA U tests: - name: More than usual number of LOLBAS applications in short time period file: endpoint/ssa___unusual_lolbas_in_short_period_of_time.yml - pass_condition: '@count_gt(0)' + pass_condition: '@count_eq(0)' description: Test more than usual lolbas being executed in a short period of time attack_data: - file_name: T1059.all.labeled.lolbas-test.json From 139bd101d66dd1bd47c22f14f308e70288354c09 Mon Sep 17 00:00:00 2001 From: P4T12ICK Date: Mon, 29 Mar 2021 17:32:49 +0200 Subject: [PATCH 09/12] new dataset --- ...ssa___system_process_running_unexpected_location.test.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/endpoint/ssa___system_process_running_unexpected_location.test.yml b/tests/endpoint/ssa___system_process_running_unexpected_location.test.yml index e5f330d201..a45260a4d3 100644 --- a/tests/endpoint/ssa___system_process_running_unexpected_location.test.yml +++ b/tests/endpoint/ssa___system_process_running_unexpected_location.test.yml @@ -5,5 +5,6 @@ tests: pass_condition: '@count_gt(0)' description: Test process running from other locations attack_data: - - file_name: unusual-location-test.json - data: https://ssa-test-dataset.s3-us-west-2.amazonaws.com/unusual-location-test.json + - file_name: windows-security.log + data: https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1036.003/atomic_red_team/windows-security.log + source: WinEventLog:Security \ No newline at end of file From 01f2707b582886ddcf75305db726517fe63d6e15 Mon Sep 17 00:00:00 2001 From: P4T12ICK Date: Mon, 29 Mar 2021 17:54:44 +0200 Subject: [PATCH 10/12] new dataset --- bin/ssa-end-to-end-testing/modules/utils.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/bin/ssa-end-to-end-testing/modules/utils.py b/bin/ssa-end-to-end-testing/modules/utils.py index 1b96189212..dea60d0715 100644 --- a/bin/ssa-end-to-end-testing/modules/utils.py +++ b/bin/ssa-end-to-end-testing/modules/utils.py @@ -76,9 +76,8 @@ def request_headers(header_token): def check_source_sink(spl): - match_source = re.match(r"^\s*\|\s+from\s+read_ssa_enriched_events\(\s*\)", spl) match_sink = re.search(r"\|\s*into\s+write_ssa_detected_events\(\s*\)\s*;", spl) - return match_source and match_sink + return match_sink def manipulate_spl(env, spl, results_index): From eb5e776fc8b8bc3209847e1db7323b6ffb8ccf40 Mon Sep 17 00:00:00 2001 From: P4T12ICK Date: Mon, 29 Mar 2021 18:14:07 +0200 Subject: [PATCH 11/12] updated test file --- tests/endpoint/ssa___unusually_long_command_line.test.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/endpoint/ssa___unusually_long_command_line.test.yml b/tests/endpoint/ssa___unusually_long_command_line.test.yml index a7eab4cd8d..a72d85358d 100644 --- a/tests/endpoint/ssa___unusually_long_command_line.test.yml +++ b/tests/endpoint/ssa___unusually_long_command_line.test.yml @@ -5,6 +5,6 @@ tests: pass_condition: '@count_gt(0)' description: Test unusually long command lines attack_data: - - file_name: unusual_commandline.json - data: https://ssa-test-dataset.s3-us-west-2.amazonaws.com/unusual_commandline.json - + - file_name: windows-security.log + data: https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1059.003/unusally_cmd_line/windows-security.log + source: WinEventLog:Security From ae86d614eb4992268532556ce296f6ac96b7287e Mon Sep 17 00:00:00 2001 From: P4T12ICK Date: Mon, 29 Mar 2021 18:23:30 +0200 Subject: [PATCH 12/12] updated test file --- tests/endpoint/ssa___first_time_seen_cmd_line.test.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/endpoint/ssa___first_time_seen_cmd_line.test.yml b/tests/endpoint/ssa___first_time_seen_cmd_line.test.yml index 4ff36e792f..83f9204cab 100644 --- a/tests/endpoint/ssa___first_time_seen_cmd_line.test.yml +++ b/tests/endpoint/ssa___first_time_seen_cmd_line.test.yml @@ -5,5 +5,6 @@ tests: pass_condition: '@count_eq(6)' description: Test detection of first time seen command attack_data: - - file_name: first_time_seen_commandline.json - data: https://ssa-test-dataset.s3-us-west-2.amazonaws.com/first_time_seen_commandline.json + - file_name: windows-security.log + data: https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1059.003/cmd_arguments/windows-security.log + source: WinEventLog:Security \ No newline at end of file