diff --git a/bin/modules/assertions/__init__.py b/bin/modules/assertions/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/bin/modules/__init__.py b/bin/ssa-end-to-end-testing/modules/assertions/__init__.py similarity index 100% rename from bin/modules/__init__.py rename to bin/ssa-end-to-end-testing/modules/assertions/__init__.py diff --git a/bin/modules/assertions/application/assertions.py b/bin/ssa-end-to-end-testing/modules/assertions/application/assertions.py similarity index 100% rename from bin/modules/assertions/application/assertions.py rename to bin/ssa-end-to-end-testing/modules/assertions/application/assertions.py diff --git a/bin/modules/assertions/assertions_parser.py b/bin/ssa-end-to-end-testing/modules/assertions/assertions_parser.py similarity index 100% rename from bin/modules/assertions/assertions_parser.py rename to bin/ssa-end-to-end-testing/modules/assertions/assertions_parser.py diff --git a/bin/modules/assertions/cloud/assertions.py b/bin/ssa-end-to-end-testing/modules/assertions/cloud/assertions.py similarity index 100% rename from bin/modules/assertions/cloud/assertions.py rename to bin/ssa-end-to-end-testing/modules/assertions/cloud/assertions.py diff --git a/bin/modules/assertions/endpoint/assertions.py b/bin/ssa-end-to-end-testing/modules/assertions/endpoint/assertions.py similarity index 100% rename from bin/modules/assertions/endpoint/assertions.py rename to bin/ssa-end-to-end-testing/modules/assertions/endpoint/assertions.py diff --git a/bin/modules/assertions/global_assertions.py b/bin/ssa-end-to-end-testing/modules/assertions/global_assertions.py similarity index 100% rename from bin/modules/assertions/global_assertions.py rename to bin/ssa-end-to-end-testing/modules/assertions/global_assertions.py diff --git a/bin/modules/assertions/network/assertions.py b/bin/ssa-end-to-end-testing/modules/assertions/network/assertions.py similarity index 100% rename from bin/modules/assertions/network/assertions.py rename to bin/ssa-end-to-end-testing/modules/assertions/network/assertions.py diff --git a/bin/modules/assertions/web/assertions.py b/bin/ssa-end-to-end-testing/modules/assertions/web/assertions.py similarity index 100% rename from bin/modules/assertions/web/assertions.py rename to bin/ssa-end-to-end-testing/modules/assertions/web/assertions.py diff --git a/bin/modules/ssa_utils.py b/bin/ssa-end-to-end-testing/modules/ssa_utils.py similarity index 100% rename from bin/modules/ssa_utils.py rename to bin/ssa-end-to-end-testing/modules/ssa_utils.py diff --git a/bin/ssa-end-to-end-testing/modules/test_ssa_detections.py b/bin/ssa-end-to-end-testing/modules/test_ssa_detections.py index fde01d0534..fef35db18b 100644 --- a/bin/ssa-end-to-end-testing/modules/test_ssa_detections.py +++ b/bin/ssa-end-to-end-testing/modules/test_ssa_detections.py @@ -6,6 +6,7 @@ import sys from http import HTTPStatus from modules.streams_service_api_helper import DSPApi from modules.utils import check_source_sink, manipulate_spl, read_spl, read_data +from ssa_test import assert_results # Logger logging.basicConfig(level=os.environ.get("LOGLEVEL", "INFO")) @@ -67,7 +68,8 @@ class SSADetectionTesting: file_path_attack_data = test_obj["attack_data_file_path"] test_results = self.ssa_detection_test(test_obj["detection_obj"]["search"], file_path_attack_data, - "SSA Smoke Test " + test_obj["test_obj"]["name"]) + "SSA Smoke Test " + test_obj["test_obj"]["name"], + test_obj['test_obj']['tests'][0]['pass_condition']) return test_results @@ -110,7 +112,7 @@ class SSADetectionTesting: else: LOGGER.warning("Found and deleted an old pipeline: %s", pipeline['name']) - def ssa_detection_test_main(self, spl, source, test_name): + def ssa_detection_test_main(self, spl, source, test_name, pass_condition): self.execution_passed = True self.wait_time(SLEEP_TIME_CREATE_INDEX) @@ -169,7 +171,12 @@ class SSADetectionTesting: LOGGER.info( f"Search didn't return any results. Retrying in {WAIT_CYCLE}s, max execution time left {self.max_execution_time}s") - assert len(results) > 0, "Search job didn't return any results" + if not results: + LOGGER.warning("Search job didn't return any results") + + LOGGER.info('Received %s result(s)', len(results)) + test_passed = assert_results(pass_condition, results) + assert test_passed, f"Pass condition {pass_condition} not satisfied" msg = f"Detection test successful for {test_name}" LOGGER.info(msg) @@ -197,10 +204,10 @@ class SSADetectionTesting: else: LOGGER.info("Testing successfully cleaned up") - def ssa_detection_test(self, spl, source, test_name): + def ssa_detection_test(self, spl, source, test_name, pass_condition='@count_gt(0)'): self.ssa_detection_test_init() try: - test_result = self.ssa_detection_test_main(spl, source, test_name) + test_result = self.ssa_detection_test_main(spl, source, test_name, pass_condition) self.ssa_detection_test_teardown() return test_result except AssertionError as e: diff --git a/bin/modules/testing_utils.py b/bin/ssa-end-to-end-testing/modules/testing_utils.py similarity index 100% rename from bin/modules/testing_utils.py rename to bin/ssa-end-to-end-testing/modules/testing_utils.py diff --git a/bin/ssa-end-to-end-testing/requirements.txt b/bin/ssa-end-to-end-testing/requirements.txt index 6d5ea588d1..a1cea2a4d9 100644 --- a/bin/ssa-end-to-end-testing/requirements.txt +++ b/bin/ssa-end-to-end-testing/requirements.txt @@ -1,6 +1,7 @@ attrs==20.3.0 certifi==2020.12.5 chardet==4.0.0 +coloredlogs==14.0 gitdb==4.0.5 GitPython==3.1.14 idna==2.10 @@ -17,6 +18,7 @@ pytz==2021.1 PyYAML==5.4.1 requests==2.25.1 six==1.15.0 +sly==0.4 smmap==3.0.5 toml==0.10.2 u-msgpack-python==2.7.1 diff --git a/bin/ssa_test.py b/bin/ssa-end-to-end-testing/ssa_test.py similarity index 100% rename from bin/ssa_test.py rename to bin/ssa-end-to-end-testing/ssa_test.py diff --git a/bin/ssa_validate.py b/bin/ssa-end-to-end-testing/ssa_validate.py similarity index 100% rename from bin/ssa_validate.py rename to bin/ssa-end-to-end-testing/ssa_validate.py diff --git a/detections/endpoint/ssa___detect_kerberoasting.yml b/detections/endpoint/ssa___detect_kerberoasting.yml index ea4b3fe19e..2f4e0612ed 100644 --- a/detections/endpoint/ssa___detect_kerberoasting.yml +++ b/detections/endpoint/ssa___detect_kerberoasting.yml @@ -1,6 +1,6 @@ name: Detect Kerberoasting id: dabdd6d7-3e10-42be-8711-4e124f7a3850 -version: 1 +version: 2 date: '2020-10-21' author: Xiao Lin, Splunk type: streaming @@ -12,7 +12,8 @@ search: ' | from read_ssa_enriched_events() | eval _time=map_get(input_event, "_ "ticket_options"), TicketEncryptionType=map_get(input_event, "ticket_encryption_type"), ServiceName=map_get(input_event, "service_name"), ServiceID=map_get(input_event, "service_id"), dest_user_id=ucast(map_get(input_event, "dest_user_id"), "string", - null), dest_device_id=ucast(map_get(input_event, "dest_device_id") | where EventCode="4769" + null), dest_device_id=ucast(map_get(input_event, "dest_device_id"), "string", null) + | where EventCode="4769" AND TicketOptions="0x40810000" AND TicketEncryptionType="0x17" | first_time_event input_columns=["EventCode","TicketOptions","TicketEncryptionType","ServiceName","ServiceID"] | where first_time_EventCode_TicketOptions_TicketEncryptionType_ServiceName_ServiceID diff --git a/detections/endpoint/ssa___first_time_seen_cmd_line.yml b/detections/endpoint/ssa___first_time_seen_cmd_line.yml index 355cff51a4..0baf036de5 100644 --- a/detections/endpoint/ssa___first_time_seen_cmd_line.yml +++ b/detections/endpoint/ssa___first_time_seen_cmd_line.yml @@ -1,6 +1,6 @@ name: First time seen command line argument id: fc0edc95-ff2b-48b0-9f6f-63da3789fd23 -version: 2 +version: 3 date: 2021-2-1 author: Ignacio Bermudez Corrales, Splunk type: streaming @@ -19,7 +19,7 @@ search: '| from read_ssa_enriched_events() | eval timestamp=parse_long(ucast(map */ cmd_line_norm=replace(cmd_line_norm, /\w:\\[^:]*(?=\\.*\.\w{3}(\s|$)+)/, "\\PATH"), /* replaces "C:\\Something\\Something\\command.ext" => "PATH\\command.ext" */ cmd_line_norm=replace(cmd_line_norm, /\d+/, "N") | where process_name="cmd.exe" AND match_regex(ucast(cmd_line, "string", - ""), /.* \/[cC] .*/)=true | select cmd_line, cmd_line_norm, timestamp, dest_device_id, + ""), /.* \/[cC] .*/)=true | select process_name, cmd_line, cmd_line_norm, timestamp, dest_device_id, dest_user_id | first_time_event input_columns=["cmd_line_norm"] | where first_time_cmd_line_norm | eval start_time = timestamp, end_time = timestamp, entities = mvappend(dest_device_id, dest_user_id), body=create_map(["cmd_line", cmd_line, "process_name", process_name]) diff --git a/detections/endpoint/ssa___system_process_running_unexpected_location.yml b/detections/endpoint/ssa___system_process_running_unexpected_location.yml index ca13c9e39c..973a926295 100644 --- a/detections/endpoint/ssa___system_process_running_unexpected_location.yml +++ b/detections/endpoint/ssa___system_process_running_unexpected_location.yml @@ -1,6 +1,6 @@ name: System Process Running from Unexpected Location id: 28179107-099a-464a-94d3-08301e6c055f -version: 2 +version: 3 date: '2020-08-25' author: Ignacio Bermudez Corrales, Splunk type: streaming @@ -224,8 +224,9 @@ search: ' $ssa_input = | from read_ssa_enriched_events() | eval device=ucast(map OR process_name="xcopy.exe" OR process_name="xpsrchvw.exe" OR process_name="xwizard.exe"; | from $cond_1 | union $cond_2 | union $cond_3 | union $cond_4 | union $cond_5 | - union $cond_6 | where process_path NOT LIKE "%\\windows\\system32%" AND process_path - NOT LIKE "%\\windows\\syswow64%" | eval start_time=timestamp, end_time=timestamp, + union $cond_6 | where match_regex(process_path, /(?i)\\windows\\system32/)=false + AND match_regex(process_path, /(?i)\\windows\\syswow64/)=false + | eval start_time=timestamp, end_time=timestamp, entities=mvappend(device, user), body=create_map(["process_path", process_path, "process_name", process_name]) | into write_ssa_detected_events();' how_to_implement: Collect endpoint data such as sysmon or 4688 events. diff --git a/detections/endpoint/ssa___unusual_lolbas_in_short_period_of_time.yml b/detections/endpoint/ssa___unusual_lolbas_in_short_period_of_time.yml index 91a6f580d2..9dc4ddb0b6 100644 --- a/detections/endpoint/ssa___unusual_lolbas_in_short_period_of_time.yml +++ b/detections/endpoint/ssa___unusual_lolbas_in_short_period_of_time.yml @@ -1,6 +1,6 @@ name: More than usual number of LOLBAS applications in short time period id: 59c0dd70-169c-4900-9a1f-bfcf13302f93 -version: 1 +version: 2 date: '2020-08-25' author: Ignacio Bermudez Corrales, Splunk type: streaming @@ -40,8 +40,8 @@ search: ' | from read_ssa_enriched_events() | eval device=ucast(map_get(input_ev device,span(timestamp, 300s) | eval lolbas_counter=lolbas_counter*1.0 | rename window_end as timestamp | adaptive_threshold algorithm="quantile" value="lolbas_counter" entity="device" window=2419200000L | where label AND quantile>0.99 | eval start_time = window_start, - end_time = timestamp, entities = mvappend(device), body=create_map(["process_name", - process_name]) | into write_null();' + end_time = timestamp, entities = mvappend(device), body=create_map(["lolbas_counter", + lolbas_counter]) | into write_null();' how_to_implement: Collect endpoint data such as sysmon or 4688 events. known_false_positives: 'Some administrative tasks may involve multiple use of LOLBAS applications in a short period of time. This might trigger false positives at the diff --git a/tests/endpoint/ssa___first_time_seen_cmd_line.test.yml b/tests/endpoint/ssa___first_time_seen_cmd_line.test.yml index 83f9204cab..e2dfca0320 100644 --- a/tests/endpoint/ssa___first_time_seen_cmd_line.test.yml +++ b/tests/endpoint/ssa___first_time_seen_cmd_line.test.yml @@ -2,9 +2,9 @@ name: First time seen command line argument - SSA Unit Test tests: - name: First time seen command line argument file: endpoint/ssa___first_time_seen_cmd_line.yml - pass_condition: '@count_eq(6)' + pass_condition: '@count_gt(0)' description: Test detection of first time seen command attack_data: - file_name: windows-security.log data: https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1059.003/cmd_arguments/windows-security.log - source: WinEventLog:Security \ No newline at end of file + source: WinEventLog:Security diff --git a/tests/endpoint/ssa___system_process_running_unexpected_location.test.yml b/tests/endpoint/ssa___system_process_running_unexpected_location.test.yml index 1b1d33dac1..d7ac0d8556 100644 --- a/tests/endpoint/ssa___system_process_running_unexpected_location.test.yml +++ b/tests/endpoint/ssa___system_process_running_unexpected_location.test.yml @@ -2,7 +2,7 @@ name: System Process Running from Unexpected Location - SSA Unit Test tests: - name: System Process Running from Unexpected Location file: endpoint/ssa___system_process_running_unexpected_location.yml - pass_condition: '@count_eq(0)' + pass_condition: '@count_eq(1)' description: Test System Process Running from an unexpected location attack_data: - file_name: windows-security.log diff --git a/tests/endpoint/ssa___unusual_lolbas_in_short_period_of_time.test.yml b/tests/endpoint/ssa___unusual_lolbas_in_short_period_of_time.test.yml index 559b718080..4fab68397c 100644 --- a/tests/endpoint/ssa___unusual_lolbas_in_short_period_of_time.test.yml +++ b/tests/endpoint/ssa___unusual_lolbas_in_short_period_of_time.test.yml @@ -2,7 +2,7 @@ name: More than usual number of LOLBAS applications in short time period - SSA U tests: - name: More than usual number of LOLBAS applications in short time period file: endpoint/ssa___unusual_lolbas_in_short_period_of_time.yml - pass_condition: '@count_eq(0)' + pass_condition: '@count_gt(0)' description: Test more than usual lolbas being executed in a short period of time attack_data: - file_name: T1059.all.labeled.lolbas-test.json