Merge pull request #1301 from splunk/ssa_ci_testing_update

Ssa ci testing update

.gitlab-ci.yml

@@ -25,6 +25,7 @@ validate:
     - python3 bin/testing_coverage.py --type streaming --min-coverage 1.0
   only:
     - /^ssa.*$/
+    - develop
 
 publish_deployer:
   stage: publish_deployer
@@ -38,6 +39,7 @@ publish_deployer:
     - docker push ${K8_DEPLOYER_CONTAINER}:${CI_COMMIT_SHORT_SHA}
   only:
     - /^ssa.*$/
+    - develop
 
 publish_smoketest_runner:
   stage: publish_smoketest_runner
@@ -50,6 +52,7 @@ publish_smoketest_runner:
     - docker push ${SMOKETEST_RUNNER}:${CI_COMMIT_SHORT_SHA}
   only:
     - /^ssa.*$/
+    - develop
 
 smoketest_staging:
   stage: smoketest_staging
@@ -69,3 +72,4 @@ smoketest_staging:
     SMOKETEST_RUNNER_IMAGE: ${SMOKETEST_RUNNER}:${CI_COMMIT_SHORT_SHA}
   only:
     - /^ssa.*$/
+    - develop

bin/ssa-end-to-end-testing/modules/github_service.py

@@ -9,7 +9,7 @@ import glob
 logging.basicConfig(level=os.environ.get("LOGLEVEL", "INFO"))
 LOGGER = logging.getLogger(__name__)
 
-SECURITY_CONTENT_URL = f"https://github.com/splunk/security_content"
+SECURITY_CONTENT_URL = "https://github.com/splunk/security_content"
 
 
 class GithubService:

bin/ssa-end-to-end-testing/modules/test_ssa_detections.py

@@ -16,7 +16,7 @@ SLEEP_TIME_CREATE_INDEX = 10
 SLEEP_TIME_ACTIVATE_PIPELINE = 10
 SLEEP_TIME_SEND_DATA = 30
 WAIT_CYCLE = 20
-MAX_EXECUTION_TIME_LIMIT = 600  # per detection test
+MAX_EXECUTION_TIME_LIMIT = 300  # per detection test
 
 TEST_DATASET = 'windows-security_small.txt'
 
@@ -187,11 +187,13 @@ class SSADetectionTesting:
         except AssertionError as e:
             self.ssa_detection_test_teardown()
             LOGGER.error(e.args[0])
+            LOGGER.error(f"Detection test failure for {test_name}")
             return {"result": False,
                     "msg": f"Detection test failure for {test_name}"}
         except Exception as e:
             self.ssa_detection_test_teardown()
             LOGGER.error(e)
+            LOGGER.error(f"Detection test failure for {test_name} (perhaps SCS problems)")
             return {"result": False,
                     "msg": f"Detection test failure for {test_name} (perhaps SCS problems)"}
 

bin/ssa-end-to-end-testing/modules/utils.py

@@ -76,9 +76,8 @@ def request_headers(header_token):
 
 
 def check_source_sink(spl):
-    match_source = re.match(r"^\s*\|\s+from\s+read_ssa_enriched_events\(\s*\)", spl)
     match_sink = re.search(r"\|\s*into\s+write_ssa_detected_events\(\s*\)\s*;", spl)
-    return match_source and match_sink
+    return match_sink
 
 
 def manipulate_spl(env, spl, results_index):

bin/ssa-end-to-end-testing/run_ssa_smoketest.py

@@ -24,16 +24,29 @@ def main(args):
                         help="specify the tenant in the environment")
     parser.add_argument("-b", "--branch", required=True,
                         help="specify the security content branch")
+    parser.add_argument("-tf", "--test_file", required=False,
+                        help="specify the path to the ssa test file")
+    parser.add_argument("-f", "--fast", required=False, default=False, action='store_true', 
+                        help="skips testing of SSA and DSP availability")
 
     args = parser.parse_args()
     token = args.token
     env = args.env
     tenant = args.tenant
     branch = args.branch
+    test_file = args.test_file
+    fast = args.fast
 
     # Retrieve Security Content
     github_service = GithubService(branch)
-    test_files_ssa = github_service.get_changed_test_files_ssa()
+    if test_file:
+        if not os.path.isfile('security_content/tests/' + test_file):
+            LOGGER.error('Can not find specified test file')
+            sys.exit(1)
+        test_files_ssa = [str("tests/" + test_file)]
+    else:
+        test_files_ssa = github_service.get_changed_test_files_ssa()
+    
     LOGGER.info('changed/added GitHub files:')
     for test_file in test_files_ssa:
         LOGGER.info(test_file)
@@ -44,10 +57,11 @@ def main(args):
 
     # test DSP and SSA pipeline
     ssa_detection_testing = SSADetectionTesting(env, tenant, token)
-    test_result_passed = ssa_detection_testing.test_dsp_pipeline()
+    if not fast:
+        test_result_passed = ssa_detection_testing.test_dsp_pipeline()
 
-    if not test_result_passed:
-        sys.exit(1)
+        if not test_result_passed:
+            sys.exit(1)
 
     # # test SSA detections
     test_results = []

detections/endpoint/ssa___detect_kerberoasting.yml

@@ -15,7 +15,7 @@ search: ' | from read_ssa_enriched_events() | eval _time=map_get(input_event, "_
   | first_time_event input_columns=["EventCode","TicketOptions","TicketEncryptionType","ServiceName","ServiceID"]
   | where first_time_EventCode_TicketOptions_TicketEncryptionType_ServiceName_ServiceID
   | eval start_time=_time, end_time=_time, body="TBD", entities="TBD" | select start_time,
-  end_time, entities, body | into write_ssa_detected_events(); '
+  end_time, entities, body | into write_null(); '
 how_to_implement: The test data is converted from Windows Security Event logs generated
   from Attach Range simulation and used in SPL search and extended to SPL2
 known_false_positives: Older systems that support kerberos RC4 by default NetApp may

detections/endpoint/ssa___detect_pass_hash.yml

@@ -7,15 +7,19 @@ type: streaming
 datamodel: []
 description: This search looks for specific authentication events from the Windows
   Security Event logs to detect potential attempts using Pass-the-Hash technique.
-search: ' | from read_ssa_enriched_events() | eval _time=map_get(input_event, "_time"),
-  EventCode=map_get(input_event, "event_code"), LogonType=map_get(input_event, "logon_type"),
-  LogonProcess=map_get(input_event, "logon_process"), ComputerName=map_get(input_event,
-  "dest_ip_primary_artifact"), AccountName=map_get(input_event, "dest_user_primary_artifact")
-  | where (LogonType="3" AND LogonProcess="NtLmSsp" AND AccountName IS NOT NULL) OR
-  (LogonType="9" AND LogonProcess="seclogo") | first_time_event input_columns=["EventCode","LogonProcess","ComputerName"]
-  | where first_time_EventCode_LogonProcess_ComputerName | eval start_time=_time,
-  end_time=_time, body="TBD", entities="TBD" | select start_time, end_time, entities,
-  body | into write_ssa_detected_events(); '
+search: ' | from read_ssa_enriched_events() 
+  
+  | eval timestamp=parse_long(ucast(map_get(input_event, "_time"), "string", null))
+  | eval signature_id=map_get(input_event, "signature_id"), authentication_type=map_get(input_event, "authentication_type"),
+  authentication_method=map_get(input_event, "authentication_method"), origin_device_domain=map_get(input_event,
+  "origin_device_domain"), dest_user_id=ucast(map_get(input_event, "dest_user_id"), "string", null),
+  dest_device_id=ucast(map_get(input_event, "dest_device_id"), "string", null)
+  
+  | where (authentication_type="3" AND authentication_method="NtLmSsp") OR
+  (authentication_type="9" AND authentication_method="seclogo") 
+
+  | eval start_time=timestamp, end_time=timestamp, entities=mvappend(dest_device_id,
+  dest_user_id), body="TBD" | into write_ssa_detected_events();'
 how_to_implement: The test data is converted from Windows Security Event logs generated
   from Attach Range simulation and used in SPL search and extended to SPL2
 known_false_positives: Legitimate logon activity by authorized NTLM systems may be

detections/endpoint/ssa___unusual_lolbas_in_short_period_of_time.yml

@@ -40,7 +40,7 @@ search: ' | from read_ssa_enriched_events() | eval device=ucast(map_get(input_ev
   device,span(timestamp, 300s) | eval lolbas_counter=lolbas_counter*1.0 | rename window_end
   as timestamp | adaptive_threshold algorithm="quantile" value="lolbas_counter" entity="device"
   window=2419200000L | where label AND quantile>0.99 | eval start_time = window_start,
-  end_time = timestamp, entities = mvappend(device), body = "TBD" | into write_ssa_detected_events();'
+  end_time = timestamp, entities = mvappend(device), body = "TBD" | into write_null();'
 how_to_implement: Collect endpoint data such as sysmon or 4688 events.
 known_false_positives: 'Some administrative tasks may involve multiple use of LOLBAS
   applications in a short period of time. This might trigger false positives at the

tests/endpoint/ssa___detect_kerberoasting.test.yml

@@ -2,9 +2,10 @@ name: Detect Kerberoasting - SSA Unit test
 tests:
   - name: Detect kerberoasting
     file: endpoint/ssa___detect_kerberoasting.yml 
-    pass_condition: '@count_gt(0)'
+    pass_condition: '@count_eq(0)'
     description: Test detection of kerberoasting
     attack_data:
-      - file_name: T1558.003.json
-        data: https://attack-range-attack-data.s3-us-west-2.amazonaws.com/T1558.003/T1558.003.json
+      - file_name: windows-security.log
+        data: https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1558.003/atomic_red_team/windows-security.log
+        source: WinEventLog:Security
 

tests/endpoint/ssa___detect_pass_hash.test.yml

@@ -5,6 +5,8 @@ tests:
     pass_condition: '@count_gt(0)'
     description: Test detection of pass-the-hash
     attack_data:
-      - file_name: T1550.002.json
-        data: https://attack-range-attack-data.s3-us-west-2.amazonaws.com/T1550.002/T1550.002.json
+      - file_name: windows-security.log
+        data: https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1550.002/atomic_red_team/windows-security.log
+        source: WinEventLog:Security
+
 

tests/endpoint/ssa___detect_prohibited_applications_spawning_cmd_exe.test.yml

@@ -5,6 +5,6 @@ tests:
     pass_condition: '@count_gt(0)'
     description: Test credential dumping detections
     attack_data:
-      - file_name: unit_test_detect_prohibited_applications_spawning_cmd_exe.json
-        data: https://ssa-test-dataset.s3-us-west-2.amazonaws.com/unit_test_detect_prohibited_applications_spawning_cmd_exe.json
-
+      - file_name: windows-security.log
+        data: https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1059.003/powershell_spawn_cmd/windows-security.log
+        source: WinEventLog:Security

tests/endpoint/ssa___first_time_seen_cmd_line.test.yml

@@ -5,5 +5,6 @@ tests:
     pass_condition: '@count_eq(6)'
     description: Test detection of first time seen command
     attack_data:
-      - file_name: first_time_seen_commandline.json
-        data: https://ssa-test-dataset.s3-us-west-2.amazonaws.com/first_time_seen_commandline.json
+      - file_name: windows-security.log
+        data: https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1059.003/cmd_arguments/windows-security.log
+        source: WinEventLog:Security

tests/endpoint/ssa___prohibited_apps_spawning_cmdprompt.test.yml

@@ -5,5 +5,6 @@ tests:
     pass_condition: '@count_gt(0)'
     description: Test prohibited apps spawning cmd.exe
     attack_data:
-      - file_name: unit_test_detect_prohibited_applications_spawning_cmd_exe.json
-        data: https://ssa-test-dataset.s3-us-west-2.amazonaws.com/unit_test_detect_prohibited_applications_spawning_cmd_exe.json
+      - file_name: windows-security.log
+        data: https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1059.003/powershell_spawn_cmd/windows-security.log
+        source: WinEventLog:Security

tests/endpoint/ssa___system_process_running_unexpected_location.test.yml

@@ -5,5 +5,6 @@ tests:
     pass_condition: '@count_gt(0)'
     description: Test process running from other locations
     attack_data:
-      - file_name: unusual-location-test.json
-        data: https://ssa-test-dataset.s3-us-west-2.amazonaws.com/unusual-location-test.json
+      - file_name: windows-security.log
+        data: https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1036.003/atomic_red_team/windows-security.log
+        source: WinEventLog:Security

tests/endpoint/ssa___unusual_lolbas_in_short_period_of_time.test.yml

@@ -2,7 +2,7 @@ name: More than usual number of LOLBAS applications in short time period - SSA U
 tests:
   - name: More than usual number of LOLBAS applications in short time period
     file: endpoint/ssa___unusual_lolbas_in_short_period_of_time.yml
-    pass_condition: '@count_gt(0)'
+    pass_condition: '@count_eq(0)'
     description: Test more than usual lolbas being executed in a short period of time
     attack_data:
       - file_name: T1059.all.labeled.lolbas-test.json

tests/endpoint/ssa___unusually_long_command_line.test.yml

@@ -5,6 +5,6 @@ tests:
     pass_condition: '@count_gt(0)'
     description: Test unusually long command lines
     attack_data:
-      - file_name: unusual_commandline.json
-        data: https://ssa-test-dataset.s3-us-west-2.amazonaws.com/unusual_commandline.json
-
+      - file_name: windows-security.log
+        data: https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1059.003/unusally_cmd_line/windows-security.log
+        source: WinEventLog:Security