Branch was auto-updated.

bin/ssa-end-to-end-testing/modules/streams_service_api_helper.py

@@ -429,7 +429,7 @@ class DSPApi:
         return preview_id
 
 
-    def ingest_data(self, data):
+    def ingest_data(self, data, sourcetype):
         """
         Send events
 
@@ -443,9 +443,11 @@ class DSPApi:
         response
             response body in JSON format
         """
+        if sourcetype == "WinEventLog:Security":
+            sourcetype = "WinEventLog"
         data = [{
             "body": event,
-            "sourcetype": "WinEventLog"
+            "sourcetype": sourcetype
             } for event in data]
         response = requests.post(self.return_api_endpoint(INGEST_ENDPOINT), json=data, headers=request_headers(self.header_token))
         if response.status_code != HTTPStatus.OK:

bin/ssa-end-to-end-testing/modules/test_ssa_detections.py

@@ -1,5 +1,6 @@
 import logging
 import os
+from re import S
 import time
 import sys
 import uuid
@@ -48,7 +49,7 @@ class SSADetectionTesting:
         for i in range(0, len(test_spls)):
             self.max_execution_time = MAX_EXECUTION_TIME_LIMIT
             test_id = str(uuid.uuid4())
-            test_result = self.ssa_detection_test(read_spl(file_path_spl, test_spls[i]), file_path_data, test_names[i], test_id)
+            test_result = self.ssa_detection_test(read_spl(file_path_spl, test_spls[i]), file_path_data, test_names[i], test_id, "WinEventLog")
             test_results.append(test_result.copy())
 
         passed = True
@@ -70,6 +71,7 @@ class SSADetectionTesting:
         test_id = str(uuid.uuid4())
         test_results = self.ssa_detection_test(test_obj["detection_obj"]["search"], file_path_attack_data,
                                                "SSA Smoke Test " + test_obj["test_obj"]["name"], test_id,
+                                               test_obj['test_obj']['tests'][0]['attack_data'][0]['source'],
                                                test_obj['test_obj']['tests'][0]['pass_condition'])
 
         return test_results
@@ -113,7 +115,7 @@ class SSADetectionTesting:
                 else:
                     LOGGER.warning("Found and deleted an old pipeline: %s", pipeline['name'])
 
-    def ssa_detection_test_main(self, spl, source, test_name, pass_condition, test_id):
+    def ssa_detection_test_main(self, spl, source, test_name, pass_condition, test_id, sourcetype):
         self.execution_passed = True
 
         self.wait_time(SLEEP_TIME_CREATE_INDEX)
@@ -147,12 +149,12 @@ class SSADetectionTesting:
             self.test_results["msg"] = msg
             return self.test_results
 
-        data = read_data(source)
+        data = read_data(source, sourcetype)
         LOGGER.info("Sending (%d) events" % (len(data)))
 
         assert len(data) > 0, "No events to send, skip to next test."
 
-        data_uploaded = self.api.ingest_data(data)
+        data_uploaded = self.api.ingest_data(data, sourcetype)
         assert data_uploaded, "Failed to upload test data"
 
         self.wait_time(SLEEP_TIME_SEND_DATA)
@@ -212,10 +214,10 @@ class SSADetectionTesting:
         else:
             LOGGER.info("Testing successfully cleaned up")
 
-    def ssa_detection_test(self, spl, source, test_name, test_id, pass_condition='@count_gt(0)'):
+    def ssa_detection_test(self, spl, source, test_name, test_id, sourcetype, pass_condition='@count_gt(0)'):
         self.ssa_detection_test_init()
         try:
-            test_result = self.ssa_detection_test_main(spl, source, test_name, pass_condition, test_id)
+            test_result = self.ssa_detection_test_main(spl, source, test_name, pass_condition, test_id, sourcetype)
             self.ssa_detection_test_teardown()
             return test_result
         except AssertionError as e:

bin/ssa-end-to-end-testing/modules/utils.py

@@ -3,6 +3,7 @@ import logging
 import os
 import fileinput
 import re
+import io
 
 from .data_manipulation import DataManipulation
 
@@ -104,31 +105,35 @@ def replace_ssa_macros(source, sink, spl):
     return spl
 
 
-def read_data(file_path):
-    data_manipulation = DataManipulation()
-    modified_file = data_manipulation.manipulate_timestamp(file_path, 'xmlwineventlog', 'WinEventLog:Security')
+def read_data(file_path, sourcetype):
     data = []
-    date_rex = r'\d{2}/\d{2}/\d{4} \d{2}:\d{2}:\d{2} [AP]M'
-    count = len(open(modified_file).readlines())
+    if sourcetype == "WinEventLog:Security" or sourcetype == "WinEventLog":
+        data_manipulation = DataManipulation()
+        modified_file = data_manipulation.manipulate_timestamp(file_path, 'xmlwineventlog', 'WinEventLog:Security')
+        date_rex = r'\d{2}/\d{2}/\d{4} \d{2}:\d{2}:\d{2} [AP]M'
+        count = len(open(modified_file).readlines())
 
-    i = 0
-    file = fileinput.input(files=modified_file)
-    event = file[0]
-    start_position = 0
+        i = 0
+        file = fileinput.input(files=modified_file)
+        event = file[0]
+        start_position = 0
 
-    for i in range(1, count):
-        line = file[i]
-        i = i + 1
-        if re.match(date_rex, line):
-            data.append(event)
-            start_position = i
-            event = line
-        else:
-            event = event + line
+        for i in range(1, count):
+            line = file[i]
+            i = i + 1
+            if re.match(date_rex, line):
+                data.append(event)
+                start_position = i
+                event = line
+            else:
+                event = event + line
 
 
-    data.append(event)
-    fileinput.close()
+        data.append(event)
+        fileinput.close()
+    elif sourcetype == "xmlwineventlog":
+        for line in fileinput.input(files=file_path):
+            data.append(line)
 
     return data
 

detections/endpoint/ssa___excessive_number_of_office_files_copied.yml

@@ -0,0 +1,54 @@
+name: Excessive Number of Office Files Copied
+id: 3c6594a9-8df6-45a1-9357-d73b62083c63
+version: 1
+date: '2021-12-07'
+author: Patrick Bareiss, Splunk
+type: Anomaly
+datamodel:
+- Endpoint_Filesystem
+description: This detection detects a high amount of office file copied.
+  This can be an indicator for a malicious insider.
+search: '| from read_ssa_enriched_events() 
+  | eval timestamp=parse_long(ucast(map_get(input_event, "_time"), "string", null)) 
+  | eval action=ucast(map_get(input_event, "action"), "string", null), 
+  process=ucast(map_get(input_event, "process"), "string", null), 
+  file_name=ucast(map_get(input_event, "file_name"), "string", null), 
+  file_path=ucast(map_get(input_event, "file_path"), "string", null), 
+  dest_user_id=ucast(map_get(input_event, "dest_user_id"), "string", null), 
+  dest_device_id=ucast(map_get(input_event, "dest_device_id"), "string", null) 
+  | where "Endpoint_Filesystem" IN(_datamodels) 
+  | where action="created" 
+  | where like(file_name, "%.doc%") OR like(file_name, "%.xls%") OR like(file_name, "%.ppt%") 
+  | stats count(file_name) AS count BY dest_user_id, dest_device_id, span(timestamp, 10m)
+  | where count > 20
+  | eval start_time=window_start, end_time=window_end, entities=mvappend(dest_user_id, dest_device_id), body=create_map(["count", count]) 
+  | into write_ssa_detected_events();'
+how_to_implement: To successfully implement this search you need to be ingesting information
+  on process that include the name of the process responsible for the changes from
+  your endpoints into the `Endpoint` datamodel in the `Filesytem` node.
+known_false_positives: user may copy a lot of office fies from one folder to another
+references: []
+tags:
+  analytic_story: []
+  confidence: 80
+  context:
+  - Source:Endpoint
+  - Stage:Exfitration
+  dataset:
+   - https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1048.003/mass_file_creation/windows-sysmon.log
+  impact: 90
+  kill_chain_phases:
+  - Exploitation
+  message: High number of files copied
+  mitre_attack_id:
+  - T1048.003
+  product:
+  - Splunk Behavioral Analytics
+  required_fields:
+  - action
+  - process
+  - file_name
+  - file_path
+  risk_score: 72
+  security_domain: endpoint
+

detections/endpoint/ssa___high_file_deletion_frequency.yml

@@ -0,0 +1,75 @@
+name: High File Deletion Frequency
+id: b6200efd-13bd-4336-920a-057b25bbcfaf
+version: 1
+date: '2021-12-07'
+author: Patrick Bareiss, Splunk
+type: Anomaly
+datamodel:
+- Endpoint_Filesystem
+description: This detection detects a high amount of file deletions in a short time for specific file types.
+  This can be an indicator for a malicious insider.
+search: '| from read_ssa_enriched_events() | eval timestamp=parse_long(ucast(map_get(input_event, "_time"), "string", null)) 
+  | eval action=ucast(map_get(input_event, "action"), "string", null), 
+  process=ucast(map_get(input_event, "process"), "string", null), 
+  file_name=ucast(map_get(input_event, "file_name"), "string", null), 
+  file_path=ucast(map_get(input_event, "file_path"), "string", null), 
+  dest_user_id=ucast(map_get(input_event, "dest_user_id"), "string", null), 
+  dest_device_id=ucast(map_get(input_event, "dest_device_id"), "string", null) 
+  | where "Endpoint_Filesystem" IN(_datamodels) 
+  | where action="deleted" 
+  | where like(file_name, "%.cmd") OR like(file_name, "%.ini") OR like(file_name, "%.gif") 
+  OR like(file_name, "%.jpg") OR like(file_name, "%.jpeg") OR like(file_name, "%.db") 
+  OR like(file_name, "%.doc%") OR like(file_name, "%.ps1") OR like(file_name, "%.xls%") 
+  OR like(file_name, "%.ppt%") OR like(file_name, "%.bmp") OR like(file_name, "%.zip") 
+  OR like(file_name, "%.rar") OR like(file_name, "%.7z") OR like(file_name, "%.chm") 
+  OR like(file_name, "%.png") OR like(file_name, "%.log") OR like(file_name, "%.vbs") 
+  OR like(file_name, "%.js")
+  | stats count(file_name) AS count BY dest_user_id, dest_device_id, span(timestamp, 10m) 
+  | where count > 20
+  | eval start_time=window_start, end_time=window_end, entities=mvappend(dest_user_id, dest_device_id), body=create_map(["count", count]) 
+  | into write_ssa_detected_events();'
+how_to_implement: To successfully implement this search you need to be ingesting information
+  on process that include the name of the process responsible for the changes from
+  your endpoints into the `Endpoint` datamodel in the `Filesytem` node.
+known_false_positives: user may delete bunch of pictures or files in a folder.
+references:
+- https://www.fireeye.com/blog/threat-research/2020/10/fin11-email-campaigns-precursor-for-ransomware-data-theft.html
+- https://blog.virustotal.com/2020/11/keep-your-friends-close-keep-ransomware.html
+tags:
+  analytic_story:
+  - Clop Ransomware
+  confidence: 80
+  context:
+  - Source:Endpoint
+  - Stage:Execution
+  dataset:
+   - https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1485/excessive_file_deletions/windows-sysmon.log
+  impact: 90
+  kill_chain_phases:
+  - Exploitation
+  message: High frequency file deletion activity detected on host $Computer$
+  mitre_attack_id:
+  - T1485
+  observable:
+  - name: user
+    type: User
+    role:
+    - Victim
+  - name: Computer
+    type: Endpoint
+    role:
+    - Victim
+  - name: deleted_files
+    type: File Name
+    role:
+    - Target
+  product:
+  - Splunk Behavioral Analytics
+  required_fields:
+  - action
+  - process
+  - file_name
+  - file_path
+  risk_score: 72
+  security_domain: endpoint
+

tests/endpoint/ssa___excessive_number_of_office_files_copied.test.yml

@@ -0,0 +1,10 @@
+name: Excessive Number of Office Files Copied Unit Test
+tests:
+  - name: Excessive Number of Office Files Copied
+    file: endpoint/ssa___excessive_number_of_office_files_copied.yml
+    pass_condition: '@count_gt(0)'
+    description: Test for Excessive Number of Office Files Copied
+    attack_data:
+      - file_name: sysmon.log
+        data: https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1048.003/mass_file_creation/windows-sysmon.log
+        source: xmlwineventlog

tests/endpoint/ssa___high_file_deletion_frequency.test.yml

@@ -0,0 +1,10 @@
+name: High File Deletion Frequency Unit Test
+tests:
+  - name: High File Deletion Frequency
+    file: endpoint/ssa___high_file_deletion_frequency.yml
+    pass_condition: '@count_gt(0)'
+    description: Test for High File Deletion Frequency
+    attack_data:
+      - file_name: sysmon.log
+        data: https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1485/excessive_file_deletions/windows-sysmon.log
+        source: xmlwineventlog