From b31ee54c206eb8f2a013d354197707fb0dcbaac1 Mon Sep 17 00:00:00 2001 From: Xiao Lin Date: Thu, 20 Jan 2022 19:30:11 -0800 Subject: [PATCH 1/8] Unusual Volume of Data Download from Internal Server Per Entity --- ...l_volume_download_from_internal_server.yml | 68 +++++++++++++++++++ ...ume_download_from_internal_server.test.yml | 13 ++++ 2 files changed, 81 insertions(+) create mode 100644 detections/network/ssa___unusual_volume_download_from_internal_server.yml create mode 100644 tests/network/ssa___unusual_volume_download_from_internal_server.test.yml diff --git a/detections/network/ssa___unusual_volume_download_from_internal_server.yml b/detections/network/ssa___unusual_volume_download_from_internal_server.yml new file mode 100644 index 0000000000..0c24720ca0 --- /dev/null +++ b/detections/network/ssa___unusual_volume_download_from_internal_server.yml @@ -0,0 +1,68 @@ +name: Unusual Volume of Data Download from Internal Server Per Entity +id: cca028f4-77dd-11ec-bc09-acde48001122 +version: 1 +date: '2022-01-17' +author: Xiao Lin, Splunk +type: Anomaly +datamodel: +- Network_Traffic +description: Insider might conduct information collection before data exfiltration, and unusual volume of data + download from internal server is an indicator of such potential threat. This detection evaluates the total + bytes downloaded from internal servers at specific time window per entity level, and then flagged these that + are higher than 99.999% percentile as an anamaly. A behavior will be reported as long as the downloaded byte + volume is unusual even though that operation is benigh, which causes false positive. It is therefore advised + to adjust threshold and time window based on detection performance whenever necessary. It should be noted that + seasonality is not modeled in the current approach. +search: '| from read_ssa_enriched_events() + | eval sourcetype = ucast(map_get(input_event, "sourcetype"), "string", null) + | where sourcetype == "pan:traffic" + | eval dest_device = map_get(input_event, "dest_device_ips") + | where dest_device IS NOT NULL AND dest_device_scope == "INTERNAL" + | eval src_device = map_get(input_event, "source_device_ips") + | where src_device IS NOT NULL AND src_device_scope == "INTERNAL" + | eval download_bytes = map_get(input_event, "bytes_in") + | where download_bytes IS NOT NULL + | eval tenant = map_get(input_event, "_tenant"), event_id = map_get(input_event, "event_id") + | adaptive_threshold algorithm="quantile" value="download_bytes" entity="dest_device" threshold=0.001 window=86400L + | eval end_time = _time + | eval start_time = end_time - 86400 + | eval body = create_map(["event_id", event_id, "tenant", tenant]) + | select start_time, end_time, dest_device, label, body + | into write_ssa_detected_events();' +how_to_implement: ingest PAN traffic logs +known_false_positives: benigh large volume data download might be flagged as (false) positive. +references: +- https://github.com/twitter/AnomalyDetection +tags: + analytic_story: + - insider threat detection, information collection + dataset: + - https://ssa-test-dataset.s3-us-west-2.amazonaws.com/unusual_volume_data_download.txt + kill_chain_phases: + - Weaponization + mitre_attack_id: + - T1213 + - T1039 + product: + - Splunk Behavioral Analytics + required_fields: + - _time + - dest_device_scope + - bytes_in + - src_device_ips + security_domain: network + impact: 50 + confidence: 50 + risk_score: 25 + risk_severity: low + context: + message: $src_device_ip downloaded unusually amount of data from internal server within one day + observable: + - name: src_device_ip + type: Host IP + role: + - Others + nist: + - DE.AE + cis20: + - CIS 13 \ No newline at end of file diff --git a/tests/network/ssa___unusual_volume_download_from_internal_server.test.yml b/tests/network/ssa___unusual_volume_download_from_internal_server.test.yml new file mode 100644 index 0000000000..907f064f1e --- /dev/null +++ b/tests/network/ssa___unusual_volume_download_from_internal_server.test.yml @@ -0,0 +1,13 @@ +name: Unusual Volume of Data Download from Internal Server Per Entity - SSA Unit Test +tests: +- name: Unusual Volume of Data Download from Internal Server Per Entity + file: network/ssa___unusual_volume_download_from_internal_server.yml + pass_condition: '| stats count | where count > 0' + description: Test unusual volume of data download from internal server per entity + earliest_time: '-24h' + latest_time: 'now' + attack_data: + - file_name: unusual_volume_data_download.txt + data: https://ssa-test-dataset.s3-us-west-2.amazonaws.com/unusual_volume_data_download.txt + source: PAN Traffic Log + sourcetype: 'pan:traffic' \ No newline at end of file From 62ab57673386cc275a6cdbf07a6edca2147c2a3a Mon Sep 17 00:00:00 2001 From: Bhavin Patel Date: Wed, 9 Feb 2022 12:06:59 -0800 Subject: [PATCH 2/8] Update ssa___unusual_volume_download_from_internal_server.yml --- ...ssa___unusual_volume_download_from_internal_server.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/detections/network/ssa___unusual_volume_download_from_internal_server.yml b/detections/network/ssa___unusual_volume_download_from_internal_server.yml index 0c24720ca0..bcee2070cb 100644 --- a/detections/network/ssa___unusual_volume_download_from_internal_server.yml +++ b/detections/network/ssa___unusual_volume_download_from_internal_server.yml @@ -29,13 +29,13 @@ search: '| from read_ssa_enriched_events() | eval body = create_map(["event_id", event_id, "tenant", tenant]) | select start_time, end_time, dest_device, label, body | into write_ssa_detected_events();' -how_to_implement: ingest PAN traffic logs -known_false_positives: benigh large volume data download might be flagged as (false) positive. +how_to_implement: Ingest PAN traffic logs +known_false_positives: Benig large volume data download might be flagged as (false) positive. references: - https://github.com/twitter/AnomalyDetection tags: analytic_story: - - insider threat detection, information collection + - Insider Threat dataset: - https://ssa-test-dataset.s3-us-west-2.amazonaws.com/unusual_volume_data_download.txt kill_chain_phases: @@ -65,4 +65,4 @@ tags: nist: - DE.AE cis20: - - CIS 13 \ No newline at end of file + - CIS 13 From ddd492fe23595d3219bb4287aa0f769770f08686 Mon Sep 17 00:00:00 2001 From: Xiao Lin Date: Wed, 9 Feb 2022 14:23:27 -0800 Subject: [PATCH 3/8] update data link to attack_range, move detection to experimental --- .../ssa___unusual_volume_download_from_internal_server.yml | 2 +- ...sa___unusual_volume_download_from_internal_server.test.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) rename detections/{ => experimental}/network/ssa___unusual_volume_download_from_internal_server.yml (95%) rename tests/{network => experimental}/ssa___unusual_volume_download_from_internal_server.test.yml (67%) diff --git a/detections/network/ssa___unusual_volume_download_from_internal_server.yml b/detections/experimental/network/ssa___unusual_volume_download_from_internal_server.yml similarity index 95% rename from detections/network/ssa___unusual_volume_download_from_internal_server.yml rename to detections/experimental/network/ssa___unusual_volume_download_from_internal_server.yml index bcee2070cb..786de16d3d 100644 --- a/detections/network/ssa___unusual_volume_download_from_internal_server.yml +++ b/detections/experimental/network/ssa___unusual_volume_download_from_internal_server.yml @@ -37,7 +37,7 @@ tags: analytic_story: - Insider Threat dataset: - - https://ssa-test-dataset.s3-us-west-2.amazonaws.com/unusual_volume_data_download.txt + - https://github.com/splunk/attack_data/blob/master/datasets/suspicious_behaviour/unusual_data_download/unusual_volume_data_download.txt kill_chain_phases: - Weaponization mitre_attack_id: diff --git a/tests/network/ssa___unusual_volume_download_from_internal_server.test.yml b/tests/experimental/ssa___unusual_volume_download_from_internal_server.test.yml similarity index 67% rename from tests/network/ssa___unusual_volume_download_from_internal_server.test.yml rename to tests/experimental/ssa___unusual_volume_download_from_internal_server.test.yml index 907f064f1e..f7a54963c3 100644 --- a/tests/network/ssa___unusual_volume_download_from_internal_server.test.yml +++ b/tests/experimental/ssa___unusual_volume_download_from_internal_server.test.yml @@ -1,13 +1,13 @@ name: Unusual Volume of Data Download from Internal Server Per Entity - SSA Unit Test tests: - name: Unusual Volume of Data Download from Internal Server Per Entity - file: network/ssa___unusual_volume_download_from_internal_server.yml + file: experimental/network/ssa___unusual_volume_download_from_internal_server.yml pass_condition: '| stats count | where count > 0' description: Test unusual volume of data download from internal server per entity earliest_time: '-24h' latest_time: 'now' attack_data: - file_name: unusual_volume_data_download.txt - data: https://ssa-test-dataset.s3-us-west-2.amazonaws.com/unusual_volume_data_download.txt + data: https://github.com/splunk/attack_data/blob/master/datasets/suspicious_behaviour/unusual_data_download/unusual_volume_data_download.txt source: PAN Traffic Log sourcetype: 'pan:traffic' \ No newline at end of file From 9a04a4e833538e184e2ce72aa5d6db49d5cc1316 Mon Sep 17 00:00:00 2001 From: Xiao Lin Date: Wed, 9 Feb 2022 14:54:23 -0800 Subject: [PATCH 4/8] fix a typo --- .../ssa___unusual_volume_download_from_internal_server.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/detections/experimental/network/ssa___unusual_volume_download_from_internal_server.yml b/detections/experimental/network/ssa___unusual_volume_download_from_internal_server.yml index 786de16d3d..ec1f51e299 100644 --- a/detections/experimental/network/ssa___unusual_volume_download_from_internal_server.yml +++ b/detections/experimental/network/ssa___unusual_volume_download_from_internal_server.yml @@ -30,7 +30,7 @@ search: '| from read_ssa_enriched_events() | select start_time, end_time, dest_device, label, body | into write_ssa_detected_events();' how_to_implement: Ingest PAN traffic logs -known_false_positives: Benig large volume data download might be flagged as (false) positive. +known_false_positives: Benign large volume data download might be flagged as (false) positive. references: - https://github.com/twitter/AnomalyDetection tags: From a1e6d3e2257aba33e867dde25877751ecd92ab43 Mon Sep 17 00:00:00 2001 From: Xiao Lin Date: Wed, 9 Feb 2022 15:45:33 -0800 Subject: [PATCH 5/8] update search string --- ...ssa___unusual_volume_download_from_internal_server.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/detections/experimental/network/ssa___unusual_volume_download_from_internal_server.yml b/detections/experimental/network/ssa___unusual_volume_download_from_internal_server.yml index ec1f51e299..a782af9c45 100644 --- a/detections/experimental/network/ssa___unusual_volume_download_from_internal_server.yml +++ b/detections/experimental/network/ssa___unusual_volume_download_from_internal_server.yml @@ -15,16 +15,20 @@ description: Insider might conduct information collection before data exfiltrati seasonality is not modeled in the current approach. search: '| from read_ssa_enriched_events() | eval sourcetype = ucast(map_get(input_event, "sourcetype"), "string", null) + | eval timestamp = parse_long(ucast(map_get(input_event, "_time"), "string", null)) | where sourcetype == "pan:traffic" + | eval src_device_scope = map_get(input_event, "src_device_scope") + | eval dest_device_scope = map_get(input_event, "src_device_scope") + | where src_device_scope IS NOT NULL AND dest_device_scope IS NOT NULL | eval dest_device = map_get(input_event, "dest_device_ips") | where dest_device IS NOT NULL AND dest_device_scope == "INTERNAL" | eval src_device = map_get(input_event, "source_device_ips") | where src_device IS NOT NULL AND src_device_scope == "INTERNAL" - | eval download_bytes = map_get(input_event, "bytes_in") + | eval download_bytes = parse_double(ucast(map_get(input_event, "bytes_in"), "string", null)) | where download_bytes IS NOT NULL | eval tenant = map_get(input_event, "_tenant"), event_id = map_get(input_event, "event_id") | adaptive_threshold algorithm="quantile" value="download_bytes" entity="dest_device" threshold=0.001 window=86400L - | eval end_time = _time + | eval end_time = timestamp | eval start_time = end_time - 86400 | eval body = create_map(["event_id", event_id, "tenant", tenant]) | select start_time, end_time, dest_device, label, body From 4c5f48c507dca93758691bb82136d9205d66da07 Mon Sep 17 00:00:00 2001 From: Xiao Lin Date: Wed, 9 Feb 2022 16:40:45 -0800 Subject: [PATCH 6/8] add detection for unusal volume of data download --- ...al_volume_download_from_internal_server.test.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 tests/experimental/network/ssa___unusual_volume_download_from_internal_server.test.yml diff --git a/tests/experimental/network/ssa___unusual_volume_download_from_internal_server.test.yml b/tests/experimental/network/ssa___unusual_volume_download_from_internal_server.test.yml new file mode 100644 index 0000000000..f7a54963c3 --- /dev/null +++ b/tests/experimental/network/ssa___unusual_volume_download_from_internal_server.test.yml @@ -0,0 +1,13 @@ +name: Unusual Volume of Data Download from Internal Server Per Entity - SSA Unit Test +tests: +- name: Unusual Volume of Data Download from Internal Server Per Entity + file: experimental/network/ssa___unusual_volume_download_from_internal_server.yml + pass_condition: '| stats count | where count > 0' + description: Test unusual volume of data download from internal server per entity + earliest_time: '-24h' + latest_time: 'now' + attack_data: + - file_name: unusual_volume_data_download.txt + data: https://github.com/splunk/attack_data/blob/master/datasets/suspicious_behaviour/unusual_data_download/unusual_volume_data_download.txt + source: PAN Traffic Log + sourcetype: 'pan:traffic' \ No newline at end of file From ee2ee9874235c1ad81f92874f9bc83afccc9df74 Mon Sep 17 00:00:00 2001 From: Xiao Lin Date: Wed, 9 Feb 2022 16:44:17 -0800 Subject: [PATCH 7/8] remove duplicate --- ...al_volume_download_from_internal_server.test.yml | 13 ------------- 1 file changed, 13 deletions(-) delete mode 100644 tests/experimental/ssa___unusual_volume_download_from_internal_server.test.yml diff --git a/tests/experimental/ssa___unusual_volume_download_from_internal_server.test.yml b/tests/experimental/ssa___unusual_volume_download_from_internal_server.test.yml deleted file mode 100644 index f7a54963c3..0000000000 --- a/tests/experimental/ssa___unusual_volume_download_from_internal_server.test.yml +++ /dev/null @@ -1,13 +0,0 @@ -name: Unusual Volume of Data Download from Internal Server Per Entity - SSA Unit Test -tests: -- name: Unusual Volume of Data Download from Internal Server Per Entity - file: experimental/network/ssa___unusual_volume_download_from_internal_server.yml - pass_condition: '| stats count | where count > 0' - description: Test unusual volume of data download from internal server per entity - earliest_time: '-24h' - latest_time: 'now' - attack_data: - - file_name: unusual_volume_data_download.txt - data: https://github.com/splunk/attack_data/blob/master/datasets/suspicious_behaviour/unusual_data_download/unusual_volume_data_download.txt - source: PAN Traffic Log - sourcetype: 'pan:traffic' \ No newline at end of file From baab60365787975d78fc6af4c02b89244bfd6f58 Mon Sep 17 00:00:00 2001 From: Bhavin Patel Date: Wed, 9 Feb 2022 16:45:28 -0800 Subject: [PATCH 8/8] Update ssa___unusual_volume_download_from_internal_server.yml --- .../ssa___unusual_volume_download_from_internal_server.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/detections/experimental/network/ssa___unusual_volume_download_from_internal_server.yml b/detections/experimental/network/ssa___unusual_volume_download_from_internal_server.yml index a782af9c45..e87ce92d78 100644 --- a/detections/experimental/network/ssa___unusual_volume_download_from_internal_server.yml +++ b/detections/experimental/network/ssa___unusual_volume_download_from_internal_server.yml @@ -65,7 +65,7 @@ tags: - name: src_device_ip type: Host IP role: - - Others + - Others nist: - DE.AE cis20: