From 1da0f22a6a6f497eff0e217552cc7384086fc968 Mon Sep 17 00:00:00 2001 From: bpatel Date: Fri, 9 Oct 2020 09:59:07 -0700 Subject: [PATCH 1/6] baseline re write --- ...usly_seen_cloud_compute_images_initial.yml | 18 +++++++++--------- ...ously_seen_cloud_compute_images_update.yml | 19 ++++++++++--------- 2 files changed, 19 insertions(+), 18 deletions(-) diff --git a/baselines/previously_seen_cloud_compute_images_initial.yml b/baselines/previously_seen_cloud_compute_images_initial.yml index 775f87d217..e10fecdf9a 100644 --- a/baselines/previously_seen_cloud_compute_images_initial.yml +++ b/baselines/previously_seen_cloud_compute_images_initial.yml @@ -1,19 +1,19 @@ name: Previously Seen Cloud Compute Images - Initial id: 7744597f-d07a-4cea-94a7-e0f8aaebc410 version: 1 -date: '2018-03-12' +date: '2020-10-08' description: This search builds a table of previously seen images used to launch cloud compute instances how_to_implement: You must be ingesting the approrpiate cloud infrastructure logs - and have the Security Research cloud data model installed. + and have the latest Change Datamodel accelerated author: David Dorsey, Splunk -search: '| tstats earliest(_time) as firstTime, latest(_time) as lastTime from datamodel=Change - where All_Changes.action=created `previously_seen_cloud_compute_image_input_filter` - by All_Changes.Instance_Changes.image_id | `drop_dm_object_name("All_Changes.Instance_Changes")` - | eventstats min(firstTime) as globalFirstTime - | eval enough_data = if(globalFirstTime <= relative_time(now(), "-14d@d"), 1, 0) - | outputlookup previously_seen_cloud_compute_images - | stats count' +search: '| tstats earliest(_time) as firstTime, latest(_time) as lastTime from datamodel=Change where All_Changes.action=created by All_Changes.Instance_Changes.image_id +| `drop_dm_object_name("All_Changes")` +| `drop_dm_object_name("Instance_Changes")` +| where image_id != "unknown" +| eventstats min(firstTime) as globalFirstTime +| eval enough_data = if(globalFirstTime <= relative_time(now(), "-7d@d"), 1, 0) +| outputlookup previously_seen_cloud_compute_images' tags: analytics_story: - Cloud Cryptomining diff --git a/baselines/previously_seen_cloud_compute_images_update.yml b/baselines/previously_seen_cloud_compute_images_update.yml index 80811729e4..202291d783 100644 --- a/baselines/previously_seen_cloud_compute_images_update.yml +++ b/baselines/previously_seen_cloud_compute_images_update.yml @@ -7,15 +7,16 @@ description: This search builds a table of previously seen images used to launch how_to_implement: You must be ingesting the approrpiate cloud infrastructure logs author: David Dorsey, Splunk search: '| tstats earliest(_time) as firstTime, latest(_time) as lastTime from datamodel=Change - where All_Changes.action=created `previously_seen_cloud_compute_image_input_filter` by All_Changes.Instance_Changes.image_id - | `drop_dm_object_name("All_Changes.Instance_Changes")` - | inputlookup append=t previously_seen_cloud_compute_images - | stats min(firstTime) as firstTime max(lastTime) as lastTime by image_id - | where lastTime > relative_time(now(), `previously_seen_cloud_compute_images_forget_window`) - | eventstats min(firstTime) as globalFirstTime - | eval enough_data = if(globalFirstTime <= relative_time(now(), "-14d@d"), 1, 0) - | outputlookup previously_seen_cloud_compute_images - | stats count' + where All_Changes.action=created by All_Changes.Instance_Changes.image_id +| `drop_dm_object_name("All_Changes")` +| `drop_dm_object_name("Instance_Changes")` +| where image_id != "unknown" +| inputlookup append=t previously_seen_cloud_compute_images +| stats min(firstTime) as firstTime max(lastTime) as lastTime by image_id +| where lastTime > relative_time(now(), `previously_seen_cloud_compute_images_forget_window`) +| eventstats min(firstTime) as globalFirstTime +| eval enough_data = if(globalFirstTime <= relative_time(now(), "-7d@d"), 1, 0) +| outputlookup previously_seen_cloud_compute_images' tags: analytics_story: - Cloud Cryptomining From aac5f2b4bc10720d7f8d8c4a7024cea182dd71ef Mon Sep 17 00:00:00 2001 From: bpatel Date: Mon, 12 Oct 2020 10:29:19 -0700 Subject: [PATCH 2/6] unseen image --- ...usly_seen_cloud_compute_images_initial.yml | 4 ++-- ...ously_seen_cloud_compute_images_update.yml | 8 +++---- ...e_created_with_previously_unseen_image.yml | 22 +++++++++++-------- 3 files changed, 19 insertions(+), 15 deletions(-) diff --git a/baselines/previously_seen_cloud_compute_images_initial.yml b/baselines/previously_seen_cloud_compute_images_initial.yml index e10fecdf9a..851647a37f 100644 --- a/baselines/previously_seen_cloud_compute_images_initial.yml +++ b/baselines/previously_seen_cloud_compute_images_initial.yml @@ -7,11 +7,11 @@ description: This search builds a table of previously seen images used to launch how_to_implement: You must be ingesting the approrpiate cloud infrastructure logs and have the latest Change Datamodel accelerated author: David Dorsey, Splunk -search: '| tstats earliest(_time) as firstTime, latest(_time) as lastTime from datamodel=Change where All_Changes.action=created by All_Changes.Instance_Changes.image_id +search: '| tstats earliest(_time) as firstTimeSeen, latest(_time) as lastTimeSeen from datamodel=Change where All_Changes.action=created by All_Changes.Instance_Changes.image_id | `drop_dm_object_name("All_Changes")` | `drop_dm_object_name("Instance_Changes")` | where image_id != "unknown" -| eventstats min(firstTime) as globalFirstTime +| eventstats min(firstTimeSeen) as globalFirstTime | eval enough_data = if(globalFirstTime <= relative_time(now(), "-7d@d"), 1, 0) | outputlookup previously_seen_cloud_compute_images' tags: diff --git a/baselines/previously_seen_cloud_compute_images_update.yml b/baselines/previously_seen_cloud_compute_images_update.yml index 202291d783..71cc62651e 100644 --- a/baselines/previously_seen_cloud_compute_images_update.yml +++ b/baselines/previously_seen_cloud_compute_images_update.yml @@ -6,15 +6,15 @@ description: This search builds a table of previously seen images used to launch compute instances how_to_implement: You must be ingesting the approrpiate cloud infrastructure logs author: David Dorsey, Splunk -search: '| tstats earliest(_time) as firstTime, latest(_time) as lastTime from datamodel=Change +search: '| tstats earliest(_time) as firstTimeSeen, latest(_time) as lastTimeSeen from datamodel=Change where All_Changes.action=created by All_Changes.Instance_Changes.image_id | `drop_dm_object_name("All_Changes")` | `drop_dm_object_name("Instance_Changes")` | where image_id != "unknown" | inputlookup append=t previously_seen_cloud_compute_images -| stats min(firstTime) as firstTime max(lastTime) as lastTime by image_id -| where lastTime > relative_time(now(), `previously_seen_cloud_compute_images_forget_window`) -| eventstats min(firstTime) as globalFirstTime +| stats min(firstTimeSeen) as firstTimeSeen max(lastTimeSeen) as lastTimeSeen by image_id +| where lastTimeSeen > relative_time(now(), `previously_seen_cloud_compute_images_forget_window`) +| eventstats min(firstTimeSeen) as globalFirstTime | eval enough_data = if(globalFirstTime <= relative_time(now(), "-7d@d"), 1, 0) | outputlookup previously_seen_cloud_compute_images' tags: diff --git a/detections/cloud_compute_instance_created_with_previously_unseen_image.yml b/detections/cloud_compute_instance_created_with_previously_unseen_image.yml index 9980550229..fea2a61368 100644 --- a/detections/cloud_compute_instance_created_with_previously_unseen_image.yml +++ b/detections/cloud_compute_instance_created_with_previously_unseen_image.yml @@ -13,15 +13,19 @@ how_to_implement: "You must be ingesting your cloud infrastructure logs from you type: ESCU references: [] author: David Dorsey, Splunk -search: '| tstats earliest(_time) as firstTimeSeen, latest(_time) as lastTimeSeen values(All_Changes.object_id) as dest, count - from datamodel=Change where All_Changes.action=created `previously_seen_cloud_compute_instance_types_input_filter` - by All_Changes.Instance_Changes.image_id, All_Changes.user - | `drop_dm_object_name("All_Changes")` | `drop_dm_object_name("Instance_Changes")` - | lookup previously_seen_cloud_compute_images image_id as image_id OUTPUT firstTime - | where isnull(firstTime) OR firstTime > relative_time(now(), "-70m@m") - | `security_content_ctime(firstTimeSeen)` - | table firstTimeSeen, user, image_id, count, dest - | `cloud_compute_instance_created_with_previously_unseen_image_filter`' +search: '| tstats count earliest(_time) as firstTime, latest(_time) as lastTime values(All_Changes.object_id) as dest + from datamodel=Change where All_Changes.action=created + by All_Changes.Instance_Changes.image_id, All_Changes.user +| `drop_dm_object_name("All_Changes")` +| `drop_dm_object_name("Instance_Changes")` +| lookup previously_seen_cloud_compute_images image_id as image_id OUTPUT firstTimeSeen, enough_data +| eventstats max(enough_data) as enough_data +| where enough_data=1 +| eval firstTimeSeenImage=min(firstTimeSeen) +| where isnull(firstTimeSeenImage) OR firstTimeSeenImage > relative_time(now(), "-24h@h") +| table firstTime, user, image_id, count, dest +| `security_content_ctime(firstTime)` +| `cloud_compute_instance_created_with_previously_unseen_image_filter`' known_false_positives: After a new image is created, the first systems created with that image will cause this alert to fire. Verify that the image being used was created by a legitimate user. From d237a2d9842b4a65b94bc5e19684d19e8a76072e Mon Sep 17 00:00:00 2001 From: bpatel Date: Mon, 12 Oct 2020 11:26:57 -0700 Subject: [PATCH 3/6] lookup errors and consistency --- baselines/previously_seen_cloud_regions_initial.yml | 4 ++-- baselines/previously_seen_cloud_regions_update.yml | 8 ++++---- ...te_instance_created_in_previously_unused_region.yml | 10 +++++++++- ...e_instance_created_with_previously_unseen_image.yml | 1 + lookups/previously_seen_cloud_regions.csv | 2 +- lookups/previously_seen_cloud_regions.yml | 2 +- 6 files changed, 18 insertions(+), 9 deletions(-) diff --git a/baselines/previously_seen_cloud_regions_initial.yml b/baselines/previously_seen_cloud_regions_initial.yml index 7683496fbc..46c342d63c 100644 --- a/baselines/previously_seen_cloud_regions_initial.yml +++ b/baselines/previously_seen_cloud_regions_initial.yml @@ -9,10 +9,10 @@ description: This search looks for cloud compute events where a compute instance how_to_implement: You must be ingesting the approrpiate cloud infrastructure logs and have the Security Research cloud data model installed. author: David Dorsey, Splunk -search: '| tstats earliest(_time) as firstTime, latest(_time) as lastTime from datamodel=Change +search: '| tstats earliest(_time) as firstTimeSeen, latest(_time) as lastTimeSeen from datamodel=Change where All_Changes.action=created by All_Changes.vendor_region | `drop_dm_object_name("All_Changes")` - | eventstats min(firstTime) as globalFirstTime + | eventstats min(firstTimeSeen) as globalFirstTime | eval enough_data = if(globalFirstTime <= relative_time(now(), "-14d@d"), 1, 0) | outputlookup previously_seen_cloud_regions | stats count' diff --git a/baselines/previously_seen_cloud_regions_update.yml b/baselines/previously_seen_cloud_regions_update.yml index c6af949d53..d56f977afc 100644 --- a/baselines/previously_seen_cloud_regions_update.yml +++ b/baselines/previously_seen_cloud_regions_update.yml @@ -9,12 +9,12 @@ description: This search looks for cloud compute events where a compute instance how_to_implement: You must be ingesting the approrpiate cloud infrastructure logs and have the Security Research cloud data model installed. author: David Dorsey, Splunk -search: '| tstats earliest(_time) as firstTime, latest(_time) as lastTime from datamodel=Change +search: '| tstats earliest(_time) as firstTimeSeen, latest(_time) as lastTimeSeen from datamodel=Change where All_Changes.action=created by All_Changes.vendor_region | `drop_dm_object_name("All_Changes")` | inputlookup append=t previously_seen_cloud_regions - | stats min(firstTime) as firstTime max(lastTime) as lastTime by vendor_region - | where lastTime > relative_time(now(), `previously_seen_cloud_region_forget_window`) - | eventstats min(firstTime) as globalFirstTime + | stats min(firstTimeSeen) as firstTimeSeen max(lastTimeSeen) as lastTimeSeen by vendor_region + | where lastTimeSeen > relative_time(now(), `previously_seen_cloud_region_forget_window`) + | eventstats min(firstTimeSeen) as globalFirstTime | eval enough_data = if(globalFirstTime <= relative_time(now(), "-14d@d"), 1, 0) | outputlookup previously_seen_cloud_regions | stats count' tags: diff --git a/detections/cloud_compute_instance_created_in_previously_unused_region.yml b/detections/cloud_compute_instance_created_in_previously_unused_region.yml index d03a8b4700..b9a12ca6e4 100644 --- a/detections/cloud_compute_instance_created_in_previously_unused_region.yml +++ b/detections/cloud_compute_instance_created_in_previously_unused_region.yml @@ -14,7 +14,15 @@ how_to_implement: "You must be ingesting your cloud infrastructure logs from you type: ESCU references: [] author: David Dorsey, Splunk -search: '| tstats earliest(_time) as firstTimeSeen, values(All_Changes.object_id) as dest, count from datamodel=Change where All_Changes.action=created by All_Changes.vendor_region, All_Changes.user | `drop_dm_object_name("All_Changes")`| lookup previously_seen_cloud_regions vendor_region as vendor_region OUTPUT firstTime | where isnull(firstTime) OR firstTime > relative_time(now(), "-70m@m") | `security_content_ctime(firstTimeSeen)`| table firstTimeSeen, user, vendor_region, count, dest | `cloud_compute_instance_created_in_previously_unused_region_filter`' +search: '| tstats earliest(_time) as firstTime latest(_time) as lastTime values(All_Changes.object_id) as dest, count from datamodel=Change where All_Changes.action=created by All_Changes.vendor_region, All_Changes.user +| `drop_dm_object_name("All_Changes")` | lookup previously_seen_cloud_regions vendor_region as vendor_region OUTPUTNEW firstTimeSeen, enough_data +| eventstats max(enough_data) as enough_data +| where enough_data=1 +| eval firstTimeSeenRegion=min(firstTimeSeen) +| where isnull(firstTimeSeenRegion) OR firstTimeSeenRegion > relative_time(now(), "-24h@h") +| table firstTime, user, dest, count +| `security_content_ctime(firstTime)` +| `cloud_compute_instance_created_in_previously_unused_region_filter`' known_false_positives: It's possible that a user has unknowingly started an instance in a new region. Please verify that this activity is legitimate. tags: diff --git a/detections/cloud_compute_instance_created_with_previously_unseen_image.yml b/detections/cloud_compute_instance_created_with_previously_unseen_image.yml index fea2a61368..5e6d6364da 100644 --- a/detections/cloud_compute_instance_created_with_previously_unseen_image.yml +++ b/detections/cloud_compute_instance_created_with_previously_unseen_image.yml @@ -18,6 +18,7 @@ search: '| tstats count earliest(_time) as firstTime, latest(_time) as lastTime by All_Changes.Instance_Changes.image_id, All_Changes.user | `drop_dm_object_name("All_Changes")` | `drop_dm_object_name("Instance_Changes")` +| where image_id != "unknown" | lookup previously_seen_cloud_compute_images image_id as image_id OUTPUT firstTimeSeen, enough_data | eventstats max(enough_data) as enough_data | where enough_data=1 diff --git a/lookups/previously_seen_cloud_regions.csv b/lookups/previously_seen_cloud_regions.csv index 91c87685c9..b209530df1 100644 --- a/lookups/previously_seen_cloud_regions.csv +++ b/lookups/previously_seen_cloud_regions.csv @@ -1 +1 @@ -firstTime,lastTime,region +firstTimeSeen, lastTimeSeen, vendor_region, enough_data \ No newline at end of file diff --git a/lookups/previously_seen_cloud_regions.yml b/lookups/previously_seen_cloud_regions.yml index 23e11f6ca4..02d536714b 100644 --- a/lookups/previously_seen_cloud_regions.yml +++ b/lookups/previously_seen_cloud_regions.yml @@ -1,4 +1,4 @@ description: A table of vendor_region values and the first and last time that they have been observed in cloud provisioning activities collection: previously_seen_cloud_regions name: previously_seen_cloud_regions -fields_list: _key, firstTime, lastTime, vendor_region, enough_data +fields_list: _key, firstTimeSeen, lastTimeSeen, vendor_region, enough_data From 97ac9c5f62f55532b5f07e3a4ace875256f66347 Mon Sep 17 00:00:00 2001 From: bpatel Date: Mon, 12 Oct 2020 11:49:37 -0700 Subject: [PATCH 4/6] spl updates and kvstore file --- ...n_cloud_compute_instance_types_initial.yml | 14 ++++++----- ...en_cloud_compute_instance_types_update.yml | 20 ++++++++-------- ...d_with_previously_unseen_instance_type.yml | 23 +++++++++++-------- ...usly_seen_cloud_compute_instance_types.csv | 1 - ...usly_seen_cloud_compute_instance_types.yml | 5 ++-- 5 files changed, 35 insertions(+), 28 deletions(-) delete mode 100644 lookups/previously_seen_cloud_compute_instance_types.csv diff --git a/baselines/previously_seen_cloud_compute_instance_types_initial.yml b/baselines/previously_seen_cloud_compute_instance_types_initial.yml index e7ddf35a1c..c4e5e2dbe7 100644 --- a/baselines/previously_seen_cloud_compute_instance_types_initial.yml +++ b/baselines/previously_seen_cloud_compute_instance_types_initial.yml @@ -7,12 +7,14 @@ description: This search builds a table of previously seen cloud compute instanc how_to_implement: You must be ingesting the approrpiate cloud infrastructure logs and have the Security Research cloud data model installed. author: David Dorsey, Splunk -search: '| tstats earliest(_time) as firstTime, latest(_time) as lastTime from datamodel=Change - where All_Changes.action=created `previously_seen_cloud_compute_instance_types_input_filter` - by All_Changes.Instance_Changes.instance_type | `drop_dm_object_name("All_Changes.Instance_Changes")` - | eventstats min(firstTime) as globalFirstTime - | eval enough_data = if(globalFirstTime <= relative_time(now(), "-14d@d"), 1, 0) - | outputlookup previously_seen_cloud_compute_instance_types | stats count' +search: '| tstats earliest(_time) as firstTimeSeen, latest(_time) as lastTimeSeen from datamodel=Change + where All_Changes.action=created + by All_Changes.Instance_Changes.instance_type +| `drop_dm_object_name("All_Changes.Instance_Changes")` +| where instance_type != "unknown" +| eventstats min(firstTimeSeen) as globalFirstTime +| eval enough_data = if(globalFirstTime <= relative_time(now(), "-14d@d"), 1, 0) +| outputlookup previously_seen_cloud_compute_instance_types' tags: analytics_story: - Cloud Cryptomining diff --git a/baselines/previously_seen_cloud_compute_instance_types_update.yml b/baselines/previously_seen_cloud_compute_instance_types_update.yml index 309648933c..6bb4ff4a78 100644 --- a/baselines/previously_seen_cloud_compute_instance_types_update.yml +++ b/baselines/previously_seen_cloud_compute_instance_types_update.yml @@ -6,15 +6,17 @@ description: This search builds a table of previously seen cloud compute instanc types how_to_implement: You must be ingesting the approrpiate cloud infrastructure logs author: David Dorsey, Splunk -search: '| tstats earliest(_time) as firstTime, latest(_time) as lastTime from datamodel=Change - where All_Changes.action=created `previously_seen_cloud_compute_instance_types_input_filter` - by All_Changes.Instance_Changes.instance_type | `drop_dm_object_name("All_Changes.Instance_Changes")` - | inputlookup append=t previously_seen_cloud_compute_instance_types - | stats min(firstTime) as firstTime max(lastTime) as lastTime by instance_type - | where lastTime > relative_time(now(), `previously_seen_cloud_compute_instance_type_forget_window`) - | eventstats min(firstTime) as globalFirstTime - | eval enough_data = if(globalFirstTime <= relative_time(now(), "-14d@d"), 1, 0) - | outputlookup previously_seen_cloud_compute_instance_types | stats count' +search: '| tstats earliest(_time) as firstTimeSeen, latest(_time) as lastTimeSeen from datamodel=Change + where All_Changes.action=created + by All_Changes.Instance_Changes.instance_type +| `drop_dm_object_name("All_Changes.Instance_Changes")` +| where instance_type != "unknown" +| inputlookup append=t previously_seen_cloud_compute_instance_types +| stats min(firstTimeSeen) as firstTimeSeen max(lastTimeSeen) as lastTimeSeen by instance_type +| where lastTimeSeen > relative_time(now(), `previously_seen_cloud_compute_instance_type_forget_window`) +| eventstats min(firstTimeSeen) as globalFirstTime +| eval enough_data = if(globalFirstTime <= relative_time(now(), "-14d@d"), 1, 0) +| outputlookup previously_seen_cloud_compute_instance_types' tags: analytics_story: - Cloud Cryptomining diff --git a/detections/cloud_compute_instance_created_with_previously_unseen_instance_type.yml b/detections/cloud_compute_instance_created_with_previously_unseen_instance_type.yml index 716181e6e4..86fad5692c 100644 --- a/detections/cloud_compute_instance_created_with_previously_unseen_instance_type.yml +++ b/detections/cloud_compute_instance_created_with_previously_unseen_instance_type.yml @@ -12,15 +12,20 @@ how_to_implement: "You must be ingesting your cloud infrastructure logs from you type: ESCU references: [] author: David Dorsey, Splunk -search: '| tstats earliest(_time) as firstTimeSeen, latest(_time) as lastTimeSeen values(All_Changes.object_id) as dest, count - from datamodel=Change where All_Changes.action=created `previously_seen_cloud_compute_instance_types_input_filter` - by All_Changes.Instance_Changes.instance_type, All_Changes.user - | `drop_dm_object_name("All_Changes")` | `drop_dm_object_name("Instance_Changes")` - | lookup previously_seen_cloud_compute_instance_types instance_type as instance_type OUTPUT firstTime - | where isnull(firstTime) OR firstTime > relative_time(now(), "-70m@m") - | `security_content_ctime(firstTimeSeen)` - | table firstTimeSeen, user, instance_type, count, dest - | `cloud_compute_instance_created_with_previously_unseen_instance_type_filter`' +search: '| tstats earliest(_time) as firstTime, latest(_time) as lastTime values(All_Changes.object_id) as dest, count + from datamodel=Change where All_Changes.action=created + by All_Changes.Instance_Changes.instance_type, All_Changes.user +| `drop_dm_object_name("All_Changes")` +| `drop_dm_object_name("Instance_Changes")` +| where instance_type != "unknown" +| lookup previously_seen_cloud_compute_instance_types instance_type as instance_type OUTPUTNEW firstTimeSeen, enough_data +| eventstats max(enough_data) as enough_data +| where enough_data=1 +| eval firstTimeSeenInstanceType=min(firstTimeSeen) +| where isnull(firstTimeSeenInstanceType) OR firstTimeSeenInstanceType > relative_time(now(), "-24h@h") +| table firstTime, user, dest, count, instance_type +| `security_content_ctime(firstTime)` +| `cloud_compute_instance_created_with_previously_unseen_instance_type_filter`' known_false_positives: It is possible that an admin will create a new system using a new instance type that has never been used before. Verify with the creator that they intended to create the system with the new instance type. diff --git a/lookups/previously_seen_cloud_compute_instance_types.csv b/lookups/previously_seen_cloud_compute_instance_types.csv deleted file mode 100644 index 8683fd2023..0000000000 --- a/lookups/previously_seen_cloud_compute_instance_types.csv +++ /dev/null @@ -1 +0,0 @@ -firstTime,lastTime,instance_type diff --git a/lookups/previously_seen_cloud_compute_instance_types.yml b/lookups/previously_seen_cloud_compute_instance_types.yml index e7b9fd088a..d6529c41ef 100644 --- a/lookups/previously_seen_cloud_compute_instance_types.yml +++ b/lookups/previously_seen_cloud_compute_instance_types.yml @@ -1,5 +1,4 @@ -default_match: 'false' description: A place holder for a list of used cloud compute instance types -filename: previously_seen_cloud_compute_instance_types.csv -min_matches: 1 +collection: previously_seen_cloud_compute_instance_types name: previously_seen_cloud_compute_instance_types +fields_list: _key, firstTimeSeen, lastTimeSeen, instance_type, enough_data From abac1c8b6243368006d537aed5bad71375adef6d Mon Sep 17 00:00:00 2001 From: bpatel Date: Mon, 12 Oct 2020 12:14:50 -0700 Subject: [PATCH 5/6] convert to kvstore and spl updates --- ...loud_compute_creations_by_user_initial.yml | 11 +++++----- ...cloud_compute_creations_by_user_update.yml | 15 ++++++++------ ...ance_created_by_previously_unseen_user.yml | 20 +++++++++++-------- ...ce_created_in_previously_unused_region.yml | 2 +- ...y_seen_cloud_compute_creations_by_user.csv | 1 - ...y_seen_cloud_compute_creations_by_user.yml | 4 ++-- .../previously_seen_cloud_compute_images.csv | 1 - .../previously_seen_cloud_compute_images.yml | 7 +++---- 8 files changed, 33 insertions(+), 28 deletions(-) delete mode 100644 lookups/previously_seen_cloud_compute_creations_by_user.csv delete mode 100644 lookups/previously_seen_cloud_compute_images.csv diff --git a/baselines/previously_seen_cloud_compute_creations_by_user_initial.yml b/baselines/previously_seen_cloud_compute_creations_by_user_initial.yml index c70cf86cae..ebe57d3de7 100644 --- a/baselines/previously_seen_cloud_compute_creations_by_user_initial.yml +++ b/baselines/previously_seen_cloud_compute_creations_by_user_initial.yml @@ -7,11 +7,12 @@ description: This search builds a table of previously seen users that have launc how_to_implement: You must be ingesting the approrpiate cloud infrastructure logs and have the proper TAs installed. author: Rico Valdez, Splunk -search: '| tstats earliest(_time) as firstTime, latest(_time) as lastTime from datamodel=Change - where All_Changes.action=created AND All_Changes.object_category=instance - by All_Changes.user - | `drop_dm_object_name("All_Changes")` | outputlookup previously_seen_cloud_compute_creations_by_user - | stats count' +search: '| tstats earliest(_time) as firstTimeSeen, latest(_time) as lastTimeSeen from datamodel=Change + where All_Changes.action=created AND All_Changes.object_category=instance + by All_Changes.user +| `drop_dm_object_name("All_Changes")` +| outputlookup previously_seen_cloud_compute_creations_by_user +| stats count' tags: analytics_story: - Cloud Cryptomining diff --git a/baselines/previously_seen_cloud_compute_creations_by_user_update.yml b/baselines/previously_seen_cloud_compute_creations_by_user_update.yml index 718a74548c..a162e686ec 100644 --- a/baselines/previously_seen_cloud_compute_creations_by_user_update.yml +++ b/baselines/previously_seen_cloud_compute_creations_by_user_update.yml @@ -7,12 +7,15 @@ description: This search builds a table of previously seen users that have launc how_to_implement: You must be ingesting the approrpiate cloud infrastructure logs and have the proper TAs installed. author: Rico Valdez, Splunk -search: '| tstats earliest(_time) as firstTime, latest(_time) as lastTime from datamodel=Change - where All_Changes.action=created AND All_Changes.object_category=instance - by All_Changes.user - | `drop_dm_object_name("All_Changes")` | inputlookup append=t previously_seen_cloud_compute_creations_by_user - | stats min(firstTime) as firstTime max(lastTime) as lastTime by user - | outputlookup previously_seen_cloud_compute_creations_by_user | stats count' +search: '| tstats earliest(_time) as firstTimeSeen, latest(_time) as lastTimeSeen from datamodel=Change + where All_Changes.action=created AND All_Changes.object_category=instance + by All_Changes.user| `drop_dm_object_name("All_Changes")` +| inputlookup append=t previously_seen_cloud_compute_creations_by_user +| stats min(firstTimeSeen) as firstTimeSeen max(lastTimeSeen) as lastTimeSeen by user +| where lastTimeSeen > relative_time(now(), "-24h@h") +| eventstats min(firstTimeSeen) as globalFirstTime +| eval enough_data = if(globalFirstTime <= relative_time(now(), "-7d@d"), 1, 0) +| outputlookup previously_seen_cloud_compute_creations_by_user' tags: analytics_story: - Cloud Cryptomining diff --git a/detections/cloud_compute_instance_created_by_previously_unseen_user.yml b/detections/cloud_compute_instance_created_by_previously_unseen_user.yml index b6eb707671..69f116a25f 100644 --- a/detections/cloud_compute_instance_created_by_previously_unseen_user.yml +++ b/detections/cloud_compute_instance_created_by_previously_unseen_user.yml @@ -10,14 +10,18 @@ how_to_implement: You must be ingesting the appropriate cloud-infrastructure log type: ESCU references: [] author: Rico Valdez, Splunk -search: '| tstats `security_content_summariesonly` earliest(_time) as firstTimeSeen, latest(_time) - as lastTimeSeen values(All_Changes.object) as dest from datamodel=Change - where All_Changes.action=created by All_Changes.user | `drop_dm_object_name("All_Changes")` - | lookup previously_seen_cloud_compute_creations_by_user user as user OUTPUT firstTime - | where isnull(firstTime) OR firstTime > relative_time(now(), "-70m@m") - | `security_content_ctime(firstTimeSeen)` - | table firstTimeSeen, user, dest - | `cloud_compute_instance_created_by_previously_unseen_user_filter`' +search: '| tstats `security_content_summariesonly` earliest(_time) as firstTime, latest(_time) + as lastTime values(All_Changes.object) as dest from datamodel=Change + where All_Changes.action=created by All_Changes.user +| `drop_dm_object_name("All_Changes")` +| lookup previously_seen_cloud_compute_creations_by_user user as user OUTPUTNEW firstTimeSeen, enough_data +| eventstats max(enough_data) as enough_data +| where enough_data=1 +| eval firstTimeSeenUser=min(firstTimeSeen) +| where isnull(firstTimeSeenUser) OR firstTimeSeenUser > relative_time(now(), "-24h@h") +| table firstTime, user, dest, count vendor_region +| `security_content_ctime(firstTime)` +| `cloud_compute_instance_created_by_previously_unseen_user_filter`' known_false_positives: It's possible that a user will start to create compute instances for the first time, for any number of reasons. Verify with the user launching instances that this is the intended behavior. diff --git a/detections/cloud_compute_instance_created_in_previously_unused_region.yml b/detections/cloud_compute_instance_created_in_previously_unused_region.yml index b9a12ca6e4..0a39901b27 100644 --- a/detections/cloud_compute_instance_created_in_previously_unused_region.yml +++ b/detections/cloud_compute_instance_created_in_previously_unused_region.yml @@ -20,7 +20,7 @@ search: '| tstats earliest(_time) as firstTime latest(_time) as lastTime values( | where enough_data=1 | eval firstTimeSeenRegion=min(firstTimeSeen) | where isnull(firstTimeSeenRegion) OR firstTimeSeenRegion > relative_time(now(), "-24h@h") -| table firstTime, user, dest, count +| table firstTime, user, dest, count , vendor_region | `security_content_ctime(firstTime)` | `cloud_compute_instance_created_in_previously_unused_region_filter`' known_false_positives: It's possible that a user has unknowingly started an instance diff --git a/lookups/previously_seen_cloud_compute_creations_by_user.csv b/lookups/previously_seen_cloud_compute_creations_by_user.csv deleted file mode 100644 index 2b729b02a5..0000000000 --- a/lookups/previously_seen_cloud_compute_creations_by_user.csv +++ /dev/null @@ -1 +0,0 @@ -firstTime,lastTime,src_user diff --git a/lookups/previously_seen_cloud_compute_creations_by_user.yml b/lookups/previously_seen_cloud_compute_creations_by_user.yml index fcf301c0cd..1e862ec71e 100644 --- a/lookups/previously_seen_cloud_compute_creations_by_user.yml +++ b/lookups/previously_seen_cloud_compute_creations_by_user.yml @@ -1,4 +1,4 @@ -description: A table of source IPs, geographic locations, and the first and last time that they have that done cloud provisioning activities +description: A table of previously seen users creating cloud instances collection: previously_seen_cloud_compute_creations_by_user name: previously_seen_cloud_compute_creations_by_user -fields_list: _key, firstTime, lastTime, user, enough_data +fields_list: _key, firstTimeSeen, lastTimeSeen, user, enough_data diff --git a/lookups/previously_seen_cloud_compute_images.csv b/lookups/previously_seen_cloud_compute_images.csv deleted file mode 100644 index 8ac3d8191f..0000000000 --- a/lookups/previously_seen_cloud_compute_images.csv +++ /dev/null @@ -1 +0,0 @@ -firstTime,lastTime,image_id diff --git a/lookups/previously_seen_cloud_compute_images.yml b/lookups/previously_seen_cloud_compute_images.yml index 140660caec..a0d0628d02 100644 --- a/lookups/previously_seen_cloud_compute_images.yml +++ b/lookups/previously_seen_cloud_compute_images.yml @@ -1,5 +1,4 @@ -default_match: 'false' -description: A place holder for a list of used cloud compute images -filename: previously_seen_cloud_compute_images.csv -min_matches: 1 +description: A table of previously seen Cloud image IDs +collection: previously_seen_cloud_compute_images name: previously_seen_cloud_compute_images +fields_list: _key, firstTimeSeen, lastTimeSeen, image_id, enough_data From dac1086b0400f38fb8af8f7656e872d1e599fcf1 Mon Sep 17 00:00:00 2001 From: bpatel Date: Mon, 12 Oct 2020 12:37:05 -0700 Subject: [PATCH 6/6] spl bugs - fully tested --- ...previously_seen_cloud_compute_creations_by_user_update.yml | 2 +- ...oud_compute_instance_created_by_previously_unseen_user.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/baselines/previously_seen_cloud_compute_creations_by_user_update.yml b/baselines/previously_seen_cloud_compute_creations_by_user_update.yml index a162e686ec..adb335345d 100644 --- a/baselines/previously_seen_cloud_compute_creations_by_user_update.yml +++ b/baselines/previously_seen_cloud_compute_creations_by_user_update.yml @@ -12,7 +12,7 @@ search: '| tstats earliest(_time) as firstTimeSeen, latest(_time) as lastTimeSee by All_Changes.user| `drop_dm_object_name("All_Changes")` | inputlookup append=t previously_seen_cloud_compute_creations_by_user | stats min(firstTimeSeen) as firstTimeSeen max(lastTimeSeen) as lastTimeSeen by user -| where lastTimeSeen > relative_time(now(), "-24h@h") +| where lastTimeSeen > relative_time(now(), "-90d@d") | eventstats min(firstTimeSeen) as globalFirstTime | eval enough_data = if(globalFirstTime <= relative_time(now(), "-7d@d"), 1, 0) | outputlookup previously_seen_cloud_compute_creations_by_user' diff --git a/detections/cloud_compute_instance_created_by_previously_unseen_user.yml b/detections/cloud_compute_instance_created_by_previously_unseen_user.yml index 69f116a25f..1f7e763877 100644 --- a/detections/cloud_compute_instance_created_by_previously_unseen_user.yml +++ b/detections/cloud_compute_instance_created_by_previously_unseen_user.yml @@ -10,9 +10,9 @@ how_to_implement: You must be ingesting the appropriate cloud-infrastructure log type: ESCU references: [] author: Rico Valdez, Splunk -search: '| tstats `security_content_summariesonly` earliest(_time) as firstTime, latest(_time) +search: '| tstats `security_content_summariesonly` count earliest(_time) as firstTime, latest(_time) as lastTime values(All_Changes.object) as dest from datamodel=Change - where All_Changes.action=created by All_Changes.user + where All_Changes.action=created by All_Changes.user All_Changes.vendor_region | `drop_dm_object_name("All_Changes")` | lookup previously_seen_cloud_compute_creations_by_user user as user OUTPUTNEW firstTimeSeen, enough_data | eventstats max(enough_data) as enough_data