From 30ad4d6eab9a53c194a9a80740bfce43aac0d063 Mon Sep 17 00:00:00 2001 From: patel-bhavin Date: Mon, 22 May 2023 17:35:13 -0700 Subject: [PATCH 1/4] only kvstore conversion --- ...seen_aws_provisioning_activity_sources.yml | 2 +- .../deprecated/previously_seen_ec2_amis.yml | 2 +- .../previously_seen_ec2_instance_types.yml | 2 +- .../previously_seen_ec2_launches_by_user.yml | 2 +- ...en_aws_cross_account_activity___update.yml | 3 +-- baselines/previously_seen_aws_regions.yml | 2 +- .../adapter/obj_to_conf_adapter.py | 6 +++++- .../adapter/templates/collections.j2 | 1 + .../cloud/detect_s3_access_from_a_new_ip.yml | 4 ++-- .../baseline_blocked_outbound_connections.csv | 1 - dist/escu/lookups/escu_search_id.csv | 1 - .../lookups/network_acl_activity_baseline.csv | 1 - ...eviously_seen_S3_access_from_remote_ip.csv | 1 - ...viously_seen_api_calls_from_user_roles.csv | 1 - ...iously_seen_aws_cross_account_activity.csv | 1 - .../lookups/previously_seen_aws_regions.csv | 1 - .../previously_seen_cmd_line_arguments.csv | 1 - ...viously_seen_ec2_modifications_by_user.csv | 1 - ...seen_gcp_storage_access_from_remote_ip.csv | 1 - dist/escu/lookups/s3_deletion_baseline.csv | 1 - .../security_group_activity_baseline.csv | 1 - lookups/csc_lookup.csv | 21 ------------------- lookups/csc_lookup.yml | 4 ---- lookups/escu_search_id.csv | 1 - lookups/escu_search_id_lookup.yml | 3 --- ...eviously_seen_S3_access_from_remote_ip.csv | 1 - ...eviously_seen_S3_access_from_remote_ip.yml | 3 ++- ...viously_seen_api_calls_from_user_roles.csv | 1 - ...viously_seen_api_calls_from_user_roles.yml | 7 ++++--- ...iously_seen_aws_cross_account_activity.csv | 1 - ...iously_seen_aws_cross_account_activity.yml | 3 ++- lookups/previously_seen_aws_regions.csv | 1 - lookups/previously_seen_aws_regions.yml | 5 ++--- ...seen_gcp_storage_access_from_remote_ip.csv | 1 - ...seen_gcp_storage_access_from_remote_ip.yml | 7 +++---- lookups/s3_deletion_baseline.csv | 1 - lookups/s3_deletion_baseline.yml | 3 ++- lookups/security_group_activity_baseline.csv | 1 - lookups/security_group_activity_baseline.yml | 3 ++- lookups/uncommon_processes_default.csv | 9 -------- lookups/uncommon_processes_local.csv | 1 - 41 files changed, 31 insertions(+), 82 deletions(-) delete mode 100644 dist/escu/lookups/baseline_blocked_outbound_connections.csv delete mode 100644 dist/escu/lookups/escu_search_id.csv delete mode 100644 dist/escu/lookups/network_acl_activity_baseline.csv delete mode 100644 dist/escu/lookups/previously_seen_S3_access_from_remote_ip.csv delete mode 100644 dist/escu/lookups/previously_seen_api_calls_from_user_roles.csv delete mode 100644 dist/escu/lookups/previously_seen_aws_cross_account_activity.csv delete mode 100644 dist/escu/lookups/previously_seen_aws_regions.csv delete mode 100644 dist/escu/lookups/previously_seen_cmd_line_arguments.csv delete mode 100644 dist/escu/lookups/previously_seen_ec2_modifications_by_user.csv delete mode 100644 dist/escu/lookups/previously_seen_gcp_storage_access_from_remote_ip.csv delete mode 100644 dist/escu/lookups/s3_deletion_baseline.csv delete mode 100644 dist/escu/lookups/security_group_activity_baseline.csv delete mode 100644 lookups/csc_lookup.csv delete mode 100644 lookups/csc_lookup.yml delete mode 100644 lookups/escu_search_id.csv delete mode 100644 lookups/escu_search_id_lookup.yml delete mode 100644 lookups/previously_seen_S3_access_from_remote_ip.csv delete mode 100644 lookups/previously_seen_api_calls_from_user_roles.csv delete mode 100644 lookups/previously_seen_aws_cross_account_activity.csv delete mode 100644 lookups/previously_seen_aws_regions.csv delete mode 100644 lookups/previously_seen_gcp_storage_access_from_remote_ip.csv delete mode 100644 lookups/s3_deletion_baseline.csv delete mode 100644 lookups/security_group_activity_baseline.csv delete mode 100644 lookups/uncommon_processes_default.csv delete mode 100644 lookups/uncommon_processes_local.csv diff --git a/baselines/deprecated/previously_seen_aws_provisioning_activity_sources.yml b/baselines/deprecated/previously_seen_aws_provisioning_activity_sources.yml index 1b62095d7d..9c977eb85c 100644 --- a/baselines/deprecated/previously_seen_aws_provisioning_activity_sources.yml +++ b/baselines/deprecated/previously_seen_aws_provisioning_activity_sources.yml @@ -10,7 +10,7 @@ description: This search builds a table of the first and last times seen for eve activity. This is broadly defined as any event that runs or creates something. search: '`cloudtrail` (eventName=Run* OR eventName=Create*) | iplocation sourceIPAddress | stats earliest(_time) as firstTime, latest(_time) as lastTime by sourceIPAddress, - City, Region, Country | outputlookup previously_seen_provisioning_activity_src.csv + City, Region, Country | outputlookup previously_seen_provisioning_activity_src | stats count' how_to_implement: You must install the AWS App for Splunk (version 5.1.0 or later) and Splunk Add-on for AWS (version 4.4.0 or later), then configure your CloudTrail diff --git a/baselines/deprecated/previously_seen_ec2_amis.yml b/baselines/deprecated/previously_seen_ec2_amis.yml index 2d4db54842..7c07f5b16c 100644 --- a/baselines/deprecated/previously_seen_ec2_amis.yml +++ b/baselines/deprecated/previously_seen_ec2_amis.yml @@ -9,7 +9,7 @@ description: This search builds a table of previously seen AMIs used to launch E instances search: '`cloudtrail` eventName=RunInstances errorCode=success | rename requestParameters.instancesSet.items{}.imageId as amiID | stats earliest(_time) as firstTime latest(_time) as lastTime by amiID - | outputlookup previously_seen_ec2_amis.csv | stats count' + | outputlookup previously_seen_ec2_amis | stats count' how_to_implement: You must install the AWS App for Splunk (version 5.1.0 or later) and Splunk Add-on for AWS version (4.4.0 or later), then configure your CloudTrail inputs. diff --git a/baselines/deprecated/previously_seen_ec2_instance_types.yml b/baselines/deprecated/previously_seen_ec2_instance_types.yml index 07f828a99f..caa4874b07 100644 --- a/baselines/deprecated/previously_seen_ec2_instance_types.yml +++ b/baselines/deprecated/previously_seen_ec2_instance_types.yml @@ -8,7 +8,7 @@ datamodel: [] description: This search builds a table of previously seen EC2 instance types search: '`cloudtrail` eventName=RunInstances errorCode=success | rename requestParameters.instanceType as instanceType | fillnull value="m1.small" instanceType | stats earliest(_time) - as earliest latest(_time) as latest by instanceType | outputlookup previously_seen_ec2_instance_types.csv + as earliest latest(_time) as latest by instanceType | outputlookup previously_seen_ec2_instance_types | stats count' how_to_implement: You must install the AWS App for Splunk (version 5.1.0 or later) and Splunk Add-on for AWS version (4.4.0 or later), then configure your CloudTrail diff --git a/baselines/deprecated/previously_seen_ec2_launches_by_user.yml b/baselines/deprecated/previously_seen_ec2_launches_by_user.yml index 8593df9832..b9055ec06d 100644 --- a/baselines/deprecated/previously_seen_ec2_launches_by_user.yml +++ b/baselines/deprecated/previously_seen_ec2_launches_by_user.yml @@ -9,7 +9,7 @@ description: This search builds a table of previously seen ARNs that have launch a EC2 instance. search: '`cloudtrail` eventName=RunInstances errorCode=success | rename userIdentity.arn as arn | stats earliest(_time) as firstTime latest(_time) as lastTime by arn | outputlookup - previously_seen_ec2_launches_by_user.csv | stats count' + previously_seen_ec2_launches_by_user | stats count' how_to_implement: You must install the AWS App for Splunk (version 5.1.0 or later) and Splunk Add-on for AWS version (4.4.0 or later), then configure your CloudTrail inputs. diff --git a/baselines/previously_seen_aws_cross_account_activity___update.yml b/baselines/previously_seen_aws_cross_account_activity___update.yml index dfc483e769..c067f69b15 100644 --- a/baselines/previously_seen_aws_cross_account_activity___update.yml +++ b/baselines/previously_seen_aws_cross_account_activity___update.yml @@ -20,8 +20,7 @@ search: '| tstats earliest(_time) as firstTime latest(_time) as lastTime from da how_to_implement: You must install and configure the Splunk Add-on for AWS (version 5.1.0 or later) and Enterprise Security 6.2, which contains the required updates to the Authentication data model for cloud use cases. Validate the user name entries - in `previously_seen_aws_cross_account_activity.csv`, a lookup file created by this - support search. + in `previously_seen_aws_cross_account_activity` kvstore known_false_positives: none references: [] tags: diff --git a/baselines/previously_seen_aws_regions.yml b/baselines/previously_seen_aws_regions.yml index 39109032a1..81db14b95d 100644 --- a/baselines/previously_seen_aws_regions.yml +++ b/baselines/previously_seen_aws_regions.yml @@ -10,7 +10,7 @@ description: This search looks for CloudTrail events where an AWS instance is st we've seen this region in our dataset grouped by the value awsRegion for the last 30 days search: '`cloudtrail` StartInstances | stats earliest(_time) as earliest latest(_time) - as latest by awsRegion | outputlookup previously_seen_aws_regions.csv | stats count' + as latest by awsRegion | outputlookup previously_seen_aws_regions| stats count' how_to_implement: You must install the AWS App for Splunk (version 5.1.0 or later) and Splunk Add-on for AWS version (4.4.0 or later), then configure your CloudTrail inputs. diff --git a/bin/contentctl_project/contentctl_infrastructure/adapter/obj_to_conf_adapter.py b/bin/contentctl_project/contentctl_infrastructure/adapter/obj_to_conf_adapter.py index f9f5a37cdb..3b8350a4f8 100644 --- a/bin/contentctl_project/contentctl_infrastructure/adapter/obj_to_conf_adapter.py +++ b/bin/contentctl_project/contentctl_infrastructure/adapter/obj_to_conf_adapter.py @@ -97,9 +97,13 @@ class ObjToConfAdapter(Adapter): for file in files: if os.path.isfile(file): shutil.copy(file, os.path.join(output_path, 'lookups')) + + files = glob.iglob(os.path.join(self.input_path, 'lookups', '*.mlmodel')) + for file in files: + if os.path.isfile(file): + shutil.copy(file, os.path.join(output_path, 'lookups')) elif type == SecurityContentType.macros: ConfWriter.writeConfFile('macros.j2', os.path.join(output_path, 'default/macros.conf'), objects) - diff --git a/bin/contentctl_project/contentctl_infrastructure/adapter/templates/collections.j2 b/bin/contentctl_project/contentctl_infrastructure/adapter/templates/collections.j2 index 349f21770a..06e491406a 100644 --- a/bin/contentctl_project/contentctl_infrastructure/adapter/templates/collections.j2 +++ b/bin/contentctl_project/contentctl_infrastructure/adapter/templates/collections.j2 @@ -4,5 +4,6 @@ [{{ lookup.name }}] enforceTypes = false replicate = false + {% endif %} {% endfor %} \ No newline at end of file diff --git a/detections/cloud/detect_s3_access_from_a_new_ip.yml b/detections/cloud/detect_s3_access_from_a_new_ip.yml index f7bcb2a934..4a974e398a 100644 --- a/detections/cloud/detect_s3_access_from_a_new_ip.yml +++ b/detections/cloud/detect_s3_access_from_a_new_ip.yml @@ -10,9 +10,9 @@ description: This search looks at S3 bucket-access logs and detects new or previ data_source: [] search: '`aws_s3_accesslogs` http_status=200 [search `aws_s3_accesslogs` http_status=200 | stats earliest(_time) as firstTime latest(_time) as lastTime by bucket_name remote_ip - | inputlookup append=t previously_seen_S3_access_from_remote_ip.csv | stats min(firstTime) + | inputlookup append=t previously_seen_S3_access_from_remote_ip | stats min(firstTime) as firstTime, max(lastTime) as lastTime by bucket_name remote_ip | outputlookup - previously_seen_S3_access_from_remote_ip.csv | eval newIP=if(firstTime >= relative_time(now(), + previously_seen_S3_access_from_remote_ip| eval newIP=if(firstTime >= relative_time(now(), "-70m@m"), 1, 0) | where newIP=1 | `security_content_ctime(firstTime)`| `security_content_ctime(lastTime)` | table bucket_name remote_ip]| iplocation remote_ip |rename remote_ip as src_ip | table _time bucket_name src_ip City Country operation request_uri | `detect_s3_access_from_a_new_ip_filter`' diff --git a/dist/escu/lookups/baseline_blocked_outbound_connections.csv b/dist/escu/lookups/baseline_blocked_outbound_connections.csv deleted file mode 100644 index da66bfd95c..0000000000 --- a/dist/escu/lookups/baseline_blocked_outbound_connections.csv +++ /dev/null @@ -1 +0,0 @@ -src_ip,numDataPoints,latestCount,avgBlockedConnections,stdevBlockedConnections \ No newline at end of file diff --git a/dist/escu/lookups/escu_search_id.csv b/dist/escu/lookups/escu_search_id.csv deleted file mode 100644 index 08e997f06b..0000000000 --- a/dist/escu/lookups/escu_search_id.csv +++ /dev/null @@ -1 +0,0 @@ -savedsearch_name, search_id, user, _time, usage diff --git a/dist/escu/lookups/network_acl_activity_baseline.csv b/dist/escu/lookups/network_acl_activity_baseline.csv deleted file mode 100644 index 4fec3cadd9..0000000000 --- a/dist/escu/lookups/network_acl_activity_baseline.csv +++ /dev/null @@ -1 +0,0 @@ -arn,latestCount,numDataPoints,avgApiCalls,stdevApiCalls diff --git a/dist/escu/lookups/previously_seen_S3_access_from_remote_ip.csv b/dist/escu/lookups/previously_seen_S3_access_from_remote_ip.csv deleted file mode 100644 index 7f64ce933f..0000000000 --- a/dist/escu/lookups/previously_seen_S3_access_from_remote_ip.csv +++ /dev/null @@ -1 +0,0 @@ -bucket_name,remote_ip,earliest,latest \ No newline at end of file diff --git a/dist/escu/lookups/previously_seen_api_calls_from_user_roles.csv b/dist/escu/lookups/previously_seen_api_calls_from_user_roles.csv deleted file mode 100644 index 16491b9d75..0000000000 --- a/dist/escu/lookups/previously_seen_api_calls_from_user_roles.csv +++ /dev/null @@ -1 +0,0 @@ -earliest,latest,userName,eventName diff --git a/dist/escu/lookups/previously_seen_aws_cross_account_activity.csv b/dist/escu/lookups/previously_seen_aws_cross_account_activity.csv deleted file mode 100644 index 30a8931b74..0000000000 --- a/dist/escu/lookups/previously_seen_aws_cross_account_activity.csv +++ /dev/null @@ -1 +0,0 @@ -firstTime,lastTime,requestingAccountId,requestedAccountId diff --git a/dist/escu/lookups/previously_seen_aws_regions.csv b/dist/escu/lookups/previously_seen_aws_regions.csv deleted file mode 100644 index 82335d2260..0000000000 --- a/dist/escu/lookups/previously_seen_aws_regions.csv +++ /dev/null @@ -1 +0,0 @@ -earliest,latest,awsRegion diff --git a/dist/escu/lookups/previously_seen_cmd_line_arguments.csv b/dist/escu/lookups/previously_seen_cmd_line_arguments.csv deleted file mode 100644 index ee6ce88f4c..0000000000 --- a/dist/escu/lookups/previously_seen_cmd_line_arguments.csv +++ /dev/null @@ -1 +0,0 @@ -firstTime,lastTime,process diff --git a/dist/escu/lookups/previously_seen_ec2_modifications_by_user.csv b/dist/escu/lookups/previously_seen_ec2_modifications_by_user.csv deleted file mode 100644 index 225fcaa19a..0000000000 --- a/dist/escu/lookups/previously_seen_ec2_modifications_by_user.csv +++ /dev/null @@ -1 +0,0 @@ -arn,firstTime,lastTime diff --git a/dist/escu/lookups/previously_seen_gcp_storage_access_from_remote_ip.csv b/dist/escu/lookups/previously_seen_gcp_storage_access_from_remote_ip.csv deleted file mode 100644 index 8a5c209fa3..0000000000 --- a/dist/escu/lookups/previously_seen_gcp_storage_access_from_remote_ip.csv +++ /dev/null @@ -1 +0,0 @@ -firstTime, lastTime, bucket_name, remote_ip, operation, request_uri \ No newline at end of file diff --git a/dist/escu/lookups/s3_deletion_baseline.csv b/dist/escu/lookups/s3_deletion_baseline.csv deleted file mode 100644 index 3f0d026011..0000000000 --- a/dist/escu/lookups/s3_deletion_baseline.csv +++ /dev/null @@ -1 +0,0 @@ -arn, latestCount, numDataPoints, avgApiCalls, stdevApiCalls \ No newline at end of file diff --git a/dist/escu/lookups/security_group_activity_baseline.csv b/dist/escu/lookups/security_group_activity_baseline.csv deleted file mode 100644 index 4fec3cadd9..0000000000 --- a/dist/escu/lookups/security_group_activity_baseline.csv +++ /dev/null @@ -1 +0,0 @@ -arn,latestCount,numDataPoints,avgApiCalls,stdevApiCalls diff --git a/lookups/csc_lookup.csv b/lookups/csc_lookup.csv deleted file mode 100644 index 58970a91f6..0000000000 --- a/lookups/csc_lookup.csv +++ /dev/null @@ -1,21 +0,0 @@ -number, name -1, Inventory of Authorized and Unauthorized Devices -2, Inventory of Authorized and Unauthorized Software -3, Secure Configuration of End-User Devices -4, Continuous Vulnerability Assessment & Remediation -5, Controlled Use of Administrative Privileges -6, Maintenance Monitoring and Analysis of Audit Logs -7, Email & Web Browser Protections -8, Malware Defense -9, Limitation & Control of Network Ports-Protocols & Services -10, Data Recovery Capability -11, Secure Configuration of Network Devices -12, Boundary Defense -13, Data Protection -14, Controlled Access Based on Need to Know -15, Wireless Access Control -16, Account Monitoring and Control -17, Security Skills Assessment and Appropriate Training -18, Application Software Security -19, Incident Response and Management -20, Penetration Tests and Red Team Exercises \ No newline at end of file diff --git a/lookups/csc_lookup.yml b/lookups/csc_lookup.yml deleted file mode 100644 index a9614d622e..0000000000 --- a/lookups/csc_lookup.yml +++ /dev/null @@ -1,4 +0,0 @@ -description: The CSC control numbers and names -filename: csc_lookup.csv -min_matches: 1 -name: csc_lookup diff --git a/lookups/escu_search_id.csv b/lookups/escu_search_id.csv deleted file mode 100644 index 08e997f06b..0000000000 --- a/lookups/escu_search_id.csv +++ /dev/null @@ -1 +0,0 @@ -savedsearch_name, search_id, user, _time, usage diff --git a/lookups/escu_search_id_lookup.yml b/lookups/escu_search_id_lookup.yml deleted file mode 100644 index 78f6fd9653..0000000000 --- a/lookups/escu_search_id_lookup.yml +++ /dev/null @@ -1,3 +0,0 @@ -description: A placeholder lookup file to hold information for ESCU Usage dashboard -filename: escu_search_id.csv -name: escu_search_id_lookup diff --git a/lookups/previously_seen_S3_access_from_remote_ip.csv b/lookups/previously_seen_S3_access_from_remote_ip.csv deleted file mode 100644 index 7f64ce933f..0000000000 --- a/lookups/previously_seen_S3_access_from_remote_ip.csv +++ /dev/null @@ -1 +0,0 @@ -bucket_name,remote_ip,earliest,latest \ No newline at end of file diff --git a/lookups/previously_seen_S3_access_from_remote_ip.yml b/lookups/previously_seen_S3_access_from_remote_ip.yml index 25096955ff..9eaa612562 100644 --- a/lookups/previously_seen_S3_access_from_remote_ip.yml +++ b/lookups/previously_seen_S3_access_from_remote_ip.yml @@ -1,3 +1,4 @@ description: A placeholder for a list of IPs that have access S3 -filename: previously_seen_S3_access_from_remote_ip.csv +collection: previously_seen_S3_access_from_remote_ip name: previously_seen_S3_access_from_remote_ip +fields_list: _key, bucket_name,remote_ip,earliest,latest \ No newline at end of file diff --git a/lookups/previously_seen_api_calls_from_user_roles.csv b/lookups/previously_seen_api_calls_from_user_roles.csv deleted file mode 100644 index 16491b9d75..0000000000 --- a/lookups/previously_seen_api_calls_from_user_roles.csv +++ /dev/null @@ -1 +0,0 @@ -earliest,latest,userName,eventName diff --git a/lookups/previously_seen_api_calls_from_user_roles.yml b/lookups/previously_seen_api_calls_from_user_roles.yml index d52d798c92..9eaa612562 100644 --- a/lookups/previously_seen_api_calls_from_user_roles.yml +++ b/lookups/previously_seen_api_calls_from_user_roles.yml @@ -1,3 +1,4 @@ -description: A placeholder for a list of AWS API calls for each user role -filename: previously_seen_api_calls_from_user_roles.csv -name: previously_seen_api_calls_from_user_roles +description: A placeholder for a list of IPs that have access S3 +collection: previously_seen_S3_access_from_remote_ip +name: previously_seen_S3_access_from_remote_ip +fields_list: _key, bucket_name,remote_ip,earliest,latest \ No newline at end of file diff --git a/lookups/previously_seen_aws_cross_account_activity.csv b/lookups/previously_seen_aws_cross_account_activity.csv deleted file mode 100644 index 30a8931b74..0000000000 --- a/lookups/previously_seen_aws_cross_account_activity.csv +++ /dev/null @@ -1 +0,0 @@ -firstTime,lastTime,requestingAccountId,requestedAccountId diff --git a/lookups/previously_seen_aws_cross_account_activity.yml b/lookups/previously_seen_aws_cross_account_activity.yml index ae39a854bd..04f34c480c 100644 --- a/lookups/previously_seen_aws_cross_account_activity.yml +++ b/lookups/previously_seen_aws_cross_account_activity.yml @@ -1,3 +1,4 @@ description: A placeholder for a list of AWS accounts and assumed roles -filename: previously_seen_aws_cross_account_activity.csv +collection: previously_seen_aws_cross_account_activity name: previously_seen_aws_cross_account_activity +fields_list: _key,firstTime,lastTime,requestingAccountId,requestedAccountId \ No newline at end of file diff --git a/lookups/previously_seen_aws_regions.csv b/lookups/previously_seen_aws_regions.csv deleted file mode 100644 index 82335d2260..0000000000 --- a/lookups/previously_seen_aws_regions.csv +++ /dev/null @@ -1 +0,0 @@ -earliest,latest,awsRegion diff --git a/lookups/previously_seen_aws_regions.yml b/lookups/previously_seen_aws_regions.yml index 15706035aa..52ade4494c 100644 --- a/lookups/previously_seen_aws_regions.yml +++ b/lookups/previously_seen_aws_regions.yml @@ -1,5 +1,4 @@ -default_match: 'false' description: A place holder for a list of used AWS regions -filename: previously_seen_aws_regions.csv -min_matches: 1 +collection: previously_seen_aws_regions name: previously_seen_aws_regions +fields_list: _key,earliest,latest,awsRegion \ No newline at end of file diff --git a/lookups/previously_seen_gcp_storage_access_from_remote_ip.csv b/lookups/previously_seen_gcp_storage_access_from_remote_ip.csv deleted file mode 100644 index 8a5c209fa3..0000000000 --- a/lookups/previously_seen_gcp_storage_access_from_remote_ip.csv +++ /dev/null @@ -1 +0,0 @@ -firstTime, lastTime, bucket_name, remote_ip, operation, request_uri \ No newline at end of file diff --git a/lookups/previously_seen_gcp_storage_access_from_remote_ip.yml b/lookups/previously_seen_gcp_storage_access_from_remote_ip.yml index f2aa889a6d..164c1b92a9 100644 --- a/lookups/previously_seen_gcp_storage_access_from_remote_ip.yml +++ b/lookups/previously_seen_gcp_storage_access_from_remote_ip.yml @@ -1,5 +1,4 @@ -default_match: 'false' description: A place holder for a list of GCP storage access from remote IPs -filename: previously_seen_gcp_storage_access_from_remote_ip.csv -min_matches: 1 -name: previously_seen_gcp_storage_access_from_remote_ip \ No newline at end of file +collection: previously_seen_gcp_storage_access_from_remote_ip +name: previously_seen_gcp_storage_access_from_remote_ip +fields_list: _key, firstTime, lastTime, bucket_name, remote_ip, operation, request_uri \ No newline at end of file diff --git a/lookups/s3_deletion_baseline.csv b/lookups/s3_deletion_baseline.csv deleted file mode 100644 index 3f0d026011..0000000000 --- a/lookups/s3_deletion_baseline.csv +++ /dev/null @@ -1 +0,0 @@ -arn, latestCount, numDataPoints, avgApiCalls, stdevApiCalls \ No newline at end of file diff --git a/lookups/s3_deletion_baseline.yml b/lookups/s3_deletion_baseline.yml index b7360bfea5..14e7532ed4 100644 --- a/lookups/s3_deletion_baseline.yml +++ b/lookups/s3_deletion_baseline.yml @@ -1,3 +1,4 @@ description: A placeholder for the baseline information for AWS S3 deletions -filename: s3_deletion_baseline.csv +collection: s3_deletion_baseline name: s3_deletion_baseline +fields_list: _key, arn, latestCount, numDataPoints, avgApiCalls, stdevApiCalls \ No newline at end of file diff --git a/lookups/security_group_activity_baseline.csv b/lookups/security_group_activity_baseline.csv deleted file mode 100644 index 4fec3cadd9..0000000000 --- a/lookups/security_group_activity_baseline.csv +++ /dev/null @@ -1 +0,0 @@ -arn,latestCount,numDataPoints,avgApiCalls,stdevApiCalls diff --git a/lookups/security_group_activity_baseline.yml b/lookups/security_group_activity_baseline.yml index 7de80b6283..7e27a12aa7 100644 --- a/lookups/security_group_activity_baseline.yml +++ b/lookups/security_group_activity_baseline.yml @@ -1,3 +1,4 @@ description: A placeholder for the baseline information for AWS security groups -filename: security_group_activity_baseline.csv +collection: security_group_activity_baseline name: security_group_activity_baseline +fields_list: _key, arn,latestCount,numDataPoints,avgApiCalls,stdevApiCalls \ No newline at end of file diff --git a/lookups/uncommon_processes_default.csv b/lookups/uncommon_processes_default.csv deleted file mode 100644 index c6e2588f4b..0000000000 --- a/lookups/uncommon_processes_default.csv +++ /dev/null @@ -1,9 +0,0 @@ -process_name,uncommon_default,category_default,analytic_story_default,kill_chain_phase_default,mitre_attack_default -sethc.exe,true,needs_accessibility,Windows Privilege Escalation,Actions on Objectives,Execution|Accessibility Features -utilman.exe,true,needs_accessibility,Windows Privilege Escalation,Actions on Objectives,Execution|Accessibility Features -osk.exe,true,needs_accessibility,Windows Privilege Escalation,Actions on Objectives,Execution|Accessibility Features -magnify.exe,true,needs_accessibility,Windows Privilege Escalation,Actions on Objectives,Execution|Accessibility Features -narrator.exe,true,needs_accessibility,Windows Privilege Escalation,Actions on Objectives,Execution|Accessibility Features -displayswitch.exe,true,needs_accessibility,Windows Privilege Escalation,Actions on Objectives,Execution|Accessibility Features -atbroker.exe,true,needs_accessibility,Windows Privilege Escalation,Actions on Objectives,Execution|Accessibility Features -quser.exe,true,,DHS Report TA18-074A|Unusual Processes,Actions on Objectives,Execution diff --git a/lookups/uncommon_processes_local.csv b/lookups/uncommon_processes_local.csv deleted file mode 100644 index 169cd6fb47..0000000000 --- a/lookups/uncommon_processes_local.csv +++ /dev/null @@ -1 +0,0 @@ -process_name,uncommon_local,category_local,analytic_story_local,kill_chain_phase_local,mitre_attack_local From cefd9349b161398522d599637cce77d98c5fca7b Mon Sep 17 00:00:00 2001 From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com> Date: Fri, 26 May 2023 16:27:31 -0700 Subject: [PATCH 2/4] Ensure that each lookup contains either a filename or a collection field defined in its yml. Make sure that is a filename is declared, then that file actually exists in the lookups folder. Make sure that every csv declared in the lookups folder is actually used by at least one lookup. If it is not, then print a warning but do not cause a failure. --- .../application/factory/factory.py | 18 ++++++++ .../application/factory/utils/utils.py | 46 +++++++++++++------ .../contentctl_core/domain/entities/lookup.py | 43 +++++++++++++++-- 3 files changed, 91 insertions(+), 16 deletions(-) diff --git a/bin/contentctl_project/contentctl_core/application/factory/factory.py b/bin/contentctl_project/contentctl_core/application/factory/factory.py index 8f6c71b397..a51eb84629 100644 --- a/bin/contentctl_project/contentctl_core/application/factory/factory.py +++ b/bin/contentctl_project/contentctl_core/application/factory/factory.py @@ -59,6 +59,24 @@ class Factory(): validation_errors = [] # order matters to load and enrich security content types validation_errors.extend(self.createSecurityContent(SecurityContentType.lookups)) + + lookups_directory = pathlib.Path(input_dto.input_path) / "lookups" + csv_lookups = [ + l.name + for l in Utils.get_all_csv_files_from_directory(str(lookups_directory)) + ] + try: + csv_lookups.remove("mitre_enrichment.csv") + except ValueError as e: + # mitre_enrichment.csv didn't exist in the lookups directory. That's okay. + pass + for lookup in self.output_dto.lookups: + if lookup.filename != None and lookup.filename in csv_lookups: + csv_lookups.remove(lookup.filename) + if len(csv_lookups) > 0: + print("The following lookups were unused. Should they be removed?\n\t- ",end="") + print("\n\t- ".join([str(p) for p in csv_lookups])) + validation_errors.extend(self.createSecurityContent(SecurityContentType.macros)) validation_errors.extend(self.createSecurityContent(SecurityContentType.deployments)) validation_errors.extend(self.createSecurityContent(SecurityContentType.baselines)) diff --git a/bin/contentctl_project/contentctl_core/application/factory/utils/utils.py b/bin/contentctl_project/contentctl_core/application/factory/utils/utils.py index 4123c0a6fc..a09693a1aa 100644 --- a/bin/contentctl_project/contentctl_core/application/factory/utils/utils.py +++ b/bin/contentctl_project/contentctl_core/application/factory/utils/utils.py @@ -3,40 +3,60 @@ import pathlib from typing import Tuple from pydantic import ValidationError -from bin.contentctl_project.contentctl_core.domain.entities.security_content_object import SecurityContentObject +from bin.contentctl_project.contentctl_core.domain.entities.security_content_object import ( + SecurityContentObject, +) + class Utils: + @staticmethod + def get_all_csv_files_from_directory(path: str) -> list[pathlib.Path]: + listOfFiles: list[pathlib.Path] = [] + for dirpath, dirnames, filenames in os.walk(path): + for file in filenames: + if file.endswith(".csv"): + listOfFiles.append(pathlib.Path(os.path.join(dirpath, file))) + + return sorted(listOfFiles) @staticmethod def get_all_yml_files_from_directory(path: str) -> list[pathlib.Path]: - listOfFiles:list[pathlib.Path] = [] - for (dirpath, dirnames, filenames) in os.walk(path): + listOfFiles: list[pathlib.Path] = [] + for dirpath, dirnames, filenames in os.walk(path): for file in filenames: if file.endswith(".yml"): listOfFiles.append(pathlib.Path(os.path.join(dirpath, file))) - + return sorted(listOfFiles) - @staticmethod - def add_id(id_dict:dict[str, list[pathlib.Path]], obj:SecurityContentObject, path:pathlib.Path) -> None: + def add_id( + id_dict: dict[str, list[pathlib.Path]], + obj: SecurityContentObject, + path: pathlib.Path, + ) -> None: if hasattr(obj, "id"): obj_id = obj.id if obj_id in id_dict: id_dict[obj_id].append(path) else: id_dict[obj_id] = [path] + # Otherwise, no ID so nothing to add.... - + @staticmethod - def check_ids_for_duplicates(id_dict:dict[str, list[pathlib.Path]])->list[Tuple[pathlib.Path, ValidationError]]: - validation_errors:list[Tuple[pathlib.Path, ValidationError]] = [] + def check_ids_for_duplicates( + id_dict: dict[str, list[pathlib.Path]] + ) -> list[Tuple[pathlib.Path, ValidationError]]: + validation_errors: list[Tuple[pathlib.Path, ValidationError]] = [] for key, values in id_dict.items(): if len(values) > 1: error_file_path = pathlib.Path("MULTIPLE") - all_files = '\n\t'.join(str(pathlib.Path(p)) for p in values) - exception = ValueError(f"Error validating id [{key}] - duplicate ID was used in the following files: \n\t{all_files}") + all_files = "\n\t".join(str(pathlib.Path(p)) for p in values) + exception = ValueError( + f"Error validating id [{key}] - duplicate ID was used in the following files: \n\t{all_files}" + ) validation_errors.append((error_file_path, exception)) - - return validation_errors \ No newline at end of file + + return validation_errors diff --git a/bin/contentctl_project/contentctl_core/domain/entities/lookup.py b/bin/contentctl_project/contentctl_core/domain/entities/lookup.py index 6b4cca798c..1aa8e6844a 100644 --- a/bin/contentctl_project/contentctl_core/domain/entities/lookup.py +++ b/bin/contentctl_project/contentctl_core/domain/entities/lookup.py @@ -1,16 +1,53 @@ -from pydantic import BaseModel, validator, ValidationError - -from bin.contentctl_project.contentctl_core.domain.entities.security_content_object import SecurityContentObject +import pathlib +from pydantic import BaseModel, validator, root_validator, ValidationError +from bin.contentctl_project.contentctl_core.domain.entities.security_content_object import ( + SecurityContentObject, +) class Lookup(BaseModel, SecurityContentObject): name: str description: str + # Collection indicates a KV Store that will be created and/or updated during detection runtime collection: str = None fields_list: str = None + + # Filename points to a lookup file that must exist during app build time filename: str = None default_match: str = None match_type: str = None min_matches: int = None case_sensitive_match: str = None + + @root_validator(pre=True) + def ensure_collection_or_filename_exists(cls, values): + # Exactly one of the fields "collection" or "filename" MUST be defined + # Check max length only for ESCU searches, SSA does not have that constraint + + if ( + values.get("collection", None) == None + and values.get("filename", None) == None + ): + raise ValueError( + "Error in lookup. Eaxctly one of 'collection' or 'filename' filename MUST be defined, but NEITHER was defined." + ) + + if ( + values.get("collection", None) != None + and values.get("filename", None) != None + ): + raise ValueError( + "Error in lookup. Exactly one of 'collection' or 'filename' filename MUST be defined, but BOTH were defined." + ) + return values + + @validator("filename") + def filename_validate(cls, v, values): + lookup_file_path = pathlib.Path(".") / "lookups" / str(v) + if not lookup_file_path.is_file(): + raise ValueError( + f"Lookup references lookup file '{lookup_file_path}', but that file does not exist." + ) + + return v From 266a2a775b1d5d4e0a2e17e38de51d5a1dcbe4ff Mon Sep 17 00:00:00 2001 From: Bhavin Patel Date: Mon, 12 Jun 2023 15:24:40 -0700 Subject: [PATCH 3/4] adding lookups --- lookups/uncommon_processes_default.csv | 9 +++++++++ lookups/uncommon_processes_local.csv | 1 + 2 files changed, 10 insertions(+) create mode 100644 lookups/uncommon_processes_default.csv create mode 100644 lookups/uncommon_processes_local.csv diff --git a/lookups/uncommon_processes_default.csv b/lookups/uncommon_processes_default.csv new file mode 100644 index 0000000000..4cd4daa1dc --- /dev/null +++ b/lookups/uncommon_processes_default.csv @@ -0,0 +1,9 @@ +process_name,uncommon_default,category_default,analytic_story_default,kill_chain_phase_default,mitre_attack_default +sethc.exe,true,needs_accessibility,Windows Privilege Escalation,Actions on Objectives,Execution|Accessibility Features +utilman.exe,true,needs_accessibility,Windows Privilege Escalation,Actions on Objectives,Execution|Accessibility Features +osk.exe,true,needs_accessibility,Windows Privilege Escalation,Actions on Objectives,Execution|Accessibility Features +magnify.exe,true,needs_accessibility,Windows Privilege Escalation,Actions on Objectives,Execution|Accessibility Features +narrator.exe,true,needs_accessibility,Windows Privilege Escalation,Actions on Objectives,Execution|Accessibility Features +displayswitch.exe,true,needs_accessibility,Windows Privilege Escalation,Actions on Objectives,Execution|Accessibility Features +atbroker.exe,true,needs_accessibility,Windows Privilege Escalation,Actions on Objectives,Execution|Accessibility Features +quser.exe,true,,DHS Report TA18-074A|Unusual Processes,Actions on Objectives,Execution \ No newline at end of file diff --git a/lookups/uncommon_processes_local.csv b/lookups/uncommon_processes_local.csv new file mode 100644 index 0000000000..49adc09176 --- /dev/null +++ b/lookups/uncommon_processes_local.csv @@ -0,0 +1 @@ +process_name,uncommon_local,category_local,analytic_story_local,kill_chain_phase_local,mitre_attack_local \ No newline at end of file From 7cbe7c87c95ba4591fc309431392530437d55b2a Mon Sep 17 00:00:00 2001 From: Bhavin Patel Date: Mon, 12 Jun 2023 15:40:37 -0700 Subject: [PATCH 4/4] Update previously_seen_api_calls_from_user_roles.yml --- lookups/previously_seen_api_calls_from_user_roles.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lookups/previously_seen_api_calls_from_user_roles.yml b/lookups/previously_seen_api_calls_from_user_roles.yml index 9eaa612562..93525ddd14 100644 --- a/lookups/previously_seen_api_calls_from_user_roles.yml +++ b/lookups/previously_seen_api_calls_from_user_roles.yml @@ -1,4 +1,4 @@ description: A placeholder for a list of IPs that have access S3 -collection: previously_seen_S3_access_from_remote_ip -name: previously_seen_S3_access_from_remote_ip -fields_list: _key, bucket_name,remote_ip,earliest,latest \ No newline at end of file +collection: previously_seen_api_calls_from_user_roles +name: previously_seen_api_calls_from_user_roles +fields_list: _key,earliest,latest,userName,eventName