updating usage details configuations

bin/generate.py

@@ -829,9 +829,15 @@ def write_use_case_lib_conf(stories, detections, investigations, baselines, OUTP
         output_file.write("known_false_positives = {0}\n".format(known_false_positives))
         output_file.write("providing_technologies = {0}\n".format(json.dumps(baseline['providing_technologies'])))
         output_file.write("\n")
-    output_file.write("### END BASELINES ###")
-
+    output_file.write("\n### END ESCU BASELINES ###\n\n")
+    output_file.write("\n### USAGE DASHBOARD CONFIGURATIONS ###\n\n")
+    usconf = open('usage_searches.conf', 'r')
+    usage_searches = usconf.read()
+    usconf.close()
+    output_file.write(usage_searches)
+    output_file.write("\n\n### END OF USAGE DASHBOARD CONFIGURATIONS ###")
     output_file.close()
+
     story_count = len(complete_stories.keys())
     return story_count, use_case_lib_path
 
@@ -1086,7 +1092,16 @@ def write_savedsearches_confv1(detections, investigations, baselines, OUTPUT_DIR
         output_file.write("schedule_window = auto\n")
         output_file.write("is_visible = false\n")
         output_file.write("search = {0}\n".format(baseline['search']))
-    output_file.write("\n### END ESCU BASELINES ###")
+    output_file.write("\n### END ESCU BASELINES ###\n\n")
+    output_file.write("\n### USAGE DASHBOARD CONFIGURATIONS ###\n\n")
+
+    usconf = open('usage_searches.conf', 'r')
+    usage_searches = usconf.read()
+    usconf.close()
+    output_file.write(usage_searches)
+    output_file.write("\n\n### END OF USAGE DASHBOARD CONFIGURATIONS ###")
+
+    output_file.close()
 
     detections_count = len(detections)
     investigations_count = len(investigations)

bin/usage_searches.conf

@@ -0,0 +1,73 @@
+[escu-metrics-usage]
+action.email.useNSSubject = 1
+alert.digest_mode = True
+alert.suppress = 0
+alert.track = 0
+auto_summarize.dispatch.earliest_time = -1d@h
+dispatchAs = user
+search = index=_audit sourcetype="audittrail" \
+"ESCU - "\
+`comment("Find all the search names in the audittrail.")`\
+| stats count(search) by search savedsearch_name user\
+| eval usage=(if(savedsearch_name=="","Adhoc","Scheduled")) \
+`comment("If the savedsearch_name field in the audittrail is empty, the search was run adhoc. Otherwise it was run as a scheduled search")`\
+| rex field=search "\"(?<savedsearch_name>.*)\""\
+`comment("Extract the name of the search from the search string")`\
+| table savedsearch_name count(search) usage user | join savedsearch_name max=0 type=left [search sourcetype="manifests" | spath searches{} | mvexpand searches{} | spath input=searches{} | table category search_name | rename search_name as savedsearch_name | dedup savedsearch_name] | search category=*
+
+[escu-metrics-search]
+action.email.useNSSubject = 1
+alert.suppress = 0
+alert.track = 0
+auto_summarize.dispatch.earliest_time = -1d@h
+enableSched = 1
+cron_schedule = 0 0 * * *
+dispatch.earliest_time = -4h@h
+dispatch.latest_time = -1h@h
+search = index=_audit action=search | transaction search_id maxspan=3m | search ESCU | stats sum(total_run_time) avg(total_run_time) max(total_run_time) sum(result_count)
+
+[escu-metrics-search-events]
+action.email.useNSSubject = 1
+alert.digest_mode = True
+alert.suppress = 0
+alert.track = 0
+auto_summarize.dispatch.earliest_time = -1d@h
+cron_schedule = 0 0 * * *
+enableSched = 1
+dispatch.earliest_time = -4h@h
+dispatch.latest_time = -1h@h
+search = [search index=_audit sourcetype="audittrail" \"ESCU NOT "index=_audit" | where search !="" | dedup search_id | rex field=search "\"(?<search_name>.*)\"" | rex field=_raw "user=(?<user>[a-zA-Z0-9_\-]+)" | eval usage=if(savedsearch_name!="", "scheduled", "adhoc") | eval savedsearch_name=if(savedsearch_name != "", savedsearch_name, search_name) | table savedsearch_name search_id user _time usage | outputlookup escu_search_id.csv | table search_id] index=_audit total_run_time event_count result_count NOT "index=_audit" | lookup escu_search_id.csv search_id | stats count(savedsearch_name) AS search_count avg(total_run_time) AS search_avg_run_time sum(total_run_time) AS search_total_run_time sum(result_count) AS search_total_results earliest(_time) AS firsts latest(_time) AS lasts by savedsearch_name user usage| eval first_run=strftime(firsts, "%B %d %Y") | eval last_run=strftime(lasts, "%B %d %Y")
+
+[escu-metrics-search-longest-runtime]
+action.email.useNSSubject = 1
+alert.digest_mode = True
+alert.suppress = 0
+alert.track = 0
+auto_summarize.dispatch.earliest_time = -1d@h
+enableSched = 1
+cron_schedule = 0 0 * * *
+disabled = 1
+dispatch.earliest_time = -4h@h
+dispatch.latest_time = -1h@h
+search = index=_* ESCU [search index=_* action=search latest=-2h earliest=-1d| transaction search_id maxspan=3m | search ESCU | stats values(total_run_time) AS run by search_id | sort -run | head 1| table search_id] | table search search_id
+
+[escu-metrics-usage-search]
+action.email.useNSSubject = 1
+alert.digest_mode = True
+alert.suppress = 0
+alert.track = 0
+auto_summarize.dispatch.earliest_time = -1d@h
+cron_schedule = 0 0 * * *
+dispatch.earliest_time = -4h@h
+dispatch.latest_time = -1h@h
+enableSched = 1
+dispatchAs = user
+search = index=_audit sourcetype="audittrail" \
+"ESCU - "\
+`comment("Find all the search names in the audittrail. Ignore the last few minutes so we can exclude this search's text from the result.")`\
+| stats count(search) by search savedsearch_name user\
+| eval usage=(if(savedsearch_name=="","Adhoc","Scheduled")) \
+`comment("If the savedsearch_name field in the audittrail is empty, the search was run adhoc. Otherwise it was run as a scheduled search")`\
+| rex field=search "\"(?<savedsearch_name>.*)\""\
+`comment("Extract the name of the search from the search string")`\
+| table savedsearch_name count(search) usage user | join savedsearch_name max=0 type=left [search sourcetype="manifests" | spath searches{} | mvexpand searches{} | spath input=searches{} | table category search_name | rename search_name as savedsearch_name | dedup savedsearch_name] | search category=*