From fcda138cc8468ebba60bc81859722eddfcec8e05 Mon Sep 17 00:00:00 2001 From: bpatel Date: Wed, 10 Jul 2019 15:14:39 -0700 Subject: [PATCH] updating usage details configuations --- bin/generate.py | 21 ++++++++++-- bin/usage_searches.conf | 73 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 91 insertions(+), 3 deletions(-) create mode 100644 bin/usage_searches.conf diff --git a/bin/generate.py b/bin/generate.py index 9db17a963b..cc5311df5c 100644 --- a/bin/generate.py +++ b/bin/generate.py @@ -829,9 +829,15 @@ def write_use_case_lib_conf(stories, detections, investigations, baselines, OUTP output_file.write("known_false_positives = {0}\n".format(known_false_positives)) output_file.write("providing_technologies = {0}\n".format(json.dumps(baseline['providing_technologies']))) output_file.write("\n") - output_file.write("### END BASELINES ###") - + output_file.write("\n### END ESCU BASELINES ###\n\n") + output_file.write("\n### USAGE DASHBOARD CONFIGURATIONS ###\n\n") + usconf = open('usage_searches.conf', 'r') + usage_searches = usconf.read() + usconf.close() + output_file.write(usage_searches) + output_file.write("\n\n### END OF USAGE DASHBOARD CONFIGURATIONS ###") output_file.close() + story_count = len(complete_stories.keys()) return story_count, use_case_lib_path @@ -1086,7 +1092,16 @@ def write_savedsearches_confv1(detections, investigations, baselines, OUTPUT_DIR output_file.write("schedule_window = auto\n") output_file.write("is_visible = false\n") output_file.write("search = {0}\n".format(baseline['search'])) - output_file.write("\n### END ESCU BASELINES ###") + output_file.write("\n### END ESCU BASELINES ###\n\n") + output_file.write("\n### USAGE DASHBOARD CONFIGURATIONS ###\n\n") + + usconf = open('usage_searches.conf', 'r') + usage_searches = usconf.read() + usconf.close() + output_file.write(usage_searches) + output_file.write("\n\n### END OF USAGE DASHBOARD CONFIGURATIONS ###") + + output_file.close() detections_count = len(detections) investigations_count = len(investigations) diff --git a/bin/usage_searches.conf b/bin/usage_searches.conf new file mode 100644 index 0000000000..6ba308dde1 --- /dev/null +++ b/bin/usage_searches.conf @@ -0,0 +1,73 @@ +[escu-metrics-usage] +action.email.useNSSubject = 1 +alert.digest_mode = True +alert.suppress = 0 +alert.track = 0 +auto_summarize.dispatch.earliest_time = -1d@h +dispatchAs = user +search = index=_audit sourcetype="audittrail" \ +"ESCU - "\ +`comment("Find all the search names in the audittrail.")`\ +| stats count(search) by search savedsearch_name user\ +| eval usage=(if(savedsearch_name=="","Adhoc","Scheduled")) \ +`comment("If the savedsearch_name field in the audittrail is empty, the search was run adhoc. Otherwise it was run as a scheduled search")`\ +| rex field=search "\"(?.*)\""\ +`comment("Extract the name of the search from the search string")`\ +| table savedsearch_name count(search) usage user | join savedsearch_name max=0 type=left [search sourcetype="manifests" | spath searches{} | mvexpand searches{} | spath input=searches{} | table category search_name | rename search_name as savedsearch_name | dedup savedsearch_name] | search category=* + +[escu-metrics-search] +action.email.useNSSubject = 1 +alert.suppress = 0 +alert.track = 0 +auto_summarize.dispatch.earliest_time = -1d@h +enableSched = 1 +cron_schedule = 0 0 * * * +dispatch.earliest_time = -4h@h +dispatch.latest_time = -1h@h +search = index=_audit action=search | transaction search_id maxspan=3m | search ESCU | stats sum(total_run_time) avg(total_run_time) max(total_run_time) sum(result_count) + +[escu-metrics-search-events] +action.email.useNSSubject = 1 +alert.digest_mode = True +alert.suppress = 0 +alert.track = 0 +auto_summarize.dispatch.earliest_time = -1d@h +cron_schedule = 0 0 * * * +enableSched = 1 +dispatch.earliest_time = -4h@h +dispatch.latest_time = -1h@h +search = [search index=_audit sourcetype="audittrail" \"ESCU NOT "index=_audit" | where search !="" | dedup search_id | rex field=search "\"(?.*)\"" | rex field=_raw "user=(?[a-zA-Z0-9_\-]+)" | eval usage=if(savedsearch_name!="", "scheduled", "adhoc") | eval savedsearch_name=if(savedsearch_name != "", savedsearch_name, search_name) | table savedsearch_name search_id user _time usage | outputlookup escu_search_id.csv | table search_id] index=_audit total_run_time event_count result_count NOT "index=_audit" | lookup escu_search_id.csv search_id | stats count(savedsearch_name) AS search_count avg(total_run_time) AS search_avg_run_time sum(total_run_time) AS search_total_run_time sum(result_count) AS search_total_results earliest(_time) AS firsts latest(_time) AS lasts by savedsearch_name user usage| eval first_run=strftime(firsts, "%B %d %Y") | eval last_run=strftime(lasts, "%B %d %Y") + +[escu-metrics-search-longest-runtime] +action.email.useNSSubject = 1 +alert.digest_mode = True +alert.suppress = 0 +alert.track = 0 +auto_summarize.dispatch.earliest_time = -1d@h +enableSched = 1 +cron_schedule = 0 0 * * * +disabled = 1 +dispatch.earliest_time = -4h@h +dispatch.latest_time = -1h@h +search = index=_* ESCU [search index=_* action=search latest=-2h earliest=-1d| transaction search_id maxspan=3m | search ESCU | stats values(total_run_time) AS run by search_id | sort -run | head 1| table search_id] | table search search_id + +[escu-metrics-usage-search] +action.email.useNSSubject = 1 +alert.digest_mode = True +alert.suppress = 0 +alert.track = 0 +auto_summarize.dispatch.earliest_time = -1d@h +cron_schedule = 0 0 * * * +dispatch.earliest_time = -4h@h +dispatch.latest_time = -1h@h +enableSched = 1 +dispatchAs = user +search = index=_audit sourcetype="audittrail" \ +"ESCU - "\ +`comment("Find all the search names in the audittrail. Ignore the last few minutes so we can exclude this search's text from the result.")`\ +| stats count(search) by search savedsearch_name user\ +| eval usage=(if(savedsearch_name=="","Adhoc","Scheduled")) \ +`comment("If the savedsearch_name field in the audittrail is empty, the search was run adhoc. Otherwise it was run as a scheduled search")`\ +| rex field=search "\"(?.*)\""\ +`comment("Extract the name of the search from the search string")`\ +| table savedsearch_name count(search) usage user | join savedsearch_name max=0 type=left [search sourcetype="manifests" | spath searches{} | mvexpand searches{} | spath input=searches{} | table category search_name | rename search_name as savedsearch_name | dedup savedsearch_name] | search category=* \ No newline at end of file