From 8a7fe2cb8bc98e6c2ad09df92fea5149679b2b46 Mon Sep 17 00:00:00 2001 From: divious1 Date: Tue, 8 Oct 2019 08:37:12 -0400 Subject: [PATCH] generating object works now --- package/bin/investigate.py | 317 +++++++++++++++++-------------------- 1 file changed, 146 insertions(+), 171 deletions(-) diff --git a/package/bin/investigate.py b/package/bin/investigate.py index b772956c93..3013c4a38e 100644 --- a/package/bin/investigate.py +++ b/package/bin/investigate.py @@ -1,4 +1,3 @@ - from splunklib.searchcommands import dispatch, StreamingCommand, Configuration import sys import json @@ -8,20 +7,20 @@ import splunklib.results from splunklib.searchcommands.validators import Boolean import splunk.mining.dcutils import time +import re from datetime import datetime, timedelta - + @Configuration() class Investigate(StreamingCommand): + logger = splunk.mining.dcutils.getLogger() + investigative_searches_to_run = [] + final_search = "" + COLLECTION_NAME = "mp_detect_new" + STORY = "Malicious PowerShell" + collection_results = {} - logger = splunk.mining.dcutils.getLogger() - investigative_searches_to_run = [] - final_search = "" - COLLECTION_NAME = "mp_detect" - STORY = "Malicious PowerShell" - collection_results = {} - - def _investigative_searches(self, content): + def _investigative_searches(self, content): investigative_data = {} investigative_data['search_name'] = content['action.escu.full_search_name'] investigative_data['search_description'] = content['description'] @@ -30,208 +29,184 @@ class Investigate(StreamingCommand): self.investigative_searches_to_run.append(investigative_data) return self.investigative_searches_to_run - # def _parse_results(self, job_results, job): + def _store_collections(self, collection, investigations_results): - # # if there are results lets process them - # investigation_results = [] - # if job['resultCount'] > "0": - # # place to store results and entity results - # # process results - # for result in job_results: - # # add store detection results - # investigation_results.append(dict(result)) - - # return investigation_results - - def _store_collections(self, collection,investigations_results): - self.collection_results['investigations'] = investigations_results collection.data.insert(json.dumps(self.collection_results)) self.logger.info("investigate.py - Entered into KV: {0}") + def _execute_investigations(self, investigations, earliest_time, latest_time, service): + investigations_with_results = [] - def _execute_investigations(self, investigations, earliest_time, latest_time, service): - investigations_with_results = [] - + for investigation in investigations: - for investigation in investigations: - - for investigation_name, investigation_value in investigation.items(): + for investigation_name, investigation_value in investigation.items(): - kwargs = {"exec_mode": "normal", "earliest_time": earliest_time, "latest_time": latest_time} - - for search in investigation_value['final_search']: - - search = search + "| head 1" - test = {} - if search: - job = service.jobs.create(search, **kwargs) - time.sleep(1) - while True: - job.refresh() - if job['isDone'] == "1": - #self.logger.info("investigate.py - finished investigative search: {0}".format(search)) - break + kwargs = {"exec_mode": "normal", "earliest_time": earliest_time, "latest_time": latest_time} - job_results = splunklib.results.ResultsReader(job.results()) + for search in investigation_value['final_search']: - investigation_results = [] - - if job['resultCount'] > "0": - for result in job_results: - # add store detection results - investigation_results.append(dict(result)) - - test['search_name'] = investigation_name - test['results'] = investigation_results - - investigations_with_results.append((test)) - #results = self._parse_results(job_results, job) + search = search + "| head 1" + test = {} + if search: + job = service.jobs.create(search, **kwargs) + time.sleep(1) + while True: + job.refresh() + if job['isDone'] == "1": + # self.logger.info("investigate.py - finished investigative search: {0}".format(search)) + break - self.logger.info("investigate.py - XX Results --------------- : {0}") - - - - return investigations_with_results + job_results = splunklib.results.ResultsReader(job.results()) - def _generate_investigation_objects(self, detected_entities): - investigations = [] + investigation_results = [] - # iterate through all the investigations - for investigative_search in self.investigative_searches_to_run: - - investigation = dict() + if job['resultCount'] > "0": + for result in job_results: + # add store detection results + investigation_results.append(dict(result)) - # get the data we need from the investigative search - investigative_search_name = investigative_search['search_name'] - search = investigative_search['search'] - investigative_entities = json.loads(investigative_search['entities']) + test['search_name'] = investigation_name + test['results'] = investigation_results + + investigations_with_results.append((test)) + # results = self._parse_results(job_results, job) + + self.logger.info("investigate.py - XX Results --------------- : {0}") + + return investigations_with_results + + def _generate_investigation_objects(self, detected_entities): + investigations = [] + + # iterate through all the investigations + for investigative_search in self.investigative_searches_to_run: + # self.logger.info("investigate.py - {0} ".format(investigative_search['search_name'])) + if investigative_search['search_name'] == "ESCU - Get Parent Process Info": + + i = dict() + + # get the data we need from the investigative search + search_name = investigative_search['search_name'] + search = investigative_search['search'] + investigative_entities = json.loads(investigative_search['entities']) + + for investigative_entity_name in investigative_entities: + # iterate through all the detection entities and grab their results + self.logger.info("investigate.py - {0} ".format(investigative_entity_name)) + i[investigative_entity_name] = [] + + for e in sorted(detected_entities): + # self.logger.info("investigate.py - search {0} - entities {1} ".format(investigative_search['search_name'],e)) + for detected_entity_name, detected_entity_value in sorted(e.items()): + if investigative_entity_name == detected_entity_name: + self.logger.info( + "investigate.py - investigative_entity_name {2} | detected_entity_name {0} | detected_entity_value {1} ".format( + detected_entity_name, detected_entity_value, investigative_entity_name)) + for v in detected_entity_value['entity_results']: + i[investigative_entity_name].append(v) - investigation[investigative_search_name] = {} - investigation[investigative_search_name]['entity'] = [] + self.logger.info("investigate.py {0} ||| object: {1}".format(search_name, json.dumps(i, indent=4))) + searches = {} + searches['searches'] = [] + searches['entities'] = [] + searches['values'] = [] + for entity_name, values in sorted(i.items()): + modified_entity_name = "{" + entity_name + "}" + for v in values: - final_search = "" - final_search_object = [] - # iterate through all the detection entities and grab their results - for entity_detected_name, entity_detected_value in detected_entities.items(): + # check if this is not our first entity and if is not in the list then we must update all our searches + if len(searches['entities']) > 0 and entity_name not in searches['entities']: + # self.logger.info("investigate.py {0} ||| haven't seen entity: {1} | updated_search: {2}".format(search_name, entity_name, searches['searches'])) + updated_searches = [] - # check if the detected entity matches that of the investigation - if entity_detected_name in investigative_entities: - # store the entity we replaced - entity_name = "{" + entity_detected_name + "}" - - #replace one one entitiy - if len(investigative_entities) == 1: + # update all searches store + for s in searches['searches']: + updated_search = s.replace(modified_entity_name, v) + updated_searches.append(updated_search) + searches['searches'] = updated_searches - for v in entity_detected_value['entity_results']: - final_search = "" - if final_search == "": - final_search = (search.replace(entity_name, v)) - investigation[investigative_search_name]['entity'].append(v) - final_search_object.append(final_search) + searches['entities'].append(entity_name) + searches['values'].append(v) + updated_search = search.replace(modified_entity_name, v) + searches['searches'].append(updated_search) + # self.logger.info("investigate.py {0} ||| entity: {1} | updated_search: {2}".format(search_name, entity_name, updated_search)) + + self.logger.info("investigate.py {0} ||| FINAL OBJECT | entity: {1} | values: {2} | searches: {3}".format(search_name, searches['entities'], searches['values'], json.dumps(searches['searches'], indent=4))) + investigations.append(searches) + return investigations - # # BROKEN - replace 2 different entities. replace all combinations of entities. Currenttly it replace one of the dests + def stream(self, records): - if len(investigative_entities) == 2: - # grab every value of that entity and perform the replacement, if already processed just overwrite - - for v in entity_detected_value['entity_results']: - if final_search == "": - final_search = search.replace(entity_name, v) - investigation[investigative_search_name]['entity'].append(v) + search_results = self.search_results_info + port = splunk.getDefault('port') + service = splunklib.client.connect(token=self._metadata.searchinfo.session_key, port=port, owner="nobody") + savedsearches = service.saved_searches - else: - final_search = final_search.replace(entity_name, v) - investigation[investigative_search_name]['entity'].append(v) - final_search_object.append(final_search) + if hasattr(search_results, 'search_et') and hasattr(search_results, 'search_lt'): + earliest_time = search_results.search_et + latest_time = search_results.search_lt - investigation[investigative_search_name]['final_search'] = final_search_object + collection = service.kvstore[self.COLLECTION_NAME] - - investigations.append(investigation) - + if self.COLLECTION_NAME in service.kvstore: + self.logger.info("investigate.py - Collection: {0}".format(self.COLLECTION_NAME)) - return investigations + for savedsearch in savedsearches: + content = savedsearch.content + if 'action.escu.analytic_story' in content: + stories = str(content['action.escu.analytic_story']).strip('][').replace('"', '').split(', ') + for s in stories: + if s == self.STORY and content['action.escu.search_type'] == 'investigative': + self.investigative_searches_to_run = self._investigative_searches(content) - def stream(self, records): - - search_results = self.search_results_info - port = splunk.getDefault('port') - service = splunklib.client.connect(token=self._metadata.searchinfo.session_key, port=port, owner = "nobody") - savedsearches = service.saved_searches + detection_results = (collection.data.query()) - if hasattr(search_results, 'search_et') and hasattr(search_results, 'search_lt'): - earliest_time = search_results.search_et - latest_time = search_results.search_lt + # grab investigations to execute + collection_results = {} + collection_results['investigations'] = [] - collection = service.kvstore[self.COLLECTION_NAME] + # testing investigation in KV store - if self.COLLECTION_NAME in service.kvstore: - self.logger.info("investigate.py - Collection: {0}".format(self.COLLECTION_NAME)) + investigate_kvstore = "investigate_kvstore" + if investigate_kvstore in service.kvstore: + service.kvstore.delete(investigate_kvstore) - for savedsearch in savedsearches: - content = savedsearch.content - if 'action.escu.analytic_story' in content and self.STORY in content['action.escu.analytic_story']: - if content['action.escu.search_type'] == 'investigative': - self.investigative_searches_to_run = self._investigative_searches(content) - - detection_results = (collection.data.query()) - - # grab investigations to execute - collection_results = {} - collection_results['investigations'] = [] + # Let's create it and then make sure it exists + service.kvstore.create(investigate_kvstore) + investigate_collection = service.kvstore[investigate_kvstore] - #testing investigation in KV store + for detection_result in detection_results: + for each_result in detection_result['detections']: + # Generate invetigsation searches to run + investigations = self._generate_investigation_objects(each_result['entities']) - investigate_kvstore = "investigate_kvstore" - if investigate_kvstore in service.kvstore: - service.kvstore.delete(investigate_kvstore) - - # Let's create it and then make sure it exists - service.kvstore.create(investigate_kvstore) - investigate_collection = service.kvstore[investigate_kvstore] - - - for detection_result in detection_results: + # Execute investigation searches + # investigations_results = self._execute_investigations(investigations, earliest_time, latest_time, service) + # self.logger.info("investigate.py - YY Results --------------- : {0}".format(investigations_results)) - for each_result in detection_result['detections']: - - # Loop through values in the entities key - for entity in each_result['entities']: + # self._store_collections(investigate_collection, investigations_results) + # investigation_final_results = [] - # Generate invetigsation searches to run - investigations = self._generate_investigation_objects(entity) - #self.logger.info("investigate.py - -------Collection: {0}".format(((investigations)))) + # for i in investigations_results: - # Execute investigation searches - investigations_results = self._execute_investigations(investigations, earliest_time, latest_time, service) - #self.logger.info("investigate.py - YY Results --------------- : {0}".format(investigations_results)) + # investigation_final_results.append((i)) - self._store_collections(investigate_collection,investigations_results) - #investigation_final_results = [] - - # for i in investigations_results: - - # investigation_final_results.append((i)) - - # self.logger.info("investigate.py - ZZZZZ Results --------------- : {0}".format(investigation_final_results)) - # self.logger.info("investigate.py - Collection Enter: {0}") + # self.logger.info("investigate.py - ZZZZZ Results --------------- : {0}".format(investigation_final_results)) + # self.logger.info("investigate.py - Collection Enter: {0}") - # collection_results['investigations'] = investigation_final_results - # self.logger.info("investigate.py - Collection Entry: {0}".format(collection_results['investigations'])) + # collection_results['investigations'] = investigation_final_results + # self.logger.info("investigate.py - Collection Entry: {0}".format(collection_results['investigations'])) - # collection.data.insert((collection_results)) - # self.logger.info("investigate.py - Collection Entry DONE: {0}") + # collection.data.insert((collection_results)) + # self.logger.info("investigate.py - Collection Entry DONE: {0}") - - - for record in records: - - yield record + for record in records: + yield record if __name__ == "__main__": - dispatch(Investigate, sys.argv, sys.stdin, sys.stdout, __name__) + dispatch(Investigate, sys.argv, sys.stdin, sys.stdout, __name__)