From 9b21029beaac4986a7687568cf8147eb0b86c642 Mon Sep 17 00:00:00 2001 From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com> Date: Tue, 12 Jul 2022 14:26:34 -0700 Subject: [PATCH 1/2] Added logic for checking to see if there are duplicate ids for any/all kinds of content that have an ID associated with it. --- .../application/factory/factory.py | 132 +++++++----------- 1 file changed, 50 insertions(+), 82 deletions(-) diff --git a/bin/contentctl_project/contentctl_core/application/factory/factory.py b/bin/contentctl_project/contentctl_core/application/factory/factory.py index 72c549a498..cc97a8a60b 100644 --- a/bin/contentctl_project/contentctl_core/application/factory/factory.py +++ b/bin/contentctl_project/contentctl_core/application/factory/factory.py @@ -15,6 +15,7 @@ from bin.contentctl_project.contentctl_core.application.builder.playbook_builder from bin.contentctl_project.contentctl_core.application.builder.director import Director from bin.contentctl_project.contentctl_core.application.factory.utils.utils import Utils from bin.contentctl_project.contentctl_core.domain.entities.link_validator import LinkValidator +from bin.contentctl_project.contentctl_core.domain.entities.security_content_object import SecurityContentObject @dataclass(frozen=True) class FactoryInputDto: @@ -42,15 +43,35 @@ class FactoryOutputDto: lookups: list tests: list + class Factory(): input_dto: FactoryInputDto output_dto: FactoryOutputDto + ids: dict[str,list[str]] = {} def __init__(self, output_dto: FactoryOutputDto) -> None: self.output_dto = output_dto + def add_id(self, obj:SecurityContentObject, path:str) -> None: + if hasattr(obj, "id"): + obj_id = obj.id + if obj_id in self.ids: + self.ids[obj_id].append(path) + else: + self.ids[obj_id] = [path] + # Otherwise, no ID so nothing to add.... + + def check_ids_for_duplicates(self)->bool: + validation_error = False + for key, values in self.ids.items(): + if len(values) > 1: + validation_error = True + id_conflicts_string = '\n\t* '.join(values) + print(f"\nError validating id [{key}] - duplicate ID is used for the following content: \n\t* {id_conflicts_string}") + return validation_error + def execute(self, input_dto: FactoryInputDto) -> None: self.input_dto = input_dto @@ -78,85 +99,12 @@ class Factory(): else: files = Utils.get_all_yml_files_from_directory(os.path.join(self.input_dto.input_path, str(type.name))) + # Instead of failing on the first error, just keep track of + # whether or not an error was found. This way, we can + # report all of the errors on a single run so that the + # user can see all the errors they need to fix. validation_error_found = False - - #Code below us commented out, but allows threaded construction of detections - #No issues have been observed running construction in parallel - ''' - import copy - import threading - NUM_THREADS = 1 - - def make_detections_thread(index:int): - my_dto = copy.deepcopy(self.input_dto) - for i in range(index, len(files), NUM_THREADS): - file = files[i] - if 'ssa__' in file: - continue - try: - if type == SecurityContentType.lookups: - my_dto.director.constructLookup(my_dto.basic_builder, file) - self.output_dto.lookups.append(my_dto.basic_builder.getObject()) - - elif type == SecurityContentType.macros: - my_dto.director.constructMacro(my_dto.basic_builder, file) - self.output_dto.macros.append(my_dto.basic_builder.getObject()) - - elif type == SecurityContentType.deployments: - my_dto.director.constructDeployment(my_dto.basic_builder, file) - self.output_dto.deployments.append(my_dto.basic_builder.getObject()) - - elif type == SecurityContentType.playbooks: - my_dto.director.constructPlaybook(my_dto.playbook_builder, file) - self.output_dto.playbooks.append(my_dto.playbook_builder.getObject()) - - elif type == SecurityContentType.baselines: - my_dto.director.constructBaseline(my_dto.baseline_builder, file, self.output_dto.deployments) - baseline = my_dto.baseline_builder.getObject() - self.output_dto.baselines.append(baseline) - - elif type == SecurityContentType.investigations: - my_dto.director.constructInvestigation(my_dto.investigation_builder, file) - investigation = my_dto.investigation_builder.getObject() - self.output_dto.investigations.append(investigation) - - elif type == SecurityContentType.stories: - my_dto.director.constructStory(my_dto.story_builder, file, - self.output_dto.detections, self.output_dto.baselines, self.output_dto.investigations) - story = my_dto.story_builder.getObject() - self.output_dto.stories.append(story) - elif type == SecurityContentType.detections: - my_dto.director.constructDetection(my_dto.detection_builder, file, - self.output_dto.deployments, self.output_dto.playbooks, self.output_dto.baselines, - self.output_dto.tests, my_dto.attack_enrichment, self.output_dto.macros, - self.output_dto.lookups, my_dto.force_cached_or_offline) - detection = my_dto.detection_builder.getObject() - self.output_dto.detections.append(detection) - - elif type == SecurityContentType.unit_tests: - my_dto.director.constructTest(my_dto.basic_builder, file) - test = my_dto.basic_builder.getObject() - self.output_dto.tests.append(test) - - except ValidationError as e: - print('\nValidation Error for file ' + file) - print(e) - validation_error_found = True - - - - builder_threads = [] - - for t in range(NUM_THREADS): - thread = threading.Thread(target=make_detections_thread,args=(t,)) - thread.start() - builder_threads.append(thread) - - for thread in builder_threads: - thread.join() - ''' - already_ran = False progress_percent = 0 type_string = "UNKNOWN TYPE" @@ -173,33 +121,43 @@ class Factory(): if type == SecurityContentType.lookups: type_string = "Lookups" self.input_dto.director.constructLookup(self.input_dto.basic_builder, file) - self.output_dto.lookups.append(self.input_dto.basic_builder.getObject()) + lookup = self.input_dto.basic_builder.getObject() + self.add_id(lookup, file) + self.output_dto.lookups.append(lookup) elif type == SecurityContentType.macros: type_string = "Macros" self.input_dto.director.constructMacro(self.input_dto.basic_builder, file) - self.output_dto.macros.append(self.input_dto.basic_builder.getObject()) + macro = self.input_dto.basic_builder.getObject() + self.add_id(macro, file) + self.output_dto.macros.append(macro) elif type == SecurityContentType.deployments: type_string = "Deployments" self.input_dto.director.constructDeployment(self.input_dto.basic_builder, file) - self.output_dto.deployments.append(self.input_dto.basic_builder.getObject()) + deployment = self.input_dto.basic_builder.getObject() + self.add_id(deployment, file) + self.output_dto.deployments.append(deployment) elif type == SecurityContentType.playbooks: type_string = "Playbooks" self.input_dto.director.constructPlaybook(self.input_dto.playbook_builder, file) - self.output_dto.playbooks.append(self.input_dto.playbook_builder.getObject()) + playbook = self.input_dto.playbook_builder.getObject() + self.add_id(playbook, file) + self.output_dto.playbooks.append(playbook) elif type == SecurityContentType.baselines: type_string = "Baselines" self.input_dto.director.constructBaseline(self.input_dto.baseline_builder, file, self.output_dto.deployments) baseline = self.input_dto.baseline_builder.getObject() + self.add_id(baseline, file) self.output_dto.baselines.append(baseline) elif type == SecurityContentType.investigations: type_string = "Investigations" self.input_dto.director.constructInvestigation(self.input_dto.investigation_builder, file) investigation = self.input_dto.investigation_builder.getObject() + self.add_id(investigation, file) self.output_dto.investigations.append(investigation) elif type == SecurityContentType.stories: @@ -207,6 +165,7 @@ class Factory(): self.input_dto.director.constructStory(self.input_dto.story_builder, file, self.output_dto.detections, self.output_dto.baselines, self.output_dto.investigations) story = self.input_dto.story_builder.getObject() + self.add_id(story, file) self.output_dto.stories.append(story) elif type == SecurityContentType.detections: @@ -216,12 +175,14 @@ class Factory(): self.output_dto.tests, self.input_dto.attack_enrichment, self.output_dto.macros, self.output_dto.lookups, self.input_dto.force_cached_or_offline) detection = self.input_dto.detection_builder.getObject() + self.add_id(detection, file) self.output_dto.detections.append(detection) elif type == SecurityContentType.unit_tests: type_string = "Unit Tests" self.input_dto.director.constructTest(self.input_dto.basic_builder, file) test = self.input_dto.basic_builder.getObject() + self.add_id(test, file) self.output_dto.tests.append(test) else: @@ -235,7 +196,14 @@ class Factory(): print('\nValidation Error for file ' + file) print(e) validation_error_found = True - + + + #Check for any duplicate IDs. The structure is uses + # to track them, self.ids, is populated previously in this + # function every time content is adde. + # This will also print out the duplicates if they exist. + validation_error_found |= self.check_ids_for_duplicates() + print(f"\r{f'{type_string} Progress'.rjust(23)}: [{progress_percent:3.0f}%]...", end="", flush=True) print("Done!") From 20ab9f27ce0ca732dc3dd90757443bc8d54814fd Mon Sep 17 00:00:00 2001 From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com> Date: Tue, 12 Jul 2022 15:06:35 -0700 Subject: [PATCH 2/2] Moved id collision checking into the Utils module since it will be used by the Factory and the BA Factory. Updated a duplicate UUID to mitigate the collision issue raised on GitHub. --- .../application/factory/ba_factory.py | 9 +++++ .../application/factory/factory.py | 40 +++++-------------- .../application/factory/utils/utils.py | 24 ++++++++++- .../web/spring4shell_payload_url_request.yml | 4 +- 4 files changed, 45 insertions(+), 32 deletions(-) diff --git a/bin/contentctl_project/contentctl_core/application/factory/ba_factory.py b/bin/contentctl_project/contentctl_core/application/factory/ba_factory.py index a28f26145b..a52bb26d63 100644 --- a/bin/contentctl_project/contentctl_core/application/factory/ba_factory.py +++ b/bin/contentctl_project/contentctl_core/application/factory/ba_factory.py @@ -27,6 +27,7 @@ class BAFactoryOutputDto: class BAFactory(): input_dto: BAFactoryInputDto output_dto: BAFactoryOutputDto + ids: dict[str,list[str]] = {} def __init__(self, output_dto: BAFactoryOutputDto) -> None: self.output_dto = output_dto @@ -68,12 +69,14 @@ class BAFactory(): type_string = "Detections" self.input_dto.director.constructDetection(self.input_dto.detection_builder, file, [], [], [], self.output_dto.tests, {}, [], []) detection = self.input_dto.detection_builder.getObject() + Utils.add_id(self.ids, detection, file) if not detection.deprecated and not detection.experimental: self.output_dto.detections.append(detection) elif type == SecurityContentType.unit_tests: type_string = "Unit Tests" self.input_dto.director.constructTest(self.input_dto.basic_builder, file) test = self.input_dto.basic_builder.getObject() + Utils.add_id(self.ids, test, file) self.output_dto.tests.append(test) else: raise(Exception(f"Unsupported content type: [{type}]")) @@ -87,6 +90,12 @@ class BAFactory(): print(e) validation_error_found = True + #Check for any duplicate IDs. The structure is uses + # to track them, self.ids, is populated previously in this + # function every time content is adde. + # This will also print out the duplicates if they exist. + validation_error_found |= Utils.check_ids_for_duplicates(self.ids) + print(f"\r{f'{type_string} Progress'.rjust(23)}: [{progress_percent:3.0f}%]...", end="", flush=True) print("Done!") diff --git a/bin/contentctl_project/contentctl_core/application/factory/factory.py b/bin/contentctl_project/contentctl_core/application/factory/factory.py index cc97a8a60b..a074bb6cca 100644 --- a/bin/contentctl_project/contentctl_core/application/factory/factory.py +++ b/bin/contentctl_project/contentctl_core/application/factory/factory.py @@ -48,29 +48,12 @@ class FactoryOutputDto: class Factory(): input_dto: FactoryInputDto output_dto: FactoryOutputDto - ids: dict[str,list[str]] = {} def __init__(self, output_dto: FactoryOutputDto) -> None: self.output_dto = output_dto - def add_id(self, obj:SecurityContentObject, path:str) -> None: - if hasattr(obj, "id"): - obj_id = obj.id - if obj_id in self.ids: - self.ids[obj_id].append(path) - else: - self.ids[obj_id] = [path] - # Otherwise, no ID so nothing to add.... - - def check_ids_for_duplicates(self)->bool: - validation_error = False - for key, values in self.ids.items(): - if len(values) > 1: - validation_error = True - id_conflicts_string = '\n\t* '.join(values) - print(f"\nError validating id [{key}] - duplicate ID is used for the following content: \n\t* {id_conflicts_string}") - return validation_error + def execute(self, input_dto: FactoryInputDto) -> None: @@ -90,7 +73,6 @@ class Factory(): def createSecurityContent(self, type: SecurityContentType) -> list: - objects = [] if type == SecurityContentType.deployments: files = Utils.get_all_yml_files_from_directory(os.path.join(self.input_dto.input_path, str(type.name), 'ESCU')) @@ -122,42 +104,42 @@ class Factory(): type_string = "Lookups" self.input_dto.director.constructLookup(self.input_dto.basic_builder, file) lookup = self.input_dto.basic_builder.getObject() - self.add_id(lookup, file) + Utils.add_id(self.ids, lookup, file) self.output_dto.lookups.append(lookup) elif type == SecurityContentType.macros: type_string = "Macros" self.input_dto.director.constructMacro(self.input_dto.basic_builder, file) macro = self.input_dto.basic_builder.getObject() - self.add_id(macro, file) + Utils.add_id(self.ids, macro, file) self.output_dto.macros.append(macro) elif type == SecurityContentType.deployments: type_string = "Deployments" self.input_dto.director.constructDeployment(self.input_dto.basic_builder, file) deployment = self.input_dto.basic_builder.getObject() - self.add_id(deployment, file) + Utils.add_id(self.ids, deployment, file) self.output_dto.deployments.append(deployment) elif type == SecurityContentType.playbooks: type_string = "Playbooks" self.input_dto.director.constructPlaybook(self.input_dto.playbook_builder, file) playbook = self.input_dto.playbook_builder.getObject() - self.add_id(playbook, file) + Utils.add_id(self.ids, playbook, file) self.output_dto.playbooks.append(playbook) elif type == SecurityContentType.baselines: type_string = "Baselines" self.input_dto.director.constructBaseline(self.input_dto.baseline_builder, file, self.output_dto.deployments) baseline = self.input_dto.baseline_builder.getObject() - self.add_id(baseline, file) + Utils.add_id(self.ids, baseline, file) self.output_dto.baselines.append(baseline) elif type == SecurityContentType.investigations: type_string = "Investigations" self.input_dto.director.constructInvestigation(self.input_dto.investigation_builder, file) investigation = self.input_dto.investigation_builder.getObject() - self.add_id(investigation, file) + Utils.add_id(self.ids, investigation, file) self.output_dto.investigations.append(investigation) elif type == SecurityContentType.stories: @@ -165,7 +147,7 @@ class Factory(): self.input_dto.director.constructStory(self.input_dto.story_builder, file, self.output_dto.detections, self.output_dto.baselines, self.output_dto.investigations) story = self.input_dto.story_builder.getObject() - self.add_id(story, file) + Utils.add_id(self.ids, story, file) self.output_dto.stories.append(story) elif type == SecurityContentType.detections: @@ -175,14 +157,14 @@ class Factory(): self.output_dto.tests, self.input_dto.attack_enrichment, self.output_dto.macros, self.output_dto.lookups, self.input_dto.force_cached_or_offline) detection = self.input_dto.detection_builder.getObject() - self.add_id(detection, file) + Utils.add_id(self.ids, detection, file) self.output_dto.detections.append(detection) elif type == SecurityContentType.unit_tests: type_string = "Unit Tests" self.input_dto.director.constructTest(self.input_dto.basic_builder, file) test = self.input_dto.basic_builder.getObject() - self.add_id(test, file) + Utils.add_id(self.ids, test, file) self.output_dto.tests.append(test) else: @@ -202,7 +184,7 @@ class Factory(): # to track them, self.ids, is populated previously in this # function every time content is adde. # This will also print out the duplicates if they exist. - validation_error_found |= self.check_ids_for_duplicates() + validation_error_found |= Utils.check_ids_for_duplicates(self.ids) print(f"\r{f'{type_string} Progress'.rjust(23)}: [{progress_percent:3.0f}%]...", end="", flush=True) print("Done!") diff --git a/bin/contentctl_project/contentctl_core/application/factory/utils/utils.py b/bin/contentctl_project/contentctl_core/application/factory/utils/utils.py index 5df5d2fb5b..35fb65f5a7 100644 --- a/bin/contentctl_project/contentctl_core/application/factory/utils/utils.py +++ b/bin/contentctl_project/contentctl_core/application/factory/utils/utils.py @@ -1,5 +1,7 @@ import os +from bin.contentctl_project.contentctl_core.domain.entities.security_content_object import SecurityContentObject + class Utils: @staticmethod @@ -10,4 +12,24 @@ class Utils: if file.endswith(".yml"): listOfFiles.append(os.path.join(dirpath, file)) - return sorted(listOfFiles) \ No newline at end of file + return sorted(listOfFiles) + + @staticmethod + def add_id(id_dict:dict[str, list[str]], obj:SecurityContentObject, path:str) -> None: + if hasattr(obj, "id"): + obj_id = obj.id + if obj_id in id_dict: + id_dict[obj_id].append(path) + else: + id_dict[obj_id] = [path] + # Otherwise, no ID so nothing to add.... + + @staticmethod + def check_ids_for_duplicates(id_dict:dict[str, list[str]])->bool: + validation_error = False + for key, values in id_dict.items(): + if len(values) > 1: + validation_error = True + id_conflicts_string = '\n\t* '.join(values) + print(f"\nError validating id [{key}] - duplicate ID is used for the following content: \n\t* {id_conflicts_string}") + return validation_error \ No newline at end of file diff --git a/detections/web/spring4shell_payload_url_request.yml b/detections/web/spring4shell_payload_url_request.yml index 34c689188d..775e50f351 100644 --- a/detections/web/spring4shell_payload_url_request.yml +++ b/detections/web/spring4shell_payload_url_request.yml @@ -1,7 +1,7 @@ name: Spring4Shell Payload URL Request -id: 2850c734-2d44-4431-8139-1a56f6f54c01 +id: 9d44d649-7d67-4559-95c1-8022ff49420b version: 1 -date: '2022-04-05' +date: '2022-07-12' author: Michael Haag, Splunk type: TTP datamodel: