From 8b7f7d20eb475ffa682cae0e0a806f61cd6be0e7 Mon Sep 17 00:00:00 2001 From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com> Date: Wed, 29 Jun 2022 16:53:17 -0700 Subject: [PATCH 1/6] in-memory optimization for fetching CVE enrichment and app enrichment, even if skip_enrichment or cached_and_offline are disabled. --- .../builder/cve_enrichment.py | 52 ++++++++--------- .../builder/splunk_app_enrichment.py | 57 +++++++++++-------- 2 files changed, 58 insertions(+), 51 deletions(-) diff --git a/bin/contentctl_project/contentctl_infrastructure/builder/cve_enrichment.py b/bin/contentctl_project/contentctl_infrastructure/builder/cve_enrichment.py index 0e509377a5..01c9a770aa 100644 --- a/bin/contentctl_project/contentctl_infrastructure/builder/cve_enrichment.py +++ b/bin/contentctl_project/contentctl_infrastructure/builder/cve_enrichment.py @@ -3,32 +3,35 @@ from pycvesearch import CVESearch import functools import os import shelve - +import time CVESSEARCH_API_URL = 'https://cve.circl.lu' CVE_CACHE_FILENAME = "lookups/CVE_CACHE.db" -@functools.cache -def cvesearch_helper(url:str, cve_id:str): - if not os.path.exists(CVE_CACHE_FILENAME): - print(f"Cache at {CVE_CACHE_FILENAME} not found - Creating it.") - cache = shelve.open(CVE_CACHE_FILENAME, flag='c', writeback=True) - if cve_id in cache: - req = cache[cve_id] - cache.close() - return req - +NON_PERSISTENT_CACHE = {} - - try: - cve = cvesearch_id_helper(url) - result = cve.id(cve_id) - except Exception as e: - raise(Exception(f"The option 'force_cached_or_offline' was used, but {cve_id} not found in {CVE_CACHE_FILENAME} and unable to connect to {CVESSEARCH_API_URL}")) - if result is None: - raise(Exception(f'CveEnrichment for [ {cve_id} ] failed - CVE does not exist')) - cache[cve_id] = result - cache.close() +@functools.cache +def cvesearch_helper(url:str, cve_id:str, force_cached_or_offline:bool=False): + if force_cached_or_offline: + if not os.path.exists(CVE_CACHE_FILENAME): + print(f"Cache at {CVE_CACHE_FILENAME} not found - Creating it.") + cache = shelve.open(CVE_CACHE_FILENAME, flag='c', writeback=True) + else: + cache = NON_PERSISTENT_CACHE + if cve_id in cache: + result = cache[cve_id] + #print(f"hit cve_enrichment: {time.time() - start:.2f}") + else: + try: + cve = cvesearch_id_helper(url) + result = cve.id(cve_id) + except Exception as e: + raise(Exception(f"The option 'force_cached_or_offline' was used, but {cve_id} not found in {CVE_CACHE_FILENAME} and unable to connect to {CVESSEARCH_API_URL}")) + if result is None: + raise(Exception(f'CveEnrichment for [ {cve_id} ] failed - CVE does not exist')) + cache[cve_id] = result + if force_cached_or_offline: + cache.close() return result @@ -47,11 +50,8 @@ class CveEnrichment(): def enrich_cve(self, cve_id: str, force_cached_or_offline: bool = False) -> dict: cve_enriched = dict() try: - if force_cached_or_offline is True: - result = cvesearch_helper(CVESSEARCH_API_URL, cve_id) - else: - cve = CVESearch(CVESSEARCH_API_URL) - result = cve.id(cve_id) + + result = cvesearch_helper(CVESSEARCH_API_URL, cve_id) cve_enriched['id'] = cve_id cve_enriched['cvss'] = result['cvss'] cve_enriched['summary'] = result['summary'] diff --git a/bin/contentctl_project/contentctl_infrastructure/builder/splunk_app_enrichment.py b/bin/contentctl_project/contentctl_infrastructure/builder/splunk_app_enrichment.py index 8dbcab908f..eedab9d13f 100644 --- a/bin/contentctl_project/contentctl_infrastructure/builder/splunk_app_enrichment.py +++ b/bin/contentctl_project/contentctl_infrastructure/builder/splunk_app_enrichment.py @@ -5,43 +5,52 @@ import functools import pickle import shelve import os +import time SPLUNKBASE_API_URL = "https://apps.splunk.com/api/apps/entriesbyid/" APP_ENRICHMENT_CACHE_FILENAME = "lookups/APP_ENRICHMENT_CACHE.db" +NON_PERSISTENT_CACHE = {} + @functools.cache -def requests_get_helper(url:str)->bytes: - if not os.path.exists(APP_ENRICHMENT_CACHE_FILENAME): - print(f"Cache at {APP_ENRICHMENT_CACHE_FILENAME} not found - Creating it.") - cache = shelve.open(APP_ENRICHMENT_CACHE_FILENAME, flag='c', writeback=True) +def requests_get_helper(url:str, force_cached_or_offline:bool = False)->bytes: + if force_cached_or_offline: + if not os.path.exists(APP_ENRICHMENT_CACHE_FILENAME): + print(f"Cache at {APP_ENRICHMENT_CACHE_FILENAME} not found - Creating it.") + cache = shelve.open(APP_ENRICHMENT_CACHE_FILENAME, flag='c', writeback=True) + else: + cache = NON_PERSISTENT_CACHE + if url in cache: req_content = cache[url] + else: + try: + req = requests.get(url) + req_content = req.content + cache[url] = req_content + except Exception as e: + raise(Exception(f"ERROR - Failed to get Splunk App Enrichment at {SPLUNKBASE_API_URL}")) + + if force_cached_or_offline: cache.close() - return req_content - try: - req = requests.get(url) - req_content = req.content - cache[url] = req_content - cache.close() - return req_content - except Exception as e: - raise(Exception(f"ERROR - Failed to get Splunk App Enrichment at {SPLUNKBASE_API_URL}")) + + return req_content class SplunkAppEnrichment(): @classmethod def enrich_splunk_app(self, splunk_ta: str, force_cached_or_offline: bool = False) -> dict: + appurl = SPLUNKBASE_API_URL + splunk_ta splunk_app_enriched = dict() + try: - if force_cached_or_offline is True: - content = requests_get_helper(appurl) - response_dict = xmltodict.parse(content) - else: - response = requests.get(appurl) - response_dict = xmltodict.parse(response.content) + + content = requests_get_helper(appurl, force_cached_or_offline) + response_dict = xmltodict.parse(content) + # check if list since data changes depending on answer url, results = self._parse_splunkbase_response(response_dict) # grab the app name @@ -50,17 +59,15 @@ class SplunkAppEnrichment(): splunk_app_enriched['name'] = i['#text'] # grab out the splunkbase url if 'entriesbyid' in url: - if force_cached_or_offline is True: - content = requests_get_helper(url) - response_dict = xmltodict.parse(content) - else: - response = requests.get(url) - response_dict = xmltodict.parse(response.content) + content = requests_get_helper(url, force_cached_or_offline) + response_dict = xmltodict.parse(content) + #print(json.dumps(response_dict, indent=2)) url, results = self._parse_splunkbase_response(response_dict) # chop the url so we grab the splunkbase portion but not direct download splunk_app_enriched['url'] = url.rsplit('/', 4)[0] except requests.exceptions.ConnectionError as connErr: + print(f"There was a connErr for ta {splunk_ta}: {connErr}") # there was a connection error lets just capture the name splunk_app_enriched['name'] = splunk_ta splunk_app_enriched['url'] = '' From 45736b5ed7f39cbe019f5b46bc4a9f35d8a3040f Mon Sep 17 00:00:00 2001 From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com> Date: Thu, 30 Jun 2022 10:52:00 -0700 Subject: [PATCH 2/6] Updated CI/CD Workflow and python code in contentctl so that progress update are not output when running in CICD environment. Outputting all of this data causes thousands of lines of output logs to be generated in CICD. This info is only relevant when running locally in a tty. --- .github/workflows/build-and-validate.yml | 12 ++++----- .../application/factory/ba_factory.py | 12 ++++++--- .../application/factory/factory.py | 25 ++++++++++++------- .../adapter/obj_to_md_adapter.py | 11 +++++--- .../builder/attack_enrichment.py | 5 ++-- 5 files changed, 41 insertions(+), 24 deletions(-) diff --git a/.github/workflows/build-and-validate.yml b/.github/workflows/build-and-validate.yml index 475e1cd6e4..cf3bf42fba 100644 --- a/.github/workflows/build-and-validate.yml +++ b/.github/workflows/build-and-validate.yml @@ -97,8 +97,8 @@ jobs: - name: content_ctl validate run: | source .venv/bin/activate - python contentctl.py -p . ${{ steps.vars.outputs.skip_enrichment_var }} validate -pr ESCU > /dev/null - python contentctl.py -p . ${{ steps.vars.outputs.skip_enrichment_var }} validate -pr SSA > /dev/null + python contentctl.py -p . ${{ steps.vars.outputs.skip_enrichment_var }} validate -pr ESCU + python contentctl.py -p . ${{ steps.vars.outputs.skip_enrichment_var }} validate -pr SSA #Now generate the documentation (uses Node) @@ -120,8 +120,8 @@ jobs: run: | source .venv/bin/activate rm -rf dist/escu/default/data/ui/panels/*.xml - python contentctl.py --path . ${{ steps.vars.outputs.skip_enrichment_var }} generate --product ESCU --output dist/escu > /dev/null - python contentctl.py --path . ${{ steps.vars.outputs.skip_enrichment_var }} generate --product SSA --output dist/ssa > /dev/null + python contentctl.py --path . ${{ steps.vars.outputs.skip_enrichment_var }} generate --product ESCU --output dist/escu + python contentctl.py --path . ${{ steps.vars.outputs.skip_enrichment_var }} generate --product SSA --output dist/ssa - name: Copy lookups .mlmodel files run: | @@ -353,12 +353,12 @@ jobs: - name: Run doc-gen run: | source venv/bin/activate - python3 contentctl.py -p . docgen -o docs > /dev/null + python3 contentctl.py -p . docgen -o docs - name: Run reporting run: | source venv/bin/activate - python3 contentctl.py -p . reporting > /dev/null + python3 contentctl.py -p . reporting - name: Update github with new docs and package bits run: | diff --git a/bin/contentctl_project/contentctl_core/application/factory/ba_factory.py b/bin/contentctl_project/contentctl_core/application/factory/ba_factory.py index d5050d854c..7b7670ddb2 100644 --- a/bin/contentctl_project/contentctl_core/application/factory/ba_factory.py +++ b/bin/contentctl_project/contentctl_core/application/factory/ba_factory.py @@ -59,18 +59,24 @@ class BAFactory(): if 'ssa__' in file: progress_percent = ((index+1)/len(files_with_ssa)) * 100 try: + type_string = "UNKNOWN TYPE" if type == SecurityContentType.detections: - print(f"\r{'Detections Progress'.rjust(23)}: [{progress_percent:3.0f}%]...", end="", flush=True) + type_string = "Detections" self.input_dto.director.constructDetection(self.input_dto.detection_builder, file, [], [], [], self.output_dto.tests, {}, [], []) detection = self.input_dto.detection_builder.getObject() if not detection.deprecated and not detection.experimental: self.output_dto.detections.append(detection) elif type == SecurityContentType.unit_tests: - print(f"\r{'Unit Tests Progress'.rjust(23)}: [{progress_percent:3.0f}%]...", end="", flush=True) + type_string = "Unit Tests" self.input_dto.director.constructTest(self.input_dto.basic_builder, file) test = self.input_dto.basic_builder.getObject() self.output_dto.tests.append(test) - + else: + raise(Exception(f"Unsupported content type: [{type}]")) + + if (sys.stdout.isatty() and sys.stdin.isatty() and sys.stderr.isatty()): + print(f"\r{f'{type_string} Progress'.rjust(23)}: [{progress_percent:3.0f}%]...", end="", flush=True) + except ValidationError as e: print('\nValidation Error for file ' + file) print(e) diff --git a/bin/contentctl_project/contentctl_core/application/factory/factory.py b/bin/contentctl_project/contentctl_core/application/factory/factory.py index 9fa6e81cbe..3e4e760aef 100644 --- a/bin/contentctl_project/contentctl_core/application/factory/factory.py +++ b/bin/contentctl_project/contentctl_core/application/factory/factory.py @@ -165,47 +165,48 @@ class Factory(): # that printouts end at 100%, not some other number progress_percent = ((index+1)/len(files_without_ssa)) * 100 try: + type_string = "UNKNOWN TYPE" if type == SecurityContentType.lookups: - print(f"\r{'Lookups Progress'.rjust(23)}: [{progress_percent:3.0f}%]...", end="", flush=True) + type_string = "Lookups" self.input_dto.director.constructLookup(self.input_dto.basic_builder, file) self.output_dto.lookups.append(self.input_dto.basic_builder.getObject()) elif type == SecurityContentType.macros: - print(f"\r{'Playbooks Progress'.rjust(23)}: [{progress_percent:3.0f}%]...", end="", flush=True) + type_string = "Macros" self.input_dto.director.constructMacro(self.input_dto.basic_builder, file) self.output_dto.macros.append(self.input_dto.basic_builder.getObject()) elif type == SecurityContentType.deployments: - print(f"\r{'Deployments Progress'.rjust(23)}: [{progress_percent:3.0f}%]...", end="", flush=True) + type_string = "Deployments" self.input_dto.director.constructDeployment(self.input_dto.basic_builder, file) self.output_dto.deployments.append(self.input_dto.basic_builder.getObject()) elif type == SecurityContentType.playbooks: - print(f"\r{'Playbooks Progress'.rjust(23)}: [{progress_percent:3.0f}%]...", end="", flush=True) + type_string = "Playbooks" self.input_dto.director.constructPlaybook(self.input_dto.playbook_builder, file) self.output_dto.playbooks.append(self.input_dto.playbook_builder.getObject()) elif type == SecurityContentType.baselines: - print(f"\r{'Baselines Progress'.rjust(23)}: [{progress_percent:3.0f}%]...", end="", flush=True) + type_string = "Baselines" self.input_dto.director.constructBaseline(self.input_dto.baseline_builder, file, self.output_dto.deployments) baseline = self.input_dto.baseline_builder.getObject() self.output_dto.baselines.append(baseline) elif type == SecurityContentType.investigations: - print(f"\r{'Investigations Progress'.rjust(23)}: [{progress_percent:3.0f}%]...", end="", flush=True) + type_string = "Investigations" self.input_dto.director.constructInvestigation(self.input_dto.investigation_builder, file) investigation = self.input_dto.investigation_builder.getObject() self.output_dto.investigations.append(investigation) elif type == SecurityContentType.stories: - print(f"\r{'Stories Progress'.rjust(23)}: [{progress_percent:3.0f}%]...", end="", flush=True) + type_string = "Stories" self.input_dto.director.constructStory(self.input_dto.story_builder, file, self.output_dto.detections, self.output_dto.baselines, self.output_dto.investigations) story = self.input_dto.story_builder.getObject() self.output_dto.stories.append(story) elif type == SecurityContentType.detections: - print(f"\r{'Detections Progress'.rjust(23)}: [{progress_percent:3.0f}%]...", end="", flush=True) + type_string = "Detections" self.input_dto.director.constructDetection(self.input_dto.detection_builder, file, self.output_dto.deployments, self.output_dto.playbooks, self.output_dto.baselines, self.output_dto.tests, self.input_dto.attack_enrichment, self.output_dto.macros, @@ -214,10 +215,16 @@ class Factory(): self.output_dto.detections.append(detection) elif type == SecurityContentType.unit_tests: - print(f"\r{'Unit Tests Progress'.rjust(23)}: [{progress_percent:3.0f}%]...", end="", flush=True) + type_string = "Unit Tests" self.input_dto.director.constructTest(self.input_dto.basic_builder, file) test = self.input_dto.basic_builder.getObject() self.output_dto.tests.append(test) + + else: + raise Exception(f"Unsupported type: [{type}]") + + if (sys.stdout.isatty() and sys.stdin.isatty() and sys.stderr.isatty()): + print(f"\r{f'{type_string} Progress'.rjust(23)}: [{progress_percent:3.0f}%]...", end="", flush=True) except ValidationError as e: print('\nValidation Error for file ' + file) diff --git a/bin/contentctl_project/contentctl_infrastructure/adapter/obj_to_md_adapter.py b/bin/contentctl_project/contentctl_infrastructure/adapter/obj_to_md_adapter.py index e8cdfb5d03..0ec3efd1e8 100644 --- a/bin/contentctl_project/contentctl_infrastructure/adapter/obj_to_md_adapter.py +++ b/bin/contentctl_project/contentctl_infrastructure/adapter/obj_to_md_adapter.py @@ -1,6 +1,6 @@ import os import asyncio - +import sys from bin.contentctl_project.contentctl_core.application.adapter.adapter import Adapter from bin.contentctl_project.contentctl_core.domain.entities.enums.enums import SecurityContentType from bin.contentctl_project.contentctl_infrastructure.adapter.jinja_writer import JinjaWriter @@ -14,7 +14,8 @@ class ObjToMdAdapter(Adapter): self.files_to_write = sum([len(obj) for obj in objects]) self.index = 0 progress_percent = ((self.index+1)/self.files_to_write) * 100 - print(f"\r{'Docgen Progress'.rjust(23)}: [{progress_percent:3.0f}%]...", end="", flush=True) + if (sys.stdout.isatty() and sys.stdin.isatty() and sys.stderr.isatty()): + print(f"\r{'Docgen Progress'.rjust(23)}: [{progress_percent:3.0f}%]...", end="", flush=True) attack_tactics = set() datamodels = set() @@ -65,7 +66,8 @@ class ObjToMdAdapter(Adapter): for obj in objects: progress_percent = ((self.index+1)/self.files_to_write) * 100 self.index+=1 - print(f"\r{'Docgen Progress'.rjust(23)}: [{progress_percent:3.0f}%]...", end="", flush=True) + if (sys.stdout.isatty() and sys.stdin.isatty() and sys.stderr.isatty()): + print(f"\r{'Docgen Progress'.rjust(23)}: [{progress_percent:3.0f}%]...", end="", flush=True) JinjaWriter.writeObject(template_name, os.path.join(output_path, obj.name.lower().replace(' ', '_') + '.md'), obj) @@ -73,6 +75,7 @@ class ObjToMdAdapter(Adapter): for obj in objects: progress_percent = ((self.index+1)/self.files_to_write) * 100 self.index+=1 - print(f"\r{'Docgen Progress'.rjust(23)}: [{progress_percent:3.0f}%]...", end="", flush=True) + if (sys.stdout.isatty() and sys.stdin.isatty() and sys.stderr.isatty()): + print(f"\r{'Docgen Progress'.rjust(23)}: [{progress_percent:3.0f}%]...", end="", flush=True) JinjaWriter.writeObject(template_name, os.path.join(output_path, obj.date + '-' + obj.name.lower().replace(' ', '_') + '.md'), obj) \ No newline at end of file diff --git a/bin/contentctl_project/contentctl_infrastructure/builder/attack_enrichment.py b/bin/contentctl_project/contentctl_infrastructure/builder/attack_enrichment.py index 5a648fca71..02e5c3174d 100644 --- a/bin/contentctl_project/contentctl_infrastructure/builder/attack_enrichment.py +++ b/bin/contentctl_project/contentctl_infrastructure/builder/attack_enrichment.py @@ -3,7 +3,7 @@ import csv import os from posixpath import split from typing import Optional - +import sys from attackcti import attack_client import logging @@ -43,7 +43,8 @@ class AttackEnrichment(): for index, technique in enumerate(all_enterprise['techniques']): progress_percent = ((index+1)/len(all_enterprise['techniques'])) * 100 - print(f"\r\t{'MITRE Technique Progress'.rjust(23)}: [{progress_percent:3.0f}%]...", end="", flush=True) + if (sys.stdout.isatty() and sys.stdin.isatty() and sys.stderr.isatty()): + print(f"\r\t{'MITRE Technique Progress'.rjust(23)}: [{progress_percent:3.0f}%]...", end="", flush=True) apt_groups = [] for relationship in enterprise_relationships: if (relationship['target_ref'] == technique['id']) and relationship['source_ref'].startswith('intrusion-set'): From cb1ba3c884ed5b1a8a3bb3b43c0e2c203de3803b Mon Sep 17 00:00:00 2001 From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com> Date: Thu, 30 Jun 2022 11:12:50 -0700 Subject: [PATCH 3/6] Added slightly more verbose printing to track high-level progress in CICD. --- .../contentctl_core/application/factory/ba_factory.py | 9 ++++++++- .../contentctl_core/application/factory/factory.py | 9 ++++++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/bin/contentctl_project/contentctl_core/application/factory/ba_factory.py b/bin/contentctl_project/contentctl_core/application/factory/ba_factory.py index 7b7670ddb2..a28f26145b 100644 --- a/bin/contentctl_project/contentctl_core/application/factory/ba_factory.py +++ b/bin/contentctl_project/contentctl_core/application/factory/ba_factory.py @@ -50,6 +50,10 @@ class BAFactory(): validation_error_found = False files_with_ssa = [f for f in files if 'ssa___' in f] + + already_ran = False + progress_percent = 0 + type_string = "UNKNOWN TYPE" for index,file in enumerate(files_with_ssa): #Index + 1 because we are zero indexed, not 1 indexed. This ensures @@ -74,13 +78,16 @@ class BAFactory(): else: raise(Exception(f"Unsupported content type: [{type}]")) - if (sys.stdout.isatty() and sys.stdin.isatty() and sys.stderr.isatty()): + if (sys.stdout.isatty() and sys.stdin.isatty() and sys.stderr.isatty()) or not already_ran: + already_ran = True print(f"\r{f'{type_string} Progress'.rjust(23)}: [{progress_percent:3.0f}%]...", end="", flush=True) except ValidationError as e: print('\nValidation Error for file ' + file) print(e) validation_error_found = True + + print(f"\r{f'{type_string} Progress'.rjust(23)}: [{progress_percent:3.0f}%]...", end="", flush=True) print("Done!") if validation_error_found: diff --git a/bin/contentctl_project/contentctl_core/application/factory/factory.py b/bin/contentctl_project/contentctl_core/application/factory/factory.py index 3e4e760aef..72c549a498 100644 --- a/bin/contentctl_project/contentctl_core/application/factory/factory.py +++ b/bin/contentctl_project/contentctl_core/application/factory/factory.py @@ -157,6 +157,10 @@ class Factory(): thread.join() ''' + already_ran = False + progress_percent = 0 + type_string = "UNKNOWN TYPE" + #Non threaded, production version of the construction code files_without_ssa = [f for f in files if 'ssa___' not in f] for index,file in enumerate(files_without_ssa): @@ -223,13 +227,16 @@ class Factory(): else: raise Exception(f"Unsupported type: [{type}]") - if (sys.stdout.isatty() and sys.stdin.isatty() and sys.stderr.isatty()): + if (sys.stdout.isatty() and sys.stdin.isatty() and sys.stderr.isatty()) or not already_ran: + already_ran = True print(f"\r{f'{type_string} Progress'.rjust(23)}: [{progress_percent:3.0f}%]...", end="", flush=True) except ValidationError as e: print('\nValidation Error for file ' + file) print(e) validation_error_found = True + + print(f"\r{f'{type_string} Progress'.rjust(23)}: [{progress_percent:3.0f}%]...", end="", flush=True) print("Done!") if validation_error_found: From 52abbc940808b3b1926cab196537be2c524ab253 Mon Sep 17 00:00:00 2001 From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com> Date: Thu, 30 Jun 2022 15:47:29 -0700 Subject: [PATCH 4/6] Changing enrichment failures from error prints to error prints AND failures. --- .../contentctl_infrastructure/builder/cve_enrichment.py | 6 ++++-- .../builder/splunk_app_enrichment.py | 5 +++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/bin/contentctl_project/contentctl_infrastructure/builder/cve_enrichment.py b/bin/contentctl_project/contentctl_infrastructure/builder/cve_enrichment.py index 01c9a770aa..9a57f2a423 100644 --- a/bin/contentctl_project/contentctl_infrastructure/builder/cve_enrichment.py +++ b/bin/contentctl_project/contentctl_infrastructure/builder/cve_enrichment.py @@ -4,6 +4,7 @@ import functools import os import shelve import time +import sys CVESSEARCH_API_URL = 'https://cve.circl.lu' CVE_CACHE_FILENAME = "lookups/CVE_CACHE.db" @@ -58,9 +59,10 @@ class CveEnrichment(): except TypeError as TypeErr: # there was a error calling the circl api lets just empty the object print("WARNING, issue enriching {0}, with error: {1}".format(cve_id, str(TypeErr))) - cve_enriched = dict() + sys.exit(1) + except Exception as e: print("WARNING - {0}".format(str(e))) - cve_enriched = dict() + sys.exit(1) return cve_enriched \ No newline at end of file diff --git a/bin/contentctl_project/contentctl_infrastructure/builder/splunk_app_enrichment.py b/bin/contentctl_project/contentctl_infrastructure/builder/splunk_app_enrichment.py index eedab9d13f..3dd11a40bb 100644 --- a/bin/contentctl_project/contentctl_infrastructure/builder/splunk_app_enrichment.py +++ b/bin/contentctl_project/contentctl_infrastructure/builder/splunk_app_enrichment.py @@ -71,6 +71,11 @@ class SplunkAppEnrichment(): # there was a connection error lets just capture the name splunk_app_enriched['name'] = splunk_ta splunk_app_enriched['url'] = '' + except Exception as e: + print(f"There was an unknown error enriching the Splunk TA [{splunk_ta}]: {str(e)}") + splunk_app_enriched['name'] = splunk_ta + splunk_app_enriched['url'] = '' + return splunk_app_enriched From 80af56d3970c5472eef981e44bc7af9e822c4dcc Mon Sep 17 00:00:00 2001 From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com> Date: Thu, 7 Jul 2022 14:33:57 -0700 Subject: [PATCH 5/6] Fixing the rare case where enrichment fails due to CVESearch API failure, causing the entire workflow to fail. Now, we will retry enrichment up to a total of 3 times. This should virtually eliminate all CVE Enrichment failures. If they do still occur, we will receive a descriptive message indicating why there was a failure instead of the cryptic message that we were getting before. --- .../builder/cve_enrichment.py | 32 +++++++++++++++---- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/bin/contentctl_project/contentctl_infrastructure/builder/cve_enrichment.py b/bin/contentctl_project/contentctl_infrastructure/builder/cve_enrichment.py index 9a57f2a423..8260979f02 100644 --- a/bin/contentctl_project/contentctl_infrastructure/builder/cve_enrichment.py +++ b/bin/contentctl_project/contentctl_infrastructure/builder/cve_enrichment.py @@ -11,8 +11,13 @@ CVE_CACHE_FILENAME = "lookups/CVE_CACHE.db" NON_PERSISTENT_CACHE = {} + + @functools.cache -def cvesearch_helper(url:str, cve_id:str, force_cached_or_offline:bool=False): +def cvesearch_helper(url:str, cve_id:str, force_cached_or_offline:bool=False, max_api_attempts:int=3): + if max_api_attempts < 1: + raise(Exception(f"The minimum number of CVESearch API attempts is 1. You have passed {max_api_attempts}")) + if force_cached_or_offline: if not os.path.exists(CVE_CACHE_FILENAME): print(f"Cache at {CVE_CACHE_FILENAME} not found - Creating it.") @@ -23,14 +28,27 @@ def cvesearch_helper(url:str, cve_id:str, force_cached_or_offline:bool=False): result = cache[cve_id] #print(f"hit cve_enrichment: {time.time() - start:.2f}") else: - try: - cve = cvesearch_id_helper(url) - result = cve.id(cve_id) - except Exception as e: - raise(Exception(f"The option 'force_cached_or_offline' was used, but {cve_id} not found in {CVE_CACHE_FILENAME} and unable to connect to {CVESSEARCH_API_URL}")) + api_attempts_remaining = max_api_attempts + while api_attempts_remaining > 0: + + api_attempts_remaining -= 1 + + start = time.time() + try: + cve = cvesearch_id_helper(url) + result = cve.id(cve_id) + break + except Exception as e: + if api_attempts_remaining > 0: + print(f"The option 'force_cached_or_offline' was used, but {cve_id} not found in {CVE_CACHE_FILENAME} and unable to connect to {CVESSEARCH_API_URL}: {str(e)}") + print(f"Retrying the CVESearch API up to {api_attempts_remaining} more times...") + else: + raise(Exception(f"The option 'force_cached_or_offline' was used, but {cve_id} not found in {CVE_CACHE_FILENAME} and unable to connect to {CVESSEARCH_API_URL} after {max_api_attempts} attempts: {str(e)}")) + if result is None: raise(Exception(f'CveEnrichment for [ {cve_id} ] failed - CVE does not exist')) cache[cve_id] = result + if force_cached_or_offline: cache.close() @@ -52,7 +70,7 @@ class CveEnrichment(): cve_enriched = dict() try: - result = cvesearch_helper(CVESSEARCH_API_URL, cve_id) + result = cvesearch_helper(CVESSEARCH_API_URL, cve_id, force_cached_or_offline) cve_enriched['id'] = cve_id cve_enriched['cvss'] = result['cvss'] cve_enriched['summary'] = result['summary'] From 8ed93724019e631002952a329357782aa9adb38a Mon Sep 17 00:00:00 2001 From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com> Date: Thu, 7 Jul 2022 14:40:39 -0700 Subject: [PATCH 6/6] On CVESearch API failure, added a brief sleep to allow the API to begin working again before trying to resolve it again. --- .../contentctl_infrastructure/builder/cve_enrichment.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/bin/contentctl_project/contentctl_infrastructure/builder/cve_enrichment.py b/bin/contentctl_project/contentctl_infrastructure/builder/cve_enrichment.py index 8260979f02..ba158219e6 100644 --- a/bin/contentctl_project/contentctl_infrastructure/builder/cve_enrichment.py +++ b/bin/contentctl_project/contentctl_infrastructure/builder/cve_enrichment.py @@ -14,7 +14,7 @@ NON_PERSISTENT_CACHE = {} @functools.cache -def cvesearch_helper(url:str, cve_id:str, force_cached_or_offline:bool=False, max_api_attempts:int=3): +def cvesearch_helper(url:str, cve_id:str, force_cached_or_offline:bool=False, max_api_attempts:int=3, retry_sleep_seconds:int=5): if max_api_attempts < 1: raise(Exception(f"The minimum number of CVESearch API attempts is 1. You have passed {max_api_attempts}")) @@ -41,7 +41,8 @@ def cvesearch_helper(url:str, cve_id:str, force_cached_or_offline:bool=False, ma except Exception as e: if api_attempts_remaining > 0: print(f"The option 'force_cached_or_offline' was used, but {cve_id} not found in {CVE_CACHE_FILENAME} and unable to connect to {CVESSEARCH_API_URL}: {str(e)}") - print(f"Retrying the CVESearch API up to {api_attempts_remaining} more times...") + print(f"Retrying the CVESearch API up to {api_attempts_remaining} more times after a sleep of {retry_sleep_seconds} seconds...") + time.sleep(retry_sleep_seconds) else: raise(Exception(f"The option 'force_cached_or_offline' was used, but {cve_id} not found in {CVE_CACHE_FILENAME} and unable to connect to {CVESSEARCH_API_URL} after {max_api_attempts} attempts: {str(e)}"))