diff --git a/cmd/enrichment/enrichment/bootstrap.py b/cmd/enrichment/enrichment/bootstrap.py index 6b5539b..a34e7e6 100644 --- a/cmd/enrichment/enrichment/bootstrap.py +++ b/cmd/enrichment/enrichment/bootstrap.py @@ -69,12 +69,13 @@ def validate_task_names(input_tasks: List[str], all_tasks: List[str]): @inject -def main(container: Container, config=Provide[Container.config]): - configure_logger(False, config["log_level"], config["environment"].value) +def main(container: Container, config: EnrichmentSettings = Provide[Container.config2]): + configure_logger(config.environment, config.log_level, config.log_color_enabled) + logger = structlog.get_logger(module=__name__) loop = asyncio.get_event_loop() - if config["environment"].is_development(): + if config.environment.is_development(): # loop.set_debug(True) loop.slow_callback_duration = 1 diff --git a/cmd/enrichment/enrichment/cli/submit_to_nemesis/submit_to_nemesis.py b/cmd/enrichment/enrichment/cli/submit_to_nemesis/submit_to_nemesis.py index 3389618..3f140a4 100644 --- a/cmd/enrichment/enrichment/cli/submit_to_nemesis/submit_to_nemesis.py +++ b/cmd/enrichment/enrichment/cli/submit_to_nemesis/submit_to_nemesis.py @@ -23,6 +23,7 @@ import yaml from enrichment.cli.submit_to_nemesis.file_monitoring import monitor_directory from nemesiscommon.apiclient import FileUploadRequest, NemesisApiClient from nemesiscommon.logging import configure_logger +from nemesiscommon.settings import EnvironmentSettings from structlog.typing import FilteringBoundLogger urllib3.disable_warnings() @@ -124,7 +125,7 @@ async def get_config() -> dict[str, str]: config["log_level"] = args.log_level config["cookies"] = args.cookies - configure_logger(False, config["log_level"]) + configure_logger(EnvironmentSettings.DEVELOPMENT, config["log_level"], True) global logger logger = structlog.getLogger() logger.debug("Config", config=config) @@ -148,7 +149,7 @@ async def get_timestamp(days_to_add=0): return dt.strftime("%Y-%m-%dT%H:%M:%S.000Z") -async def get_metadata(config, data_type="file_data"): +async def get_metadata(config: dict[str, str], data_type: str): metadata = {} metadata["agent_id"] = config["operator_name"] metadata["agent_type"] = "submit_to_nemesis" @@ -184,7 +185,7 @@ async def get_nemesis_api_client(config) -> httpx.AsyncClient: return api_client -async def nemesis_post_file(config: Dict[str, str], file_path: str): +async def nemesis_post_file(config: dict[str, str], file_path: str): """ Takes a series of raw file bytes and POSTs it to the NEMESIS /file API endpoint. @@ -226,7 +227,7 @@ async def nemesis_post_file(config: Dict[str, str], file_path: str): return resp.object_id -async def nemesis_post_data(config, data): +async def nemesis_post_data(config: dict[str, str], data): """ Takes a json blob and POSTs it to the NEMESIS /data API endpoint. @@ -286,149 +287,112 @@ async def submit_random_cookies(config, num_cookies=1000) -> uuid.UUID | None: return uuid.UUID(resp["object_id"]) if resp else None -async def process_file(config, file_path) -> uuid.UUID | None: +def string_matches_regexes(regexes: list[str], file_path: str) -> bool: """ - Takes a configuration dictionary and file path and uploads the file to Nemesis, - then posting the file_data message. + Returns True if the regex, False otherwise. """ - if re.match(r".*services_api.*\.json$", file_path): # NOTE: This must not conflict with the example Seatbelt services either - with open(file_path, "r") as f: - services_json_raw = f.read() - services_json = json.loads(services_json_raw) - resp = await nemesis_post_data(config, services_json) - return uuid.UUID(resp["object_id"]) if resp else None + for matcher in regexes: + if re.match(matcher, file_path): + return True - elif re.match(r".*file_information.*\.json$", file_path): - with open(file_path, "r") as f: - file_information = f.read() - file_information_json = json.loads(file_information) - resp = await nemesis_post_data(config, file_information_json) - return uuid.UUID(resp["object_id"]) if resp else None + return False - elif re.match(r".*registry_value.*\.json$", file_path): - with open(file_path, "r") as f: - registry_value = f.read() - registry_value_json = json.loads(registry_value) - resp = await nemesis_post_data(config, registry_value_json) - return uuid.UUID(resp["object_id"]) if resp else None - elif re.match(r".*authentication_data.*\.json$", file_path): - with open(file_path, "r") as f: - authentication_data = f.read() - authentication_data_json = json.loads(authentication_data) - resp = await nemesis_post_data(config, authentication_data_json) - return uuid.UUID(resp["object_id"]) if resp else None +async def post_json_to_api(config: dict[str, str], file_path: str) -> uuid.UUID | None: + with open(file_path, "r") as f: + services_json_raw = f.read() + services_json = json.loads(services_json_raw) + resp = await nemesis_post_data(config, services_json) - elif re.match(r".*cookies.*\.json$", file_path): - with open(file_path, "r") as f: - cookie = f.read() - cookie_json = json.loads(cookie) - resp = await nemesis_post_data(config, cookie_json) - return uuid.UUID(resp["object_id"]) if resp else None - - elif re.match(r".*named_pipes.*\.json$", file_path): - with open(file_path, "r") as f: - named_pipes = f.read() - named_pipes_json = json.loads(named_pipes) - resp = await nemesis_post_data(config, named_pipes_json) - return uuid.UUID(resp["object_id"]) if resp else None - - elif re.match(r".*network_connections.*\.json$", file_path): - with open(file_path, "r") as f: - network_connections = f.read() - network_connections_json = json.loads(network_connections) - resp = await nemesis_post_data(config, network_connections_json) - return uuid.UUID(resp["object_id"]) if resp else None - - elif "bof_reg_collect.nemesis" in file_path: - object_id = await nemesis_post_file(config, file_path) - if not object_id: - logger.error("No nemesis_file_id returned when uploading bof_reg_collect.nemesis") - return None - else: - logger.debug("bof_reg_collect.nemesis uploaded to Nemesis", file_uuid=object_id) - - metadata = await get_metadata(config, "raw_data") - - raw_data = {} - raw_data["tags"] = ["bof_reg_collect"] - raw_data["data"] = object_id - raw_data["is_file"] = True - - resp = await nemesis_post_data(config, {"metadata": metadata, "data": [raw_data]}) if resp: - logger.debug("bof_reg_collect.nemesis data sent to Nemesis", file_uuid=resp["object_id"]) + logger.debug("File data sent to Nemesis", message_uuid=resp["object_id"], file_path=file_path) return uuid.UUID(resp["object_id"]) else: return None - elif "dpapi_domain_backupkey.json" in file_path: - object_id = await nemesis_post_file(config, file_path) - if not object_id: - logger.error("No nemesis_file_id returned when uploading DPAPI domain backup key") - return None - else: - logger.debug("DPAPI domain backup key file uploaded to Nemesis", file_uuid=object_id) - metadata = await get_metadata(config, "raw_data") - - raw_data = {} - raw_data["tags"] = ["dpapi_domain_backupkey"] - raw_data["data"] = object_id - raw_data["is_file"] = True - - resp = await nemesis_post_data(config, {"metadata": metadata, "data": [raw_data]}) - if resp: - logger.debug("DPAPI backupkey file data sent to Nemesis", file_uuid=resp["object_id"]) - return uuid.UUID(resp["object_id"]) - else: - return None - - elif "seatbelt" in file_path: - object_id = await nemesis_post_file(config, file_path) - if not object_id: - logger.error("No nemesis_file_id returned from seatbelt file upload", file_path=file_path) - return None - else: - logger.debug("Seatbelt file uploaded to Nemesis", file_uuid=object_id, file_path=file_path) - - metadata = await get_metadata(config, "raw_data") - - raw_data = {} - raw_data["tags"] = ["seatbelt_json"] - raw_data["data"] = object_id - raw_data["is_file"] = True - - resp = await nemesis_post_data(config, {"metadata": metadata, "data": [raw_data]}) - if resp: - logger.debug("Seatbelt NDJSON file data sent to Nemesis", file_uuid=resp["object_id"]) - return uuid.UUID(resp["object_id"]) - else: - return None +async def submit_file_with_raw_data_tag(config: dict[str, str], file_path: str, tags: List[str]) -> uuid.UUID | None: + object_id = await nemesis_post_file(config, file_path) + if not object_id: + logger.error("No nemesis_file_id returned when uploading raw_data file", tags=tags, file_path=file_path) + return None else: - file_data = {} - file_path = os.path.abspath(file_path) - file_data["path"] = file_path + logger.debug("raw_data file uploaded to Nemesis", file_uuid=object_id, file_path=file_path) - file_data["size"] = os.path.getsize(file_path) + metadata = await get_metadata(config, "raw_data") - object_id = await nemesis_post_file(config, file_path) - if not object_id: - return - else: - logger.debug("File uploaded to Nemesis", file_uuid=object_id, file_path=file_path) + raw_data = {} + raw_data["tags"] = tags + raw_data["data"] = object_id + raw_data["is_file"] = True - file_data["object_id"] = object_id - metadata = await get_metadata(config) + resp = await nemesis_post_data(config, {"metadata": metadata, "data": [raw_data]}) + if resp: + logger.debug("raw_data file data sent to Nemesis", message_uuid=resp["object_id"], file_path=file_path, tags=tags) + return uuid.UUID(resp["object_id"]) + else: + return None - # post to the Nemesis data API (`data` needs to be an array of dictionaries!) - resp = await nemesis_post_data(config, {"metadata": metadata, "data": [file_data]}) - if resp: - logger.debug("File data submitted to Nemesis", file_uuid=resp["object_id"], path=file_path) - return uuid.UUID(resp["object_id"]) - else: - return None + +async def submit_file(config: dict[str, str], file_path: str) -> uuid.UUID | None: + file_data = {} + file_path = os.path.abspath(file_path) + file_data["path"] = file_path + + file_data["size"] = os.path.getsize(file_path) + + object_id = await nemesis_post_file(config, file_path) + if not object_id: + return + else: + logger.debug("File uploaded to Nemesis", file_uuid=object_id, file_path=file_path) + + file_data["object_id"] = object_id + metadata = await get_metadata(config, "file_data") + + # post to the Nemesis data API (`data` needs to be an array of dictionaries!) + resp = await nemesis_post_data(config, {"metadata": metadata, "data": [file_data]}) + if resp: + logger.debug("File data submitted to Nemesis", file_uuid=resp["object_id"], path=file_path) + return uuid.UUID(resp["object_id"]) + else: + return None + + +async def process_file(config: dict[str, str], file_path: str) -> uuid.UUID | None: + """ + Takes a configuration dictionary and file path, and uploads the file to Nemesis depending on what the filename is. + """ + api_json_regexes = [ + r".*authentication_data.*\.json$", + r".*cookies.*\.json$", + # r".*file_data.*\.json$", # Removing for now since normal file uploads use this + r".*file_information.*\.json$", + r".*named_pipes.*\.json$", + r".*network_connections.*\.json$", + r".*path_list.*\.json$", + r".*process_data.*\.json$", + r".*registry_value.*\.json$", + r".*services_api.*\.json$", # NOTE: This must not conflict with the example Seatbelt services either + ] + + dpapi_domain_backupkey_regex = r".*dpapi_domain_backupkey.*\.json$" + seatbelt_json_regex = r".*seatbelt.*\.json$" + bof_reg_collect_regex = r".*bof_reg_collect.*\.nemesis$" + + # Process files differently depending on how they're named + if string_matches_regexes(api_json_regexes, file_path): + return await post_json_to_api(config, file_path) + elif string_matches_regexes([dpapi_domain_backupkey_regex], file_path): + return await submit_file_with_raw_data_tag(config, file_path, ["dpapi_domain_backupkey_json"]) + elif string_matches_regexes([seatbelt_json_regex], file_path): + return await submit_file_with_raw_data_tag(config, file_path, ["seatbelt_json"]) + elif string_matches_regexes([bof_reg_collect_regex], file_path): + return await submit_file_with_raw_data_tag(config, file_path, ["bof_reg_collect"]) + else: + # Default case: just upload the file + return await submit_file(config, file_path) async def process_folder(config, folder_path) -> AsyncIterator: @@ -546,7 +510,7 @@ def exception_handler(e, args): logger.exception("Error processing file", args=args) -async def submit_paths_concurrently(config, paths: List[str], workers: int, delay: float = 0) -> AsyncIterator[Tuple[str, uuid.UUID]]: +async def submit_paths_concurrently(config: dict[str, str], paths: List[str], workers: int, delay: float = 0) -> AsyncIterator[Tuple[str, uuid.UUID]]: """Submits files to Nemesis concurrently. Args: diff --git a/cmd/enrichment/enrichment/tasks/elastic_connector.py b/cmd/enrichment/enrichment/tasks/elastic_connector.py index 11d10f2..20fd67a 100644 --- a/cmd/enrichment/enrichment/tasks/elastic_connector.py +++ b/cmd/enrichment/enrichment/tasks/elastic_connector.py @@ -257,7 +257,7 @@ class ElasticConnector(TaskInterface): @aio.time(Summary("send_to_elastic", "Time spent submitting data directly to Elastic")) # type: ignore async def send_to_elastic(self, index: ElasticIndex, data: dict[str, Any]): - await logger.adebug("Submitting document to the Elastic", index=index) + await logger.adebug("Submitting document to Elastic", index=index) # try: # resp = await self.es_client.index(index=index, document=data) diff --git a/cmd/nlp/nlp/__main__.py b/cmd/nlp/nlp/__main__.py index f70b85c..90ed4a0 100644 --- a/cmd/nlp/nlp/__main__.py +++ b/cmd/nlp/nlp/__main__.py @@ -6,11 +6,10 @@ from typing import Any, Optional # 3rd Party Libraries import structlog from nemesiscommon.logging import configure_logger - from nlp.app import App from nlp.settings import config -configure_logger(False, config.log_level, config.environment.value) +configure_logger(config.environment, config.log_level, config.log_color_enabled) logger = structlog.get_logger(module=__name__) @@ -31,9 +30,7 @@ def main(): loop.run_until_complete(task) -async def shutdown( - loop: asyncio.AbstractEventLoop, signal: Optional[signal.Signals] = None -): +async def shutdown(loop: asyncio.AbstractEventLoop, signal: Optional[signal.Signals] = None): """Cleanup tasks tied to the service's shutdown.""" if signal: diff --git a/cmd/passwordcracker/passwordcracker/bootstrap.py b/cmd/passwordcracker/passwordcracker/bootstrap.py index 231c952..7d6fc4e 100644 --- a/cmd/passwordcracker/passwordcracker/bootstrap.py +++ b/cmd/passwordcracker/passwordcracker/bootstrap.py @@ -29,12 +29,12 @@ async def amain(container: Container): @inject -def main(container: Container, config=Provide[Container.config]): - configure_logger(False, config["log_level"], config["environment"].value) +def main(container: Container, config: PasswordCrackerSettings = Provide[Container.config2]): + configure_logger(config.environment, config.log_level, config.log_color_enabled) logger = structlog.get_logger(module=__name__) loop = asyncio.get_event_loop() - if config["environment"].is_development(): + if config.environment.is_development(): loop.slow_callback_duration = 1 signals = (signal.SIGHUP, signal.SIGTERM, signal.SIGINT) diff --git a/docs/nemesis-cli.md b/docs/nemesis-cli.md index fe9f212..e121732 100644 --- a/docs/nemesis-cli.md +++ b/docs/nemesis-cli.md @@ -22,6 +22,7 @@ If a required configuration value is not supplied, nemesis-cli will check if it | STORAGE_PROVIDER | storage_provider | --storage_provider | Storage provider to use, either `minio` (default) or `aws` | | ASSESSMENT_ID | assessment_id | --assessment_id | An ID for the assessment | | NEMESIS_HTTP_SERVER | nemesis_http_server | --nemesis_http_server | The public HTTP server of the Nemesis server (for link creation). The port used here must match the port of the ingress-nginx-controller service in skaffold.yaml (port 8080 by default) | +| ENVIRONMENT | environment | --environment | The environment Nemesis is running in. Possible value: development,production,testing. Production results in JSON-structured logs. Other environments result in human-readable logs. | | LOG_LEVEL | log_level | --log_level | (optional) Python logging level. Possible values: DEBUG, INFO, WARNING, ERROR, CRITICAL | | DATA_EXPIRATION_DAYS | data_expiration_days | --data_expiration_days | The number of days to set for data expiration (default 100) | | DISABLE_SLACK_ALERTING | disable_slack_alerting | --disable_slack_alerting | Should slack alerting be disabled? Possible values: True/False | @@ -44,10 +45,10 @@ If a required configuration value is not supplied, nemesis-cli will check if it | RABBITMQ_ERLANG_COOKIE | rabbitmq_erlang_cookie | --rabbitmq_erlang_cookie | Password to allow RabbitMQ nodes to communicate (default: random 24 characters) | # Example: Specifying Nemesis options using a configuration file -1. In the root of the repo, copy the example config to another file: +1. In the root of the repo, copy the example config `nemesis.config.example` to another file: ``` cd /path/to/Nemesis -cp nemesis.config.example nemesis.config +cp nemesis.config.example my.nemesis.config ``` 2. Edit the options in `my.nemesis.config` to you desired values 3. Setup the Kubernetes environment by running `python3 nemesis-cli.py -c my.nemesis.config` @@ -58,6 +59,7 @@ The following configures Nemesis using CLI arguments, setting all services to us python3 nemesis-cli.py \ --assessment_id ASSESS-TEST \ --nemesis_http_server http://192.168.230.42:8080/ \ + --environment development \ --disable_slack_alerting True \ --basic_auth_password PASSWORD \ --basic_auth_user nemesis \ diff --git a/kubernetes/dotnet/deployment.yaml b/kubernetes/dotnet/deployment.yaml index 61c586b..dda4832 100644 --- a/kubernetes/dotnet/deployment.yaml +++ b/kubernetes/dotnet/deployment.yaml @@ -27,7 +27,10 @@ spec: - name: DATA_DOWNLOAD_DIR value: "/tmp" - name: ENVIRONMENT - value: development + valueFrom: + configMapKeyRef: + name: operation-config + key: environment - name: PROMETHEUS_DISABLE_CREATED_SERIES value: "True" # - name: PROMETHEUS_PORT diff --git a/kubernetes/enrichment/deployment.yaml b/kubernetes/enrichment/deployment.yaml index bf994ce..32d2658 100644 --- a/kubernetes/enrichment/deployment.yaml +++ b/kubernetes/enrichment/deployment.yaml @@ -40,7 +40,10 @@ spec: - name: DATA_DOWNLOAD_DIR value: "/tmp" - name: ENVIRONMENT - value: development + valueFrom: + configMapKeyRef: + name: operation-config + key: environment - name: CRACK_WORDLIST_TOP_WORDS value: "10000" # either 10,000 or 100,000 diff --git a/kubernetes/nlp/deployment.yaml b/kubernetes/nlp/deployment.yaml index a9ba0ad..ca6f625 100644 --- a/kubernetes/nlp/deployment.yaml +++ b/kubernetes/nlp/deployment.yaml @@ -20,7 +20,10 @@ spec: containers: - env: - name: ENVIRONMENT - value: development + valueFrom: + configMapKeyRef: + name: operation-config + key: environment - name: LOG_LEVEL valueFrom: configMapKeyRef: diff --git a/kubernetes/passwordcracker/deployment.yaml b/kubernetes/passwordcracker/deployment.yaml index bee2e27..4b61d10 100644 --- a/kubernetes/passwordcracker/deployment.yaml +++ b/kubernetes/passwordcracker/deployment.yaml @@ -31,7 +31,10 @@ spec: - name: DISABLE_ALERTING value: "False" - name: ENVIRONMENT - value: development + valueFrom: + configMapKeyRef: + name: operation-config + key: environment - name: LOG_LEVEL valueFrom: configMapKeyRef: diff --git a/kubernetes/pgadmin/statefulset.yaml b/kubernetes/pgadmin/statefulset.yaml index 9a21c35..937cc6b 100644 --- a/kubernetes/pgadmin/statefulset.yaml +++ b/kubernetes/pgadmin/statefulset.yaml @@ -37,7 +37,7 @@ spec: - name: SCRIPT_NAME value: /pgadmin/ - name: MAX_LOGIN_ATTEMPTS - value: "10" + value: "15" ports: - name: http containerPort: 80 diff --git a/nemesis-cli.py b/nemesis-cli.py index a79a88d..8924fd2 100755 --- a/nemesis-cli.py +++ b/nemesis-cli.py @@ -29,7 +29,7 @@ try: import boto3 from passlib.hash import apr_md5_crypt from vyper import v -except: +except ModuleNotFoundError: logger.error("Please run `pip3 install boto3 vyper-config passlib`") sys.exit(1) @@ -61,9 +61,7 @@ def print_logo(): def get_random_password(length=24): """Gets a random password of the specified length.""" - return "".join( - secrets.choice(string.ascii_letters + string.digits) for i in range(length) - ) + return "".join(secrets.choice(string.ascii_letters + string.digits) for i in range(length)) def run_cmd(cmd, show_error=False): @@ -95,174 +93,109 @@ def get_kubectl_value(key): """Gets a specific aws configmap/key value that is already set in kubectl.""" if key == "aws_region": - return run_cmd( - "kubectl get configmaps aws-config -o=go-template='{{index .data \"aws-default-region\"}}'" - ) + return run_cmd("kubectl get configmaps aws-config -o=go-template='{{index .data \"aws-default-region\"}}'") elif key == "aws_bucket": - return run_cmd( - "kubectl get configmaps aws-config -o=go-template='{{index .data \"aws-bucket\"}}'" - ) + return run_cmd("kubectl get configmaps aws-config -o=go-template='{{index .data \"aws-bucket\"}}'") elif key == "aws_kms_key_alias": - return run_cmd( - "kubectl get configmaps aws-config -o=go-template='{{index .data \"aws-kms-key-alias\"}}'" - ) + return run_cmd("kubectl get configmaps aws-config -o=go-template='{{index .data \"aws-kms-key-alias\"}}'") elif key == "aws_access_key_id": - return run_cmd( - "kubectl get secret aws-creds -o=go-template='{{index .data \"aws_access_key_id\"}}' | base64 -d" - ) + return run_cmd("kubectl get secret aws-creds -o=go-template='{{index .data \"aws_access_key_id\"}}' | base64 -d") elif key == "aws_secret_key": - return run_cmd( - "kubectl get secret aws-creds -o=go-template='{{index .data \"aws_secret_key\"}}' | base64 -d" - ) + return run_cmd("kubectl get secret aws-creds -o=go-template='{{index .data \"aws_secret_key\"}}' | base64 -d") elif key == "minio_root_user": - return run_cmd( - "kubectl get secret minio-creds -o=go-template='{{index .data \"minio_root_user\"}}' | base64 -d" - ) + return run_cmd("kubectl get secret minio-creds -o=go-template='{{index .data \"minio_root_user\"}}' | base64 -d") elif key == "minio_root_password": - return run_cmd( - "kubectl get secret minio-creds -o=go-template='{{index .data \"minio_root_password\"}}' | base64 -d" - ) + return run_cmd("kubectl get secret minio-creds -o=go-template='{{index .data \"minio_root_password\"}}' | base64 -d") elif key == "minio_storage_size": - return run_cmd( - "kubectl get configmaps operation-config -o=go-template='{{index .data \"minio_storage_size\"}}'" - ) + return run_cmd("kubectl get configmaps operation-config -o=go-template='{{index .data \"minio_storage_size\"}}'") elif key == "storage_provider": - return run_cmd( - "kubectl get configmaps operation-config -o=go-template='{{index .data \"storage_provider\"}}'" - ) + return run_cmd("kubectl get configmaps operation-config -o=go-template='{{index .data \"storage_provider\"}}'") elif key == "assessment_id": - return run_cmd( - "kubectl get configmaps operation-config -o=go-template='{{index .data \"assessment-id\"}}'" - ) + return run_cmd("kubectl get configmaps operation-config -o=go-template='{{index .data \"assessment-id\"}}'") elif key == "log_level": - return run_cmd( - "kubectl get configmaps operation-config -o=go-template='{{index .data \"log-level\"}}'" - ) + return run_cmd("kubectl get configmaps operation-config -o=go-template='{{index .data \"log-level\"}}'") elif key == "data_expiration_days": - return run_cmd( - "kubectl get configmaps operation-config -o=go-template='{{index .data \"data-expiration-days\"}}'" - ) - - elif key == "disable_slack_alerting": - return run_cmd( - "kubectl get configmaps operation-config -o=go-template='{{index .data \"disable-slack-alerting\"}}'" - ) + return run_cmd("kubectl get configmaps operation-config -o=go-template='{{index .data \"data-expiration-days\"}}'") elif key == "nemesis_http_server": - return run_cmd( - "kubectl get configmaps operation-config -o=go-template='{{index .data \"nemesis-http-server\"}}'" - ) + return run_cmd("kubectl get configmaps operation-config -o=go-template='{{index .data \"nemesis-http-server\"}}'") + + elif key == "environment": + return run_cmd("kubectl get configmaps operation-config -o=go-template='{{index .data \"environment\"}}'") + + elif key == "disable_slack_alerting": + return run_cmd("kubectl get configmaps operation-config -o=go-template='{{index .data \"disable-slack-alerting\"}}'") elif key == "slack_channel": - return run_cmd( - "kubectl get configmaps operation-config -o=go-template='{{index .data \"slack-alert-channel\"}}'" - ) + return run_cmd("kubectl get configmaps operation-config -o=go-template='{{index .data \"slack-alert-channel\"}}'") elif key == "slack_webhook": - return run_cmd( - "kubectl get secret operation-creds -o=go-template='{{index .data \"slack_web_hook\"}}' | base64 -d" - ) + return run_cmd("kubectl get secret operation-creds -o=go-template='{{index .data \"slack_web_hook\"}}' | base64 -d") elif key == "basic_auth_password": - return run_cmd( - "kubectl get secret operation-creds -o=go-template='{{index .data \"basic-auth-password\"}}' | base64 -d" - ) + return run_cmd("kubectl get secret operation-creds -o=go-template='{{index .data \"basic-auth-password\"}}' | base64 -d") elif key == "basic_auth_user": - return run_cmd( - "kubectl get secret operation-creds -o=go-template='{{index .data \"basic-auth-user\"}}' | base64 -d" - ) + return run_cmd("kubectl get secret operation-creds -o=go-template='{{index .data \"basic-auth-user\"}}' | base64 -d") elif key == "elasticsearch_password": - return run_cmd( - "kubectl get secret elasticsearch-users -o=go-template='{{index .data \"password\"}}' | base64 -d" - ) + return run_cmd("kubectl get secret elasticsearch-users -o=go-template='{{index .data \"password\"}}' | base64 -d") elif key == "elasticsearch_user": - return run_cmd( - "kubectl get secret elasticsearch-users -o=go-template='{{index .data \"username\"}}' | base64 -d" - ) + return run_cmd("kubectl get secret elasticsearch-users -o=go-template='{{index .data \"username\"}}' | base64 -d") elif key == "grafana_password": - return run_cmd( - "kubectl get secret grafana-creds --namespace=monitoring -o=go-template='{{index .data \"username\"}}' | base64 -d" - ) + return run_cmd("kubectl get secret grafana-creds --namespace=monitoring -o=go-template='{{index .data \"username\"}}' | base64 -d") elif key == "grafana_user": - return run_cmd( - "kubectl get secret grafana-creds --namespace=monitoring -o=go-template='{{index .data \"password\"}}' | base64 -d" - ) + return run_cmd("kubectl get secret grafana-creds --namespace=monitoring -o=go-template='{{index .data \"password\"}}' | base64 -d") elif key == "postgres_user": - return run_cmd( - "kubectl get secret postgres-creds -o=go-template='{{index .data \"postgres-user\"}}' | base64 -d" - ) + return run_cmd("kubectl get secret postgres-creds -o=go-template='{{index .data \"postgres-user\"}}' | base64 -d") elif key == "postgres_password": - return run_cmd( - "kubectl get secret postgres-creds -o=go-template='{{index .data \"postgres-password\"}}' | base64 -d" - ) + return run_cmd("kubectl get secret postgres-creds -o=go-template='{{index .data \"postgres-password\"}}' | base64 -d") elif key == "dashboard_user": - return run_cmd( - "kubectl get secret dashboard-creds -o=go-template='{{index .data \"dashboard-user\"}}' | base64 -d" - ) + return run_cmd("kubectl get secret dashboard-creds -o=go-template='{{index .data \"dashboard-user\"}}' | base64 -d") elif key == "dashboard_password": - return run_cmd( - "kubectl get secret dashboard-creds -o=go-template='{{index .data \"dashboard-password\"}}' | base64 -d" - ) + return run_cmd("kubectl get secret dashboard-creds -o=go-template='{{index .data \"dashboard-password\"}}' | base64 -d") elif key == "dashboard_user": - return run_cmd( - "kubectl get secret dashboard-creds -o=go-template='{{index .data \"dashboard-user\"}}' | base64 -d" - ) + return run_cmd("kubectl get secret dashboard-creds -o=go-template='{{index .data \"dashboard-user\"}}' | base64 -d") elif key == "dashboard_password": - return run_cmd( - "kubectl get secret dashboard-creds -o=go-template='{{index .data \"dashboard-password\"}}' | base64 -d" - ) + return run_cmd("kubectl get secret dashboard-creds -o=go-template='{{index .data \"dashboard-password\"}}' | base64 -d") elif key == "pgadmin_email": - return run_cmd( - "kubectl get secret postgres-creds -o=go-template='{{index .data \"pgadmin-email\"}}' | base64 -d" - ) + return run_cmd("kubectl get secret postgres-creds -o=go-template='{{index .data \"pgadmin-email\"}}' | base64 -d") elif key == "pgadmin_password": - return run_cmd( - "kubectl get secret postgres-creds -o=go-template='{{index .data \"pgadmin-password\"}}' | base64 -d" - ) + return run_cmd("kubectl get secret postgres-creds -o=go-template='{{index .data \"pgadmin-password\"}}' | base64 -d") elif key == "rabbitmq_admin_user": - return run_cmd( - "kubectl get secret rabbitmq-creds -o=go-template='{{index .data \"rabbitmq-admin-user\"}}' | base64 -d" - ) + return run_cmd("kubectl get secret rabbitmq-creds -o=go-template='{{index .data \"rabbitmq-admin-user\"}}' | base64 -d") elif key == "rabbitmq_admin_password": - return run_cmd( - "kubectl get secret rabbitmq-creds -o=go-template='{{index .data \"rabbitmq-admin-password\"}}' | base64 -d" - ) + return run_cmd("kubectl get secret rabbitmq-creds -o=go-template='{{index .data \"rabbitmq-admin-password\"}}' | base64 -d") elif key == "rabbitmq_connectionuri": - return run_cmd( - "kubectl get secret rabbitmq-creds -o=go-template='{{index .data \"rabbitmq-connectionuri\"}}' | base64 -d" - ) + return run_cmd("kubectl get secret rabbitmq-creds -o=go-template='{{index .data \"rabbitmq-connectionuri\"}}' | base64 -d") elif key == "rabbitmq_erlang_cookie": - return run_cmd( - "kubectl get secret rabbitmq-creds -o=go-template='{{index .data \"rabbitmq-erlang-cookie\"}}' | base64 -d" - ) + return run_cmd("kubectl get secret rabbitmq-creds -o=go-template='{{index .data \"rabbitmq-erlang-cookie\"}}' | base64 -d") else: logger.error(f"Invalid config key: {key}") @@ -291,26 +224,19 @@ def set_config_values(config_values): if continue_set_values: run_cmd("kubectl delete configmap aws-config") run_cmd( - "kubectl create configmap aws-config" - + f" --from-literal=aws-bucket={v.get('aws_bucket')}" - + f" --from-literal=aws-default-region={v.get('aws_region')}" - + f" --from-literal=aws-kms-key-alias={v.get('aws_kms_key_alias')}", + "kubectl create configmap aws-config" + f" --from-literal=aws-bucket={v.get('aws_bucket')}" + f" --from-literal=aws-default-region={v.get('aws_region')}" + f" --from-literal=aws-kms-key-alias={v.get('aws_kms_key_alias')}", True, ) run_cmd("kubectl delete secret aws-creds") run_cmd( - "kubectl create secret generic aws-creds" - + f" --from-literal=aws_access_key_id={v.get('aws_access_key_id')}" - + f" --from-literal=aws_secret_key={v.get('aws_secret_key')}", + "kubectl create secret generic aws-creds" + f" --from-literal=aws_access_key_id={v.get('aws_access_key_id')}" + f" --from-literal=aws_secret_key={v.get('aws_secret_key')}", True, ) run_cmd("kubectl delete secret minio-creds") run_cmd( - "kubectl create secret generic minio-creds" - + f" --from-literal=minio_root_user={v.get('minio_root_user')}" - + f" --from-literal=minio_root_password={v.get('minio_root_password')}", + "kubectl create secret generic minio-creds" + f" --from-literal=minio_root_user={v.get('minio_root_user')}" + f" --from-literal=minio_root_password={v.get('minio_root_password')}", True, ) @@ -324,6 +250,7 @@ def set_config_values(config_values): + f" --from-literal=storage_provider={v.get('storage_provider')}" + f" --from-literal=minio_storage_size={v.get('minio_storage_size')}" + f" --from-literal=nemesis-http-server={v.get('nemesis_http_server')}" + + f" --from-literal=environment={v.get('environment')}" + f" --from-literal=data-expiration-days={v.get('data_expiration_days')}", True, ) @@ -334,11 +261,13 @@ def set_config_values(config_values): run_cmd( "kubectl create configmap operation-config --namespace=monitoring" + f" --from-literal=slack-alert-channel={v.get('slack_channel')}" + + f" --from-literal=disable-slack-alerting={v.get('disable_slack_alerting')}" + f" --from-literal=log-level={v.get('log_level')}" + f" --from-literal=assessment-id={v.get('assessment_id')}" + f" --from-literal=storage_provider={v.get('storage_provider')}" + f" --from-literal=minio_storage_size={v.get('minio_storage_size')}" + f" --from-literal=nemesis-http-server={v.get('nemesis_http_server')}" + + f" --from-literal=environment={v.get('environment')}" + f" --from-literal=data-expiration-days={v.get('data_expiration_days')}", True, ) @@ -353,15 +282,9 @@ def set_config_values(config_values): run_cmd("kubectl delete secret basic-auth") encrypted = apr_md5_crypt.hash(v.get("basic_auth_password")) - run_cmd( - "kubectl create secret generic basic-auth" - + f" --from-literal=auth='{v.get('basic_auth_user')}:{encrypted}'" - ) + run_cmd("kubectl create secret generic basic-auth" + f" --from-literal=auth='{v.get('basic_auth_user')}:{encrypted}'") run_cmd("kubectl delete secret basic-auth -n monitoring") - run_cmd( - "kubectl create secret generic basic-auth -n monitoring" - + f" --from-literal=auth='{v.get('basic_auth_user')}:{encrypted}'" - ) + run_cmd("kubectl create secret generic basic-auth -n monitoring" + f" --from-literal=auth='{v.get('basic_auth_user')}:{encrypted}'") run_cmd("kubectl delete secret operation-creds --namespace=monitoring") run_cmd( @@ -372,19 +295,10 @@ def set_config_values(config_values): ) run_cmd("kubectl delete secret grafana-creds --namespace=monitoring") - run_cmd( - "kubectl create secret generic grafana-creds --namespace=monitoring" - + f" --from-literal=username={v.get('grafana_user')}" - + f" --from-literal=password={v.get('grafana_password')}" - ) + run_cmd("kubectl create secret generic grafana-creds --namespace=monitoring" + f" --from-literal=username={v.get('grafana_user')}" + f" --from-literal=password={v.get('grafana_password')}") run_cmd("kubectl delete secret elasticsearch-users") - run_cmd( - "kubectl create secret generic elasticsearch-users" - + f" --from-literal=username={v.get('elasticsearch_user')}" - + f" --from-literal=password={v.get('elasticsearch_password')}" - + f" --from-literal=roles=superuser" - ) + run_cmd("kubectl create secret generic elasticsearch-users" + f" --from-literal=username={v.get('elasticsearch_user')}" + f" --from-literal=password={v.get('elasticsearch_password')}" + f" --from-literal=roles=superuser") run_cmd("kubectl delete secret postgres-creds") run_cmd( @@ -396,11 +310,7 @@ def set_config_values(config_values): ) run_cmd("kubectl delete secret dashboard-creds") - run_cmd( - "kubectl create secret generic dashboard-creds" - + f" --from-literal=dashboard-user={v.get('dashboard_user')}" - + f" --from-literal=dashboard-password={v.get('dashboard_password')}" - ) + run_cmd("kubectl create secret generic dashboard-creds" + f" --from-literal=dashboard-user={v.get('dashboard_user')}" + f" --from-literal=dashboard-password={v.get('dashboard_password')}") run_cmd("kubectl delete secret fluentd-creds --namespace=kube-system") run_cmd( @@ -420,9 +330,7 @@ def set_config_values(config_values): ) # hack, but application not working through skaffold - run_cmd( - f"kubectl apply --server-side=true -f ./monitoring/grafana-dashboards.yaml" - ) + run_cmd(f"kubectl apply --server-side=true -f ./monitoring/grafana-dashboards.yaml") ###################################################### @@ -435,9 +343,7 @@ def ensure_command(command: str): exitcode, output = subprocess.getstatusoutput(command) if exitcode == 127: - logger.error( - f"'{command}' command not found. Please install 'kubectl' and try again." - ) + logger.error(f"'{command}' command not found. Please install 'kubectl' and try again.") sys.exit(1) @@ -454,9 +360,7 @@ def validate_kubernetes(): logger.info("kubectl configured to use existing cluster") return else: - logger.info( - "No Kubernetes cluster found using kubectl, attempting to start Minikube" - ) + logger.info("No Kubernetes cluster found using kubectl, attempting to start Minikube") start_minikube() @@ -471,9 +375,7 @@ def start_minikube(): if not minikube_running: logger.info("Starting Minikube...") - output = subprocess.getstatusoutput( - "minikube start --network-plugin=cni --cni=calico" - ) + output = subprocess.getstatusoutput("minikube start --network-plugin=cni --cni=calico") else: logger.info("Minikube is already running") @@ -530,10 +432,7 @@ def validate_config_values(config_keys): v.set("aws_region", "not-applicable") if not v.get("aws_secret_key"): v.set("aws_secret_key", "not-applicable") - if ( - not v.get("minio_storage_size") - or v.get("minio_storage_size") == "" - ): + if not v.get("minio_storage_size") or v.get("minio_storage_size") == "": v.set("minio_storage_size", "30Gi") if not v.get("minio_root_user") or v.get("minio_root_user") == "": minio_root_user = get_kubectl_value("minio_root_user") @@ -542,10 +441,7 @@ def validate_config_values(config_keys): else: # set default username if not supplied or already set v.set("minio_root_user", "nemesis") - if ( - not v.get("minio_root_password") - or v.get("minio_root_password") == "" - ): + if not v.get("minio_root_password") or v.get("minio_root_password") == "": minio_root_password = get_kubectl_value("minio_root_password") if minio_root_password: v.set("minio_root_password", minio_root_password) @@ -571,10 +467,7 @@ def validate_config_values(config_keys): "rabbitmq_erlang_cookie", ] if config_key not in not_required_args: - if ( - config_key.startswith("slack_") - and str(v.get("disable_slack_alerting")).lower() == "true" - ): + if config_key.startswith("slack_") and str(v.get("disable_slack_alerting")).lower() == "true": continue if not v.get(config_key): @@ -582,9 +475,7 @@ def validate_config_values(config_keys): config_value = get_kubectl_value(config_key) if not config_value: # otherwise prompt - config_value = input( - f"\n[*] Please enter a value for '{config_key}' or for the default: " - ) + config_value = input(f"\n[*] Please enter a value for '{config_key}' or for the default: ") v.set(config_key, config_value) if not v.get("log_level") or v.get("log_level") == "": @@ -616,10 +507,7 @@ def validate_config_values(config_keys): else: v.set("elasticsearch_user", "nemesis") - if ( - not v.get("elasticsearch_password") - or v.get("elasticsearch_password") == "" - ): + if not v.get("elasticsearch_password") or v.get("elasticsearch_password") == "": elasticsearch_password_kubectl = get_kubectl_value("elasticsearch_password") if elasticsearch_password_kubectl: v.set("elasticsearch_password", elasticsearch_password_kubectl) @@ -687,10 +575,7 @@ def validate_config_values(config_keys): # set a random password if not supplied or already set v.set("grafana_password", get_random_password(24)) - if ( - not v.get("data_expiration_days") - or v.get("data_expiration_days") == "" - ): + if not v.get("data_expiration_days") or v.get("data_expiration_days") == "": data_expiration_days_kubectl = get_kubectl_value("data_expiration_days") if data_expiration_days_kubectl: v.set("data_expiration_days", data_expiration_days_kubectl) @@ -704,10 +589,7 @@ def validate_config_values(config_keys): else: v.set("rabbitmq_admin_user", "nemesis") - if ( - not v.get("rabbitmq_admin_password") - or v.get("rabbitmq_admin_password") == "" - ): + if not v.get("rabbitmq_admin_password") or v.get("rabbitmq_admin_password") == "": rabbitmq_admin_password_kubectl = get_kubectl_value("rabbitmq_admin_password") if rabbitmq_admin_password_kubectl: v.set("rabbitmq_admin_password", rabbitmq_admin_password_kubectl) @@ -715,10 +597,7 @@ def validate_config_values(config_keys): # set a random password if not supplied or already set v.set("rabbitmq_admin_password", get_random_password(24)) - if ( - not v.get("rabbitmq_connectionuri") - or v.get("rabbitmq_connectionuri") == "" - ): + if not v.get("rabbitmq_connectionuri") or v.get("rabbitmq_connectionuri") == "": rabbitmq_connectionuri_kubectl = get_kubectl_value("rabbitmq_connectionuri") if rabbitmq_connectionuri_kubectl: v.set("rabbitmq_connectionuri", rabbitmq_connectionuri_kubectl) @@ -731,10 +610,7 @@ def validate_config_values(config_keys): f"amqp://{rabbitmq_user}:{rabbitmq_password}@nemesis-rabbitmq-svc:5672/", ) - if ( - not v.get("rabbitmq_erlang_cookie") - or v.get("rabbitmq_erlang_cookie") == "" - ): + if not v.get("rabbitmq_erlang_cookie") or v.get("rabbitmq_erlang_cookie") == "": rabbitmq_erlang_cookie_kubectl = get_kubectl_value("rabbitmq_erlang_cookie") if rabbitmq_erlang_cookie_kubectl: v.set("rabbitmq_erlang_cookie", rabbitmq_erlang_cookie_kubectl) @@ -742,10 +618,7 @@ def validate_config_values(config_keys): # set a random password if not supplied or already set v.set("rabbitmq_erlang_cookie", get_random_password(24)) - if ( - not v.get("disable_slack_alerting") - or v.get("disable_slack_alerting") == "" - ): + if not v.get("disable_slack_alerting") or v.get("disable_slack_alerting") == "": disable_slack_alerting = get_kubectl_value("disable_slack_alerting") if disable_slack_alerting: disable_slack_alerting = bool(disable_slack_alerting) @@ -754,9 +627,7 @@ def validate_config_values(config_keys): v.set("disable_slack_alerting", "False") disable_slack_alerting = str(v.get("disable_slack_alerting")) if disable_slack_alerting.lower() not in ["true", "false"]: - logger.error( - f"The disable_slack_alerting argument must be either 'True' or 'False'. Supplied value: {disable_slack_alerting}" - ) + logger.error(f"The disable_slack_alerting argument must be either 'True' or 'False'. Supplied value: {disable_slack_alerting}") sys.exit(1) disable_slack_alerting = disable_slack_alerting.lower() == "true" @@ -771,14 +642,10 @@ def validate_config_values(config_keys): if not disable_slack_alerting: if slack_channel: if slack_channel[0] != "#": - logger.error( - f"The slack_channel argument must start with a '#'. Supplied value: {slack_channel}" - ) + logger.error(f"The slack_channel argument must start with a '#'. Supplied value: {slack_channel}") sys.exit(1) else: - logger.error( - f"The slack_channel argument must be set if slack alerting is enabled." - ) + logger.error(f"The slack_channel argument must be set if slack alerting is enabled.") sys.exit(1) if not v.get("slack_webhook") or v.get("slack_webhook") == "": @@ -789,14 +656,8 @@ def validate_config_values(config_keys): v.set("slack_webhook", None) slack_webhook = v.get("slack_webhook") - if ( - not disable_slack_alerting - and slack_webhook - and slack_webhook[0:8] != "https://" - ): - logger.error( - f"The slack_webhook argument must start with a 'https://'. Supplied value: {slack_webhook}" - ) + if not disable_slack_alerting and slack_webhook and slack_webhook[0:8] != "https://": + logger.error(f"The slack_webhook argument must start with a 'https://'. Supplied value: {slack_webhook}") sys.exit(1) # make sure we have everything set @@ -806,9 +667,7 @@ def validate_config_values(config_keys): if config_key == "slack_webhook" or config_key == "slack_channel": continue - logger.error( - f"\nRequired configuration key value '{config_key}' not supplied and not already present!\n" - ) + logger.error(f"\nRequired configuration key value '{config_key}' not supplied and not already present!\n") all_values_set = True if not all_values_set: @@ -867,9 +726,7 @@ def validate_aws_resources(): logger.info(f"S3 bucket '{aws_bucket}' exists") except: if not v.get("force"): - output = input( - f"\n[*] S3 bucket '{aws_bucket}' does not exist, do you want to create it? [Y/n] " - ) + output = input(f"\n[*] S3 bucket '{aws_bucket}' does not exist, do you want to create it? [Y/n] ") if v.get("force") or output == "" or output.lower() == "y": # create the bucket with a 'private' ACL @@ -879,9 +736,7 @@ def validate_aws_resources(): Bucket=aws_bucket, ) except Exception as e: - logger.error( - f"Error creating S3 bucket '{aws_bucket}' (bucket name is likely already taken) : {e}\n" - ) + logger.error(f"Error creating S3 bucket '{aws_bucket}' (bucket name is likely already taken) : {e}\n") sys.exit(1) # ensure the bucket/objects can't be public @@ -910,9 +765,7 @@ def validate_aws_resources(): logger.info(f"S3 bucket key '{aws_bucket}' created!\n") else: - logger.error( - f"S3 bucket '{aws_bucket}' doesn't exist, file storage will not work!\n" - ) + logger.error(f"S3 bucket '{aws_bucket}' doesn't exist, file storage will not work!\n") sys.exit(1) # check if the KMS key already exists @@ -927,34 +780,24 @@ def validate_aws_resources(): if not aws_kms_key_exists: if not v.get("force"): - output = input( - f"\n[*] KMS key '{aws_kms_key_alias}' does not exist, do you want to create it? [Y/n] " - ) + output = input(f"\n[*] KMS key '{aws_kms_key_alias}' does not exist, do you want to create it? [Y/n] ") if v.get("force") or output == "" or output.lower() == "y": response = kms_client.create_key(Description=f"key for {assessment_id}") keyId = response["KeyMetadata"]["KeyId"] - kms_client.create_alias( - AliasName=f"alias/{aws_kms_key_alias}", TargetKeyId=keyId - ) + kms_client.create_alias(AliasName=f"alias/{aws_kms_key_alias}", TargetKeyId=keyId) logger.info(f"KMS key '{aws_kms_key_alias}' (ID {keyId}) created!\n") else: - logger.error( - f"KMS key '{aws_kms_key_alias}' doesn't exist, file encryption will not work!\n" - ) + logger.error(f"KMS key '{aws_kms_key_alias}' doesn't exist, file encryption will not work!\n") sys.exit(1) elif aws_kms_key_state != "Enabled": - logger.error( - f"Key state for KMS key '{aws_kms_key_alias}' is '{aws_kms_key_state}', encryption will not work!" - ) + logger.error(f"Key state for KMS key '{aws_kms_key_alias}' is '{aws_kms_key_state}', encryption will not work!") sys.exit(1) def create_ingress_controller(): # Some jank to check if the nginx controller is installed in Kubernetes - exitcode, output = subprocess.getstatusoutput( - "kubectl get --raw /apis/admissionregistration.k8s.io/v1/validatingwebhookconfigurations | grep 'validate.nginx.ingress.kubernetes.io'" - ) + exitcode, output = subprocess.getstatusoutput("kubectl get --raw /apis/admissionregistration.k8s.io/v1/validatingwebhookconfigurations | grep 'validate.nginx.ingress.kubernetes.io'") if exitcode == 0: logger.info("Ingress controller already installed, skipping") return @@ -965,14 +808,10 @@ def create_ingress_controller(): 'openssl req -x509 -newkey rsa:4096 -sha256 -nodes -keyout /tmp/tls.key -out /tmp/tls.crt -subj "/CN=nemesis.local" -days 365 -addext "subjectAltName = DNS:nemesis.local"', show_error=True, ) - run_cmd( - "kubectl create secret tls nemesis-ingress-tls --cert=/tmp/tls.crt --key=/tmp/tls.key" - ) - run_cmd( - "kubectl create secret tls nemesis-ingress-tls --cert=/tmp/tls.crt --key=/tmp/tls.key -n monitoring" - ) + run_cmd("kubectl create secret tls nemesis-ingress-tls --cert=/tmp/tls.crt --key=/tmp/tls.key") + run_cmd("kubectl create secret tls nemesis-ingress-tls --cert=/tmp/tls.crt --key=/tmp/tls.key -n monitoring") - logger.info(f"Installing ingress-nginx controller using helm") + logger.info("Installing ingress-nginx controller using helm") run_cmd( "helm upgrade --install ingress-nginx ingress-nginx" " --repo https://kubernetes.github.io/ingress-nginx" @@ -980,7 +819,7 @@ def create_ingress_controller(): " --create-namespace" " --set prometheus.create=true" " --set prometheus.port=9113" - f' --set tcp.5044="default/nemesis-ls-beats:5044"', + ' --set tcp.5044="default/nemesis-ls-beats:5044"', show_error=True, ) @@ -1016,9 +855,7 @@ def create_minio(): def create_elastic_operator(): # Check if the elastic operator is already installed - exitcode, output = subprocess.getstatusoutput( - "kubectl get crds | grep 'k8s.elastic.co'" - ) + exitcode, output = subprocess.getstatusoutput("kubectl get crds | grep 'k8s.elastic.co'") if exitcode == 0: logger.info("ECK operator already installed, skipping") return @@ -1027,18 +864,11 @@ def create_elastic_operator(): run_cmd("helm repo add elastic https://helm.elastic.co") run_cmd("helm repo update") - run_cmd( - "helm install elastic-operator elastic/eck-operator" - " --namespace elastic-system" - " --create-namespace" - " --set managedNamespaces='{default}'" - ) + run_cmd("helm install elastic-operator elastic/eck-operator" " --namespace elastic-system" " --create-namespace" " --set managedNamespaces='{default}'") def create_metrics_server(): - exitcode, output = subprocess.getstatusoutput( - "kubectl get pods -A | grep 'metrics-server'" - ) + exitcode, output = subprocess.getstatusoutput("kubectl get pods -A | grep 'metrics-server'") if exitcode == 0: logger.info("Metrics Server already installed, skipping") return @@ -1065,6 +895,7 @@ if __name__ == "__main__": "storage_provider", "assessment_id", "nemesis_http_server", + "environment", "data_expiration_days", "log_level", "disable_slack_alerting", @@ -1120,9 +951,7 @@ if __name__ == "__main__": type=str, help="AWS region (default: us-east-1)", ) - parser.add_argument( - "--aws_bucket", "--aws-bucket", "--bucket", type=str, help="AWS S3 bucket name" - ) + parser.add_argument("--aws_bucket", "--aws-bucket", "--bucket", type=str, help="AWS S3 bucket name") parser.add_argument( "--aws_kms_key_alias", "--aws-kms-key-alias", @@ -1148,9 +977,7 @@ if __name__ == "__main__": ) # minio configs - parser.add_argument( - "--minio_root_user", "--minio-root-user", type=str, help="Minio root user" - ) + parser.add_argument("--minio_root_user", "--minio-root-user", type=str, help="Minio root user") parser.add_argument( "--minio_root_password", "--minio-root-password", @@ -1172,24 +999,28 @@ if __name__ == "__main__": ) # operation configs - parser.add_argument( - "--assessment_id", "--assessment-id", type=str, help="Asessment ID" - ) + parser.add_argument("--assessment_id", "--assessment-id", type=str, help="Asessment ID") parser.add_argument( "--nemesis_http_server", "--ip", type=str, help="Nemesis frontend HTTP server endpoint. Format: http://:", ) + parser.add_argument( + "--environment", + "--env", + type=str, + help="Environment the application is running in (default: development)", + default="production", + choices=["development", "production", "testing"], + ) parser.add_argument( "--data_expiration_days", "--exp", type=int, help="Days after ingestion to set data to expire (default: 100)", ) - parser.add_argument( - "--log_level", "--log", type=str, help="Level of logging (default: info)" - ) + parser.add_argument("--log_level", "--log", type=str, help="Level of logging (default: info)") parser.add_argument( "--disable_slack_alerting", type=str, @@ -1234,33 +1065,15 @@ if __name__ == "__main__": type=str, help="Password for Elasticsearch/Kibana", ) - parser.add_argument( - "--postgres_user", "--pg_user", type=str, help="Username for Postgres" - ) - parser.add_argument( - "--postgres_password", "--pg_password", type=str, help="Password for Postgres" - ) - parser.add_argument( - "--dashboard_user", type=str, help="Username for the Nemesis dashboard" - ) - parser.add_argument( - "--dashboard_password", type=str, help="Password for the Nemesis dashboard" - ) - parser.add_argument( - "--pgadmin_email", "--pgemail", type=str, help="Email (username) for pgAdmin" - ) - parser.add_argument( - "--pgadmin_password", "--pgpassword", type=str, help="Password for pgAdmin" - ) - parser.add_argument( - "--grafana_user", "--guser", type=str, help="Username for Grafana" - ) - parser.add_argument( - "--grafana_password", "--gpassword", type=str, help="Password for Grafana" - ) - parser.add_argument( - "--rabbitmq_admin_user", "--ruser", type=str, help="Admin username for RabbitMQ" - ) + parser.add_argument("--postgres_user", "--pg_user", type=str, help="Username for Postgres") + parser.add_argument("--postgres_password", "--pg_password", type=str, help="Password for Postgres") + parser.add_argument("--dashboard_user", type=str, help="Username for the Nemesis dashboard") + parser.add_argument("--dashboard_password", type=str, help="Password for the Nemesis dashboard") + parser.add_argument("--pgadmin_email", "--pgemail", type=str, help="Email (username) for pgAdmin") + parser.add_argument("--pgadmin_password", "--pgpassword", type=str, help="Password for pgAdmin") + parser.add_argument("--grafana_user", "--guser", type=str, help="Username for Grafana") + parser.add_argument("--grafana_password", "--gpassword", type=str, help="Password for Grafana") + parser.add_argument("--rabbitmq_admin_user", "--ruser", type=str, help="Admin username for RabbitMQ") parser.add_argument( "--rabbitmq_admin_password", "--rpassword", @@ -1308,17 +1121,9 @@ if __name__ == "__main__": logger.info("Configuration set") - logger.info( - f"Nemesis basic auth credentials: `{get_kubectl_value('basic_auth_user')}:{get_kubectl_value('basic_auth_password')}`" - ) + logger.info(f"Nemesis basic auth credentials: `{get_kubectl_value('basic_auth_user')}:{get_kubectl_value('basic_auth_password')}`") - logger.info( - "If settings were changed, you may need to restart minikube with: `minikube stop && minikube start`" - ) - logger.info( - "You can start the backend infrastructure in development mode with `./scripts/infra_start.sh`" - ) - logger.info( - "You can start the main processing services in development mode with `./scripts/services_start.sh`" - ) + logger.info("If settings were changed, you may need to restart minikube with: `minikube stop && minikube start`") + logger.info("You can start the backend infrastructure in development mode with `./scripts/infra_start.sh`") + logger.info("You can start the main processing services in development mode with `./scripts/services_start.sh`") logger.info("For non-development execution, run `skaffold run --port-forward`\n") diff --git a/nemesis.config.example b/nemesis.config.example index 22c71d7..6b55348 100644 --- a/nemesis.config.example +++ b/nemesis.config.example @@ -4,6 +4,7 @@ nemesis_http_server: http://IP_of_k8s_host:8080 assessment_id: ASSESS-TEST data_expiration_days: 100 log_level: INFO +environment: development # Slack Alerting disable_slack_alerting: True diff --git a/packages/python/nemesiscommon/nemesiscommon/logging.py b/packages/python/nemesiscommon/nemesiscommon/logging.py index 31c18aa..bf692a0 100644 --- a/packages/python/nemesiscommon/nemesiscommon/logging.py +++ b/packages/python/nemesiscommon/nemesiscommon/logging.py @@ -3,18 +3,22 @@ import logging # 3rd Party Libraries import structlog +from nemesiscommon.settings import EnvironmentSettings from rich.console import Console from rich.traceback import Traceback # TODO: Figure out how to use structlog with uvicorn's logging -def configure_logger(enable_json_logs: bool = False, log_level: str = "INFO", environment: str = "development"): +def configure_logger(environment: EnvironmentSettings, log_level: str, log_color_enabled: bool): level: int = logging.getLevelName(log_level) - if environment == "production": + if environment == EnvironmentSettings.PRODUCTION: configure_prod_logger(level) else: - configure_dev_logger(level) + if log_color_enabled: + configure_dev_logger(level, log_color_enabled) + else: + configure_dev_logger(level, log_color_enabled) def rich_traceback(sio, exc_info) -> None: @@ -36,7 +40,7 @@ def rich_traceback(sio, exc_info) -> None: ) -def configure_dev_logger(level: int): +def configure_dev_logger(level: int, colored_logging_enabled: bool): # timestamper = structlog.processors.TimeStamper(fmt="%Y-%m-%d %H:%M:%S") wrapper = structlog.make_filtering_bound_logger(level) @@ -48,7 +52,7 @@ def configure_dev_logger(level: int): structlog.processors.StackInfoRenderer(), # structlog.dev.set_exc_info, structlog.dev.ConsoleRenderer( - colors=True, + colors=colored_logging_enabled, exception_formatter=rich_traceback, ), ], diff --git a/packages/python/nemesiscommon/nemesiscommon/messaging_rabbitmq.py b/packages/python/nemesiscommon/nemesiscommon/messaging_rabbitmq.py index 6253390..6574522 100644 --- a/packages/python/nemesiscommon/nemesiscommon/messaging_rabbitmq.py +++ b/packages/python/nemesiscommon/nemesiscommon/messaging_rabbitmq.py @@ -175,7 +175,9 @@ class NemesisRabbitMQConsumer(MessageQueueConsumerInterface, Generic[T]): # If an unhandled exception happens, err on the side of not losing data and resubmit to queue # This could lead to a loop where the same troublesome message keeps getting resubmitted due to # it never parsing correctly - await message.reject(requeue=True) + # await message.reject(requeue=True) + + await message.ack() await logger.adebug("Waiting for messages", queue=self.__queue.name) await self.__queue.consume(callback=on_message, no_ack=False) diff --git a/packages/python/nemesiscommon/nemesiscommon/settings.py b/packages/python/nemesiscommon/nemesiscommon/settings.py index 4e8d0ec..214483d 100644 --- a/packages/python/nemesiscommon/nemesiscommon/settings.py +++ b/packages/python/nemesiscommon/nemesiscommon/settings.py @@ -47,6 +47,7 @@ class EnvironmentSettings(StrEnum): class NemesisServiceSettings(BaseSettings): environment: EnvironmentSettings log_level: str + log_color_enabled: bool = Field(True) prometheus_port: int = Field(None, ge=0, le=65535) assessment_id: str diff --git a/packages/python/nemesiscommon/nemesiscommon/storage_s3.py b/packages/python/nemesiscommon/nemesiscommon/storage_s3.py index 389a1b9..e56722a 100644 --- a/packages/python/nemesiscommon/nemesiscommon/storage_s3.py +++ b/packages/python/nemesiscommon/nemesiscommon/storage_s3.py @@ -12,7 +12,6 @@ from Cryptodome.Cipher import AES from nemesiscommon.nemesis_tempfile import TempFile from nemesiscommon.storage import StorageInterface - logger = structlog.get_logger(module=__name__) @@ -129,7 +128,7 @@ class StorageS3(StorageInterface): # now encrypt and write out the data while not finished: - chunk = in_file.read(1024 * self.block_size) + chunk = in_file.read(8196 * self.block_size) if len(chunk) == 0 or len(chunk) % self.block_size != 0: # final block/chunk is padded before encryption padding_length = (self.block_size - len(chunk) % self.block_size) or self.block_size @@ -167,7 +166,7 @@ class StorageS3(StorageInterface): finished = False while not finished: - chunk, next_chunk = next_chunk, cipher.decrypt(in_file.read(1024 * self.block_size)) + chunk, next_chunk = next_chunk, cipher.decrypt(in_file.read(8196 * self.block_size)) if len(next_chunk) == 0: padding_length = chunk[-1] chunk = chunk[:-padding_length]