diff --git a/helm/enrichment/Chart.yaml b/helm/enrichment/Chart.yaml new file mode 100644 index 0000000..d2f0f6b --- /dev/null +++ b/helm/enrichment/Chart.yaml @@ -0,0 +1,7 @@ +apiVersion: v2 +name: nemesis-enrichment +description: The enrichment container for Nemesis +type: application +version: 0.1.0 +appVersion: "0.1.0a" +icon: https://github.com/SpecterOps/Nemesis/raw/main/img/nemesis_white.png diff --git a/helm/enrichment/README.md b/helm/enrichment/README.md new file mode 100644 index 0000000..5bcdce3 --- /dev/null +++ b/helm/enrichment/README.md @@ -0,0 +1,3 @@ +# Helm + +Chart for the `enrichment` for use with Skaffold during development. \ No newline at end of file diff --git a/helm/enrichment/templates/NOTES.txt b/helm/enrichment/templates/NOTES.txt new file mode 100644 index 0000000..3576610 --- /dev/null +++ b/helm/enrichment/templates/NOTES.txt @@ -0,0 +1,5 @@ +------------------------------------------------------------------------ + +Enrichment container should now be deployed. + +------------------------------------------------------------------------ \ No newline at end of file diff --git a/helm/enrichment/templates/_helpers.tpl b/helm/enrichment/templates/_helpers.tpl new file mode 100644 index 0000000..9fa97bc --- /dev/null +++ b/helm/enrichment/templates/_helpers.tpl @@ -0,0 +1,62 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "nemesis.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "nemesis.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "nemesis.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "nemesis.labels" -}} +helm.sh/chart: {{ include "nemesis.chart" . }} +{{ include "nemesis.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "nemesis.selectorLabels" -}} +app.kubernetes.io/name: {{ include "nemesis.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "nemesis.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "nemesis.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/helm/enrichment/templates/enrichment.deployment.yaml b/helm/enrichment/templates/enrichment.deployment.yaml new file mode 100644 index 0000000..4dd2d04 --- /dev/null +++ b/helm/enrichment/templates/enrichment.deployment.yaml @@ -0,0 +1,240 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: enrichment + labels: + app: enrichment + component: enrichment-pipeline +spec: + replicas: 1 + selector: + matchLabels: + app: enrichment + strategy: {} + template: + metadata: + labels: + app: enrichment + component: enrichment-pipeline + spec: + volumes: + - name: cracklist + emptyDir: {} + - name: noseyparker + emptyDir: {} + - name: noseyparker-rules + configMap: + name: noseyparker-rules + initContainers: + - name: noseyparker + image: ghcr.io/praetorian-inc/noseyparker:v0.16.0 + imagePullPolicy: IfNotPresent + command: ["sh", "-c", "cp /usr/local/bin/noseyparker /opt/noseyparker/"] + volumeMounts: + - name: noseyparker + mountPath: /opt/noseyparker/ + containers: + - env: + - name: LOG_LEVEL + valueFrom: + configMapKeyRef: + name: operation-config + key: log-level + - name: DATA_DOWNLOAD_DIR + value: "/tmp" + - name: ENVIRONMENT + valueFrom: + configMapKeyRef: + name: operation-config + key: environment + - name: STORAGE_EXPIRATION_DAYS + valueFrom: + configMapKeyRef: + name: operation-config + key: storage-expiration-days + + - name: CRACK_WORDLIST_TOP_WORDS + value: "10000" # either 10,000 or 100,000 + - name: JTR_INSTANCES + value: "1" + + - name: EXTRACTED_ARCHIVE_SIZE_LIMIT # limit of extracted archives to process, 500 MB + value: "500000000" + - name: CHUNK_SIZE # number of bytes to read at a time per file + value: "128000" + - name: MODEL_WORD_LIMIT # only extract passwords for documents below this word limit + value: "100000" + - name: YARA_API_PORT + value: "9700" + - name: PLAINTEXT_SIZE_LIMIT + value: "50000000" # max number of bytes for a plaintext doc to prevent indexing of embeddings + + - name: CONTEXT_WORDS + value: "5" + - name: ML_CHUNK_SIZE + value: "5000" + - name: PROB_THRESHOLD + value: ".95" + + - name: TENSORFLOW_URI + value: "http://tensorflow-serving:8501/" + - name: TIKA_URI + value: http://tika:9998/ + - name: DOTNET_URI + value: http://dotnet:9800/ + - name: GOTENBERG_URI + value: http://gotenberg:3000/ + - name: ML_MODELS_URI + value: http://127.0.0.1:5000/ + - name: CRACK_LIST_URI + value: http://127.0.0.1:9900/ + - name: NEMESIS_HTTP_SERVER + valueFrom: + configMapKeyRef: + name: operation-config + key: nemesis-http-server + - name: PUBLIC_NEMESIS_URL + value: "$(NEMESIS_HTTP_SERVER)dashboard/" + - name: PUBLIC_KIBANA_URL + value: "$(NEMESIS_HTTP_SERVER)kibana/" + - name: WEB_API_URL + value: "$(NEMESIS_HTTP_SERVER)api/" + + - name: POSTGRES_SERVER + value: postgres + - name: POSTGRES_PORT + value: "5432" + - name: POSTGRES_DATABASE + value: nemesis + - name: POSTGRES_USER + valueFrom: + secretKeyRef: + name: postgres-creds + key: postgres-user + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: postgres-creds + key: postgres-password + - name: POSTGRES_CONNECTION_URI + value: "postgresql://$(POSTGRES_USER):$(POSTGRES_PASSWORD)@$(POSTGRES_SERVER):$(POSTGRES_PORT)/$(POSTGRES_DATABASE)" + - name: DB_ITERATION_SIZE + value: "1000" + - name: PROMETHEUS_DISABLE_CREATED_SERIES + value: "True" + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: aws-creds + key: aws_access_key_id + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: aws-creds + key: aws_secret_key + - name: AWS_BUCKET + valueFrom: + configMapKeyRef: + name: aws-config + key: aws-bucket + - name: AWS_DEFAULT_REGION + valueFrom: + configMapKeyRef: + name: aws-config + key: aws-default-region + - name: AWS_KMS_KEY_ALIAS + valueFrom: + configMapKeyRef: + name: aws-config + key: aws-kms-key-alias + + - name: STORAGE_PROVIDER + valueFrom: + configMapKeyRef: + name: operation-config + key: storage_provider + - name: MINIO_ROOT_USER + valueFrom: + secretKeyRef: + name: minio-creds + key: root-user + - name: MINIO_ROOT_PASSWORD + valueFrom: + secretKeyRef: + name: minio-creds + key: root-password + + - name: ASSESSMENT_ID + valueFrom: + configMapKeyRef: + name: operation-config + key: assessment-id + - name: RABBITMQ_CONNECTION_URI + valueFrom: + secretKeyRef: + name: rabbitmq-creds + key: rabbitmq-connectionuri + - name: ELASTICSEARCH_URL + value: http://nemesis-es-internal-http:9200/ + - name: KIBANA_URL + value: http://nemesis-kb-http:5601/ + - name: ELASTICSEARCH_USER + valueFrom: + secretKeyRef: + name: elasticsearch-users + key: username + - name: ELASTICSEARCH_PASSWORD + valueFrom: + secretKeyRef: + name: elasticsearch-users + key: password + - name: PROMETHEUS_PORT + value: "8000" + - name: SLACK_CHANNEL + valueFrom: + configMapKeyRef: + name: operation-config + key: slack-alert-channel + - name: SLACK_WEBHOOK_URL + valueFrom: + secretKeyRef: + name: operation-creds + key: slack_web_hook + - name: SLACK_USERNAME + value: Nemesis + - name: SLACK_EMOJI + value: ":smiling_imp:" + - name: DISABLE_ALERTING + valueFrom: + configMapKeyRef: + name: operation-config + key: disable-slack-alerting + - name: REPROCESSING_WORKERS + value: "5" + + image: "{{ .Values.enrichment.image.repository }}:{{ .Values.enrichment.image.tag }}" + imagePullPolicy: {{ .Values.enrichment.image.pullPolicy }} + name: enrichment + ports: + - containerPort: 9700 + name: yara-http + resources: {{ toYaml .Values.enrichment.resources | nindent 12 }} + volumeMounts: + - name: cracklist + mountPath: /opt/cracklist/ + - name: noseyparker + mountPath: /opt/noseyparker/ + - name: noseyparker-rules + mountPath: /opt/noseyparker/noseyparker.rules + subPath: noseyparker.rules + readOnly: true + readinessProbe: + httpGet: + path: /ready + port: yara-http + initialDelaySeconds: 5 + periodSeconds: 5 + timeoutSeconds: 5 + failureThreshold: 10 + restartPolicy: Always +status: {} diff --git a/helm/enrichment/templates/enrichment.ingress.yaml b/helm/enrichment/templates/enrichment.ingress.yaml new file mode 100644 index 0000000..3d3dc81 --- /dev/null +++ b/helm/enrichment/templates/enrichment.ingress.yaml @@ -0,0 +1,47 @@ +{{- if ne .Values.operation.environment "development" }} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: enrichment-ingress + annotations: + nginx.ingress.kubernetes.io/proxy-body-size: "1000m" + nginx.ingress.kubernetes.io/auth-type: basic + nginx.ingress.kubernetes.io/auth-secret: basic-auth + nginx.ingress.kubernetes.io/auth-realm: 'Authentication Required' + nginx.ingress.kubernetes.io/ssl-redirect: "false" + nginx.ingress.kubernetes.io/rewrite-target: /$2 +spec: + ingressClassName: nginx + rules: + - http: + paths: + - path: /crack-list(/|$)(.*) + pathType: ImplementationSpecific + backend: + service: + name: enrichment-cracklist + port: + number: 9900 + - path: /yara(/|$)(.*) + pathType: ImplementationSpecific + backend: + service: + name: enrichment-yara + port: + number: 9700 + - path: /api(/|$)(.*) + pathType: ImplementationSpecific + backend: + service: + name: enrichment-webapi + port: + number: 9910 + - path: / + pathType: ImplementationSpecific + backend: + service: + name: enrichment-landingpage + port: + number: 9920 + +{{- end }} \ No newline at end of file diff --git a/helm/enrichment/templates/enrichment.service.yaml b/helm/enrichment/templates/enrichment.service.yaml new file mode 100644 index 0000000..ec3bddb --- /dev/null +++ b/helm/enrichment/templates/enrichment.service.yaml @@ -0,0 +1,97 @@ +{{- if ne .Values.operation.environment "development" }} +apiVersion: v1 +kind: Service +metadata: + labels: + app: enrichment + component: enrichment-pipeline + name: enrichment-yara +spec: + type: NodePort + ports: + - name: "9700" + port: 9700 + targetPort: 9700 + nodePort: {{ .Values.enrichment.yara.nodePort }} + selector: + app: enrichment +status: + loadBalancer: {} +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app: enrichment + component: enrichment-pipeline + name: enrichment-cracklist +spec: + type: NodePort + ports: + - name: "9900" + port: 9900 + targetPort: 9900 + nodePort: {{ .Values.enrichment.cracklist.nodePort }} + selector: + app: enrichment +status: + loadBalancer: {} +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app: enrichment + component: enrichment-pipeline + name: enrichment-webapi +spec: + type: NodePort + ports: + - name: "9910" + port: 9910 + targetPort: 9910 + nodePort: {{ .Values.enrichment.webapi.nodePort }} + selector: + app: enrichment +status: + loadBalancer: {} +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app: enrichment + component: enrichment-landingpage + name: enrichment-landingpage +spec: + type: NodePort + ports: + - name: "9920" + port: 9920 + targetPort: 9920 + nodePort: {{ .Values.enrichment.landingpage.nodePort }} + selector: + app: enrichment +status: + loadBalancer: {} +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app: enrichment + component: enrichment-monitoring + name: enrichment-monitoring +spec: + type: NodePort + ports: + - name: "8000" + port: 8000 + targetPort: 8000 + nodePort: {{ .Values.enrichment.monitoring.nodePort }} + selector: + app: enrichment +status: + loadBalancer: {} + +{{- end }} \ No newline at end of file diff --git a/helm/enrichment/values.schema.json b/helm/enrichment/values.schema.json new file mode 100644 index 0000000..46dc245 --- /dev/null +++ b/helm/enrichment/values.schema.json @@ -0,0 +1,129 @@ +{ + "type": "object", + "$schema": "http://json-schema.org/draft-07/schema", + "required": [], + "properties": { + "enrichment": { + "type": "object", + "required": [], + "properties": { + "image": { + "type": "object", + "required": [], + "properties": { + "repository": { + "type": [ + "string", + "boolean", + "number", + "object", + "array" + ], + "default": "specterops/nemesis-enrichment" + }, + "tag": { + "type": [ + "string", + "boolean", + "number", + "object", + "array" + ], + "default": "latest" + }, + "pullPolicy": { + "type": [ + "string", + "boolean", + "number", + "object", + "array" + ], + "default": "IfNotPresent" + } + } + }, + "cracklist": { + "type": "object", + "required": [], + "properties": { + "nodePort": { + "type": [ + "string", + "boolean", + "number", + "object", + "array" + ], + "default": "30089" + } + } + }, + "yara": { + "type": "object", + "required": [], + "properties": { + "nodePort": { + "type": [ + "string", + "boolean", + "number", + "object", + "array" + ], + "default": "30079" + } + } + }, + "webapi": { + "type": "object", + "required": [], + "properties": { + "nodePort": { + "type": [ + "string", + "boolean", + "number", + "object", + "array" + ], + "default": "30808" + } + } + }, + "landingpage": { + "type": "object", + "required": [], + "properties": { + "nodePort": { + "type": [ + "string", + "boolean", + "number", + "object", + "array" + ], + "default": "30809" + } + } + }, + "monitoring": { + "type": "object", + "required": [], + "properties": { + "nodePort": { + "type": [ + "string", + "boolean", + "number", + "object", + "array" + ], + "default": "31693" + } + } + } + } + } + } +} \ No newline at end of file diff --git a/helm/enrichment/values.yaml b/helm/enrichment/values.yaml new file mode 100644 index 0000000..4cd965b --- /dev/null +++ b/helm/enrichment/values.yaml @@ -0,0 +1,15 @@ +enrichment: + image: + repository: specterops/nemesis-enrichment-dev + tag: latest + pullPolicy: IfNotPresent + cracklist: + nodePort: 30089 + yara: + nodePort: 30079 + webapi: + nodePort: 30808 + landingpage: + nodePort: 30809 + monitoring: + nodePort: 31693 diff --git a/helm/monitoring/values.yaml b/helm/monitoring/values.yaml index d2dd954..581741c 100644 --- a/helm/monitoring/values.yaml +++ b/helm/monitoring/values.yaml @@ -87,6 +87,6 @@ postgresExporter: nemesisWaiter: image: - repository: t94j0/nemesis-waiter + repository: specterops/nemesis-waiter tag: latest pullPolicy: IfNotPresent \ No newline at end of file diff --git a/helm/nemesis/templates/enrichment.deployment.yaml b/helm/nemesis/templates/enrichment.deployment.yaml index 3bdef87..e68f25f 100644 --- a/helm/nemesis/templates/enrichment.deployment.yaml +++ b/helm/nemesis/templates/enrichment.deployment.yaml @@ -54,6 +54,11 @@ spec: configMapKeyRef: name: operation-config key: environment + - name: STORAGE_EXPIRATION_DAYS + valueFrom: + configMapKeyRef: + name: operation-config + key: storage-expiration-days - name: CRACK_WORDLIST_TOP_WORDS value: "10000" # either 10,000 or 100,000 diff --git a/helm/nemesis/templates/operation-config.configmap.yaml b/helm/nemesis/templates/operation-config.configmap.yaml index 9a22f38..0dba9ab 100644 --- a/helm/nemesis/templates/operation-config.configmap.yaml +++ b/helm/nemesis/templates/operation-config.configmap.yaml @@ -10,4 +10,4 @@ data: storage_provider: {{ default "minio" .Values.operation.storageProvider | quote }} nemesis-http-server: {{ .Values.operation.nemesisHttpServer | quote }} environment: {{ default "production" .Values.operation.environment | quote }} - data-expiration-days: {{ default "100" .Values.operation.dataExpirationDays | quote }} \ No newline at end of file + storage-expiration-days: {{ default "100" .Values.operation.storageExpirationDays | quote }} \ No newline at end of file diff --git a/helm/nemesis/values.yaml b/helm/nemesis/values.yaml index d174f0a..44136f4 100644 --- a/helm/nemesis/values.yaml +++ b/helm/nemesis/values.yaml @@ -15,7 +15,7 @@ operation: basicAuthUser: "nemesis" # Default password for basic auth # sets to a random value if not specified - # basicAuthPassword: "" + # basicAuthPassword: "Qwerty12345" storage: minio @@ -95,7 +95,7 @@ postgres: dashboard: nodePort: 30073 image: - repository: t94j0/nemesis-dashboard + repository: specterops/nemesis-dashboard tag: latest pullPolicy: IfNotPresent username: "nemesis" @@ -105,7 +105,7 @@ dashboard: nemesisWaiter: image: - repository: t94j0/nemesis-waiter + repository: specterops/nemesis-waiter tag: latest pullPolicy: IfNotPresent @@ -127,13 +127,13 @@ dotnet: nodePort: 31800 resources: {} image: - repository: t94j0/nemesis-dotnet + repository: specterops/nemesis-dotnet tag: latest pullPolicy: IfNotPresent enrichment: image: - repository: t94j0/nemesis-enrichment + repository: specterops/nemesis-enrichment tag: latest pullPolicy: IfNotPresent cracklist: @@ -150,7 +150,7 @@ enrichment: jupyter: nodePort: 31888 image: - repository: t94j0/nemesis-jupyter + repository: specterops/nemesis-jupyter tag: latest pullPolicy: IfNotPresent @@ -158,7 +158,7 @@ nlp: prometheusNodePort: 31692 nodePort: 31803 image: - repository: t94j0/nemesis-nlp + repository: specterops/nemesis-nlp tag: latest pullPolicy: IfNotPresent resources: @@ -172,7 +172,7 @@ nlp: passwordcracker: nodePort: 31990 image: - repository: t94j0/nemesis-passwordcracker + repository: specterops/nemesis-passwordcracker tag: latest pullPolicy: IfNotPresent resources: diff --git a/skaffold.yaml b/skaffold.yaml index e1bf7e7..823acca 100644 --- a/skaffold.yaml +++ b/skaffold.yaml @@ -40,7 +40,6 @@ build: sync: infer: - "**/*" - deploy: helm: releases: @@ -80,8 +79,7 @@ deploy: upgrade: ["--timeout=30m"] install: - ["--timeout=30m"] - + ["--timeout=45m"] portForward: - resourceType: service resourceName: ingress-nginx-controller @@ -93,4 +91,37 @@ portForward: resourceName: jupyter namespace: default port: 8888 - address: 0.0.0.0 \ No newline at end of file + address: 0.0.0.0 +--- +apiVersion: skaffold/v4beta6 +kind: Config +metadata: + name: enrichment +build: + tagPolicy: + sha256: {} + local: + push: false + tryImportMissing: false + useBuildkit: true + concurrency: 0 + artifacts: + - image: enrichment-dev + docker: + dockerfile: dockerfiles/enrichment.Dockerfile + sync: + manual: + - src: 'cmd/enrichment/**/*.py' + dest: '/app/cmd/enrichment' + strip: 'cmd/enrichment' +deploy: + helm: + releases: + - name: enrichment + chartPath: helm/enrichment + setValueTemplates: + operation.environment: test # force no storage + enrichment.image.repository: "{{.IMAGE_REPO_enrichment_dev}}" + enrichment.image.tag: "{{.IMAGE_REPO_enrichment_dev}}@{{.IMAGE_DIGEST_enrichment_dev}}" + setValues: + enrichment.image.pullPolicy: IfNotPresent \ No newline at end of file