admin/SpecterOps-Nemesis
1
0
Fork 0
mirror of https://github.com/SpecterOps/Nemesis synced 2026-06-08 12:36:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

Skaffold and container location modifications

Browse Source 
- `enrichment` dev profile for Skaffold
- container locations moved to specterops dockerhub
This commit is contained in:
Will
2024-03-12 13:36:36 -07:00
parent 39b66f5d77
commit eb86360a55
14 changed files with 655 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
helm/enrichment/Chart.yaml
+7
View File
@@ -0,0 +1,7 @@
apiVersion: v2
name: nemesis-enrichment
description: The enrichment container for Nemesis
type: application
version: 0.1.0
appVersion: "0.1.0a"
icon: https://github.com/SpecterOps/Nemesis/raw/main/img/nemesis_white.png
helm/enrichment/README.md
+3
View File
@@ -0,0 +1,3 @@
# Helm
Chart for the `enrichment` for use with Skaffold during development.
helm/enrichment/templates/NOTES.txt
+5
View File
@@ -0,0 +1,5 @@
------------------------------------------------------------------------
Enrichment container should now be deployed.
------------------------------------------------------------------------
helm/enrichment/templates/_helpers.tpl
+62
View File
@@ -0,0 +1,62 @@
{{/*
Expand the name of the chart.
*/}}
{{- define "nemesis.name" -}}
{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
{{- end }}
{{/*
Create a default fully qualified app name.
We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
If release name contains chart name it will be used as a full name.
*/}}
{{- define "nemesis.fullname" -}}
{{- if .Values.fullnameOverride }}
{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
{{- else }}
{{- $name := default .Chart.Name .Values.nameOverride }}
{{- if contains $name .Release.Name }}
{{- .Release.Name | trunc 63 | trimSuffix "-" }}
{{- else }}
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
{{- end }}
{{- end }}
{{- end }}
{{/*
Create chart name and version as used by the chart label.
*/}}
{{- define "nemesis.chart" -}}
{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
{{- end }}
{{/*
Common labels
*/}}
{{- define "nemesis.labels" -}}
helm.sh/chart: {{ include "nemesis.chart" . }}
{{ include "nemesis.selectorLabels" . }}
{{- if .Chart.AppVersion }}
app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
{{- end }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
{{- end }}
{{/*
Selector labels
*/}}
{{- define "nemesis.selectorLabels" -}}
app.kubernetes.io/name: {{ include "nemesis.name" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
{{- end }}
{{/*
Create the name of the service account to use
*/}}
{{- define "nemesis.serviceAccountName" -}}
{{- if .Values.serviceAccount.create }}
{{- default (include "nemesis.fullname" .) .Values.serviceAccount.name }}
{{- else }}
{{- default "default" .Values.serviceAccount.name }}
{{- end }}
{{- end }}
helm/enrichment/templates/enrichment.deployment.yaml
+240
View File
@@ -0,0 +1,240 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: enrichment
labels:
app: enrichment
component: enrichment-pipeline
spec:
replicas: 1
selector:
matchLabels:
app: enrichment
strategy: {}
template:
metadata:
labels:
app: enrichment
component: enrichment-pipeline
spec:
volumes:
- name: cracklist
emptyDir: {}
- name: noseyparker
emptyDir: {}
- name: noseyparker-rules
configMap:
name: noseyparker-rules
initContainers:
- name: noseyparker
image: ghcr.io/praetorian-inc/noseyparker:v0.16.0
imagePullPolicy: IfNotPresent
command: ["sh", "-c", "cp /usr/local/bin/noseyparker /opt/noseyparker/"]
volumeMounts:
- name: noseyparker
mountPath: /opt/noseyparker/
containers:
- env:
- name: LOG_LEVEL
valueFrom:
configMapKeyRef:
name: operation-config
key: log-level
- name: DATA_DOWNLOAD_DIR
value: "/tmp"
- name: ENVIRONMENT
valueFrom:
configMapKeyRef:
name: operation-config
key: environment
- name: STORAGE_EXPIRATION_DAYS
valueFrom:
configMapKeyRef:
name: operation-config
key: storage-expiration-days
- name: CRACK_WORDLIST_TOP_WORDS
value: "10000" # either 10,000 or 100,000
- name: JTR_INSTANCES
value: "1"
- name: EXTRACTED_ARCHIVE_SIZE_LIMIT # limit of extracted archives to process, 500 MB
value: "500000000"
- name: CHUNK_SIZE # number of bytes to read at a time per file
value: "128000"
- name: MODEL_WORD_LIMIT # only extract passwords for documents below this word limit
value: "100000"
- name: YARA_API_PORT
value: "9700"
- name: PLAINTEXT_SIZE_LIMIT
value: "50000000" # max number of bytes for a plaintext doc to prevent indexing of embeddings
- name: CONTEXT_WORDS
value: "5"
- name: ML_CHUNK_SIZE
value: "5000"
- name: PROB_THRESHOLD
value: ".95"
- name: TENSORFLOW_URI
value: "http://tensorflow-serving:8501/"
- name: TIKA_URI
value: http://tika:9998/
- name: DOTNET_URI
value: http://dotnet:9800/
- name: GOTENBERG_URI
value: http://gotenberg:3000/
- name: ML_MODELS_URI
value: http://127.0.0.1:5000/
- name: CRACK_LIST_URI
value: http://127.0.0.1:9900/
- name: NEMESIS_HTTP_SERVER
valueFrom:
configMapKeyRef:
name: operation-config
key: nemesis-http-server
- name: PUBLIC_NEMESIS_URL
value: "$(NEMESIS_HTTP_SERVER)dashboard/"
- name: PUBLIC_KIBANA_URL
value: "$(NEMESIS_HTTP_SERVER)kibana/"
- name: WEB_API_URL
value: "$(NEMESIS_HTTP_SERVER)api/"
- name: POSTGRES_SERVER
value: postgres
- name: POSTGRES_PORT
value: "5432"
- name: POSTGRES_DATABASE
value: nemesis
- name: POSTGRES_USER
valueFrom:
secretKeyRef:
name: postgres-creds
key: postgres-user
- name: POSTGRES_PASSWORD
valueFrom:
secretKeyRef:
name: postgres-creds
key: postgres-password
- name: POSTGRES_CONNECTION_URI
value: "postgresql://$(POSTGRES_USER):$(POSTGRES_PASSWORD)@$(POSTGRES_SERVER):$(POSTGRES_PORT)/$(POSTGRES_DATABASE)"
- name: DB_ITERATION_SIZE
value: "1000"
- name: PROMETHEUS_DISABLE_CREATED_SERIES
value: "True"
- name: AWS_ACCESS_KEY_ID
valueFrom:
secretKeyRef:
name: aws-creds
key: aws_access_key_id
- name: AWS_SECRET_ACCESS_KEY
valueFrom:
secretKeyRef:
name: aws-creds
key: aws_secret_key
- name: AWS_BUCKET
valueFrom:
configMapKeyRef:
name: aws-config
key: aws-bucket
- name: AWS_DEFAULT_REGION
valueFrom:
configMapKeyRef:
name: aws-config
key: aws-default-region
- name: AWS_KMS_KEY_ALIAS
valueFrom:
configMapKeyRef:
name: aws-config
key: aws-kms-key-alias
- name: STORAGE_PROVIDER
valueFrom:
configMapKeyRef:
name: operation-config
key: storage_provider
- name: MINIO_ROOT_USER
valueFrom:
secretKeyRef:
name: minio-creds
key: root-user
- name: MINIO_ROOT_PASSWORD
valueFrom:
secretKeyRef:
name: minio-creds
key: root-password
- name: ASSESSMENT_ID
valueFrom:
configMapKeyRef:
name: operation-config
key: assessment-id
- name: RABBITMQ_CONNECTION_URI
valueFrom:
secretKeyRef:
name: rabbitmq-creds
key: rabbitmq-connectionuri
- name: ELASTICSEARCH_URL
value: http://nemesis-es-internal-http:9200/
- name: KIBANA_URL
value: http://nemesis-kb-http:5601/
- name: ELASTICSEARCH_USER
valueFrom:
secretKeyRef:
name: elasticsearch-users
key: username
- name: ELASTICSEARCH_PASSWORD
valueFrom:
secretKeyRef:
name: elasticsearch-users
key: password
- name: PROMETHEUS_PORT
value: "8000"
- name: SLACK_CHANNEL
valueFrom:
configMapKeyRef:
name: operation-config
key: slack-alert-channel
- name: SLACK_WEBHOOK_URL
valueFrom:
secretKeyRef:
name: operation-creds
key: slack_web_hook
- name: SLACK_USERNAME
value: Nemesis
- name: SLACK_EMOJI
value: ":smiling_imp:"
- name: DISABLE_ALERTING
valueFrom:
configMapKeyRef:
name: operation-config
key: disable-slack-alerting
- name: REPROCESSING_WORKERS
value: "5"
image: "{{ .Values.enrichment.image.repository }}:{{ .Values.enrichment.image.tag }}"
imagePullPolicy: {{ .Values.enrichment.image.pullPolicy }}
name: enrichment
ports:
- containerPort: 9700
name: yara-http
resources: {{ toYaml .Values.enrichment.resources | nindent 12 }}
volumeMounts:
- name: cracklist
mountPath: /opt/cracklist/
- name: noseyparker
mountPath: /opt/noseyparker/
- name: noseyparker-rules
mountPath: /opt/noseyparker/noseyparker.rules
subPath: noseyparker.rules
readOnly: true
readinessProbe:
httpGet:
path: /ready
port: yara-http
initialDelaySeconds: 5
periodSeconds: 5
timeoutSeconds: 5
failureThreshold: 10
restartPolicy: Always
status: {}
helm/enrichment/templates/enrichment.ingress.yaml
+47
View File
@@ -0,0 +1,47 @@
{{- if ne .Values.operation.environment "development" }}
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: enrichment-ingress
annotations:
nginx.ingress.kubernetes.io/proxy-body-size: "1000m"
nginx.ingress.kubernetes.io/auth-type: basic
nginx.ingress.kubernetes.io/auth-secret: basic-auth
nginx.ingress.kubernetes.io/auth-realm: 'Authentication Required'
nginx.ingress.kubernetes.io/ssl-redirect: "false"
nginx.ingress.kubernetes.io/rewrite-target: /$2
spec:
ingressClassName: nginx
rules:
- http:
paths:
- path: /crack-list(/|$)(.*)
pathType: ImplementationSpecific
backend:
service:
name: enrichment-cracklist
port:
number: 9900
- path: /yara(/|$)(.*)
pathType: ImplementationSpecific
backend:
service:
name: enrichment-yara
port:
number: 9700
- path: /api(/|$)(.*)
pathType: ImplementationSpecific
backend:
service:
name: enrichment-webapi
port:
number: 9910
- path: /
pathType: ImplementationSpecific
backend:
service:
name: enrichment-landingpage
port:
number: 9920
{{- end }}
helm/enrichment/templates/enrichment.service.yaml
+97
View File
@@ -0,0 +1,97 @@
{{- if ne .Values.operation.environment "development" }}
apiVersion: v1
kind: Service
metadata:
labels:
app: enrichment
component: enrichment-pipeline
name: enrichment-yara
spec:
type: NodePort
ports:
- name: "9700"
port: 9700
targetPort: 9700
nodePort: {{ .Values.enrichment.yara.nodePort }}
selector:
app: enrichment
status:
loadBalancer: {}
---
apiVersion: v1
kind: Service
metadata:
labels:
app: enrichment
component: enrichment-pipeline
name: enrichment-cracklist
spec:
type: NodePort
ports:
- name: "9900"
port: 9900
targetPort: 9900
nodePort: {{ .Values.enrichment.cracklist.nodePort }}
selector:
app: enrichment
status:
loadBalancer: {}
---
apiVersion: v1
kind: Service
metadata:
labels:
app: enrichment
component: enrichment-pipeline
name: enrichment-webapi
spec:
type: NodePort
ports:
- name: "9910"
port: 9910
targetPort: 9910
nodePort: {{ .Values.enrichment.webapi.nodePort }}
selector:
app: enrichment
status:
loadBalancer: {}
---
apiVersion: v1
kind: Service
metadata:
labels:
app: enrichment
component: enrichment-landingpage
name: enrichment-landingpage
spec:
type: NodePort
ports:
- name: "9920"
port: 9920
targetPort: 9920
nodePort: {{ .Values.enrichment.landingpage.nodePort }}
selector:
app: enrichment
status:
loadBalancer: {}
---
apiVersion: v1
kind: Service
metadata:
labels:
app: enrichment
component: enrichment-monitoring
name: enrichment-monitoring
spec:
type: NodePort
ports:
- name: "8000"
port: 8000
targetPort: 8000
nodePort: {{ .Values.enrichment.monitoring.nodePort }}
selector:
app: enrichment
status:
loadBalancer: {}
{{- end }}
helm/enrichment/values.schema.json
+129
View File
@@ -0,0 +1,129 @@
{
"type": "object",
"$schema": "http://json-schema.org/draft-07/schema",
"required": [],
"properties": {
"enrichment": {
"type": "object",
"required": [],
"properties": {
"image": {
"type": "object",
"required": [],
"properties": {
"repository": {
"type": [
"string",
"boolean",
"number",
"object",
"array"
],
"default": "specterops/nemesis-enrichment"
},
"tag": {
"type": [
"string",
"boolean",
"number",
"object",
"array"
],
"default": "latest"
},
"pullPolicy": {
"type": [
"string",
"boolean",
"number",
"object",
"array"
],
"default": "IfNotPresent"
}
}
},
"cracklist": {
"type": "object",
"required": [],
"properties": {
"nodePort": {
"type": [
"string",
"boolean",
"number",
"object",
"array"
],
"default": "30089"
}
}
},
"yara": {
"type": "object",
"required": [],
"properties": {
"nodePort": {
"type": [
"string",
"boolean",
"number",
"object",
"array"
],
"default": "30079"
}
}
},
"webapi": {
"type": "object",
"required": [],
"properties": {
"nodePort": {
"type": [
"string",
"boolean",
"number",
"object",
"array"
],
"default": "30808"
}
}
},
"landingpage": {
"type": "object",
"required": [],
"properties": {
"nodePort": {
"type": [
"string",
"boolean",
"number",
"object",
"array"
],
"default": "30809"
}
}
},
"monitoring": {
"type": "object",
"required": [],
"properties": {
"nodePort": {
"type": [
"string",
"boolean",
"number",
"object",
"array"
],
"default": "31693"
}
}
}
}
}
}
}
helm/enrichment/values.yaml
+15
View File
@@ -0,0 +1,15 @@
enrichment:
image:
repository: specterops/nemesis-enrichment-dev
tag: latest
pullPolicy: IfNotPresent
cracklist:
nodePort: 30089
yara:
nodePort: 30079
webapi:
nodePort: 30808
landingpage:
nodePort: 30809
monitoring:
nodePort: 31693
helm/monitoring/values.yaml
+1 -1
View File
@@ -87,6 +87,6 @@ postgresExporter:
nemesisWaiter:
image:
repository: t94j0/nemesis-waiter
repository: specterops/nemesis-waiter
tag: latest
pullPolicy: IfNotPresent
helm/nemesis/templates/enrichment.deployment.yaml
+5
View File
@@ -54,6 +54,11 @@ spec:
configMapKeyRef:
name: operation-config
key: environment
- name: STORAGE_EXPIRATION_DAYS
valueFrom:
configMapKeyRef:
name: operation-config
key: storage-expiration-days
- name: CRACK_WORDLIST_TOP_WORDS
value: "10000" # either 10,000 or 100,000
helm/nemesis/templates/operation-config.configmap.yaml
+1 -1
View File
@@ -10,4 +10,4 @@ data:
storage_provider: {{ default "minio" .Values.operation.storageProvider | quote }}
nemesis-http-server: {{ .Values.operation.nemesisHttpServer | quote }}
environment: {{ default "production" .Values.operation.environment | quote }}
data-expiration-days: {{ default "100" .Values.operation.dataExpirationDays | quote }}
storage-expiration-days: {{ default "100" .Values.operation.storageExpirationDays | quote }}
helm/nemesis/values.yaml
+8 -8
View File
@@ -15,7 +15,7 @@ operation:
basicAuthUser: "nemesis"
# Default password for basic auth
# sets to a random value if not specified
# basicAuthPassword: ""
# basicAuthPassword: "Qwerty12345"
storage: minio
@@ -95,7 +95,7 @@ postgres:
dashboard:
nodePort: 30073
image:
repository: t94j0/nemesis-dashboard
repository: specterops/nemesis-dashboard
tag: latest
pullPolicy: IfNotPresent
username: "nemesis"
@@ -105,7 +105,7 @@ dashboard:
nemesisWaiter:
image:
repository: t94j0/nemesis-waiter
repository: specterops/nemesis-waiter
tag: latest
pullPolicy: IfNotPresent
@@ -127,13 +127,13 @@ dotnet:
nodePort: 31800
resources: {}
image:
repository: t94j0/nemesis-dotnet
repository: specterops/nemesis-dotnet
tag: latest
pullPolicy: IfNotPresent
enrichment:
image:
repository: t94j0/nemesis-enrichment
repository: specterops/nemesis-enrichment
tag: latest
pullPolicy: IfNotPresent
cracklist:
@@ -150,7 +150,7 @@ enrichment:
jupyter:
nodePort: 31888
image:
repository: t94j0/nemesis-jupyter
repository: specterops/nemesis-jupyter
tag: latest
pullPolicy: IfNotPresent
@@ -158,7 +158,7 @@ nlp:
prometheusNodePort: 31692
nodePort: 31803
image:
repository: t94j0/nemesis-nlp
repository: specterops/nemesis-nlp
tag: latest
pullPolicy: IfNotPresent
resources:
@@ -172,7 +172,7 @@ nlp:
passwordcracker:
nodePort: 31990
image:
repository: t94j0/nemesis-passwordcracker
repository: specterops/nemesis-passwordcracker
tag: latest
pullPolicy: IfNotPresent
resources:
skaffold.yaml
+35 -4
View File
@@ -40,7 +40,6 @@ build:
sync:
infer:
- "**/*"
deploy:
helm:
releases:
@@ -80,8 +79,7 @@ deploy:
upgrade:
["--timeout=30m"]
install:
["--timeout=30m"]
["--timeout=45m"]
portForward:
- resourceType: service
resourceName: ingress-nginx-controller
@@ -93,4 +91,37 @@ portForward:
resourceName: jupyter
namespace: default
port: 8888
address: 0.0.0.0
address: 0.0.0.0
---
apiVersion: skaffold/v4beta6
kind: Config
metadata:
name: enrichment
build:
tagPolicy:
sha256: {}
local:
push: false
tryImportMissing: false
useBuildkit: true
concurrency: 0
artifacts:
- image: enrichment-dev
docker:
dockerfile: dockerfiles/enrichment.Dockerfile
sync:
manual:
- src: 'cmd/enrichment/**/*.py'
dest: '/app/cmd/enrichment'
strip: 'cmd/enrichment'
deploy:
helm:
releases:
- name: enrichment
chartPath: helm/enrichment
setValueTemplates:
operation.environment: test # force no storage
enrichment.image.repository: "{{.IMAGE_REPO_enrichment_dev}}"
enrichment.image.tag: "{{.IMAGE_REPO_enrichment_dev}}@{{.IMAGE_DIGEST_enrichment_dev}}"
setValues:
enrichment.image.pullPolicy: IfNotPresent