mirror of
https://github.com/trailofbits/buttercup
synced 2026-06-21 14:11:39 +00:00
Henrik Brodin
1e4e56490f
This PR introduces a NodeLocal storage concept. Data is always processed on the local node, and only replicated to the azure file storage as needed.
A DaemonSet is introduced that syncs any corpus on the local node with the global corpus directory.
The following path structure is introduced:
/node_data/crs_scratch
/node_data/tasks_storage
/node_data/scratch
The first two directories is the equivalent of the previous usage of
/crs_scratch
/tasks_storage
The scratch is for basing temporary files and directories on the same filesystem as node-local version to be able to do atomic renames.
Further, ChallengeTask data is used normally on the /node_data/... paths, but is stored as /tasks_storage/{uuid}.tgz on the "remote" storage. This is to speed up copying.
Corpus and CrashSets are synced as is.
70 lines
2.7 KiB
Plaintext
70 lines
2.7 KiB
Plaintext
NODE_DATA_DIR=/node_data
|
|
DOCKER_HOST=tcp://dind:2375
|
|
|
|
BUTTERCUP_LITELLM_KEY=sk-1234
|
|
BUTTERCUP_LITELLM_HOSTNAME=http://litellm:8080
|
|
|
|
BUTTERCUP_DOWNLOADER_SERVE__REDIS_URL=redis://redis:6379
|
|
BUTTERCUP_DOWNLOADER_LOG_LEVEL=debug
|
|
BUTTERCUP_DOWNLOADER_DOWNLOAD_DIR=/node_data/tasks_storage
|
|
BUTTERCUP_DOWNLOADER_SERVE__SLEEP_TIME=5
|
|
|
|
BUTTERCUP_TASK_SERVER_REDIS_URL=redis://redis:6379
|
|
BUTTERCUP_TASK_SERVER_LOG_LEVEL=debug
|
|
BUTTERCUP_TASK_SERVER_HOST=0.0.0.0
|
|
BUTTERCUP_TASK_SERVER_PORT=8000
|
|
BUTTERCUP_TASK_SERVER_API_KEY_ID=515cc8a0-3019-4c9f-8c1c-72d0b54ae561
|
|
BUTTERCUP_TASK_SERVER_API_TOKEN_HASH='$argon2id$v=19$m=65536,t=3,p=4$Dg1v6NPGTyXPoOPF4ozD5A$wa/85ttk17bBsIASSwdR/uGz5UKN/bZuu4wu+JIy1iA'
|
|
BUTTERCUP_TASK_SERVER_COMPETITION_API_URL=http://scantron:31323
|
|
BUTTERCUP_TASK_SERVER_COMPETITION_API_USERNAME=11111111-1111-1111-1111-111111111111
|
|
BUTTERCUP_TASK_SERVER_COMPETITION_API_PASSWORD=secret
|
|
|
|
# Client Authentication:
|
|
# CRS_API_KEY_ID=515cc8a0-3019-4c9f-8c1c-72d0b54ae561
|
|
# CRS_API_TOKEN=VGuAC8axfOnFXKBB7irpNDOKcDjOlnyB
|
|
|
|
# Competition API Authentication
|
|
COMPETITION_API_KEY_ID=11111111-1111-1111-1111-111111111111
|
|
COMPETITION_API_KEY_TOKEN=secret
|
|
|
|
BUTTERCUP_SCHEDULER_SERVE__REDIS_URL=redis://redis:6379
|
|
BUTTERCUP_SCHEDULER_LOG_LEVEL=debug
|
|
BUTTERCUP_SCHEDULER_SERVE__SLEEP_TIME=5
|
|
BUTTERCUP_SCHEDULER_SERVE__COMPETITION_API_URL=http://scantron:31323
|
|
BUTTERCUP_SCHEDULER_TASKS_STORAGE_DIR=/node_data/tasks_storage
|
|
BUTTERCUP_SCHEDULER_SCRATCH_DIR=/node_data/crs_scratch
|
|
|
|
BUTTERCUP_PATCHER_SERVE__REDIS_URL=redis://redis:6379
|
|
BUTTERCUP_PATCHER_SERVE__SLEEP_TIME=5
|
|
BUTTERCUP_PATCHER_LOG_LEVEL=debug
|
|
BUTTERCUP_PATCHER_TASK_STORAGE_DIR=/node_data/crs_scratch
|
|
BUTTERCUP_PATCHER_SCRATCH_DIR=/node_data/crs_scratch
|
|
|
|
BUTTERCUP_PROGRAM_MODEL_SERVE__REDIS_URL=redis://redis:6379
|
|
BUTTERCUP_PROGRAM_MODEL_SERVE__SLEEP_TIME=5
|
|
BUTTERCUP_PROGRAM_MODEL_LOG_LEVEL=debug
|
|
BUTTERCUP_PROGRAM_MODEL_SCRATCH_DIR=/node_data/crs_scratch
|
|
BUTTERCUP_PROGRAM_MODEL_GRAPHDB_ENABLED=false
|
|
|
|
BUTTERCUP_FUZZER_WDIR=/node_data/crs_scratch
|
|
|
|
# Keep these short for local development, there's just one worker for each
|
|
# service and we want to retry as soon as possible usually.
|
|
BUILD_TASK_TIMEOUT_MS=120000
|
|
BUILD_OUTPUT_TASK_TIMEOUT_MS=120000
|
|
DOWNLOAD_TASK_TIMEOUT_MS=120000
|
|
READY_TASK_TIMEOUT_MS=120000
|
|
DELETE_TASK_TIMEOUT_MS=120000
|
|
CRASH_TASK_TIMEOUT_MS=120000
|
|
PATCH_TASK_TIMEOUT_MS=120000
|
|
CONFIRMED_VULNERABILITIES_TASK_TIMEOUT_MS=120000
|
|
INDEX_TASK_TIMEOUT_MS=120000
|
|
INDEX_OUTPUT_TASK_TIMEOUT_MS=120000
|
|
TRACED_VULNERABILITIES_TASK_TIMEOUT_MS=120000
|
|
|
|
# OTel
|
|
OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4317
|
|
# token is b64 encoded "username:password"
|
|
OTEL_EXPORTER_OTLP_HEADERS="Authorization=Basic dXNlcm5hbWU6cGFzc3dvcmQ="
|
|
OTEL_EXPORTER_OTLP_PROTOCOL="grpc"
|