trailofbits-buttercup/dev/docker-compose/compose.yaml

include:
  - ../../competition-server/compose.yaml
  - ../../competition-server/signoz/compose.yaml

services:
  # ugh the only solution i can come up with is a dind instance paired with every single build-bot we will need like k8s groups or something
  dind:
    expose:
      - "2375"
    image: docker:29-dind
    command: ["dockerd", "-H", "tcp://0.0.0.0:2375", "--tls=false", "--storage-driver=overlay2"]
    restart: always
    privileged: true  # This must run with privlege to support nested virtualization within the public Linux CP for `virtme-ng`
    environment:
      - DOCKER_TLS_CERTDIR  # intentionally blank to optimize runtime
    volumes:
      - type: bind
        source: ../../crs_scratch
        target: /crs_scratch
      - shared-tmp:/tmp
      - ../../node_data_storage/:/node_data

  redis:
    image: redis:7.4.2
    restart: always
    ports:
      - '127.0.0.1:6379:6379'
    # NOTE: Do not use the configuration file for now, so every time you restart
    # the container, it will reset the redis instance
    # command: redis-server /mount/redis.conf
    # volumes:
      # - cache:/data
      # - ../../redis:/mount
    healthcheck:
      test: [ "CMD-SHELL", "redis-cli ping | grep PONG" ]
      interval: 1s
      timeout: 3s
      retries: 5

  program-model:
    build:
      context: ../../
      dockerfile: ./program-model/Dockerfile
    command: ["buttercup-program-model", "serve"]
    env_file:
      - env.dev.compose
      - .env
    volumes:
      - ../../crs_scratch:/crs_scratch
      - ../../tasks_storage:/tasks_storage
      - ../../node_data_storage/:/node_data
    depends_on:
      redis:
        condition: service_healthy
      dind:
        condition: service_started
      # graphdb:
      #   condition: service_healthy

  coverage-bot:
    command: ["buttercup-coverage-bot", "--wdir", "/node_data/crs_scratch", "--redis_url", "redis://redis:6379"]
    build:
      context: ../../
      dockerfile: ./fuzzer/Dockerfile
    volumes:
      - ../../crs_scratch:/crs_scratch
      - ../../tasks_storage:/tasks_storage
      - ../../node_data_storage/:/node_data
    env_file:
      - env.dev.compose
      - .env
    depends_on:
      redis:
        condition: service_healthy
      dind:
        condition: service_started

  build-bot:
    command: ["buttercup-fuzzer-builder", "--wdir", "/node_data/crs_scratch", "--redis_url", "redis://redis:6379", "--timer", "5000"]
    build:
      context: ../../
      dockerfile: ./fuzzer/Dockerfile
    deploy:
      replicas: 4
      mode: replicated
    volumes:
      - ../../crs_scratch:/crs_scratch
      - ../../tasks_storage:/tasks_storage
      - ../../node_data_storage/:/node_data
    env_file:
      - env.dev.compose
      - .env
    depends_on:
      redis:
        condition: service_healthy
      dind:
        condition: service_started

  tracer-bot:
    command: ["buttercup-tracer-bot", "--wdir", "/node_data/crs_scratch", "--redis_url", "redis://redis:6379"]
    build:
      context: ../../
      dockerfile: ./fuzzer/Dockerfile
    deploy:
      replicas: 2
      mode: replicated
    volumes:
      - ../../crs_scratch:/crs_scratch
      - ../../tasks_storage:/tasks_storage
      - ../../node_data_storage/:/node_data
    env_file:
      - env.dev.compose
      - .env
    depends_on:
      redis:
        condition: service_healthy
      dind:
        condition: service_started

  fuzzer-bot:
    build:
      context: ../../
      dockerfile: ./fuzzer/Dockerfile
    command: ["buttercup-fuzzer"]
    deploy:
      replicas: 8
      mode: replicated
    env_file:
      - env.dev.compose
      - .env
    volumes:
      - ../../crs_scratch:/crs_scratch
      - ../../node_data_storage/:/node_data
    depends_on:
      redis:
        condition: service_healthy

  task-downloader:
    build:
      context: ../../
      dockerfile: ./orchestrator/Dockerfile
    command: ["buttercup-task-downloader", "serve"]
    volumes:
      - ../../tasks_storage:/tasks_storage
      - ../../node_data_storage/:/node_data
    env_file:
      - env.dev.compose
      - .env
    depends_on:
      redis:
        condition: service_healthy

  task-server:
    build:
      context: ../../
      dockerfile: ./orchestrator/Dockerfile
    command: ["buttercup-task-server"]
    ports:
      - "127.0.0.1:8000:8000"
    env_file:
      - env.dev.compose
      - .env
    depends_on:
      redis:
        condition: service_healthy

  scheduler:
    build:
      context: ../../
      dockerfile: ./orchestrator/Dockerfile
    command: ["buttercup-scheduler", "serve"]
    volumes:
      - ../../crs_scratch:/crs_scratch
      - ../../tasks_storage:/tasks_storage
      - ../../node_data_storage/:/node_data
    env_file:
      - env.dev.compose
      - .env
    depends_on:
      redis:
        condition: service_healthy

  seed-gen:
    build:
      context: ../../
      dockerfile: ./seed-gen/Dockerfile
    command: >
      /bin/sh -c 'timeout=120; interval=2; max=$$((timeout/interval)); i=0;
      while [ ! -s "$$LITELLM_API_KEY_FILE" ] && [ $$i -lt $$max ]; do
        echo "Waiting for LiteLLM key... ($$i/$$max)"; sleep $$interval; i=$$((i+1));
      done;
      if [ ! -s "$$LITELLM_API_KEY_FILE" ]; then
        echo "Timed out waiting for LiteLLM key"; exit 1;
      fi;
      export LITELLM_API_KEY=$$(cat "$$LITELLM_API_KEY_FILE"); exec seed-gen server'
    deploy:
      replicas: 4
      mode: replicated
    volumes:
      - ../../crs_scratch:/crs_scratch
      - ../../node_data_storage/:/node_data
      - litellm-user-keys-output:/secrets:ro
    env_file:
      - env.dev.compose
      - .env
    environment:
      - LANGFUSE_HOST=${LANGFUSE_HOST}
      - LANGFUSE_PUBLIC_KEY=${LANGFUSE_PUBLIC_KEY}
      - LANGFUSE_SECRET_KEY=${LANGFUSE_SECRET_KEY}
      - LITELLM_API_KEY_FILE=/secrets/llm-user_api_key
    depends_on:
      redis:
        condition: service_healthy
      litellm:
        condition: service_started
      dind:
        condition: service_started

  patcher:
    build:
      context: ../../
      dockerfile: ./patcher/Dockerfile
    command: >
      /bin/sh -c 'timeout=120; interval=2; max=$$((timeout/interval)); i=0;
      while [ ! -s "$$LITELLM_API_KEY_FILE" ] && [ $$i -lt $$max ]; do
        echo "Waiting for LiteLLM key... ($$i/$$max)"; sleep $$interval; i=$$((i+1));
      done;
      if [ ! -s "$$LITELLM_API_KEY_FILE" ]; then
        echo "Timed out waiting for LiteLLM key"; exit 1;
      fi;
      export LITELLM_API_KEY=$$(cat "$$LITELLM_API_KEY_FILE"); exec buttercup-patcher serve'
    volumes:
      - ../../crs_scratch:/crs_scratch
      - ../../node_data_storage/:/node_data
      - litellm-user-keys-output:/secrets:ro
    env_file:
      - env.dev.compose
      - .env
    environment:
      - LANGFUSE_HOST=${LANGFUSE_HOST}
      - LANGFUSE_PUBLIC_KEY=${LANGFUSE_PUBLIC_KEY}
      - LANGFUSE_SECRET_KEY=${LANGFUSE_SECRET_KEY}
      - LITELLM_API_KEY_FILE=/secrets/llm-user_api_key
    depends_on:
      redis:
        condition: service_healthy
      litellm:
        condition: service_started
      dind:
        condition: service_started

  litellm:
    image: ghcr.io/berriai/litellm:litellm_stable_release_branch-v1.57.8-stable
    configs:
      - source: litellm_config
        target: /app/config.yaml
    env_file: .env
    ports:
      - "127.0.0.1:8080:8080"
    environment:
        DATABASE_URL: "postgresql://litellm_user:litellm_password11@litellm-db:5432/litellm"

    # note: litellm recommends 1 worker for kubernetes: https://docs.litellm.ai/docs/proxy/prod
    command: ["--config", "/app/config.yaml", "--port", "8080", "--num_workers", "8"]
    depends_on:
      litellm-db:
        condition: service_healthy

  # One-time service to create LiteLLM user API keys (mirrors k8s Job)
  litellm-user-keys-setup:
    image: curlimages/curl:8.3.0
    user: "0"
    restart: "no"
    command: >
      /bin/sh -c 'if [ -s /output/llm-user_api_key ]; then echo "API key already present, skipping creation"; exit 0; fi; \
        echo "Waiting for LiteLLM..."; \
        until curl -sSf "$$LITELLM_URL/health/readiness" >/dev/null 2>&1; do sleep 2; done; \
        echo "Creating key for llm-user"; \
        resp=$$(curl -sS -X POST "$$LITELLM_URL/key/generate" -H "Authorization: Bearer $$LITELLM_MASTER_KEY" -H "Content-Type: application/json" -d "{\"user\":\"llm-user\",\"max_budget\":$${LITELLM_MAX_BUDGET:-100}}" -o -); \
        key=$$(echo "$$resp" | sed -n "s/.*\"key\"[[:space:]]*:[[:space:]]*\"\\([^\"]*\\)\".*/\\1/p"); \
        if [ -z "$$key" ]; then echo "Failed to obtain key"; echo "response: $$resp"; exit 1; fi; \
        printf "%s" "$$key" > /output/llm-user_api_key && echo "Key saved to /output/llm-user_api_key"'
    env_file:
      - .env
    environment:
      - LITELLM_URL=http://litellm:8080
      - LITELLM_MASTER_KEY=${BUTTERCUP_LITELLM_KEY}
      - LITELLM_MAX_BUDGET=${LITELLM_MAX_BUDGET:-100}
    volumes:
      - ../../crs_scratch:/crs_scratch
      - litellm-user-keys-output:/output
    depends_on:
      litellm:
        condition: service_started

  litellm-db:
    image: postgres:17.2
    restart: always
    environment:
      POSTGRES_DB: litellm
      POSTGRES_USER: litellm_user
      POSTGRES_PASSWORD: litellm_password11
    healthcheck:
      test: ["CMD-SHELL", "pg_isready -d $${POSTGRES_DB} -U $${POSTGRES_USER}"]
      interval: 5s
      timeout: 5s
      retries: 10

  buttercup-ui:
    build:
      context: ../../
      dockerfile: ./orchestrator/Dockerfile
    command: ["buttercup-ui"]
    ports:
      - "127.0.0.1:31323:31323"
    env_file:
      - env.dev.compose
      - .env


configs:
  litellm_config:
    file: ../../litellm/litellm_config.yaml

volumes:
  cache:
    driver: local
  graphdb_data:
    driver: local
  shared-tmp:
  litellm-user-keys-output: