Test and improve AKS deployment (#336)

* Test configuration for Azure deployment
* ci: re-enable docker build/push
* ci: docker push labeled PRs
* Use `make deploy` for both local and azure deployments
* doc: update AKS_DEPLOYMENT doc
* add confirmation in Makefile
* Make resource group location configurable

.github/workflows/docker.yml

@@ -17,6 +17,7 @@ on:
     branches: ["main"]
   pull_request:
     branches: ["main"]
+    types: [labeled, synchronize]
 
 env:
   REGISTRY: ghcr.io
@@ -76,7 +77,7 @@ jobs:
         with:
           context: .
           file: ${{ matrix.dockerfile }}
-          push: ${{ (github.event_name != 'pull_request' || startsWith(github.head_ref, 'ci/') || inputs.push) && 'true' || 'false' }}
+          push: ${{ (github.event_name != 'pull_request' || startsWith(github.head_ref, 'ci/') || inputs.push || (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'docker-push'))) && 'true' || 'false' }}
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
           cache-from: type=gha

.github/workflows/integration.yml

@@ -105,7 +105,7 @@ jobs:
         working-directory: deployment
 
       - name: Run CRS
-        run: make deploy-local
+        run: make deploy
 
       - name: Submit custom task to the CRS
         if: github.event_name == 'workflow_dispatch'

CONTRIBUTING.md

@@ -9,7 +9,7 @@ Thank you for contributing to the Buttercup Cyber Reasoning System!
 git clone --recurse-submodules https://github.com/trailofbits/buttercup.git
 cd buttercup
 make setup-local      # Automated setup
-make deploy-local     # Start environment
+make deploy           # Start environment
 
 # Setup development tools (optional but recommended)
 pip install pre-commit
@@ -50,7 +50,7 @@ make undeploy                # Clean up resources
 
 3. **Full System** (90+ min): Weekly or with `full-integration` label
    ```bash
-   make deploy-local && make send-libpng-task
+   make deploy && make send-libpng-task
    ```
 
 ## Project Structure

Makefile

@@ -1,6 +1,6 @@
 # Makefile for Trail of Bits AIxCC Finals CRS
 
-.PHONY: help setup-local setup-azure validate deploy deploy-local deploy-azure test undeploy install-cscope lint lint-component clean-local wait-crs check-crs crs-instance-id status send-integration-task
+.PHONY: help setup-local setup-azure validate deploy test undeploy install-cscope lint lint-component clean-local wait-crs check-crs crs-instance-id status send-integration-task
 
 # Default target
 help:
@@ -13,8 +13,6 @@ help:
 	@echo ""
 	@echo "Deployment:"
 	@echo "  deploy            - Deploy to current environment (local or azure)"
-	@echo "  deploy-local      - Deploy to local Minikube environment"
-	@echo "  deploy-azure      - Deploy to production AKS environment"
 	@echo ""
 	@echo "Status:"
 	@echo "  status              - Check the status of the deployment"
@@ -48,16 +46,6 @@ validate:
 	@echo "Validating setup..."
 	./scripts/validate-setup.sh
 
-# Deployment targets
-deploy:
-	@echo "Deploying to current environment..."
-	@if [ ! -f external/buttercup-cscope/configure.ac ]; then \
-		echo "Error: The git submodules have not been initialized. Run 'git submodule update --init --recursive' first."; \
-		exit 1; \
-	fi
-	cd deployment && make up
-	make wait-crs
-
 wait-crs:
 	@echo "Waiting for CRS deployment to be ready..."
 	@if ! kubectl get namespace $${BUTTERCUP_NAMESPACE:-crs} >/dev/null 2>&1; then \
@@ -96,35 +84,29 @@ crs-instance-id:
 	fi
 	echo "CRS instance ID: $$(kubectl get configmap -n $${BUTTERCUP_NAMESPACE:-crs} crs-instance-id -o jsonpath='{.data.crs-instance-id}')"
 
-deploy-local:
-	@echo "Deploying to local Minikube environment..."
+deploy:
+	@echo "Deploying environment..."
 	@if [ ! -f deployment/env ]; then \
-		echo "Error: Configuration file not found. Run 'make setup-local' first."; \
+		echo "Error: Configuration file not found. Run 'make setup-local' or `make setup-azure` first."; \
 		exit 1; \
 	fi
 	@if [ ! -f external/buttercup-cscope/configure.ac ]; then \
 		echo "Error: The git submodules have not been initialized. Run 'git submodule update --init --recursive' first."; \
 		exit 1; \
 	fi
+	@echo "Deployment configuration:"
+	@grep 'CLUSTER_TYPE=' deployment/env || echo "CLUSTER_TYPE not set"
+	@grep 'K8S_VALUES_TEMPLATE' deployment/env || echo "K8S_VALUES_TEMPLATE not set"
+	@echo "Are you sure you want to deploy with these settings? Type 'yes' to continue:"
+	@read ans; \
+	ans_lc=$$(echo "$$ans" | tr '[:upper:]' '[:lower:]'); \
+	if [ "$$ans_lc" != "yes" ] && [ "$$ans_lc" != "y" ]; then \
+		echo "Aborted by user."; \
+		exit 1; \
+	fi
 	cd deployment && make up
 	make crs-instance-id
 	make wait-crs
-
-deploy-azure:
-	@echo "Deploying to production AKS environment..."
-	@if [ ! -f deployment/env ]; then \
-		echo "Error: Configuration file not found. Run 'make setup-azure' first."; \
-		exit 1; \
-	fi
-	@if [ ! -f external/buttercup-cscope/configure.ac ]; then \
-		echo "Error: The git submodules have not been initialized. Run 'git submodule update --init --recursive' first."; \
-		exit 1; \
-	fi
-	cd deployment && make up
-	crs_instance_id=$$(make crs-instance-id)
-	echo "CRS instance ID: $$crs_instance_id"
-	make wait-crs
-
 status:
 	@echo "----------PODS------------"
 	@kubectl get pods -n $${BUTTERCUP_NAMESPACE:-crs}

README.md

@@ -23,7 +23,7 @@
 
 **Note:** Buttercup uses third-party AI providers (LLMs from companies like OpenAI, Anthropic and Google), which cost money. Please ensure that you manage per-deployment costs by using the built-in LLM budget setting.
 
-**Note:** Buttercup works best with access to models from OpenAI **and** Anthropic, but can be run with at least one API key from one third-party provider (support for Gemini coming soon).
+**Note:** Buttercup works best with access to models from OpenAI **and** Anthropic, but can be run with at least one API key from one third-party provider. You can use a combination of OpenAI, Anthropic, and Google LLMs.
 
 ### Supported Systems
 
@@ -78,7 +78,7 @@ This script will install all dependencies, configure the environment, and guide
 1. Start Buttercup locally
 
 ```bash
-make deploy-local
+make deploy
 ```
 
 1. Verify local deployment:

deployment/.terraform.lock.hcl

@@ -5,6 +5,7 @@ provider "registry.terraform.io/azure/azapi" {
   version     = "2.2.0"
   constraints = "2.2.0"
   hashes = [
+    "h1:Us5LvK2ju2qo3MQlXVtDDKCt5SMFRDIHUL8ubVdCEUg=",
     "h1:yckm1jqUMUGSeS57a3uR0gG/V7scwvjpkRVXnQIUAo4=",
     "zh:062be5d8272cac297a88c2057449f449ea6906c4121ba3dfdeb5cecb3ff91178",
     "zh:1fd9abec3ffcbf8d0244408334e9bfc8f49ada50978cd73ee0ed5f8560987267",
@@ -25,6 +26,7 @@ provider "registry.terraform.io/hashicorp/azurerm" {
   version     = "4.17.0"
   constraints = "4.17.0"
   hashes = [
+    "h1:VgnUh7PiRa/76P+0NFk8vmrmfLnPT6+tOZ/AP6h4TeQ=",
     "h1:gpFgaBSkRTxhavgPAuqQcElHJqmRJ1RpQGr1K0dvVW8=",
     "zh:163b81a3bf29c8f161a1c100a48164b1bd1af434cd564b44596cb71a6c33f03d",
     "zh:2996b107d3c05a9db14458b32b6f22f8cde0adb96263196d82d3dc302907a257",
@@ -46,6 +48,7 @@ provider "registry.terraform.io/hashicorp/helm" {
   constraints = "2.17.0"
   hashes = [
     "h1:K5FEjxvDnxb1JF1kG1xr8J3pNGxoaR3Z0IBG9Csm/Is=",
+    "h1:kQMkcPVvHOguOqnxoEU2sm1ND9vCHiT8TvZ2x6v/Rsw=",
     "zh:06fb4e9932f0afc1904d2279e6e99353c2ddac0d765305ce90519af410706bd4",
     "zh:104eccfc781fc868da3c7fec4385ad14ed183eb985c96331a1a937ac79c2d1a7",
     "zh:129345c82359837bb3f0070ce4891ec232697052f7d5ccf61d43d818912cf5f3",
@@ -66,6 +69,7 @@ provider "registry.terraform.io/hashicorp/random" {
   constraints = "3.6.3"
   hashes = [
     "h1:Fnaec9vA8sZ8BXVlN3Xn9Jz3zghSETIKg7ch8oXhxno=",
+    "h1:zG9uFP8l9u+yGZZvi5Te7PV62j50azpgwPunq2vTm1E=",
     "zh:04ceb65210251339f07cd4611885d242cd4d0c7306e86dda9785396807c00451",
     "zh:448f56199f3e99ff75d5c0afacae867ee795e4dfda6cb5f8e3b2a72ec3583dd8",
     "zh:4b4c11ccfba7319e901df2dac836b1ae8f12185e37249e8d870ee10bb87a13fe",
@@ -86,6 +90,7 @@ provider "registry.terraform.io/hashicorp/time" {
   constraints = "0.12.1"
   hashes = [
     "h1:6BhxSYBJdBBKyuqatOGkuPKVenfx6UmLdiI13Pb3his=",
+    "h1:JzYsPugN8Fb7C4NlfLoFu7BBPuRVT2/fCOdCaxshveI=",
     "zh:090023137df8effe8804e81c65f636dadf8f9d35b79c3afff282d39367ba44b2",
     "zh:26f1e458358ba55f6558613f1427dcfa6ae2be5119b722d0b3adb27cd001efea",
     "zh:272ccc73a03384b72b964918c7afeb22c2e6be22460d92b150aaf28f29a7d511",

deployment/RELEASE.md

@@ -9,6 +9,7 @@
      - Microsoft.Compute
      - Microsoft.Storage
      - Microsoft.Network
+     - Microsoft.ContainerService
 
 2. Verify and adjust Azure quotas:
    - Total Regional vCPUs: ~2000

deployment/env.template

@@ -22,6 +22,7 @@ export CLUSTER_TYPE=minikube # or "aks"
 # export TF_VAR_ARM_SUBSCRIPTION_ID="<your-sub-id>"
 # export TF_VAR_usr_node_count=50
 # export TF_VAR_resource_group_name_prefix="<resource-prefix>"
+# export TF_VAR_resource_group_name="<existing-resource-group-name>"
 
 # TailScale variables, necessary only for production/staging deployments
 export TAILSCALE_ENABLED=false # or true

deployment/k8s/values-upstream-aks.template

@@ -0,0 +1,283 @@
+global:
+  environment: "aks"
+  orchestratorImage:
+    tag: "latest"
+  fuzzerImage:
+    tag: "latest"
+  seedGenImage:
+    tag: "latest"
+  patcherImage:
+    tag: "latest"
+  programModelImage:
+    tag: "latest"
+
+  langfuse:
+    enabled: ${LANGFUSE_ENABLED}
+    host: "${LANGFUSE_HOST}"
+    publicKey: "${LANGFUSE_PUBLIC_KEY}"
+    secretKey: "${LANGFUSE_SECRET_KEY}"
+
+  signoz:
+    deployed: ${DEPLOY_SIGNOZ}
+
+  crs:
+    api_key_id: ${CRS_KEY_ID}
+    api_key_token: ${CRS_KEY_TOKEN}
+    api_key_token_hash: "${CRS_KEY_TOKEN_HASH}"
+    api_url: "${CRS_URL}"
+    hostname: "${CRS_API_HOSTNAME}"
+    competition_api_key_id: ${COMPETITION_API_KEY_ID}
+    competition_api_key_token: ${COMPETITION_API_KEY_TOKEN}
+    competition_api_url: "ui"
+
+  otel:
+    endpoint: "${OTEL_ENDPOINT}"
+    token: "${OTEL_TOKEN}"
+    protocol: "${OTEL_PROTOCOL}"
+
+  queueTimeouts:
+    buildTaskTimeoutMs: 1200000
+    buildOutputTaskTimeoutMs: 120000
+    downloadTaskTimeoutMs: 720000
+    readyTaskTimeoutMs: 120000
+    deleteTaskTimeoutMs: 120000
+    crashTaskTimeoutMs: 900000
+    patchTaskTimeoutMs: 120000
+    confirmedVulnerabilitiesTaskTimeoutMs: 1800000
+    indexTaskTimeoutMs: 800000
+    indexOutputTaskTimeoutMs: 120000
+    tracedVulnerabilitiesTaskTimeoutMs: 120000
+
+volumes:
+  tasks_storage:
+    storageClass: "azurefile-csi-nfs-crs"
+    size: "10Gi"
+  crs_scratch:
+    storageClass: "azurefile-csi-nfs-crs"
+    size: "100Gi"
+
+# Dind daemon chart configuration
+dind-daemon:
+  enabled: true
+  resources:
+    limits:
+      cpu: 16000m
+      memory: 32Gi
+    requests:
+      cpu: 1000m
+      memory: 6Gi
+
+redis:
+  master:
+    persistence:
+      enabled: true
+      size: 10Gi
+    resources:
+      limits:
+        cpu: 8000m
+        memory: 32Gi
+      requests:
+        cpu: 1000m
+        memory: 12Gi
+
+litellm:
+  masterKey: "${LITELLM_MASTER_KEY}"
+  maxBudget: "${LITELLM_MAX_BUDGET}"
+  azure:
+    apiBase: "${AZURE_API_BASE}"
+    apiKey: "${AZURE_API_KEY}"
+  openai:
+    apiKey: "${OPENAI_API_KEY}"
+  anthropic:
+    apiKey: "${ANTHROPIC_API_KEY}"
+  gemini:
+    apiKey: "${GEMINI_API_KEY}"
+
+litellm-helm:
+  resources:
+    limits:
+      cpu: 4000m
+      memory: 8Gi
+    requests:
+      cpu: 1000m
+      memory: 1Gi
+
+  postgresql:
+    primary:
+      resources:
+        limits:
+          cpu: 4000m
+          memory: 8Gi
+        requests:
+          cpu: 500m
+          memory: 1Gi
+
+ui:
+  enabled: true
+
+competition-api:
+  enabled: false
+
+image-preloader:
+  baseImages:
+    - gcr.io/oss-fuzz/base-runner
+    - gcr.io/oss-fuzz/base-builder
+    - gcr.io/oss-fuzz/base-image
+    - gcr.io/oss-fuzz/base-clang
+    - gcr.io/oss-fuzz/base-runner-debug
+    - gcr.io/oss-fuzz-base/base-builder-jvm
+    - gcr.io/oss-fuzz-base/base-builder-ruby
+    - gcr.io/oss-fuzz-base/base-builder-go
+    - gcr.io/oss-fuzz-base/base-builder-python
+    - gcr.io/oss-fuzz-base/base-builder-rust
+  versions:
+    - latest
+
+signoz:
+  enabled: ${DEPLOY_SIGNOZ}
+
+registry-cache:
+  enabled: true
+  persistence:
+    enabled: true
+    size: 50Gi
+  resources:
+    limits:
+      cpu: 4000m
+      memory: 8Gi
+    requests:
+      cpu: 500m
+      memory: 2Gi
+
+task-server:
+  enabled: true
+  resources:
+    limits:
+      cpu: 1000m
+      memory: 1Gi
+    requests:
+      cpu: 250m
+      memory: 1Gi
+
+task-downloader:
+  enabled: true
+  replicaCount: 1
+  resources:
+    limits:
+      cpu: 4000m
+      memory: 8Gi
+    requests:
+      cpu: 250m
+      memory: 512Mi
+
+scheduler:
+  enabled: true
+  patchRequestsPerVulnerability: 1
+  resources:
+    limits:
+      cpu: 4000m
+      memory: 3Gi
+    requests:
+      cpu: 250m
+      memory: 1Gi
+
+program-model:
+  enabled: true
+  replicaCount: 1
+  resources:
+    limits:
+      cpu: 4000m
+      memory: 4Gi
+    requests:
+      cpu: 250m
+      memory: 512Mi
+
+build-bot:
+  enabled: true
+  replicaCount: 3
+  resources:
+    limits:
+      cpu: 4000m
+      memory: 4Gi
+    requests:
+      cpu: 250m
+      memory: 512Mi
+
+fuzzer-bot:
+  enabled: true
+  replicaCount: 4
+  resources:
+    limits:
+      cpu: 16000m
+      memory: 32Gi
+    requests:
+      cpu: 500m
+      memory: 512Mi
+
+merger-bot:
+  enabled: true
+  resources:
+    limits:
+      cpu: 8000m
+      memory: 16Gi
+    requests:
+      cpu: 250m
+      memory: 1Gi
+
+coverage-bot:
+  enabled: true
+  replicaCount: 2
+  resources:
+    limits:
+      cpu: 4000m
+      memory: 32Gi
+    requests:
+      cpu: 250m
+      memory: 1Gi
+
+tracer-bot:
+  enabled: true
+  replicaCount: 1
+  resources:
+    limits:
+      cpu: 4000m
+      memory: 4Gi
+    requests:
+      cpu: 200m
+      memory: 1Gi
+
+pov-reproducer:
+  enabled: true
+  replicaCount: 2
+  resources:
+    limits:
+      cpu: 4000m
+      memory: 2Gi
+    requests:
+      cpu: 200m
+      memory: 1Gi
+
+seed-gen:
+  enabled: true
+  replicaCount: 5
+  resources:
+    limits:
+      cpu: 2000m
+      memory: 4Gi
+    requests:
+      cpu: 200m
+      memory: 1Gi
+
+patcher:
+  enabled: true
+  replicaCount: 2
+  maxPatchRetries: 10
+  maxContextRetrieverRecursionLimit: 100
+  maxMinutesRunPOVs: 30
+  resources:
+    limits:
+      cpu: 2000m
+      memory: 3Gi
+    requests:
+      cpu: 200m
+      memory: 1Gi

deployment/main.tf

@@ -16,8 +16,8 @@ resource "azapi_resource_action" "ssh_public_key_gen" {
 resource "azapi_resource" "ssh_public_key" {
   type      = "Microsoft.Compute/sshPublicKeys@2022-11-01"
   name      = random_pet.ssh_key_name.id
-  location  = azurerm_resource_group.rg.location
-  parent_id = azurerm_resource_group.rg.id
+  location  = local.resource_group_location
+  parent_id = local.resource_group_id
 }
 
 output "key_data" {
@@ -30,16 +30,31 @@ resource "random_pet" "rg_name" {
   prefix = var.resource_group_name_prefix
 }
 
+# Data source to reference existing resource group if provided
+data "azurerm_resource_group" "existing" {
+  count = var.resource_group_name != null ? 1 : 0
+  name  = var.resource_group_name
+}
+
+# Create new resource group only if resource_group_name is not provided
 resource "azurerm_resource_group" "rg" {
+  count = var.resource_group_name == null ? 1 : 0
   #ts:skip=AC_AZURE_0389 Locks not required
   location = var.resource_group_location
   name     = random_pet.rg_name.id
 }
 
+# Local values to reference the appropriate resource group
+locals {
+  resource_group_name     = var.resource_group_name != null ? var.resource_group_name : azurerm_resource_group.rg[0].name
+  resource_group_location = var.resource_group_name != null ? data.azurerm_resource_group.existing[0].location : azurerm_resource_group.rg[0].location
+  resource_group_id       = var.resource_group_name != null ? data.azurerm_resource_group.existing[0].id : azurerm_resource_group.rg[0].id
+}
+
 # Optional: Adds resource lock to prevent deletion of the RG. Requires additional configuration
 #resource "azurerm_management_lock" "resource-group-level" {
 #  name       = "resource-group-cannotdelete-lock"
-#  scope      = azurerm_resource_group.rg.id
+#  scope      = local.resource_group_id
 #  lock_level = "CanNotDelete"
 #  notes      = "This Resource Group is set to CanNotDelete to prevent accidental deletion."
 #}
@@ -54,9 +69,9 @@ resource "random_pet" "azurerm_kubernetes_cluster_dns_prefix" {
 }
 
 resource "azurerm_kubernetes_cluster" "primary" {
-  location            = azurerm_resource_group.rg.location
+  location            = local.resource_group_location
   name                = random_pet.azurerm_kubernetes_cluster_name.id
-  resource_group_name = azurerm_resource_group.rg.name
+  resource_group_name = local.resource_group_name
   dns_prefix          = random_pet.azurerm_kubernetes_cluster_dns_prefix.id
   sku_tier            = "Standard"
   # Disable automatic upgrades to prevent disruption
@@ -110,9 +125,9 @@ resource "azurerm_kubernetes_cluster_node_pool" "user" {
 
 #Monitoring Log Anayltics
 resource "azurerm_log_analytics_workspace" "aks_logs" {
-  name                = "${random_pet.rg_name.id}-logs"
-  location            = azurerm_resource_group.rg.location
-  resource_group_name = azurerm_resource_group.rg.name
+  name                = "${local.resource_group_name}-logs"
+  location            = local.resource_group_location
+  resource_group_name = local.resource_group_name
   sku                 = "PerGB2018"
   retention_in_days   = 30
 }

deployment/outputs.tf

@@ -1,5 +1,5 @@
 output "resource_group_name" {
-  value = azurerm_resource_group.rg.name
+  value = local.resource_group_name
 }
 
 output "kubernetes_cluster_name" {

deployment/providers.tf

@@ -31,4 +31,5 @@ provider "azurerm" {
   tenant_id       = var.ARM_TENANT_ID
   client_id       = var.ARM_CLIENT_ID
   client_secret   = var.ARM_CLIENT_SECRET
+  resource_provider_registrations = "none"
 }

deployment/variables.tf

@@ -10,6 +10,12 @@ variable "resource_group_name_prefix" {
   description = "Prefix of the resource group name that's combined with a random ID so name is unique in your Azure subscription."
 }
 
+variable "resource_group_name" {
+  type        = string
+  default     = null
+  description = "Name of the existing resource group to use. If not set, a new resource group will be created with a random name using resource_group_name_prefix."
+}
+
 variable "sys_node_count" {
   type        = number
   description = "The initial quantity of nodes for the node pool."

dev/docker-compose/README.md

@@ -28,5 +28,5 @@ cd dev/docker-compose && docker-compose down
 
 - This setup is **not recommended for production** - use the Kubernetes deployment instead
 - Primarily useful for developers who need to run individual components for testing
-- The main deployment method is now Kubernetes via `make deploy-local` from the repository root
-- See the main [README](../../README.md) for the recommended setup instructions
+- The main deployment method is now Kubernetes via `make deploy` from the repository root
+- See the main [README](../../README.md) for the recommended setup instructions

guides/AKS_DEPLOYMENT.md

@@ -1,11 +1,6 @@
 # Production AKS Deployment Guide
 
-> **⚠️ Notice:**
-> The following production deployment instructions have **not been fully tested**.
-> Please proceed with caution and verify each step in your environment.
-> If you encounter issues, consult the script comments and configuration files for troubleshooting.
-
-Full production deployment of the **Buttercup CRS** on Azure Kubernetes Service with proper networking, monitoring, and scaling for the DARPA AIxCC competition.
+Full production deployment of the **Buttercup CRS** on Azure Kubernetes Service with proper networking, monitoring.
 
 ## Quick Setup (Recommended)
 
@@ -16,66 +11,28 @@ make setup-azure
 ```
 
 This script will check prerequisites, help create service principals, configure the environment, and validate your setup.
-
-## Manual Setup
-
-If you prefer to set up manually, follow these steps:
-
-### Prerequisites
-
-- Azure CLI installed and configured
-- Terraform installed
-- Active Azure subscription
-- Access to competition Tailscale tailnet
-
-### Azure Setup
-
-1. **Login to Azure:**
-
-```bash
-az login --tenant <your-tenant-here>
-```
-
-2. **Create Service Principal:**
-
-```bash
-# Get your subscription ID
-az account show --query "{SubscriptionID:id}" --output table
-
-# Create service principal (replace with your subscription ID)
-az ad sp create-for-rbac --name "ButtercupCRS" --role Contributor --scopes /subscriptions/<YOUR-SUBSCRIPTION-ID>
-```
+You probably need at least `Contributor` role in your Azure subscription to deploy Buttercup.
 
 ### Production Configuration
 
-1. **Configure environment file:**
+The setup-azure make target will help you configure a simple production-ready
+cluster, however each deployment is different based on requirements: cost,
+number of tasks that you want to solve at the same time, size of projects, etc.
 
-```bash
-cp deployment/env.template deployment/env
-```
+For more fine-grained adjustments that are not configurable during the automated setup script, modify `deployment/env` and `deployment/values-upstream-aks.template`. In particular you may want to change the number of nodes in the k8s cluster, the type of nodes, the number of pods for different kind of services, the storage, etc.
 
-2. **Update `deployment/env` for production:**
-
-Look at the comments in the `deployment/env.template` for how to set variables.
-In particular, set `TF_VAR_*` variables, and `TAILSCALE_*` if used.
 
 ## Deploy to AKS
 
 **Deploy the cluster and services:**
 
 ```bash
-make deploy-azure
-```
-
-**Alternative manual command:**
-
-```bash
-cd deployment && make up
+make deploy
 ```
 
 ## Scaling and Management
 
-- **Scale nodes:** Update `TF_VAR_usr_node_count` in your env file and run `make up`
+- **Scale nodes:** Update `TF_VAR_usr_node_count` in your env file and run `make deploy` again
 - **View logs:** `kubectl logs -n crs <pod-name>`
 - **Monitor resources:** `kubectl top pods -A`
 

scripts/common.sh

@@ -72,7 +72,7 @@ install_docker() {
         print_success "Docker is already installed"
     fi
     
-    # Install buildx plugin (required for deploy-local target)
+    # Install buildx plugin (required for deploy target)
     print_status "Installing Docker buildx plugin..."
     sudo apt install -y docker-buildx-plugin
     print_success "Docker buildx plugin installed"
@@ -222,7 +222,7 @@ setup_config_file() {
     else
         print_warning "Configuration file already exists"
         if [ "$overwrite_existing" = "true" ]; then
-            read -p "Do you want to overwrite it? (y/n): " -n 1 -r
+            read -p "Do you want to overwrite it? (y/N): " -n 1 -r
             echo
             if [[ $REPLY =~ ^[Yy]$ ]]; then
                 cp deployment/env.template deployment/env

scripts/setup-azure.sh

@@ -23,44 +23,109 @@ setup_service_principal() {
     local subscription_name=$(az account show --query name -o tsv)
     
     print_status "Current subscription: $subscription_name ($subscription_id)"
-    
-    read -p "Do you want to create a new service principal? (y/n): " -n 1 -r
-    echo
-    if [[ $REPLY =~ ^[Yy]$ ]]; then
-        local sp_name="ButtercupCRS-$(date +%Y%m%d-%H%M%S)"
-        print_status "Creating service principal: $sp_name"
-        
-        local sp_output=$(az ad sp create-for-rbac --name "$sp_name" --role Contributor --scopes "/subscriptions/$subscription_id" --output json)
-        
-        local app_id=$(echo "$sp_output" | jq -r '.appId')
-        local password=$(echo "$sp_output" | jq -r '.password')
-        local tenant_id=$(echo "$sp_output" | jq -r '.tenant')
-        
-        print_success "Service principal created successfully"
-        echo
-        print_status "Service Principal Details:"
-        echo "  Name: $sp_name"
-        echo "  App ID: $app_id"
-        echo "  Tenant ID: $tenant_id"
-        echo "  Password: $password"
-        echo
-        print_warning "Save these credentials securely!"
-        echo
-        
-        # Export environment variables
-        export TF_ARM_TENANT_ID="$tenant_id"
-        export TF_ARM_CLIENT_ID="$app_id"
-        export TF_ARM_CLIENT_SECRET="$password"
-        export TF_ARM_SUBSCRIPTION_ID="$subscription_id"
-        
-        print_status "Environment variables exported for current session"
+
+    # Source deployment/env to get existing RESOURCE_GROUP_NAME if set
+    if [ -f "deployment/env" ]; then
+        source deployment/env
+    fi
+
+    RESOURCE_GROUP_NAME="${TF_VAR_resource_group_name:-}"
+    if [ -z "$RESOURCE_GROUP_NAME" ] || [ "$RESOURCE_GROUP_NAME" = "<your-resource-group-name>" ]; then
+        print_status "Please specify the resource group where all Azure resources will be deployed:"
+        read -p "Enter resource group name (e.g., buttercup-crs-rg): " RESOURCE_GROUP_NAME
+
+        if [ -z "$RESOURCE_GROUP_NAME" ]; then
+            print_error "Resource group name is required"
+            exit 1
+        fi
     else
-        print_status "Using existing service principal"
-        print_status "Please set the following environment variables:"
-        echo "  export TF_ARM_TENANT_ID=\"<your-tenant-id>\""
-        echo "  export TF_ARM_CLIENT_ID=\"<your-client-id>\""
-        echo "  export TF_ARM_CLIENT_SECRET=\"<your-client-secret>\""
-        echo "  export TF_ARM_SUBSCRIPTION_ID=\"<your-subscription-id>\""
+        print_success "Using RESOURCE_GROUP_NAME from deployment/env: $RESOURCE_GROUP_NAME"
+    fi
+    RESOURCE_GROUP_LOCATION="${TF_VAR_resource_group_location:-}"
+    if [ -z "$RESOURCE_GROUP_LOCATION" ] || [ "$RESOURCE_GROUP_LOCATION" = "<your-resource-group-location>" ]; then
+        read -p "Enter resource group location (default: eastus): " RESOURCE_GROUP_LOCATION
+        if [ -z "$RESOURCE_GROUP_LOCATION" ]; then
+            RESOURCE_GROUP_LOCATION="eastus"
+            print_status "No value provided. Using default: $RESOURCE_GROUP_LOCATION"
+        fi
+    else
+        print_success "Using RESOURCE_GROUP_LOCATION from deployment/env: $RESOURCE_GROUP_LOCATION"
+    fi
+
+    # Check if resource group exists, if not ask to create it
+    if ! az group show --name "$RESOURCE_GROUP_NAME" >/dev/null 2>&1; then
+        print_status "Resource group '$RESOURCE_GROUP_NAME' does not exist"
+        read -p "Do you want to create it? (Y/n): " -n 1 -r
+        echo
+        if [[ $REPLY =~ ^[Nn]$ ]]; then
+            print_error "Resource group must exist before proceeding"
+            exit 1
+        fi
+
+        print_status "Creating resource group: $RESOURCE_GROUP_NAME"
+        az group create --name "$RESOURCE_GROUP_NAME" --location "$RESOURCE_GROUP_LOCATION"
+        print_success "Resource group created successfully"
+    else
+        print_success "Using existing resource group: $RESOURCE_GROUP_NAME"
+    fi
+
+    # Write RESOURCE_GROUP_NAME to deployment/env (uncomment or add line)
+    if grep -q '^[# ]*export TF_VAR_resource_group_name=' deployment/env; then
+        portable_sed "s|^[# ]*export TF_VAR_resource_group_name=.*|export TF_VAR_resource_group_name=\"$RESOURCE_GROUP_NAME\"|" deployment/env
+    else
+        echo "export TF_VAR_resource_group_name=\"$RESOURCE_GROUP_NAME\"" >> deployment/env
+    fi
+
+    # Write RESOURCE_GROUP_LOCATION to deployment/env (uncomment or add line)
+    if grep -q '^[# ]*export TF_VAR_resource_group_location=' deployment/env; then
+        portable_sed "s|^[# ]*export TF_VAR_resource_group_location=.*|export TF_VAR_resource_group_location=\"$RESOURCE_GROUP_LOCATION\"|" deployment/env
+    else
+        echo "export TF_VAR_resource_group_location=\"$RESOURCE_GROUP_LOCATION\"" >> deployment/env
+    fi
+
+    # Only prompt to create a new service principal if TF_VAR_ARM_CLIENT_ID and TF_VAR_ARM_CLIENT_SECRET are not set
+    if [ -z "$TF_VAR_ARM_CLIENT_ID" ] || [ "$TF_VAR_ARM_CLIENT_ID" = "<your-client-id>" ] || [ -z "$TF_VAR_ARM_CLIENT_SECRET" ] || [ "$TF_VAR_ARM_CLIENT_SECRET" = "<your-client-secret>" ]; then
+        read -p "Do you want to create a new service principal? (y/N): " -n 1 -r
+        echo
+        if [[ $REPLY =~ ^[Yy]$ ]]; then
+            local sp_name="ButtercupCRS-$(date +%Y%m%d-%H%M%S)"
+            print_status "Creating service principal: $sp_name"
+
+            local sp_output=$(az ad sp create-for-rbac --name "$sp_name" --role Contributor --scopes "/subscriptions/$subscription_id/resourceGroups/$RESOURCE_GROUP_NAME" --output json)
+
+            local app_id=$(echo "$sp_output" | jq -r '.appId')
+            local password=$(echo "$sp_output" | jq -r '.password')
+            local tenant_id=$(echo "$sp_output" | jq -r '.tenant')
+
+            print_success "Service principal created successfully"
+            echo
+            print_status "Service Principal Details:"
+            echo "  Name: $sp_name"
+            echo "  App ID: $app_id"
+            echo "  Tenant ID: $tenant_id"
+            echo "  Password: $password"
+            echo
+            print_warning "Save these credentials securely!"
+            echo
+
+            # Write environment variables to deployment/env (uncomment or add line)
+            portable_sed "s|^[# ]*export TF_VAR_ARM_TENANT_ID=.*|export TF_VAR_ARM_TENANT_ID=\"$tenant_id\"|" deployment/env || echo "export TF_VAR_ARM_TENANT_ID=\"$tenant_id\"" >> deployment/env
+            portable_sed "s|^[# ]*export TF_VAR_ARM_CLIENT_ID=.*|export TF_VAR_ARM_CLIENT_ID=\"$app_id\"|" deployment/env || echo "export TF_VAR_ARM_CLIENT_ID=\"$app_id\"" >> deployment/env
+            portable_sed "s|^[# ]*export TF_VAR_ARM_CLIENT_SECRET=.*|export TF_VAR_ARM_CLIENT_SECRET=\"$password\"|" deployment/env || echo "export TF_VAR_ARM_CLIENT_SECRET=\"$password\"" >> deployment/env
+            portable_sed "s|^[# ]*export TF_VAR_ARM_SUBSCRIPTION_ID=.*|export TF_VAR_ARM_SUBSCRIPTION_ID=\"$subscription_id\"|" deployment/env || echo "export TF_VAR_ARM_SUBSCRIPTION_ID=\"$subscription_id\"" >> deployment/env
+
+            print_status "Service principal credentials written to deployment/env"
+        else
+            print_status "Using existing service principal"
+            print_status "Please set the following environment variables:"
+            echo "  export TF_VAR_ARM_TENANT_ID=\"<your-tenant-id>\""
+            echo "  export TF_VAR_ARM_CLIENT_ID=\"<your-client-id>\""
+            echo "  export TF_VAR_ARM_CLIENT_SECRET=\"<your-client-secret>\""
+            echo "  export TF_VAR_ARM_SUBSCRIPTION_ID=\"<your-subscription-id>\""
+            exit 1
+        fi
+    else
+        print_success "Using existing service principal credentials from deployment/env"
     fi
 }
 
@@ -70,6 +135,11 @@ setup_config() {
     
     setup_config_file "true"
     
+    # Set the Kubernetes values template for AKS deployment
+    print_status "Setting Kubernetes values template for AKS deployment..."
+    portable_sed "s|.*export BUTTERCUP_K8S_VALUES_TEMPLATE=.*|export BUTTERCUP_K8S_VALUES_TEMPLATE=\"k8s/values-upstream-aks.template\"|" deployment/env
+    print_success "Kubernetes values template set to: k8s/values-upstream-aks.template"
+
     # Configure required API keys
     configure_local_api_keys
 
@@ -100,20 +170,23 @@ validate_config() {
 setup_remote_state() {
     print_status "Setting up Terraform remote state..."
     
-    read -p "Do you want to configure remote state storage? (y/n): " -n 1 -r
+    read -p "Do you want to configure remote state storage? (y/N): " -n 1 -r
     echo
     if [[ $REPLY =~ ^[Yy]$ ]]; then
-        print_status "This will create Azure resources for remote state storage"
+        print_status "This will create Azure resources for remote state storage in resource group: $RESOURCE_GROUP_NAME"
         
-        # Get resource group name
-        read -p "Enter resource group name for state storage (e.g., buttercup-tfstate-rg): " state_rg
-        read -p "Enter storage account name (e.g., buttercuptfstate): " storage_account
-        
-        # Create resource group
-        print_status "Creating resource group: $state_rg"
-        az group create --name "$state_rg" --location eastus
-        
-        # Create storage account
+        # Use the same resource group as specified in setup_service_principal
+        local state_rg="$RESOURCE_GROUP_NAME"
+        # Generate a short random suffix (4 alphanumeric chars)
+        local rand_suffix=$(LC_CTYPE=C tr -dc 'a-z0-9' </dev/urandom | head -c 4)
+        local default_storage_account="buttercuptf${rand_suffix}"
+        read -p "Enter storage account name (default: $default_storage_account): " storage_account
+        if [[ -z "$storage_account" ]]; then
+            storage_account="$default_storage_account"
+            print_status "No storage account name provided. Using default: $storage_account"
+        fi
+
+        # Create storage account in the existing resource group
         print_status "Creating storage account: $storage_account"
         az storage account create --resource-group "$state_rg" --name "$storage_account" --sku Standard_LRS --encryption-services blob
         
@@ -140,19 +213,72 @@ EOF
     fi
 }
 
+setup_aks_resources() {
+    # Load existing value from env file if present
+    if [[ -f "deployment/env" ]]; then
+        source "deployment/env"
+    fi
+
+    # Set cluster type to aks
+    portable_sed "s|^[# ]*export CLUSTER_TYPE=.*|export CLUSTER_TYPE=\"aks\"|" deployment/env
+
+    # Set VM size
+    read -p "Enter the VM size for the AKS cluster (default: Standard_L8s): " vm_size
+    if [[ -z "$vm_size" ]]; then
+        vm_size="Standard_L8s"
+        print_status "No value provided. Using default: $vm_size"
+    fi
+    if grep -q '^[# ]*export TF_VAR_vm_size=' deployment/env; then
+        portable_sed "s|^[# ]*export TF_VAR_vm_size=.*|export TF_VAR_vm_size=\"$vm_size\"|" deployment/env
+    else
+        echo "export TF_VAR_vm_size=\"$vm_size\"" >> deployment/env
+    fi
+    print_success "Set TF_VAR_vm_size to $vm_size in deployment/env"
+
+    # Register resource providers
+    az provider register --namespace Microsoft.ContainerService
+    az provider register --namespace Microsoft.Storage
+    az provider register --namespace Microsoft.Network
+    az provider register --namespace Microsoft.Compute
+
+    print_warning "Please note that number of nodes, pods, storage sizes, and other resources are correlated, so if you change one, you may need to change the others. Review the Kubernetes values.yaml template (e.g. k8s/values-upstream-aks.template) for more details."
+
+    # Set TF_VAR_usr_node_count
+    current_value="${TF_VAR_usr_node_count:-}"
+    if [[ -n "$current_value" ]]; then
+        print_status "Current TF_VAR_usr_node_count is set to: $current_value"
+        read -p "Do you want to modify the number of user nodes? (y/N): " -n 1 -r
+        echo
+        if [[ ! $REPLY =~ ^[Yy]$ ]]; then
+            print_status "Keeping existing TF_VAR_usr_node_count: $current_value"
+            return
+        fi
+    fi
+
+    # Prompt for number of user nodes, default to 3
+    read -p "Enter the number of user nodes for the AKS cluster (default: 3): " usr_node_count
+    if [[ -z "$usr_node_count" ]]; then
+        usr_node_count=3
+        print_status "No value provided. Using default: $usr_node_count"
+    fi
+
+    # Update or add TF_VAR_usr_node_count in the env file
+    portable_sed "s|^[# ]*export TF_VAR_usr_node_count=.*|export TF_VAR_usr_node_count=\"$usr_node_count\"|" deployment/env
+    print_success "Set TF_VAR_usr_node_count to $usr_node_count in deployment/env"
+}
+
 # Function to provide deployment instructions
 deployment_instructions() {
     print_success "Production setup completed!"
     echo
     print_status "Next steps for deployment:"
-    echo "  1. Run: make deploy-azure"
+    echo "  1. Run: make deploy"
     echo "  2. Monitor deployment: kubectl get pods -A"
     echo "  3. Get cluster credentials: az aks get-credentials --name <cluster-name> --resource-group <rg-name>"
-    echo "  4. Access via Tailscale: kubectl get -n crs-webservice ingress"
     echo
     print_status "Useful commands:"
     echo "  - View logs: kubectl logs -n crs <pod-name>"
-    echo "  - Scale nodes: Update TF_VAR_usr_node_count and run make deploy-azure"
+    echo "  - Scale nodes: Update TF_VAR_usr_node_count and run make deploy"
     echo "  - Cleanup: make clean"
 }
 
@@ -162,9 +288,10 @@ main() {
     
     check_azure_cli
     check_terraform
-    setup_service_principal
     setup_config
+    setup_service_principal
     setup_remote_state
+    setup_aks_resources
     
     print_status "Configuration validation..."
     if validate_config; then

scripts/setup-local.sh

@@ -39,7 +39,7 @@ verify_setup() {
     if make validate >/dev/null 2>&1; then
         print_success "Setup verification completed successfully!"
         print_status "Next steps:"
-        echo "  1. Run: make deploy-local"
+        echo "  1. Run: make deploy"
         echo "  2. Test with: make send-libpng-task"
     else
         print_error "Setup verification failed. Run 'make validate' for details."