deployment: support Tailscale (#235)

2026-06-21 14:11:39 +00:00 · 2025-03-26 08:49:25 +01:00
parent cc9e863649
commit f4c4f75a2c
39 changed files with 5683 additions and 436 deletions
@@ -10,7 +10,9 @@ errored.tfstate

 ## Generated Kubernetes Resources
 .k8s
-deployment/k8s/values-override.yaml
+deployment/k8s/values-*.yaml
+deployment/k8s/base/tailscale-coredns/coredns-custom.yaml
+deployment/k8s/base/tailscale-operator/operator.yaml

 ## Python
 __pycache__/
@@ -45,4 +47,4 @@ pdm.lock
 common/common/datastructures/fuzzer_msg*

 crs_scratch/*
-tasks_storage/
+tasks_storage/*
@@ -2,61 +2,62 @@
 # Manual edits may be lost in future updates.

 provider "registry.terraform.io/azure/azapi" {
-  version     = "2.0.1"
-  constraints = "2.0.1"
+  version     = "2.2.0"
+  constraints = "2.2.0"
  hashes = [
-    "h1:NybiPNJ4MsKsTtRUnxV5vrhkbPkK+puqgfa0j0Q5nlU=",
-    "zh:3df16ed604be5f4ccd5d52a02c2681d8eb2f5a4462625c983cb17c20cdf0bfb2",
-    "zh:4efd9961ea52990e21385086f0b3324edfb534ea6a8f0f6ba146a74bfb56aa63",
-    "zh:5561418efc9744c9873855a146226608778e29b4c0c3b3872634ef2da2d86593",
-    "zh:7ebcb4c6ca71c87850df67d4e5f79ce4a036d4131b8c11ae0b9b8787353843b8",
-    "zh:81a9259cb1e45507e9431794fbd354dd4d8b78c6a9508b0bfa108b00e6ad23cb",
-    "zh:8c1836fa186272347f97c7a3884556979618d1b93721e8a24203d90ff4efbd40",
-    "zh:a72bdd43a11a383525764720d24cb78ec5d9f1167f129d05448108fef1ba7af3",
-    "zh:ade9d17c6b8717e7b04af5a9d1a948d047ac4dcf6affb2485afa3ad0a2eaee15",
-    "zh:b3c5bfcab98251cb0c157dbe78dc6d0864c9bf364d316003c84c1e624a3c3524",
-    "zh:c33b872a2473a9b052add89e4557d361b0ebaa42865e99b95465050d2c858d43",
-    "zh:efe425f8ecd4d79448214c93ef10881b3b74cf2d9b5211d76f05aced22621eb4",
-    "zh:ff704c5e73e832507367d9d962b6b53c0ca3c724689f0974feffd5339c3db18a",
+    "h1:yckm1jqUMUGSeS57a3uR0gG/V7scwvjpkRVXnQIUAo4=",
+    "zh:062be5d8272cac297a88c2057449f449ea6906c4121ba3dfdeb5cecb3ff91178",
+    "zh:1fd9abec3ffcbf8d0244408334e9bfc8f49ada50978cd73ee0ed5f8560987267",
+    "zh:48e84b0302af99d7e7f4248a724088fb1c34aeee78c9ca63ec5a9464ec5054a0",
+    "zh:4e7302883fd9dd83bfbbcd72ebd55f83d8b16ccc6d12d1573d578058e604d5cf",
+    "zh:5b6e181e32cbf62f5d2ce34f9d6d9ffe17192e24943450bbe335e1baf0494e66",
+    "zh:62d525d426c6d5f10109ab04a9abc231b204ea413238f5690f69b420a8b8583a",
+    "zh:90aab23497ec9c7af44ad9ea1a1d6063dc3331334915e1c549527a73c2c6948d",
+    "zh:91ecf30a01df5e832191e0c55c87f8403a1f584796fd70f9c9c913d35c2e2a37",
+    "zh:bc3a5db5e4b9695a69dff47cf1e7184eaf5564d3dc50f231cbcbf535dd140d19",
+    "zh:cb566bec2676511bf4722e24d0dfc9bf58aff78af38b8e0864970f20d263118f",
+    "zh:d4fa0c1462b389cee313e1c152e00f5dfc175a1be3615d3b23b526a8581e39a5",
+    "zh:f8136b0f41045a1e5a6dedc6b6fb055faee3d825f84a3192312e3ac5d057ff72",
  ]
 }

 provider "registry.terraform.io/hashicorp/azurerm" {
-  version     = "4.7.0"
-  constraints = "4.7.0"
+  version     = "4.17.0"
+  constraints = "4.17.0"
  hashes = [
-    "h1:gCTgXiVJIPfPmeMtWHEkzcuzmJzBBjCo5FF0tIShiPc=",
-    "zh:220994020593f3bc3e8d73bfaa98d7f0abccea2f51f76029bb9f6dc2a5bb3436",
-    "zh:2cdfb75f3b375410da63deb17b7dadec1e3ddfac7e2f5168624a4fc51dd3e177",
-    "zh:37ee931a839228c4a74fa96cd053b8e0c3de4637cc85fba963fccaf3d67120a4",
-    "zh:4b12d06e4b782af96df2cfd3445c330c8553f5ac9cf73b2ab045004cf02edbe5",
-    "zh:7c80ee772836ddf66087a688d50ce404eeaa4a6e72a5f8f5f3f3232068a0c0ca",
-    "zh:826acb1bf9b335ca6992796bfd3898b15d0f99a888f02d9c97b7771c31765456",
-    "zh:83dea535e067547215595141ffd954aab36dd67f646a53a12dc811557fb38acd",
-    "zh:8d5d08e4367f679b877eec33a9e0062d66de0aedd557412ab88b696bdf08c7ef",
-    "zh:c2c4efbc861248d89620db3e36370ce3053d886930db38d87c6e24a73ca5017e",
-    "zh:d7e568f956c6e2a764b0d8f465e5c971cc6edab58bb8232f63ea423c459ffc85",
+    "h1:gpFgaBSkRTxhavgPAuqQcElHJqmRJ1RpQGr1K0dvVW8=",
+    "zh:163b81a3bf29c8f161a1c100a48164b1bd1af434cd564b44596cb71a6c33f03d",
+    "zh:2996b107d3c05a9db14458b32b6f22f8cde0adb96263196d82d3dc302907a257",
+    "zh:361abd84b6e73016ebebb9ef9cd14c237d8b1e4500ea75f73243ff0534e5e4fb",
+    "zh:4872445dcb109fe8bbaba439d3dffaaef849a92645df3f8a854d3a40ac962f68",
+    "zh:61974eb7379acadbceb47b001ae1de2cdefe8cf078a15fff3a6fcc753cd24273",
+    "zh:75c60ca6e7851fe1d52fe9f5a0ae3d219e300ee5aa63bc8f807e3e0cab569ff0",
+    "zh:7c79305cff7849e6c5d9d60fe570510f95fb2e2bd5ae801da0281702f21dd779",
+    "zh:964b7da03f2dc55583cda3c277fef3511824b183a3a88344ae4ff9823af79109",
+    "zh:cad1593d364eb22b68578a1da4fd4d84749dc81f20e6591b27c6cb1eed9d2072",
+    "zh:db1a2ca17aae78813e8e0676bb9ef941e1a1e32d9fc6e1b239c24661605a8425",
+    "zh:e3a65d2f6f5a63cd1beeeb60a23e7e6b7328ebbd46ffe994792aaac6738186c3",
    "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c",
-    "zh:f866502c6a996e5c2cf42b262a5e5c35e13f3bcb40f364f90c9f4e8e64db733d",
  ]
 }

-provider "registry.terraform.io/hashicorp/kubernetes" {
-  version = "2.35.1"
+provider "registry.terraform.io/hashicorp/helm" {
+  version     = "2.17.0"
+  constraints = "2.17.0"
  hashes = [
-    "h1:zgXeWvp4//Ry+4glwNrLMpPFOU8QBQlARNmR9WCNe9o=",
-    "zh:12212ca5ae47823ce14bfafb909eeb6861faf1e2435fb2fc4a8b334b3544b5f5",
-    "zh:3f49b3d77182df06b225ab266667de69681c2e75d296867eb2cf06a8f8db768c",
-    "zh:40832494d19f8a2b3cd0c18b80294d0b23ef6b82f6f6897b5fe00248a9997460",
-    "zh:739a5ddea61a77925ee7006a29c8717377a2e9d0a79a0bbd98738d92eec12c0d",
-    "zh:a02b472021753627c5c39447a56d125a32214c29ff9108fc499f2dcdf4f1cc4f",
-    "zh:b78865b3867065aa266d6758c9601a2756741478f5735a838c20d633d65e085b",
-    "zh:d362e87464683f5632790e66920ea803adb54c2bc0cb24b6fd9a314d2b1efffd",
-    "zh:d98206fe88c2c9a52b8d2d0cb2c877c812a4a51d19f9d8428e63cbd5fd8a304d",
-    "zh:dfa320946b1ce3f3615c42b3447a28dc9f604c06d8b9a6fe289855ab2ade4d11",
+    "h1:K5FEjxvDnxb1JF1kG1xr8J3pNGxoaR3Z0IBG9Csm/Is=",
+    "zh:06fb4e9932f0afc1904d2279e6e99353c2ddac0d765305ce90519af410706bd4",
+    "zh:104eccfc781fc868da3c7fec4385ad14ed183eb985c96331a1a937ac79c2d1a7",
+    "zh:129345c82359837bb3f0070ce4891ec232697052f7d5ccf61d43d818912cf5f3",
+    "zh:3956187ec239f4045975b35e8c30741f701aa494c386aaa04ebabffe7749f81c",
+    "zh:66a9686d92a6b3ec43de3ca3fde60ef3d89fb76259ed3313ca4eb9bb8c13b7dd",
+    "zh:88644260090aa621e7e8083585c468c8dd5e09a3c01a432fb05da5c4623af940",
+    "zh:a248f650d174a883b32c5b94f9e725f4057e623b00f171936dcdcc840fad0b3e",
+    "zh:aa498c1f1ab93be5c8fbf6d48af51dc6ef0f10b2ea88d67bcb9f02d1d80d3930",
+    "zh:bf01e0f2ec2468c53596e027d376532a2d30feb72b0b5b810334d043109ae32f",
+    "zh:c46fa84cc8388e5ca87eb575a534ebcf68819c5a5724142998b487cb11246654",
+    "zh:d0c0f15ffc115c0965cbfe5c81f18c2e114113e7a1e6829f6bfd879ce5744fbb",
    "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c",
-    "zh:fc1debd2e695b5222d2ccc8b24dab65baba4ee2418ecce944e64d42e79474cb5",
-    "zh:fdaf960443720a238c09e519aeb30faf74f027ac5d1e0a309c3b326888e031d7",
  ]
 }

@@ -64,7 +65,7 @@ provider "registry.terraform.io/hashicorp/random" {
  version     = "3.6.3"
  constraints = "3.6.3"
  hashes = [
-    "h1:zG9uFP8l9u+yGZZvi5Te7PV62j50azpgwPunq2vTm1E=",
+    "h1:Fnaec9vA8sZ8BXVlN3Xn9Jz3zghSETIKg7ch8oXhxno=",
    "zh:04ceb65210251339f07cd4611885d242cd4d0c7306e86dda9785396807c00451",
    "zh:448f56199f3e99ff75d5c0afacae867ee795e4dfda6cb5f8e3b2a72ec3583dd8",
    "zh:4b4c11ccfba7319e901df2dac836b1ae8f12185e37249e8d870ee10bb87a13fe",
@@ -84,7 +85,7 @@ provider "registry.terraform.io/hashicorp/time" {
  version     = "0.12.1"
  constraints = "0.12.1"
  hashes = [
-    "h1:JzYsPugN8Fb7C4NlfLoFu7BBPuRVT2/fCOdCaxshveI=",
+    "h1:6BhxSYBJdBBKyuqatOGkuPKVenfx6UmLdiI13Pb3his=",
    "zh:090023137df8effe8804e81c65f636dadf8f9d35b79c3afff282d39367ba44b2",
    "zh:26f1e458358ba55f6558613f1427dcfa6ae2be5119b722d0b3adb27cd001efea",
    "zh:272ccc73a03384b72b964918c7afeb22c2e6be22460d92b150aaf28f29a7d511",
@@ -0,0 +1,46 @@
+# CHANGELOG.md
+
+## 2025-03-07
+
+- Adds tailscale configuration for ingress
+- Removes nginx-ingress component
+- Removes cert-manager / letsencrypt component
+- Removes cluster issuer component
+- Updates to env variables
+- Updates to README.md
+
+## 2025-03-01
+
+- Modified variable names
+- Uppdated example deployment to use echo server from GHCR authenticated registry
+- Implmented `Makefile` to drive deployment and teardown of architecture
+- DNS records created during deploy (up) are removed during destroy (down)
+- Updates to README.md
+
+## 2025-02-20
+
+- Moved kuberenetes resources out of terraform HCL due to inconsistencies and reliability concerns
+- Terraform now only creates the initial AKS infrastructure
+- Introduced kustomize manifests to deploy kubernetes resources into the AKS cluster
+- Iintroduced wrapper script, crs-architecture, to manage the proper deployment of all environments and variables within
+- Leveraging generic template files for kustomize usage of environment variables to simplify end user burden
+
+## 2025-01-19
+
+- Updates to outputs.tf
+- Updates to variables.tf
+- Changes hardcoded credentials environment variables
+- Updates to README.md
+
+## 2025-01-15
+
+- Adds example CRS webservice docker image via deployment.tf
+- Adds private load balancer for CRS webservice via deployment.tf
+- Adds docker registry auth via secrets.tf
+- Updates to providers.tf to supprt `kubernetes` provider
+- Updates to README.md
+
+## 2024-12-03
+
+- Initial commit of example-crs-architecture
+- Added linter rules to root of `example-crs-architecture` repository
@@ -0,0 +1,39 @@
+# Makefile
+
+# Variables
+SCRIPT := crs-architecture.sh
+GENERATED_FILES := k8s/base/tailscale-coredns/coredns-custom.yaml \
+		   k8s/base/tailscale-operator/operator.yaml \
+		   k8s/values-aks.yaml
+
+# Default target when just running `make`
+.PHONY: help
+help:
+	@echo "Usage:"
+	@echo "  make up     # Deploys the AKS cluster and resources"
+	@echo "  make down   # Destroys the AKS cluster and resources"
+	@echo "  make clean  # Removes generated files from templates (optional)"
+
+#check script exists and is executable
+.PHONY: check
+check:
+	@test -f $(SCRIPT) || (echo "Error: $(SCRIPT) not found" && exit 1)
+	@test -x $(SCRIPT) || (echo "Error: $(SCRIPT) not executable" && exit 1)
+
+# up target
+.PHONY: up
+up: check clean
+	@bash $(SCRIPT) up
+
+# down target
+.PHONY: down
+down: check
+	@bash $(SCRIPT) down
+
+down-k8s:
+	@bash $(SCRIPT) down-k8s
+
+# Clean up generated files (optional)
+.PHONY: clean
+clean:
+	@rm -f $(GENERATED_FILES)
@@ -1,12 +1,29 @@
-# Deployment of the Trail of Bits AIxCC Finals CRS
+# Example AKS Cluster

-## Pre-requisites
+## Overview
+
+The following is an example on how to deploy an Azure Kubernetes Service cluster within an Azure subscription.
+
+This configuration deploys an AKS cluster (`primary`) with two node pools:
+
+- The default node pool (`sys`) - contains 2 system mode nodes
+- A User node pool (`usr`) - contains 3 user mode nodes
+
+The resource group name is generated with the `"random_pet"` resource from the `hashicorp/random` provider; and are prefixed with `example`
+
+The VM Size for both pools are using `Standard_D5_v2` in this example. You can change this to suit your needs by editing the `vm_size` values in `main.tf`.
+
+The standard `azure` networking profile is used.
+
+## Prerequisites

 - Azure CLI installed: [az cli install instructions](https://learn.microsoft.com/en-us/cli/azure/install-azure-cli)
 - Terraform installed: [terraform install instructions](https://developer.hashicorp.com/terraform/tutorials/azure-get-started/install-cli)
 - Kubernetes CLI installed: [kubectl install instructions](https://kubernetes.io/docs/tasks/tools/#kubectl)
+- `gettext` package
 - An active Azure subscription.
 - An account in Azure Entra ID.
+- Access credentials to the competition Tailscale tailnet.

 ### Azure

@@ -30,14 +47,15 @@ SubscriptionID                        Tenant

 A service principal account (SPA) is required to automate the creation of resources and objects within your subscription.

-You can create a SPA several ways, the following describes using azure cli.
+- You can create a SPA several ways, the following describes using Azure cli.

 ```bash
 az ad sp create-for-rbac --name "ExampleSPA" --role Contributor --scopes /subscriptions/<YOUR-SUBSCRIPTION-ID>
 ```

-> Replace "ExampleSPA" with the name of the SPA you wish to create. Replace `<YOUR-SUBSCRIPTION-ID>` with your azure subscription ID.
-> If using resource group locks, additional configuration may be neccessary which is out of scope of this example; e.g. adding the role `Microsoft.Authorization/locks/` for write, read and delete to the SPA.
+> Replace "ExampleSPA" with the name of the SPA you wish to create.  
+> Replace `<YOUR-SUBSCRIPTION-ID>` with your Azure subscription ID.  
+> If using resource group locks, additional configuration may be necessary which is out of scope of this example; e.g. adding the role `Microsoft.Authorization/locks/` for write, read and delete to the SPA.

 On successful SPA creation, you will receive output similar to the following:

@@ -50,39 +68,84 @@ On successful SPA creation, you will receive output similar to the following:
 }
 ```

-Make note of these values, they will be used in the AKS deployment as the following environment variables:
+Make note of these values, they will be needed in the AKS deployment as the following environment variables:

 ```bash
-ARM_TENANT_ID="<tenant-value>"
-ARM_CLIENT_ID="<appID-value>"
-ARM_CLIENT_SECRET="<password-value>"
-ARM_SUBSCRIPTION_ID="<YOUR-SUBSCRIPTION-ID>"
+TF_ARM_TENANT_ID="<tenant-value>"
+TF_ARM_CLIENT_ID="<appID-value>"
+TF_ARM_CLIENT_SECRET="<password-value>"
+TF_ARM_SUBSCRIPTION_ID="<YOUR-SUBSCRIPTION-ID>"
 ```

-You can export these as environment variables from the host you're deploying from.
+### Environment Variables
+
+The following environment variables are required to be passed into the terraform and kubernetes configurations:
+
+| Variable Name                | Description                                                                   |
+| ---------------------------- | ----------------------------------------------------------------------------- |
+| `TF_VAR_ARM_SUBSCRIPTION_ID` | Azure subscription ID                                                         |
+| `TF_VAR_ARM_TENANT_ID`       | Azure tenant ID                                                               |
+| `TF_VAR_ARM_CLIENT_ID`       | Azure client ID (service principal account)                                   |
+| `TF_VAR_ARM_CLIENT_SECRET`   | Azure client ID secret                                                        |
+| `CRS_API_HOSTNAME`           | The hostname you want to assign to your API. Exmaple: `teamX-api`             |
+| `TS_CLIENT_ID`               | Tailscale oauth client ID (provided by the Organizers)                        |
+| `TS_CLIENT_SECRET`           | Tailscale oauth client secret (provided by the Organizers)                    |
+| `TS_OP_TAG`                  | Tailscale operator tag (provided by the Organizers)                           |
+| `COMPETITION_API_KEY_ID`     | HTTP basic auth username for the competition API (provided by the Organizers) |
+| `COMPETITION_API_KEY_TOKEN`  | HTTP basic auth password for the competition API (provided by the Organizers) |
+| `CRS_KEY_ID`                 | HTTP basic auth username for the CRS API                                      |
+| `CRS_KEY_TOKEN`              | HTTP basic auth password for the CRS API                                      |
+| `GHCR_AUTH`                  | Base64 encoded credentials for GHCR                                           |
+
+**These variables are stored in `./env` , and must be updated with accurate values.**
+
+### CRS HTTP basic auth
+
+_WIP (Current example is using the `jmalloc/echo-server` image as a PoC)_  
+The crs-webapp image expects the following environment variables to be passed to it for HTTP basic authentication:
+
+- `CRS_KEY_ID` - The CRS's username/ID
+- `CRS_KEY_TOKEN` - The CRS's password
+- `COMPETITION_API_KEY_ID` - The competition APIs username/ID
+- `COMPETITION_API_KEY_TOKEN` - The competition APIs password
+
+These values can be generated with the following python calls:

 ```bash
-export ARM_CLIENT_ID="00000000-0000-0000-0000-000000000000"
-export ARM_CLIENT_SECRET="12345678-0000-0000-0000-000000000000"
-export ARM_TENANT_ID="10000000-0000-0000-0000-000000000000"
-export ARM_SUBSCRIPTION_ID="20000000-0000-0000-0000-000000000000"
+key_id:
+python3 -c 'import uuid; print(str(uuid.uuid4()))'
+
+key_token:
+python3 -c 'import secrets, string; print("".join(secrets.choice(string.ascii_uppercase + string.ascii_lowercase + string.digits) for _ in range(32)))'
 ```

+### GitHub personal access token
+
+_WIP (Current example is using the `jmalloc/echo-server` image as a PoC)_  
+You will need to have a GitHub personal access token (PAT) scoped to at least `read:packages`.
+
+To create the PAT, go to your account, `Settings` > `Developer settings` > `Personal access tokens`, and generate a Token (classic) with the scopes needed for your use case.
+
+For this example, the `read:packages` scope is required.
+
+Once you have your PAT, you will need to base64 encode it for use within `secrets.tf`:
+
+```bash
+echo -n "ghcr_username:ghcr_token" | base64
+```
+
+> replace `ghcr_username` and `ghcr_token` with your GitHub username and your PAT respectively.
+
+Add your base64 encoded credentials to `GHCR_AUTH`
+
 ## Remote Terraform State Storage

-By default the terraform state for the CRS is saved on Azure in the following resource:
-```
-    resource_group_name  = "tfstate-rg"
-    storage_account_name = "tfstateserviceact"
-    container_name       = "tfstate"
-    key                  = "terraform.tfstate"
-```
-
-If you want to change the resources used to save the Terraform state, you may need to create other resources.
+By default, terraform stores its state locally. It is best practice to store terraform state in a remote location.  
+This can help with collaboration, security, recovery and scalability. To do this within Azure, you need to create resources to do so.

 ### Azure CLI

-The following is an example of how to create the resources needed for remote state configuration.
+The following is an example of how to create the resources needed for remote state configuration.  
 These resources will be used in the `backend.tf` configuration file.

 - Create remote state resource group.
@@ -94,13 +157,13 @@ az group create --name example-tfstate-rg --location eastus
 - Create storage account for remote state.

 ```bash
-az storage account create --resource-group example-tfstate-rg --name exampleserviceaccountname --sku Standard_LRS --encryption-services blob
+az storage account create --resource-group example-tfstate-rg --name examplestorageaccountname --sku Standard_LRS --encryption-services blob
 ```

 - Create storage container for remote state

 ```bash
-az storage container create --name tfstate --account-name exampleserviceaccountname --auth-mode login
+az storage container create --name tfstate --account-name examplestorageaccountname --auth-mode login
 ```

 ### backend.tf
@@ -111,25 +174,39 @@ Replace the values for `resource_group_name`, `storage_account_name`, `container
 terraform {
  backend "azurerm" {
    resource_group_name  = "example-tfstate-rg"
-    storage_account_name = "exampleserviceaccountname"
+    storage_account_name = "examplestorageaccountname"
    container_name       = "tfstate"
    key                  = "terraform.tfstate"
  }
 }
 ```

+## Makefile
+
+The deployment of the AKS cluster and its resources are performed by the `Makefile`, which leverages `crs-architecture.sh`. This wrapper utilizes several mechanisms to properly configure both the teraform and kubernetes environments.
+
 ## Deploy

 - Log into your Azure tenant with `az login --tenant aixcc.tech`
- Export the environment variables for your [SPA Configuration](#service-principal-account) if needed.
- Initialize terraform: `terraform init`
- Run plan: `terraform plan` - review output
- Deploy: `terraform apply`
-  - type `yes` when prompted to apply
+- Clone this repository if needed: `git clone git@github.com:aixcc-finals/example-crs-architecture.git /<local_dir>`
+- Make required changes to `backend.tf`
+- Make any wanted changes to `main.tf`, `outputs.tf`, `providers.tf`, and `variables.tf`
+- Update `./env` with accurate values for each variable
+- Run `make up` from within the `example-crs-architecture` directory
+  This will execute a deployment of your cluster via a combination of terraform and kubectl based on your unique values in `./env`

-A handful of outputs will be provided based on `outputs.tf` when the apply completes.
+## Useful Cluster Commands

-You can see the outputs values with `terraform output` or `terraform output <output-name>`.
+- `az aks get-credentials --name <your-cluster-name> --resource-group <your-resource-group>` - retrieves access credentials and updates kubeconfig for the current cluster
+- `kubectl get namespaces` - lists all namespaces
+- `kubectl get -n crs-webservice all` - lists all resources within the crs-webservice namespace
+- `kubectl get -n tailscale all` - lists all resources within the tailscale namespace
+- `kubectl get pods -A` - lists all pods in all namespaces
+- `kubectl config get-contexts` - lists the current contexts in your kubeconfig
+- `kubectl describe deployment -n crs-webservice crs-webapp` - print detailed information about the deployment, crs-webapp
+- `kubectl logs <podName>` - retrieves the stdout and stderr streams from the containers within the specified pod
+- `kubectl get svc -A` - lists status of all services
+- `kubectl get -n crs-webservice ingress` - lists the tailscale ingress address of your API

 ## State

@@ -138,61 +215,16 @@ You can see the outputs values with `terraform output` or `terraform output <out

 ## Destroy

-To teardown your AKS cluster run the following:
+To tear down your AKS cluster run the following:

- `terraform destroy`
- Review the output on what is to be destroyed
- Type `yes` at the prompt
+- Run `make down` from within the `example-crs-architecture` directory

-## Kubernetes interactions
+## Reset / Redeploy CRS

-Save the kube config file and make it available through `KUBECONFIG` env var:
-```shell
-terraform output --raw  kube_config >! kube.config
-export KUBECONFIG=$(pwd)/kube.config
-```
+To reset or redeploy a running CRS cluster:

-```shell
-kubectl get namespaces
-```
+- Run `make down && make up` from within the `example-crs-architecture` directory

-## Access kubernetes cluster nodes / services
-```shell
-kubectl port-forward service/orchestrator -n crs 18000:8000
-```
+## Clean

-Then access `127.0.0.1:18000`.
-
-
-## Troubleshooting
-
-### Error acquiring the state lock
-```
-Error: Error acquiring the state lock
-│
-│ Error message: state blob is already locked
-│ Lock Info:
-│   ID:        7a304f3a-6b83-34a1-6773-b70ec456e6cc
-│   Path:      tfstate/terraform.tfstate
-│   Operation: OperationTypeApply
-│   Who:       ret2libc@macbookpro.lan
-│   Version:   1.10.4
-│   Created:   2025-01-14 12:56:49.352199 +0000 UTC
-│   Info:
-│
-│
-│ Terraform acquires a state lock to protect the state from being written
-│ by multiple users at the same time. Please resolve the issue above and try
-│ again. For most commands, you can disable locking with the "-lock=false"
-│ flag, but this is not recommended.
-╵
-```
-
-First, ensure that someone else is not really running some `terraform` command
-concurrently. If that's not the case, the state file might have not been
-unlocked (e.g. you CTRL-C terraform at some point and it did not unlocked the
-file before exiting), you can try adding the `-lock=false` as specified, or, if
-that does not work, go to the [Azure portal](https://portal.azure.com), open the
-`tfstate-rg` Resource Group, `tfstateserviceact`, then in Data Storage >
-Containers select `tfstate`. Select the `terraform.tfstate` blob and click
-`Break lease.
+Optionally, you can run `make clean` to remove any generated files create from the included templates at runtime. This action is executed during `make up`.
@@ -0,0 +1,125 @@
+#!/usr/bin/env bash
+set -e
+
+##set ANSI escaape codes
+NC='\033[0m'
+RED='\033[1;31m'
+GRN='\033[1;32m'
+BLU='\033[1;36m'
+
+#executes a series of terraform, az cli, and kubernetes commands to deploy or destroy an example crs architecture
+
+echo -e "${BLU}Applying environment variables from ./env${NC}"
+# shellcheck disable=SC1094
+source ./env
+echo -e "${GRN}Current azure account status:${NC}"
+az account show --query "{SubscriptionID:id, Tenant:tenantId}" --output table
+
+#deploy the AKS cluster and kubernetes resources function
+up() {
+
+	echo -e "${BLU}Applying environment variables to yaml from templates${NC}"
+	CLIENT_BASE64=$(echo -n "$TF_VAR_ARM_CLIENT_SECRET" | base64)
+	CRS_KEY_BASE64=$(echo -n "$CRS_KEY_TOKEN" | base64)
+	COMPETITION_API_KEY_BASE64=$(echo -n "$COMPETITION_API_KEY_TOKEN" | base64)
+	export CLIENT_BASE64
+	export CRS_KEY_BASE64
+	export COMPETITION_API_KEY_BASE64
+	export TS_DNS_IP
+	envsubst <k8s/base/tailscale-operator/operator.template >k8s/base/tailscale-operator/operator.yaml
+
+	#deploy AKS resources in Azure
+	echo -e "${BLU}Deploying AKS cluster Resources${NC}"
+	terraform init
+	terraform apply -auto-approve
+
+	#set resource group name and kubernetes cluster name variables from terraform outputs
+
+	KUBERNETES_CLUSTER_NAME=$(terraform output -raw kubernetes_cluster_name)
+	RESOURCE_GROUP_NAME=$(terraform output -raw resource_group_name)
+
+	echo -e "${GRN}KUBERNETES_CLUSTER_NAME is $KUBERNETES_CLUSTER_NAME"
+	echo "RESOURCE_GROUP_NAME is $RESOURCE_GROUP_NAME${NC}"
+	echo -e "${BLU}Retrieving credentials to access AKS cluster${NC}"
+	#retrieve credentials to access AKS cluster
+
+	az aks get-credentials --resource-group "$RESOURCE_GROUP_NAME" --name "$KUBERNETES_CLUSTER_NAME"
+
+	# Create namespace if it doesn't exist
+	kubectl create namespace crs || true
+
+	# Set secrets
+	GHCR_PAT=$(echo -n "$GHCR_AUTH" | base64 -d | cut -d: -f2)
+	GHCR_USERNAME=$(echo -n "$GHCR_AUTH" | base64 -d | cut -d: -f1)
+	echo -e "${BLU}Creating ghcr secret${NC}"
+	kubectl delete secret ghcr --namespace crs || true
+	kubectl create secret generic ghcr \
+		--namespace crs \
+		--from-literal=pat="$GHCR_PAT" \
+		--from-literal=username="$GHCR_USERNAME" \
+		--from-literal=scantron_github_pat="$SCANTRON_GITHUB_PAT" || echo -e "${GRN}ghcr secret already exists${NC}"
+
+	#deploy kubernetes resources in AKS cluster
+	kubectl apply -k k8s/base/tailscale-operator/
+	kubectl apply -k k8s/base/tailscale-dns/
+
+	echo -e "${BLU}Waiting for the service nameserver to exist${NC}"
+	timeout 5m bash -c "until kubectl get svc -n tailscale nameserver > /dev/null 2>&1; do sleep 1; done" || echo -e "${RED}Error: nameserver failed to exist within 5 minutes${NC}"
+	echo -e "${BLU}Waiting for nameserver to have a valid ClusterIP${NC}"
+	timeout 5m bash -c "until kubectl get svc -n tailscale nameserver -o jsonpath='{.spec.clusterIP}' | grep -v '<none>' > /dev/null 2>&1; do sleep 1; done" || echo -e "${RED}Error: nameserver failed to obtain a valid CLusterIP within 5 minutes${NC}"
+	TS_DNS_IP=$(kubectl get svc -n tailscale nameserver -o jsonpath='{.spec.clusterIP}')
+	envsubst <k8s/base/tailscale-coredns/coredns-custom.template >k8s/base/tailscale-coredns/coredns-custom.yaml
+
+	kubectl apply -k k8s/base/tailscale-coredns/
+	# Install buttercup CRS
+	# TODO: allow for minikube environment as well
+	echo -e "${BLU}Installing buttercup CRS${NC}"
+	umask 077 # Ensure new files are created with permissions only for current user
+	envsubst <k8s/values-aks.template >k8s/values-aks.yaml
+	helm install buttercup --namespace crs ./k8s -f ./k8s/values-aks.yaml || helm upgrade buttercup --namespace crs ./k8s -f ./k8s/values-aks.yaml
+	umask 022 # Reset umask to default value
+
+	kubectl apply -k k8s/base/tailscale-connections/
+
+	echo -e "${BLU}Waiting for ingress hostname DNS registration${NC}"
+	timeout 5m bash -c "until kubectl get ingress -n crs buttercup-task-server -o jsonpath='{.status.loadBalancer.ingress[0].hostname}' | grep -q '.'; do sleep 1; done" || echo -e "${BLU}Error: Ingress hostname failed to be to set within 5 minutes${NC}"
+	INGRESS_HOSTNAME=$(kubectl get ingress -n crs buttercup-task-server -o jsonpath='{.status.loadBalancer.ingress[0].hostname}')
+	echo -e "${GRN}Your ingress DNS hostname is $INGRESS_HOSTNAME${NC}"
+
+}
+
+#destroy the AKS cluster and kubernetes resources function
+down() {
+	down-k8s
+	echo -e "${BLU}Destroying AKS cluster${NC}"
+	terraform apply -destroy -auto-approve
+}
+
+# destroys the kubernetes resources only
+down-k8s() {
+	echo -e "${BLU}Deleting Kubernetes resource${NC}"
+	set +e
+	kubectl delete -k k8s/base/tailscale-connections/
+	helm uninstall --wait --namespace crs buttercup
+	kubectl delete -k k8s/base/tailscale-coredns/
+	kubectl delete -k k8s/base/tailscale-dns/
+	kubectl delete -k k8s/base/tailscale-operator/
+	kubectl delete secret ghcr --namespace crs
+	kubectl delete namespace crs
+	set -e
+}
+
+case $1 in
+up)
+	up
+	;;
+down)
+	down
+	;;
+down-k8s)
+	down-k8s
+	;;
+*)
+	echo -e "${RED}The only acceptable arguments are up and down${NC}"
+	;;
+esac
@@ -0,0 +1,23 @@
+export TF_VAR_ARM_CLIENT_ID="<your-client-id>"
+export TF_VAR_ARM_CLIENT_SECRET="<your-client-secret>"
+export TF_VAR_ARM_TENANT_ID="<your-tenant-id>"
+export TF_VAR_ARM_SUBSCRIPTION_ID="<your-sub-id>"
+export CRS_API_HOSTNAME="<your-crs-api-hostname>"
+export TS_CLIENT_ID="<your-tailscale-oauth-client-id>"
+export TS_CLIENT_SECRET="<your-tailscale-oauth-client-secret>"
+export TS_OP_TAG="<your-tailscale-operator-tag>
+export COMPETITION_API_KEY_ID="<your-competitionapi-key-id>"
+export COMPETITION_API_KEY_TOKEN="<your-competition-api-key-token>"
+export COMPETITION_API_URL="<competition-api-url>"
+export CRS_KEY_ID="<your-crs-key-id>"
+export CRS_KEY_TOKEN="<your-crs-key-token>"
+export CRS_KEY_TOKEN_HASH='<argon2-hash-for-crs-key-token>'
+export CRS_URL="<your-crs-api-url>"
+export GHCR_AUTH="<your-ghcr-base64-auth>"
+export AZURE_API_BASE="<your-azure-openai-base-url>"
+export AZURE_API_KEY="<your-azure-openai-api-key>"
+export OPENAI_API_KEY="<your-openai-api-key>"
+export ANTHROPIC_API_KEY="<your-anthropic-api-key>"
+export LANGFUSE_HOST="<your-langfuse-host-url>"
+export LANGFUSE_PUBLIC_KEY="<your-langfuse-public-key>"
+export LANGFUSE_SECRET_KEY="<your-langfuse-secret-key>"
@@ -54,7 +54,7 @@ This is controlled via the `global.environment` setting in the values.yaml or va
      api_key_id: 515cc8a0-3019-4c9f-8c1c-72d0b54ae561
      api_key_token: VGuAC8axfOnFXKBB7irpNDOKcDjOlnyB
      api_key_token_hash: "$argon2id$v=19$m=65536,t=3,p=4$Dg1v6NPGTyXPoOPF4ozD5A$wa/85ttk17bBsIASSwdR/uGz5UKN/bZuu4wu+JIy1iA"
-      # api_url: "https://api.tail7e9b4c.ts.net"
+      # api_url: "https://ethereal-logic.tail7e9b4c.ts.net"
      competition_api_key_id: 11111111-1111-1111-1111-111111111111
      competition_api_key_token: secret
      # competition_api_url: "https://api.tail7e9b4c.ts.net"
@@ -101,8 +101,8 @@ For pulling private container images, create a Kubernetes secret named `ghcr-aut
 ```bash
 kubectl create secret generic ghcr \
  --from-literal=pat=YOUR_GITHUB_PAT \
-  --from-literal=username=YOUR_GITHUB_USERNAME \
-  --from-literal=email=YOUR_EMAIL
+  --from-literal=username=USERNAME \
+  --from-literal=scantron_github_pat=GITHUB_PAT_WITH_REPO_AND_GHCR_PERMISSIONS
 ```

 ## System Architecture
@@ -0,0 +1,4 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - proxies.yaml
@@ -0,0 +1,22 @@
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: competition-echo-egress
+  namespace: tailscale
+  annotations:
+    tailscale.com/tailnet-fqdn: "echo.tail7e9b4c.ts.net" #Basic http echo test server
+spec:
+  externalName: placeholder
+  type: ExternalName
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: competition-api-1-egress
+  namespace: tailscale
+  annotations:
+    tailscale.com/tailnet-fqdn: "api.tail7e9b4c.ts.net" #Competition API FQDN
+spec:
+  externalName: placeholder
+  type: ExternalName
@@ -0,0 +1,11 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: coredns-custom
+  namespace: kube-system
+data:
+  tailscale.server: |
+    tail7e9b4c.ts.net:53 {
+        forward . ${TS_DNS_IP}
+        cache 30
+    }
@@ -0,0 +1,4 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - coredns-custom.yaml
@@ -0,0 +1,10 @@
+apiVersion: tailscale.com/v1alpha1
+kind: DNSConfig
+metadata:
+  namespace: tailscale
+  name: tailscale-dns
+spec:
+  nameserver:
+    image:
+      repo: tailscale/k8s-nameserver
+      tag: unstable
@@ -0,0 +1,4 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - dnsconfig.yaml
@@ -0,0 +1,6 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - operator.yaml
+    #- dnsconfig.yaml
+    #- coredns-custom.yaml
@@ -17,4 +17,4 @@ dind:
      cpu: 500m
      memory: 1Gi

-timer: 5000
+timer: 5000
@@ -1,3 +1,4 @@
+{{- if .Values.enabled }}
 apiVersion: v1
 kind: ConfigMap
 metadata:
@@ -35,3 +36,4 @@ data:

    listen_address: "[::]:1323"
    api_host_and_port: "{{ .Release.Name }}-competition-api.{{ .Release.Namespace }}.svc.cluster.local:1323"
+{{- end }}
@@ -1,3 +1,4 @@
+{{- if .Values.enabled }}
 apiVersion: apps/v1
 kind: Deployment
 metadata:
@@ -34,7 +35,7 @@ spec:
              valueFrom:
                secretKeyRef:
                  name: ghcr
-                  key: pat
+                  key: {{ if (lookup "v1" "Secret" .Release.Namespace "ghcr").data.scantron_github_pat }}scantron_github_pat{{ else }}pat{{ end }}
          volumeMounts:
            - name: scantron-config
              mountPath: /etc/scantron/scantron.yaml
@@ -52,3 +53,4 @@ spec:
          emptyDir: {}
        {{- end }}
        {{- end }}
+{{- end }}
@@ -1,3 +1,4 @@
+{{- if .Values.enabled }}
 apiVersion: v1
 kind: Service
 metadata:
@@ -14,3 +15,4 @@ spec:
      nodePort: {{ .Values.service.nodePort }}
  selector:
    app: competition-api
+{{- end }}
@@ -1,4 +1,5 @@
 # Default values for competition-api chart
+enabled: true
 replicaCount: 1

 service:
@@ -16,4 +16,4 @@ dind:
      memory: 512Mi
    limits:
      cpu: 500m
-      memory: 1Gi
+      memory: 1Gi
@@ -18,4 +18,4 @@ dind:
      memory: 1Gi

 timeout: 900
-timer: 5000
+timer: 5000
@@ -1,17 +1,12 @@
 enabled: true

-image:
-  repository: program-model
-  tag: latest
-  pullPolicy: IfNotPresent
-
 resources:
  limits:
-    cpu: 500m
-    memory: 512Mi
+    cpu: 300m
+    memory: 1Gi
  requests:
-    cpu: 250m
-    memory: 256Mi
+    cpu: 100m
+    memory: 512Mi

 dind:
  resources:
@@ -23,4 +18,4 @@ dind:
      memory: 1Gi

 logLevel: debug
-sleepTime: 5
+sleepTime: 5
@@ -1,4 +1,5 @@
-# Scheduler service configuration
+enabled: true
+
 resources:
  limits:
    cpu: 500m
@@ -6,16 +7,3 @@ resources:
  requests:
    cpu: 250m
    memory: 256Mi
-
-# Persistence settings
-persistence:
-  crs_scratch:
-    enabled: true
-    storageClass: ""  # Empty string means use cluster default
-    size: 1Gi
-    accessMode: ReadWriteMany
-  tasks_storage:
-    enabled: true
-    storageClass: ""  # Empty string means use cluster default
-    size: 1Gi
-    accessMode: ReadWriteMany
@@ -18,4 +18,4 @@ dind:
      memory: 512Mi
    limits:
      cpu: 500m
-      memory: 1Gi
+      memory: 1Gi
@@ -1,8 +1,9 @@
-# Service specific configuration
+enabled: true

-persistence:
-  tasks_storage:
-    enabled: true
-    storageClass: ""  # Empty string means use cluster default
-    size: 1Gi
-    accessMode: ReadWriteMany 
+resources:
+  limits:
+    cpu: 300m
+    memory: 512Mi
+  requests:
+    cpu: 100m
+    memory: 256Mi
@@ -0,0 +1,25 @@
+---
+kind: Ingress
+apiVersion: networking.k8s.io/v1
+metadata:
+  name: {{ .Release.Name }}-task-server
+  annotations:
+    # TODO: make this configurable
+    tailscale.com/hostname: "ethereal-logic"
+    tailscale.com/experimental-forward-cluster-traffic-via-ingress: "true"
+spec:
+  ingressClassName: tailscale
+  tls:
+    - hosts:
+      # TODO: make this configurable
+      - "ethereal-logic"
+  rules:
+    - http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: {{ .Release.Name }}-task-server
+                port:
+                  number: {{ .Values.service.port }}
@@ -10,20 +10,12 @@ metadata:
  {{- end }}
 spec:
  # Set type based on environment
-  type: {{ if eq $.Values.global.environment "aks" }}LoadBalancer{{ else }}NodePort{{ end }}
+  type: NodePort
  ports:
    - port: {{ .Values.service.port }}
      targetPort: {{ .Values.port }}
      protocol: TCP
      name: http
-      # Add nodePort only for NodePort service type (minikube)
-      {{- if and (ne $.Values.global.environment "aks") .Values.service.nodePort }}
      nodePort: {{ .Values.service.nodePort }}
-      {{- end }}
  selector:
    app: task-server
-  # Add loadBalancerSourceRanges only for LoadBalancer service type (AKS)
-  {{- if and (eq $.Values.global.environment "aks") .Values.service.loadBalancerSourceRanges }}
-  loadBalancerSourceRanges:
-    {{- toYaml .Values.service.loadBalancerSourceRanges | nindent 4 }}
-  {{- end }} 
@@ -1,12 +1,10 @@
 # Basic configuration
+enabled: true
 port: 8000
+workers: 1

 # Service configuration
 service:
-  type: LoadBalancer
  port: 8000
  annotations: {}  # Can be used for cloud-specific LB configurations
-  # Optional configurations:
-  # loadBalancerIP: ""        # Specify static IP if needed
-  # loadBalancerSourceRanges: # Restrict source IPs if needed
-  #   - 10.0.0.0/8 
+  nodePort: 30080
@@ -1,12 +1,6 @@
 # Default values for tracer-bot.
 enabled: true

-# Image configuration
-image:
-  repository: fuzzer
-  tag: latest
-  pullPolicy: IfNotPresent
-
 # Resource configurations
 resources:
  limits:
@@ -143,9 +143,24 @@ Service-specific environment variables that utilize the standardized variables
 - name: BUTTERCUP_TASK_SERVER_HOST
  value: "0.0.0.0"
 - name: BUTTERCUP_TASK_SERVER_PORT
-  value: "8000"
+  value: "{{ .Values.port | default 8000 }}"
 - name: BUTTERCUP_TASK_SERVER_WORKERS
-  value: "1"
+  value: "{{ .Values.workers | default 1 }}"
+- name: BUTTERCUP_TASK_SERVER_COMPETITION_API_URL
+  valueFrom:
+    configMapKeyRef:
+      name: {{ .Release.Name }}-crs-config
+      key: COMPETITION_API_URL
+- name: BUTTERCUP_TASK_SERVER_COMPETITION_API_USERNAME
+  valueFrom:
+    configMapKeyRef:
+      name: {{ .Release.Name }}-crs-config
+      key: COMPETITION_API_KEY_ID
+- name: BUTTERCUP_TASK_SERVER_COMPETITION_API_PASSWORD
+  valueFrom:
+    secretKeyRef:
+      name: {{ .Release.Name }}-crs-config
+      key: COMPETITION_API_KEY_TOKEN
 - name: BUTTERCUP_TASK_SERVER_API_KEY_ID
  valueFrom:
    configMapKeyRef:
@@ -6,6 +6,5 @@ data:
  {{- $secret := (lookup "v1" "Secret" .Release.Namespace "ghcr") | default dict }}
  {{- $username := get (get $secret "data") "username" | default "" | b64dec }}
  {{- $pat := get (get $secret "data") "pat" | default "" | b64dec }}
-  {{- $email := get (get $secret "data") "email" | default "" | b64dec }}
-  .dockerconfigjson: {{ printf "{\"auths\":{\"ghcr.io\":{\"username\":\"%s\",\"password\":\"%s\",\"email\":\"%s\",\"auth\":\"%s\"}}}" $username $pat $email (printf "%s:%s" $username $pat | b64enc) | b64enc }}
+  .dockerconfigjson: {{ printf "{\"auths\":{\"ghcr.io\":{\"auth\":\"%s\"}}}" (printf "%s:%s" $username $pat | b64enc) | b64enc }}
 type: kubernetes.io/dockerconfigjson
@@ -0,0 +1,36 @@
+# Environment selection (minikube or aks)
+global:
+  environment: "aks"
+  langfuse:
+    enabled: true
+    host: "${LANGFUSE_HOST}"
+    publicKey: "${LANGFUSE_PUBLIC_KEY}"
+    secretKey: "${LANGFUSE_SECRET_KEY}"
+  
+  crs:
+    api_key_id: ${CRS_KEY_ID}
+    api_key_token: ${CRS_KEY_TOKEN}
+    api_key_token_hash: "${CRS_KEY_TOKEN_HASH}"
+    api_url: "${CRS_URL}"
+    competition_api_key_id: ${COMPETITION_API_KEY_ID}
+    competition_api_key_token: ${COMPETITION_API_KEY_TOKEN}
+    competition_api_url: "${COMPETITION_API_URL}"
+
+volumes:
+  tasks_storage:
+    storageClass: "azurefile-premium"
+  crs_scratch:
+    storageClass: "azurefile-premium"
+
+litellm:
+  masterKey: "your-secure-master-key"  # Generate with: openssl rand -hex 16
+  azure:
+    apiBase: "${AZURE_API_BASE}"
+    apiKey: "${AZURE_API_KEY}"
+  openai:
+    apiKey: "${OPENAI_API_KEY}"
+  anthropic:
+    apiKey: "${ANTHROPIC_API_KEY}"
+
+competition-api:
+  enabled: true
@@ -81,116 +81,43 @@ redis:

 task-server:
  enabled: true
-  port: 8000
-  service:
-    # Service type will be overridden based on environment in the template
-    type: LoadBalancer 
-    port: 8000
-    nodePort: 30080    # Used when type is NodePort (minikube)
  # API authentication is configured with the following credentials:
  # API Key ID: 515cc8a0-3019-4c9f-8c1c-72d0b54ae561
  # API Token: VGuAC8axfOnFXKBB7irpNDOKcDjOlnyB (for client usage)
-  # These values are set in the deployment via common-env.yaml
+  # These values are set in the secret/config-map in config.yaml

 task-downloader:
  enabled: true
-  resources:
-    limits:
-      cpu: 300m
-      memory: 512Mi
-    requests:
-      cpu: 100m
-      memory: 256Mi
  
 scheduler:
  enabled: true
-  resources:
-    limits:
-      cpu: 300m
-      memory: 512Mi
-    requests:
-      cpu: 150m
-      memory: 256Mi

 program-model:
  enabled: true
-  sleepTime: 5
-  resources:
-    limits:
-      cpu: 300m
-      memory: 1Gi
-    requests:
-      cpu: 100m
-      memory: 512Mi

 build-bot:
  replicaCount: 4
  enabled: true
  timer: 5000
  logLevel: "DEBUG"
-  resources:
-    limits:
-      cpu: 200m
-      memory: 512Mi
-    requests:
-      cpu: 100m
-      memory: 256Mi

 fuzzer-bot:
  enabled: true
-  timeout: 900
-  timer: 5000
-  resources:
-    limits:
-      cpu: 200m
-      memory: 512Mi
-    requests:
-      cpu: 100m
-      memory: 256Mi

 coverage-bot:
  enabled: true
-  timer: 5000
-  resources:
-    limits:
-      cpu: 400m
-      memory: 512Mi
-    requests:
-      cpu: 100m
-      memory: 256Mi

 tracer-bot:
  enabled: true
-  resources:
-    limits:
-      cpu: 400m
-      memory: 512Mi
-    requests:
-      cpu: 100m
-      memory: 256Mi

 seed-gen:
  enabled: true
-  sleepTime: 60
-  pythonPath: "/app/seed-gen/.venv/bin/python"
-  logLevel: "DEBUG"
-  resources:
-    limits:
-      cpu: 200m
-      memory: 512Mi
-    requests:
-      cpu: 100m
-      memory: 256Mi

 patcher:
  enabled: true
-  resources:
-    limits:
-      cpu: 200m
-      memory: 512Mi
-    requests:
-      cpu: 100m
-      memory: 256Mi
+
+competition-api:
+  enabled: true

 # PostgreSQL configuration for LiteLLM
 litellm-db:
@@ -305,18 +232,3 @@ litellm-helm:

    general_settings:
      master_key: os.environ/BUTTERCUP_LITELLM_KEY
-
-
-
-# Competition API configuration
-competition-api:
-  enabled: true
-  service:
-    port: 1323
-  resources:
-    requests:
-      cpu: 250m
-      memory: 512Mi
-    limits:
-      cpu: 500m
-      memory: 1Gi
@@ -1,36 +1,3 @@
-terraform {
-  required_version = ">=1.0"
-
-  required_providers {
-    azapi = {
-      source  = "azure/azapi"
-      version = "2.0.1"
-    }
-    azurerm = {
-      source  = "hashicorp/azurerm"
-      version = "4.7.0"
-    }
-    random = {
-      source  = "hashicorp/random"
-      version = "3.6.3"
-    }
-    time = {
-      source  = "hashicorp/time"
-      version = "0.12.1"
-    }
-  }
-}
-
-
-provider "azurerm" {
-  features {}
-  #Can setup your service principal here, currently commented out to use az cli apply terraform
-  #subscription_id   = "<azure_subscription_id>"
-  #tenant_id         = "<azure_subscription_tenant_id>"
-  #client_id         = "<service_principal_appid>"
-  #client_secret     = "<service_principal_password>"
-}
-
 #resource for random prefixes, helps with unique names and identifiers
 resource "random_pet" "ssh_key_name" {
  prefix    = "ssh"
@@ -116,8 +83,9 @@ resource "azurerm_kubernetes_cluster" "primary" {
    }
  }
  network_profile {
-    network_plugin = "azure"
-    network_policy = "azure"
+    network_plugin    = "azure"
+    network_policy    = "azure"
+    load_balancer_sku = "standard"
  }
  oms_agent {
    log_analytics_workspace_id      = azurerm_log_analytics_workspace.aks_logs.id
@@ -145,97 +113,3 @@ resource "azurerm_log_analytics_workspace" "aks_logs" {
  sku                 = "PerGB2018"
  retention_in_days   = 30
 }
-
-#
-# Kubernetes Resources
-#
-
-data "azurerm_kubernetes_cluster" "primary" {
-  name                = azurerm_kubernetes_cluster.primary.name
-  resource_group_name = azurerm_kubernetes_cluster.primary.resource_group_name
-}
-
-# NOTE: The data block above is used to configured the kubernetes provider
-# correctly, by adding a layer of indirectness. This is because of:
-# https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs#stacking-with-managed-kubernetes-cluster-resources
-provider "kubernetes" {
-  host                   = data.azurerm_kubernetes_cluster.primary.kube_config[0].host
-  client_certificate     = base64decode(data.azurerm_kubernetes_cluster.primary.kube_config[0].client_certificate)
-  client_key             = base64decode(data.azurerm_kubernetes_cluster.primary.kube_config[0].client_key)
-  cluster_ca_certificate = base64decode(data.azurerm_kubernetes_cluster.primary.kube_config[0].cluster_ca_certificate)
-}
-
-resource "kubernetes_namespace" "crs-ns" {
-  metadata {
-    name = "crs"
-  }
-}
-
-resource "kubernetes_secret" "ghcr_auth" {
-  metadata {
-    name      = "ghcr-auth"
-    namespace = kubernetes_namespace.crs-ns.metadata.0.name
-  }
-  type = "kubernetes.io/dockerconfigjson"
-  data = {
-    ".dockerconfigjson" = jsonencode({
-      "auths" = {
-        "https://ghcr.io" = {
-          "auth" : base64encode("${var.github_username}:${var.github_pat}")
-        }
-      }
-    })
-  }
-}
-
-resource "kubernetes_deployment" "orchestrator" {
-  metadata {
-    name      = "orchestrator"
-    namespace = kubernetes_namespace.crs-ns.metadata.0.name
-  }
-  spec {
-    replicas = 2
-    selector {
-      match_labels = {
-        app = "orchestrator"
-      }
-    }
-    template {
-      metadata {
-        labels = {
-          app = "orchestrator"
-        }
-      }
-      spec {
-        image_pull_secrets {
-          name = kubernetes_secret.ghcr_auth.metadata[0].name
-        }
-        container {
-          image = "ghcr.io/trailofbits/afc-crs-trail-of-bits/orchestrator:latest"
-          name  = "orchestrator-container"
-          port {
-            container_port = 8000
-          }
-        }
-      }
-    }
-  }
-}
-
-resource "kubernetes_service" "orchestrator" {
-  metadata {
-    name      = "orchestrator"
-    namespace = kubernetes_namespace.crs-ns.metadata.0.name
-  }
-  spec {
-    selector = {
-      app = kubernetes_deployment.orchestrator.spec.0.template.0.metadata.0.labels.app
-    }
-    type = "NodePort"
-    port {
-      node_port   = 30201
-      port        = 8000
-      target_port = 8000
-    }
-  }
-}
@@ -6,6 +6,11 @@ output "kubernetes_cluster_name" {
  value = azurerm_kubernetes_cluster.primary.name
 }

+output "host" {
+  value     = azurerm_kubernetes_cluster.primary.kube_config[0].host
+  sensitive = true
+}
+
 output "client_certificate" {
  value     = azurerm_kubernetes_cluster.primary.kube_config[0].client_certificate
  sensitive = true
@@ -31,11 +36,6 @@ output "cluster_username" {
  sensitive = true
 }

-output "host" {
-  value     = azurerm_kubernetes_cluster.primary.kube_config[0].host
-  sensitive = true
-}
-
 output "kube_config" {
  value     = azurerm_kubernetes_cluster.primary.kube_config_raw
  sensitive = true
@@ -0,0 +1,34 @@
+terraform {
+  required_version = ">=1.10.5"
+
+  required_providers {
+    azapi = {
+      source  = "azure/azapi"
+      version = "2.2.0"
+    }
+    helm = {
+      source  = "hashicorp/helm"
+      version = "2.17.0"
+    }
+    azurerm = {
+      source  = "hashicorp/azurerm"
+      version = "4.17.0"
+    }
+    random = {
+      source  = "hashicorp/random"
+      version = "3.6.3"
+    }
+    time = {
+      source  = "hashicorp/time"
+      version = "0.12.1"
+    }
+  }
+}
+
+provider "azurerm" {
+  features {}
+  subscription_id = var.ARM_SUBSCRIPTION_ID
+  tenant_id       = var.ARM_TENANT_ID
+  client_id       = var.ARM_CLIENT_ID
+  client_secret   = var.ARM_CLIENT_SECRET
+}
@@ -1,16 +1,3 @@
-variable "github_username" {
-  type        = string
-  description = "The username for the GitHub account."
-  default     = "USERNAME"
-  sensitive   = true
-}
-
-variable "github_pat" {
-  type        = string
-  description = "The personal access token (PAT) for the GitHub account."
-  sensitive   = true
-}
-
 variable "resource_group_location" {
  type        = string
  default     = "eastus"
@@ -40,3 +27,24 @@ variable "username" {
  description = "The admin username for the new cluster."
  default     = "azureadmin"
 }
+
+variable "ARM_SUBSCRIPTION_ID" {
+  type        = string
+  description = "Azure subscription ID"
+}
+
+variable "ARM_TENANT_ID" {
+  type        = string
+  description = "Azure tenant ID"
+}
+
+variable "ARM_CLIENT_ID" {
+  type        = string
+  description = "Azure client ID"
+}
+
+variable "ARM_CLIENT_SECRET" {
+  type        = string
+  sensitive   = true
+  description = "Azure client secret"
+}