Files
Will Schroeder a36be30b39 K8s k3d/k3s support (#106)
* Add k8s/k3d/k3s deployment with Helm, KEDA autoscaling, and connection pool fixes

- Helm chart for full Nemesis deployment (Traefik, Dapr, KEDA, all services)
- KEDA autoscaling for file-enrichment and document-conversion on RabbitMQ queue depth
- Fix PostgreSQL connection exhaustion: configurable pool size (default 20, was 100)
- Cap Dapr statestore connections (maxConns=10), increase postgres max_connections to 300
- Add Prometheus metrics and /system/pool-stats endpoint for connection pool monitoring
- Background pool stats logger with pressure warnings
- Fix Prometheus scrape targets to correct service ports
- Helm-based Dapr install (v1.16.9) instead of CLI
- k8s setup/deploy/build/verify/teardown scripts
- Kubernetes deployment docs and references in CLAUDE.md, AGENTS.md, mkdocs

* Add KEDA autoscaling for titus-scanner and dotnet-service

* Add PgBouncer connection pooling, KEDA gotenberg CPU scaling, and tuning

- Add PgBouncer deployment between services and PostgreSQL (transaction pooling)
- Route all DB connections through pgbouncer instead of postgres directly
- Add KEDA CPU-based autoscaling for gotenberg (1-3 replicas)
- Mirror external images (pgbouncer) into k3d registry via build-and-push.sh
- Reduce queue-based cooldowns to 60s, gotenberg stabilization to 30s
- Add 180s terminationGracePeriodSeconds to gotenberg for in-flight conversions
- Update docs with PgBouncer architecture, connection tuning, and troubleshooting

* Add k3s support, unify Traefik on port 7443, rename scripts

- Add setup-cluster-k3s.sh and teardown-cluster-k3s.sh for native k3s
- Rename scripts with -k3d suffix to distinguish runtimes
- Unify k3s Traefik to HTTPS-only on port 7443 (matching k3d)
- Add k3d registry check for --build flag in deploy.sh
- Fix verify.sh arithmetic under set -e with || true
- Update docs with k3s quick start, comparison table, and teardown
- Add image-sync workflow and pgbouncer registry override for values-dev

* Add monitoring, Jupyter, and LLM optional stacks to Helm chart

- Monitoring: Prometheus, Grafana (with dashboards), Loki, Promtail,
  Jaeger, OTEL Collector, Node Exporter, Postgres Exporter
- Jupyter: notebook service with Hasura integration
- LLM: Agents (Dapr-enabled), LiteLLM proxy, Phoenix observability
- Add ingress routes, strip-prefix middleware, litellm database
- Update deploy.sh, build script, README, and test connectivity checks

* Add k3s --build support with auto-detection in deploy.sh

- deploy.sh --build auto-detects k3d registry vs k3s and uses appropriate build script
- Add build-and-load-k3s.sh for building and importing images into k3s containerd
- Add values-dev-k3s.yaml with imagePullPolicy: Never (no pgbouncer registry override)
- Fix jupyter Dockerfile context path in build-and-push-k3d.sh
- Update docs for k3s local build support

---------

Co-authored-by: Lee Chagolla-Christensen <lee@localhost>
2026-03-05 10:29:03 -08:00

192 lines
5.6 KiB
Bash
Executable File

#!/usr/bin/env bash
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
K8S_DIR="$(dirname "$SCRIPT_DIR")"
CHART_DIR="${K8S_DIR}/helm/nemesis"
NAMESPACE="${NAMESPACE:-nemesis}"
RELEASE_NAME="${RELEASE_NAME:-nemesis}"
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
NC='\033[0m'
log() { echo -e "${GREEN}[+]${NC} $*"; }
warn() { echo -e "${YELLOW}[!]${NC} $*"; }
error() { echo -e "${RED}[-]${NC} $*" >&2; }
usage() {
cat <<EOF
Deploy Nemesis to a Kubernetes cluster using Helm.
Usage: $0 <action> [options]
Actions:
install Install/upgrade Nemesis
uninstall Uninstall Nemesis
status Show deployment status
Options:
--build Build images locally before deploying (auto-detects k3d or k3s)
--monitoring Enable monitoring stack (Prometheus, Grafana, Loki, Jaeger, etc.)
--jupyter Enable Jupyter notebook stack
--llm Enable LLM stack (LiteLLM, Phoenix, Agents)
--values FILE Additional values file
--set KEY=VAL Override a specific value
--dry-run Render templates without deploying
-h, --help Show this help
Examples:
$0 install # Deploy using ghcr.io images
$0 install --build # Build locally (k3d registry or k3s ctr import)
$0 install --monitoring # Deploy with monitoring enabled
$0 uninstall # Remove deployment
$0 status # Check pod/service status
EOF
exit 0
}
if [[ $# -lt 1 ]]; then
usage
fi
ACTION="$1"
shift
BUILD=false
MONITORING=false
JUPYTER=false
LLM=false
DRY_RUN=false
EXTRA_VALUES=()
EXTRA_SETS=()
while [[ $# -gt 0 ]]; do
case "$1" in
--build) BUILD=true; shift ;;
--monitoring) MONITORING=true; shift ;;
--jupyter) JUPYTER=true; shift ;;
--llm) LLM=true; shift ;;
--dry-run) DRY_RUN=true; shift ;;
--values) EXTRA_VALUES+=("$2"); shift 2 ;;
--set) EXTRA_SETS+=("$2"); shift 2 ;;
-h|--help) usage ;;
*) error "Unknown option: $1"; usage ;;
esac
done
do_install() {
# Verify namespace exists (created by setup-cluster-k3d.sh or setup-cluster-k3s.sh)
if ! kubectl get namespace "$NAMESPACE" &>/dev/null; then
error "Namespace '$NAMESPACE' does not exist. Run setup-cluster-k3d.sh or setup-cluster-k3s.sh first."
exit 1
fi
log "Deploying Nemesis to namespace: $NAMESPACE"
# Build images if requested (auto-detects k3d or k3s)
if [[ "$BUILD" == "true" ]]; then
local registry="${REGISTRY:-k3d-nemesis-registry.localhost:5111}"
if curl -sf "http://${registry}/v2/" &>/dev/null; then
BUILD_TARGET="k3d"
log "Detected k3d registry at ${registry}"
log "Building and pushing images to k3d registry..."
"${SCRIPT_DIR}/build-and-push-k3d.sh"
elif command -v k3s &>/dev/null; then
BUILD_TARGET="k3s"
log "Detected k3s cluster"
log "Building images and loading into k3s containerd..."
"${SCRIPT_DIR}/build-and-load-k3s.sh"
else
error "--build requires either a k3d cluster with a local registry or a k3s installation."
error "Use setup-cluster-k3d.sh or setup-cluster-k3s.sh first, or deploy without --build to pull from ghcr.io."
exit 1
fi
fi
# Build helm command
local -a HELM_CMD=(
helm upgrade --install "$RELEASE_NAME" "$CHART_DIR"
--namespace "$NAMESPACE"
-f "${CHART_DIR}/values.yaml"
)
# Use dev values (local registry/containerd) when building locally
if [[ "$BUILD" == "true" ]]; then
if [[ "${BUILD_TARGET:-k3d}" == "k3s" ]]; then
HELM_CMD+=(-f "${CHART_DIR}/values-dev-k3s.yaml")
else
HELM_CMD+=(-f "${CHART_DIR}/values-dev.yaml")
fi
fi
# Optional stack toggles
if [[ "$MONITORING" == "true" ]]; then
HELM_CMD+=(--set "monitoring.enabled=true")
fi
if [[ "$JUPYTER" == "true" ]]; then
HELM_CMD+=(--set "jupyter.enabled=true")
fi
if [[ "$LLM" == "true" ]]; then
HELM_CMD+=(--set "llm.enabled=true")
fi
# Extra values files
for f in "${EXTRA_VALUES[@]+"${EXTRA_VALUES[@]}"}"; do
HELM_CMD+=(-f "$f")
done
# Extra --set overrides
for s in "${EXTRA_SETS[@]+"${EXTRA_SETS[@]}"}"; do
HELM_CMD+=(--set "$s")
done
if [[ "$DRY_RUN" == "true" ]]; then
HELM_CMD+=(--dry-run --debug)
else
HELM_CMD+=(--wait --timeout 10m)
fi
echo ""
log "Running: ${HELM_CMD[*]}"
echo ""
"${HELM_CMD[@]}"
if [[ "$DRY_RUN" != "true" ]]; then
echo ""
log "Deployment complete!"
echo ""
do_status
fi
}
do_uninstall() {
log "Uninstalling Nemesis from namespace: $NAMESPACE"
helm uninstall "$RELEASE_NAME" --namespace "$NAMESPACE" 2>/dev/null || warn "Release not found"
log "Uninstall complete. PVCs are retained — delete manually if needed:"
echo " kubectl delete pvc --all -n $NAMESPACE"
}
do_status() {
echo "=== Pods ==="
kubectl get pods -n "$NAMESPACE" -o wide 2>/dev/null || warn "No pods found"
echo ""
echo "=== Services ==="
kubectl get svc -n "$NAMESPACE" 2>/dev/null || warn "No services found"
echo ""
echo "=== Dapr Components ==="
kubectl get components.dapr.io -n "$NAMESPACE" 2>/dev/null || warn "No Dapr components found"
echo ""
echo "=== KEDA ScaledObjects ==="
kubectl get scaledobject -n "$NAMESPACE" 2>/dev/null || warn "No KEDA objects found"
}
case "$ACTION" in
install) do_install ;;
uninstall) do_uninstall ;;
status) do_status ;;
*) error "Unknown action: $ACTION"; usage ;;
esac