admin/trailofbits-buttercup
1
0
Fork 0
mirror of https://github.com/trailofbits/buttercup synced 2026-06-21 14:11:39 +00:00
Code Issues Packages Projects Releases Wiki Activity

deployment: do not deploy SigNoz locally

Browse Source 
Local SigNoz/Langfuse use a lot of resources that make local testing
much more resource heavy (e.g. clickhouse is quite big). Instead, just
assume users who wants this will deploy these resources separately
elsewhere.

This also has the advantage that each CRS deployment/un-deployment does
not recreate the observability tools as well. The resources in minikube
will be only for the real CRS functionalities and deploying and
un-deploying the system will be faster.
This commit is contained in:
Riccardo Schirone
2025-12-05 11:11:36 +01:00
parent ab27eb2fab
commit 77e92f5fca
18 changed files with 78 additions and 124 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/ci-env.template
-1
View File
@@ -1,6 +1,5 @@
export BUTTERCUP_K8S_VALUES_TEMPLATE=./k8s/values-upstream-minikube.template
export DEPLOY_CLUSTER=true
export DEPLOY_SIGNOZ=false
export CLUSTER_TYPE=minikube
export TAILSCALE_ENABLED=false
export COMPETITION_API_ENABLED=false
Makefile
-24
View File
@@ -18,7 +18,6 @@ help:
@echo " status - Check the status of the deployment"
@echo " crs-instance-id - Get the CRS instance ID"
@echo " download-artifacts - Download submitted artifacts from the CRS"
@echo " signoz-ui - Open the SigNoz UI"
@echo ""
@echo "Testing:"
@echo " send-integration-task - Run integration-test task"
@@ -203,28 +202,5 @@ clean-local:
install-cscope:
cd external/buttercup-cscope/ && autoreconf -i -s && ./configure && make && sudo make install
signoz-ui:
@echo "Opening SigNoz UI..."
@echo ""
@echo "NOTE: If you plan to redeploy and access SigNoz, clear your browser cookies for http://localhost:33301 to avoid login issues."
@echo ""
@if ! kubectl get namespace $${BUTTERCUP_NAMESPACE:-crs} >/dev/null 2>&1; then \
echo "Error: CRS namespace not found. Deploy first with 'make deploy'."; \
exit 1; \
fi
@if ! kubectl get service/buttercup-signoz-frontend -n $${BUTTERCUP_NAMESPACE:-crs} >/dev/null 2>&1; then \
echo "Error: SigNoz is not deployed. Set DEPLOY_SIGNOZ=true in deployment/env and redeploy."; \
exit 1; \
fi
kubectl port-forward -n $${BUTTERCUP_NAMESPACE:-crs} service/buttercup-signoz-frontend 33301:3301 &
@sleep 3
@if command -v xdg-open >/dev/null 2>&1; then \
xdg-open http://localhost:33301; \
elif command -v open >/dev/null 2>&1; then \
open http://localhost:33301; \
else \
echo "Please open http://localhost:33301 in your browser."; \
fi
web-ui:
@./scripts/web_ui.sh
README.md
+16 -11
View File
@@ -119,22 +119,27 @@ In the GUI you can monitor active tasks and see when Buttercup finds bugs and ge
make undeploy
```
## Accessing Logs
## Accessing Logs and Observability
Buttercup includes local SigNoz deployment by default for comprehensive system observability. You can access logs, traces, and metrics through the SigNoz UI:
Buttercup supports OpenTelemetry for comprehensive system observability. To view distributed traces, metrics, and logs, you need to deploy an observability backend separately:
- **SigNoz**: [Kubernetes deployment guide](https://signoz.io/docs/install/kubernetes/)
- **LangFuse** (for LLM observability): [Self-hosting guide](https://langfuse.com/self-hosting)
Configure the OTEL endpoint in your `deployment/env` file:
```bash
make signoz-ui
export OTEL_ENDPOINT="<your-signoz-otel-collector-endpoint>"
export OTEL_TOKEN="<your-auth-token>"
export OTEL_PROTOCOL=grpc
```
Then navigate to `http://localhost:33301` in your web browser to view:
- Distributed traces
- Application metrics
- Error monitoring
- Performance insights
If you configured LangFuse during setup, you can also monitor LLM usage and costs there.
For LangFuse integration, configure:
```bash
export LANGFUSE_ENABLED=true
export LANGFUSE_HOST="<your-langfuse-url>"
export LANGFUSE_PUBLIC_KEY="<your-public-key>"
export LANGFUSE_SECRET_KEY="<your-secret-key>"
```
For additional log access methods, see the [Quick Reference Guide](guides/QUICK_REFERENCE.md).
deployment/crs-architecture.sh
-5
View File
@@ -38,7 +38,6 @@ fi
BUTTERCUP_NAMESPACE=${BUTTERCUP_NAMESPACE:-crs}
DEPLOY_CLUSTER=${DEPLOY_CLUSTER:-true}
DEPLOY_SIGNOZ=${DEPLOY_SIGNOZ:-false}
CLUSTER_TYPE=${CLUSTER_TYPE:-minikube}
if [ "$DEPLOY_CLUSTER" = "true" ] && [ "$CLUSTER_TYPE" = "aks" ]; then
@@ -270,9 +269,6 @@ down-k8s() {
set -x
kubectl delete -k k8s/base/tailscale-connections/
helm uninstall --wait --namespace "$BUTTERCUP_NAMESPACE" buttercup
# Remove finalizers from clickhouse installation as stated in https://signoz.io/docs/operate/kubernetes/#uninstall-signoz
# Without this, the namespace would not be deleted
kubectl -n "$BUTTERCUP_NAMESPACE" patch clickhouseinstallations.clickhouse.altinity.com/buttercup-clickhouse -p '{"metadata":{"finalizers":[]}}' --type=merge
kubectl delete -k k8s/base/tailscale-coredns/
kubectl delete -k k8s/base/tailscale-dns/
kubectl delete -k k8s/base/tailscale-operator/
@@ -280,7 +276,6 @@ down-k8s() {
kubectl delete namespace "$BUTTERCUP_NAMESPACE"
set +x
echo -e "${GRN}Cleanup complete.${NC}"
echo -e "${BLU}Note: If you plan to redeploy and access SigNoz, clear your browser cookies for http://localhost:33301 to avoid login issues.${NC}"
set -e
}
deployment/env.template
+5 -6
View File
@@ -68,18 +68,17 @@ export OPENAI_API_KEY="<your-openai-api-key>"
export ANTHROPIC_API_KEY="<your-anthropic-api-key>"
# export GEMINI_API_KEY="<your-gemini-api-key>"
# LangFuse settings, instructing LLM-applications to log their LLM traces
# Optional LangFuse settings, instructing LLM-applications to log their LLM traces
# See https://langfuse.com/self-hosting for self-hosting documentation
# export LANGFUSE_ENABLED=true
# export LANGFUSE_HOST="<your-langfuse-host-url>"
# export LANGFUSE_PUBLIC_KEY="<your-langfuse-public-key>"
# export LANGFUSE_SECRET_KEY="<your-langfuse-secret-key>"
# OpenTelemetry endpoint settings (e.g. SigNoz)
export DEPLOY_SIGNOZ=false
# Your otel endpoint
# Optional OpenTelemetry endpoint settings for observability (e.g. SigNoz)
# See https://signoz.io/docs/install/kubernetes/ for SigNoz configuration
# export OTEL_ENDPOINT="<your-otel-endpoint>"
# Your otel token, including basic or bearer
# export OTEL_TOKEN="dXNlcm5hbWU6cGFzc3dvcmQ="
# export OTEL_TOKEN="<your-otel-auth-token>" # Include Basic or Bearer prefix
# export OTEL_PROTOCOL=grpc
# Docker Hub credentials for doing logged requests while getting Container
deployment/k8s/Chart.lock
+2 -5
View File
@@ -53,8 +53,5 @@ dependencies:
- name: ui
repository: file://charts/ui
version: 0.1.0
- name: signoz
repository: https://charts.signoz.io
version: 0.71.0
digest: sha256:d406395eef32321d26f3dee82bd5ab91e80f476dc8f271bac7edbcd04790cc34
generated: "2025-09-29T12:14:43.37270009Z"
digest: sha256:127f211095649787969caa6ac55652a0fda0d2be8e532510de584f4d3e8fc20a
generated: "2025-12-04T14:47:04.214674+01:00"
deployment/k8s/Chart.yaml
-4
View File
@@ -58,7 +58,3 @@ dependencies:
- name: ui
version: "0.1.0"
repository: "file://charts/ui"
- name: signoz
version: "0.71.0"
repository: "https://charts.signoz.io"
condition: signoz.enabled
deployment/k8s/README.md
+27
View File
@@ -91,6 +91,33 @@ If Langfuse integration is enabled:
- **LANGFUSE_PUBLIC_KEY**: Langfuse public key
- **LANGFUSE_SECRET_KEY**: Langfuse secret key
## Observability Configuration
Buttercup uses OpenTelemetry for distributed tracing and observability. You must deploy an observability backend separately (e.g., SigNoz) and configure the system to send telemetry data to it.
### OpenTelemetry Configuration
Configure the following in your `values-override.yaml`:
```yaml
global:
otel:
endpoint: "https://your-signoz-otel-collector:4317"
token: "Basic <base64-encoded-credentials>"
protocol: "grpc"
```
### Deploying SigNoz
For Kubernetes deployment of SigNoz, see:
- [SigNoz Kubernetes Installation](https://signoz.io/docs/install/kubernetes/)
- [SigNoz Helm Deployment](https://signoz.io/docs/install/kubernetes/others/)
### Deploying LangFuse
For self-hosting LangFuse for LLM observability:
- [LangFuse Self-Hosting Guide](https://langfuse.com/self-hosting)
### 3. Container Registry Auth
For pulling private container images, create a Kubernetes secret named `ghcr-auth`:
deployment/k8s/charts/signoz-0.71.0.tgz
BIN
View File
Binary file not shown.
deployment/k8s/templates/common-env.yaml
-9
View File
@@ -85,21 +85,12 @@ Define Docker Host environment variable for Unix socket
{{- end }}
{{- define "buttercup.env.telemetry" }}
{{- if .Values.global.signoz.deployed }}
- name: OTEL_EXPORTER_OTLP_ENDPOINT
value: "http://{{ .Release.Name }}-signoz-otel-collector:4317"
- name: OTEL_EXPORTER_OTLP_HEADERS
value: "Authorization=Basic dXNlcm5hbWU6cGFzc3dvcmQ="
- name: OTEL_EXPORTER_OTLP_PROTOCOL
value: "grpc"
{{- else }}
- name: OTEL_EXPORTER_OTLP_ENDPOINT
value: "{{ .Values.global.otel.endpoint }}"
- name: OTEL_EXPORTER_OTLP_HEADERS
value: "Authorization={{ .Values.global.otel.token }}"
- name: OTEL_EXPORTER_OTLP_PROTOCOL
value: "{{ .Values.global.otel.protocol | default "grpc" }}"
{{- end }}
- name: CRS_INSTANCE_ID
valueFrom:
configMapKeyRef:
deployment/k8s/values-upstream-aks.template
-6
View File
@@ -17,9 +17,6 @@ global:
publicKey: "${LANGFUSE_PUBLIC_KEY}"
secretKey: "${LANGFUSE_SECRET_KEY}"
signoz:
deployed: ${DEPLOY_SIGNOZ}
crs:
api_key_id: ${CRS_KEY_ID}
api_key_token: ${CRS_KEY_TOKEN}
@@ -133,9 +130,6 @@ image-preloader:
versions:
- latest
signoz:
enabled: ${DEPLOY_SIGNOZ}
registry-cache:
enabled: true
persistence:
deployment/k8s/values-upstream-minikube.template
-6
View File
@@ -27,9 +27,6 @@ global:
publicKey: "${LANGFUSE_PUBLIC_KEY}"
secretKey: "${LANGFUSE_SECRET_KEY}"
signoz:
deployed: ${DEPLOY_SIGNOZ}
crs:
api_key_id: ${CRS_KEY_ID}
api_key_token: ${CRS_KEY_TOKEN}
@@ -86,6 +83,3 @@ image-preloader:
- gcr.io/oss-fuzz-base/base-builder-rust
versions:
- latest
signoz:
enabled: ${DEPLOY_SIGNOZ}
deployment/k8s/values.yaml
-10
View File
@@ -47,8 +47,6 @@ global:
host: "https://cloud.langfuse.com"
publicKey: "pk-lf-..." # Replace with your actual public key
secretKey: "sk-lf-..." # Replace with your actual secret key
signoz:
deployed: false
crs:
api_key_id: 515cc8a0-3019-4c9f-8c1c-72d0b54ae561
api_key_token: VGuAC8axfOnFXKBB7irpNDOKcDjOlnyB
@@ -403,11 +401,3 @@ litellm-helm:
# store_prompts_in_spend_logs: true
restartPolicy: Always
# SigNoz observability platform
signoz:
enabled: false
clickhouse:
zookeeper:
image:
repository: bitnamilegacy/zookeeper
guides/MANUAL_SETUP.md
+6 -12
View File
@@ -69,22 +69,16 @@ cp deployment/env.template deployment/env
Look at the comments in the `deployment/env.template` for how to set variables.
### External SigNoz Configuration
### OpenTelemetry Configuration (Optional)
If you want to use an external SigNoz instance instead of the default local deployment, you can configure custom OpenTelemetry settings:
Buttercup supports sending telemetry data to an external observability platform like SigNoz. You must deploy SigNoz separately - see the [SigNoz Kubernetes deployment guide](https://signoz.io/docs/install/kubernetes/).
1. **Disable local SigNoz deployment:**
To configure the OTEL endpoint:
```bash
# In deployment/env
export DEPLOY_SIGNOZ=false
```
2. **Configure external OTEL endpoint:**
```bash
# In deployment/env
export OTEL_ENDPOINT="https://your-signoz-instance.com"
export OTEL_PROTOCOL="http" # or "grpc"
export OTEL_TOKEN="your-otel-token" # optional
export OTEL_ENDPOINT="https://your-signoz-otel-collector:4317"
export OTEL_PROTOCOL="grpc" # or "http"
export OTEL_TOKEN="your-otel-auth-token" # Include Basic or Bearer prefix
```
## Start Services Manually
guides/QUICK_REFERENCE.md
+1 -1
View File
@@ -43,7 +43,7 @@ kubectl get services -n crs
# Port forwarding
kubectl port-forward -n crs service/buttercup-competition-api 31323:1323
# View logs (alternative to SigNoz UI)
# View logs
kubectl get pods -n crs
kubectl logs -n crs <pod-name>
kubectl logs -n crs <pod-name> -f # Follow logs in real-time
scripts/common.sh
+15 -18
View File
@@ -644,29 +644,27 @@ configure_llm_budget() {
}
# Function to configure OTEL telemetry (simplified for local deployment)
# Function to configure OTEL telemetry
configure_otel() {
print_linebreak
print_status "Configuring SigNoz for local observability..."
print_status "Configuring OpenTelemetry / SigNoz (optional observability)..."
# Source the env file to check current values
if [ -f "deployment/env" ]; then
source deployment/env
fi
print_status "SigNoz: Local observability platform for distributed tracing and metrics."
print_status "Provides detailed performance monitoring and system observability for debugging."
# Check if already configured and user wants to keep it
if [ "$DEPLOY_SIGNOZ" = "true" ]; then
if ! prompt_for_update "DEPLOY_SIGNOZ" "SigNoz deployment" "local" "" false; then
return 0 # User chose to keep existing value, exit early
fi
print_status "OpenTelemetry: Optional distributed tracing and observability platform."
print_status "Sends traces and metrics to an external backend like SigNoz for debugging and monitoring."
print_status "Deploy SigNoz separately: https://signoz.io/docs/install/kubernetes/"
# Use a meaningful current value for the check
local current_otel_config=""
if [ -n "$OTEL_ENDPOINT" ] && [ "$OTEL_ENDPOINT" != "" ] && [ "$OTEL_ENDPOINT" != "<your-otel-endpoint>" ]; then
current_otel_config="$OTEL_ENDPOINT"
fi
# Enable local SigNoz deployment by default for quickstart
portable_sed "s|.*export DEPLOY_SIGNOZ=.*|export DEPLOY_SIGNOZ=true|" deployment/env
print_success "Local SigNoz deployment enabled for observability"
configure_service "OTEL" "OpenTelemetry/SigNoz configuration" "$current_otel_config" "" false "configure_otel_wrapper"
}
# Function to check configuration file
@@ -772,10 +770,9 @@ check_aks_config() {
done
fi
# Check optional SigNoz/OTEL configuration
if [ "$DEPLOY_SIGNOZ" = "true" ]; then
print_status "SigNoz local deployment is enabled"
elif [ -n "$OTEL_ENDPOINT" ] && [ "$OTEL_ENDPOINT" != "" ]; then
# Check optional OTEL configuration
if [ -n "$OTEL_ENDPOINT" ] && [ "$OTEL_ENDPOINT" != "" ]; then
print_status "External OTEL endpoint is configured"
if [ -z "$OTEL_PROTOCOL" ] || [ "$OTEL_PROTOCOL" = "<your-*>" ]; then
print_error "OTEL_PROTOCOL is not set when OTEL_ENDPOINT is configured"
errors=$((errors + 1))
scripts/setup-azure.sh
+4 -4
View File
@@ -151,12 +151,12 @@ setup_config() {
# Configure LLM Budget
configure_llm_budget
# Configure LangFuse (optional)
configure_langfuse
# TODO: For production, we don't deploy SigNoz, as it needs to be done properly
# Just allow to configure access to an existing SigNoz instance
# Configure OpenTelemetry / SigNoz (optional)
configure_otel
}
# Function to validate configuration
scripts/setup-local.sh
+2 -2
View File
@@ -32,8 +32,8 @@ setup_config() {
# Configure LangFuse (optional)
configure_langfuse
# Configure SigNoz deployment (optional)
# Configure OpenTelemetry (optional)
configure_otel
}