From d72887d630f0d03f950c640abba8f25af650c8a0 Mon Sep 17 00:00:00 2001
From: Riccardo Schirone <562321+ret2libc@users.noreply.github.com>
Date: Tue, 13 May 2025 18:16:08 +0200
Subject: [PATCH] deployment: push logs to signoz (#591)

* Save logs in SigNoz

* deployment: push logs to signoz

* fix removal

* gitignore node_data_storage

* deployment: print SERVICE_INSTANCE_ID

* ci: configure otel
---
 .github/ci-env.template                       |  2 +
 .gitignore                                    |  1 +
 .../buttercup/common/challenge_task_cli.py    |  2 +-
 common/src/buttercup/common/logger.py         | 39 ++++++++++++++++++-
 common/src/buttercup/common/util_cli.py       |  2 +-
 deployment/crs-architecture.sh                | 10 +++++
 deployment/k8s/templates/common-env.yaml      |  5 +++
 deployment/k8s/values-ci.template             |  1 +
 .../buttercup/fuzzing_infra/builder_bot.py    |  2 +-
 .../buttercup/fuzzing_infra/corpus_merger.py  |  2 +-
 .../buttercup/fuzzing_infra/corpus_sync.py    |  2 +-
 .../buttercup/fuzzing_infra/coverage_bot.py   |  2 +-
 .../src/buttercup/fuzzing_infra/fuzzer_bot.py |  2 +-
 .../buttercup/fuzzing_infra/orchestrator.py   |  2 +-
 fuzzer/src/buttercup/fuzzing_infra/runner.py  |  2 +-
 .../src/buttercup/fuzzing_infra/tracer_bot.py |  2 +-
 .../orchestrator/downloader/__cli__.py        |  2 +-
 .../orchestrator/scheduler/__cli__.py         |  2 +-
 .../orchestrator/task_server/server.py        |  2 +-
 patcher/src/buttercup/patcher/__cli__.py      |  2 +-
 .../src/buttercup/program_model/__cli__.py    |  2 +-
 seed-gen/eval/vuln_discovery_base.py          |  2 +-
 seed-gen/src/buttercup/seed_gen/_cli.py       |  2 +-
 23 files changed, 73 insertions(+), 19 deletions(-)

diff --git a/.github/ci-env.template b/.github/ci-env.template
index 8b71ee05..d2421da4 100644
--- a/.github/ci-env.template
+++ b/.github/ci-env.template
@@ -11,6 +11,8 @@ export CRS_KEY_TOKEN_HASH='$argon2id$v=19$m=65536,t=3,p=4$Dg1v6NPGTyXPoOPF4ozD5A
 export LITELLM_MASTER_KEY="06beca4d4debbb6bd0933cddd28a3f60"
 export AZURE_API_BASE="ignore-me"
 export AZURE_API_KEY="ignore-me"
+export OTEL_ENDPOINT="https://aixcc-crone-bronx-cuprous-wiggly.gateway.trailofbits.com"
+export OTEL_PROTOCOL="http"
 
 # Secrets/Env replaced
 export BUTTERCUP_NAMESPACE=replace-me
diff --git a/.gitignore b/.gitignore
index 9ad84e13..b2f89a0c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -49,3 +49,4 @@ common/common/datastructures/fuzzer_msg*
 
 crs_scratch/*
 tasks_storage/*
+node_data_storage/*
diff --git a/common/src/buttercup/common/challenge_task_cli.py b/common/src/buttercup/common/challenge_task_cli.py
index 3b99237a..1448e163 100644
--- a/common/src/buttercup/common/challenge_task_cli.py
+++ b/common/src/buttercup/common/challenge_task_cli.py
@@ -115,7 +115,7 @@ def get_task_copy(task: ChallengeTask, use_copy: bool = False) -> Iterator[Chall
 
 def main():
     settings = Settings()
-    setup_package_logger(__name__, "DEBUG")
+    setup_package_logger("challenge-task-cli", __name__, "DEBUG")
     if settings.rw:
         task = ChallengeTask(
             read_only_task_dir=settings.task_dir,
diff --git a/common/src/buttercup/common/logger.py b/common/src/buttercup/common/logger.py
index bc9f003f..ed04e396 100644
--- a/common/src/buttercup/common/logger.py
+++ b/common/src/buttercup/common/logger.py
@@ -1,12 +1,23 @@
-import logging
+from opentelemetry._logs import set_logger_provider
 import os
+
+if os.environ.get("OTEL_EXPORTER_OTLP_PROTOCOL") == "grpc":
+    from opentelemetry.exporter.otlp.proto.grpc._log_exporter import OTLPLogExporter
+else:
+    from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
+
+from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
+from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
+from opentelemetry.sdk.resources import Resource
+import uuid
+import logging
 import tempfile
 
 _is_initialized = False
 PACKAGE_LOGGER_NAME = "buttercup"
 
 
-def setup_package_logger(logger_name: str, log_level: str = "info") -> logging.Logger:
+def setup_package_logger(application_name: str, logger_name: str, log_level: str = "info") -> logging.Logger:
     global _is_initialized
 
     if not _is_initialized:
@@ -16,6 +27,27 @@ def setup_package_logger(logger_name: str, log_level: str = "info") -> logging.L
             for handler in root.handlers:
                 root.removeHandler(handler)
 
+        # Create resource with service and environment information
+        resource = Resource.create(
+            attributes={
+                "service.name": application_name,
+                "service.instance.id": os.getenv("SERVICE_INSTANCE_ID", str(uuid.uuid4())),
+            }
+        )
+
+        # Initialize the LoggerProvider with the created resource.
+        logger_provider = LoggerProvider(resource=resource)
+
+        # Configure the span exporter and processor based on whether the endpoint is effectively set.
+        otlp_handler = None
+        if os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT"):
+            set_logger_provider(logger_provider)
+            exporter = OTLPLogExporter()
+
+            # add the batch processors to the trace provider
+            logger_provider.add_log_record_processor(BatchLogRecordProcessor(exporter))
+            otlp_handler = LoggingHandler(level=logging.DEBUG, logger_provider=logger_provider)
+
         persistent_log_dir = os.getenv("PERSISTENT_LOG_DIR", None)
 
         handlers = [
@@ -28,6 +60,9 @@ def setup_package_logger(logger_name: str, log_level: str = "info") -> logging.L
 
             handlers.append(logging.FileHandler(os.path.join(persistent_log_dir, f"{logger_name}.log")))
 
+        if otlp_handler:
+            handlers.append(otlp_handler)
+
         # Configure root logger
         logging.basicConfig(
             format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
diff --git a/common/src/buttercup/common/util_cli.py b/common/src/buttercup/common/util_cli.py
index 406622c1..11f06e1b 100644
--- a/common/src/buttercup/common/util_cli.py
+++ b/common/src/buttercup/common/util_cli.py
@@ -85,7 +85,7 @@ class Settings(BaseSettings):
 
 def main():
     settings = Settings()
-    setup_package_logger(__name__, settings.log_level)
+    setup_package_logger("util-cli", __name__, settings.log_level)
 
     redis = Redis.from_url(settings.redis_url, decode_responses=False)
     command = get_subcommand(settings)
diff --git a/deployment/crs-architecture.sh b/deployment/crs-architecture.sh
index 63897fe5..7454bd2a 100755
--- a/deployment/crs-architecture.sh
+++ b/deployment/crs-architecture.sh
@@ -107,6 +107,16 @@ up() {
 		--from-literal=username="$GHCR_USERNAME" \
 		--from-literal=scantron_github_pat="$SCANTRON_GITHUB_PAT" || echo -e "${GRN}ghcr secret already exists${NC}"
 
+	echo -e "${BLU}Creating SERVICE_INSTANCE_ID${NC}"
+	SERVICE_INSTANCE_ID=$(echo $RANDOM | md5sum | head -c 20)
+	kubectl create configmap service-instance-id \
+		--namespace "$BUTTERCUP_NAMESPACE" \
+		--from-literal=service-instance-id="$SERVICE_INSTANCE_ID" || echo -e "${GRN}service-instance-id configmap already exists${NC}"
+
+	SERVICE_INSTANCE_ID=$(kubectl get configmap service-instance-id \
+		--namespace "$BUTTERCUP_NAMESPACE" \
+		-o jsonpath='{.data.service-instance-id}')
+	echo -e "${GRN}SERVICE_INSTANCE_ID is $SERVICE_INSTANCE_ID${NC}"
 
 	kubectl create secret docker-registry docker-auth \
 		--namespace "$BUTTERCUP_NAMESPACE" \
diff --git a/deployment/k8s/templates/common-env.yaml b/deployment/k8s/templates/common-env.yaml
index 97bb23df..daa9b7dd 100644
--- a/deployment/k8s/templates/common-env.yaml
+++ b/deployment/k8s/templates/common-env.yaml
@@ -88,6 +88,11 @@ Define Docker Host environment variable for Unix socket
   value: "Authorization=Basic {{ .Values.global.otel.token }}"
 - name: OTEL_EXPORTER_OTLP_PROTOCOL
   value: "{{ .Values.global.otel.protocol | default "grpc" }}"
+- name: SERVICE_INSTANCE_ID
+  valueFrom:
+    configMapKeyRef:
+      name: service-instance-id
+      key: service-instance-id
 {{- end }}
 
 {{- define "buttercup.env.timeouts" }}
diff --git a/deployment/k8s/values-ci.template b/deployment/k8s/values-ci.template
index 1e1f1016..e71fb5d9 100644
--- a/deployment/k8s/values-ci.template
+++ b/deployment/k8s/values-ci.template
@@ -31,6 +31,7 @@ global:
   otel:
     endpoint: "${OTEL_ENDPOINT}"
     token: "${OTEL_TOKEN}"
+    protocol: "${OTEL_PROTOCOL}"
 
 volumes:
   tasks_storage:
diff --git a/fuzzer/src/buttercup/fuzzing_infra/builder_bot.py b/fuzzer/src/buttercup/fuzzing_infra/builder_bot.py
index 4ac99dc2..15b662fa 100644
--- a/fuzzer/src/buttercup/fuzzing_infra/builder_bot.py
+++ b/fuzzer/src/buttercup/fuzzing_infra/builder_bot.py
@@ -15,7 +15,7 @@ logger = logging.getLogger(__name__)
 
 def main():
     args = BuilderBotSettings()
-    setup_package_logger(__name__, args.log_level)
+    setup_package_logger("builder-bot", __name__, args.log_level)
 
     logger.info(f"Starting builder bot ({args.wdir})")
     redis = Redis.from_url(args.redis_url)
diff --git a/fuzzer/src/buttercup/fuzzing_infra/corpus_merger.py b/fuzzer/src/buttercup/fuzzing_infra/corpus_merger.py
index afad3ac0..d2776356 100644
--- a/fuzzer/src/buttercup/fuzzing_infra/corpus_merger.py
+++ b/fuzzer/src/buttercup/fuzzing_infra/corpus_merger.py
@@ -102,7 +102,7 @@ class MergerBot(TaskLoop):
 
 def main():
     args = FuzzerBotSettings()
-    setup_package_logger(__name__, args.log_level)
+    setup_package_logger("corpus-merger", __name__, args.log_level)
 
     os.makedirs(args.wdir, exist_ok=True)
     logger.info(f"Starting fuzzer (wdir: {args.wdir} crs_scratch_dir: {args.crs_scratch_dir})")
diff --git a/fuzzer/src/buttercup/fuzzing_infra/corpus_sync.py b/fuzzer/src/buttercup/fuzzing_infra/corpus_sync.py
index 40294724..4cbb6939 100644
--- a/fuzzer/src/buttercup/fuzzing_infra/corpus_sync.py
+++ b/fuzzer/src/buttercup/fuzzing_infra/corpus_sync.py
@@ -18,7 +18,7 @@ def main():
     parser.add_argument("--redis-url", type=str, required=True)
     args = parser.parse_args()
 
-    setup_package_logger(__name__, args.log_level)
+    setup_package_logger("corpus-sync", __name__, args.log_level)
     last_sync = 0
     logger.info(f"Syncing corpora from {args.wdir}")
     redis = Redis.from_url(args.redis_url)
diff --git a/fuzzer/src/buttercup/fuzzing_infra/coverage_bot.py b/fuzzer/src/buttercup/fuzzing_infra/coverage_bot.py
index 4221c344..2521a5af 100644
--- a/fuzzer/src/buttercup/fuzzing_infra/coverage_bot.py
+++ b/fuzzer/src/buttercup/fuzzing_infra/coverage_bot.py
@@ -194,7 +194,7 @@ class CoverageBot(TaskLoop):
 def main():
     args = CoverageBotSettings()
 
-    setup_package_logger(__name__, args.log_level)
+    setup_package_logger("coverage-bot", __name__, args.log_level)
 
     os.makedirs(args.wdir, exist_ok=True)
     logger.info(f"Starting coverage bot (wdir: {args.wdir})")
diff --git a/fuzzer/src/buttercup/fuzzing_infra/fuzzer_bot.py b/fuzzer/src/buttercup/fuzzing_infra/fuzzer_bot.py
index 8e59dcaf..89d3c8ca 100644
--- a/fuzzer/src/buttercup/fuzzing_infra/fuzzer_bot.py
+++ b/fuzzer/src/buttercup/fuzzing_infra/fuzzer_bot.py
@@ -114,7 +114,7 @@ class FuzzerBot(TaskLoop):
 
 def main():
     args = FuzzerBotSettings()
-    setup_package_logger(__name__, args.log_level)
+    setup_package_logger("fuzzer-bot", __name__, args.log_level)
     init_telemetry("fuzzer")
     os.makedirs(args.wdir, exist_ok=True)
     logger.info(f"Starting fuzzer (wdir: {args.wdir} crs_scratch_dir: {args.crs_scratch_dir})")
diff --git a/fuzzer/src/buttercup/fuzzing_infra/orchestrator.py b/fuzzer/src/buttercup/fuzzing_infra/orchestrator.py
index 0c2d9e6c..8de6eb63 100644
--- a/fuzzer/src/buttercup/fuzzing_infra/orchestrator.py
+++ b/fuzzer/src/buttercup/fuzzing_infra/orchestrator.py
@@ -15,7 +15,7 @@ from buttercup.common.clusterfuzz_utils import get_fuzz_targets
 import os
 from buttercup.common.logger import setup_package_logger
 
-logger = setup_package_logger(__name__)
+logger = setup_package_logger("fuzzer-orchestrator", __name__)
 DEFAULT_WEIGHT = 1.0
 
 
diff --git a/fuzzer/src/buttercup/fuzzing_infra/runner.py b/fuzzer/src/buttercup/fuzzing_infra/runner.py
index 513c2b6b..388f645d 100644
--- a/fuzzer/src/buttercup/fuzzing_infra/runner.py
+++ b/fuzzer/src/buttercup/fuzzing_infra/runner.py
@@ -61,7 +61,7 @@ def main():
     prsr.add_argument("target")
     args = prsr.parse_args()
 
-    setup_package_logger(__name__, "DEBUG")
+    setup_package_logger("fuzzer-runner", __name__, "DEBUG")
 
     conf = Conf(args.timeout)
     fuzzconf = FuzzConfiguration(args.corpusdir, args.target, args.engine, args.sanitizer)
diff --git a/fuzzer/src/buttercup/fuzzing_infra/tracer_bot.py b/fuzzer/src/buttercup/fuzzing_infra/tracer_bot.py
index 71fd41e7..55136dc0 100644
--- a/fuzzer/src/buttercup/fuzzing_infra/tracer_bot.py
+++ b/fuzzer/src/buttercup/fuzzing_infra/tracer_bot.py
@@ -80,7 +80,7 @@ class TracerBot:
 
 def main():
     args = TracerSettings()
-    setup_package_logger(__name__, "DEBUG")
+    setup_package_logger("tracer-bot", __name__, "DEBUG")
 
     os.makedirs(args.wdir, exist_ok=True)
     logger.info(f"Starting tracer-bot (wdir: {args.wdir})")
diff --git a/orchestrator/src/buttercup/orchestrator/downloader/__cli__.py b/orchestrator/src/buttercup/orchestrator/downloader/__cli__.py
index 805f9b1b..1aea5a50 100644
--- a/orchestrator/src/buttercup/orchestrator/downloader/__cli__.py
+++ b/orchestrator/src/buttercup/orchestrator/downloader/__cli__.py
@@ -50,7 +50,7 @@ def prepare_task(command: DownloaderProcessCommand, session: requests.Session) -
 
 def main():
     settings = DownloaderSettings()
-    setup_package_logger(__name__, settings.log_level)
+    setup_package_logger("task-downloader", __name__, settings.log_level)
     command = get_subcommand(settings)
     if isinstance(command, DownloaderServeCommand):
         redis = Redis.from_url(command.redis_url, decode_responses=False)
diff --git a/orchestrator/src/buttercup/orchestrator/scheduler/__cli__.py b/orchestrator/src/buttercup/orchestrator/scheduler/__cli__.py
index a3945eac..b76990c5 100644
--- a/orchestrator/src/buttercup/orchestrator/scheduler/__cli__.py
+++ b/orchestrator/src/buttercup/orchestrator/scheduler/__cli__.py
@@ -34,7 +34,7 @@ def _prepare_build_output(command: ProcessBuildOutputCommand) -> BuildOutput:
 
 def main():
     settings = Settings()
-    setup_package_logger(__name__, settings.log_level)
+    setup_package_logger("scheduler", __name__, settings.log_level)
     logger.debug(f"Settings: {settings}")
     command = get_subcommand(settings)
     if isinstance(command, ServeCommand):
diff --git a/orchestrator/src/buttercup/orchestrator/task_server/server.py b/orchestrator/src/buttercup/orchestrator/task_server/server.py
index 40cd46b6..d67f7cd3 100644
--- a/orchestrator/src/buttercup/orchestrator/task_server/server.py
+++ b/orchestrator/src/buttercup/orchestrator/task_server/server.py
@@ -32,7 +32,7 @@ from buttercup.orchestrator.competition_api_client.api.ping_api import PingApi
 from buttercup.orchestrator.competition_api_client.models.types_ping_response import TypesPingResponse
 
 settings = get_settings()
-logger = setup_package_logger(__name__, settings.log_level)
+logger = setup_package_logger("task-server", __name__, settings.log_level)
 logger.info("Redis URL: %s", settings.redis_url)
 logger.info("Competition API URL: %s", settings.competition_api_url)
 logger.info("Competition API Key ID: %s", settings.competition_api_username)
diff --git a/patcher/src/buttercup/patcher/__cli__.py b/patcher/src/buttercup/patcher/__cli__.py
index 188b485b..7ffee5fa 100644
--- a/patcher/src/buttercup/patcher/__cli__.py
+++ b/patcher/src/buttercup/patcher/__cli__.py
@@ -18,7 +18,7 @@ logger = logging.getLogger(__name__)
 def main() -> None:
     settings = Settings()
     command = get_subcommand(settings)
-    setup_package_logger(__name__, settings.log_level)
+    setup_package_logger("patcher", __name__, settings.log_level)
 
     logger.info("Starting patcher")
     logger.debug("Settings: %s", settings)
diff --git a/program-model/src/buttercup/program_model/__cli__.py b/program-model/src/buttercup/program_model/__cli__.py
index 0ecfde2d..6a79b86a 100644
--- a/program-model/src/buttercup/program_model/__cli__.py
+++ b/program-model/src/buttercup/program_model/__cli__.py
@@ -26,7 +26,7 @@ def prepare_task(command: ProcessCommand) -> IndexRequest:
 def main() -> None:
     settings = Settings()
     command = get_subcommand(settings)
-    setup_package_logger(__name__, settings.log_level)
+    setup_package_logger("program-model", __name__, settings.log_level)
     if isinstance(command, ServeCommand):
         init_telemetry("program-model")  # type: ignore[unreachable]
         redis = Redis.from_url(command.redis_url, decode_responses=False)
diff --git a/seed-gen/eval/vuln_discovery_base.py b/seed-gen/eval/vuln_discovery_base.py
index e8209f14..27367d25 100644
--- a/seed-gen/eval/vuln_discovery_base.py
+++ b/seed-gen/eval/vuln_discovery_base.py
@@ -28,7 +28,7 @@ from buttercup.common.logger import setup_package_logger
 from buttercup.seed_gen.sandbox.sandbox import sandbox_exec_funcs
 from buttercup.seed_gen.utils import extract_md
 
-logger = setup_package_logger(__name__, "DEBUG")
+logger = setup_package_logger("vuln-discovery-base", __name__, "DEBUG")
 
 
 @dataclass
diff --git a/seed-gen/src/buttercup/seed_gen/_cli.py b/seed-gen/src/buttercup/seed_gen/_cli.py
index 45f017ed..45f4b6e6 100644
--- a/seed-gen/src/buttercup/seed_gen/_cli.py
+++ b/seed-gen/src/buttercup/seed_gen/_cli.py
@@ -49,6 +49,6 @@ def main() -> None:
         help="Maximum number of crashes in the crash dir for a single token",
     )
     args = parser.parse_args()
-    setup_package_logger(__name__, os.getenv("LOG_LEVEL", "INFO").upper())
+    setup_package_logger("seed-gen", __name__, os.getenv("LOG_LEVEL", "INFO").upper())
     if args.command == "server":
         command_server(args)