deployment: push logs to signoz (#591)

* Save logs in SigNoz

* deployment: push logs to signoz

* fix removal

* gitignore node_data_storage

* deployment: print SERVICE_INSTANCE_ID

* ci: configure otel

.github/ci-env.template

@@ -11,6 +11,8 @@ export CRS_KEY_TOKEN_HASH='$argon2id$v=19$m=65536,t=3,p=4$Dg1v6NPGTyXPoOPF4ozD5A
 export LITELLM_MASTER_KEY="06beca4d4debbb6bd0933cddd28a3f60"
 export AZURE_API_BASE="ignore-me"
 export AZURE_API_KEY="ignore-me"
+export OTEL_ENDPOINT="https://aixcc-crone-bronx-cuprous-wiggly.gateway.trailofbits.com"
+export OTEL_PROTOCOL="http"
 
 # Secrets/Env replaced
 export BUTTERCUP_NAMESPACE=replace-me

.gitignore

@@ -49,3 +49,4 @@ common/common/datastructures/fuzzer_msg*
 
 crs_scratch/*
 tasks_storage/*
+node_data_storage/*

common/src/buttercup/common/challenge_task_cli.py

@@ -115,7 +115,7 @@ def get_task_copy(task: ChallengeTask, use_copy: bool = False) -> Iterator[Chall
 
 def main():
     settings = Settings()
-    setup_package_logger(__name__, "DEBUG")
+    setup_package_logger("challenge-task-cli", __name__, "DEBUG")
     if settings.rw:
         task = ChallengeTask(
             read_only_task_dir=settings.task_dir,

common/src/buttercup/common/logger.py

@@ -1,12 +1,23 @@
-import logging
+from opentelemetry._logs import set_logger_provider
 import os
+
+if os.environ.get("OTEL_EXPORTER_OTLP_PROTOCOL") == "grpc":
+    from opentelemetry.exporter.otlp.proto.grpc._log_exporter import OTLPLogExporter
+else:
+    from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
+
+from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
+from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
+from opentelemetry.sdk.resources import Resource
+import uuid
+import logging
 import tempfile
 
 _is_initialized = False
 PACKAGE_LOGGER_NAME = "buttercup"
 
 
-def setup_package_logger(logger_name: str, log_level: str = "info") -> logging.Logger:
+def setup_package_logger(application_name: str, logger_name: str, log_level: str = "info") -> logging.Logger:
     global _is_initialized
 
     if not _is_initialized:
@@ -16,6 +27,27 @@ def setup_package_logger(logger_name: str, log_level: str = "info") -> logging.L
             for handler in root.handlers:
                 root.removeHandler(handler)
 
+        # Create resource with service and environment information
+        resource = Resource.create(
+            attributes={
+                "service.name": application_name,
+                "service.instance.id": os.getenv("SERVICE_INSTANCE_ID", str(uuid.uuid4())),
+            }
+        )
+
+        # Initialize the LoggerProvider with the created resource.
+        logger_provider = LoggerProvider(resource=resource)
+
+        # Configure the span exporter and processor based on whether the endpoint is effectively set.
+        otlp_handler = None
+        if os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT"):
+            set_logger_provider(logger_provider)
+            exporter = OTLPLogExporter()
+
+            # add the batch processors to the trace provider
+            logger_provider.add_log_record_processor(BatchLogRecordProcessor(exporter))
+            otlp_handler = LoggingHandler(level=logging.DEBUG, logger_provider=logger_provider)
+
         persistent_log_dir = os.getenv("PERSISTENT_LOG_DIR", None)
 
         handlers = [
@@ -28,6 +60,9 @@ def setup_package_logger(logger_name: str, log_level: str = "info") -> logging.L
 
             handlers.append(logging.FileHandler(os.path.join(persistent_log_dir, f"{logger_name}.log")))
 
+        if otlp_handler:
+            handlers.append(otlp_handler)
+
         # Configure root logger
         logging.basicConfig(
             format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",

common/src/buttercup/common/util_cli.py

@@ -85,7 +85,7 @@ class Settings(BaseSettings):
 
 def main():
     settings = Settings()
-    setup_package_logger(__name__, settings.log_level)
+    setup_package_logger("util-cli", __name__, settings.log_level)
 
     redis = Redis.from_url(settings.redis_url, decode_responses=False)
     command = get_subcommand(settings)

deployment/crs-architecture.sh

@@ -107,6 +107,16 @@ up() {
 		--from-literal=username="$GHCR_USERNAME" \
 		--from-literal=scantron_github_pat="$SCANTRON_GITHUB_PAT" || echo -e "${GRN}ghcr secret already exists${NC}"
 
+	echo -e "${BLU}Creating SERVICE_INSTANCE_ID${NC}"
+	SERVICE_INSTANCE_ID=$(echo $RANDOM | md5sum | head -c 20)
+	kubectl create configmap service-instance-id \
+		--namespace "$BUTTERCUP_NAMESPACE" \
+		--from-literal=service-instance-id="$SERVICE_INSTANCE_ID" || echo -e "${GRN}service-instance-id configmap already exists${NC}"
+
+	SERVICE_INSTANCE_ID=$(kubectl get configmap service-instance-id \
+		--namespace "$BUTTERCUP_NAMESPACE" \
+		-o jsonpath='{.data.service-instance-id}')
+	echo -e "${GRN}SERVICE_INSTANCE_ID is $SERVICE_INSTANCE_ID${NC}"
 
 	kubectl create secret docker-registry docker-auth \
 		--namespace "$BUTTERCUP_NAMESPACE" \

deployment/k8s/templates/common-env.yaml

@@ -88,6 +88,11 @@ Define Docker Host environment variable for Unix socket
   value: "Authorization=Basic {{ .Values.global.otel.token }}"
 - name: OTEL_EXPORTER_OTLP_PROTOCOL
   value: "{{ .Values.global.otel.protocol | default "grpc" }}"
+- name: SERVICE_INSTANCE_ID
+  valueFrom:
+    configMapKeyRef:
+      name: service-instance-id
+      key: service-instance-id
 {{- end }}
 
 {{- define "buttercup.env.timeouts" }}

deployment/k8s/values-ci.template

@@ -31,6 +31,7 @@ global:
   otel:
     endpoint: "${OTEL_ENDPOINT}"
     token: "${OTEL_TOKEN}"
+    protocol: "${OTEL_PROTOCOL}"
 
 volumes:
   tasks_storage:

fuzzer/src/buttercup/fuzzing_infra/builder_bot.py

@@ -15,7 +15,7 @@ logger = logging.getLogger(__name__)
 
 def main():
     args = BuilderBotSettings()
-    setup_package_logger(__name__, args.log_level)
+    setup_package_logger("builder-bot", __name__, args.log_level)
 
     logger.info(f"Starting builder bot ({args.wdir})")
     redis = Redis.from_url(args.redis_url)

fuzzer/src/buttercup/fuzzing_infra/corpus_merger.py

@@ -102,7 +102,7 @@ class MergerBot(TaskLoop):
 
 def main():
     args = FuzzerBotSettings()
-    setup_package_logger(__name__, args.log_level)
+    setup_package_logger("corpus-merger", __name__, args.log_level)
 
     os.makedirs(args.wdir, exist_ok=True)
     logger.info(f"Starting fuzzer (wdir: {args.wdir} crs_scratch_dir: {args.crs_scratch_dir})")

fuzzer/src/buttercup/fuzzing_infra/corpus_sync.py

@@ -18,7 +18,7 @@ def main():
     parser.add_argument("--redis-url", type=str, required=True)
     args = parser.parse_args()
 
-    setup_package_logger(__name__, args.log_level)
+    setup_package_logger("corpus-sync", __name__, args.log_level)
     last_sync = 0
     logger.info(f"Syncing corpora from {args.wdir}")
     redis = Redis.from_url(args.redis_url)

fuzzer/src/buttercup/fuzzing_infra/coverage_bot.py

@@ -194,7 +194,7 @@ class CoverageBot(TaskLoop):
 def main():
     args = CoverageBotSettings()
 
-    setup_package_logger(__name__, args.log_level)
+    setup_package_logger("coverage-bot", __name__, args.log_level)
 
     os.makedirs(args.wdir, exist_ok=True)
     logger.info(f"Starting coverage bot (wdir: {args.wdir})")

fuzzer/src/buttercup/fuzzing_infra/fuzzer_bot.py

@@ -114,7 +114,7 @@ class FuzzerBot(TaskLoop):
 
 def main():
     args = FuzzerBotSettings()
-    setup_package_logger(__name__, args.log_level)
+    setup_package_logger("fuzzer-bot", __name__, args.log_level)
     init_telemetry("fuzzer")
     os.makedirs(args.wdir, exist_ok=True)
     logger.info(f"Starting fuzzer (wdir: {args.wdir} crs_scratch_dir: {args.crs_scratch_dir})")

fuzzer/src/buttercup/fuzzing_infra/orchestrator.py

@@ -15,7 +15,7 @@ from buttercup.common.clusterfuzz_utils import get_fuzz_targets
 import os
 from buttercup.common.logger import setup_package_logger
 
-logger = setup_package_logger(__name__)
+logger = setup_package_logger("fuzzer-orchestrator", __name__)
 DEFAULT_WEIGHT = 1.0
 
 

fuzzer/src/buttercup/fuzzing_infra/runner.py

@@ -61,7 +61,7 @@ def main():
     prsr.add_argument("target")
     args = prsr.parse_args()
 
-    setup_package_logger(__name__, "DEBUG")
+    setup_package_logger("fuzzer-runner", __name__, "DEBUG")
 
     conf = Conf(args.timeout)
     fuzzconf = FuzzConfiguration(args.corpusdir, args.target, args.engine, args.sanitizer)

fuzzer/src/buttercup/fuzzing_infra/tracer_bot.py

@@ -80,7 +80,7 @@ class TracerBot:
 
 def main():
     args = TracerSettings()
-    setup_package_logger(__name__, "DEBUG")
+    setup_package_logger("tracer-bot", __name__, "DEBUG")
 
     os.makedirs(args.wdir, exist_ok=True)
     logger.info(f"Starting tracer-bot (wdir: {args.wdir})")

orchestrator/src/buttercup/orchestrator/downloader/__cli__.py

@@ -50,7 +50,7 @@ def prepare_task(command: DownloaderProcessCommand, session: requests.Session) -
 
 def main():
     settings = DownloaderSettings()
-    setup_package_logger(__name__, settings.log_level)
+    setup_package_logger("task-downloader", __name__, settings.log_level)
     command = get_subcommand(settings)
     if isinstance(command, DownloaderServeCommand):
         redis = Redis.from_url(command.redis_url, decode_responses=False)

orchestrator/src/buttercup/orchestrator/scheduler/__cli__.py

@@ -34,7 +34,7 @@ def _prepare_build_output(command: ProcessBuildOutputCommand) -> BuildOutput:
 
 def main():
     settings = Settings()
-    setup_package_logger(__name__, settings.log_level)
+    setup_package_logger("scheduler", __name__, settings.log_level)
     logger.debug(f"Settings: {settings}")
     command = get_subcommand(settings)
     if isinstance(command, ServeCommand):

orchestrator/src/buttercup/orchestrator/task_server/server.py

@@ -32,7 +32,7 @@ from buttercup.orchestrator.competition_api_client.api.ping_api import PingApi
 from buttercup.orchestrator.competition_api_client.models.types_ping_response import TypesPingResponse
 
 settings = get_settings()
-logger = setup_package_logger(__name__, settings.log_level)
+logger = setup_package_logger("task-server", __name__, settings.log_level)
 logger.info("Redis URL: %s", settings.redis_url)
 logger.info("Competition API URL: %s", settings.competition_api_url)
 logger.info("Competition API Key ID: %s", settings.competition_api_username)

patcher/src/buttercup/patcher/__cli__.py

@@ -18,7 +18,7 @@ logger = logging.getLogger(__name__)
 def main() -> None:
     settings = Settings()
     command = get_subcommand(settings)
-    setup_package_logger(__name__, settings.log_level)
+    setup_package_logger("patcher", __name__, settings.log_level)
 
     logger.info("Starting patcher")
     logger.debug("Settings: %s", settings)

program-model/src/buttercup/program_model/__cli__.py

@@ -26,7 +26,7 @@ def prepare_task(command: ProcessCommand) -> IndexRequest:
 def main() -> None:
     settings = Settings()
     command = get_subcommand(settings)
-    setup_package_logger(__name__, settings.log_level)
+    setup_package_logger("program-model", __name__, settings.log_level)
     if isinstance(command, ServeCommand):
         init_telemetry("program-model")  # type: ignore[unreachable]
         redis = Redis.from_url(command.redis_url, decode_responses=False)

seed-gen/eval/vuln_discovery_base.py

@@ -28,7 +28,7 @@ from buttercup.common.logger import setup_package_logger
 from buttercup.seed_gen.sandbox.sandbox import sandbox_exec_funcs
 from buttercup.seed_gen.utils import extract_md
 
-logger = setup_package_logger(__name__, "DEBUG")
+logger = setup_package_logger("vuln-discovery-base", __name__, "DEBUG")
 
 
 @dataclass

seed-gen/src/buttercup/seed_gen/_cli.py

@@ -49,6 +49,6 @@ def main() -> None:
         help="Maximum number of crashes in the crash dir for a single token",
     )
     args = parser.parse_args()
-    setup_package_logger(__name__, os.getenv("LOG_LEVEL", "INFO").upper())
+    setup_package_logger("seed-gen", __name__, os.getenv("LOG_LEVEL", "INFO").upper())
     if args.command == "server":
         command_server(args)