diff --git a/compose.yaml b/compose.yaml index c7cbe4c..72bc84e 100644 --- a/compose.yaml +++ b/compose.yaml @@ -84,7 +84,12 @@ services: - MINIO_ROOT_PASSWORD=${MINIO_ROOT_PASSWORD:?} - MINIO_ROOT_USER=${MINIO_ROOT_USER:?} - RABBITMQ_CONNECTION_STRING=amqp://${RABBITMQ_USER}:${RABBITMQ_PASSWORD}@rabbitmq:5672 - - POSTGRES_CONNECTION_STRING=postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/enrichment?sslmode=disable + - POSTGRES_USER=${POSTGRES_USER:?} + - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:?} + - POSTGRES_HOST=${POSTGRES_HOST:-postgres} + - POSTGRES_PORT=${POSTGRES_PORT:-5432} + - POSTGRES_DB=${POSTGRES_DB:-enrichment} + - POSTGRES_PARAMETERS=${POSTGRES_PARAMETERS:-sslmode=disable} - RABBITMQ_PASSWORD=${RABBITMQ_PASSWORD:?} - RABBITMQ_USER=${RABBITMQ_USER:?} depends_on: @@ -277,7 +282,12 @@ services: - ./infra/dapr/components/workflowstate.yaml:/dapr/components/workflowstate.yaml:ro - ./infra/dapr/configuration/config_monitoring_${NEMESIS_MONITORING:-disabled}.yaml:/dapr/configuration/config.yaml:ro environment: - - POSTGRES_CONNECTION_STRING=postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/enrichment?sslmode=disable + - POSTGRES_USER=${POSTGRES_USER:?} + - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:?} + - POSTGRES_HOST=${POSTGRES_HOST:-postgres} + - POSTGRES_PORT=${POSTGRES_PORT:-5432} + - POSTGRES_DB=${POSTGRES_DB:-enrichment} + - POSTGRES_PARAMETERS=${POSTGRES_PARAMETERS:-sslmode=disable} - RABBITMQ_CONNECTION_STRING=amqp://${RABBITMQ_USER}:${RABBITMQ_PASSWORD}@rabbitmq:5672 - MINIO_ROOT_USER=${MINIO_ROOT_USER:?} - MINIO_ROOT_PASSWORD=${MINIO_ROOT_PASSWORD:?} @@ -413,7 +423,7 @@ services: # Phoenix LLM tracing configuration - PHOENIX_ENABLED=${PHOENIX_ENABLED:-false} - PHOENIX_ENDPOINT=${PHOENIX_ENDPOINT:-http://phoenix:6006/v1/traces} - - PHOENIX_SQL_DATABASE_URL=postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/phoenix + - PHOENIX_SQL_DATABASE_URL=postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST:-postgres}:${POSTGRES_PORT:-5432}/phoenix - NEMESIS_MONITORING=${NEMESIS_MONITORING:-disabled} - OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=http://otel-collector:4317 - OTEL_EXPORTER_OTLP_TRACES_ENDPOINT_INSECURE=true @@ -456,7 +466,12 @@ services: - ./infra/dapr/components/workflowstate.yaml:/dapr/components/workflowstate.yaml:ro - ./infra/dapr/configuration/config_monitoring_${NEMESIS_MONITORING:-disabled}.yaml:/dapr/configuration/config.yaml:ro environment: - - POSTGRES_CONNECTION_STRING=postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/enrichment?sslmode=disable + - POSTGRES_USER=${POSTGRES_USER:?} + - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:?} + - POSTGRES_HOST=${POSTGRES_HOST:-postgres} + - POSTGRES_PORT=${POSTGRES_PORT:-5432} + - POSTGRES_DB=${POSTGRES_DB:-enrichment} + - POSTGRES_PARAMETERS=${POSTGRES_PARAMETERS:-sslmode=disable} - HASURA_ADMIN_SECRET=${HASURA_ADMIN_SECRET:-pass456} - RABBITMQ_CONNECTION_STRING=amqp://${RABBITMQ_USER}:${RABBITMQ_PASSWORD}@rabbitmq:5672 - MINIO_ROOT_USER=${MINIO_ROOT_USER:?} @@ -514,7 +529,12 @@ services: environment: - MINIO_ROOT_PASSWORD=${MINIO_ROOT_PASSWORD:?} - MINIO_ROOT_USER=${MINIO_ROOT_USER:?} - - POSTGRES_CONNECTION_STRING=postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/enrichment?sslmode=disable + - POSTGRES_USER=${POSTGRES_USER:?} + - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:?} + - POSTGRES_HOST=${POSTGRES_HOST:-postgres} + - POSTGRES_PORT=${POSTGRES_PORT:-5432} + - POSTGRES_DB=${POSTGRES_DB:-enrichment} + - POSTGRES_PARAMETERS=${POSTGRES_PARAMETERS:-sslmode=disable} depends_on: housekeeping: { condition: service_started } placement: { condition: service_started } @@ -584,7 +604,12 @@ services: environment: - MINIO_ROOT_PASSWORD=${MINIO_ROOT_PASSWORD:?} - MINIO_ROOT_USER=${MINIO_ROOT_USER:?} - - POSTGRES_CONNECTION_STRING=postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/enrichment?sslmode=disable + - POSTGRES_USER=${POSTGRES_USER:?} + - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:?} + - POSTGRES_HOST=${POSTGRES_HOST:-postgres} + - POSTGRES_PORT=${POSTGRES_PORT:-5432} + - POSTGRES_DB=${POSTGRES_DB:-enrichment} + - POSTGRES_PARAMETERS=${POSTGRES_PARAMETERS:-sslmode=disable} - RABBITMQ_CONNECTION_STRING=amqp://${RABBITMQ_USER}:${RABBITMQ_PASSWORD}@rabbitmq:5672 depends_on: document-conversion: { condition: service_started } @@ -726,7 +751,7 @@ services: profiles: ["monitoring"] image: prometheuscommunity/postgres-exporter:latest environment: - DATA_SOURCE_NAME: "postgresql://${POSTGRES_USER:?}:${POSTGRES_PASSWORD:?}@postgres:5432/enrichment?sslmode=disable" + DATA_SOURCE_NAME: "postgresql://${POSTGRES_USER:?}:${POSTGRES_PASSWORD:?}@${POSTGRES_HOST:-postgres}:${POSTGRES_PORT:-5432}/${POSTGRES_DB:-enrichment}?${POSTGRES_PARAMETERS:-sslmode=disable}" volumes: - ./infra/postgres-exporter/postgres_exporter.yml:/postgres_exporter.yml:ro depends_on: @@ -739,13 +764,13 @@ services: environment: HASURA_GRAPHQL_ADMIN_SECRET: "${HASURA_ADMIN_SECRET:-pass456}" HASURA_GRAPHQL_BASE_PATH: "/hasura" - HASURA_GRAPHQL_DATABASE_URL: postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/enrichment + HASURA_GRAPHQL_DATABASE_URL: postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST:-postgres}:${POSTGRES_PORT:-5432}/${POSTGRES_DB:-enrichment} HASURA_GRAPHQL_ENABLE_CONSOLE: "true" HASURA_GRAPHQL_ENABLE_METADATA_SYNC: "true" HASURA_GRAPHQL_ENABLE_TELEMETRY: "false" HASURA_GRAPHQL_ENABLED_LOG_TYPES: startup, http-log, webhook-log, websocket-log, query-log HASURA_GRAPHQL_LOG_LEVEL: "warn" - HASURA_GRAPHQL_METADATA_DATABASE_URL: postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/enrichment + HASURA_GRAPHQL_METADATA_DATABASE_URL: postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST:-postgres}:${POSTGRES_PORT:-5432}/${POSTGRES_DB:-enrichment} HASURA_GRAPHQL_METADATA_DIR: /hasura-metadata HASURA_GRAPHQL_UNAUTHORIZED_ROLE: anonymous volumes: @@ -845,7 +870,7 @@ services: environment: - PHOENIX_WORKING_DIR=/data - PHOENIX_HOST_ROOT_PATH=/phoenix - - PHOENIX_SQL_DATABASE_URL=postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/phoenix + - PHOENIX_SQL_DATABASE_URL=postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST:-postgres}:${POSTGRES_PORT:-5432}/phoenix - PHOENIX_DEFAULT_RETENTION_POLICY_DAYS=100 volumes: - phoenix_data:/data @@ -995,7 +1020,7 @@ services: profiles: ["llm"] image: ghcr.io/berriai/litellm:v1.74.0-stable environment: - - DATABASE_URL=postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/litellm + - DATABASE_URL=postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST:-postgres}:${POSTGRES_PORT:-5432}/litellm - STORE_MODEL_IN_DB=True - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID:-} - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY:-} diff --git a/docs/dapr.md b/docs/dapr.md index b40a40f..c15105e 100644 --- a/docs/dapr.md +++ b/docs/dapr.md @@ -20,13 +20,13 @@ The [document_conversion](https://github.com/SpecterOps/Nemesis/tree/main/projec ## Secrets -Nemesis uses the [Dapr Secrets management](https://docs.dapr.io/developing-applications/building-blocks/secrets/secrets-overview/) building block to protect secrets internally (like Postgres connection strings). Currently the [Local environment variables](https://docs.dapr.io/reference/components-reference/supported-secret-stores/envvar-secret-store/) component is used. These secrets are also refereced within some Dapr files such as [pubsub.yaml](https://github.com/SpecterOps/Nemesis/tree/main/infra/dapr/components/pubsub.yaml). +Nemesis uses the [Dapr Secrets management](https://docs.dapr.io/developing-applications/building-blocks/secrets/secrets-overview/) building block to protect secrets internally (like PostgreSQL connection parameters). Currently the [Local environment variables](https://docs.dapr.io/reference/components-reference/supported-secret-stores/envvar-secret-store/) component is used. These secrets are also referenced within some Dapr files such as [pubsub.yaml](https://github.com/SpecterOps/Nemesis/tree/main/infra/dapr/components/pubsub.yaml). This reason for using this abstraction is so alternative secret management systems like [Vault or Kubernetes secrets](https://docs.dapr.io/reference/components-reference/supported-secret-stores/) can be used in the future: ![Dapr Secrets](images/dapr-secrets-overview-cloud-stores.png) -An example of retrieving a secret is at the top of the the [housekeeping code](https://github.com/SpecterOps/Nemesis/blob/main/projects/housekeeping/housekeeping/main.py) to retrieve the `POSTGRES_CONNECTION_STRING` string. +An example of retrieving secrets is in [libs/common/common/db.py](https://github.com/SpecterOps/Nemesis/blob/main/libs/common/common/db.py) which retrieves individual PostgreSQL connection parameters (`POSTGRES_USER`, `POSTGRES_PASSWORD`, `POSTGRES_HOST`, `POSTGRES_PORT`, `POSTGRES_DB`, `POSTGRES_PARAMETERS`) and constructs the connection string. ## Service Invocation diff --git a/env.example b/env.example index 11a8cee..e9598d1 100644 --- a/env.example +++ b/env.example @@ -1,10 +1,17 @@ MINIO_ROOT_PASSWORD=Qwerty12345 MINIO_ROOT_USER=nemesis -POSTGRES_PASSWORD=Qwerty12345 -POSTGRES_USER=nemesis + RABBITMQ_PASSWORD=Qwerty12345 RABBITMQ_USER=nemesis +POSTGRES_PASSWORD=Qwerty12345 +POSTGRES_USER=nemesis +POSTGRES_HOST=postgres +POSTGRES_PORT=5432 +POSTGRES_DB=enrichment +POSTGRES_PARAMETERS=sslmode=disable + + # You can change the password used by HASURA. If not defined a default password will be used (pass456) # Example: diff --git a/infra/dapr/components/workflowstate.yaml b/infra/dapr/components/workflowstate.yaml index 8698389..24273d2 100644 --- a/infra/dapr/components/workflowstate.yaml +++ b/infra/dapr/components/workflowstate.yaml @@ -7,9 +7,27 @@ spec: version: v1 metadata: - name: connectionString + value: "postgresql://username:password@hostname:5432/databasename?sslmode=disable" + - name: host secretKeyRef: - name: POSTGRES_CONNECTION_STRING - key: POSTGRES_CONNECTION_STRING + name: POSTGRES_HOST + key: POSTGRES_HOST + - name: port + secretKeyRef: + name: POSTGRES_PORT + key: POSTGRES_PORT + - name: database + secretKeyRef: + name: POSTGRES_DB + key: POSTGRES_DB + - name: user + secretKeyRef: + name: POSTGRES_USER + key: POSTGRES_USER + - name: password + secretKeyRef: + name: POSTGRES_PASSWORD + key: POSTGRES_PASSWORD - name: actorStateStore value: "true" - name: table diff --git a/libs/common/common/db.py b/libs/common/common/db.py index 1255537..df9a310 100644 --- a/libs/common/common/db.py +++ b/libs/common/common/db.py @@ -1,24 +1,47 @@ from functools import lru_cache +from urllib.parse import quote_plus from dapr.clients import DaprClient -_POSTGRES_SECRET_NAME = "POSTGRES_CONNECTION_STRING" _DAPR_SECRET_STORE_NAME = "nemesis-secret-store" @lru_cache(maxsize=1) def get_postgres_connection_str(dapr_client: DaprClient | None = None) -> str: - """Get PostgreSQL connection string from Dapr.""" + """Get PostgreSQL connection string from Dapr secrets by building it from individual parameters.""" + + def fetch_secrets(client: DaprClient) -> dict: + """Fetch all required PostgreSQL secrets.""" + secrets = {} + secret_keys = ["POSTGRES_USER", "POSTGRES_PASSWORD", "POSTGRES_HOST", "POSTGRES_PORT", "POSTGRES_DB", "POSTGRES_PARAMETERS"] + + for key in secret_keys: + try: + secret = client.get_secret(store_name=_DAPR_SECRET_STORE_NAME, key=key) + secrets[key] = secret.secret[key] + except Exception as e: + raise ValueError(f"Failed to fetch {key} from Dapr secret store: {e}") from e + + return secrets if dapr_client: - secret = dapr_client.get_secret(store_name=_DAPR_SECRET_STORE_NAME, key=_POSTGRES_SECRET_NAME) - output = secret.secret[_POSTGRES_SECRET_NAME] + secrets = fetch_secrets(dapr_client) else: with DaprClient() as client: - secret = client.get_secret(store_name=_DAPR_SECRET_STORE_NAME, key=_POSTGRES_SECRET_NAME) - output = secret.secret[_POSTGRES_SECRET_NAME] + secrets = fetch_secrets(client) + + # Build the connection string from individual parameters + # URL-encode user and password to handle special characters like @, :, /, etc. + user = quote_plus(secrets["POSTGRES_USER"]) + password = quote_plus(secrets["POSTGRES_PASSWORD"]) + host = secrets["POSTGRES_HOST"] + port = secrets["POSTGRES_PORT"] + db = secrets["POSTGRES_DB"] + parameters = secrets["POSTGRES_PARAMETERS"] + + output = f"postgresql://{user}:{password}@{host}:{port}/{db}?{parameters}" if not output.startswith("postgresql://"): - raise ValueError("POSTGRES_CONNECTION_STRING must start with 'postgresql://' to be used with the DpapiManager") + raise ValueError("Constructed POSTGRES connection string must start with 'postgresql://'") return output diff --git a/projects/alerting/alerting/main.py b/projects/alerting/alerting/main.py index 293df4d..68dbf8a 100644 --- a/projects/alerting/alerting/main.py +++ b/projects/alerting/alerting/main.py @@ -61,7 +61,9 @@ async def lifespan(app: FastAPI): logger.info(f"[alerting] adding Apprise URL: {url} (tag: {tag})") apobj.add(f"{url}?footer=no", tag=tag) else: - logger.warning("No Apprise services were added during initialization") + # Use test endpoint as default when APPRISE_URLS is not configured + logger.info("No APPRISE_URLS configured, using test endpoint as default") + apobj.add("json://localhost:8000/test/alert?footer=no", tag="default") is_initialized = True @@ -120,12 +122,13 @@ async def handle_feedback_subscription(): while True: try: transport = WebsocketsTransport( - url="ws://hasura:8080/v1/graphql", headers={"x-hasura-admin-secret": hasura_admin_secret} + url="ws://hasura:8080/v1/graphql", + headers={"x-hasura-admin-secret": hasura_admin_secret} ) async with Client( transport=transport, - fetch_schema_from_transport=True, + fetch_schema_from_transport=False, # Disable schema fetching to avoid large payload ) as session: async for result in session.subscribe(SUBSCRIPTION): if result is None: diff --git a/projects/web_api/web_api/main.py b/projects/web_api/web_api/main.py index 771d189..1395815 100644 --- a/projects/web_api/web_api/main.py +++ b/projects/web_api/web_api/main.py @@ -263,7 +263,6 @@ async def upload_file( logger.info( "Received file upload request", filename=file.filename, - has_metadata=metadata is not None, metadata=file_metadata.model_dump(), ) @@ -285,7 +284,7 @@ async def upload_file( file_model = FileModel.from_file_metadata(file_metadata, str(object_id)) submission_id = await submit_file(file_model) - logger.info("Metadata submitted", submission_id=submission_id) + logger.info("Metadata submitted", submission_id=str(submission_id)) return FileWithMetadataResponse(object_id=object_id, submission_id=submission_id) except Exception as e: