diff --git a/.github/workflows/bootstrap-cluster.yml b/.github/workflows/bootstrap-cluster.yml index aa7654b..be882b5 100644 --- a/.github/workflows/bootstrap-cluster.yml +++ b/.github/workflows/bootstrap-cluster.yml @@ -401,17 +401,21 @@ jobs: - name: Collect the alert webhook URLs env: - # Looked up by name rather than listed one per type, because one - # cluster can host installations that belong to different people and - # want different channels: Bonn and Mannheim share the `eduide` - # cluster and each has its own Discord. A channel's secretKey - # `discord-mannheim` reads `ALERT_WEBHOOK_DISCORD_MANNHEIM`. + # One JSON object mapping each channel's secretKey to its webhook + # URL, because one cluster can host installations belonging to + # different people: Bonn and Mannheim share the `eduide` cluster and + # each has its own Discord, so a single URL per type is not enough. # - # GitHub expressions cannot index `secrets` by a computed name, so the - # whole map is passed in and one key is picked out with jq. Nothing in - # this step echoes it, and Actions masks secret values in logs - # regardless. - ALL_SECRETS: ${{ toJSON(secrets) }} + # {"discord-mannheim": "https://...", "discord-bonn": "https://..."} + # + # GitHub expressions cannot index `secrets` by a computed name, and + # `toJSON(secrets)` is not the way round it: Actions refuses to + # schedule a run that does that at all. The run completes in about a + # second with zero jobs and conclusion `action_required`, and no error + # is reported anywhere - it looks like the workflow simply did not + # run. It would also have handed KUBECONFIG and the wildcard private + # key to this step for no reason. + ALERT_WEBHOOKS: ${{ secrets.ALERT_WEBHOOKS }} run: | set -euo pipefail # A Slack or Discord webhook URL is a credential: anyone holding it can @@ -427,6 +431,16 @@ jobs: echo "alerting is off for ${CLUSTER}" exit 0 fi + if [[ -z "${ALERT_WEBHOOKS:-}" ]]; then + echo "::error::alerting is enabled on ${CLUSTER} but ALERT_WEBHOOKS is not set on the" + echo "::error::'${{ needs.resolve.outputs.environment }}' environment. It is a JSON object mapping" + echo "::error::each channel's secretKey to its webhook URL. See docs/monitoring-setup.md." + exit 1 + fi + if ! printf '%s' "$ALERT_WEBHOOKS" | jq -e 'type == "object"' >/dev/null 2>&1; then + echo "::error::ALERT_WEBHOOKS on '${{ needs.resolve.outputs.environment }}' is not a JSON object." + exit 1 + fi # Every channel names the key it reads. A channel whose secret is not # set would render an AlertmanagerConfig that notifies nobody and # reports no error, so fail here instead. @@ -450,11 +464,9 @@ jobs: echo "::error::The prefix selects the webhook secret; the rest has to be a valid Kubernetes Secret data key." exit 1 fi - # discord-mannheim -> ALERT_WEBHOOK_DISCORD_MANNHEIM - secret_name="ALERT_WEBHOOK_$(printf '%s' "$key" | tr '[:lower:]-' '[:upper:]_')" - value=$(printf '%s' "$ALL_SECRETS" | jq -r --arg n "$secret_name" '.[$n] // ""') + value=$(printf '%s' "$ALERT_WEBHOOKS" | jq -r --arg n "$key" '.[$n] // ""') if [[ -z "$value" ]]; then - MISSING+=("${key} (expected secret ${secret_name})") + MISSING+=("$key") continue fi echo " ${key}: \"$(printf '%s' "$value" | base64 | tr -d '\n')\"" >> alert-secrets.yaml diff --git a/clusters/eduide.yaml b/clusters/eduide.yaml index 0e0ff0e..7b6f697 100644 --- a/clusters/eduide.yaml +++ b/clusters/eduide.yaml @@ -96,10 +96,10 @@ spec: monitorCertManager: true # Where alerts go. The channel list is here; the webhook URLs are not - they - # are credentials and live in this cluster's GitHub Environment. A channel's - # `secretKey` names the secret: `discord-mannheim` is read from - # ALERT_WEBHOOK_DISCORD_MANNHEIM, so one cluster can hold a different webhook - # per installation. + # are credentials and live in this cluster's GitHub Environment as + # ALERT_WEBHOOKS, a JSON object keyed by each channel's `secretKey`. One + # object rather than one secret per channel, so a cluster can hold a + # different webhook per installation without editing the workflow. # # `minSeverity` is what reaches the channels, not what fires. Everything below # it still fires and is visible in Alertmanager and on the dashboards; it just diff --git a/clusters/tum-production.yaml b/clusters/tum-production.yaml index 452623c..e9e9378 100644 --- a/clusters/tum-production.yaml +++ b/clusters/tum-production.yaml @@ -81,10 +81,10 @@ spec: monitorCertManager: true # Where alerts go. The channel list is here; the webhook URLs are not - they - # are credentials and live in this cluster's GitHub Environment. A channel's - # `secretKey` names the secret: `discord-mannheim` is read from - # ALERT_WEBHOOK_DISCORD_MANNHEIM, so one cluster can hold a different webhook - # per installation. + # are credentials and live in this cluster's GitHub Environment as + # ALERT_WEBHOOKS, a JSON object keyed by each channel's `secretKey`. One + # object rather than one secret per channel, so a cluster can hold a + # different webhook per installation without editing the workflow. # # `minSeverity` is what reaches the channels, not what fires. Everything below # it still fires and is visible in Alertmanager and on the dashboards; it just diff --git a/clusters/tum-student.yaml b/clusters/tum-student.yaml index 5e19c0c..7c6b8e6 100644 --- a/clusters/tum-student.yaml +++ b/clusters/tum-student.yaml @@ -47,10 +47,10 @@ spec: monitorCertManager: true # Where alerts go. The channel list is here; the webhook URLs are not - they - # are credentials and live in this cluster's GitHub Environment. A channel's - # `secretKey` names the secret: `discord-mannheim` is read from - # ALERT_WEBHOOK_DISCORD_MANNHEIM, so one cluster can hold a different webhook - # per installation. + # are credentials and live in this cluster's GitHub Environment as + # ALERT_WEBHOOKS, a JSON object keyed by each channel's `secretKey`. One + # object rather than one secret per channel, so a cluster can hold a + # different webhook per installation without editing the workflow. # # `minSeverity` is what reaches the channels, not what fires. Everything below # it still fires and is visible in Alertmanager and on the dashboards; it just diff --git a/docs/monitoring-setup.md b/docs/monitoring-setup.md index 2c1ff78..8d873c2 100644 --- a/docs/monitoring-setup.md +++ b/docs/monitoring-setup.md @@ -169,24 +169,30 @@ Webhook URLs are credentials. They never go in a manifest, a values file or a `--set`, which would put them in the process list and in Actions debug logs. 1. Create the incoming webhook in Slack or Discord. -2. Put it on the **cluster** GitHub Environment. The secret is named after the - channel's `secretKey`, uppercased with hyphens as underscores and prefixed - `ALERT_WEBHOOK_`, so one cluster can hold a different webhook per - installation: - - | `secretKey` | GitHub Environment secret | - |---|---| - | `discord-mannheim` | `ALERT_WEBHOOK_DISCORD_MANNHEIM` | - | `discord-bonn` | `ALERT_WEBHOOK_DISCORD_BONN` | - | `slack-platform` | `ALERT_WEBHOOK_SLACK_PLATFORM` | +2. Put it on the **cluster** GitHub Environment in `ALERT_WEBHOOKS`, a single + JSON object keyed by each channel's `secretKey`: + + ```json + { + "discord-mannheim": "https://discord.com/api/webhooks/...", + "discord-bonn": "https://discord.com/api/webhooks/..." + } + ``` ```bash REPO=EduIDE/EduIDE-deployment - gh secret set ALERT_WEBHOOK_DISCORD_MANNHEIM --repo "$REPO" --env cluster-eduide < webhook.txt + gh secret set ALERT_WEBHOOKS --repo "$REPO" --env cluster-eduide < webhooks.json ``` - Pipe from a file or use `--body`; never paste a webhook URL into a shell you - share, and never into a manifest. + One object rather than one secret per channel, because GitHub expressions + cannot index `secrets` by a computed name and the obvious workaround does not + exist: **Actions refuses to schedule a run whose workflow uses + `toJSON(secrets)`** - it completes in about a second with zero jobs and + conclusion `action_required`, reporting no error at all. Passing one named + secret also keeps `KUBECONFIG` and the wildcard private key out of that step. + + Pipe from a file; never paste a webhook URL into a shell you share, and never + into a manifest. Delete the file afterwards. 3. Add the channel to `clusters/.yaml`. The `secretKey` prefix decides which webhook type it is, so it must be `slack-*` for a Slack channel and