From 1191b01382b515d0c9e87c973a6ca3befe7799ef Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Sun, 15 Nov 2026 19:06:18 +0100 Subject: [PATCH 1/7] feat(customer-identity): add a one-shot sign-in recovery for the administrator The setup CronJob gains a recovery step that runs only when ADMIN_RECOVERY_NONCE holds a value no run has recorded yet. For the administrator alone it unlocks the account, removes every second factor and passkey, keeps the password and sends ZITADEL's own password-reset email. It records the value on the client Secret once the factors are gone, so later runs do nothing, even after a new factor is enrolled. The login policy still forces MFA, so the next sign-in enrols one. Nothing the reset creates is printed. This change sets the trigger once; a follow-up removes it. Co-Authored-By: Claude Opus 5.5 --- .../customer-identity/cron-job-setup.yaml | 69 ++++++++++++++++++- 1 file changed, 68 insertions(+), 1 deletion(-) diff --git a/k8s/providers/hetzner/apps/customer-identity/cron-job-setup.yaml b/k8s/providers/hetzner/apps/customer-identity/cron-job-setup.yaml index d6ce4565a..ad9750d1b 100644 --- a/k8s/providers/hetzner/apps/customer-identity/cron-job-setup.yaml +++ b/k8s/providers/hetzner/apps/customer-identity/cron-job-setup.yaml @@ -13,7 +13,12 @@ # - the administrator's one-time password: until the maintainer's first # sign-in replaces it, it stays equal to zitadel-first-admin, so the # commissioning step works even after that Secret is regenerated. Once he has -# chosen his own password it is never touched again. +# chosen his own password it is never touched again; +# - a one-shot sign-in recovery for that administrator, run only when +# ADMIN_RECOVERY_NONCE holds a value no run has recorded yet: it unlocks him, +# removes his second factors and passkeys, keeps his password and sends +# ZITADEL's own password-reset email. The login policy still forces MFA, so +# his next sign-in enrols a new factor. Remove the value once it has run. # A run that finds nothing to change writes nothing. A duplicate, a deactivated # client, a missing Secret, an API error or a refused write stops the run # before it changes anything further; the next run starts again. @@ -145,6 +150,11 @@ spec: value: /run/setup-token/token - name: ADMIN_EMAIL value: ned@devantler.tech + # The one-shot recovery trigger (see the top of this file). The + # run records it on the client Secret and never acts on it + # again; a follow-up change removes it. + - name: ADMIN_RECOVERY_NONCE + value: admin-recovery-2026-11-15 volumeMounts: - name: setup-token mountPath: /run/setup-token @@ -191,6 +201,7 @@ spec: # one Secret it may write, so a run with nothing new changes nothing. fingerprint_annotation=platform.devantler.tech/smtp-credential-sha256 admin_annotation=platform.devantler.tech/first-admin-password-sha256 + recovery_annotation=platform.devantler.tech/admin-recovery-nonce project_name="Customer sign-in" app_name="Customer Dex" dex_callback=https://login.devantler.tech/callback @@ -451,6 +462,62 @@ spec: fi fi + # --- One-shot sign-in recovery for the administrator --------------------------- + # A reviewed change sets ADMIN_RECOVERY_NONCE to a new value when the maintainer + # can no longer pass his second factor. The first run that sees the value + # unlocks him, removes every second factor and passkey he has, keeps his + # password, and has ZITADEL send its own password-reset email. The value is + # recorded once the factors are gone, so no later run repeats the removal, even + # after he has enrolled a new factor. The login policy still forces MFA, so his + # next sign-in makes him enrol one. Nothing the reset creates is ever printed. + if [[ -n ${ADMIN_RECOVERY_NONCE:-} ]]; then + [[ $ADMIN_RECOVERY_NONCE =~ ^[a-z0-9-]{8,64}$ ]] || fail "ADMIN_RECOVERY_NONCE must be 8 to 64 characters of a-z, 0-9 and -" + if [[ $(secret_json | jq -r --arg a "$recovery_annotation" '.metadata.annotations[$a] // ""') == "$ADMIN_RECOVERY_NONCE" ]]; then + log "recovery $ADMIN_RECOVERY_NONCE: done earlier, nothing to do" + else + user_request=$(jq -nc --arg u "$admin_id" '{userId: $u}') + list_factors() { + connect zitadel.user.v2.UserService/ListAuthenticationFactors "$(jq -nc --arg u "$admin_id" \ + '{userId: $u, states: ["AUTH_FACTOR_STATE_READY", "AUTH_FACTOR_STATE_NOT_READY"]}')" + } + removed=() unlocked=no + if [[ $(jq -r '.result[0].state' <<<"$admins") == USER_STATE_LOCKED ]]; then + connect zitadel.user.v2.UserService/UnlockUser "$user_request" >/dev/null + unlocked=yes + fi + factors=$(list_factors) + while IFS= read -r factor; do + case $(jq -r 'del(.state) | keys | join(",")' <<<"$factor") in + u2f) + connect zitadel.user.v2.UserService/RemoveU2F "$(jq -nc --arg u "$admin_id" --argjson f "$factor" \ + '{userId: $u, u2fId: $f.u2f.id}')" >/dev/null + removed+=("security key") + ;; + otp) connect zitadel.user.v2.UserService/RemoveTOTP "$user_request" >/dev/null; removed+=("authenticator app") ;; + otpSms) connect zitadel.user.v2.UserService/RemoveOTPSMS "$user_request" >/dev/null; removed+=("SMS code") ;; + otpEmail) connect zitadel.user.v2.UserService/RemoveOTPEmail "$user_request" >/dev/null; removed+=("email code") ;; + *) fail "the administrator has a second factor of an unknown type; remove it on purpose" ;; + esac + done < <(jq -c '.result // [] | .[]' <<<"$factors") + passkeys=$(connect zitadel.user.v2.UserService/ListPasskeys "$user_request") + while IFS= read -r passkey_id; do + connect zitadel.user.v2.UserService/RemovePasskey "$(jq -nc --arg u "$admin_id" --arg p "$passkey_id" \ + '{userId: $u, passkeyId: $p}')" >/dev/null + removed+=("passkey") + done < <(jq -r '.result // [] | .[].id' <<<"$passkeys") + [[ $(list_factors | jq '.result // [] | length') == 0 ]] || fail "second factors remain after the recovery" + [[ $(connect zitadel.user.v2.UserService/ListPasskeys "$user_request" | jq '.result // [] | length') == 0 ]] || + fail "passkeys remain after the recovery" + state=$(connect zitadel.user.v2.UserService/GetUserByID "$user_request" | jq -r '.user.state') + [[ $state == USER_STATE_ACTIVE ]] || fail "the administrator is $state after the recovery" + patch_secret "$(jq -nc --arg a "$recovery_annotation" --arg n "$ADMIN_RECOVERY_NONCE" '{metadata: {annotations: {($a): $n}}}')" + connect zitadel.user.v2.UserService/PasswordReset "$(jq -nc --arg u "$admin_id" \ + '{userId: $u, sendLink: {notificationType: "NOTIFICATION_TYPE_Email"}}')" >/dev/null + log "recovery $ADMIN_RECOVERY_NONCE: unlocked=$unlocked removed=$(IFS=,; echo "${removed[*]:-nothing}") password reset email sent" + changes+=("recovered the administrator's sign-in") + fi + fi + summary="project=$project_id client=$client_id application=$app_id smtp=$smtp_id administrator=$admin_state" if ((${#changes[@]})); then log "OK $summary changes: $(IFS=,; echo "${changes[*]}")" From 1b957c15015622845ab54c29d7a549a6e804873a Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Sun, 15 Nov 2026 19:17:52 +0100 Subject: [PATCH 2/7] fix(customer-identity): drop the reset email and remove recovery codes The administrator will sign in with GitHub instead of a password, so the recovery no longer sends ZITADEL's password-reset email. It now also removes recovery codes, and its final check reads the sign-in method types, which include them, so the nonce is recorded only when nothing but the password is left. Co-Authored-By: Claude Opus 5.5 --- .../customer-identity/cron-job-setup.yaml | 37 ++++++++++++------- 1 file changed, 23 insertions(+), 14 deletions(-) diff --git a/k8s/providers/hetzner/apps/customer-identity/cron-job-setup.yaml b/k8s/providers/hetzner/apps/customer-identity/cron-job-setup.yaml index ad9750d1b..2b37937cd 100644 --- a/k8s/providers/hetzner/apps/customer-identity/cron-job-setup.yaml +++ b/k8s/providers/hetzner/apps/customer-identity/cron-job-setup.yaml @@ -15,10 +15,9 @@ # commissioning step works even after that Secret is regenerated. Once he has # chosen his own password it is never touched again; # - a one-shot sign-in recovery for that administrator, run only when -# ADMIN_RECOVERY_NONCE holds a value no run has recorded yet: it unlocks him, -# removes his second factors and passkeys, keeps his password and sends -# ZITADEL's own password-reset email. The login policy still forces MFA, so -# his next sign-in enrols a new factor. Remove the value once it has run. +# ADMIN_RECOVERY_NONCE holds a value no run has recorded yet: it unlocks him +# and removes his second factors, recovery codes and passkeys. Remove the +# value once it has run. # A run that finds nothing to change writes nothing. A duplicate, a deactivated # client, a missing Secret, an API error or a refused write stops the run # before it changes anything further; the next run starts again. @@ -465,11 +464,12 @@ spec: # --- One-shot sign-in recovery for the administrator --------------------------- # A reviewed change sets ADMIN_RECOVERY_NONCE to a new value when the maintainer # can no longer pass his second factor. The first run that sees the value - # unlocks him, removes every second factor and passkey he has, keeps his - # password, and has ZITADEL send its own password-reset email. The value is - # recorded once the factors are gone, so no later run repeats the removal, even - # after he has enrolled a new factor. The login policy still forces MFA, so his - # next sign-in makes him enrol one. Nothing the reset creates is ever printed. + # unlocks him and removes every second factor, recovery code and passkey he + # has. The value is recorded once they are gone, so no later run repeats the + # removal, even after he has enrolled something new. A run that fails before + # that point is repeated by the next one, which is safe: every step is + # idempotent. Remove the value from this file once it has run, so a rebuilt + # client Secret, which carries no record, cannot trigger it again. if [[ -n ${ADMIN_RECOVERY_NONCE:-} ]]; then [[ $ADMIN_RECOVERY_NONCE =~ ^[a-z0-9-]{8,64}$ ]] || fail "ADMIN_RECOVERY_NONCE must be 8 to 64 characters of a-z, 0-9 and -" if [[ $(secret_json | jq -r --arg a "$recovery_annotation" '.metadata.annotations[$a] // ""') == "$ADMIN_RECOVERY_NONCE" ]]; then @@ -480,6 +480,9 @@ spec: connect zitadel.user.v2.UserService/ListAuthenticationFactors "$(jq -nc --arg u "$admin_id" \ '{userId: $u, states: ["AUTH_FACTOR_STATE_READY", "AUTH_FACTOR_STATE_NOT_READY"]}')" } + method_types() { + connect zitadel.user.v2.UserService/ListAuthenticationMethodTypes "$user_request" | jq -c '.authMethodTypes // []' + } removed=() unlocked=no if [[ $(jq -r '.result[0].state' <<<"$admins") == USER_STATE_LOCKED ]]; then connect zitadel.user.v2.UserService/UnlockUser "$user_request" >/dev/null @@ -499,21 +502,27 @@ spec: *) fail "the administrator has a second factor of an unknown type; remove it on purpose" ;; esac done < <(jq -c '.result // [] | .[]' <<<"$factors") + types=$(method_types) + if jq -e 'index("AUTHENTICATION_METHOD_TYPE_RECOVERY_CODE")' <<<"$types" >/dev/null; then + connect zitadel.user.v2.UserService/RemoveRecoveryCodes "$user_request" >/dev/null + removed+=("recovery codes") + fi passkeys=$(connect zitadel.user.v2.UserService/ListPasskeys "$user_request") while IFS= read -r passkey_id; do connect zitadel.user.v2.UserService/RemovePasskey "$(jq -nc --arg u "$admin_id" --arg p "$passkey_id" \ '{userId: $u, passkeyId: $p}')" >/dev/null removed+=("passkey") done < <(jq -r '.result // [] | .[].id' <<<"$passkeys") + # Pending factors appear only in the factor list; every ready method, + # recovery codes included, appears in the method types. [[ $(list_factors | jq '.result // [] | length') == 0 ]] || fail "second factors remain after the recovery" - [[ $(connect zitadel.user.v2.UserService/ListPasskeys "$user_request" | jq '.result // [] | length') == 0 ]] || - fail "passkeys remain after the recovery" + types=$(method_types) + left=$(jq -r 'map(select(. != "AUTHENTICATION_METHOD_TYPE_PASSWORD" and . != "AUTHENTICATION_METHOD_TYPE_IDP")) | join(",")' <<<"$types") + [[ -z $left ]] || fail "sign-in methods remain after the recovery: $left" state=$(connect zitadel.user.v2.UserService/GetUserByID "$user_request" | jq -r '.user.state') [[ $state == USER_STATE_ACTIVE ]] || fail "the administrator is $state after the recovery" patch_secret "$(jq -nc --arg a "$recovery_annotation" --arg n "$ADMIN_RECOVERY_NONCE" '{metadata: {annotations: {($a): $n}}}')" - connect zitadel.user.v2.UserService/PasswordReset "$(jq -nc --arg u "$admin_id" \ - '{userId: $u, sendLink: {notificationType: "NOTIFICATION_TYPE_Email"}}')" >/dev/null - log "recovery $ADMIN_RECOVERY_NONCE: unlocked=$unlocked removed=$(IFS=,; echo "${removed[*]:-nothing}") password reset email sent" + log "recovery $ADMIN_RECOVERY_NONCE: unlocked=$unlocked removed=$(IFS=,; echo "${removed[*]:-nothing}")" changes+=("recovered the administrator's sign-in") fi fi From 35e162996c73c21a58cabff2dccd7568ab11576c Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Sun, 15 Nov 2026 19:31:12 +0100 Subject: [PATCH 3/7] feat(customer-identity): sign ZITADEL's administrator in with GitHub The maintainer's ZITADEL login collided with his mail login in Apple Passwords, so he signs in with GitHub through the platform Dex instead of a ZITADEL password and passkey. - The platform Dex's shared client accepts ZITADEL's Login V1 callback. - The setup CronJob keeps that Dex configured as a generic OIDC identity provider of his organization, which neither creates nor links accounts. It sets that organization's login policy to external sign-in only, with MFA forced for local sign-in alone, and links his account to his GitHub identity as Dex presents it. - Customers get a "Customers" organization, which keeps the instance's password and MFA policy. The customer Dex requests it, so they are not sent to GitHub and no one else signs in there. - ZITADEL may reach dex.platform.devantler.tech. Co-Authored-By: Claude Opus 5.5 --- .../cilium-network-policy.yaml | 9 +- .../customer-identity/cron-job-setup.yaml | 175 +++++++++++++++++- .../external-secret-platform-dex-client.yaml | 22 +++ .../customer-identity/helm-release-dex.yaml | 6 + .../apps/customer-identity/kustomization.yaml | 1 + .../dex/patches/allow-zitadel-sign-in.yaml | 12 ++ .../controllers/kustomization.yaml | 8 + 7 files changed, 226 insertions(+), 7 deletions(-) create mode 100644 k8s/providers/hetzner/apps/customer-identity/external-secret-platform-dex-client.yaml create mode 100644 k8s/providers/hetzner/infrastructure/controllers/dex/patches/allow-zitadel-sign-in.yaml diff --git a/k8s/providers/hetzner/apps/customer-identity/cilium-network-policy.yaml b/k8s/providers/hetzner/apps/customer-identity/cilium-network-policy.yaml index e565d8b8a..9b424f79c 100644 --- a/k8s/providers/hetzner/apps/customer-identity/cilium-network-policy.yaml +++ b/k8s/providers/hetzner/apps/customer-identity/cilium-network-policy.yaml @@ -2,7 +2,8 @@ # ZITADEL (8080) and Dex (5556); only the namespace itself and the CNPG # operator reach the database. Egress: the namespace itself, the API server # (CNPG instance manager), R2 for backups, ZITADEL's public name for Dex's -# connector, and the business site's SES SMTP endpoint for ZITADEL's +# connector, the platform Dex, through which ZITADEL signs its administrator +# in with GitHub, and the business site's SES SMTP endpoint for ZITADEL's # invitation and recovery mail (platform#5400). apiVersion: cilium.io/v2 kind: CiliumNetworkPolicy @@ -50,6 +51,12 @@ spec: - ports: - port: "443" protocol: TCP + - toFQDNs: + - matchName: dex.platform.devantler.tech + toPorts: + - ports: + - port: "443" + protocol: TCP - toFQDNs: - matchName: email-smtp.eu-central-1.amazonaws.com toPorts: diff --git a/k8s/providers/hetzner/apps/customer-identity/cron-job-setup.yaml b/k8s/providers/hetzner/apps/customer-identity/cron-job-setup.yaml index 2b37937cd..f96cd3a92 100644 --- a/k8s/providers/hetzner/apps/customer-identity/cron-job-setup.yaml +++ b/k8s/providers/hetzner/apps/customer-identity/cron-job-setup.yaml @@ -14,6 +14,12 @@ # sign-in replaces it, it stays equal to zitadel-first-admin, so the # commissioning step works even after that Secret is regenerated. Once he has # chosen his own password it is never touched again; +# - the administrator's GitHub sign-in: the platform Dex (dex.platform.devantler.tech) +# as a generic OIDC identity provider of his organization, a login policy +# there that allows only it (no ZITADEL password or MFA; GitHub's two-step +# login covers that), and his account linked to his GitHub identity. A +# "Customers" organization keeps the instance's policy for customers, whom +# the customer Dex signs in there; # - a one-shot sign-in recovery for that administrator, run only when # ADMIN_RECOVERY_NONCE holds a value no run has recorded yet: it unlocks him # and removes his second factors, recovery codes and passkeys. Remove the @@ -28,12 +34,12 @@ # 10-minute token to a memory volume and exits. # - The setup container alone gets a Kubernetes token, projected to # /run/kube-api, which may only read and update the client Secret (role.yaml). -# It reads the SMTP login (zitadel-smtp) and the one-time password -# (zitadel-first-admin) as mounted files, never as environment variables. +# It reads the SMTP login (zitadel-smtp), the one-time password +# (zitadel-first-admin) and the platform Dex client secret +# (platform-dex-client) as mounted files, never as environment variables. # - Neither prints a secret. # -# The maintainer's own first sign-in (choosing his password, enrolling MFA) is -# not part of this. +# The maintainer signs in himself, with GitHub; nothing here signs in as him. apiVersion: batch/v1 kind: CronJob metadata: @@ -166,6 +172,9 @@ spec: - name: first-admin mountPath: /run/first-admin readOnly: true + - name: platform-dex + mountPath: /run/platform-dex + readOnly: true - name: kube-api mountPath: /run/kube-api readOnly: true @@ -201,6 +210,18 @@ spec: fingerprint_annotation=platform.devantler.tech/smtp-credential-sha256 admin_annotation=platform.devantler.tech/first-admin-password-sha256 recovery_annotation=platform.devantler.tech/admin-recovery-nonce + # The administrator signs in with GitHub through the platform Dex (see below). + platform_dex_secret_file=${PLATFORM_DEX_SECRET_FILE:-/run/platform-dex/client-secret} + dex_annotation=platform.devantler.tech/platform-dex-secret-sha256 + platform_dex_issuer=https://dex.platform.devantler.tech + platform_dex_client=public-client + idp_name=GitHub + # Dex's subject for GitHub user devantler (id 26203420) through its "github" + # connector: base64url of the protobuf {1: "26203420", 2: "github"}. + admin_dex_subject=CggyNjIwMzQyMBIGZ2l0aHVi + admin_github_login=devantler + customers_org=Customers + customers_domain=customers.accounts.devantler.tech project_name="Customer sign-in" app_name="Customer Dex" dex_callback=https://login.devantler.tech/callback @@ -247,6 +268,7 @@ spec: smtp_user=$(read_secret "$smtp_dir/username" "SMTP username") smtp_password=$(read_secret "$smtp_dir/password" "SMTP password") admin_password=$(read_secret "$admin_password_file" "administrator one-time password") + platform_dex_secret=$(read_secret "$platform_dex_secret_file" "platform Dex client secret") # The Kubernetes token is projected outside the default path, so kubectl gets # a kubeconfig that reads it from there. @@ -280,9 +302,11 @@ spec: local method=$1 path=$2 body=${3:-'{}'} status local -a data=() [[ $method == GET ]] || data=(--data-binary "$body") + local -a org_header=() + [[ -z ${api_org:-} ]] || org_header=(-H "x-zitadel-orgid: $api_org") status=$(curl -sS -m 20 -o "$work/response" -w '%{http_code}' -X "$method" \ -H "@$work/auth" -H "Host: $ZITADEL_HOST" -H 'Content-Type: application/json' \ - -H 'Connect-Protocol-Version: 1' "${data[@]}" "$ZITADEL_URL$path") || + -H 'Connect-Protocol-Version: 1' "${org_header[@]}" "${data[@]}" "$ZITADEL_URL$path") || fail "$method $path: no answer from ZITADEL" if [[ $status != 2?? ]]; then fail "$method $path: HTTP $status $(jq -rc '[.code, .message] | map(tostring) | join(" ")' "$work/response" 2>/dev/null || echo 'unreadable error')" @@ -290,6 +314,8 @@ spec: cat "$work/response" } connect() { api POST "/$1" "${2-}"; } + # mgmt METHOD PATH [BODY]: the management API, in the administrator's organization. + mgmt() { api_org=$org_id api "$@"; } # The client Secret is created empty by Flux and only ever patched here. secret_json() { @@ -461,6 +487,136 @@ spec: fi fi + # --- The administrator's GitHub sign-in ------------------------------------------- + # The maintainer signs in with GitHub, through the platform Dex, which admits + # only the devantler-tech maintainers team. ZITADEL knows that Dex as a generic + # OIDC identity provider owned by his organization, the instance's default one. + # That organization's login policy allows only this provider, so ZITADEL sends + # him straight to GitHub and asks for no password or MFA of its own: GitHub's + # two-step login covers it. The provider neither creates nor links accounts, so + # his pre-linked account is the only one it can sign in. Customers belong to + # their own organization, which keeps the instance's policy (password and MFA); + # the customer Dex requests that organization. + customer_orgs=$(connect zitadel.org.v2.OrganizationService/ListOrganizations "$(jq -nc --arg n "$customers_org" \ + '{queries: [{nameQuery: {name: $n, method: "TEXT_QUERY_METHOD_EQUALS"}}]}')") + case $(jq '.result // [] | length' <<<"$customer_orgs") in + 0) + connect zitadel.org.v2.OrganizationService/AddOrganization "$(jq -nc --arg n "$customers_org" '{name: $n}')" >/dev/null + changes+=("created the $customers_org organization") + customer_orgs=$(connect zitadel.org.v2.OrganizationService/ListOrganizations "$(jq -nc --arg n "$customers_org" \ + '{queries: [{nameQuery: {name: $n, method: "TEXT_QUERY_METHOD_EQUALS"}}]}')") + ;; + 1) ;; + *) fail "more than one organization named '$customers_org'" ;; + esac + customers_org_id=$(jq -er '.result[0].id' <<<"$customer_orgs") + [[ $(jq -r '.result[0].primaryDomain' <<<"$customer_orgs") == "$customers_domain" ]] || + fail "the $customers_org organization's primary domain is not $customers_domain, which the customer Dex requests" + [[ $customers_org_id != "$org_id" ]] || fail "the $customers_org organization is the administrator's own" + + # The provider. ZITADEL never returns its client secret, so a fingerprint of + # the one last applied decides whether to send it again. + dex_fingerprint=$(printf '%s' "$platform_dex_secret" | sha256sum | cut -d' ' -f1) + desired_idp=$(jq -nc --arg i "$platform_dex_issuer" --arg c "$platform_dex_client" '{ + issuer: $i, clientId: $c, scopes: ["openid", "profile", "email"], isIdTokenMapping: false, usePkce: true, + providerOptions: {isLinkingAllowed: false, isCreationAllowed: false, isAutoCreation: false, isAutoUpdate: false, + autoLinking: "AUTO_LINKING_OPTION_UNSPECIFIED"}}') + normalise_idp='{issuer: (.config.oidc.issuer // ""), clientId: (.config.oidc.clientId // ""), + scopes: (.config.oidc.scopes // []), isIdTokenMapping: (.config.oidc.isIdTokenMapping // false), + usePkce: (.config.oidc.usePkce // false), + providerOptions: {isLinkingAllowed: (.config.options.isLinkingAllowed // false), + isCreationAllowed: (.config.options.isCreationAllowed // false), + isAutoCreation: (.config.options.isAutoCreation // false), isAutoUpdate: (.config.options.isAutoUpdate // false), + autoLinking: (.config.options.autoLinking // "AUTO_LINKING_OPTION_UNSPECIFIED")}}' + list_idps() { + mgmt POST /management/v1/idps/templates/_search "$(jq -nc --arg n "$idp_name" \ + '{queries: [{idpNameQuery: {name: $n, method: "TEXT_QUERY_METHOD_EQUALS"}}]}')" | + jq -c --arg o "$org_id" '[.result // [] | .[] | select(.owner == "IDP_OWNER_TYPE_ORG" and .details.resourceOwner == $o)]' + } + idps=$(list_idps) + case $(jq length <<<"$idps") in + 0) + idp_id=$(mgmt POST /management/v1/idps/generic_oidc "$(PLATFORM_DEX_SECRET=$platform_dex_secret jq -c --arg n "$idp_name" \ + '. + {name: $n, clientSecret: env.PLATFORM_DEX_SECRET}' <<<"$desired_idp")" | jq -er '.id') + patch_secret "$(jq -nc --arg a "$dex_annotation" --arg f "$dex_fingerprint" '{metadata: {annotations: {($a): $f}}}')" + changes+=("created the GitHub identity provider") + ;; + 1) + idp_id=$(jq -er '.[0].id' <<<"$idps") + [[ $(jq -r '.[0].state' <<<"$idps") == IDP_STATE_ACTIVE ]] || + fail "identity provider '$idp_name' is not active; reactivate or remove it on purpose" + recorded_dex=$(secret_json | jq -r --arg a "$dex_annotation" '.metadata.annotations[$a] // ""') + if [[ $(jq -cS ".[0] | $normalise_idp" <<<"$idps") != "$(jq -cS . <<<"$desired_idp")" || $recorded_dex != "$dex_fingerprint" ]]; then + mgmt PUT "/management/v1/idps/generic_oidc/$idp_id" "$(PLATFORM_DEX_SECRET=$platform_dex_secret jq -c --arg n "$idp_name" \ + '. + {name: $n, clientSecret: env.PLATFORM_DEX_SECRET}' <<<"$desired_idp")" >/dev/null + [[ $(list_idps | jq -cS ".[0] | $normalise_idp") == "$(jq -cS . <<<"$desired_idp")" ]] || + fail "identity provider '$idp_name' still differs from its declared settings after the update" + patch_secret "$(jq -nc --arg a "$dex_annotation" --arg f "$dex_fingerprint" '{metadata: {annotations: {($a): $f}}}')" + changes+=("updated the GitHub identity provider") + fi + ;; + *) fail "more than one identity provider named '$idp_name'" ;; + esac + + # His organization's login policy: the instance's, but external sign-in only, + # MFA forced for local sign-in alone, and no password reset. + instance_policy=$(api GET /admin/v1/policies/login | jq -c '.policy') + desired_policy=$(jq -c '{ + allowUsernamePassword: false, allowRegister: false, allowExternalIdp: true, forceMfa: true, forceMfaLocalOnly: true, + passwordlessType: "PASSWORDLESS_TYPE_NOT_ALLOWED", hidePasswordReset: true, + ignoreUnknownUsernames: (.ignoreUnknownUsernames // false), defaultRedirectUri: (.defaultRedirectUri // ""), + passwordCheckLifetime: .passwordCheckLifetime, externalLoginCheckLifetime: .externalLoginCheckLifetime, + mfaInitSkipLifetime: .mfaInitSkipLifetime, secondFactorCheckLifetime: .secondFactorCheckLifetime, + multiFactorCheckLifetime: .multiFactorCheckLifetime, allowDomainDiscovery: false, + disableLoginWithEmail: (.disableLoginWithEmail // false), disableLoginWithPhone: (.disableLoginWithPhone // false)}' \ + <<<"$instance_policy") + normalise_policy='{allowUsernamePassword: (.allowUsernamePassword // false), allowRegister: (.allowRegister // false), + allowExternalIdp: (.allowExternalIdp // false), forceMfa: (.forceMfa // false), + forceMfaLocalOnly: (.forceMfaLocalOnly // false), + passwordlessType: (.passwordlessType // "PASSWORDLESS_TYPE_NOT_ALLOWED"), + hidePasswordReset: (.hidePasswordReset // false), ignoreUnknownUsernames: (.ignoreUnknownUsernames // false), + defaultRedirectUri: (.defaultRedirectUri // ""), passwordCheckLifetime, externalLoginCheckLifetime, + mfaInitSkipLifetime, secondFactorCheckLifetime, multiFactorCheckLifetime, + allowDomainDiscovery: (.allowDomainDiscovery // false), disableLoginWithEmail: (.disableLoginWithEmail // false), + disableLoginWithPhone: (.disableLoginWithPhone // false)}' + org_policy=$(mgmt GET /management/v1/policies/login | jq -c '.policy') + if [[ $(jq -r '.isDefault // false' <<<"$org_policy") == true ]]; then + mgmt POST /management/v1/policies/login "$(jq -c --argjson p "$instance_policy" --arg i "$idp_id" \ + '. + {secondFactors: ($p.secondFactors // []), multiFactors: ($p.multiFactors // []), + idps: [{idpId: $i, ownerType: "IDP_OWNER_TYPE_ORG"}]}' <<<"$desired_policy")" >/dev/null + changes+=("set the administrator's organization to GitHub sign-in") + elif [[ $(jq -cS "$normalise_policy" <<<"$org_policy") != "$(jq -cS . <<<"$desired_policy")" ]]; then + mgmt PUT /management/v1/policies/login "$desired_policy" >/dev/null + changes+=("restored the administrator's organization login policy") + fi + [[ $(mgmt GET /management/v1/policies/login | jq -cS ".policy | $normalise_policy") == "$(jq -cS . <<<"$desired_policy")" ]] || + fail "the administrator's organization login policy still differs from its declared settings" + # Exactly this one provider: with a second one ZITADEL would show a choice + # instead of sending him to GitHub. + policy_idps=$(mgmt POST /management/v1/policies/login/idps/_search | jq -c '[.result // [] | .[].idpId]') + if [[ $(jq -c . <<<"$policy_idps") == '[]' ]]; then + mgmt POST /management/v1/policies/login/idps "$(jq -nc --arg i "$idp_id" '{idpId: $i, ownerType: "IDP_OWNER_TYPE_ORG"}')" >/dev/null + changes+=("allowed the GitHub identity provider") + elif [[ $policy_idps != "$(jq -nc --arg i "$idp_id" '[$i]')" ]]; then + fail "the administrator's organization allows identity providers other than '$idp_name'; remove them on purpose" + fi + + # His account, linked to his GitHub identity as Dex presents it. + links=$(connect zitadel.user.v2.UserService/ListIDPLinks "$(jq -nc --arg u "$admin_id" '{userId: $u}')" | + jq -c --arg i "$idp_id" '[.result // [] | .[] | select(.idpId == $i)]') + case $(jq length <<<"$links") in + 0) + connect zitadel.user.v2.UserService/AddIDPLink "$(jq -nc --arg u "$admin_id" --arg i "$idp_id" \ + --arg s "$admin_dex_subject" --arg n "$admin_github_login" '{userId: $u, idpLink: {idpId: $i, userId: $s, userName: $n}}')" >/dev/null + changes+=("linked the administrator to GitHub") + ;; + 1) + [[ $(jq -r '.[0].userId' <<<"$links") == "$admin_dex_subject" ]] || + fail "the administrator is linked to a different GitHub identity; remove that link on purpose" + ;; + *) fail "the administrator has more than one link to '$idp_name'" ;; + esac + # --- One-shot sign-in recovery for the administrator --------------------------- # A reviewed change sets ADMIN_RECOVERY_NONCE to a new value when the maintainer # can no longer pass his second factor. The first run that sees the value @@ -527,7 +683,7 @@ spec: fi fi - summary="project=$project_id client=$client_id application=$app_id smtp=$smtp_id administrator=$admin_state" + summary="project=$project_id client=$client_id application=$app_id smtp=$smtp_id administrator=$admin_state idp=$idp_id customers=$customers_org_id" if ((${#changes[@]})); then log "OK $summary changes: $(IFS=,; echo "${changes[*]}")" else @@ -584,3 +740,10 @@ spec: items: - key: password path: password + - name: platform-dex + secret: + secretName: platform-dex-client + defaultMode: 0440 + items: + - key: client-secret + path: client-secret diff --git a/k8s/providers/hetzner/apps/customer-identity/external-secret-platform-dex-client.yaml b/k8s/providers/hetzner/apps/customer-identity/external-secret-platform-dex-client.yaml new file mode 100644 index 000000000..b783cf8cd --- /dev/null +++ b/k8s/providers/hetzner/apps/customer-identity/external-secret-platform-dex-client.yaml @@ -0,0 +1,22 @@ +# The platform Dex's shared client secret (OpenBao infrastructure/oidc/dex), +# which ZITADEL presents when it signs its administrator in with GitHub through +# that Dex. Only the setup CronJob mounts it, to keep ZITADEL's identity +# provider configured (cron-job-setup.yaml). +apiVersion: external-secrets.io/v1 +kind: ExternalSecret +metadata: + name: platform-dex-client + namespace: customer-identity +spec: + refreshInterval: 1h + secretStoreRef: + name: openbao + kind: ClusterSecretStore + target: + name: platform-dex-client + creationPolicy: Owner + data: + - secretKey: client-secret # checkov:skip=CKV_SECRET_6:names the key in the synced Secret; the value lives in OpenBao + remoteRef: + key: infrastructure/oidc/dex + property: client-secret diff --git a/k8s/providers/hetzner/apps/customer-identity/helm-release-dex.yaml b/k8s/providers/hetzner/apps/customer-identity/helm-release-dex.yaml index 8ebcc43d7..0efff8a6d 100644 --- a/k8s/providers/hetzner/apps/customer-identity/helm-release-dex.yaml +++ b/k8s/providers/hetzner/apps/customer-identity/helm-release-dex.yaml @@ -169,6 +169,12 @@ spec: scopes: - profile - email + # Customers live in ZITADEL's "Customers" organization, which + # the setup CronJob creates. This scope signs them in under + # its login policy (password and MFA) and admits no one else. + # Without it they would get the default organization's + # policy, which sends its administrator straight to GitHub. + - urn:zitadel:iam:org:domain:primary:customers.accounts.devantler.tech getUserInfo: true insecureSkipEmailVerified: false insecureEnableGroups: false diff --git a/k8s/providers/hetzner/apps/customer-identity/kustomization.yaml b/k8s/providers/hetzner/apps/customer-identity/kustomization.yaml index 9e5b549d6..497f9f4bb 100644 --- a/k8s/providers/hetzner/apps/customer-identity/kustomization.yaml +++ b/k8s/providers/hetzner/apps/customer-identity/kustomization.yaml @@ -20,6 +20,7 @@ resources: - external-secret-smtp.yaml - password-first-admin.yaml - external-secret-first-admin.yaml + - external-secret-platform-dex-client.yaml - helm-repository.yaml - helm-release.yaml # The portal's customer Dex. diff --git a/k8s/providers/hetzner/infrastructure/controllers/dex/patches/allow-zitadel-sign-in.yaml b/k8s/providers/hetzner/infrastructure/controllers/dex/patches/allow-zitadel-sign-in.yaml new file mode 100644 index 000000000..0c7b1aa34 --- /dev/null +++ b/k8s/providers/hetzner/infrastructure/controllers/dex/patches/allow-zitadel-sign-in.yaml @@ -0,0 +1,12 @@ +# Prod-only: ZITADEL (accounts.devantler.tech, apps/customer-identity) signs +# its administrator in with GitHub through this Dex, which it uses as a generic +# OIDC identity provider (business-site#302). Its Login V1 callback joins the +# shared client's redirect URIs; the customer-identity setup CronJob reads that +# client's secret from OpenBao (infrastructure/oidc/dex). The test stops the +# build if the shared client is no longer the first entry. +- op: test + path: /spec/values/config/staticClients/0/id + value: public-client +- op: add + path: /spec/values/config/staticClients/0/redirectURIs/- + value: https://accounts.devantler.tech/ui/login/login/externalidp/callback diff --git a/k8s/providers/hetzner/infrastructure/controllers/kustomization.yaml b/k8s/providers/hetzner/infrastructure/controllers/kustomization.yaml index e5a2ed11c..8c26ce5bc 100644 --- a/k8s/providers/hetzner/infrastructure/controllers/kustomization.yaml +++ b/k8s/providers/hetzner/infrastructure/controllers/kustomization.yaml @@ -141,6 +141,14 @@ patches: - path: kyverno/patches/allow-sigstore-cache-eviction.yaml # Prod-only: run Dex on the source-built, signed distribution — see file. - path: dex/patches/run-source-built-image.yaml + # Prod-only: let ZITADEL's administrator sign in through Dex — see file. + - path: dex/patches/allow-zitadel-sign-in.yaml + target: + group: helm.toolkit.fluxcd.io + version: v2 + kind: HelmRelease + name: dex + namespace: dex # Prod-only fail-closed WireGuard encryption (egress strict mode) — see file. - path: cilium/patches/enforce-wireguard-strict-mode.yaml # Require relay replicas to use separate nodes in prod; local retains soft spread. From 668dbcd4d9b7137e011b3f79bcdc273f05c1a4f8 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Sun, 15 Nov 2026 19:34:02 +0100 Subject: [PATCH 4/7] fix(customer-identity): approve the two Dex releases' new rendered surface The platform Dex gains ZITADEL's callback and the customer Dex requests the Customers organization, so both HelmReleases render differently. Their ledger entries take the hashes the authorization contract reports. Co-Authored-By: Claude Opus 5.5 --- scripts/validate-eks-ci-role-policy/approved-surface.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/validate-eks-ci-role-policy/approved-surface.txt b/scripts/validate-eks-ci-role-policy/approved-surface.txt index 8e92229b2..848671c3f 100644 --- a/scripts/validate-eks-ci-role-policy/approved-surface.txt +++ b/scripts/validate-eks-ci-role-policy/approved-surface.txt @@ -60,11 +60,11 @@ helm.toolkit.fluxcd.io/v2|HelmRelease|cnpg-system|plugin-barman-cloud 74682cb45a helm.toolkit.fluxcd.io/v2|HelmRelease|crossplane-system|crossplane fb717f7c9adb21b0f772e6a84554900e568a2675c91d375d1bb6ee76c2a0b1f3 -helm.toolkit.fluxcd.io/v2|HelmRelease|customer-identity|customer-dex 1733d0b05966c1c28a4c6a1c1caa221208958b04d36bdd6d2e1672ce0587c795 +helm.toolkit.fluxcd.io/v2|HelmRelease|customer-identity|customer-dex 55a9ceabb55c712138a754c1666cf9e55b62b47357e3ac3cddea423a17531051 helm.toolkit.fluxcd.io/v2|HelmRelease|customer-identity|zitadel eeb9881018664637cdefc9c5c0afd6953aa958e5dbfbc9cd8c5d521501e52431 -helm.toolkit.fluxcd.io/v2|HelmRelease|dex|dex e2cbe6e6b5ace19ebded8d0e75f508277dd450c079ed51062d13649e148904b9 +helm.toolkit.fluxcd.io/v2|HelmRelease|dex|dex 1a1e8181ff71a06313611577a55fe9b3cb356a1071139b3d852dcaa42dafe972 helm.toolkit.fluxcd.io/v2|HelmRelease|external-dns|external-dns 107eaf8ad3d74abff5c6b1c6d7ca1b49ac75829fe425672f81cdc1e799457511 From ec40187cc438e1e7cbc6ef913f08a10c4a12810d Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Sun, 15 Nov 2026 19:36:57 +0100 Subject: [PATCH 5/7] fix(customer-identity): stop the recovery when its record cannot be read The recovery compared the recorded nonce inside a test, where a failed Secret read is not an error under set -e: it read as "not recorded", and a later run removed whatever the administrator had enrolled since. The read is now its own assignment, so a failure stops the run. Co-Authored-By: Claude Opus 5.5 --- .../hetzner/apps/customer-identity/cron-job-setup.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/k8s/providers/hetzner/apps/customer-identity/cron-job-setup.yaml b/k8s/providers/hetzner/apps/customer-identity/cron-job-setup.yaml index 2b37937cd..b1d9003e1 100644 --- a/k8s/providers/hetzner/apps/customer-identity/cron-job-setup.yaml +++ b/k8s/providers/hetzner/apps/customer-identity/cron-job-setup.yaml @@ -472,7 +472,10 @@ spec: # client Secret, which carries no record, cannot trigger it again. if [[ -n ${ADMIN_RECOVERY_NONCE:-} ]]; then [[ $ADMIN_RECOVERY_NONCE =~ ^[a-z0-9-]{8,64}$ ]] || fail "ADMIN_RECOVERY_NONCE must be 8 to 64 characters of a-z, 0-9 and -" - if [[ $(secret_json | jq -r --arg a "$recovery_annotation" '.metadata.annotations[$a] // ""') == "$ADMIN_RECOVERY_NONCE" ]]; then + # A separate assignment, so a failed read stops the run instead of reading + # as "not recorded" and removing what he has enrolled since. + recorded_nonce=$(secret_json | jq -r --arg a "$recovery_annotation" '.metadata.annotations[$a] // ""') + if [[ $recorded_nonce == "$ADMIN_RECOVERY_NONCE" ]]; then log "recovery $ADMIN_RECOVERY_NONCE: done earlier, nothing to do" else user_request=$(jq -nc --arg u "$admin_id" '{userId: $u}') From defce8f2fd4e6a476f6aba93505216e29e175624 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Sun, 15 Nov 2026 19:52:46 +0100 Subject: [PATCH 6/7] fix(customer-identity): disarm the administrator's sign-in recovery The one-shot recovery has run and recorded its nonce on the client Secret, so the trigger is removed. A rebuilt Secret, which carries no record, can no longer start it again. The recovery step stays in the setup script as break-glass: it does nothing while ADMIN_RECOVERY_NONCE is unset, and a future recovery is a reviewed change setting a new value. Co-Authored-By: Claude Opus 5.5 --- .../hetzner/apps/customer-identity/cron-job-setup.yaml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/k8s/providers/hetzner/apps/customer-identity/cron-job-setup.yaml b/k8s/providers/hetzner/apps/customer-identity/cron-job-setup.yaml index a2569708b..8bb8c749a 100644 --- a/k8s/providers/hetzner/apps/customer-identity/cron-job-setup.yaml +++ b/k8s/providers/hetzner/apps/customer-identity/cron-job-setup.yaml @@ -155,11 +155,6 @@ spec: value: /run/setup-token/token - name: ADMIN_EMAIL value: ned@devantler.tech - # The one-shot recovery trigger (see the top of this file). The - # run records it on the client Secret and never acts on it - # again; a follow-up change removes it. - - name: ADMIN_RECOVERY_NONCE - value: admin-recovery-2026-11-15 volumeMounts: - name: setup-token mountPath: /run/setup-token From 0fbe10f6f64a39084d1c1610e73162e24067572c Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Sun, 15 Nov 2026 22:03:43 +0100 Subject: [PATCH 7/7] fix(customer-identity): disarm the administrator's sign-in recovery Merging main brought the trigger back, because the branch never added it relative to the old base. Remove ADMIN_RECOVERY_NONCE again so this PR's diff is the disarm alone. Co-Authored-By: Claude Opus 5.5 --- .../hetzner/apps/customer-identity/cron-job-setup.yaml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/k8s/providers/hetzner/apps/customer-identity/cron-job-setup.yaml b/k8s/providers/hetzner/apps/customer-identity/cron-job-setup.yaml index a2569708b..8bb8c749a 100644 --- a/k8s/providers/hetzner/apps/customer-identity/cron-job-setup.yaml +++ b/k8s/providers/hetzner/apps/customer-identity/cron-job-setup.yaml @@ -155,11 +155,6 @@ spec: value: /run/setup-token/token - name: ADMIN_EMAIL value: ned@devantler.tech - # The one-shot recovery trigger (see the top of this file). The - # run records it on the client Secret and never acts on it - # again; a follow-up change removes it. - - name: ADMIN_RECOVERY_NONCE - value: admin-recovery-2026-11-15 volumeMounts: - name: setup-token mountPath: /run/setup-token