From 985950bef23d063168230a1b982046e06c7001f1 Mon Sep 17 00:00:00 2001 From: Boris Bobrov Date: Sun, 22 Mar 2026 09:24:09 +0100 Subject: [PATCH 01/56] Add TLS termination and ingress passthrough support to Barbican Add pod-level TLS on top of the Apache migration. Barbican now serves on two ports: - Port 9311 (HTTP): internal traffic, protected by Linkerd mTLS (unchanged) - Port 443 (HTTPS): external traffic via ingress TLS passthrough (new) New file: - _tls-hardening.conf.tpl: TLS 1.3/1.2 cipher suite hardening with brainpool curve preference, AEAD-only, PFS-only cipher suites Modified files: - _wsgi-barbican.conf.tpl: add conditional HTTPS VirtualHost on 443 with SSL - api-deployment.yaml: TLS init container (copies key from K8s Secret to tmpfs at /mnt/secrets), TLS volumes, port 443, hash annotation, Stakater reloader - etc-configmap.yaml: add tls-hardening.conf entry when TLS enabled - service.yaml: add port 443 when TLS enabled - ingress.yaml: ssl-passthrough mode with Linkerd skip-inbound-ports annotation - values.yaml: tls section with all configuration options When tls.enabled is false, all behavior is identical to the Apache migration. Envelope encryption defaults to noop (ready for vault-transit via config flag). Co-authored-by: Rajiv Mucheli --- .../barbican/templates/api-deployment.yaml | 48 ++++++++++++++++++- .../barbican/templates/etc-configmap.yaml | 4 ++ .../templates/etc/_tls-hardening.conf.tpl | 33 +++++++++++++ .../templates/etc/_wsgi-barbican.conf.tpl | 27 +++++++++++ openstack/barbican/templates/ingress.yaml | 11 ++++- openstack/barbican/templates/service.yaml | 5 ++ openstack/barbican/values.yaml | 19 ++++++++ 7 files changed, 145 insertions(+), 2 deletions(-) create mode 100644 openstack/barbican/templates/etc/_tls-hardening.conf.tpl diff --git a/openstack/barbican/templates/api-deployment.yaml b/openstack/barbican/templates/api-deployment.yaml index fc6b81ed2f2..1a609e1e5d6 100644 --- a/openstack/barbican/templates/api-deployment.yaml +++ b/openstack/barbican/templates/api-deployment.yaml @@ -12,7 +12,7 @@ metadata: type: api component: barbican annotations: - secret.reloader.stakater.com/reload: "{{ .Release.Name }}-secrets" + secret.reloader.stakater.com/reload: "{{ .Release.Name }}-secrets{{- if .Values.tls.enabled }},{{ .Values.tls.secretName | default "barbican-tls" }}{{- end }}" deployment.reloader.stakater.com/pause-period: "60s" spec: replicas: {{ required ".Values.api.replicas is missing" .Values.api.replicas }} @@ -59,6 +59,29 @@ spec: {{- if .Values.proxysql.native_sidecar }} {{- include "utils.proxysql.container" . | indent 6 }} {{- end }} + {{- if .Values.tls.enabled }} + - name: tls-init + image: {{ required ".Values.global.registry is missing" .Values.global.registry }}/loci-barbican:{{ required "Values.imageVersionBarbicanApi is missing" .Values.imageVersionBarbicanApi }} + command: + - bash + - -c + - | + set -ex + # Copy TLS key material from K8s Secret to tmpfs + # In noop envelope-encryption mode, the key is plaintext in the Secret. + # When vault-transit or TPM is enabled, this step would decrypt first. + cp /tls-input/tls.key /mnt/secrets/tls.key + cp /tls-input/tls.crt /mnt/secrets/tls.crt + chmod 0440 /mnt/secrets/tls.key + chmod 0444 /mnt/secrets/tls.crt + echo "TLS key material placed on tmpfs" + volumeMounts: + - name: tls-input + mountPath: /tls-input + readOnly: true + - name: tls-secrets + mountPath: /mnt/secrets + {{- end }} containers: - name: barbican-api image: {{required ".Values.global.registry is missing" .Values.global.registry }}/loci-barbican:{{required "Values.imageVersionBarbicanApi is missing" .Values.imageVersionBarbicanApi}} @@ -115,6 +138,10 @@ spec: ports: - name: barbican-api containerPort: {{.Values.api_port_internal}} + {{- if .Values.tls.enabled }} + - name: barbican-tls + containerPort: 443 + {{- end }} volumeMounts: - name: etcbarbican mountPath: /etc/barbican @@ -138,6 +165,15 @@ spec: mountPath: /etc/barbican/logging.ini subPath: logging.ini readOnly: true + {{- if .Values.tls.enabled }} + - name: tls-secrets + mountPath: /mnt/secrets + readOnly: true + - name: barbican-etc + mountPath: /etc/apache2/conf-enabled/tls-hardening.conf + subPath: tls-hardening.conf + readOnly: true + {{- end }} - mountPath: /etc/barbican/barbican.conf.d name: barbican-etc-confd readOnly: true @@ -334,3 +370,13 @@ spec: {{- end }} {{- include "utils.proxysql.volumes" . | indent 8 }} {{- include "utils.trust_bundle.volumes" . | indent 8 }} + {{- if .Values.tls.enabled }} + - name: tls-input + secret: + secretName: {{ .Values.tls.secretName | default "barbican-tls" }} + defaultMode: 0440 + - name: tls-secrets + emptyDir: + medium: Memory + sizeLimit: 1Mi + {{- end }} diff --git a/openstack/barbican/templates/etc-configmap.yaml b/openstack/barbican/templates/etc-configmap.yaml index e735e25c787..6d93bcce19a 100644 --- a/openstack/barbican/templates/etc-configmap.yaml +++ b/openstack/barbican/templates/etc-configmap.yaml @@ -11,6 +11,10 @@ data: {{ include (print .Template.BasePath "/etc/_barbican-api-paste.ini.tpl") . | indent 4 }} barbican.conf: | {{ include (print .Template.BasePath "/etc/_barbican.conf.tpl") . | indent 4 }} +{{- if .Values.tls.enabled }} + tls-hardening.conf: | +{{ include (print .Template.BasePath "/etc/_tls-hardening.conf.tpl") . | indent 4 }} +{{- end }} policy.yaml: | {{ include (print .Template.BasePath "/etc/_barbican-policy.yaml.tpl") . | indent 4 }} logging.ini: | diff --git a/openstack/barbican/templates/etc/_tls-hardening.conf.tpl b/openstack/barbican/templates/etc/_tls-hardening.conf.tpl new file mode 100644 index 00000000000..5598d9dd0ce --- /dev/null +++ b/openstack/barbican/templates/etc/_tls-hardening.conf.tpl @@ -0,0 +1,33 @@ +{{/* +Apache TLS protocol hardening configuration. +Configures TLS 1.3/1.2 with compliant cipher suites, PFS, AEAD modes, +and brainpool curve preference. Shared between Keystone and Barbican. +*/}} + +# TLS Protocol Versions: only 1.3 and 1.2 +SSLProtocol -all +TLSv1.3 +TLSv1.2 + +# TLS 1.2 Cipher Suites (ECDHE + AEAD only, PFS required) +SSLCipherSuite ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-GCM-SHA256 + +# TLS 1.3 Cipher Suites +SSLCipherSuite TLSv1.3 TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256 + +# Server chooses cipher order +SSLHonorCipherOrder on + +# ECDHE Curve Preference: brainpool first, NIST as fallback +SSLOpenSSLConfCmd Curves brainpoolP256r1:brainpoolP384r1:prime256v1:secp384r1 + +# Signature algorithms preference +SSLOpenSSLConfCmd SignatureAlgorithms ecdsa_secp256r1_sha256:ecdsa_secp384r1_sha384:rsa_pss_rsae_sha256:rsa_pss_rsae_sha384 + +# Session cache +SSLSessionCache shmcb:/run/apache2/ssl_scache(512000) +SSLSessionCacheTimeout 300 + +# Disable session tickets (for PFS) +SSLSessionTickets off + +# Strict SNI +SSLStrictSNIVHostCheck on diff --git a/openstack/barbican/templates/etc/_wsgi-barbican.conf.tpl b/openstack/barbican/templates/etc/_wsgi-barbican.conf.tpl index 2a4441db1a5..8e66f25096a 100644 --- a/openstack/barbican/templates/etc/_wsgi-barbican.conf.tpl +++ b/openstack/barbican/templates/etc/_wsgi-barbican.conf.tpl @@ -16,6 +16,33 @@ WSGIDaemonProcess barbican-api processes={{ .Values.api.processes | default 1 }} Listen 0.0.0.0:{{ .Values.api_port_internal }} +{{- if .Values.tls.enabled }} +# External HTTPS endpoint (via Ingress TLS passthrough) +Listen 0.0.0.0:443 + + + SSLEngine on + SSLCertificateFile /mnt/secrets/tls.crt + SSLCertificateKeyFile /mnt/secrets/tls.key + + Include /etc/apache2/conf-enabled/tls-hardening.conf + + WSGIDaemonProcess barbican-api-tls processes={{ .Values.api.processes | default 1 }} threads=1 user=barbican group=barbican display-name=%{GROUP} + WSGIProcessGroup barbican-api-tls + WSGIScriptAlias / /var/www/cgi-bin/barbican/barbican-wsgi-api + WSGIApplicationGroup %{GLOBAL} + WSGIPassAuthorization On + + = 2.4> + ErrorLogFormat "%{cu}t %M" + + ErrorLog /dev/stderr + + KeepAliveTimeout 61 + +{{- end }} + +# Internal HTTP endpoint (protected by Linkerd mTLS at the network layer) ServerName {{ include "barbican_api_endpoint_host_public" . }} diff --git a/openstack/barbican/templates/ingress.yaml b/openstack/barbican/templates/ingress.yaml index 20ebb94a242..59cf32b7e98 100644 --- a/openstack/barbican/templates/ingress.yaml +++ b/openstack/barbican/templates/ingress.yaml @@ -13,7 +13,12 @@ metadata: type: api component: barbican annotations: - {{- if .Values.tlsacme }} + {{- if and .Values.tls.enabled .Values.tls.passthrough }} + nginx.ingress.kubernetes.io/ssl-passthrough: "true" + nginx.ingress.kubernetes.io/backend-protocol: "HTTPS" + config.linkerd.io/skip-inbound-ports: "443" + disco: "true" + {{- else if .Values.tlsacme }} kubernetes.io/tls-acme: "true" disco: "true" {{- end }} @@ -38,4 +43,8 @@ spec: service: name: barbican-api port: + {{- if and .Values.tls.enabled .Values.tls.passthrough }} + number: 443 + {{- else }} number: {{.Values.api_port_internal}} + {{- end }} diff --git a/openstack/barbican/templates/service.yaml b/openstack/barbican/templates/service.yaml index 9e9f6ca2899..2616d5b6e6b 100644 --- a/openstack/barbican/templates/service.yaml +++ b/openstack/barbican/templates/service.yaml @@ -23,3 +23,8 @@ spec: ports: - name: barbican-api port: {{.Values.api_port_internal}} + {{- if .Values.tls.enabled }} + - name: barbican-tls + port: 443 + targetPort: 443 + {{- end }} diff --git a/openstack/barbican/values.yaml b/openstack/barbican/values.yaml index 2993252f39f..bc008c51f27 100644 --- a/openstack/barbican/values.yaml +++ b/openstack/barbican/values.yaml @@ -489,6 +489,25 @@ ingress: enableConnectionsLimit: true limitConnections: 20 +# Pod-level TLS termination in Apache +tls: + enabled: false + # Name of the K8s Secret containing tls.key and tls.crt (created by TLS bootstrap Job) + secretName: barbican-tls + # Enable TLS passthrough on the ingress (sends encrypted stream directly to pod) + passthrough: false + # Envelope encryption mode for the private key: "none" (plaintext) or "vault-transit" + envelopeEncryption: none + # TLS lifecycle manager binary for bootstrap Job and rotation CronJob + lifecycle: + image: keppel.eu-de-1.cloud.sap/ccloud/tls-lifecycle-manager + imageTag: latest + # cert-manager issuer configuration + issuerName: digicert-issuer + issuerKind: ClusterDigicertIssuer + issuerGroup: certmanager.cloud.sap + keyAlgorithm: ECDSAP256 + # sapcc/openstack-rate-limit-middleware sapcc_rate_limit: enabled: true From c17f9f3145c6cca362e952ccaf4f592a781d389e Mon Sep 17 00:00:00 2001 From: Boris Bobrov Date: Sun, 22 Mar 2026 09:28:20 +0100 Subject: [PATCH 02/56] Add TLS termination and ingress passthrough support to Keystone Dual-port Apache: HTTPS on 443 (external) + HTTP on 5000 (internal). TLS hardening config (shared with Barbican): brainpool curves, PFS, AEAD. Init container copies key from K8s Secret to tmpfs at /mnt/secrets. Ingress passthrough mode with source-IP affinity and Linkerd skip-inbound-ports. SAML federation Location blocks included in the HTTPS VirtualHost. When tls.enabled is false, all behavior is identical to current deployment. --- .../keystone/templates/configmap-etc.yaml | 4 ++ .../keystone/templates/deployment-api.yaml | 46 ++++++++++++++++++- .../templates/etc/_tls-hardening.conf.tpl | 33 +++++++++++++ .../templates/etc/_wsgi-keystone.conf.tpl | 40 ++++++++++++++++ openstack/keystone/templates/ingress-api.yaml | 13 +++++- openstack/keystone/templates/service-api.yaml | 6 +++ openstack/keystone/values.yaml | 14 ++++++ 7 files changed, 153 insertions(+), 3 deletions(-) create mode 100644 openstack/keystone/templates/etc/_tls-hardening.conf.tpl diff --git a/openstack/keystone/templates/configmap-etc.yaml b/openstack/keystone/templates/configmap-etc.yaml index 810015e3084..175d0caa2bb 100644 --- a/openstack/keystone/templates/configmap-etc.yaml +++ b/openstack/keystone/templates/configmap-etc.yaml @@ -21,6 +21,10 @@ data: {{ include (print .Template.BasePath "/etc/_sso_callback_template.html.tpl") . | indent 4 }} wsgi-keystone.conf: | {{ include (print .Template.BasePath "/etc/_wsgi-keystone.conf.tpl") . | indent 4 }} +{{- if .Values.tls.enabled }} + tls-hardening.conf: | +{{ include (print .Template.BasePath "/etc/_tls-hardening.conf.tpl") . | indent 4 }} +{{- end }} {{- if .Values.watcher.enabled }} watcher.yaml: | {{ include (print .Template.BasePath "/etc/_watcher.yaml.tpl") . | indent 4 }} diff --git a/openstack/keystone/templates/deployment-api.yaml b/openstack/keystone/templates/deployment-api.yaml index 0b7718e5486..4cb6b2b304b 100644 --- a/openstack/keystone/templates/deployment-api.yaml +++ b/openstack/keystone/templates/deployment-api.yaml @@ -12,7 +12,7 @@ metadata: component: keystone type: api annotations: - secret.reloader.stakater.com/reload: "{{ .Release.Name }}-secrets,{{ .Release.Name }}-federation{{- if .Values.federation.saml.enabled }},{{ .Release.Name }}-federation-saml,{{ .Values.federation.saml.idp.spKeySecretName | default "keystone-saml-sp" }}{{- end }}" + secret.reloader.stakater.com/reload: "{{ .Release.Name }}-secrets,{{ .Release.Name }}-federation{{- if .Values.federation.saml.enabled }},{{ .Release.Name }}-federation-saml,{{ .Values.federation.saml.idp.spKeySecretName | default "keystone-saml-sp" }}{{- end }}{{- if .Values.tls.enabled }},{{ .Values.tls.secretName | default "keystone-tls" }}{{- end }}" {{- if .Values.federation.saml.enabled }} configmap.reloader.stakater.com/reload: "keystone-saml-shibboleth-config,keystone-saml-federation-saml-tenants,{{ .Values.federation.saml.idp.metadataConfigMap }}" {{- end }} @@ -54,6 +54,9 @@ spec: {{- if .Values.federation.saml.enabled }} federation-saml-hash: {{ include (print $.Template.BasePath "/federation-saml.yaml") . | sha256sum }} {{- end }} + {{- if .Values.tls.enabled }} + tls-secret-hash: {{ .Values.tls.secretName | default "keystone-tls" }} + {{- end }} {{- if .Values.api.metrics.enabled }} prometheus.io/scrape: "true" prometheus.io/targets: {{ required ".Values.alerts.prometheus missing" .Values.alerts.prometheus | quote }} @@ -107,6 +110,24 @@ spec: - name: saml-sp-tmpfs mountPath: /saml-sp-keys {{- end }} + {{- if .Values.tls.enabled }} + - name: tls-init + image: {{ required ".Values.global.dockerHubMirror is missing" .Values.global.dockerHubMirror }}/library/busybox + command: ["sh", "-c"] + args: + - | + cp /tls-input/tls.key /mnt/secrets/tls.key + cp /tls-input/tls.crt /mnt/secrets/tls.crt + chmod 0440 /mnt/secrets/tls.key + chmod 0444 /mnt/secrets/tls.crt + echo "TLS key material placed on tmpfs" + volumeMounts: + - name: tls-input + mountPath: /tls-input + readOnly: true + - name: tls-secrets + mountPath: /mnt/secrets + {{- end }} containers: - name: keystone-api image: {{ required ".Values.global.registryAlternateRegion is missing" .Values.global.registryAlternateRegion }}/{{ .Values.api.image }}:{{ required ".Values.api.imageTag is missing" .Values.api.imageTag }} @@ -156,6 +177,10 @@ spec: ports: - name: public containerPort: 5000 + {{- if .Values.tls.enabled }} + - name: public-tls + containerPort: 443 + {{- end }} lifecycle: preStop: exec: @@ -244,6 +269,15 @@ spec: mountPath: /etc/shibboleth/metadata readOnly: true {{- end }} + {{- if .Values.tls.enabled }} + - name: tls-secrets + mountPath: /mnt/secrets + readOnly: true + - name: keystone-etc + mountPath: /etc/apache2/conf-enabled/tls-hardening.conf + subPath: tls-hardening.conf + readOnly: true + {{- end }} - name: keystone-etc mountPath: /etc/apache2/mods-available/mpm_event.conf subPath: mpm_event.conf @@ -367,3 +401,13 @@ spec: name: keystone-sentry-filter-rules {{- include "utils.proxysql.volumes" . | indent 8 }} {{- include "utils.trust_bundle.volumes" . | indent 8 }} + {{- if .Values.tls.enabled }} + - name: tls-input + secret: + secretName: {{ .Values.tls.secretName | default "keystone-tls" }} + defaultMode: 0440 + - name: tls-secrets + emptyDir: + medium: Memory + sizeLimit: 1Mi + {{- end }} diff --git a/openstack/keystone/templates/etc/_tls-hardening.conf.tpl b/openstack/keystone/templates/etc/_tls-hardening.conf.tpl new file mode 100644 index 00000000000..5598d9dd0ce --- /dev/null +++ b/openstack/keystone/templates/etc/_tls-hardening.conf.tpl @@ -0,0 +1,33 @@ +{{/* +Apache TLS protocol hardening configuration. +Configures TLS 1.3/1.2 with compliant cipher suites, PFS, AEAD modes, +and brainpool curve preference. Shared between Keystone and Barbican. +*/}} + +# TLS Protocol Versions: only 1.3 and 1.2 +SSLProtocol -all +TLSv1.3 +TLSv1.2 + +# TLS 1.2 Cipher Suites (ECDHE + AEAD only, PFS required) +SSLCipherSuite ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-GCM-SHA256 + +# TLS 1.3 Cipher Suites +SSLCipherSuite TLSv1.3 TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256 + +# Server chooses cipher order +SSLHonorCipherOrder on + +# ECDHE Curve Preference: brainpool first, NIST as fallback +SSLOpenSSLConfCmd Curves brainpoolP256r1:brainpoolP384r1:prime256v1:secp384r1 + +# Signature algorithms preference +SSLOpenSSLConfCmd SignatureAlgorithms ecdsa_secp256r1_sha256:ecdsa_secp384r1_sha384:rsa_pss_rsae_sha256:rsa_pss_rsae_sha384 + +# Session cache +SSLSessionCache shmcb:/run/apache2/ssl_scache(512000) +SSLSessionCacheTimeout 300 + +# Disable session tickets (for PFS) +SSLSessionTickets off + +# Strict SNI +SSLStrictSNIVHostCheck on diff --git a/openstack/keystone/templates/etc/_wsgi-keystone.conf.tpl b/openstack/keystone/templates/etc/_wsgi-keystone.conf.tpl index 94b960cece6..6b20198807d 100644 --- a/openstack/keystone/templates/etc/_wsgi-keystone.conf.tpl +++ b/openstack/keystone/templates/etc/_wsgi-keystone.conf.tpl @@ -73,6 +73,46 @@ CustomLog /dev/stdout proxy env=forwarded WSGIServerMetrics On {{- end }} +{{- if .Values.tls.enabled }} +# External HTTPS endpoint (via Ingress TLS passthrough) +Listen 0.0.0.0:443 + + + ServerName {{ .Values.services.public.host }}.{{ .Values.global.region }}.{{ .Values.global.tld }} + + SSLEngine on + SSLCertificateFile /mnt/secrets/tls.crt + SSLCertificateKeyFile /mnt/secrets/tls.key + Include /etc/apache2/conf-enabled/tls-hardening.conf + + WSGIDaemonProcess keystone-tls processes=8 threads=1 user=keystone group=keystone display-name=%{GROUP} + WSGIProcessGroup keystone-tls + WSGIScriptAlias / /var/www/cgi-bin/keystone/keystone-wsgi-public + WSGIApplicationGroup %{GLOBAL} + WSGIPassAuthorization On + LimitRequestBody 114688 + LimitRequestFieldSize 16380 + = 2.4> + ErrorLogFormat "%{cu}t %M" + + ErrorLog /dev/stdout + + SetEnvIf X-Forwarded-For "^.*\..*\..*\..*" forwarded + CustomLog /dev/stdout combined env=!forwarded + CustomLog /dev/stdout proxy env=forwarded + + KeepAliveTimeout 61 + + {{- if .Values.federation.saml.enabled }} + + SetHandler shib + + IncludeOptional /etc/apache2/conf-enabled/federation-saml.conf + {{- end }} + +{{- end }} + +# Internal HTTP endpoint (protected by Linkerd mTLS at the network layer) ServerName {{ .Values.services.public.host }}.{{ .Values.global.region }}.{{ .Values.global.tld }} WSGIDaemonProcess keystone-public processes=8 threads=1 user=keystone group=keystone display-name=%{GROUP} diff --git a/openstack/keystone/templates/ingress-api.yaml b/openstack/keystone/templates/ingress-api.yaml index 22b68557f49..20a05f30764 100644 --- a/openstack/keystone/templates/ingress-api.yaml +++ b/openstack/keystone/templates/ingress-api.yaml @@ -46,7 +46,12 @@ metadata: component: keystone type: api annotations: - {{- if or .Values.federation.oidc.enabled .Values.federation.saml.enabled }} + {{- if and .Values.tls.enabled .Values.tls.passthrough }} + nginx.ingress.kubernetes.io/ssl-passthrough: "true" + nginx.ingress.kubernetes.io/backend-protocol: "HTTPS" + config.linkerd.io/skip-inbound-ports: "443" + nginx.ingress.kubernetes.io/upstream-hash-by: "$remote_addr" + {{- else if or .Values.federation.oidc.enabled .Values.federation.saml.enabled }} # the affinity bits are required for federation; they do not affect the # cases when the API is used, but will affect browser-based workflows ingress.kubernetes.io/affinity: cookie @@ -79,7 +84,7 @@ metadata: ingress.kubernetes.io/auth-tls-verify-depth: "3" nginx.ingress.kubernetes.io/auth-tls-verify-depth: "3" {{- end }} - {{- if .Values.services.ingress.tlsacme }} + {{- if and .Values.services.ingress.tlsacme (not (and .Values.tls.enabled .Values.tls.passthrough)) }} kubernetes.io/tls-acme: "true" {{- end }} {{- if .Values.global.is_global_region }} @@ -123,5 +128,9 @@ spec: service: name: "{{if .Values.global.is_global_region }}keystone-global{{else}}keystone{{end}}" port: + {{- if and .Values.tls.enabled .Values.tls.passthrough }} + number: 443 + {{- else }} number: 5000 + {{- end }} {{- end }} diff --git a/openstack/keystone/templates/service-api.yaml b/openstack/keystone/templates/service-api.yaml index 15f7d815eb0..baf7e0d6226 100644 --- a/openstack/keystone/templates/service-api.yaml +++ b/openstack/keystone/templates/service-api.yaml @@ -30,6 +30,12 @@ spec: protocol: "TCP" port: 5000 targetPort: 5000 + {{- if .Values.tls.enabled }} + - name: public-tls + protocol: "TCP" + port: 443 + targetPort: 443 + {{- end }} {{- if .Values.global.keystone_internal_ip }} externalIPs: - {{ .Values.global.keystone_internal_ip | quote }} diff --git a/openstack/keystone/values.yaml b/openstack/keystone/values.yaml index 117193f9df4..48165fab39e 100644 --- a/openstack/keystone/values.yaml +++ b/openstack/keystone/values.yaml @@ -57,6 +57,20 @@ osprofiler-redis: alerts: support_group: identity +# Pod-level TLS termination in Apache +tls: + enabled: false + secretName: keystone-tls + passthrough: false + envelopeEncryption: none + lifecycle: + image: keppel.eu-de-1.cloud.sap/ccloud/tls-lifecycle-manager + imageTag: latest + issuerName: digicert-issuer + issuerKind: ClusterDigicertIssuer + issuerGroup: certmanager.cloud.sap + keyAlgorithm: ECDSAP256 + federation: oidc: enabled: false From a0c6294ac8bc47d23c15740d8a5d419b1eed6e5f Mon Sep 17 00:00:00 2001 From: Boris Bobrov Date: Sun, 22 Mar 2026 09:29:07 +0100 Subject: [PATCH 03/56] Add TLS bootstrap Job and weekly rotation CronJob to Keystone Bootstrap Job (Helm pre-install/pre-upgrade hook, weight -5) ensures a valid TLS certificate exists before pods start. Rotation CronJob runs weekly to renew certificates 30 days before expiry. Both use the shared tls-lifecycle-manager Go binary with noop envelope encryption. RBAC: ServiceAccount + Role for Secrets, CertificateRequests, Deployments. --- .../templates/cronjob-tls-rotation.yaml | 41 +++++++++++ .../keystone/templates/job-tls-bootstrap.yaml | 69 +++++++++++++++++++ 2 files changed, 110 insertions(+) create mode 100644 openstack/keystone/templates/cronjob-tls-rotation.yaml create mode 100644 openstack/keystone/templates/job-tls-bootstrap.yaml diff --git a/openstack/keystone/templates/cronjob-tls-rotation.yaml b/openstack/keystone/templates/cronjob-tls-rotation.yaml new file mode 100644 index 00000000000..1c71c0acbff --- /dev/null +++ b/openstack/keystone/templates/cronjob-tls-rotation.yaml @@ -0,0 +1,41 @@ +{{- if .Values.tls.enabled }} +apiVersion: batch/v1 +kind: CronJob +metadata: + name: {{ .Release.Name }}-tls-rotation + labels: + app: {{ template "fullname" . }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: "{{ .Release.Name }}" + component: keystone + type: tls-rotation +spec: + schedule: "0 3 * * 1" + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 3 + jobTemplate: + spec: + template: + spec: + restartPolicy: Never + serviceAccountName: {{ .Release.Name }}-tls-manager + containers: + - name: tls-rotation + image: {{ .Values.tls.lifecycle.image }}:{{ .Values.tls.lifecycle.imageTag }} + imagePullPolicy: IfNotPresent + args: + - --mode=rotation + - --namespace={{ .Release.Namespace }} + - --secret-name={{ .Values.tls.secretName | default "keystone-tls" }} + - --key-algorithm={{ .Values.tls.keyAlgorithm | default "ECDSAP256" }} + - --issuer-name={{ .Values.tls.issuerName | default "digicert-issuer" }} + - --issuer-kind={{ .Values.tls.issuerKind | default "ClusterDigicertIssuer" }} + - --issuer-group={{ .Values.tls.issuerGroup | default "certmanager.cloud.sap" }} + - --common-name={{ .Values.services.public.host }}.{{ .Values.global.region }}.{{ .Values.global.tld }} + - --san-dns={{ .Values.services.public.host }}.{{ .Values.global.region }}.{{ .Values.global.tld }} + - --renewal-threshold-days=30 + - --deployment-name={{ .Release.Name }}-api + - --envelope-encryption={{ .Values.tls.envelopeEncryption | default "none" }} + backoffLimit: 3 +{{- end }} diff --git a/openstack/keystone/templates/job-tls-bootstrap.yaml b/openstack/keystone/templates/job-tls-bootstrap.yaml new file mode 100644 index 00000000000..8d3115e038c --- /dev/null +++ b/openstack/keystone/templates/job-tls-bootstrap.yaml @@ -0,0 +1,69 @@ +{{- if .Values.tls.enabled }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ .Release.Name }}-tls-bootstrap + labels: + app: {{ template "fullname" . }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: "{{ .Release.Name }}" + component: keystone + type: tls-bootstrap + annotations: + "helm.sh/hook": pre-install,pre-upgrade + "helm.sh/hook-weight": "-5" + "helm.sh/hook-delete-policy": before-hook-creation +spec: + template: + spec: + restartPolicy: Never + serviceAccountName: {{ .Release.Name }}-tls-manager + containers: + - name: tls-bootstrap + image: {{ .Values.tls.lifecycle.image }}:{{ .Values.tls.lifecycle.imageTag }} + imagePullPolicy: IfNotPresent + args: + - --mode=bootstrap + - --namespace={{ .Release.Namespace }} + - --secret-name={{ .Values.tls.secretName | default "keystone-tls" }} + - --key-algorithm={{ .Values.tls.keyAlgorithm | default "ECDSAP256" }} + - --issuer-name={{ .Values.tls.issuerName | default "digicert-issuer" }} + - --issuer-kind={{ .Values.tls.issuerKind | default "ClusterDigicertIssuer" }} + - --issuer-group={{ .Values.tls.issuerGroup | default "certmanager.cloud.sap" }} + - --common-name={{ .Values.services.public.host }}.{{ .Values.global.region }}.{{ .Values.global.tld }} + - --san-dns={{ .Values.services.public.host }}.{{ .Values.global.region }}.{{ .Values.global.tld }} + - --envelope-encryption={{ .Values.tls.envelopeEncryption | default "none" }} + backoffLimit: 3 +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Release.Name }}-tls-manager +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ .Release.Name }}-tls-manager +rules: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "create", "update"] + - apiGroups: ["cert-manager.io"] + resources: ["certificaterequests"] + verbs: ["create", "get", "delete", "watch"] + - apiGroups: ["apps"] + resources: ["deployments"] + verbs: ["patch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ .Release.Name }}-tls-manager +subjects: + - kind: ServiceAccount + name: {{ .Release.Name }}-tls-manager +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ .Release.Name }}-tls-manager +{{- end }} From 0822dbd3f86308697d446046810ffbb39ecc317f Mon Sep 17 00:00:00 2001 From: Boris Bobrov Date: Sun, 22 Mar 2026 09:29:16 +0100 Subject: [PATCH 04/56] Add TLS bootstrap Job and weekly rotation CronJob to Barbican Same pattern as Keystone. Bootstrap Job runs at weight -5 (before the existing migration-job at weight -1). Rotation CronJob runs weekly. Co-authored-by: Rajiv Mucheli --- .../templates/cronjob-tls-rotation.yaml | 41 +++++++++++ .../barbican/templates/job-tls-bootstrap.yaml | 69 +++++++++++++++++++ 2 files changed, 110 insertions(+) create mode 100644 openstack/barbican/templates/cronjob-tls-rotation.yaml create mode 100644 openstack/barbican/templates/job-tls-bootstrap.yaml diff --git a/openstack/barbican/templates/cronjob-tls-rotation.yaml b/openstack/barbican/templates/cronjob-tls-rotation.yaml new file mode 100644 index 00000000000..2b25309f0d7 --- /dev/null +++ b/openstack/barbican/templates/cronjob-tls-rotation.yaml @@ -0,0 +1,41 @@ +{{- if .Values.tls.enabled }} +apiVersion: batch/v1 +kind: CronJob +metadata: + name: {{ .Release.Name }}-tls-rotation + labels: + app: {{ template "fullname" . }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: "{{ .Release.Name }}" + component: barbican + type: tls-rotation +spec: + schedule: "0 3 * * 1" + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 3 + jobTemplate: + spec: + template: + spec: + restartPolicy: Never + serviceAccountName: {{ .Release.Name }}-tls-manager + containers: + - name: tls-rotation + image: {{ .Values.tls.lifecycle.image }}:{{ .Values.tls.lifecycle.imageTag }} + imagePullPolicy: IfNotPresent + args: + - --mode=rotation + - --namespace={{ .Release.Namespace }} + - --secret-name={{ .Values.tls.secretName | default "barbican-tls" }} + - --key-algorithm={{ .Values.tls.keyAlgorithm | default "ECDSAP256" }} + - --issuer-name={{ .Values.tls.issuerName | default "digicert-issuer" }} + - --issuer-kind={{ .Values.tls.issuerKind | default "ClusterDigicertIssuer" }} + - --issuer-group={{ .Values.tls.issuerGroup | default "certmanager.cloud.sap" }} + - --common-name={{ include "barbican_api_endpoint_host_public" . }} + - --san-dns={{ include "barbican_api_endpoint_host_public" . }} + - --renewal-threshold-days=30 + - --deployment-name=barbican-api + - --envelope-encryption={{ .Values.tls.envelopeEncryption | default "none" }} + backoffLimit: 3 +{{- end }} diff --git a/openstack/barbican/templates/job-tls-bootstrap.yaml b/openstack/barbican/templates/job-tls-bootstrap.yaml new file mode 100644 index 00000000000..5f801b8126a --- /dev/null +++ b/openstack/barbican/templates/job-tls-bootstrap.yaml @@ -0,0 +1,69 @@ +{{- if .Values.tls.enabled }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ .Release.Name }}-tls-bootstrap + labels: + app: {{ template "fullname" . }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: "{{ .Release.Name }}" + component: barbican + type: tls-bootstrap + annotations: + "helm.sh/hook": pre-install,pre-upgrade + "helm.sh/hook-weight": "-5" + "helm.sh/hook-delete-policy": before-hook-creation +spec: + template: + spec: + restartPolicy: Never + serviceAccountName: {{ .Release.Name }}-tls-manager + containers: + - name: tls-bootstrap + image: {{ .Values.tls.lifecycle.image }}:{{ .Values.tls.lifecycle.imageTag }} + imagePullPolicy: IfNotPresent + args: + - --mode=bootstrap + - --namespace={{ .Release.Namespace }} + - --secret-name={{ .Values.tls.secretName | default "barbican-tls" }} + - --key-algorithm={{ .Values.tls.keyAlgorithm | default "ECDSAP256" }} + - --issuer-name={{ .Values.tls.issuerName | default "digicert-issuer" }} + - --issuer-kind={{ .Values.tls.issuerKind | default "ClusterDigicertIssuer" }} + - --issuer-group={{ .Values.tls.issuerGroup | default "certmanager.cloud.sap" }} + - --common-name={{ include "barbican_api_endpoint_host_public" . }} + - --san-dns={{ include "barbican_api_endpoint_host_public" . }} + - --envelope-encryption={{ .Values.tls.envelopeEncryption | default "none" }} + backoffLimit: 3 +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Release.Name }}-tls-manager +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ .Release.Name }}-tls-manager +rules: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "create", "update"] + - apiGroups: ["cert-manager.io"] + resources: ["certificaterequests"] + verbs: ["create", "get", "delete", "watch"] + - apiGroups: ["apps"] + resources: ["deployments"] + verbs: ["patch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ .Release.Name }}-tls-manager +subjects: + - kind: ServiceAccount + name: {{ .Release.Name }}-tls-manager +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ .Release.Name }}-tls-manager +{{- end }} From 1ca16e3ccf033cc87b3223981875f347ac69ce14 Mon Sep 17 00:00:00 2001 From: Boris Bobrov Date: Wed, 25 Mar 2026 11:16:31 +0100 Subject: [PATCH 05/56] Add Prometheus alerts for TLS certificate rotation failures MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fires when the weekly TLS rotation CronJob fails for either service. Severity: critical — if rotation keeps failing, the TLS certificate will expire and external HTTPS access will break. Keystone: OpenstackKeystoneTLSRotationFailed Barbican: OpenstackBarbicanTLSRotationFailed Both trigger after 1 hour of failure (one retry window). --- .../openstack/openstack-barbican.alerts | 14 ++++++++++ .../keystone/alerts/openstack/keystone.alerts | 27 +++++++++++++++++++ 2 files changed, 41 insertions(+) diff --git a/openstack/barbican/alerts/openstack/openstack-barbican.alerts b/openstack/barbican/alerts/openstack/openstack-barbican.alerts index 368ec46d229..2710e32d31b 100644 --- a/openstack/barbican/alerts/openstack/openstack-barbican.alerts +++ b/openstack/barbican/alerts/openstack/openstack-barbican.alerts @@ -116,3 +116,17 @@ groups: annotations: description: All barbican-api server pods are down. summary: barbican is unavailable. + + - alert: OpenstackBarbicanTLSRotationFailed + expr: kube_job_status_failed{job_name=~"barbican-tls-rotation.*"} > 0 + for: 1h + labels: + context: security + dashboard: barbican + service: barbican + severity: critical + tier: os + support_group: identity + annotations: + description: The weekly TLS certificate rotation CronJob for Barbican has failed. If not resolved, the TLS certificate will expire and external HTTPS access will break. Check the CronJob logs. + summary: Barbican TLS certificate rotation failed diff --git a/openstack/keystone/alerts/openstack/keystone.alerts b/openstack/keystone/alerts/openstack/keystone.alerts index b9ed5b48d6d..2320e99e3d5 100644 --- a/openstack/keystone/alerts/openstack/keystone.alerts +++ b/openstack/keystone/alerts/openstack/keystone.alerts @@ -91,3 +91,30 @@ groups: description: Api Requests to Keystone is {{ $value }}ops/s summary: Api Requests to Keystone is {{ $value }}ops/s + - alert: OpenstackKeystoneSAMLAssertionValidationFailed + expr: increase(apache_error_log_saml_failures_total{service="keystone"}[5m]) > 0 + for: 1m + labels: + context: security + dashboard: keystone + service: keystone + severity: warning + tier: os + support_group: identity + annotations: + description: mod_shib rejected {{ $value }} SAML assertion(s) in the last 5 minutes. Check Apache error logs for signature, timing, or issuer validation failures. + summary: SAML assertion validation failures detected in Keystone + + - alert: OpenstackKeystoneTLSRotationFailed + expr: kube_job_status_failed{job_name=~"keystone-tls-rotation.*"} > 0 + for: 1h + labels: + context: security + dashboard: keystone + service: keystone + severity: critical + tier: os + support_group: identity + annotations: + description: The weekly TLS certificate rotation CronJob for Keystone has failed. If not resolved, the TLS certificate will expire and external HTTPS access will break. Check the CronJob logs. + summary: Keystone TLS certificate rotation failed From 1cbbd6c6630eafd04320b6115b35a980bc66a556 Mon Sep 17 00:00:00 2001 From: Boris Bobrov Date: Sun, 29 Mar 2026 23:18:15 +0200 Subject: [PATCH 06/56] Use Keystone LOCI image for TLS init container instead of busybox The init container only needs bash and cp, which are available in the Keystone LOCI image. This avoids pulling an external busybox image and ensures all images come from Keppel. --- openstack/keystone/templates/deployment-api.yaml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/openstack/keystone/templates/deployment-api.yaml b/openstack/keystone/templates/deployment-api.yaml index 4cb6b2b304b..46fb9460f5b 100644 --- a/openstack/keystone/templates/deployment-api.yaml +++ b/openstack/keystone/templates/deployment-api.yaml @@ -112,10 +112,12 @@ spec: {{- end }} {{- if .Values.tls.enabled }} - name: tls-init - image: {{ required ".Values.global.dockerHubMirror is missing" .Values.global.dockerHubMirror }}/library/busybox - command: ["sh", "-c"] - args: + image: {{required ".Values.global.registryAlternateRegion is missing" .Values.global.registryAlternateRegion}}/loci-keystone:{{required ".Values.api.imageTag is missing" .Values.api.imageTag}} + command: + - bash + - -c - | + set -ex cp /tls-input/tls.key /mnt/secrets/tls.key cp /tls-input/tls.crt /mnt/secrets/tls.crt chmod 0440 /mnt/secrets/tls.key From 9916149a2768c7fa60f46e554a4ba6bccbba33c9 Mon Sep 17 00:00:00 2001 From: Boris Bobrov Date: Mon, 30 Mar 2026 08:26:34 +0200 Subject: [PATCH 07/56] Lower TLS rotation alert severity from critical to warning MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The rotation CronJob renews certificates 30 days before expiry. Even if several weekly runs fail, there are weeks of runway before impact. Warning severity is appropriate — critical should be reserved for imminent outages. --- openstack/barbican/alerts/openstack/openstack-barbican.alerts | 2 +- openstack/keystone/alerts/openstack/keystone.alerts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/openstack/barbican/alerts/openstack/openstack-barbican.alerts b/openstack/barbican/alerts/openstack/openstack-barbican.alerts index 2710e32d31b..f839c6dd282 100644 --- a/openstack/barbican/alerts/openstack/openstack-barbican.alerts +++ b/openstack/barbican/alerts/openstack/openstack-barbican.alerts @@ -124,7 +124,7 @@ groups: context: security dashboard: barbican service: barbican - severity: critical + severity: warning tier: os support_group: identity annotations: diff --git a/openstack/keystone/alerts/openstack/keystone.alerts b/openstack/keystone/alerts/openstack/keystone.alerts index 2320e99e3d5..f4587ed6e8f 100644 --- a/openstack/keystone/alerts/openstack/keystone.alerts +++ b/openstack/keystone/alerts/openstack/keystone.alerts @@ -112,7 +112,7 @@ groups: context: security dashboard: keystone service: keystone - severity: critical + severity: warning tier: os support_group: identity annotations: From 603bda8da3e269dd14ecda264edcd156fc79974e Mon Sep 17 00:00:00 2001 From: Boris Bobrov Date: Mon, 30 Mar 2026 08:53:31 +0200 Subject: [PATCH 08/56] Use DigicertIssuer (namespace-scoped) instead of ClusterDigicertIssuer The ClusterDigicertIssuer controller is not active in any region. The namespace-scoped DigicertIssuer in kube-system is the one that processes CertificateRequests. The digicert-issuer controller falls back to kube-system when it doesn't find the issuer in the CR's namespace. --- openstack/barbican/templates/cronjob-tls-rotation.yaml | 2 +- openstack/barbican/templates/job-tls-bootstrap.yaml | 2 +- openstack/barbican/values.yaml | 2 +- openstack/keystone/templates/cronjob-tls-rotation.yaml | 2 +- openstack/keystone/templates/job-tls-bootstrap.yaml | 2 +- openstack/keystone/values.yaml | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/openstack/barbican/templates/cronjob-tls-rotation.yaml b/openstack/barbican/templates/cronjob-tls-rotation.yaml index 2b25309f0d7..a778cf1f023 100644 --- a/openstack/barbican/templates/cronjob-tls-rotation.yaml +++ b/openstack/barbican/templates/cronjob-tls-rotation.yaml @@ -30,7 +30,7 @@ spec: - --secret-name={{ .Values.tls.secretName | default "barbican-tls" }} - --key-algorithm={{ .Values.tls.keyAlgorithm | default "ECDSAP256" }} - --issuer-name={{ .Values.tls.issuerName | default "digicert-issuer" }} - - --issuer-kind={{ .Values.tls.issuerKind | default "ClusterDigicertIssuer" }} + - --issuer-kind={{ .Values.tls.issuerKind | default "DigicertIssuer" }} - --issuer-group={{ .Values.tls.issuerGroup | default "certmanager.cloud.sap" }} - --common-name={{ include "barbican_api_endpoint_host_public" . }} - --san-dns={{ include "barbican_api_endpoint_host_public" . }} diff --git a/openstack/barbican/templates/job-tls-bootstrap.yaml b/openstack/barbican/templates/job-tls-bootstrap.yaml index 5f801b8126a..798dc7278fd 100644 --- a/openstack/barbican/templates/job-tls-bootstrap.yaml +++ b/openstack/barbican/templates/job-tls-bootstrap.yaml @@ -28,7 +28,7 @@ spec: - --secret-name={{ .Values.tls.secretName | default "barbican-tls" }} - --key-algorithm={{ .Values.tls.keyAlgorithm | default "ECDSAP256" }} - --issuer-name={{ .Values.tls.issuerName | default "digicert-issuer" }} - - --issuer-kind={{ .Values.tls.issuerKind | default "ClusterDigicertIssuer" }} + - --issuer-kind={{ .Values.tls.issuerKind | default "DigicertIssuer" }} - --issuer-group={{ .Values.tls.issuerGroup | default "certmanager.cloud.sap" }} - --common-name={{ include "barbican_api_endpoint_host_public" . }} - --san-dns={{ include "barbican_api_endpoint_host_public" . }} diff --git a/openstack/barbican/values.yaml b/openstack/barbican/values.yaml index bc008c51f27..71f019a6110 100644 --- a/openstack/barbican/values.yaml +++ b/openstack/barbican/values.yaml @@ -504,7 +504,7 @@ tls: imageTag: latest # cert-manager issuer configuration issuerName: digicert-issuer - issuerKind: ClusterDigicertIssuer + issuerKind: DigicertIssuer issuerGroup: certmanager.cloud.sap keyAlgorithm: ECDSAP256 diff --git a/openstack/keystone/templates/cronjob-tls-rotation.yaml b/openstack/keystone/templates/cronjob-tls-rotation.yaml index 1c71c0acbff..a2537eef4ac 100644 --- a/openstack/keystone/templates/cronjob-tls-rotation.yaml +++ b/openstack/keystone/templates/cronjob-tls-rotation.yaml @@ -30,7 +30,7 @@ spec: - --secret-name={{ .Values.tls.secretName | default "keystone-tls" }} - --key-algorithm={{ .Values.tls.keyAlgorithm | default "ECDSAP256" }} - --issuer-name={{ .Values.tls.issuerName | default "digicert-issuer" }} - - --issuer-kind={{ .Values.tls.issuerKind | default "ClusterDigicertIssuer" }} + - --issuer-kind={{ .Values.tls.issuerKind | default "DigicertIssuer" }} - --issuer-group={{ .Values.tls.issuerGroup | default "certmanager.cloud.sap" }} - --common-name={{ .Values.services.public.host }}.{{ .Values.global.region }}.{{ .Values.global.tld }} - --san-dns={{ .Values.services.public.host }}.{{ .Values.global.region }}.{{ .Values.global.tld }} diff --git a/openstack/keystone/templates/job-tls-bootstrap.yaml b/openstack/keystone/templates/job-tls-bootstrap.yaml index 8d3115e038c..c84456f6d7f 100644 --- a/openstack/keystone/templates/job-tls-bootstrap.yaml +++ b/openstack/keystone/templates/job-tls-bootstrap.yaml @@ -28,7 +28,7 @@ spec: - --secret-name={{ .Values.tls.secretName | default "keystone-tls" }} - --key-algorithm={{ .Values.tls.keyAlgorithm | default "ECDSAP256" }} - --issuer-name={{ .Values.tls.issuerName | default "digicert-issuer" }} - - --issuer-kind={{ .Values.tls.issuerKind | default "ClusterDigicertIssuer" }} + - --issuer-kind={{ .Values.tls.issuerKind | default "DigicertIssuer" }} - --issuer-group={{ .Values.tls.issuerGroup | default "certmanager.cloud.sap" }} - --common-name={{ .Values.services.public.host }}.{{ .Values.global.region }}.{{ .Values.global.tld }} - --san-dns={{ .Values.services.public.host }}.{{ .Values.global.region }}.{{ .Values.global.tld }} diff --git a/openstack/keystone/values.yaml b/openstack/keystone/values.yaml index 48165fab39e..7f4e11cefa3 100644 --- a/openstack/keystone/values.yaml +++ b/openstack/keystone/values.yaml @@ -67,7 +67,7 @@ tls: image: keppel.eu-de-1.cloud.sap/ccloud/tls-lifecycle-manager imageTag: latest issuerName: digicert-issuer - issuerKind: ClusterDigicertIssuer + issuerKind: DigicertIssuer issuerGroup: certmanager.cloud.sap keyAlgorithm: ECDSAP256 From f6ef1b04bf3a5ae11ed4e02da430eb23f5ad0f6d Mon Sep 17 00:00:00 2001 From: Boris Bobrov Date: Mon, 30 Mar 2026 09:07:52 +0200 Subject: [PATCH 09/56] Move TLS manager RBAC out of Helm hook into regular templates The ServiceAccount, Role, and RoleBinding were in the same template as the pre-install/pre-upgrade hook Job. Helm treats all resources in a hook template as hooks, causing the SA to either not exist when the Job starts or get deleted by the before-hook-creation policy. Move RBAC to separate rbac-tls-manager.yaml files (regular templates, not hooks). These are created as normal Helm resources before any hooks run, ensuring the ServiceAccount exists when the bootstrap Job starts. --- .../barbican/templates/job-tls-bootstrap.yaml | 32 --------------- .../barbican/templates/rbac-tls-manager.yaml | 39 +++++++++++++++++++ .../keystone/templates/job-tls-bootstrap.yaml | 32 --------------- .../keystone/templates/rbac-tls-manager.yaml | 39 +++++++++++++++++++ 4 files changed, 78 insertions(+), 64 deletions(-) create mode 100644 openstack/barbican/templates/rbac-tls-manager.yaml create mode 100644 openstack/keystone/templates/rbac-tls-manager.yaml diff --git a/openstack/barbican/templates/job-tls-bootstrap.yaml b/openstack/barbican/templates/job-tls-bootstrap.yaml index 798dc7278fd..f0599aada41 100644 --- a/openstack/barbican/templates/job-tls-bootstrap.yaml +++ b/openstack/barbican/templates/job-tls-bootstrap.yaml @@ -34,36 +34,4 @@ spec: - --san-dns={{ include "barbican_api_endpoint_host_public" . }} - --envelope-encryption={{ .Values.tls.envelopeEncryption | default "none" }} backoffLimit: 3 ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ .Release.Name }}-tls-manager ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: {{ .Release.Name }}-tls-manager -rules: - - apiGroups: [""] - resources: ["secrets"] - verbs: ["get", "create", "update"] - - apiGroups: ["cert-manager.io"] - resources: ["certificaterequests"] - verbs: ["create", "get", "delete", "watch"] - - apiGroups: ["apps"] - resources: ["deployments"] - verbs: ["patch"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: {{ .Release.Name }}-tls-manager -subjects: - - kind: ServiceAccount - name: {{ .Release.Name }}-tls-manager -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: {{ .Release.Name }}-tls-manager {{- end }} diff --git a/openstack/barbican/templates/rbac-tls-manager.yaml b/openstack/barbican/templates/rbac-tls-manager.yaml new file mode 100644 index 00000000000..7ecd546ca6d --- /dev/null +++ b/openstack/barbican/templates/rbac-tls-manager.yaml @@ -0,0 +1,39 @@ +{{- if .Values.tls.enabled }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Release.Name }}-tls-manager + labels: + app: {{ template "fullname" . }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: "{{ .Release.Name }}" + component: barbican + type: tls-manager +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ .Release.Name }}-tls-manager +rules: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "create", "update"] + - apiGroups: ["cert-manager.io"] + resources: ["certificaterequests"] + verbs: ["create", "get", "delete", "watch"] + - apiGroups: ["apps"] + resources: ["deployments"] + verbs: ["patch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ .Release.Name }}-tls-manager +subjects: + - kind: ServiceAccount + name: {{ .Release.Name }}-tls-manager +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ .Release.Name }}-tls-manager +{{- end }} diff --git a/openstack/keystone/templates/job-tls-bootstrap.yaml b/openstack/keystone/templates/job-tls-bootstrap.yaml index c84456f6d7f..916c5a5a800 100644 --- a/openstack/keystone/templates/job-tls-bootstrap.yaml +++ b/openstack/keystone/templates/job-tls-bootstrap.yaml @@ -34,36 +34,4 @@ spec: - --san-dns={{ .Values.services.public.host }}.{{ .Values.global.region }}.{{ .Values.global.tld }} - --envelope-encryption={{ .Values.tls.envelopeEncryption | default "none" }} backoffLimit: 3 ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ .Release.Name }}-tls-manager ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: {{ .Release.Name }}-tls-manager -rules: - - apiGroups: [""] - resources: ["secrets"] - verbs: ["get", "create", "update"] - - apiGroups: ["cert-manager.io"] - resources: ["certificaterequests"] - verbs: ["create", "get", "delete", "watch"] - - apiGroups: ["apps"] - resources: ["deployments"] - verbs: ["patch"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: {{ .Release.Name }}-tls-manager -subjects: - - kind: ServiceAccount - name: {{ .Release.Name }}-tls-manager -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: {{ .Release.Name }}-tls-manager {{- end }} diff --git a/openstack/keystone/templates/rbac-tls-manager.yaml b/openstack/keystone/templates/rbac-tls-manager.yaml new file mode 100644 index 00000000000..59903402fdf --- /dev/null +++ b/openstack/keystone/templates/rbac-tls-manager.yaml @@ -0,0 +1,39 @@ +{{- if .Values.tls.enabled }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Release.Name }}-tls-manager + labels: + app: {{ template "fullname" . }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: "{{ .Release.Name }}" + component: keystone + type: tls-manager +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ .Release.Name }}-tls-manager +rules: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "create", "update"] + - apiGroups: ["cert-manager.io"] + resources: ["certificaterequests"] + verbs: ["create", "get", "delete", "watch"] + - apiGroups: ["apps"] + resources: ["deployments"] + verbs: ["patch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ .Release.Name }}-tls-manager +subjects: + - kind: ServiceAccount + name: {{ .Release.Name }}-tls-manager +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ .Release.Name }}-tls-manager +{{- end }} From 308c494dd2ed8862647ef466ae3b9528ef43a489 Mon Sep 17 00:00:00 2001 From: Boris Bobrov Date: Mon, 30 Mar 2026 09:18:44 +0200 Subject: [PATCH 10/56] Make TLS manager RBAC a pre-install hook with weight -10 Regular Helm resources are applied AFTER hooks, so the ServiceAccount didn't exist when the bootstrap Job (hook weight -5) started. Making the RBAC resources hooks with weight -10 ensures they're created before the Job at weight -5. --- openstack/barbican/templates/rbac-tls-manager.yaml | 12 ++++++++++++ openstack/keystone/templates/rbac-tls-manager.yaml | 12 ++++++++++++ 2 files changed, 24 insertions(+) diff --git a/openstack/barbican/templates/rbac-tls-manager.yaml b/openstack/barbican/templates/rbac-tls-manager.yaml index 7ecd546ca6d..ba73c685a5c 100644 --- a/openstack/barbican/templates/rbac-tls-manager.yaml +++ b/openstack/barbican/templates/rbac-tls-manager.yaml @@ -9,11 +9,19 @@ metadata: release: "{{ .Release.Name }}" component: barbican type: tls-manager + annotations: + "helm.sh/hook": pre-install,pre-upgrade + "helm.sh/hook-weight": "-10" + "helm.sh/hook-delete-policy": before-hook-creation --- apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: name: {{ .Release.Name }}-tls-manager + annotations: + "helm.sh/hook": pre-install,pre-upgrade + "helm.sh/hook-weight": "-10" + "helm.sh/hook-delete-policy": before-hook-creation rules: - apiGroups: [""] resources: ["secrets"] @@ -29,6 +37,10 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: name: {{ .Release.Name }}-tls-manager + annotations: + "helm.sh/hook": pre-install,pre-upgrade + "helm.sh/hook-weight": "-10" + "helm.sh/hook-delete-policy": before-hook-creation subjects: - kind: ServiceAccount name: {{ .Release.Name }}-tls-manager diff --git a/openstack/keystone/templates/rbac-tls-manager.yaml b/openstack/keystone/templates/rbac-tls-manager.yaml index 59903402fdf..9f02a0ba9b4 100644 --- a/openstack/keystone/templates/rbac-tls-manager.yaml +++ b/openstack/keystone/templates/rbac-tls-manager.yaml @@ -9,11 +9,19 @@ metadata: release: "{{ .Release.Name }}" component: keystone type: tls-manager + annotations: + "helm.sh/hook": pre-install,pre-upgrade + "helm.sh/hook-weight": "-10" + "helm.sh/hook-delete-policy": before-hook-creation --- apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: name: {{ .Release.Name }}-tls-manager + annotations: + "helm.sh/hook": pre-install,pre-upgrade + "helm.sh/hook-weight": "-10" + "helm.sh/hook-delete-policy": before-hook-creation rules: - apiGroups: [""] resources: ["secrets"] @@ -29,6 +37,10 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: name: {{ .Release.Name }}-tls-manager + annotations: + "helm.sh/hook": pre-install,pre-upgrade + "helm.sh/hook-weight": "-10" + "helm.sh/hook-delete-policy": before-hook-creation subjects: - kind: ServiceAccount name: {{ .Release.Name }}-tls-manager From 302c2799bd0602aa9b29df1991253c007489d4c8 Mon Sep 17 00:00:00 2001 From: Boris Bobrov Date: Mon, 30 Mar 2026 09:36:05 +0200 Subject: [PATCH 11/56] Enable mod_ssl for TLS termination via read-only conf for Barbican Keystone enables mod_ssl at startup with a2enmod, which writes a symlink into /etc/apache2/mods-enabled. Barbican now runs with a read-only root filesystem, so a2enmod fails there with a read-only filesystem error. Load mod_ssl (and its socache_shmcb dependency) through a LoadModule config mounted read-only into conf-enabled instead. Because conf-enabled is parsed after ports.conf, the ssl_module gate that would bind port 443 is not yet satisfied, so the TLS virtual host declares Listen 443 explicitly. The TLS hardening include is placed at server scope since SSLSessionCache is not valid inside a VirtualHost. --- openstack/barbican/templates/api-deployment.yaml | 4 ++++ openstack/barbican/templates/etc-configmap.yaml | 2 ++ openstack/barbican/templates/etc/_ssl-load.conf.tpl | 7 +++++++ openstack/barbican/templates/etc/_wsgi-barbican.conf.tpl | 5 +++-- openstack/keystone/templates/bin/_keystone_api.sh.tpl | 4 ++++ 5 files changed, 20 insertions(+), 2 deletions(-) create mode 100644 openstack/barbican/templates/etc/_ssl-load.conf.tpl diff --git a/openstack/barbican/templates/api-deployment.yaml b/openstack/barbican/templates/api-deployment.yaml index 1a609e1e5d6..91a30659062 100644 --- a/openstack/barbican/templates/api-deployment.yaml +++ b/openstack/barbican/templates/api-deployment.yaml @@ -169,6 +169,10 @@ spec: - name: tls-secrets mountPath: /mnt/secrets readOnly: true + - name: barbican-etc + mountPath: /etc/apache2/conf-enabled/00-ssl-load.conf + subPath: ssl-load.conf + readOnly: true - name: barbican-etc mountPath: /etc/apache2/conf-enabled/tls-hardening.conf subPath: tls-hardening.conf diff --git a/openstack/barbican/templates/etc-configmap.yaml b/openstack/barbican/templates/etc-configmap.yaml index 6d93bcce19a..2408bbf5eaf 100644 --- a/openstack/barbican/templates/etc-configmap.yaml +++ b/openstack/barbican/templates/etc-configmap.yaml @@ -12,6 +12,8 @@ data: barbican.conf: | {{ include (print .Template.BasePath "/etc/_barbican.conf.tpl") . | indent 4 }} {{- if .Values.tls.enabled }} + ssl-load.conf: | +{{ include (print .Template.BasePath "/etc/_ssl-load.conf.tpl") . | indent 4 }} tls-hardening.conf: | {{ include (print .Template.BasePath "/etc/_tls-hardening.conf.tpl") . | indent 4 }} {{- end }} diff --git a/openstack/barbican/templates/etc/_ssl-load.conf.tpl b/openstack/barbican/templates/etc/_ssl-load.conf.tpl new file mode 100644 index 00000000000..86117cbc2af --- /dev/null +++ b/openstack/barbican/templates/etc/_ssl-load.conf.tpl @@ -0,0 +1,7 @@ +{{/* +Load mod_ssl and its socache dependency via read-only conf so the module is +available under a read-only root filesystem without a2enmod. +*/}} + +LoadModule socache_shmcb_module /usr/lib/apache2/modules/mod_socache_shmcb.so +LoadModule ssl_module /usr/lib/apache2/modules/mod_ssl.so diff --git a/openstack/barbican/templates/etc/_wsgi-barbican.conf.tpl b/openstack/barbican/templates/etc/_wsgi-barbican.conf.tpl index 8e66f25096a..2ec4afecc91 100644 --- a/openstack/barbican/templates/etc/_wsgi-barbican.conf.tpl +++ b/openstack/barbican/templates/etc/_wsgi-barbican.conf.tpl @@ -20,13 +20,14 @@ Listen 0.0.0.0:{{ .Values.api_port_internal }} # External HTTPS endpoint (via Ingress TLS passthrough) Listen 0.0.0.0:443 +# TLS hardening at server level (must be outside VirtualHost) +Include /etc/apache2/conf-enabled/tls-hardening.conf + SSLEngine on SSLCertificateFile /mnt/secrets/tls.crt SSLCertificateKeyFile /mnt/secrets/tls.key - Include /etc/apache2/conf-enabled/tls-hardening.conf - WSGIDaemonProcess barbican-api-tls processes={{ .Values.api.processes | default 1 }} threads=1 user=barbican group=barbican display-name=%{GROUP} WSGIProcessGroup barbican-api-tls WSGIScriptAlias / /var/www/cgi-bin/barbican/barbican-wsgi-api diff --git a/openstack/keystone/templates/bin/_keystone_api.sh.tpl b/openstack/keystone/templates/bin/_keystone_api.sh.tpl index 76f5c6dcb86..0ef2523402f 100644 --- a/openstack/keystone/templates/bin/_keystone_api.sh.tpl +++ b/openstack/keystone/templates/bin/_keystone_api.sh.tpl @@ -26,6 +26,10 @@ function start () { a2dismod status + {{- if .Values.tls.enabled }} + a2enmod ssl + {{- end }} + if [ -f /etc/apache2/envvars ]; then # Loading Apache2 ENV variables source /etc/apache2/envvars From 8477e1aef0efc3b2d01b7519ae26b80763ff9530 Mon Sep 17 00:00:00 2001 From: Boris Bobrov Date: Mon, 30 Mar 2026 10:15:26 +0200 Subject: [PATCH 12/56] Move TLS hardening Include outside VirtualHost block SSLSessionCache, SSLSessionCacheTimeout, and SSLStrictSNIVHostCheck are server-level directives that cannot appear inside a VirtualHost section. Moving the Include to before the VirtualHost puts all directives at server level where they apply globally. The per-VirtualHost directives (SSLProtocol, SSLCipherSuite, etc.) inherit correctly from server level. Verified locally: apache2 -t confirms Syntax OK with the Include outside VirtualHost, and confirms the error with Include inside. --- openstack/barbican/templates/etc/_wsgi-barbican.conf.tpl | 3 +++ openstack/keystone/templates/etc/_wsgi-keystone.conf.tpl | 4 +++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/openstack/barbican/templates/etc/_wsgi-barbican.conf.tpl b/openstack/barbican/templates/etc/_wsgi-barbican.conf.tpl index 2ec4afecc91..04c398523aa 100644 --- a/openstack/barbican/templates/etc/_wsgi-barbican.conf.tpl +++ b/openstack/barbican/templates/etc/_wsgi-barbican.conf.tpl @@ -17,6 +17,9 @@ WSGIDaemonProcess barbican-api processes={{ .Values.api.processes | default 1 }} Listen 0.0.0.0:{{ .Values.api_port_internal }} {{- if .Values.tls.enabled }} +# TLS hardening at server level (must be outside VirtualHost) +Include /etc/apache2/conf-enabled/tls-hardening.conf + # External HTTPS endpoint (via Ingress TLS passthrough) Listen 0.0.0.0:443 diff --git a/openstack/keystone/templates/etc/_wsgi-keystone.conf.tpl b/openstack/keystone/templates/etc/_wsgi-keystone.conf.tpl index 6b20198807d..396ba5dfb4c 100644 --- a/openstack/keystone/templates/etc/_wsgi-keystone.conf.tpl +++ b/openstack/keystone/templates/etc/_wsgi-keystone.conf.tpl @@ -74,6 +74,9 @@ WSGIServerMetrics On {{- end }} {{- if .Values.tls.enabled }} +# TLS hardening at server level (must be outside VirtualHost) +Include /etc/apache2/conf-enabled/tls-hardening.conf + # External HTTPS endpoint (via Ingress TLS passthrough) Listen 0.0.0.0:443 @@ -83,7 +86,6 @@ Listen 0.0.0.0:443 SSLEngine on SSLCertificateFile /mnt/secrets/tls.crt SSLCertificateKeyFile /mnt/secrets/tls.key - Include /etc/apache2/conf-enabled/tls-hardening.conf WSGIDaemonProcess keystone-tls processes=8 threads=1 user=keystone group=keystone display-name=%{GROUP} WSGIProcessGroup keystone-tls From d5ae952f22dd106d650b3a70fa8675e2d72033ec Mon Sep 17 00:00:00 2001 From: Boris Bobrov Date: Mon, 30 Mar 2026 10:25:28 +0200 Subject: [PATCH 13/56] =?UTF-8?q?Remove=20duplicate=20Listen=20443=20?= =?UTF-8?q?=E2=80=94=20ports.conf=20already=20declares=20it?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When a2enmod ssl is run, Apache's default /etc/apache2/ports.conf activates 'Listen 443' inside an block. Our WSGI templates also declared 'Listen 0.0.0.0:443', causing a duplicate bind error at startup. Remove our Listen directive and let ports.conf handle it. Verified locally: apache2 -t confirms duplicate Listen causes 'Cannot define multiple Listeners on the same IP:port', and removing ours resolves it. --- openstack/barbican/templates/etc/_wsgi-barbican.conf.tpl | 2 +- openstack/keystone/templates/etc/_wsgi-keystone.conf.tpl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/openstack/barbican/templates/etc/_wsgi-barbican.conf.tpl b/openstack/barbican/templates/etc/_wsgi-barbican.conf.tpl index 04c398523aa..cbe5bbe18b9 100644 --- a/openstack/barbican/templates/etc/_wsgi-barbican.conf.tpl +++ b/openstack/barbican/templates/etc/_wsgi-barbican.conf.tpl @@ -21,7 +21,7 @@ Listen 0.0.0.0:{{ .Values.api_port_internal }} Include /etc/apache2/conf-enabled/tls-hardening.conf # External HTTPS endpoint (via Ingress TLS passthrough) -Listen 0.0.0.0:443 +# Note: Listen 443 is provided by /etc/apache2/ports.conf when mod_ssl is enabled # TLS hardening at server level (must be outside VirtualHost) Include /etc/apache2/conf-enabled/tls-hardening.conf diff --git a/openstack/keystone/templates/etc/_wsgi-keystone.conf.tpl b/openstack/keystone/templates/etc/_wsgi-keystone.conf.tpl index 396ba5dfb4c..402d4e73b52 100644 --- a/openstack/keystone/templates/etc/_wsgi-keystone.conf.tpl +++ b/openstack/keystone/templates/etc/_wsgi-keystone.conf.tpl @@ -78,7 +78,7 @@ WSGIServerMetrics On Include /etc/apache2/conf-enabled/tls-hardening.conf # External HTTPS endpoint (via Ingress TLS passthrough) -Listen 0.0.0.0:443 +# Note: Listen 443 is provided by /etc/apache2/ports.conf when mod_ssl is enabled ServerName {{ .Values.services.public.host }}.{{ .Values.global.region }}.{{ .Values.global.tld }} From 9f6730a5a0500a4353692966743f0797028ffe70 Mon Sep 17 00:00:00 2001 From: Boris Bobrov Date: Mon, 30 Mar 2026 10:41:57 +0200 Subject: [PATCH 14/56] Disable Apache default site when TLS is enabled The default 000-default.conf site on port 80 interferes with VirtualHost routing when TLS passthrough is active. Apache misroutes HTTP/2 requests after a successful TLS handshake on port 443 to the default port 80 site, returning '400 Bad Request: speaking plain HTTP to an SSL-enabled server port'. Disabling the default site fixes the routing. --- openstack/keystone/templates/bin/_keystone_api.sh.tpl | 1 + 1 file changed, 1 insertion(+) diff --git a/openstack/keystone/templates/bin/_keystone_api.sh.tpl b/openstack/keystone/templates/bin/_keystone_api.sh.tpl index 0ef2523402f..189bd2565f8 100644 --- a/openstack/keystone/templates/bin/_keystone_api.sh.tpl +++ b/openstack/keystone/templates/bin/_keystone_api.sh.tpl @@ -28,6 +28,7 @@ function start () { {{- if .Values.tls.enabled }} a2enmod ssl + a2dissite 000-default {{- end }} if [ -f /etc/apache2/envvars ]; then From 755cc70209bdb7838f61a4fdd2db49c7620353e5 Mon Sep 17 00:00:00 2001 From: Boris Bobrov Date: Mon, 30 Mar 2026 15:41:29 +0200 Subject: [PATCH 15/56] Move x509 client cert verification from ingress to Apache in passthrough mode The auth-tls-* ingress annotations are incompatible with ssl-passthrough because NGINX cannot verify client certificates on an encrypted stream it doesn't decrypt. When passthrough is enabled, suppress the ingress annotations and configure Apache to do the client cert verification instead: - SSLVerifyClient optional + SSLVerifyDepth 3 in the HTTPS VirtualHost - SSLCACertificateFile from the x509-ca Secret mounted into the pod - RequestHeader directives pass client cert info to the WSGI app - a2enmod headers for the RequestHeader directive The x509-ca Secret (already created by the ingress template) is mounted at /etc/apache2/x509-ca/ when both passthrough and x509 are enabled. --- .../keystone/templates/bin/_keystone_api.sh.tpl | 1 + openstack/keystone/templates/deployment-api.yaml | 11 +++++++++++ .../keystone/templates/etc/_wsgi-keystone.conf.tpl | 13 +++++++++++++ openstack/keystone/templates/ingress-api.yaml | 2 +- 4 files changed, 26 insertions(+), 1 deletion(-) diff --git a/openstack/keystone/templates/bin/_keystone_api.sh.tpl b/openstack/keystone/templates/bin/_keystone_api.sh.tpl index 189bd2565f8..381a31da107 100644 --- a/openstack/keystone/templates/bin/_keystone_api.sh.tpl +++ b/openstack/keystone/templates/bin/_keystone_api.sh.tpl @@ -28,6 +28,7 @@ function start () { {{- if .Values.tls.enabled }} a2enmod ssl + a2enmod headers a2dissite 000-default {{- end }} diff --git a/openstack/keystone/templates/deployment-api.yaml b/openstack/keystone/templates/deployment-api.yaml index 46fb9460f5b..6aeb0e17ba2 100644 --- a/openstack/keystone/templates/deployment-api.yaml +++ b/openstack/keystone/templates/deployment-api.yaml @@ -279,6 +279,11 @@ spec: mountPath: /etc/apache2/conf-enabled/tls-hardening.conf subPath: tls-hardening.conf readOnly: true + {{- if .Values.services.ingress.x509.ca }} + - name: x509-ca + mountPath: /etc/apache2/x509-ca + readOnly: true + {{- end }} {{- end }} - name: keystone-etc mountPath: /etc/apache2/mods-available/mpm_event.conf @@ -412,4 +417,10 @@ spec: emptyDir: medium: Memory sizeLimit: 1Mi + {{- if .Values.services.ingress.x509.ca }} + - name: x509-ca + secret: + secretName: {{ .Release.Name }}-x509-ca + defaultMode: 0444 + {{- end }} {{- end }} diff --git a/openstack/keystone/templates/etc/_wsgi-keystone.conf.tpl b/openstack/keystone/templates/etc/_wsgi-keystone.conf.tpl index 402d4e73b52..6ef5c7bc969 100644 --- a/openstack/keystone/templates/etc/_wsgi-keystone.conf.tpl +++ b/openstack/keystone/templates/etc/_wsgi-keystone.conf.tpl @@ -87,6 +87,19 @@ Include /etc/apache2/conf-enabled/tls-hardening.conf SSLCertificateFile /mnt/secrets/tls.crt SSLCertificateKeyFile /mnt/secrets/tls.key + {{- if .Values.services.ingress.x509.ca }} + # Client certificate verification (moved from ingress to Apache in passthrough mode) + SSLVerifyClient optional + SSLVerifyDepth 3 + SSLCACertificateFile /etc/apache2/x509-ca/ca.crt + + # Pass client cert info to WSGI app via request headers + # Replaces the headers that NGINX used to forward when it terminated TLS + RequestHeader set SSL_CLIENT_CERT "%{SSL_CLIENT_CERT}s" + RequestHeader set SSL_CLIENT_I_DN "%{SSL_CLIENT_I_DN}s" + RequestHeader set SSL_CLIENT_VERIFY "%{SSL_CLIENT_VERIFY}s" + {{- end }} + WSGIDaemonProcess keystone-tls processes=8 threads=1 user=keystone group=keystone display-name=%{GROUP} WSGIProcessGroup keystone-tls WSGIScriptAlias / /var/www/cgi-bin/keystone/keystone-wsgi-public diff --git a/openstack/keystone/templates/ingress-api.yaml b/openstack/keystone/templates/ingress-api.yaml index 20a05f30764..26fc907a55f 100644 --- a/openstack/keystone/templates/ingress-api.yaml +++ b/openstack/keystone/templates/ingress-api.yaml @@ -74,7 +74,7 @@ metadata: ingress.kubernetes.io/limit-connections: {{ .Values.services.ingress.limitConnections | quote }} nginx.ingress.kubernetes.io/limit-connections: {{ .Values.services.ingress.limitConnections | quote }} {{- end }} - {{- if .Values.services.ingress.x509.ca }} + {{- if and .Values.services.ingress.x509.ca (not (and .Values.tls.enabled .Values.tls.passthrough)) }} ingress.kubernetes.io/auth-tls-secret: {{ .Release.Namespace }}/{{ .Release.Name }}-x509-ca nginx.ingress.kubernetes.io/auth-tls-secret: {{ .Release.Namespace }}/{{ .Release.Name }}-x509-ca ingress.kubernetes.io/auth-tls-pass-certificate-to-upstream: "true" From 8ee71b73ce0b9ef857e6255085881690b3392649 Mon Sep 17 00:00:00 2001 From: Boris Bobrov Date: Mon, 30 Mar 2026 16:47:38 +0200 Subject: [PATCH 16/56] Add ingress.kubernetes.io prefix for ssl-passthrough and backend-protocol The ingress controller is configured with --annotations-prefix=ingress.kubernetes.io so it only reads annotations with that prefix. The ssl-passthrough annotation was only set with the nginx.ingress.kubernetes.io prefix, causing the controller to ignore it entirely. Add both prefixes, matching the dual-prefix pattern already used by auth-tls-* and other annotations in these templates. --- openstack/barbican/templates/ingress.yaml | 2 ++ openstack/keystone/templates/ingress-api.yaml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/openstack/barbican/templates/ingress.yaml b/openstack/barbican/templates/ingress.yaml index 59cf32b7e98..378149e2356 100644 --- a/openstack/barbican/templates/ingress.yaml +++ b/openstack/barbican/templates/ingress.yaml @@ -14,7 +14,9 @@ metadata: component: barbican annotations: {{- if and .Values.tls.enabled .Values.tls.passthrough }} + ingress.kubernetes.io/ssl-passthrough: "true" nginx.ingress.kubernetes.io/ssl-passthrough: "true" + ingress.kubernetes.io/backend-protocol: "HTTPS" nginx.ingress.kubernetes.io/backend-protocol: "HTTPS" config.linkerd.io/skip-inbound-ports: "443" disco: "true" diff --git a/openstack/keystone/templates/ingress-api.yaml b/openstack/keystone/templates/ingress-api.yaml index 26fc907a55f..2ae8cd1a96d 100644 --- a/openstack/keystone/templates/ingress-api.yaml +++ b/openstack/keystone/templates/ingress-api.yaml @@ -47,7 +47,9 @@ metadata: type: api annotations: {{- if and .Values.tls.enabled .Values.tls.passthrough }} + ingress.kubernetes.io/ssl-passthrough: "true" nginx.ingress.kubernetes.io/ssl-passthrough: "true" + ingress.kubernetes.io/backend-protocol: "HTTPS" nginx.ingress.kubernetes.io/backend-protocol: "HTTPS" config.linkerd.io/skip-inbound-ports: "443" nginx.ingress.kubernetes.io/upstream-hash-by: "$remote_addr" From d8692e4a37bcffb7625dae57a8843d2841855532 Mon Sep 17 00:00:00 2001 From: Boris Bobrov Date: Mon, 30 Mar 2026 17:16:43 +0200 Subject: [PATCH 17/56] Change issuerKind default from DigicertIssuer to ClusterIssuer The operations documentation specifies kind: ClusterIssuer with group: certmanager.cloud.sap for DigiCert certificates. The group field routes to the digicert-issuer controller. All working CertificateRequests in the cluster use ClusterIssuer. Using DigicertIssuer caused a nil map panic in the digicert-issuer controller. --- openstack/barbican/templates/cronjob-tls-rotation.yaml | 2 +- openstack/barbican/templates/job-tls-bootstrap.yaml | 2 +- openstack/barbican/values.yaml | 2 +- openstack/keystone/templates/cronjob-tls-rotation.yaml | 2 +- openstack/keystone/templates/job-tls-bootstrap.yaml | 2 +- openstack/keystone/values.yaml | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/openstack/barbican/templates/cronjob-tls-rotation.yaml b/openstack/barbican/templates/cronjob-tls-rotation.yaml index a778cf1f023..5e298348f4f 100644 --- a/openstack/barbican/templates/cronjob-tls-rotation.yaml +++ b/openstack/barbican/templates/cronjob-tls-rotation.yaml @@ -30,7 +30,7 @@ spec: - --secret-name={{ .Values.tls.secretName | default "barbican-tls" }} - --key-algorithm={{ .Values.tls.keyAlgorithm | default "ECDSAP256" }} - --issuer-name={{ .Values.tls.issuerName | default "digicert-issuer" }} - - --issuer-kind={{ .Values.tls.issuerKind | default "DigicertIssuer" }} + - --issuer-kind={{ .Values.tls.issuerKind | default "ClusterIssuer" }} - --issuer-group={{ .Values.tls.issuerGroup | default "certmanager.cloud.sap" }} - --common-name={{ include "barbican_api_endpoint_host_public" . }} - --san-dns={{ include "barbican_api_endpoint_host_public" . }} diff --git a/openstack/barbican/templates/job-tls-bootstrap.yaml b/openstack/barbican/templates/job-tls-bootstrap.yaml index f0599aada41..e5675b4fe17 100644 --- a/openstack/barbican/templates/job-tls-bootstrap.yaml +++ b/openstack/barbican/templates/job-tls-bootstrap.yaml @@ -28,7 +28,7 @@ spec: - --secret-name={{ .Values.tls.secretName | default "barbican-tls" }} - --key-algorithm={{ .Values.tls.keyAlgorithm | default "ECDSAP256" }} - --issuer-name={{ .Values.tls.issuerName | default "digicert-issuer" }} - - --issuer-kind={{ .Values.tls.issuerKind | default "DigicertIssuer" }} + - --issuer-kind={{ .Values.tls.issuerKind | default "ClusterIssuer" }} - --issuer-group={{ .Values.tls.issuerGroup | default "certmanager.cloud.sap" }} - --common-name={{ include "barbican_api_endpoint_host_public" . }} - --san-dns={{ include "barbican_api_endpoint_host_public" . }} diff --git a/openstack/barbican/values.yaml b/openstack/barbican/values.yaml index 71f019a6110..ccc98724c69 100644 --- a/openstack/barbican/values.yaml +++ b/openstack/barbican/values.yaml @@ -504,7 +504,7 @@ tls: imageTag: latest # cert-manager issuer configuration issuerName: digicert-issuer - issuerKind: DigicertIssuer + issuerKind: ClusterIssuer issuerGroup: certmanager.cloud.sap keyAlgorithm: ECDSAP256 diff --git a/openstack/keystone/templates/cronjob-tls-rotation.yaml b/openstack/keystone/templates/cronjob-tls-rotation.yaml index a2537eef4ac..6bd457ce433 100644 --- a/openstack/keystone/templates/cronjob-tls-rotation.yaml +++ b/openstack/keystone/templates/cronjob-tls-rotation.yaml @@ -30,7 +30,7 @@ spec: - --secret-name={{ .Values.tls.secretName | default "keystone-tls" }} - --key-algorithm={{ .Values.tls.keyAlgorithm | default "ECDSAP256" }} - --issuer-name={{ .Values.tls.issuerName | default "digicert-issuer" }} - - --issuer-kind={{ .Values.tls.issuerKind | default "DigicertIssuer" }} + - --issuer-kind={{ .Values.tls.issuerKind | default "ClusterIssuer" }} - --issuer-group={{ .Values.tls.issuerGroup | default "certmanager.cloud.sap" }} - --common-name={{ .Values.services.public.host }}.{{ .Values.global.region }}.{{ .Values.global.tld }} - --san-dns={{ .Values.services.public.host }}.{{ .Values.global.region }}.{{ .Values.global.tld }} diff --git a/openstack/keystone/templates/job-tls-bootstrap.yaml b/openstack/keystone/templates/job-tls-bootstrap.yaml index 916c5a5a800..03d56369249 100644 --- a/openstack/keystone/templates/job-tls-bootstrap.yaml +++ b/openstack/keystone/templates/job-tls-bootstrap.yaml @@ -28,7 +28,7 @@ spec: - --secret-name={{ .Values.tls.secretName | default "keystone-tls" }} - --key-algorithm={{ .Values.tls.keyAlgorithm | default "ECDSAP256" }} - --issuer-name={{ .Values.tls.issuerName | default "digicert-issuer" }} - - --issuer-kind={{ .Values.tls.issuerKind | default "DigicertIssuer" }} + - --issuer-kind={{ .Values.tls.issuerKind | default "ClusterIssuer" }} - --issuer-group={{ .Values.tls.issuerGroup | default "certmanager.cloud.sap" }} - --common-name={{ .Values.services.public.host }}.{{ .Values.global.region }}.{{ .Values.global.tld }} - --san-dns={{ .Values.services.public.host }}.{{ .Values.global.region }}.{{ .Values.global.tld }} diff --git a/openstack/keystone/values.yaml b/openstack/keystone/values.yaml index 7f4e11cefa3..060bde805e5 100644 --- a/openstack/keystone/values.yaml +++ b/openstack/keystone/values.yaml @@ -67,7 +67,7 @@ tls: image: keppel.eu-de-1.cloud.sap/ccloud/tls-lifecycle-manager imageTag: latest issuerName: digicert-issuer - issuerKind: DigicertIssuer + issuerKind: ClusterIssuer issuerGroup: certmanager.cloud.sap keyAlgorithm: ECDSAP256 From 726f600e3c3921c46210554fcd0435fe77a17e70 Mon Sep 17 00:00:00 2001 From: Boris Bobrov Date: Tue, 31 Mar 2026 17:30:46 +0200 Subject: [PATCH 18/56] Enable ssl-passthrough on the ingress-nginx controller Services might decide to use the nginx ssl-passthrough annotation to forward encrypted TLS streams directly to pods for end-to-end encryption. Without --enable-ssl-passthrough in the controller args, this annotation is silently ignored and the controller terminates TLS at the ingress instead of passing it through. Add controller.extraArgs.enable-ssl-passthrough to the ingress-nginx subchart values so passthrough works globally across all clusters. --- system/kube-system-metal/values.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/system/kube-system-metal/values.yaml b/system/kube-system-metal/values.yaml index 8dfe2357aa3..afa57e71137 100644 --- a/system/kube-system-metal/values.yaml +++ b/system/kube-system-metal/values.yaml @@ -86,6 +86,10 @@ ingress-nginx: # See https://github.com/kubernetes/ingress-nginx/issues/7837 . allowSnippetAnnotations: false + extraArgs: + # Required for ssl-passthrough ingress annotations to take effect. + enable-ssl-passthrough: true + config: custom-http-errors: "495" From 397c0edc0e25f21f9e09ef1932fb5c6a29c05cb3 Mon Sep 17 00:00:00 2001 From: Boris Bobrov Date: Wed, 1 Apr 2026 11:58:43 +0200 Subject: [PATCH 19/56] Suppress tls-acme ingress annotation whenever pod-level TLS is enabled When tls.enabled is true, the bootstrap Job provisions the certificate via DigiCert. The tls-acme annotation must be suppressed regardless of whether tls.passthrough is also set, otherwise both the bootstrap Job and the ingress cert-manager flow provision certificates for the same hostname. Previously, tls-acme was only suppressed when both tls.enabled and tls.passthrough were true. With tls.enabled alone (passthrough not yet active), both certificate paths were active simultaneously. --- openstack/barbican/templates/ingress.yaml | 2 +- openstack/keystone/templates/ingress-api.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/openstack/barbican/templates/ingress.yaml b/openstack/barbican/templates/ingress.yaml index 378149e2356..3b6614e3b7a 100644 --- a/openstack/barbican/templates/ingress.yaml +++ b/openstack/barbican/templates/ingress.yaml @@ -20,7 +20,7 @@ metadata: nginx.ingress.kubernetes.io/backend-protocol: "HTTPS" config.linkerd.io/skip-inbound-ports: "443" disco: "true" - {{- else if .Values.tlsacme }} + {{- else if and .Values.tlsacme (not .Values.tls.enabled) }} kubernetes.io/tls-acme: "true" disco: "true" {{- end }} diff --git a/openstack/keystone/templates/ingress-api.yaml b/openstack/keystone/templates/ingress-api.yaml index 2ae8cd1a96d..dd8ac97f1be 100644 --- a/openstack/keystone/templates/ingress-api.yaml +++ b/openstack/keystone/templates/ingress-api.yaml @@ -86,7 +86,7 @@ metadata: ingress.kubernetes.io/auth-tls-verify-depth: "3" nginx.ingress.kubernetes.io/auth-tls-verify-depth: "3" {{- end }} - {{- if and .Values.services.ingress.tlsacme (not (and .Values.tls.enabled .Values.tls.passthrough)) }} + {{- if and .Values.services.ingress.tlsacme (not .Values.tls.enabled) }} kubernetes.io/tls-acme: "true" {{- end }} {{- if .Values.global.is_global_region }} From cbbf33258dd25e84cd7614355aa48337a53abb65 Mon Sep 17 00:00:00 2001 From: Boris Bobrov Date: Thu, 2 Apr 2026 10:03:20 +0200 Subject: [PATCH 20/56] Drop Deployment RBAC and --deployment-name from TLS rotation CronJobs Reloader watches the TLS secret and triggers rolling restarts when it changes, so the tls-lifecycle-manager no longer needs to patch Deployments directly. This removes the --deployment-name argument from both CronJob templates and the apps/deployments RBAC rule from both Role definitions, narrowing the ServiceAccount permissions to only secrets and certificaterequests. --- openstack/barbican/templates/cronjob-tls-rotation.yaml | 1 - openstack/barbican/templates/rbac-tls-manager.yaml | 3 --- openstack/keystone/templates/cronjob-tls-rotation.yaml | 1 - openstack/keystone/templates/rbac-tls-manager.yaml | 3 --- 4 files changed, 8 deletions(-) diff --git a/openstack/barbican/templates/cronjob-tls-rotation.yaml b/openstack/barbican/templates/cronjob-tls-rotation.yaml index 5e298348f4f..87991909aca 100644 --- a/openstack/barbican/templates/cronjob-tls-rotation.yaml +++ b/openstack/barbican/templates/cronjob-tls-rotation.yaml @@ -35,7 +35,6 @@ spec: - --common-name={{ include "barbican_api_endpoint_host_public" . }} - --san-dns={{ include "barbican_api_endpoint_host_public" . }} - --renewal-threshold-days=30 - - --deployment-name=barbican-api - --envelope-encryption={{ .Values.tls.envelopeEncryption | default "none" }} backoffLimit: 3 {{- end }} diff --git a/openstack/barbican/templates/rbac-tls-manager.yaml b/openstack/barbican/templates/rbac-tls-manager.yaml index ba73c685a5c..7e537d0c610 100644 --- a/openstack/barbican/templates/rbac-tls-manager.yaml +++ b/openstack/barbican/templates/rbac-tls-manager.yaml @@ -29,9 +29,6 @@ rules: - apiGroups: ["cert-manager.io"] resources: ["certificaterequests"] verbs: ["create", "get", "delete", "watch"] - - apiGroups: ["apps"] - resources: ["deployments"] - verbs: ["patch"] --- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding diff --git a/openstack/keystone/templates/cronjob-tls-rotation.yaml b/openstack/keystone/templates/cronjob-tls-rotation.yaml index 6bd457ce433..e06badcd416 100644 --- a/openstack/keystone/templates/cronjob-tls-rotation.yaml +++ b/openstack/keystone/templates/cronjob-tls-rotation.yaml @@ -35,7 +35,6 @@ spec: - --common-name={{ .Values.services.public.host }}.{{ .Values.global.region }}.{{ .Values.global.tld }} - --san-dns={{ .Values.services.public.host }}.{{ .Values.global.region }}.{{ .Values.global.tld }} - --renewal-threshold-days=30 - - --deployment-name={{ .Release.Name }}-api - --envelope-encryption={{ .Values.tls.envelopeEncryption | default "none" }} backoffLimit: 3 {{- end }} diff --git a/openstack/keystone/templates/rbac-tls-manager.yaml b/openstack/keystone/templates/rbac-tls-manager.yaml index 9f02a0ba9b4..511a28d9828 100644 --- a/openstack/keystone/templates/rbac-tls-manager.yaml +++ b/openstack/keystone/templates/rbac-tls-manager.yaml @@ -29,9 +29,6 @@ rules: - apiGroups: ["cert-manager.io"] resources: ["certificaterequests"] verbs: ["create", "get", "delete", "watch"] - - apiGroups: ["apps"] - resources: ["deployments"] - verbs: ["patch"] --- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding From fc365c95d9fac447288d2de0b5cb2501701855c5 Mon Sep 17 00:00:00 2001 From: Boris Bobrov Date: Tue, 7 Apr 2026 12:56:36 +0200 Subject: [PATCH 21/56] Add pluggable key lifecycle configuration with Helm validation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace envelopeEncryption with three explicit per-region operator choices: keyGeneration, keyWrapping, keyStorage. All three are required when tls.enabled — there are no defaults. Add Helm template validation that fails the render when: - Any of the three options is missing - The forbidden combination (wrapping=none + storage=k8s-secret) is used without allowInsecureStorage: true Replace bash cp init container with tls-lifecycle-manager --mode=unwrap which reads from volume mount, unwraps via the configured wrapper, validates PEM, and writes to tmpfs. Works for all wrapping modes. Templates use required() as a second layer of enforcement alongside the validation helper. --- openstack/barbican/templates/_helpers.tpl | 17 +++++++++++++++ .../barbican/templates/api-deployment.yaml | 21 +++++++------------ .../templates/cronjob-tls-rotation.yaml | 4 +++- .../barbican/templates/job-tls-bootstrap.yaml | 5 ++++- openstack/barbican/values.yaml | 9 ++++++-- openstack/keystone/templates/_helpers.tpl | 17 +++++++++++++++ .../templates/cronjob-tls-rotation.yaml | 4 +++- .../keystone/templates/deployment-api.yaml | 18 +++++++--------- .../keystone/templates/job-tls-bootstrap.yaml | 5 ++++- openstack/keystone/values.yaml | 8 ++++++- 10 files changed, 76 insertions(+), 32 deletions(-) diff --git a/openstack/barbican/templates/_helpers.tpl b/openstack/barbican/templates/_helpers.tpl index fe785f70456..593b189cd8d 100644 --- a/openstack/barbican/templates/_helpers.tpl +++ b/openstack/barbican/templates/_helpers.tpl @@ -39,3 +39,20 @@ We truncate at 63 chars because some Kubernetes name fields are limited to this {{- .Release.Name }}-{{ $name }}-{{ substr 0 4 $hash }}-{{ .Values.imageVersionBarbicanApi | required "Please set barbican.imageVersionBarbicanApi" }} {{- end }} {{- end }} + +{{- define "barbican.tls.validate" -}} +{{- if .Values.tls.enabled }} + {{- if not .Values.tls.keyGeneration }} + {{- fail "tls.keyGeneration is required when tls.enabled (options: go-crypto, hsm-entropy, hsm-full, tpm-entropy)" }} + {{- end }} + {{- if not .Values.tls.keyWrapping }} + {{- fail "tls.keyWrapping is required when tls.enabled (options: none, vault-transit, hsm, tpm)" }} + {{- end }} + {{- if not .Values.tls.keyStorage }} + {{- fail "tls.keyStorage is required when tls.enabled (options: k8s-secret, vault-secret)" }} + {{- end }} + {{- if and (eq .Values.tls.keyWrapping "none") (eq .Values.tls.keyStorage "k8s-secret") (not .Values.tls.allowInsecureStorage) }} + {{- fail "tls: unwrapped keys cannot be stored as plain-text K8s Secrets. Set tls.keyWrapping or tls.keyStorage, or set tls.allowInsecureStorage: true to acknowledge." }} + {{- end }} +{{- end }} +{{- end }} diff --git a/openstack/barbican/templates/api-deployment.yaml b/openstack/barbican/templates/api-deployment.yaml index 91a30659062..38d6f99ce63 100644 --- a/openstack/barbican/templates/api-deployment.yaml +++ b/openstack/barbican/templates/api-deployment.yaml @@ -61,20 +61,13 @@ spec: {{- end }} {{- if .Values.tls.enabled }} - name: tls-init - image: {{ required ".Values.global.registry is missing" .Values.global.registry }}/loci-barbican:{{ required "Values.imageVersionBarbicanApi is missing" .Values.imageVersionBarbicanApi }} - command: - - bash - - -c - - | - set -ex - # Copy TLS key material from K8s Secret to tmpfs - # In noop envelope-encryption mode, the key is plaintext in the Secret. - # When vault-transit or TPM is enabled, this step would decrypt first. - cp /tls-input/tls.key /mnt/secrets/tls.key - cp /tls-input/tls.crt /mnt/secrets/tls.crt - chmod 0440 /mnt/secrets/tls.key - chmod 0444 /mnt/secrets/tls.crt - echo "TLS key material placed on tmpfs" + image: {{ .Values.tls.lifecycle.image }}:{{ .Values.tls.lifecycle.imageTag }} + imagePullPolicy: IfNotPresent + args: + - --mode=unwrap + - --input-dir=/tls-input + - --output-dir=/mnt/secrets + - --key-wrapping={{ required "tls.keyWrapping is required" .Values.tls.keyWrapping }} volumeMounts: - name: tls-input mountPath: /tls-input diff --git a/openstack/barbican/templates/cronjob-tls-rotation.yaml b/openstack/barbican/templates/cronjob-tls-rotation.yaml index 87991909aca..e298193fa09 100644 --- a/openstack/barbican/templates/cronjob-tls-rotation.yaml +++ b/openstack/barbican/templates/cronjob-tls-rotation.yaml @@ -35,6 +35,8 @@ spec: - --common-name={{ include "barbican_api_endpoint_host_public" . }} - --san-dns={{ include "barbican_api_endpoint_host_public" . }} - --renewal-threshold-days=30 - - --envelope-encryption={{ .Values.tls.envelopeEncryption | default "none" }} + - --key-generation={{ required "tls.keyGeneration is required" .Values.tls.keyGeneration }} + - --key-wrapping={{ required "tls.keyWrapping is required" .Values.tls.keyWrapping }} + - --key-storage={{ required "tls.keyStorage is required" .Values.tls.keyStorage }} backoffLimit: 3 {{- end }} diff --git a/openstack/barbican/templates/job-tls-bootstrap.yaml b/openstack/barbican/templates/job-tls-bootstrap.yaml index e5675b4fe17..e7670dfecd5 100644 --- a/openstack/barbican/templates/job-tls-bootstrap.yaml +++ b/openstack/barbican/templates/job-tls-bootstrap.yaml @@ -1,4 +1,5 @@ {{- if .Values.tls.enabled }} +{{- include "barbican.tls.validate" . }} apiVersion: batch/v1 kind: Job metadata: @@ -32,6 +33,8 @@ spec: - --issuer-group={{ .Values.tls.issuerGroup | default "certmanager.cloud.sap" }} - --common-name={{ include "barbican_api_endpoint_host_public" . }} - --san-dns={{ include "barbican_api_endpoint_host_public" . }} - - --envelope-encryption={{ .Values.tls.envelopeEncryption | default "none" }} + - --key-generation={{ required "tls.keyGeneration is required" .Values.tls.keyGeneration }} + - --key-wrapping={{ required "tls.keyWrapping is required" .Values.tls.keyWrapping }} + - --key-storage={{ required "tls.keyStorage is required" .Values.tls.keyStorage }} backoffLimit: 3 {{- end }} diff --git a/openstack/barbican/values.yaml b/openstack/barbican/values.yaml index ccc98724c69..b91cc585e99 100644 --- a/openstack/barbican/values.yaml +++ b/openstack/barbican/values.yaml @@ -496,8 +496,13 @@ tls: secretName: barbican-tls # Enable TLS passthrough on the ingress (sends encrypted stream directly to pod) passthrough: false - # Envelope encryption mode for the private key: "none" (plaintext) or "vault-transit" - envelopeEncryption: none + # Per-region operator choices — all three are REQUIRED when tls.enabled: true. + # There are no defaults; the operator must make an explicit choice. + # keyGeneration: go-crypto | hsm-entropy | hsm-full | tpm-entropy + # keyWrapping: none | vault-transit | hsm | tpm + # keyStorage: k8s-secret | vault-secret + # Escape hatch: set allowInsecureStorage: true to allow wrapping=none + storage=k8s-secret + allowInsecureStorage: false # TLS lifecycle manager binary for bootstrap Job and rotation CronJob lifecycle: image: keppel.eu-de-1.cloud.sap/ccloud/tls-lifecycle-manager diff --git a/openstack/keystone/templates/_helpers.tpl b/openstack/keystone/templates/_helpers.tpl index 26b71af9dcc..83a502eb88e 100644 --- a/openstack/keystone/templates/_helpers.tpl +++ b/openstack/keystone/templates/_helpers.tpl @@ -111,3 +111,20 @@ Empty values are skipped so callers may pass "" for optional dependencies. {{- printf "http://prodel.%s.svc/check-delete_project/%%(project_id)s" $ns -}} {{- end }} {{- end }} + +{{- define "keystone.tls.validate" -}} +{{- if .Values.tls.enabled }} + {{- if not .Values.tls.keyGeneration }} + {{- fail "tls.keyGeneration is required when tls.enabled (options: go-crypto, hsm-entropy, hsm-full, tpm-entropy)" }} + {{- end }} + {{- if not .Values.tls.keyWrapping }} + {{- fail "tls.keyWrapping is required when tls.enabled (options: none, vault-transit, hsm, tpm)" }} + {{- end }} + {{- if not .Values.tls.keyStorage }} + {{- fail "tls.keyStorage is required when tls.enabled (options: k8s-secret, vault-secret)" }} + {{- end }} + {{- if and (eq .Values.tls.keyWrapping "none") (eq .Values.tls.keyStorage "k8s-secret") (not .Values.tls.allowInsecureStorage) }} + {{- fail "tls: unwrapped keys cannot be stored as plain-text K8s Secrets. Set tls.keyWrapping or tls.keyStorage, or set tls.allowInsecureStorage: true to acknowledge." }} + {{- end }} +{{- end }} +{{- end }} diff --git a/openstack/keystone/templates/cronjob-tls-rotation.yaml b/openstack/keystone/templates/cronjob-tls-rotation.yaml index e06badcd416..b1ba97303e9 100644 --- a/openstack/keystone/templates/cronjob-tls-rotation.yaml +++ b/openstack/keystone/templates/cronjob-tls-rotation.yaml @@ -35,6 +35,8 @@ spec: - --common-name={{ .Values.services.public.host }}.{{ .Values.global.region }}.{{ .Values.global.tld }} - --san-dns={{ .Values.services.public.host }}.{{ .Values.global.region }}.{{ .Values.global.tld }} - --renewal-threshold-days=30 - - --envelope-encryption={{ .Values.tls.envelopeEncryption | default "none" }} + - --key-generation={{ required "tls.keyGeneration is required" .Values.tls.keyGeneration }} + - --key-wrapping={{ required "tls.keyWrapping is required" .Values.tls.keyWrapping }} + - --key-storage={{ required "tls.keyStorage is required" .Values.tls.keyStorage }} backoffLimit: 3 {{- end }} diff --git a/openstack/keystone/templates/deployment-api.yaml b/openstack/keystone/templates/deployment-api.yaml index 6aeb0e17ba2..282b8d430c1 100644 --- a/openstack/keystone/templates/deployment-api.yaml +++ b/openstack/keystone/templates/deployment-api.yaml @@ -112,17 +112,13 @@ spec: {{- end }} {{- if .Values.tls.enabled }} - name: tls-init - image: {{required ".Values.global.registryAlternateRegion is missing" .Values.global.registryAlternateRegion}}/loci-keystone:{{required ".Values.api.imageTag is missing" .Values.api.imageTag}} - command: - - bash - - -c - - | - set -ex - cp /tls-input/tls.key /mnt/secrets/tls.key - cp /tls-input/tls.crt /mnt/secrets/tls.crt - chmod 0440 /mnt/secrets/tls.key - chmod 0444 /mnt/secrets/tls.crt - echo "TLS key material placed on tmpfs" + image: {{ .Values.tls.lifecycle.image }}:{{ .Values.tls.lifecycle.imageTag }} + imagePullPolicy: IfNotPresent + args: + - --mode=unwrap + - --input-dir=/tls-input + - --output-dir=/mnt/secrets + - --key-wrapping={{ required "tls.keyWrapping is required" .Values.tls.keyWrapping }} volumeMounts: - name: tls-input mountPath: /tls-input diff --git a/openstack/keystone/templates/job-tls-bootstrap.yaml b/openstack/keystone/templates/job-tls-bootstrap.yaml index 03d56369249..841d9fbeed8 100644 --- a/openstack/keystone/templates/job-tls-bootstrap.yaml +++ b/openstack/keystone/templates/job-tls-bootstrap.yaml @@ -1,4 +1,5 @@ {{- if .Values.tls.enabled }} +{{- include "keystone.tls.validate" . }} apiVersion: batch/v1 kind: Job metadata: @@ -32,6 +33,8 @@ spec: - --issuer-group={{ .Values.tls.issuerGroup | default "certmanager.cloud.sap" }} - --common-name={{ .Values.services.public.host }}.{{ .Values.global.region }}.{{ .Values.global.tld }} - --san-dns={{ .Values.services.public.host }}.{{ .Values.global.region }}.{{ .Values.global.tld }} - - --envelope-encryption={{ .Values.tls.envelopeEncryption | default "none" }} + - --key-generation={{ required "tls.keyGeneration is required" .Values.tls.keyGeneration }} + - --key-wrapping={{ required "tls.keyWrapping is required" .Values.tls.keyWrapping }} + - --key-storage={{ required "tls.keyStorage is required" .Values.tls.keyStorage }} backoffLimit: 3 {{- end }} diff --git a/openstack/keystone/values.yaml b/openstack/keystone/values.yaml index 060bde805e5..77643b6158c 100644 --- a/openstack/keystone/values.yaml +++ b/openstack/keystone/values.yaml @@ -62,7 +62,13 @@ tls: enabled: false secretName: keystone-tls passthrough: false - envelopeEncryption: none + # Per-region operator choices — all three are REQUIRED when tls.enabled: true. + # There are no defaults; the operator must make an explicit choice. + # keyGeneration: go-crypto | hsm-entropy | hsm-full | tpm-entropy + # keyWrapping: none | vault-transit | hsm | tpm + # keyStorage: k8s-secret | vault-secret + # Escape hatch: set allowInsecureStorage: true to allow wrapping=none + storage=k8s-secret + allowInsecureStorage: false lifecycle: image: keppel.eu-de-1.cloud.sap/ccloud/tls-lifecycle-manager imageTag: latest From 8117bafee2d9571053ab3b0bfec2c01c8eb2c9e4 Mon Sep 17 00:00:00 2001 From: Boris Bobrov Date: Sat, 15 Aug 2026 16:47:35 +0200 Subject: [PATCH 22/56] Add internal-k8s-secret storage option and wire --allow-insecure-storage The tls-lifecycle-manager binary now distinguishes internal-k8s-secret (written directly to the K8s API, stays in-cluster) from k8s-secret (Secret manifest emitted for Git commit + GitOps operator review). - values.yaml (keystone, barbican): document internal-k8s-secret as the recommended keyStorage default and clarify that allowInsecureStorage now also passes --allow-insecure-storage to the binary. - _helpers.tpl tls.validate: list internal-k8s-secret in the required keyStorage options message. - bootstrap Job and rotation CronJob (keystone, barbican): pass --allow-insecure-storage when tls.allowInsecureStorage is true, so the binary permits keyWrapping=none + keyStorage=k8s-secret in dev. --- openstack/barbican/templates/_helpers.tpl | 2 +- openstack/barbican/templates/cronjob-tls-rotation.yaml | 3 +++ openstack/barbican/templates/job-tls-bootstrap.yaml | 3 +++ openstack/barbican/values.yaml | 7 +++++-- openstack/keystone/templates/_helpers.tpl | 2 +- openstack/keystone/templates/cronjob-tls-rotation.yaml | 3 +++ openstack/keystone/templates/job-tls-bootstrap.yaml | 3 +++ openstack/keystone/values.yaml | 7 +++++-- 8 files changed, 24 insertions(+), 6 deletions(-) diff --git a/openstack/barbican/templates/_helpers.tpl b/openstack/barbican/templates/_helpers.tpl index 593b189cd8d..fb4433961b1 100644 --- a/openstack/barbican/templates/_helpers.tpl +++ b/openstack/barbican/templates/_helpers.tpl @@ -49,7 +49,7 @@ We truncate at 63 chars because some Kubernetes name fields are limited to this {{- fail "tls.keyWrapping is required when tls.enabled (options: none, vault-transit, hsm, tpm)" }} {{- end }} {{- if not .Values.tls.keyStorage }} - {{- fail "tls.keyStorage is required when tls.enabled (options: k8s-secret, vault-secret)" }} + {{- fail "tls.keyStorage is required when tls.enabled (options: internal-k8s-secret, k8s-secret, vault-secret)" }} {{- end }} {{- if and (eq .Values.tls.keyWrapping "none") (eq .Values.tls.keyStorage "k8s-secret") (not .Values.tls.allowInsecureStorage) }} {{- fail "tls: unwrapped keys cannot be stored as plain-text K8s Secrets. Set tls.keyWrapping or tls.keyStorage, or set tls.allowInsecureStorage: true to acknowledge." }} diff --git a/openstack/barbican/templates/cronjob-tls-rotation.yaml b/openstack/barbican/templates/cronjob-tls-rotation.yaml index e298193fa09..b09da9ac92d 100644 --- a/openstack/barbican/templates/cronjob-tls-rotation.yaml +++ b/openstack/barbican/templates/cronjob-tls-rotation.yaml @@ -38,5 +38,8 @@ spec: - --key-generation={{ required "tls.keyGeneration is required" .Values.tls.keyGeneration }} - --key-wrapping={{ required "tls.keyWrapping is required" .Values.tls.keyWrapping }} - --key-storage={{ required "tls.keyStorage is required" .Values.tls.keyStorage }} + {{- if .Values.tls.allowInsecureStorage }} + - --allow-insecure-storage + {{- end }} backoffLimit: 3 {{- end }} diff --git a/openstack/barbican/templates/job-tls-bootstrap.yaml b/openstack/barbican/templates/job-tls-bootstrap.yaml index e7670dfecd5..99938947c3e 100644 --- a/openstack/barbican/templates/job-tls-bootstrap.yaml +++ b/openstack/barbican/templates/job-tls-bootstrap.yaml @@ -36,5 +36,8 @@ spec: - --key-generation={{ required "tls.keyGeneration is required" .Values.tls.keyGeneration }} - --key-wrapping={{ required "tls.keyWrapping is required" .Values.tls.keyWrapping }} - --key-storage={{ required "tls.keyStorage is required" .Values.tls.keyStorage }} + {{- if .Values.tls.allowInsecureStorage }} + - --allow-insecure-storage + {{- end }} backoffLimit: 3 {{- end }} diff --git a/openstack/barbican/values.yaml b/openstack/barbican/values.yaml index b91cc585e99..f716b415894 100644 --- a/openstack/barbican/values.yaml +++ b/openstack/barbican/values.yaml @@ -500,8 +500,11 @@ tls: # There are no defaults; the operator must make an explicit choice. # keyGeneration: go-crypto | hsm-entropy | hsm-full | tpm-entropy # keyWrapping: none | vault-transit | hsm | tpm - # keyStorage: k8s-secret | vault-secret - # Escape hatch: set allowInsecureStorage: true to allow wrapping=none + storage=k8s-secret + # keyStorage: internal-k8s-secret | k8s-secret | vault-secret + # internal-k8s-secret (recommended): key written directly to the K8s API; stays in-cluster, never in Git + # k8s-secret: Secret manifest emitted for Git commit + GitOps operator review + # Escape hatch: allowInsecureStorage: true also passes --allow-insecure-storage to the binary, + # permitting wrapping=none + storage=k8s-secret (development only) allowInsecureStorage: false # TLS lifecycle manager binary for bootstrap Job and rotation CronJob lifecycle: diff --git a/openstack/keystone/templates/_helpers.tpl b/openstack/keystone/templates/_helpers.tpl index 83a502eb88e..5dcc1603622 100644 --- a/openstack/keystone/templates/_helpers.tpl +++ b/openstack/keystone/templates/_helpers.tpl @@ -121,7 +121,7 @@ Empty values are skipped so callers may pass "" for optional dependencies. {{- fail "tls.keyWrapping is required when tls.enabled (options: none, vault-transit, hsm, tpm)" }} {{- end }} {{- if not .Values.tls.keyStorage }} - {{- fail "tls.keyStorage is required when tls.enabled (options: k8s-secret, vault-secret)" }} + {{- fail "tls.keyStorage is required when tls.enabled (options: internal-k8s-secret, k8s-secret, vault-secret)" }} {{- end }} {{- if and (eq .Values.tls.keyWrapping "none") (eq .Values.tls.keyStorage "k8s-secret") (not .Values.tls.allowInsecureStorage) }} {{- fail "tls: unwrapped keys cannot be stored as plain-text K8s Secrets. Set tls.keyWrapping or tls.keyStorage, or set tls.allowInsecureStorage: true to acknowledge." }} diff --git a/openstack/keystone/templates/cronjob-tls-rotation.yaml b/openstack/keystone/templates/cronjob-tls-rotation.yaml index b1ba97303e9..7a14b5f1212 100644 --- a/openstack/keystone/templates/cronjob-tls-rotation.yaml +++ b/openstack/keystone/templates/cronjob-tls-rotation.yaml @@ -38,5 +38,8 @@ spec: - --key-generation={{ required "tls.keyGeneration is required" .Values.tls.keyGeneration }} - --key-wrapping={{ required "tls.keyWrapping is required" .Values.tls.keyWrapping }} - --key-storage={{ required "tls.keyStorage is required" .Values.tls.keyStorage }} + {{- if .Values.tls.allowInsecureStorage }} + - --allow-insecure-storage + {{- end }} backoffLimit: 3 {{- end }} diff --git a/openstack/keystone/templates/job-tls-bootstrap.yaml b/openstack/keystone/templates/job-tls-bootstrap.yaml index 841d9fbeed8..ac6f39c36b2 100644 --- a/openstack/keystone/templates/job-tls-bootstrap.yaml +++ b/openstack/keystone/templates/job-tls-bootstrap.yaml @@ -36,5 +36,8 @@ spec: - --key-generation={{ required "tls.keyGeneration is required" .Values.tls.keyGeneration }} - --key-wrapping={{ required "tls.keyWrapping is required" .Values.tls.keyWrapping }} - --key-storage={{ required "tls.keyStorage is required" .Values.tls.keyStorage }} + {{- if .Values.tls.allowInsecureStorage }} + - --allow-insecure-storage + {{- end }} backoffLimit: 3 {{- end }} diff --git a/openstack/keystone/values.yaml b/openstack/keystone/values.yaml index 77643b6158c..212df450975 100644 --- a/openstack/keystone/values.yaml +++ b/openstack/keystone/values.yaml @@ -66,8 +66,11 @@ tls: # There are no defaults; the operator must make an explicit choice. # keyGeneration: go-crypto | hsm-entropy | hsm-full | tpm-entropy # keyWrapping: none | vault-transit | hsm | tpm - # keyStorage: k8s-secret | vault-secret - # Escape hatch: set allowInsecureStorage: true to allow wrapping=none + storage=k8s-secret + # keyStorage: internal-k8s-secret | k8s-secret | vault-secret + # internal-k8s-secret (recommended): key written directly to the K8s API; stays in-cluster, never in Git + # k8s-secret: Secret manifest emitted for Git commit + GitOps operator review + # Escape hatch: allowInsecureStorage: true also passes --allow-insecure-storage to the binary, + # permitting wrapping=none + storage=k8s-secret (development only) allowInsecureStorage: false lifecycle: image: keppel.eu-de-1.cloud.sap/ccloud/tls-lifecycle-manager From 2afcc2242ea884c4c0941ff3e8d23aa5a5e20a28 Mon Sep 17 00:00:00 2001 From: Boris Bobrov Date: Sun, 16 Aug 2026 16:34:24 +0200 Subject: [PATCH 23/56] Replace ingress TLS passthrough with direct /32 exposure for Keystone and Barbican Terminate TLS at Apache in the pod and expose it directly via a dedicated BGP-announced /32, instead of nginx-ingress SSL-passthrough (ADR 0406). Each chart's own tls.enabled flag now selects one of two behaviours and the tls.passthrough flag is removed: - tls.enabled=false: nginx-ingress terminates TLS (tls-acme), backend on the internal port, x509 client-cert auth at the ingress. - tls.enabled=true: Apache terminates TLS on :443, exposed via a new dedicated :443-only Service carrying the /32 (services.externalip); no Ingress is rendered; the pod carries skip-inbound-ports: "443"; x509 is verified natively in Apache. Keystone: - Gate ingress-api.yaml on the ingress-termination path and remove the passthrough annotation and backend-port branches. - Move the x509-ca Secret into its own secret-x509-ca.yaml so it is created regardless of whether the Ingress renders; the pod mounts it for the Apache SSLCACertificateFile. Gate matches the mount (services.ingress.x509.ca). - Add service-api-tls.yaml; trim the main Service to internal :5000 only; add the pod skip-inbound-ports annotation. Barbican: - Gate ingress.yaml on the ingress-termination path and remove the passthrough branch. - Add service-tls.yaml and a services.externalip value; trim the main Service to :9311 only; add the pod skip-inbound-ports annotation. Also remove the enable-ssl-passthrough ingress-nginx controller flag in system/kube-system-metal/values.yaml, which is no longer needed. Co-authored-by: Rajiv Mucheli --- .../barbican/templates/api-deployment.yaml | 3 ++ .../templates/etc/_wsgi-barbican.conf.tpl | 9 +++-- openstack/barbican/templates/ingress.yaml | 15 ++------ openstack/barbican/templates/service-tls.yaml | 29 +++++++++++++++ openstack/barbican/templates/service.yaml | 5 --- openstack/barbican/values.yaml | 5 +-- .../keystone/templates/deployment-api.yaml | 1 + .../templates/etc/_wsgi-keystone.conf.tpl | 4 +-- openstack/keystone/templates/ingress-api.yaml | 36 +++---------------- .../keystone/templates/secret-x509-ca.yaml | 15 ++++++++ .../keystone/templates/service-api-tls.yaml | 29 +++++++++++++++ openstack/keystone/templates/service-api.yaml | 9 ----- openstack/keystone/values.yaml | 6 ++-- system/kube-system-metal/values.yaml | 4 --- 14 files changed, 97 insertions(+), 73 deletions(-) create mode 100644 openstack/barbican/templates/service-tls.yaml create mode 100644 openstack/keystone/templates/secret-x509-ca.yaml create mode 100644 openstack/keystone/templates/service-api-tls.yaml diff --git a/openstack/barbican/templates/api-deployment.yaml b/openstack/barbican/templates/api-deployment.yaml index 38d6f99ce63..81c1077a5c4 100644 --- a/openstack/barbican/templates/api-deployment.yaml +++ b/openstack/barbican/templates/api-deployment.yaml @@ -39,6 +39,9 @@ spec: annotations: configmap-etc-hash: {{ include (print $.Template.BasePath "/etc-configmap.yaml") . | sha256sum }} configmap-bin-hash: {{ include (print $.Template.BasePath "/configmap-bin.yaml") . | sha256sum }} + {{- if .Values.tls.enabled }} + config.linkerd.io/skip-inbound-ports: "443" + {{- end }} {{- if .Values.proxysql.mode }} prometheus.io/scrape: "true" prometheus.io/targets: {{ required ".Values.alerts.prometheus missing" .Values.alerts.prometheus | quote }} diff --git a/openstack/barbican/templates/etc/_wsgi-barbican.conf.tpl b/openstack/barbican/templates/etc/_wsgi-barbican.conf.tpl index cbe5bbe18b9..3e30c0305d7 100644 --- a/openstack/barbican/templates/etc/_wsgi-barbican.conf.tpl +++ b/openstack/barbican/templates/etc/_wsgi-barbican.conf.tpl @@ -17,11 +17,10 @@ WSGIDaemonProcess barbican-api processes={{ .Values.api.processes | default 1 }} Listen 0.0.0.0:{{ .Values.api_port_internal }} {{- if .Values.tls.enabled }} -# TLS hardening at server level (must be outside VirtualHost) -Include /etc/apache2/conf-enabled/tls-hardening.conf - -# External HTTPS endpoint (via Ingress TLS passthrough) -# Note: Listen 443 is provided by /etc/apache2/ports.conf when mod_ssl is enabled +# External HTTPS endpoint +# mod_ssl is loaded via a conf-enabled snippet, which is parsed after +# ports.conf, so its ssl_module-gated Listen 443 does not apply here. +Listen 0.0.0.0:443 # TLS hardening at server level (must be outside VirtualHost) Include /etc/apache2/conf-enabled/tls-hardening.conf diff --git a/openstack/barbican/templates/ingress.yaml b/openstack/barbican/templates/ingress.yaml index 3b6614e3b7a..94d26106dd2 100644 --- a/openstack/barbican/templates/ingress.yaml +++ b/openstack/barbican/templates/ingress.yaml @@ -1,3 +1,4 @@ +{{- if not .Values.tls.enabled }} kind: Ingress apiVersion: networking.k8s.io/v1 @@ -13,14 +14,7 @@ metadata: type: api component: barbican annotations: - {{- if and .Values.tls.enabled .Values.tls.passthrough }} - ingress.kubernetes.io/ssl-passthrough: "true" - nginx.ingress.kubernetes.io/ssl-passthrough: "true" - ingress.kubernetes.io/backend-protocol: "HTTPS" - nginx.ingress.kubernetes.io/backend-protocol: "HTTPS" - config.linkerd.io/skip-inbound-ports: "443" - disco: "true" - {{- else if and .Values.tlsacme (not .Values.tls.enabled) }} + {{- if .Values.tlsacme }} kubernetes.io/tls-acme: "true" disco: "true" {{- end }} @@ -45,8 +39,5 @@ spec: service: name: barbican-api port: - {{- if and .Values.tls.enabled .Values.tls.passthrough }} - number: 443 - {{- else }} number: {{.Values.api_port_internal}} - {{- end }} +{{- end }} diff --git a/openstack/barbican/templates/service-tls.yaml b/openstack/barbican/templates/service-tls.yaml new file mode 100644 index 00000000000..f13dc848d59 --- /dev/null +++ b/openstack/barbican/templates/service-tls.yaml @@ -0,0 +1,29 @@ +{{- if .Values.tls.enabled }} +kind: Service +apiVersion: v1 +metadata: + name: barbican-public-tls + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "fullname" . }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" + system: openstack + type: api-tls + component: barbican + annotations: + {{- include "utils.linkerd.pod_and_service_annotation" . | indent 4 }} +spec: + selector: + name: barbican-api + ports: + - name: barbican-tls + protocol: "TCP" + port: 443 + targetPort: 443 + {{- if .Values.services.externalip }} + externalIPs: + - {{ .Values.services.externalip | quote }} + {{- end }} +{{- end }} diff --git a/openstack/barbican/templates/service.yaml b/openstack/barbican/templates/service.yaml index 2616d5b6e6b..9e9f6ca2899 100644 --- a/openstack/barbican/templates/service.yaml +++ b/openstack/barbican/templates/service.yaml @@ -23,8 +23,3 @@ spec: ports: - name: barbican-api port: {{.Values.api_port_internal}} - {{- if .Values.tls.enabled }} - - name: barbican-tls - port: 443 - targetPort: 443 - {{- end }} diff --git a/openstack/barbican/values.yaml b/openstack/barbican/values.yaml index f716b415894..503b96c3e69 100644 --- a/openstack/barbican/values.yaml +++ b/openstack/barbican/values.yaml @@ -472,6 +472,9 @@ sentry: tlsacme: true +services: + # externalip: "192.168.1.100" + # SCIBPL-209: Ingress hardening — request size cap and connection limiting. # These annotations are applied to the nginx ingress controller. # Keystone middleware already enforces authentication; these settings add a @@ -494,8 +497,6 @@ tls: enabled: false # Name of the K8s Secret containing tls.key and tls.crt (created by TLS bootstrap Job) secretName: barbican-tls - # Enable TLS passthrough on the ingress (sends encrypted stream directly to pod) - passthrough: false # Per-region operator choices — all three are REQUIRED when tls.enabled: true. # There are no defaults; the operator must make an explicit choice. # keyGeneration: go-crypto | hsm-entropy | hsm-full | tpm-entropy diff --git a/openstack/keystone/templates/deployment-api.yaml b/openstack/keystone/templates/deployment-api.yaml index 282b8d430c1..4b1231d3206 100644 --- a/openstack/keystone/templates/deployment-api.yaml +++ b/openstack/keystone/templates/deployment-api.yaml @@ -56,6 +56,7 @@ spec: {{- end }} {{- if .Values.tls.enabled }} tls-secret-hash: {{ .Values.tls.secretName | default "keystone-tls" }} + config.linkerd.io/skip-inbound-ports: "443" {{- end }} {{- if .Values.api.metrics.enabled }} prometheus.io/scrape: "true" diff --git a/openstack/keystone/templates/etc/_wsgi-keystone.conf.tpl b/openstack/keystone/templates/etc/_wsgi-keystone.conf.tpl index 6ef5c7bc969..a558b5b6b72 100644 --- a/openstack/keystone/templates/etc/_wsgi-keystone.conf.tpl +++ b/openstack/keystone/templates/etc/_wsgi-keystone.conf.tpl @@ -77,7 +77,7 @@ WSGIServerMetrics On # TLS hardening at server level (must be outside VirtualHost) Include /etc/apache2/conf-enabled/tls-hardening.conf -# External HTTPS endpoint (via Ingress TLS passthrough) +# External HTTPS endpoint # Note: Listen 443 is provided by /etc/apache2/ports.conf when mod_ssl is enabled @@ -88,7 +88,7 @@ Include /etc/apache2/conf-enabled/tls-hardening.conf SSLCertificateKeyFile /mnt/secrets/tls.key {{- if .Values.services.ingress.x509.ca }} - # Client certificate verification (moved from ingress to Apache in passthrough mode) + # Client certificate verification SSLVerifyClient optional SSLVerifyDepth 3 SSLCACertificateFile /etc/apache2/x509-ca/ca.crt diff --git a/openstack/keystone/templates/ingress-api.yaml b/openstack/keystone/templates/ingress-api.yaml index dd8ac97f1be..f6fc112f710 100644 --- a/openstack/keystone/templates/ingress-api.yaml +++ b/openstack/keystone/templates/ingress-api.yaml @@ -1,21 +1,4 @@ -{{- if .Values.services.ingress.enabled }} -{{- if .Values.services.ingress.x509.ca }} -kind: Secret -apiVersion: v1 - -metadata: - name: {{ .Release.Name }}-x509-ca - labels: - app: {{ template "fullname" . }} - chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" - release: "{{ .Release.Name }}" - heritage: "{{ .Release.Service }}" - component: keystone - type: x509-ca -data: - ca.crt: {{ .Values.services.ingress.x509.ca | b64enc }} ---- -{{- end }} +{{- if and .Values.services.ingress.enabled (not .Values.tls.enabled) }} {{- if not .Values.services.ingress.tlsacme }} {{- if .Values.services.public.tlsCertificate}} kind: Secret @@ -46,14 +29,7 @@ metadata: component: keystone type: api annotations: - {{- if and .Values.tls.enabled .Values.tls.passthrough }} - ingress.kubernetes.io/ssl-passthrough: "true" - nginx.ingress.kubernetes.io/ssl-passthrough: "true" - ingress.kubernetes.io/backend-protocol: "HTTPS" - nginx.ingress.kubernetes.io/backend-protocol: "HTTPS" - config.linkerd.io/skip-inbound-ports: "443" - nginx.ingress.kubernetes.io/upstream-hash-by: "$remote_addr" - {{- else if or .Values.federation.oidc.enabled .Values.federation.saml.enabled }} + {{- if or .Values.federation.oidc.enabled .Values.federation.saml.enabled }} # the affinity bits are required for federation; they do not affect the # cases when the API is used, but will affect browser-based workflows ingress.kubernetes.io/affinity: cookie @@ -76,7 +52,7 @@ metadata: ingress.kubernetes.io/limit-connections: {{ .Values.services.ingress.limitConnections | quote }} nginx.ingress.kubernetes.io/limit-connections: {{ .Values.services.ingress.limitConnections | quote }} {{- end }} - {{- if and .Values.services.ingress.x509.ca (not (and .Values.tls.enabled .Values.tls.passthrough)) }} + {{- if .Values.services.ingress.x509.ca }} ingress.kubernetes.io/auth-tls-secret: {{ .Release.Namespace }}/{{ .Release.Name }}-x509-ca nginx.ingress.kubernetes.io/auth-tls-secret: {{ .Release.Namespace }}/{{ .Release.Name }}-x509-ca ingress.kubernetes.io/auth-tls-pass-certificate-to-upstream: "true" @@ -86,7 +62,7 @@ metadata: ingress.kubernetes.io/auth-tls-verify-depth: "3" nginx.ingress.kubernetes.io/auth-tls-verify-depth: "3" {{- end }} - {{- if and .Values.services.ingress.tlsacme (not .Values.tls.enabled) }} + {{- if .Values.services.ingress.tlsacme }} kubernetes.io/tls-acme: "true" {{- end }} {{- if .Values.global.is_global_region }} @@ -130,9 +106,5 @@ spec: service: name: "{{if .Values.global.is_global_region }}keystone-global{{else}}keystone{{end}}" port: - {{- if and .Values.tls.enabled .Values.tls.passthrough }} - number: 443 - {{- else }} number: 5000 - {{- end }} {{- end }} diff --git a/openstack/keystone/templates/secret-x509-ca.yaml b/openstack/keystone/templates/secret-x509-ca.yaml new file mode 100644 index 00000000000..8efbad3ddc0 --- /dev/null +++ b/openstack/keystone/templates/secret-x509-ca.yaml @@ -0,0 +1,15 @@ +{{- if .Values.services.ingress.x509.ca }} +kind: Secret +apiVersion: v1 +metadata: + name: {{ .Release.Name }}-x509-ca + labels: + app: {{ template "fullname" . }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" + component: keystone + type: x509-ca +data: + ca.crt: {{ .Values.services.ingress.x509.ca | b64enc }} +{{- end }} diff --git a/openstack/keystone/templates/service-api-tls.yaml b/openstack/keystone/templates/service-api-tls.yaml new file mode 100644 index 00000000000..5307883b5f4 --- /dev/null +++ b/openstack/keystone/templates/service-api-tls.yaml @@ -0,0 +1,29 @@ +{{- if .Values.tls.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ .Release.Name }}-public-tls + labels: + app: {{ template "fullname" . }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" + system: openstack + component: keystone + type: api-tls + annotations: + {{- include "utils.linkerd.pod_and_service_annotation" . | indent 4 }} +spec: + selector: + name: {{ .Release.Name }}-api + type: {{ default "ClusterIP" .Values.services.type }} + ports: + - name: public-tls + protocol: "TCP" + port: 443 + targetPort: 443 + {{- if .Values.services.externalip }} + externalIPs: + - {{ .Values.services.externalip | quote }} + {{- end }} +{{- end }} diff --git a/openstack/keystone/templates/service-api.yaml b/openstack/keystone/templates/service-api.yaml index baf7e0d6226..d476c603368 100644 --- a/openstack/keystone/templates/service-api.yaml +++ b/openstack/keystone/templates/service-api.yaml @@ -30,16 +30,7 @@ spec: protocol: "TCP" port: 5000 targetPort: 5000 - {{- if .Values.tls.enabled }} - - name: public-tls - protocol: "TCP" - port: 443 - targetPort: 443 - {{- end }} {{- if .Values.global.keystone_internal_ip }} externalIPs: - {{ .Values.global.keystone_internal_ip | quote }} -{{- if .Values.services.externalip }} - - {{ .Values.services.externalip | quote }} -{{- end }} {{- end }} diff --git a/openstack/keystone/values.yaml b/openstack/keystone/values.yaml index 212df450975..618e58974b6 100644 --- a/openstack/keystone/values.yaml +++ b/openstack/keystone/values.yaml @@ -61,7 +61,6 @@ osprofiler-redis: tls: enabled: false secretName: keystone-tls - passthrough: false # Per-region operator choices — all three are REQUIRED when tls.enabled: true. # There are no defaults; the operator must make an explicit choice. # keyGeneration: go-crypto | hsm-entropy | hsm-full | tpm-entropy @@ -321,7 +320,10 @@ services: # let kubernetes.io/tls-acme manage tls certificates tlsacme: true - # x509 client certificate support + # x509 client certificate support. + # NOTE: `ca` is the client-certificate trust anchor, consumed by the + # nginx-ingress (auth-tls-secret) and by Apache (SSLCACertificateFile) when + # TLS terminates in the pod. The `ingress` path in this key name is legacy. x509: # ca: issuer_attribute: HTTP_SSL_CLIENT_I_DN diff --git a/system/kube-system-metal/values.yaml b/system/kube-system-metal/values.yaml index afa57e71137..8dfe2357aa3 100644 --- a/system/kube-system-metal/values.yaml +++ b/system/kube-system-metal/values.yaml @@ -86,10 +86,6 @@ ingress-nginx: # See https://github.com/kubernetes/ingress-nginx/issues/7837 . allowSnippetAnnotations: false - extraArgs: - # Required for ssl-passthrough ingress annotations to take effect. - enable-ssl-passthrough: true - config: custom-http-errors: "495" From 14e79c981fbdbd457c94d7e332e011fec6c2a42a Mon Sep 17 00:00:00 2001 From: Boris Bobrov Date: Mon, 17 Aug 2026 21:35:17 +0200 Subject: [PATCH 24/56] Announce Keystone TLS /32 via LoadBalancer and Calico BGP Expose the Keystone Apache TLS listener on :443 through a dedicated LoadBalancer Service whose /32 is announced over BGP via the Calico projectcalico.org/loadBalancerIPs annotation, with externalTrafficPolicy Local to preserve the client source IP. This replaces the previous ClusterIP plus bare externalIPs, which is not route-advertised, and aligns with the announce convention used by other exposed Services. Publish the public endpoint DNS through a standalone disco Record CR, rendered only when TLS is enabled and the external IP is set. Source the address from the shared global.keystone_external_ip value so the Service and Record stay consistent. --- openstack/keystone/templates/record.yaml | 18 ++++++++++++++++++ .../keystone/templates/service-api-tls.yaml | 10 ++++------ 2 files changed, 22 insertions(+), 6 deletions(-) create mode 100644 openstack/keystone/templates/record.yaml diff --git a/openstack/keystone/templates/record.yaml b/openstack/keystone/templates/record.yaml new file mode 100644 index 00000000000..14c3f023f00 --- /dev/null +++ b/openstack/keystone/templates/record.yaml @@ -0,0 +1,18 @@ +{{- if and .Values.tls.enabled .Values.global.keystone_external_ip }} +apiVersion: disco.stable.sap.cc/v1 +kind: Record +metadata: + name: {{ .Values.services.public.host }}.{{ .Values.global.region }}.{{ .Values.global.tld }} + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "fullname" . }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" +spec: + type: A + record: {{ .Values.global.keystone_external_ip | quote }} + hosts: + - {{ .Values.services.public.host }}.{{ .Values.global.region }}.{{ .Values.global.tld }} + zoneName: {{ .Values.global.region }}.{{ .Values.global.tld }} +{{- end }} diff --git a/openstack/keystone/templates/service-api-tls.yaml b/openstack/keystone/templates/service-api-tls.yaml index 5307883b5f4..6889eb2ebe0 100644 --- a/openstack/keystone/templates/service-api-tls.yaml +++ b/openstack/keystone/templates/service-api-tls.yaml @@ -1,4 +1,4 @@ -{{- if .Values.tls.enabled }} +{{- if and .Values.tls.enabled .Values.global.keystone_external_ip }} apiVersion: v1 kind: Service metadata: @@ -12,18 +12,16 @@ metadata: component: keystone type: api-tls annotations: + projectcalico.org/loadBalancerIPs: '["{{ .Values.global.keystone_external_ip }}"]' {{- include "utils.linkerd.pod_and_service_annotation" . | indent 4 }} spec: + type: LoadBalancer + externalTrafficPolicy: Local selector: name: {{ .Release.Name }}-api - type: {{ default "ClusterIP" .Values.services.type }} ports: - name: public-tls protocol: "TCP" port: 443 targetPort: 443 - {{- if .Values.services.externalip }} - externalIPs: - - {{ .Values.services.externalip | quote }} - {{- end }} {{- end }} From c03eb822bc39dd94f54bfbe1c648580c304f13d4 Mon Sep 17 00:00:00 2001 From: Boris Bobrov Date: Tue, 18 Aug 2026 11:40:00 +0200 Subject: [PATCH 25/56] Rebuild client-certificate headers for cc_x509 at Apache TLS When Apache terminates TLS in the pod, the cc_x509 auth method must still receive the client certificate the way it did when NGINX terminated TLS. Reconstruct the SSL-Client-Cert and SSL-Client-Verify request headers from the verified TLS session, percent-encoding the certificate to the format the plugin already parses. Strip every dash/underscore spelling of these headers before setting them so a client cannot inject a forged certificate; only the values derived from the verified handshake reach the WSGI app. Scope this to the external :443 virtual host. --- .../templates/etc/_wsgi-keystone.conf.tpl | 21 +++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/openstack/keystone/templates/etc/_wsgi-keystone.conf.tpl b/openstack/keystone/templates/etc/_wsgi-keystone.conf.tpl index a558b5b6b72..8f007db7b1c 100644 --- a/openstack/keystone/templates/etc/_wsgi-keystone.conf.tpl +++ b/openstack/keystone/templates/etc/_wsgi-keystone.conf.tpl @@ -88,16 +88,25 @@ Include /etc/apache2/conf-enabled/tls-hardening.conf SSLCertificateKeyFile /mnt/secrets/tls.key {{- if .Values.services.ingress.x509.ca }} - # Client certificate verification SSLVerifyClient optional SSLVerifyDepth 3 SSLCACertificateFile /etc/apache2/x509-ca/ca.crt - # Pass client cert info to WSGI app via request headers - # Replaces the headers that NGINX used to forward when it terminated TLS - RequestHeader set SSL_CLIENT_CERT "%{SSL_CLIENT_CERT}s" - RequestHeader set SSL_CLIENT_I_DN "%{SSL_CLIENT_I_DN}s" - RequestHeader set SSL_CLIENT_VERIFY "%{SSL_CLIENT_VERIFY}s" + # Reconstruct, from the verified TLS session, the client-certificate request + # headers cc_x509 consumes (same format NGINX forwarded when it terminated + # TLS). Every inbound spelling is stripped first so a client cannot inject + # them; only these Apache-set values, sourced from the verified handshake, + # reach the WSGI app. + RequestHeader unset SSL-Client-Cert + RequestHeader unset SSL_Client-Cert + RequestHeader unset SSL-Client_Cert + RequestHeader unset SSL_CLIENT_CERT + RequestHeader unset SSL-Client-Verify + RequestHeader unset SSL_Client-Verify + RequestHeader unset SSL-Client_Verify + RequestHeader unset SSL_CLIENT_VERIFY + RequestHeader set SSL-Client-Cert "expr=%{escape:%{SSL:SSL_CLIENT_CERT}}" + RequestHeader set SSL-Client-Verify "expr=%{SSL:SSL_CLIENT_VERIFY}" {{- end }} WSGIDaemonProcess keystone-tls processes=8 threads=1 user=keystone group=keystone display-name=%{GROUP} From 4df157c04acec12096b90d42865df068b2d7e8bb Mon Sep 17 00:00:00 2001 From: Boris Bobrov Date: Tue, 18 Aug 2026 16:58:41 +0200 Subject: [PATCH 26/56] Set pod fsGroup so the non-root tls-init can read the TLS Secret The tls-init init container runs as the distroless nonroot user (uid/gid 65532), but the tls-input Secret is mounted 0440 and, without an fsGroup, is owned root:root. The unwrap step then fails with "permission denied" on /tls-input/tls.key and the pod never starts. Add a pod-level securityContext.fsGroup: 65532 (gated on tls.enabled) to the Keystone and Barbican API deployments. The Secret files become group readable while staying non-root and non-world-readable, matching the BSI least-privilege requirement. tls-input keeps defaultMode 0440. --- openstack/barbican/templates/api-deployment.yaml | 8 ++++++++ openstack/keystone/templates/deployment-api.yaml | 8 ++++++++ 2 files changed, 16 insertions(+) diff --git a/openstack/barbican/templates/api-deployment.yaml b/openstack/barbican/templates/api-deployment.yaml index 81c1077a5c4..417321e89a0 100644 --- a/openstack/barbican/templates/api-deployment.yaml +++ b/openstack/barbican/templates/api-deployment.yaml @@ -48,6 +48,14 @@ spec: {{- end }} {{- include "utils.linkerd.pod_and_service_annotation" . | indent 8 }} spec: + {{- if .Values.tls.enabled }} + securityContext: + # distroless "nonroot" gid of the tls-lifecycle-manager image; lets the + # non-root tls-init container read the 0440 tls-input Secret while keeping + # the key group-restricted (not world-readable) per the BSI least-privilege + # requirement (Sicherheitsarchitektur 4.1: non-root, restriktive Berechtigungen). + fsGroup: 65532 + {{- end }} {{- if .Values.rbac.enabled }} serviceAccountName: {{ .Release.Name }} {{- end }} diff --git a/openstack/keystone/templates/deployment-api.yaml b/openstack/keystone/templates/deployment-api.yaml index 4b1231d3206..e3f6f91952f 100644 --- a/openstack/keystone/templates/deployment-api.yaml +++ b/openstack/keystone/templates/deployment-api.yaml @@ -64,6 +64,14 @@ spec: {{- end }} {{- include "utils.linkerd.pod_and_service_annotation" . | indent 8 }} spec: + {{- if .Values.tls.enabled }} + securityContext: + # distroless "nonroot" gid of the tls-lifecycle-manager image; lets the + # non-root tls-init container read the 0440 tls-input Secret while keeping + # the key group-restricted (not world-readable) per the BSI least-privilege + # requirement (Sicherheitsarchitektur 4.x: non-root, restriktive Berechtigungen). + fsGroup: 65532 + {{- end }} {{- if .Values.api.priorityClassName }} priorityClassName: {{ .Values.api.priorityClassName }} {{- end }} From d375298b96c86ce6f7a38036db386f7c879e8b30 Mon Sep 17 00:00:00 2001 From: Boris Bobrov Date: Tue, 18 Aug 2026 17:21:10 +0200 Subject: [PATCH 27/56] Support non-Calico external IP for Keystone TLS service The public TLS service only set type: LoadBalancer with the Calico projectcalico.org/loadBalancerIPs annotation. On metal clusters (e.g. qa-de-3) there is no Calico LoadBalancer controller, so the IP is never assigned or advertised and the /32 is unreachable (No route to host). Add a calicoLoadBalancer toggle (matching designate/bind, default true to preserve existing Calico regions). When false, render spec.externalIPs so kube-parrot/externalip-operator announces the address via BGP instead. --- openstack/keystone/templates/service-api-tls.yaml | 6 ++++++ openstack/keystone/values.yaml | 4 ++++ 2 files changed, 10 insertions(+) diff --git a/openstack/keystone/templates/service-api-tls.yaml b/openstack/keystone/templates/service-api-tls.yaml index 6889eb2ebe0..6c47be5a395 100644 --- a/openstack/keystone/templates/service-api-tls.yaml +++ b/openstack/keystone/templates/service-api-tls.yaml @@ -12,7 +12,9 @@ metadata: component: keystone type: api-tls annotations: + {{- if .Values.calicoLoadBalancer }} projectcalico.org/loadBalancerIPs: '["{{ .Values.global.keystone_external_ip }}"]' + {{- end }} {{- include "utils.linkerd.pod_and_service_annotation" . | indent 4 }} spec: type: LoadBalancer @@ -24,4 +26,8 @@ spec: protocol: "TCP" port: 443 targetPort: 443 + {{- if not .Values.calicoLoadBalancer }} + externalIPs: + - {{ .Values.global.keystone_external_ip }} + {{- end }} {{- end }} diff --git a/openstack/keystone/values.yaml b/openstack/keystone/values.yaml index 618e58974b6..13115f4c25c 100644 --- a/openstack/keystone/values.yaml +++ b/openstack/keystone/values.yaml @@ -4,6 +4,10 @@ imageVersionKubernetesEntrypoint: latest # name: value +# Announce the public TLS /32 via Calico (type: LoadBalancer + projectcalico.org/loadBalancerIPs). +# Set to false on metal clusters (kube-parrot/externalip-operator), which announce spec.externalIPs instead. +calicoLoadBalancer: true + global: # tld: cloud.sap # region: cluster From 7015eff981eb85f1088edcb368135ef640c85484 Mon Sep 17 00:00:00 2001 From: Boris Bobrov Date: Thu, 20 Aug 2026 07:57:46 +0200 Subject: [PATCH 28/56] Probe served TLS certificate and alert on expiry and stale rotation Add a blackbox Probe for the public HTTPS endpoint of Keystone and Barbican that observes the certificate actually served on the wire, and alert when it expires within 7 days. Also alert when the rotation CronJob has not completed successfully in over 8 days, covering the case where it never runs at all. The Probe is disabled by default and only renders where TLS and the external endpoint are configured, so it stays inert in regions that have not switched over. The expiry and rotation alerts carry no_alert_on_absence and match on the specific endpoint and CronJob, so they never fire where the probe or rotation is absent. --- .../kubernetes/openstack-barbican.alerts | 35 +++++++++++++++++++ openstack/barbican/templates/probe-tls.yaml | 22 ++++++++++++ openstack/barbican/values.yaml | 6 ++++ .../alerts/kubernetes/keystone.alerts | 35 +++++++++++++++++++ openstack/keystone/templates/probe-tls.yaml | 22 ++++++++++++ openstack/keystone/values.yaml | 6 ++++ 6 files changed, 126 insertions(+) create mode 100644 openstack/barbican/templates/probe-tls.yaml create mode 100644 openstack/keystone/templates/probe-tls.yaml diff --git a/openstack/barbican/alerts/kubernetes/openstack-barbican.alerts b/openstack/barbican/alerts/kubernetes/openstack-barbican.alerts index 8fe19921a6a..d676d78db38 100644 --- a/openstack/barbican/alerts/kubernetes/openstack-barbican.alerts +++ b/openstack/barbican/alerts/kubernetes/openstack-barbican.alerts @@ -17,3 +17,38 @@ groups: annotations: description: 'barbican-api pod {{ $labels.pod }} not ready; recreate pod. See playbook.' summary: 'barbican-api pod not ready' + + - alert: OpenstackBarbicanTLSCertificateExpiresCritical + expr: (probe_ssl_earliest_cert_expiry{instance=~"https://keymanager-3\\..*"} - time()) / 86400 < 7 + for: 1h + labels: + no_alert_on_absence: "true" + context: security + dashboard: barbican + service: barbican + severity: critical + tier: os + support_group: identity + playbook: 'docs/support/playbook/barbican/alerts/tls-certificate-expiry/#openstackbarbicantlscertificateexpirescritical' + annotations: + description: The TLS certificate served by Barbican at {{ $labels.instance }} expires in less than 7 days. External HTTPS access will break when it expires. Check the rotation CronJob and cert-manager. + summary: Barbican served TLS certificate expires within 7 days + + - alert: OpenstackBarbicanTLSRotationStale + expr: > + time() - kube_cronjob_created{cronjob="barbican-tls-rotation"} >= 8*24*60*60 + and on(cronjob) (time() - kube_cronjob_status_last_successful_time{cronjob="barbican-tls-rotation"} + or on(cronjob) kube_cronjob_status_last_schedule_time{cronjob="barbican-tls-rotation"}) >= 8*24*60*60 + for: 1h + labels: + no_alert_on_absence: "true" + context: security + dashboard: barbican + service: barbican + severity: warning + tier: os + support_group: identity + playbook: 'docs/support/playbook/barbican/alerts/tls-certificate-expiry/#openstackbarbicantlsrotationstale' + annotations: + description: The Barbican TLS certificate rotation CronJob has not completed successfully in over 8 days. If not resolved, the certificate will expire and external HTTPS access will break. Check the CronJob and its Jobs. + summary: Barbican TLS certificate rotation is stale diff --git a/openstack/barbican/templates/probe-tls.yaml b/openstack/barbican/templates/probe-tls.yaml new file mode 100644 index 00000000000..3bef7e91923 --- /dev/null +++ b/openstack/barbican/templates/probe-tls.yaml @@ -0,0 +1,22 @@ +{{- if and .Values.tls.enabled .Values.tls.probe.enabled .Values.services.externalip }} +apiVersion: monitoring.coreos.com/v1 +kind: Probe +metadata: + name: {{ .Release.Name }}-tls + labels: + prometheus: {{ .Values.tls.probe.prometheus }} +spec: + jobName: {{ .Release.Name }}-tls + prober: + url: prober.{{ .Values.global.region }}.cloud.sap + scheme: https + module: {{ .Values.tls.probe.module }} + interval: {{ .Values.tls.probe.interval }} + scrapeTimeout: {{ .Values.tls.probe.scrapeTimeout }} + targets: + staticConfig: + static: + - https://{{ include "barbican_api_endpoint_host_public" . }} + labels: + region: {{ .Values.global.region }} +{{- end }} diff --git a/openstack/barbican/values.yaml b/openstack/barbican/values.yaml index 503b96c3e69..d0da1fde3f1 100644 --- a/openstack/barbican/values.yaml +++ b/openstack/barbican/values.yaml @@ -516,6 +516,12 @@ tls: issuerKind: ClusterIssuer issuerGroup: certmanager.cloud.sap keyAlgorithm: ECDSAP256 + probe: + enabled: false + prometheus: kubernetes + module: http_200_403 + interval: 60s + scrapeTimeout: 55s # sapcc/openstack-rate-limit-middleware sapcc_rate_limit: diff --git a/openstack/keystone/alerts/kubernetes/keystone.alerts b/openstack/keystone/alerts/kubernetes/keystone.alerts index 8bb3d7cb083..bb15841690c 100644 --- a/openstack/keystone/alerts/kubernetes/keystone.alerts +++ b/openstack/keystone/alerts/kubernetes/keystone.alerts @@ -51,3 +51,38 @@ groups: description: The container {{ $labels.pod }}/{{ $labels.container }} is being CPU-throttled constantly. This is probably impacting performance, so check if we can increase the number of replicas or the resource requests/limits. + + - alert: OpenstackKeystoneTLSCertificateExpiresCritical + expr: (probe_ssl_earliest_cert_expiry{instance=~"https://identity-3\\..*"} - time()) / 86400 < 7 + for: 1h + labels: + no_alert_on_absence: "true" + context: security + dashboard: keystone + service: keystone + severity: critical + tier: os + support_group: identity + playbook: 'docs/support/playbook/keystone/alerts/tls-certificate-expiry/#openstackkeystonetlscertificateexpirescritical' + annotations: + description: The TLS certificate served by Keystone at {{ $labels.instance }} expires in less than 7 days. External HTTPS access will break when it expires. Check the rotation CronJob and cert-manager. + summary: Keystone served TLS certificate expires within 7 days + + - alert: OpenstackKeystoneTLSRotationStale + expr: > + time() - kube_cronjob_created{cronjob="keystone-tls-rotation"} >= 8*24*60*60 + and on(cronjob) (time() - kube_cronjob_status_last_successful_time{cronjob="keystone-tls-rotation"} + or on(cronjob) kube_cronjob_status_last_schedule_time{cronjob="keystone-tls-rotation"}) >= 8*24*60*60 + for: 1h + labels: + no_alert_on_absence: "true" + context: security + dashboard: keystone + service: keystone + severity: warning + tier: os + support_group: identity + playbook: 'docs/support/playbook/keystone/alerts/tls-certificate-expiry/#openstackkeystonetlsrotationstale' + annotations: + description: The Keystone TLS certificate rotation CronJob has not completed successfully in over 8 days. If not resolved, the certificate will expire and external HTTPS access will break. Check the CronJob and its Jobs. + summary: Keystone TLS certificate rotation is stale diff --git a/openstack/keystone/templates/probe-tls.yaml b/openstack/keystone/templates/probe-tls.yaml new file mode 100644 index 00000000000..f761cd13382 --- /dev/null +++ b/openstack/keystone/templates/probe-tls.yaml @@ -0,0 +1,22 @@ +{{- if and .Values.tls.enabled .Values.tls.probe.enabled .Values.global.keystone_external_ip }} +apiVersion: monitoring.coreos.com/v1 +kind: Probe +metadata: + name: {{ .Release.Name }}-tls + labels: + prometheus: {{ .Values.tls.probe.prometheus }} +spec: + jobName: {{ .Release.Name }}-tls + prober: + url: prober.{{ .Values.global.region }}.cloud.sap + scheme: https + module: {{ .Values.tls.probe.module }} + interval: {{ .Values.tls.probe.interval }} + scrapeTimeout: {{ .Values.tls.probe.scrapeTimeout }} + targets: + staticConfig: + static: + - https://{{ include "keystone_api_endpoint_host_public" . }} + labels: + region: {{ .Values.global.region }} +{{- end }} diff --git a/openstack/keystone/values.yaml b/openstack/keystone/values.yaml index 13115f4c25c..77e3ed85fc5 100644 --- a/openstack/keystone/values.yaml +++ b/openstack/keystone/values.yaml @@ -82,6 +82,12 @@ tls: issuerKind: ClusterIssuer issuerGroup: certmanager.cloud.sap keyAlgorithm: ECDSAP256 + probe: + enabled: false + prometheus: kubernetes + module: http_200_403 + interval: 60s + scrapeTimeout: 55s federation: oidc: From 58c154c68e18847eb94a70c348a4886a80e51e92 Mon Sep 17 00:00:00 2001 From: Boris Bobrov Date: Fri, 21 Aug 2026 15:01:19 +0200 Subject: [PATCH 29/56] Replace runtime a2enmod/a2dissite with static Apache config for Keystone The Keystone start script mutated Apache state at boot: a2enmod ssl/headers, a2dissite 000-default, and a2dismod status. These runtime changes write into /etc/apache2 and prevent a read-only root filesystem. Replace them with static config: - Load mod_ssl, mod_socache_shmcb and mod_headers via a conf-enabled snippet (ssl-load.conf). a2enmod ssl previously pulled in socache_shmcb implicitly; static loading has no dependency resolution, so it is listed explicitly. - Because conf-enabled is parsed after ports.conf, the ssl_module-gated Listen 443 there no longer fires, so the TLS vhost declares Listen 443. - Suppress the default :80 vhost with an emptyDir over sites-enabled instead of a2dissite. - mod_status is enabled by default in the image and ships a status.conf that exposes /server-status with ExtendedStatus On. Neutralize it with a conf-enabled override (status-off.conf) rather than a2dismod, since the module cannot be unloaded statically. mod_wsgi, mod_auth_openidc and mod_shib are already enabled in the image and need no action. The root filesystem stays writable for now; this change removes the config mutations that would block making it read-only later. Verified against the deployed API image: Apache starts, ssl/socache_shmcb/ headers load, port 443 binds, TLS 1.3 negotiates, the default :80 vhost is gone, and /server-status returns 403. --- .../keystone/templates/bin/_keystone_api.sh.tpl | 8 -------- openstack/keystone/templates/configmap-etc.yaml | 4 ++++ openstack/keystone/templates/deployment-api.yaml | 12 ++++++++++++ openstack/keystone/templates/etc/_ssl-load.conf.tpl | 9 +++++++++ .../keystone/templates/etc/_status-off.conf.tpl | 13 +++++++++++++ .../keystone/templates/etc/_wsgi-keystone.conf.tpl | 4 +++- 6 files changed, 41 insertions(+), 9 deletions(-) create mode 100644 openstack/keystone/templates/etc/_ssl-load.conf.tpl create mode 100644 openstack/keystone/templates/etc/_status-off.conf.tpl diff --git a/openstack/keystone/templates/bin/_keystone_api.sh.tpl b/openstack/keystone/templates/bin/_keystone_api.sh.tpl index 381a31da107..17e7f06c18a 100644 --- a/openstack/keystone/templates/bin/_keystone_api.sh.tpl +++ b/openstack/keystone/templates/bin/_keystone_api.sh.tpl @@ -24,14 +24,6 @@ function start () { cp -a $(type -p ${KEYSTONE_WSGI_SCRIPT}) /var/www/cgi-bin/keystone/ done - a2dismod status - - {{- if .Values.tls.enabled }} - a2enmod ssl - a2enmod headers - a2dissite 000-default - {{- end }} - if [ -f /etc/apache2/envvars ]; then # Loading Apache2 ENV variables source /etc/apache2/envvars diff --git a/openstack/keystone/templates/configmap-etc.yaml b/openstack/keystone/templates/configmap-etc.yaml index 175d0caa2bb..9c931932fde 100644 --- a/openstack/keystone/templates/configmap-etc.yaml +++ b/openstack/keystone/templates/configmap-etc.yaml @@ -21,7 +21,11 @@ data: {{ include (print .Template.BasePath "/etc/_sso_callback_template.html.tpl") . | indent 4 }} wsgi-keystone.conf: | {{ include (print .Template.BasePath "/etc/_wsgi-keystone.conf.tpl") . | indent 4 }} + status-off.conf: | +{{ include (print .Template.BasePath "/etc/_status-off.conf.tpl") . | indent 4 }} {{- if .Values.tls.enabled }} + ssl-load.conf: | +{{ include (print .Template.BasePath "/etc/_ssl-load.conf.tpl") . | indent 4 }} tls-hardening.conf: | {{ include (print .Template.BasePath "/etc/_tls-hardening.conf.tpl") . | indent 4 }} {{- end }} diff --git a/openstack/keystone/templates/deployment-api.yaml b/openstack/keystone/templates/deployment-api.yaml index e3f6f91952f..14e76aecf01 100644 --- a/openstack/keystone/templates/deployment-api.yaml +++ b/openstack/keystone/templates/deployment-api.yaml @@ -243,6 +243,12 @@ spec: mountPath: /etc/apache2/conf-enabled/wsgi-keystone.conf subPath: wsgi-keystone.conf readOnly: true + - name: keystone-etc + mountPath: /etc/apache2/conf-enabled/status-off.conf + subPath: status-off.conf + readOnly: true + - name: apache-sites + mountPath: /etc/apache2/sites-enabled {{- if .Values.federation.oidc.enabled }} - name: keystone-federation mountPath: /etc/apache2/conf-enabled/federation-oidc.conf @@ -280,6 +286,10 @@ spec: - name: tls-secrets mountPath: /mnt/secrets readOnly: true + - name: keystone-etc + mountPath: /etc/apache2/conf-enabled/00-ssl-load.conf + subPath: ssl-load.conf + readOnly: true - name: keystone-etc mountPath: /etc/apache2/conf-enabled/tls-hardening.conf subPath: tls-hardening.conf @@ -348,6 +358,8 @@ spec: emptyDir: {} - name: wsgi-keystone emptyDir: {} + - name: apache-sites + emptyDir: {} - name: keystone-etc configMap: name: keystone-etc diff --git a/openstack/keystone/templates/etc/_ssl-load.conf.tpl b/openstack/keystone/templates/etc/_ssl-load.conf.tpl new file mode 100644 index 00000000000..bd4546d2373 --- /dev/null +++ b/openstack/keystone/templates/etc/_ssl-load.conf.tpl @@ -0,0 +1,9 @@ +{{/* +Load mod_ssl and its socache dependency via read-only conf so the modules are +available without a2enmod. mod_headers is required for the x509 client-cert +header reconstruction in the TLS vhost. +*/}} + +LoadModule socache_shmcb_module /usr/lib/apache2/modules/mod_socache_shmcb.so +LoadModule ssl_module /usr/lib/apache2/modules/mod_ssl.so +LoadModule headers_module /usr/lib/apache2/modules/mod_headers.so diff --git a/openstack/keystone/templates/etc/_status-off.conf.tpl b/openstack/keystone/templates/etc/_status-off.conf.tpl new file mode 100644 index 00000000000..28ff85321d3 --- /dev/null +++ b/openstack/keystone/templates/etc/_status-off.conf.tpl @@ -0,0 +1,13 @@ +{{/* +Neutralize mod_status: it is enabled by default in the image and ships a +status.conf that exposes /server-status with ExtendedStatus On. This override +is parsed after mods-enabled, so it wins. +*/}} + + + ExtendedStatus Off + + SetHandler none + Require all denied + + diff --git a/openstack/keystone/templates/etc/_wsgi-keystone.conf.tpl b/openstack/keystone/templates/etc/_wsgi-keystone.conf.tpl index 8f007db7b1c..5a79cdc0687 100644 --- a/openstack/keystone/templates/etc/_wsgi-keystone.conf.tpl +++ b/openstack/keystone/templates/etc/_wsgi-keystone.conf.tpl @@ -78,7 +78,9 @@ WSGIServerMetrics On Include /etc/apache2/conf-enabled/tls-hardening.conf # External HTTPS endpoint -# Note: Listen 443 is provided by /etc/apache2/ports.conf when mod_ssl is enabled +# mod_ssl is loaded via a conf-enabled snippet, which is parsed after +# ports.conf, so its ssl_module-gated Listen 443 does not apply here. +Listen 0.0.0.0:443 ServerName {{ .Values.services.public.host }}.{{ .Values.global.region }}.{{ .Values.global.tld }} From e88f5a3deab23663702cca0734979729a761713c Mon Sep 17 00:00:00 2001 From: Boris Bobrov Date: Fri, 21 Aug 2026 18:41:34 +0200 Subject: [PATCH 30/56] Add LoadBalancer TLS service, DNS record, and Calico support for Barbican Mirror the Keystone exposure model for pod-level TLS termination: a LoadBalancer Service on port 443 that announces the public /32 via Calico loadBalancerIPs (or spec.externalIPs on metal clusters), plus a disco DNS Record for the public host. The external IP is sourced from global.barbican_external_ip with services.externalip kept as an alias. Co-authored-by: Rajiv Mucheli --- openstack/barbican/templates/_helpers.tpl | 5 +++++ openstack/barbican/templates/probe-tls.yaml | 2 +- openstack/barbican/templates/record.yaml | 19 +++++++++++++++++++ openstack/barbican/templates/service-tls.yaml | 12 +++++++++--- openstack/barbican/values.yaml | 7 +++++++ 5 files changed, 41 insertions(+), 4 deletions(-) create mode 100644 openstack/barbican/templates/record.yaml diff --git a/openstack/barbican/templates/_helpers.tpl b/openstack/barbican/templates/_helpers.tpl index fb4433961b1..7c7f858490b 100644 --- a/openstack/barbican/templates/_helpers.tpl +++ b/openstack/barbican/templates/_helpers.tpl @@ -15,6 +15,11 @@ We truncate at 63 chars because some Kubernetes name fields are limited to this {{- printf "%s-%s" .Release.Name $name | trunc 63 | replace "_" "-" | trimSuffix "-" -}} {{- end -}} +{{- define "barbican.external_ip" -}} +{{- $svc := .Values.services | default dict -}} +{{- .Values.global.barbican_external_ip | default $svc.externalip -}} +{{- end -}} + {{- define "barbican.db_service" }} {{- include "utils.db_host" . }} {{- end }} diff --git a/openstack/barbican/templates/probe-tls.yaml b/openstack/barbican/templates/probe-tls.yaml index 3bef7e91923..b2ef0eb6601 100644 --- a/openstack/barbican/templates/probe-tls.yaml +++ b/openstack/barbican/templates/probe-tls.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.tls.enabled .Values.tls.probe.enabled .Values.services.externalip }} +{{- if and .Values.tls.enabled .Values.tls.probe.enabled (include "barbican.external_ip" .) }} apiVersion: monitoring.coreos.com/v1 kind: Probe metadata: diff --git a/openstack/barbican/templates/record.yaml b/openstack/barbican/templates/record.yaml new file mode 100644 index 00000000000..99c02ebb3b4 --- /dev/null +++ b/openstack/barbican/templates/record.yaml @@ -0,0 +1,19 @@ +{{- $externalIP := include "barbican.external_ip" . }} +{{- if and .Values.tls.enabled $externalIP }} +apiVersion: disco.stable.sap.cc/v1 +kind: Record +metadata: + name: {{ include "barbican_api_endpoint_host_public" . }} + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "fullname" . }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" +spec: + type: A + record: {{ $externalIP | quote }} + hosts: + - {{ include "barbican_api_endpoint_host_public" . }} + zoneName: {{ .Values.global.region }}.{{ .Values.global.tld }} +{{- end }} diff --git a/openstack/barbican/templates/service-tls.yaml b/openstack/barbican/templates/service-tls.yaml index f13dc848d59..3d0b88119e4 100644 --- a/openstack/barbican/templates/service-tls.yaml +++ b/openstack/barbican/templates/service-tls.yaml @@ -1,4 +1,5 @@ -{{- if .Values.tls.enabled }} +{{- $externalIP := include "barbican.external_ip" . }} +{{- if and .Values.tls.enabled $externalIP }} kind: Service apiVersion: v1 metadata: @@ -13,8 +14,13 @@ metadata: type: api-tls component: barbican annotations: + {{- if .Values.calicoLoadBalancer }} + projectcalico.org/loadBalancerIPs: '["{{ $externalIP }}"]' + {{- end }} {{- include "utils.linkerd.pod_and_service_annotation" . | indent 4 }} spec: + type: LoadBalancer + externalTrafficPolicy: Local selector: name: barbican-api ports: @@ -22,8 +28,8 @@ spec: protocol: "TCP" port: 443 targetPort: 443 - {{- if .Values.services.externalip }} + {{- if not .Values.calicoLoadBalancer }} externalIPs: - - {{ .Values.services.externalip | quote }} + - {{ $externalIP }} {{- end }} {{- end }} diff --git a/openstack/barbican/values.yaml b/openstack/barbican/values.yaml index d0da1fde3f1..173f165e285 100644 --- a/openstack/barbican/values.yaml +++ b/openstack/barbican/values.yaml @@ -11,6 +11,8 @@ global: domain_seeds: skip_hcm_domain: false linkerd_requested: false + # Public /32 for the pod-level TLS LoadBalancer service and DNS record. + # barbican_external_ip: "192.168.1.100" rbac: enabled: true @@ -472,7 +474,12 @@ sentry: tlsacme: true +# Announce the public TLS /32 via Calico LoadBalancer IPs. Set to false on +# metal clusters that announce via spec.externalIPs instead. +calicoLoadBalancer: true + services: + # Backward-compat alias for global.barbican_external_ip. # externalip: "192.168.1.100" # SCIBPL-209: Ingress hardening — request size cap and connection limiting. From 4978ee28cbf7a3563ab0d485ca0ab4c4d2c629c8 Mon Sep 17 00:00:00 2001 From: Boris Bobrov Date: Mon, 24 Aug 2026 16:09:59 +0200 Subject: [PATCH 31/56] Default the TLS service to spec.externalIPs for Keystone and Barbican Most clusters do not perform automatic IP assignment, so announce the public TLS address via Service.spec.externalIPs by default. Clusters that assign IPs via the Calico annotation set calicoLoadBalancer: true. --- openstack/barbican/values.yaml | 4 +--- openstack/keystone/values.yaml | 4 +--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/openstack/barbican/values.yaml b/openstack/barbican/values.yaml index 173f165e285..88637857e0b 100644 --- a/openstack/barbican/values.yaml +++ b/openstack/barbican/values.yaml @@ -474,9 +474,7 @@ sentry: tlsacme: true -# Announce the public TLS /32 via Calico LoadBalancer IPs. Set to false on -# metal clusters that announce via spec.externalIPs instead. -calicoLoadBalancer: true +calicoLoadBalancer: false services: # Backward-compat alias for global.barbican_external_ip. diff --git a/openstack/keystone/values.yaml b/openstack/keystone/values.yaml index 77e3ed85fc5..638eb33b9cf 100644 --- a/openstack/keystone/values.yaml +++ b/openstack/keystone/values.yaml @@ -4,9 +4,7 @@ imageVersionKubernetesEntrypoint: latest # name: value -# Announce the public TLS /32 via Calico (type: LoadBalancer + projectcalico.org/loadBalancerIPs). -# Set to false on metal clusters (kube-parrot/externalip-operator), which announce spec.externalIPs instead. -calicoLoadBalancer: true +calicoLoadBalancer: false global: # tld: cloud.sap From bff7901d7365a893f0d2da5ffd2bb2d55349e7d0 Mon Sep 17 00:00:00 2001 From: Boris Bobrov Date: Mon, 24 Aug 2026 23:37:07 +0200 Subject: [PATCH 32/56] Harden Barbican pod-level TLS and pin the TLS lifecycle image - Log external HTTPS requests on the Barbican TLS VirtualHost and cap the request body size, mirroring the internal VirtualHost. - Add a PodDisruptionBudget (minAvailable: 1) for the Barbican API so a single pod eviction cannot drop all external HTTPS traffic. - Pin the tls-lifecycle-manager image tag in the Keystone and Barbican chart defaults instead of tracking latest. --- .../templates/etc/_wsgi-barbican.conf.tpl | 10 ++++++++++ openstack/barbican/templates/pdb.yaml | 17 +++++++++++++++++ openstack/barbican/values.yaml | 4 +++- openstack/keystone/values.yaml | 2 +- 4 files changed, 31 insertions(+), 2 deletions(-) create mode 100644 openstack/barbican/templates/pdb.yaml diff --git a/openstack/barbican/templates/etc/_wsgi-barbican.conf.tpl b/openstack/barbican/templates/etc/_wsgi-barbican.conf.tpl index 3e30c0305d7..2039c09fb29 100644 --- a/openstack/barbican/templates/etc/_wsgi-barbican.conf.tpl +++ b/openstack/barbican/templates/etc/_wsgi-barbican.conf.tpl @@ -35,11 +35,21 @@ Include /etc/apache2/conf-enabled/tls-hardening.conf WSGIScriptAlias / /var/www/cgi-bin/barbican/barbican-wsgi-api WSGIApplicationGroup %{GLOBAL} WSGIPassAuthorization On + LimitRequestBody 114688 = 2.4> ErrorLogFormat "%{cu}t %M" ErrorLog /dev/stderr + {{- if .Values.use_json }} + SetEnvIf X-Forwarded-For "^.*\..*\..*\..*" forwarded + CustomLog /dev/stdout json_combined env=!forwarded + CustomLog /dev/stdout json_proxy env=forwarded + {{- else }} + SetEnvIf X-Forwarded-For "^.*\..*\..*\..*" forwarded + CustomLog /dev/stdout combined env=!forwarded + CustomLog /dev/stdout proxy env=forwarded + {{- end }} KeepAliveTimeout 61 diff --git a/openstack/barbican/templates/pdb.yaml b/openstack/barbican/templates/pdb.yaml new file mode 100644 index 00000000000..54931bc33db --- /dev/null +++ b/openstack/barbican/templates/pdb.yaml @@ -0,0 +1,17 @@ +{{- if .Values.api.pdb.minAvailable }} +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: {{ .Release.Name }}-api + labels: + app: {{ template "fullname" . }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: "{{ .Release.Name }}" + component: barbican + type: api +spec: + minAvailable: {{ .Values.api.pdb.minAvailable }} + selector: + matchLabels: + name: barbican-api +{{- end }} diff --git a/openstack/barbican/values.yaml b/openstack/barbican/values.yaml index 88637857e0b..0b196c063da 100644 --- a/openstack/barbican/values.yaml +++ b/openstack/barbican/values.yaml @@ -44,6 +44,8 @@ api: replicas: 2 processes: 1 threads: 1 + pdb: + minAvailable: 1 metrics: enabled: true resources: @@ -515,7 +517,7 @@ tls: # TLS lifecycle manager binary for bootstrap Job and rotation CronJob lifecycle: image: keppel.eu-de-1.cloud.sap/ccloud/tls-lifecycle-manager - imageTag: latest + imageTag: "20260818160021" # cert-manager issuer configuration issuerName: digicert-issuer issuerKind: ClusterIssuer diff --git a/openstack/keystone/values.yaml b/openstack/keystone/values.yaml index 638eb33b9cf..d278e944184 100644 --- a/openstack/keystone/values.yaml +++ b/openstack/keystone/values.yaml @@ -75,7 +75,7 @@ tls: allowInsecureStorage: false lifecycle: image: keppel.eu-de-1.cloud.sap/ccloud/tls-lifecycle-manager - imageTag: latest + imageTag: "20260818160021" issuerName: digicert-issuer issuerKind: ClusterIssuer issuerGroup: certmanager.cloud.sap From ba5fff7cf492efc12393a0e255fe3ac91dd91e25 Mon Sep 17 00:00:00 2001 From: Boris Bobrov Date: Tue, 25 Aug 2026 10:36:26 +0200 Subject: [PATCH 33/56] Harden Apache TLS ciphers, add HTTP security headers and server hardening - Refine the TLS 1.3 cipher suites to AES-GCM and AES-CCM, and disable TLS compression on both Keystone and Barbican. - Set HSTS and standard HTTP security response headers on both TLS vhosts, loading mod_headers for Barbican. - Add a TLS-gated Apache hardening snippet (suppress the version banner, disable TRACE, and turn off directory listing) for both services. - Bump Keystone to 0.14.2 and Barbican to 0.9.3. --- .../barbican/templates/api-deployment.yaml | 4 ++++ openstack/barbican/templates/etc-configmap.yaml | 2 ++ .../templates/etc/_apache-hardening.conf.tpl | 17 +++++++++++++++++ .../barbican/templates/etc/_ssl-load.conf.tpl | 4 +++- .../templates/etc/_tls-hardening.conf.tpl | 7 +++++-- .../templates/etc/_wsgi-barbican.conf.tpl | 7 +++++++ openstack/keystone/templates/configmap-etc.yaml | 2 ++ .../keystone/templates/deployment-api.yaml | 4 ++++ .../templates/etc/_apache-hardening.conf.tpl | 17 +++++++++++++++++ .../templates/etc/_tls-hardening.conf.tpl | 7 +++++-- .../templates/etc/_wsgi-keystone.conf.tpl | 7 +++++++ 11 files changed, 73 insertions(+), 5 deletions(-) create mode 100644 openstack/barbican/templates/etc/_apache-hardening.conf.tpl create mode 100644 openstack/keystone/templates/etc/_apache-hardening.conf.tpl diff --git a/openstack/barbican/templates/api-deployment.yaml b/openstack/barbican/templates/api-deployment.yaml index 417321e89a0..c9e626e8582 100644 --- a/openstack/barbican/templates/api-deployment.yaml +++ b/openstack/barbican/templates/api-deployment.yaml @@ -181,6 +181,10 @@ spec: mountPath: /etc/apache2/conf-enabled/tls-hardening.conf subPath: tls-hardening.conf readOnly: true + - name: barbican-etc + mountPath: /etc/apache2/conf-enabled/apache-hardening.conf + subPath: apache-hardening.conf + readOnly: true {{- end }} - mountPath: /etc/barbican/barbican.conf.d name: barbican-etc-confd diff --git a/openstack/barbican/templates/etc-configmap.yaml b/openstack/barbican/templates/etc-configmap.yaml index 2408bbf5eaf..237a5eb7a96 100644 --- a/openstack/barbican/templates/etc-configmap.yaml +++ b/openstack/barbican/templates/etc-configmap.yaml @@ -16,6 +16,8 @@ data: {{ include (print .Template.BasePath "/etc/_ssl-load.conf.tpl") . | indent 4 }} tls-hardening.conf: | {{ include (print .Template.BasePath "/etc/_tls-hardening.conf.tpl") . | indent 4 }} + apache-hardening.conf: | +{{ include (print .Template.BasePath "/etc/_apache-hardening.conf.tpl") . | indent 4 }} {{- end }} policy.yaml: | {{ include (print .Template.BasePath "/etc/_barbican-policy.yaml.tpl") . | indent 4 }} diff --git a/openstack/barbican/templates/etc/_apache-hardening.conf.tpl b/openstack/barbican/templates/etc/_apache-hardening.conf.tpl new file mode 100644 index 00000000000..8ff2fa1a4db --- /dev/null +++ b/openstack/barbican/templates/etc/_apache-hardening.conf.tpl @@ -0,0 +1,17 @@ +{{/* +Server-level Apache hardening: suppress the version banner, disable the TRACE +method, and turn off directory listing for the document root. Parsed after the +packaged apache2.conf so these directives take precedence. +*/}} + +# Suppress server version banner and disable TRACE +ServerTokens Prod +ServerSignature Off +TraceEnable Off + +# Disable directory listing for the document root + + Options FollowSymLinks + AllowOverride None + Require all granted + diff --git a/openstack/barbican/templates/etc/_ssl-load.conf.tpl b/openstack/barbican/templates/etc/_ssl-load.conf.tpl index 86117cbc2af..ed96306c1db 100644 --- a/openstack/barbican/templates/etc/_ssl-load.conf.tpl +++ b/openstack/barbican/templates/etc/_ssl-load.conf.tpl @@ -1,7 +1,9 @@ {{/* Load mod_ssl and its socache dependency via read-only conf so the module is -available under a read-only root filesystem without a2enmod. +available under a read-only root filesystem without a2enmod. mod_headers is +required for the HTTP security response headers set in the TLS vhost. */}} LoadModule socache_shmcb_module /usr/lib/apache2/modules/mod_socache_shmcb.so LoadModule ssl_module /usr/lib/apache2/modules/mod_ssl.so +LoadModule headers_module /usr/lib/apache2/modules/mod_headers.so diff --git a/openstack/barbican/templates/etc/_tls-hardening.conf.tpl b/openstack/barbican/templates/etc/_tls-hardening.conf.tpl index 5598d9dd0ce..e85cd57fd4d 100644 --- a/openstack/barbican/templates/etc/_tls-hardening.conf.tpl +++ b/openstack/barbican/templates/etc/_tls-hardening.conf.tpl @@ -10,8 +10,8 @@ SSLProtocol -all +TLSv1.3 +TLSv1.2 # TLS 1.2 Cipher Suites (ECDHE + AEAD only, PFS required) SSLCipherSuite ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-GCM-SHA256 -# TLS 1.3 Cipher Suites -SSLCipherSuite TLSv1.3 TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256 +# TLS 1.3 Cipher Suites (AES-GCM and AES-CCM) +SSLCipherSuite TLSv1.3 TLS_AES_256_GCM_SHA384:TLS_AES_128_GCM_SHA256:TLS_AES_128_CCM_SHA256 # Server chooses cipher order SSLHonorCipherOrder on @@ -31,3 +31,6 @@ SSLSessionTickets off # Strict SNI SSLStrictSNIVHostCheck on + +# Disable TLS compression (CRIME mitigation) +SSLCompression off diff --git a/openstack/barbican/templates/etc/_wsgi-barbican.conf.tpl b/openstack/barbican/templates/etc/_wsgi-barbican.conf.tpl index 2039c09fb29..36344bde727 100644 --- a/openstack/barbican/templates/etc/_wsgi-barbican.conf.tpl +++ b/openstack/barbican/templates/etc/_wsgi-barbican.conf.tpl @@ -30,6 +30,13 @@ Include /etc/apache2/conf-enabled/tls-hardening.conf SSLCertificateFile /mnt/secrets/tls.crt SSLCertificateKeyFile /mnt/secrets/tls.key + # HTTP security response headers + Header always set Strict-Transport-Security "max-age=63072000; includeSubDomains" + Header always set X-Content-Type-Options "nosniff" + Header always set X-Frame-Options "DENY" + Header always set Content-Security-Policy "default-src 'self'" + Header always set Referrer-Policy "strict-origin-when-cross-origin" + WSGIDaemonProcess barbican-api-tls processes={{ .Values.api.processes | default 1 }} threads=1 user=barbican group=barbican display-name=%{GROUP} WSGIProcessGroup barbican-api-tls WSGIScriptAlias / /var/www/cgi-bin/barbican/barbican-wsgi-api diff --git a/openstack/keystone/templates/configmap-etc.yaml b/openstack/keystone/templates/configmap-etc.yaml index 9c931932fde..18d7f4a0d5e 100644 --- a/openstack/keystone/templates/configmap-etc.yaml +++ b/openstack/keystone/templates/configmap-etc.yaml @@ -28,6 +28,8 @@ data: {{ include (print .Template.BasePath "/etc/_ssl-load.conf.tpl") . | indent 4 }} tls-hardening.conf: | {{ include (print .Template.BasePath "/etc/_tls-hardening.conf.tpl") . | indent 4 }} + apache-hardening.conf: | +{{ include (print .Template.BasePath "/etc/_apache-hardening.conf.tpl") . | indent 4 }} {{- end }} {{- if .Values.watcher.enabled }} watcher.yaml: | diff --git a/openstack/keystone/templates/deployment-api.yaml b/openstack/keystone/templates/deployment-api.yaml index 14e76aecf01..d89f015ca96 100644 --- a/openstack/keystone/templates/deployment-api.yaml +++ b/openstack/keystone/templates/deployment-api.yaml @@ -294,6 +294,10 @@ spec: mountPath: /etc/apache2/conf-enabled/tls-hardening.conf subPath: tls-hardening.conf readOnly: true + - name: keystone-etc + mountPath: /etc/apache2/conf-enabled/apache-hardening.conf + subPath: apache-hardening.conf + readOnly: true {{- if .Values.services.ingress.x509.ca }} - name: x509-ca mountPath: /etc/apache2/x509-ca diff --git a/openstack/keystone/templates/etc/_apache-hardening.conf.tpl b/openstack/keystone/templates/etc/_apache-hardening.conf.tpl new file mode 100644 index 00000000000..8ff2fa1a4db --- /dev/null +++ b/openstack/keystone/templates/etc/_apache-hardening.conf.tpl @@ -0,0 +1,17 @@ +{{/* +Server-level Apache hardening: suppress the version banner, disable the TRACE +method, and turn off directory listing for the document root. Parsed after the +packaged apache2.conf so these directives take precedence. +*/}} + +# Suppress server version banner and disable TRACE +ServerTokens Prod +ServerSignature Off +TraceEnable Off + +# Disable directory listing for the document root + + Options FollowSymLinks + AllowOverride None + Require all granted + diff --git a/openstack/keystone/templates/etc/_tls-hardening.conf.tpl b/openstack/keystone/templates/etc/_tls-hardening.conf.tpl index 5598d9dd0ce..e85cd57fd4d 100644 --- a/openstack/keystone/templates/etc/_tls-hardening.conf.tpl +++ b/openstack/keystone/templates/etc/_tls-hardening.conf.tpl @@ -10,8 +10,8 @@ SSLProtocol -all +TLSv1.3 +TLSv1.2 # TLS 1.2 Cipher Suites (ECDHE + AEAD only, PFS required) SSLCipherSuite ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-GCM-SHA256 -# TLS 1.3 Cipher Suites -SSLCipherSuite TLSv1.3 TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256 +# TLS 1.3 Cipher Suites (AES-GCM and AES-CCM) +SSLCipherSuite TLSv1.3 TLS_AES_256_GCM_SHA384:TLS_AES_128_GCM_SHA256:TLS_AES_128_CCM_SHA256 # Server chooses cipher order SSLHonorCipherOrder on @@ -31,3 +31,6 @@ SSLSessionTickets off # Strict SNI SSLStrictSNIVHostCheck on + +# Disable TLS compression (CRIME mitigation) +SSLCompression off diff --git a/openstack/keystone/templates/etc/_wsgi-keystone.conf.tpl b/openstack/keystone/templates/etc/_wsgi-keystone.conf.tpl index 5a79cdc0687..7d45809cd5a 100644 --- a/openstack/keystone/templates/etc/_wsgi-keystone.conf.tpl +++ b/openstack/keystone/templates/etc/_wsgi-keystone.conf.tpl @@ -89,6 +89,13 @@ Listen 0.0.0.0:443 SSLCertificateFile /mnt/secrets/tls.crt SSLCertificateKeyFile /mnt/secrets/tls.key + # HTTP security response headers + Header always set Strict-Transport-Security "max-age=63072000; includeSubDomains" + Header always set X-Content-Type-Options "nosniff" + Header always set X-Frame-Options "DENY" + Header always set Content-Security-Policy "default-src 'self'" + Header always set Referrer-Policy "strict-origin-when-cross-origin" + {{- if .Values.services.ingress.x509.ca }} SSLVerifyClient optional SSLVerifyDepth 3 From ef5424cc7956e7f6f267e77f6f7ed5f6684f16df Mon Sep 17 00:00:00 2001 From: Boris Bobrov Date: Tue, 25 Aug 2026 11:09:28 +0200 Subject: [PATCH 34/56] Suppress absence alerting on the TLS rotation failure alerts The TLS rotation failure alerts for Keystone and Barbican lacked no_alert_on_absence, unlike the other TLS alerts. When TLS is disabled the rotation jobs do not exist and the metric is absent, so add the label to avoid spurious absence alerts. --- openstack/barbican/alerts/openstack/openstack-barbican.alerts | 1 + openstack/keystone/alerts/openstack/keystone.alerts | 1 + 2 files changed, 2 insertions(+) diff --git a/openstack/barbican/alerts/openstack/openstack-barbican.alerts b/openstack/barbican/alerts/openstack/openstack-barbican.alerts index f839c6dd282..49263ccd982 100644 --- a/openstack/barbican/alerts/openstack/openstack-barbican.alerts +++ b/openstack/barbican/alerts/openstack/openstack-barbican.alerts @@ -127,6 +127,7 @@ groups: severity: warning tier: os support_group: identity + no_alert_on_absence: "true" annotations: description: The weekly TLS certificate rotation CronJob for Barbican has failed. If not resolved, the TLS certificate will expire and external HTTPS access will break. Check the CronJob logs. summary: Barbican TLS certificate rotation failed diff --git a/openstack/keystone/alerts/openstack/keystone.alerts b/openstack/keystone/alerts/openstack/keystone.alerts index f4587ed6e8f..9dd279cf977 100644 --- a/openstack/keystone/alerts/openstack/keystone.alerts +++ b/openstack/keystone/alerts/openstack/keystone.alerts @@ -115,6 +115,7 @@ groups: severity: warning tier: os support_group: identity + no_alert_on_absence: "true" annotations: description: The weekly TLS certificate rotation CronJob for Keystone has failed. If not resolved, the TLS certificate will expire and external HTTPS access will break. Check the CronJob logs. summary: Keystone TLS certificate rotation failed From 8acb16f4081b70fb5fbb6cbe0309d7eeaf7dd992 Mon Sep 17 00:00:00 2001 From: Boris Bobrov Date: Wed, 26 Aug 2026 08:33:50 +0200 Subject: [PATCH 35/56] Remove non-functional tls-secret-hash pod annotation --- openstack/keystone/templates/deployment-api.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/openstack/keystone/templates/deployment-api.yaml b/openstack/keystone/templates/deployment-api.yaml index d89f015ca96..eeeb4f5c674 100644 --- a/openstack/keystone/templates/deployment-api.yaml +++ b/openstack/keystone/templates/deployment-api.yaml @@ -55,7 +55,6 @@ spec: federation-saml-hash: {{ include (print $.Template.BasePath "/federation-saml.yaml") . | sha256sum }} {{- end }} {{- if .Values.tls.enabled }} - tls-secret-hash: {{ .Values.tls.secretName | default "keystone-tls" }} config.linkerd.io/skip-inbound-ports: "443" {{- end }} {{- if .Values.api.metrics.enabled }} From 37f97ecbc3778597fbff19ad5ce1971892f3d969 Mon Sep 17 00:00:00 2001 From: Boris Bobrov Date: Wed, 26 Aug 2026 22:44:11 +0200 Subject: [PATCH 36/56] Include per-tenant SAML config in the TLS vhost --- openstack/keystone/templates/etc/_wsgi-keystone.conf.tpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openstack/keystone/templates/etc/_wsgi-keystone.conf.tpl b/openstack/keystone/templates/etc/_wsgi-keystone.conf.tpl index 7d45809cd5a..8ecc093d95f 100644 --- a/openstack/keystone/templates/etc/_wsgi-keystone.conf.tpl +++ b/openstack/keystone/templates/etc/_wsgi-keystone.conf.tpl @@ -140,7 +140,7 @@ Listen 0.0.0.0:443 SetHandler shib - IncludeOptional /etc/apache2/conf-enabled/federation-saml.conf + IncludeOptional /etc/apache2/conf-enabled/federation-saml.d/*.conf {{- end }} {{- end }} From 0958691f4229a89bd74e81f865c619d00670e0df Mon Sep 17 00:00:00 2001 From: Boris Bobrov Date: Thu, 27 Aug 2026 06:12:34 +0200 Subject: [PATCH 37/56] Require an external IP when pod-level TLS is enabled --- openstack/barbican/templates/_helpers.tpl | 4 ++++ openstack/keystone/templates/_helpers.tpl | 3 +++ 2 files changed, 7 insertions(+) diff --git a/openstack/barbican/templates/_helpers.tpl b/openstack/barbican/templates/_helpers.tpl index 7c7f858490b..b6b858cd389 100644 --- a/openstack/barbican/templates/_helpers.tpl +++ b/openstack/barbican/templates/_helpers.tpl @@ -59,5 +59,9 @@ We truncate at 63 chars because some Kubernetes name fields are limited to this {{- if and (eq .Values.tls.keyWrapping "none") (eq .Values.tls.keyStorage "k8s-secret") (not .Values.tls.allowInsecureStorage) }} {{- fail "tls: unwrapped keys cannot be stored as plain-text K8s Secrets. Set tls.keyWrapping or tls.keyStorage, or set tls.allowInsecureStorage: true to acknowledge." }} {{- end }} + {{- $svc := .Values.services | default dict }} + {{- if not (.Values.global.barbican_external_ip | default $svc.externalip) }} + {{- fail "tls.enabled requires an external IP (global.barbican_external_ip or services.externalip): enabling TLS removes the ingress and the public TLS Service only renders once the external IP is set." }} + {{- end }} {{- end }} {{- end }} diff --git a/openstack/keystone/templates/_helpers.tpl b/openstack/keystone/templates/_helpers.tpl index 5dcc1603622..aa9ea61c99d 100644 --- a/openstack/keystone/templates/_helpers.tpl +++ b/openstack/keystone/templates/_helpers.tpl @@ -126,5 +126,8 @@ Empty values are skipped so callers may pass "" for optional dependencies. {{- if and (eq .Values.tls.keyWrapping "none") (eq .Values.tls.keyStorage "k8s-secret") (not .Values.tls.allowInsecureStorage) }} {{- fail "tls: unwrapped keys cannot be stored as plain-text K8s Secrets. Set tls.keyWrapping or tls.keyStorage, or set tls.allowInsecureStorage: true to acknowledge." }} {{- end }} + {{- if not .Values.global.keystone_external_ip }} + {{- fail "tls.enabled requires global.keystone_external_ip: enabling TLS removes the ingress and the public TLS Service only renders once the external IP is set." }} + {{- end }} {{- end }} {{- end }} From 9dfb2b5f6fa056d21d0101d3abb3749b4c66aa7d Mon Sep 17 00:00:00 2001 From: Boris Bobrov Date: Thu, 27 Aug 2026 22:37:41 +0200 Subject: [PATCH 38/56] Harden x509 header handling, unify Barbican WSGI group, restore keystone external IP Rebuild and sanitize the client-certificate issuer and subject DN request headers on the Keystone TLS vhost so tokenless x509 auth reads verified values and clients cannot inject them. Collapse the Barbican TLS vhost onto the single barbican-api WSGI daemon group to keep one PKCS#11 session per pod and honor api.threads. Restore the services.externalip entry on the plain Keystone API Service. --- .../barbican/templates/etc/_wsgi-barbican.conf.tpl | 3 +-- .../keystone/templates/etc/_wsgi-keystone.conf.tpl | 11 +++++++++++ openstack/keystone/templates/service-api.yaml | 3 +++ 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/openstack/barbican/templates/etc/_wsgi-barbican.conf.tpl b/openstack/barbican/templates/etc/_wsgi-barbican.conf.tpl index 36344bde727..e8031d378aa 100644 --- a/openstack/barbican/templates/etc/_wsgi-barbican.conf.tpl +++ b/openstack/barbican/templates/etc/_wsgi-barbican.conf.tpl @@ -37,8 +37,7 @@ Include /etc/apache2/conf-enabled/tls-hardening.conf Header always set Content-Security-Policy "default-src 'self'" Header always set Referrer-Policy "strict-origin-when-cross-origin" - WSGIDaemonProcess barbican-api-tls processes={{ .Values.api.processes | default 1 }} threads=1 user=barbican group=barbican display-name=%{GROUP} - WSGIProcessGroup barbican-api-tls + WSGIProcessGroup barbican-api WSGIScriptAlias / /var/www/cgi-bin/barbican/barbican-wsgi-api WSGIApplicationGroup %{GLOBAL} WSGIPassAuthorization On diff --git a/openstack/keystone/templates/etc/_wsgi-keystone.conf.tpl b/openstack/keystone/templates/etc/_wsgi-keystone.conf.tpl index 8ecc093d95f..b31ad1c6d0f 100644 --- a/openstack/keystone/templates/etc/_wsgi-keystone.conf.tpl +++ b/openstack/keystone/templates/etc/_wsgi-keystone.conf.tpl @@ -114,8 +114,19 @@ Listen 0.0.0.0:443 RequestHeader unset SSL_Client-Verify RequestHeader unset SSL-Client_Verify RequestHeader unset SSL_CLIENT_VERIFY + {{- /* The issuer/subject DN env vars have four underscore-separated tokens, + so a client can spell the injected header with '-' or '_' at each of + the three separators; strip every combination. */}} + {{- range $dn := list "I" "S" }} + {{- range $s1 := list "-" "_" }} + {{- range $s2 := list "-" "_" }} + {{- range $s3 := list "-" "_" }} + RequestHeader unset SSL{{ $s1 }}Client{{ $s2 }}{{ $dn }}{{ $s3 }}DN + {{- end }}{{- end }}{{- end }}{{- end }} RequestHeader set SSL-Client-Cert "expr=%{escape:%{SSL:SSL_CLIENT_CERT}}" RequestHeader set SSL-Client-Verify "expr=%{SSL:SSL_CLIENT_VERIFY}" + RequestHeader set SSL-Client-I-DN "expr=%{SSL:SSL_CLIENT_I_DN}" + RequestHeader set SSL-Client-S-DN "expr=%{SSL:SSL_CLIENT_S_DN}" {{- end }} WSGIDaemonProcess keystone-tls processes=8 threads=1 user=keystone group=keystone display-name=%{GROUP} diff --git a/openstack/keystone/templates/service-api.yaml b/openstack/keystone/templates/service-api.yaml index d476c603368..15f7d815eb0 100644 --- a/openstack/keystone/templates/service-api.yaml +++ b/openstack/keystone/templates/service-api.yaml @@ -33,4 +33,7 @@ spec: {{- if .Values.global.keystone_internal_ip }} externalIPs: - {{ .Values.global.keystone_internal_ip | quote }} +{{- if .Values.services.externalip }} + - {{ .Values.services.externalip | quote }} +{{- end }} {{- end }} From 49198b4baeca8f6d8f59c3892bdc5ce2be96bbc8 Mon Sep 17 00:00:00 2001 From: Boris Bobrov Date: Fri, 28 Aug 2026 11:05:19 +0200 Subject: [PATCH 39/56] Allow the WebSSO auto-submit script under the Keystone CSP The TLS vhost CSP default-src 'self' blocked the inline auto-submit script in the WebSSO callback, leaving browser federation logins stuck. Allow just that script via its sha256 hash, and guard the pinned hash at render time so a change to the callback template fails loudly instead of silently breaking WebSSO. --- openstack/keystone/templates/_helpers.tpl | 13 +++++++++++++ .../keystone/templates/etc/_wsgi-keystone.conf.tpl | 5 ++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/openstack/keystone/templates/_helpers.tpl b/openstack/keystone/templates/_helpers.tpl index aa9ea61c99d..74a350470ed 100644 --- a/openstack/keystone/templates/_helpers.tpl +++ b/openstack/keystone/templates/_helpers.tpl @@ -129,5 +129,18 @@ Empty values are skipped so callers may pass "" for optional dependencies. {{- if not .Values.global.keystone_external_ip }} {{- fail "tls.enabled requires global.keystone_external_ip: enabling TLS removes the ingress and the public TLS Service only renders once the external IP is set." }} {{- end }} + {{- /* Guard the CSP script-src hash against drift: the TLS vhost pins a + sha256 of the WebSSO callback's inline auto-submit script, and Helm + cannot recompute a base64 digest, so fail loudly if the script changes + and both pins must be refreshed together. */}} + {{- $ssoTpl := include (print $.Template.BasePath "/etc/_sso_callback_template.html.tpl") . }} + {{- $ssoParts := splitList "" (index $ssoParts 1)) 0 }} + {{- if ne (sha256sum $ssoScript) "a016a19411507a6fa732cd49c20701074dc7cbc9d18797bca8419339cc6602e3" }} + {{- fail "SSO callback inline script changed; recompute the CSP script-src hash (sha256/base64 of the