From 15a4e28fea6d18b2e61419e8aef13d57aaebf971 Mon Sep 17 00:00:00 2001 From: Stefan Hipfel Date: Fri, 4 Sep 2026 13:47:07 +0200 Subject: [PATCH 1/4] ironic: add conductor-group-sync CronJob for kos_conductor: false mode When kos_conductor is false, a CronJob runs hourly to reconcile conductor_group on Ironic nodes against the groups defined in conductor.hosts. The block->group mapping is rendered from the conductor.hosts[*].blocks field (written by the sync-conductors Concourse job) at Helm deploy time. --- .../conductor-group-sync-cronjob.yaml | 63 +++++++++++++ openstack/ironic/templates/etc-configmap.yaml | 4 + .../etc/_conductor_group_sync.py.tpl | 92 +++++++++++++++++++ openstack/ironic/values.yaml | 3 + 4 files changed, 162 insertions(+) create mode 100644 openstack/ironic/templates/conductor-group-sync-cronjob.yaml create mode 100644 openstack/ironic/templates/etc/_conductor_group_sync.py.tpl diff --git a/openstack/ironic/templates/conductor-group-sync-cronjob.yaml b/openstack/ironic/templates/conductor-group-sync-cronjob.yaml new file mode 100644 index 00000000000..288c3007a22 --- /dev/null +++ b/openstack/ironic/templates/conductor-group-sync-cronjob.yaml @@ -0,0 +1,63 @@ +{{- if not .Values.kos_conductor }} +apiVersion: batch/v1 +kind: CronJob +metadata: + name: ironic-conductor-group-sync + labels: + system: openstack + type: cronjob + component: ironic +spec: + schedule: {{ .Values.conductor.groupSyncSchedule | default "17 * * * *" | quote }} + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 3 + jobTemplate: + spec: + template: + spec: + restartPolicy: OnFailure + containers: + - name: ironic-conductor-group-sync + image: {{ .Values.global.registry }}/loci-ironic:{{ .Values.imageVersion }} + imagePullPolicy: IfNotPresent + command: + - bash + args: + - -c + - | + # Source OpenStack credentials from ironic.conf [service_catalog] + [keystone_authtoken] + eval $( + cat /etc/ironic/ironic.conf | grep -Pzo '\[service_catalog\][^[]*' | tr -d '\000' | grep '=' | + while read LINE; do var="${LINE% =*}" + val="${LINE#*= }" + echo export OS_${var^^}=${val} + done) + export OS_AUTH_URL=$(grep -Pzo '\[keystone_authtoken\][^[]*' /etc/ironic/ironic.conf | tr -d '\000' | grep 'auth_url' | awk -F'= ' '{print $2}' | head -1) + export OS_IDENTITY_API_VERSION=3 + + python3 /etc/ironic/conductor-group-sync.py + volumeMounts: + - mountPath: /etc/ironic/ironic.conf.d + name: ironic-etc-confd + - mountPath: /etc/ironic/ironic.conf + name: ironic-etc + subPath: ironic.conf + readOnly: true + - mountPath: /etc/ironic/conductor-group-sync.py + name: ironic-etc + subPath: conductor-group-sync.py + readOnly: true + {{- include "utils.trust_bundle.volume_mount" . | indent 12 }} + volumes: + - name: ironic-etc-confd + secret: + secretName: {{ .Release.Name }}-secrets + items: + - key: secrets.conf + path: secrets.conf + - name: ironic-etc + configMap: + name: ironic-etc + {{- include "utils.trust_bundle.volumes" . | indent 10 }} +{{- end }} diff --git a/openstack/ironic/templates/etc-configmap.yaml b/openstack/ironic/templates/etc-configmap.yaml index 8b4d4727efc..57801ca329c 100644 --- a/openstack/ironic/templates/etc-configmap.yaml +++ b/openstack/ironic/templates/etc-configmap.yaml @@ -34,6 +34,10 @@ data: {{- if .Values.audit.enabled }} api_audit_map.yaml: | {{ include (print .Template.BasePath "/etc/_api_audit_map.yaml.tpl") . | indent 4 }} +{{- end }} +{{- if not .Values.kos_conductor }} + conductor-group-sync.py: | +{{ include "ironic_conductor_group_sync_py" . | indent 4 }} {{- end }} statsd-exporter.yaml: | defaults: diff --git a/openstack/ironic/templates/etc/_conductor_group_sync.py.tpl b/openstack/ironic/templates/etc/_conductor_group_sync.py.tpl new file mode 100644 index 00000000000..49d50e71b45 --- /dev/null +++ b/openstack/ironic/templates/etc/_conductor_group_sync.py.tpl @@ -0,0 +1,92 @@ +{{- define "ironic_conductor_group_sync_py" }} +#!/usr/bin/env python3 +""" +conductor-group-sync: update conductor_group on Ironic nodes to match the +conductor groups defined in conductor.hosts. + +Reads OpenStack credentials from the environment (sourced from ironic.conf +and secrets.conf by the CronJob wrapper). +""" + +import re +import sys +import json +import subprocess + +# Block -> conductor group mapping, rendered from conductor.hosts at deploy time +BLOCK_TO_GROUP = { +{{- range .Values.conductor.hosts }} +{{- $group := .name }} +{{- range (.blocks | default list) }} + {{ . | quote }}: {{ $group | quote }}, +{{- end }} +{{- end }} +} + + +def openstack(args): + cmd = ["openstack"] + args + ["-f", "json"] + result = subprocess.run(cmd, capture_output=True, text=True) + if result.returncode != 0: + print(f"ERROR: openstack {' '.join(args)} failed:\n{result.stderr}", file=sys.stderr) + sys.exit(1) + return json.loads(result.stdout) + + +def main(): + if not BLOCK_TO_GROUP: + print("No block-to-group mapping configured, nothing to do.") + sys.exit(0) + + print("Listing Ironic nodes...") + nodes = openstack(["baremetal", "node", "list", "--fields", "uuid", "name", "conductor_group", "--limit", "0"]) + + updates = [] + for node in nodes: + name = node.get("Name") or node.get("name") or "" + uuid = node.get("UUID") or node.get("uuid") or "" + current_group = node.get("Conductor Group") or node.get("conductor_group") or "" + + if not name or not uuid: + continue + + # Extract block from node name: last segment after - or . + m = re.search(r'[.\-]([^.\-]+)$', name) + if not m: + continue + block = m.group(1) + + expected_group = BLOCK_TO_GROUP.get(block) + if expected_group is None: + continue # block not managed by us + + if current_group != expected_group: + updates.append((uuid, name, block, current_group, expected_group)) + + if not updates: + print("All nodes already have the correct conductor_group.") + sys.exit(0) + + print(f"Updating conductor_group on {len(updates)} node(s):") + errors = 0 + for uuid, name, block, current, expected in updates: + print(f" {name} ({block}): {current!r} -> {expected!r}") + result = subprocess.run( + ["openstack", "baremetal", "node", "set", "--conductor-group", expected, uuid], + capture_output=True, text=True + ) + if result.returncode != 0: + print(f" ERROR: {result.stderr.strip()}", file=sys.stderr) + errors += 1 + + if errors: + print(f"{errors} error(s) during update.", file=sys.stderr) + sys.exit(1) + + print("Done.") + sys.exit(0) + + +if __name__ == "__main__": + main() +{{- end }} diff --git a/openstack/ironic/values.yaml b/openstack/ironic/values.yaml index bd7a6114cbd..08671865b91 100644 --- a/openstack/ironic/values.yaml +++ b/openstack/ironic/values.yaml @@ -176,6 +176,9 @@ inspector: conductor: hosts: [] + # groupSyncSchedule controls the CronJob that updates conductor_group on Ironic + # nodes when kos_conductor is false. Defaults to hourly at :17. + groupSyncSchedule: "17 * * * *" deploy: protocol: "http" port: 8088 From 20893e6f2674e1a0135f96a12e6f6586e076b660 Mon Sep 17 00:00:00 2001 From: Stefan Hipfel Date: Fri, 4 Sep 2026 15:35:05 +0200 Subject: [PATCH 2/4] Revert CronJob, keep nodeAffinity pending --- .../conductor-group-sync-cronjob.yaml | 63 ------------- openstack/ironic/templates/etc-configmap.yaml | 4 - .../etc/_conductor_group_sync.py.tpl | 92 ------------------- openstack/ironic/values.yaml | 3 - 4 files changed, 162 deletions(-) delete mode 100644 openstack/ironic/templates/conductor-group-sync-cronjob.yaml delete mode 100644 openstack/ironic/templates/etc/_conductor_group_sync.py.tpl diff --git a/openstack/ironic/templates/conductor-group-sync-cronjob.yaml b/openstack/ironic/templates/conductor-group-sync-cronjob.yaml deleted file mode 100644 index 288c3007a22..00000000000 --- a/openstack/ironic/templates/conductor-group-sync-cronjob.yaml +++ /dev/null @@ -1,63 +0,0 @@ -{{- if not .Values.kos_conductor }} -apiVersion: batch/v1 -kind: CronJob -metadata: - name: ironic-conductor-group-sync - labels: - system: openstack - type: cronjob - component: ironic -spec: - schedule: {{ .Values.conductor.groupSyncSchedule | default "17 * * * *" | quote }} - concurrencyPolicy: Forbid - successfulJobsHistoryLimit: 3 - failedJobsHistoryLimit: 3 - jobTemplate: - spec: - template: - spec: - restartPolicy: OnFailure - containers: - - name: ironic-conductor-group-sync - image: {{ .Values.global.registry }}/loci-ironic:{{ .Values.imageVersion }} - imagePullPolicy: IfNotPresent - command: - - bash - args: - - -c - - | - # Source OpenStack credentials from ironic.conf [service_catalog] + [keystone_authtoken] - eval $( - cat /etc/ironic/ironic.conf | grep -Pzo '\[service_catalog\][^[]*' | tr -d '\000' | grep '=' | - while read LINE; do var="${LINE% =*}" - val="${LINE#*= }" - echo export OS_${var^^}=${val} - done) - export OS_AUTH_URL=$(grep -Pzo '\[keystone_authtoken\][^[]*' /etc/ironic/ironic.conf | tr -d '\000' | grep 'auth_url' | awk -F'= ' '{print $2}' | head -1) - export OS_IDENTITY_API_VERSION=3 - - python3 /etc/ironic/conductor-group-sync.py - volumeMounts: - - mountPath: /etc/ironic/ironic.conf.d - name: ironic-etc-confd - - mountPath: /etc/ironic/ironic.conf - name: ironic-etc - subPath: ironic.conf - readOnly: true - - mountPath: /etc/ironic/conductor-group-sync.py - name: ironic-etc - subPath: conductor-group-sync.py - readOnly: true - {{- include "utils.trust_bundle.volume_mount" . | indent 12 }} - volumes: - - name: ironic-etc-confd - secret: - secretName: {{ .Release.Name }}-secrets - items: - - key: secrets.conf - path: secrets.conf - - name: ironic-etc - configMap: - name: ironic-etc - {{- include "utils.trust_bundle.volumes" . | indent 10 }} -{{- end }} diff --git a/openstack/ironic/templates/etc-configmap.yaml b/openstack/ironic/templates/etc-configmap.yaml index 57801ca329c..8b4d4727efc 100644 --- a/openstack/ironic/templates/etc-configmap.yaml +++ b/openstack/ironic/templates/etc-configmap.yaml @@ -34,10 +34,6 @@ data: {{- if .Values.audit.enabled }} api_audit_map.yaml: | {{ include (print .Template.BasePath "/etc/_api_audit_map.yaml.tpl") . | indent 4 }} -{{- end }} -{{- if not .Values.kos_conductor }} - conductor-group-sync.py: | -{{ include "ironic_conductor_group_sync_py" . | indent 4 }} {{- end }} statsd-exporter.yaml: | defaults: diff --git a/openstack/ironic/templates/etc/_conductor_group_sync.py.tpl b/openstack/ironic/templates/etc/_conductor_group_sync.py.tpl deleted file mode 100644 index 49d50e71b45..00000000000 --- a/openstack/ironic/templates/etc/_conductor_group_sync.py.tpl +++ /dev/null @@ -1,92 +0,0 @@ -{{- define "ironic_conductor_group_sync_py" }} -#!/usr/bin/env python3 -""" -conductor-group-sync: update conductor_group on Ironic nodes to match the -conductor groups defined in conductor.hosts. - -Reads OpenStack credentials from the environment (sourced from ironic.conf -and secrets.conf by the CronJob wrapper). -""" - -import re -import sys -import json -import subprocess - -# Block -> conductor group mapping, rendered from conductor.hosts at deploy time -BLOCK_TO_GROUP = { -{{- range .Values.conductor.hosts }} -{{- $group := .name }} -{{- range (.blocks | default list) }} - {{ . | quote }}: {{ $group | quote }}, -{{- end }} -{{- end }} -} - - -def openstack(args): - cmd = ["openstack"] + args + ["-f", "json"] - result = subprocess.run(cmd, capture_output=True, text=True) - if result.returncode != 0: - print(f"ERROR: openstack {' '.join(args)} failed:\n{result.stderr}", file=sys.stderr) - sys.exit(1) - return json.loads(result.stdout) - - -def main(): - if not BLOCK_TO_GROUP: - print("No block-to-group mapping configured, nothing to do.") - sys.exit(0) - - print("Listing Ironic nodes...") - nodes = openstack(["baremetal", "node", "list", "--fields", "uuid", "name", "conductor_group", "--limit", "0"]) - - updates = [] - for node in nodes: - name = node.get("Name") or node.get("name") or "" - uuid = node.get("UUID") or node.get("uuid") or "" - current_group = node.get("Conductor Group") or node.get("conductor_group") or "" - - if not name or not uuid: - continue - - # Extract block from node name: last segment after - or . - m = re.search(r'[.\-]([^.\-]+)$', name) - if not m: - continue - block = m.group(1) - - expected_group = BLOCK_TO_GROUP.get(block) - if expected_group is None: - continue # block not managed by us - - if current_group != expected_group: - updates.append((uuid, name, block, current_group, expected_group)) - - if not updates: - print("All nodes already have the correct conductor_group.") - sys.exit(0) - - print(f"Updating conductor_group on {len(updates)} node(s):") - errors = 0 - for uuid, name, block, current, expected in updates: - print(f" {name} ({block}): {current!r} -> {expected!r}") - result = subprocess.run( - ["openstack", "baremetal", "node", "set", "--conductor-group", expected, uuid], - capture_output=True, text=True - ) - if result.returncode != 0: - print(f" ERROR: {result.stderr.strip()}", file=sys.stderr) - errors += 1 - - if errors: - print(f"{errors} error(s) during update.", file=sys.stderr) - sys.exit(1) - - print("Done.") - sys.exit(0) - - -if __name__ == "__main__": - main() -{{- end }} diff --git a/openstack/ironic/values.yaml b/openstack/ironic/values.yaml index 08671865b91..bd7a6114cbd 100644 --- a/openstack/ironic/values.yaml +++ b/openstack/ironic/values.yaml @@ -176,9 +176,6 @@ inspector: conductor: hosts: [] - # groupSyncSchedule controls the CronJob that updates conductor_group on Ironic - # nodes when kos_conductor is false. Defaults to hourly at :17. - groupSyncSchedule: "17 * * * *" deploy: protocol: "http" port: 8088 From ea11517ebd88e716c364fbd7b0645addfa1db0a0 Mon Sep 17 00:00:00 2001 From: Stefan Hipfel Date: Fri, 4 Sep 2026 15:35:16 +0200 Subject: [PATCH 3/4] ironic: add zone nodeAffinity to conductor deployment when site is set --- .../ironic/templates/_conductor-deployment.yaml.tpl | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/openstack/ironic/templates/_conductor-deployment.yaml.tpl b/openstack/ironic/templates/_conductor-deployment.yaml.tpl index e0a5aa85c78..751186e51c2 100644 --- a/openstack/ironic/templates/_conductor-deployment.yaml.tpl +++ b/openstack/ironic/templates/_conductor-deployment.yaml.tpl @@ -60,6 +60,18 @@ spec: {{- if .Values.rbac.enabled }} serviceAccountName: {{ .Release.Name }} {{- end }} + {{- if $conductor.site }} + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - key: topology.kubernetes.io/zone + operator: In + values: + - {{ $conductor.site }} + {{- end }} {{- include "utils.proxysql.pod_settings" . | indent 6 }} initContainers: {{- tuple . (dict "service" "ironic-api,ironic-rabbitmq") | include "utils.snippets.kubernetes_entrypoint_init_container" | indent 6 }} From 95dfbbbf74afac8cd224a4d08911aa69b27f757d Mon Sep 17 00:00:00 2001 From: Stefan Hipfel Date: Fri, 4 Sep 2026 20:11:21 +0200 Subject: [PATCH 4/4] ironic: add quota-class-sync CronJob for kos_conductor: false mode --- .../templates/quota-class-sync-cronjob.yaml | 93 +++++++++++++++++++ 1 file changed, 93 insertions(+) create mode 100644 openstack/ironic/templates/quota-class-sync-cronjob.yaml diff --git a/openstack/ironic/templates/quota-class-sync-cronjob.yaml b/openstack/ironic/templates/quota-class-sync-cronjob.yaml new file mode 100644 index 00000000000..0b147e10d66 --- /dev/null +++ b/openstack/ironic/templates/quota-class-sync-cronjob.yaml @@ -0,0 +1,93 @@ +{{- if not .Values.kos_conductor }} +apiVersion: batch/v1 +kind: CronJob +metadata: + name: ironic-quota-class-sync + labels: + system: openstack + type: cronjob + component: ironic +spec: + schedule: "37 * * * *" + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 3 + jobTemplate: + spec: + template: + spec: + restartPolicy: OnFailure + containers: + - name: ironic-quota-class-sync + image: {{ .Values.global.registry }}/loci-ironic:{{ .Values.imageVersion }} + imagePullPolicy: IfNotPresent + command: + - bash + args: + - -c + - | + eval $( + cat /etc/ironic/ironic.conf | grep -Pzo '\[service_catalog\][^[]*' | tr -d '\000' | grep '=' | + while read LINE; do var="${LINE% =*}" + val="${LINE#*= }" + echo export OS_${var^^}=${val} + done) + export OS_AUTH_URL=$(grep -Pzo '\[keystone_authtoken\][^[]*' /etc/ironic/ironic.conf | tr -d '\000' | grep 'auth_url' | awk -F'= ' '{print $2}' | head -1) + export OS_IDENTITY_API_VERSION=3 + + python3 - <<'EOF' +import os, sys, json, subprocess + +def openstack(args): + result = subprocess.run(["openstack"] + args + ["-f", "json"], capture_output=True, text=True) + if result.returncode != 0: + print(f"ERROR: {result.stderr}", file=sys.stderr) + sys.exit(1) + return json.loads(result.stdout) + +nodes = openstack(["baremetal", "node", "list", "--fields", "resource_class", "--limit", "0"]) +resource_classes = { + n.get("Resource Class") or n.get("resource_class") + for n in nodes + if (n.get("Resource Class") or n.get("resource_class")) + and not (n.get("Resource Class") or n.get("resource_class", "")).startswith(("tempest-Resource_Class-", "ResClass-")) +} +print(f"Resource classes: {sorted(resource_classes)}") + +token = subprocess.run(["openstack", "token", "issue", "-f", "value", "-c", "id"], + capture_output=True, text=True).stdout.strip() +nova_url = subprocess.run(["openstack", "endpoint", "list", "--service", "compute", + "--interface", "public", "-f", "value", "-c", "URL"], + capture_output=True, text=True).stdout.strip().rstrip("/") + +import urllib.request +quotas = {"quota_class_set": {f"instances_{r}": 0 for r in resource_classes}} +req = urllib.request.Request( + f"{nova_url}/os-quota-class-sets/flavors", + data=json.dumps(quotas).encode(), + headers={"X-Auth-Token": token, "Content-Type": "application/json"}, + method="POST", +) +with urllib.request.urlopen(req) as resp: + print(f"Response: {resp.status}") +EOF + volumeMounts: + - mountPath: /etc/ironic/ironic.conf + name: ironic-etc + subPath: ironic.conf + readOnly: true + - mountPath: /etc/ironic/ironic.conf.d + name: ironic-etc-confd + {{- include "utils.trust_bundle.volume_mount" . | indent 12 }} + volumes: + - name: ironic-etc + configMap: + name: ironic-etc + - name: ironic-etc-confd + secret: + secretName: {{ .Release.Name }}-secrets + items: + - key: secrets.conf + path: secrets.conf + {{- include "utils.trust_bundle.volumes" . | indent 10 }} +{{- end }}