From 066550b91a5f99589d54ce5ff93ee1ddc3be66cb Mon Sep 17 00:00:00 2001 From: Krstan Vjestica Date: Sun, 26 Jul 2026 11:21:03 +0200 Subject: [PATCH 1/2] fix: assert the default queue has a member it can execute on register_queue only assigns instances when it creates the group. On an upgrade the group already exists, so it prints "Instance Group already registered" and leaves it empty -- and an empty regular instance group accepts job launches and never runs them. Assert membership here rather than trusting that call, and fold the separate node_type fix-up into the same block: the backend now honours FORAIL_NODE_TYPE when the task pod registers itself, but a newer chart can still meet an older image, and this Job is what makes the two converge. --- files/scripts/init.sh | 67 +++++++++++++++++++++++-------------------- 1 file changed, 36 insertions(+), 31 deletions(-) diff --git a/files/scripts/init.sh b/files/scripts/init.sh index 4b5cfa3..9f6fa35 100644 --- a/files/scripts/init.sh +++ b/files/scripts/init.sh @@ -30,44 +30,49 @@ forail-manage provision_instance --skip-checks \ --hostname="${NODE_NAME}" \ --node_type="${NODE_TYPE}" -# provision_instance only inserts if missing — it does not update an -# already-registered instance. forail-web auto-registers itself as -# 'control' on first startup before this Job runs, so without this -# explicit ORM update the cluster ends up with node_type=control and -# refuses to execute jobs (it can only orchestrate). Force-set the -# requested type so launches stay on this instance via the local -# Receptor work command instead of routing to a ContainerGroup. -forail-manage shell -c " -from forail.main.models import Instance -i = Instance.objects.filter(hostname='${NODE_NAME}').first() -if i and i.node_type != '${NODE_TYPE}': - i.node_type = '${NODE_TYPE}' - i.save(update_fields=['node_type']) - print('Updated', i.hostname, 'node_type ->', i.node_type) -else: - print('Instance node_type already', i.node_type if i else 'missing') -" - echo "==> Registering queues..." forail-manage register_queue --skip-checks --queuename=controlplane --instance_percent=100 forail-manage register_queue --skip-checks --queuename=default --instance_percent=100 -# A post_migrate signal in forail auto-creates the 'default' InstanceGroup -# as is_container_group=true when running in k8s. Without a working -# ContainerGroup that resolves to a local Receptor work type, every job -# launch errors with 'unknown work type kubernetes-incluster-auth'. -# Flip it back to a regular IG so register_queue's instance assignment -# above is honored and jobs run via the local work command. +# Three things the 'default' group needs that the commands above do not +# reliably leave behind: +# +# 1. is_container_group=false. A post_migrate signal auto-creates 'default' as +# a ContainerGroup on k8s, and without a work type that resolves locally +# every launch errors with 'unknown work type kubernetes-incluster-auth'. +# 2. Membership. On an UPGRADE the group already exists, so register_queue +# prints "Instance Group already registered" and assigns no instance. The +# group is left empty and every job sits in "pending" forever, with nothing +# in the UI or the logs to say why. +# 3. node_type, as a backstop. The real fix for that one is in the backend — +# the task pod re-runs provision_instance on every start and used to +# re-register as 'control' unconditionally, undoing whatever this Job set; +# it now honours FORAIL_NODE_TYPE. Keep the assertion so a newer chart +# paired with an older backend image still converges. forail-manage shell -c " -from forail.main.models import InstanceGroup +from forail.main.models import Instance, InstanceGroup ig = InstanceGroup.objects.filter(name='default').first() -if ig and ig.is_container_group: - ig.is_container_group = False - ig.pod_spec_override = '' - ig.save(update_fields=['is_container_group', 'pod_spec_override']) - print('default IG: is_container_group -> False') +if not ig: + print('default IG missing — register_queue did not create it') else: - print('default IG already non-container or missing') + if ig.is_container_group: + ig.is_container_group = False + ig.pod_spec_override = '' + ig.save(update_fields=['is_container_group', 'pod_spec_override']) + print('default IG: is_container_group -> False') + i = Instance.objects.filter(hostname='${NODE_NAME}').first() + if not i: + print('instance ${NODE_NAME} missing — cannot assign to default IG') + else: + if i.node_type != '${NODE_TYPE}': + i.node_type = '${NODE_TYPE}' + i.save(update_fields=['node_type']) + print('instance node_type ->', i.node_type) + if not ig.instances.filter(pk=i.pk).exists(): + ig.instances.add(i) + print('default IG: added', i.hostname) + else: + print('default IG already contains', i.hostname) " echo "==> Creating preload data..." From 91572c0d79925deecf3375ad47804c53063dd0fa Mon Sep 17 00:00:00 2001 From: Krstan Vjestica Date: Sun, 26 Jul 2026 20:30:00 +0200 Subject: [PATCH 2/2] release: chart 2026.7.1 pinning the fixed backend image Closes the changelog section that still said [Unreleased] while 2026.7.0 was already out. --- CHANGELOG.md | 18 ++++++++++++++++++ Chart.yaml | 4 ++-- values.yaml | 2 +- 3 files changed, 21 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 017e437..812f9db 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,24 @@ and the chart uses SemVer (`version`) plus the upstream Forail CalVer ## [Unreleased] +## [2026.7.1] - 2026-07-26 + +### Fixed +- **An upgrade no longer leaves the `default` queue unable to run anything.** + `register_queue` assigns instances only when it creates the group, so on an + upgrade it printed `Instance Group already registered default` and assigned + nothing — and an empty regular instance group accepts job launches and never + runs them. The init Job now asserts membership itself, alongside the + container-group flag it already asserted. + +### Changed +- `images.backend.tag` pinned to `2026.07.1`, which carries the matching backend + fix: the task pod used to re-register itself as `control` and flip `default` + back to a ContainerGroup on every start, undoing what this Job sets. The + frontend stays at `2026.07.0` — it is unchanged. + +## [2026.7.0] - 2026-07-25 + ### Added - **Job-execution RBAC** (`templates/rbac.yaml`): a `forail` ServiceAccount plus a namespaced `Role`/`RoleBinding` (`forail-job-runner`) granting diff --git a/Chart.yaml b/Chart.yaml index 6cf9aa0..d8b5837 100644 --- a/Chart.yaml +++ b/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: forail description: Forail Platform — automation, RBAC, EDA, observability (k8s deployment) type: application -version: 2026.7.0 -appVersion: "2026.07.0" +version: 2026.7.1 +appVersion: "2026.07.1" # forail-operator lives in its own repo (forail-platform/forail-operator) and # is installed separately via its own helm chart: # helm install forail-operator -n forail --set forail.token= diff --git a/values.yaml b/values.yaml index b6cb167..ba588dc 100644 --- a/values.yaml +++ b/values.yaml @@ -13,7 +13,7 @@ namespace: images: backend: repository: ghcr.io/forail-platform/forail-backend - tag: "2026.07.0" + tag: "2026.07.1" pullPolicy: IfNotPresent frontend: repository: ghcr.io/forail-platform/forail-frontend