From e97824562853d47d1774e4f72191fb9a38a382a9 Mon Sep 17 00:00:00 2001
From: Vladimir Antropov <vladimir.antropov@milaboratories.com>
Date: Thu, 16 Jul 2026 13:52:34 +0200
Subject: [PATCH 1/3] chore(gcp): bump monolithic GCP deployment to Platforma
 chart 4.2.0
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Vendor the v4.2.0 chart (from the release tag, so it matches the 4.2.0
backend image) in place of 4.1.5. Verified: helm lint passes and the chart
renders against the monolith's values (gcs main storage, Filestore
workspace, dedicated Kueue with the batch ComputeClass nodeSelector, GPU
disabled) — batch flavor keeps cloud.google.com/compute-class=platforma-batch,
--runner-max-ram-request stays 484Gi, and the job template's ResourceID field
is satisfied by the paired 4.2.0 backend.

Update the upgrade runbook to 4.2.0 and note that the one-time re-login only
applies when coming from 3.5.0 (4.1.x already has the master secret).
---
 charts/platforma/Chart.yaml                   |   4 +-
 charts/platforma/files/job-script.sh          |  14 +-
 .../configmap-job-template-kueue.yaml         |  11 ++
 charts/platforma/templates/deployment.yaml    |  31 ++++-
 .../templates/kueue-clusterqueues.yaml        |  32 +++--
 .../templates/nvidia-device-plugin.yaml       |  10 +-
 charts/platforma/values.schema.json           | 127 ++++++------------
 charts/platforma/values.yaml                  |  58 +++++---
 ...-4.1.5.md => UPGRADE-monolith-to-4.2.0.md} |  18 ++-
 9 files changed, 178 insertions(+), 127 deletions(-)
 rename infrastructure/gcp/{UPGRADE-monolith-to-4.1.5.md => UPGRADE-monolith-to-4.2.0.md} (84%)

diff --git a/charts/platforma/Chart.yaml b/charts/platforma/Chart.yaml
index 30a77f2..d95793d 100644
--- a/charts/platforma/Chart.yaml
+++ b/charts/platforma/Chart.yaml
@@ -2,8 +2,8 @@ apiVersion: v2
 name: platforma
 description: Platforma - Bioinformatics platform for Kubernetes
 type: application
-version: 4.1.5
-appVersion: "4.1.5"
+version: 4.2.0
+appVersion: "4.2.0"
 kubeVersion: ">=1.31.0-0"
 
 keywords:
diff --git a/charts/platforma/files/job-script.sh b/charts/platforma/files/job-script.sh
index f791cbb..a27be56 100755
--- a/charts/platforma/files/job-script.sh
+++ b/charts/platforma/files/job-script.sh
@@ -2,7 +2,19 @@
 set -e
 set -u
 
-# Prepend PL_JOB_PATH to PATH if set
+# Prepend GPU driver paths (set by the chart when the job runs on a GPU node).
+# Done at runtime, not via k8s env, because k8s env can't reference $(PATH) /
+# $(LD_LIBRARY_PATH) from the container image — setting them statically would
+# wipe image-provided entries (e.g. conda's /opt/conda/bin).
+if [ -n "${PL_GPU_BIN_PATH:-}" ]; then
+  export PATH="${PL_GPU_BIN_PATH}:${PATH}"
+fi
+if [ -n "${PL_GPU_LIB_PATH:-}" ]; then
+  export LD_LIBRARY_PATH="${PL_GPU_LIB_PATH}${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}"
+fi
+
+# Prepend PL_JOB_PATH to PATH if set. Applied after PL_GPU_BIN_PATH so
+# runenv-provided entries (e.g. a conda env's bin) win over GPU paths.
 if [ -n "${PL_JOB_PATH:-}" ]; then
   export PATH="${PL_JOB_PATH}:${PATH}"
 fi
diff --git a/charts/platforma/templates/configmap-job-template-kueue.yaml b/charts/platforma/templates/configmap-job-template-kueue.yaml
index 8ec37fb..8854b61 100644
--- a/charts/platforma/templates/configmap-job-template-kueue.yaml
+++ b/charts/platforma/templates/configmap-job-template-kueue.yaml
@@ -22,6 +22,7 @@ data:
       name: << .JobID >>
       labels:
         platforma.bio/job-id: << .JobID >>
+        platforma.bio/resource-id: << .ResourceID >>
         app.kubernetes.io/managed-by: platforma
         app.kubernetes.io/instance: {{ .Release.Name }}
         # Queue label for Kueue - maps Platforma queue to Kueue LocalQueue
@@ -52,6 +53,7 @@ data:
               name: << .JobID >>
               labels:
                 platforma.bio/job-id: << .JobID >>
+                platforma.bio/resource-id: << .ResourceID >>
                 app.kubernetes.io/managed-by: platforma
             spec:
               backoffLimit: 0  # AppWrapper handles retries
@@ -62,6 +64,7 @@ data:
                     cluster-autoscaler.kubernetes.io/safe-to-evict: "false"
                   labels:
                     platforma.bio/job-id: << .JobID >>
+                    platforma.bio/resource-id: << .ResourceID >>
                     app.kubernetes.io/managed-by: platforma
                 spec:
                   serviceAccountName: {{ include "platforma.jobServiceAccountName" . }}
@@ -160,6 +163,14 @@ data:
                           value: "all"
                         - name: NVIDIA_DRIVER_CAPABILITIES
                           value: "compute,utility"
+                        # These are marker vars consumed by job_script.sh,
+                        # which prepends them to PATH/LD_LIBRARY_PATH at
+                        # container start — preserving image-provided
+                        # entries that a static k8s env value would wipe.
+                        - name: PL_GPU_BIN_PATH
+                          value: "/usr/local/nvidia/bin"
+                        - name: PL_GPU_LIB_PATH
+                          value: "/usr/local/nvidia/lib64:/usr/local/nvidia/lib"
                         - name: PLATFORMA_GPU_MEMORY
                           value: "<< .Resources.GPUMemory >>"
                         <<- end >>
diff --git a/charts/platforma/templates/deployment.yaml b/charts/platforma/templates/deployment.yaml
index ecb2aed..87c0a47 100644
--- a/charts/platforma/templates/deployment.yaml
+++ b/charts/platforma/templates/deployment.yaml
@@ -205,9 +205,19 @@ spec:
             {{- if .Values.kueue.pools.gpu.enabled }}
             - "--runner-gpu-available=enabled"
             {{- if not (and .Values.kueue.maxJobResources .Values.kueue.maxJobResources.gpuMemory) }}
-            {{- fail "ERROR: kueue.maxJobResources.gpuMemory is required when kueue.pools.gpu.enabled=true.\nSet it to the largest VRAM available on a single GPU node in your cluster (e.g. 24Gi, 80Gi, 192Gi)." }}
+            {{- fail "ERROR: kueue.maxJobResources.gpuMemory is required when kueue.pools.gpu.enabled=true.\nSet it to the largest VRAM available on a single GPU node in your cluster (e.g. 24Gi, 80Gi)." }}
             {{- end }}
             - "--runner-max-gpu-memory-request={{ .Values.kueue.maxJobResources.gpuMemory }}"
+            # GPU jobs run on a separate node pool; ceiling their CPU/RAM to the largest GPU
+            # node's allocatable so oversized requests land there instead of staying Pending.
+            {{- if not .Values.kueue.maxJobResources.gpuCpu }}
+            {{- fail "ERROR: kueue.maxJobResources.gpuCpu is required when kueue.pools.gpu.enabled=true.\nSet it to the allocatable CPU cores of the largest GPU node in your cluster." }}
+            {{- end }}
+            {{- if not .Values.kueue.maxJobResources.gpuRam }}
+            {{- fail "ERROR: kueue.maxJobResources.gpuRam is required when kueue.pools.gpu.enabled=true.\nSet it to the allocatable RAM of the largest GPU node in your cluster (e.g. 128Gi, 192Gi)." }}
+            {{- end }}
+            - "--k8s-gpu-max-cpu-request={{ .Values.kueue.maxJobResources.gpuCpu }}"
+            - "--k8s-gpu-max-ram-request={{ .Values.kueue.maxJobResources.gpuRam }}"
             {{- else }}
             - "--runner-gpu-available=disabled"
             {{- end }}
@@ -273,6 +283,9 @@ spec:
             {{- if .Values.auth.sso.prompt }}
             - "--sso-idp-prompt={{ .Values.auth.sso.prompt }}"
             {{- end }}
+            {{- if .Values.auth.sso.accessType }}
+            - "--sso-idp-access-type={{ .Values.auth.sso.accessType }}"
+            {{- end }}
             {{- if .Values.auth.sso.userIdClaim }}
             - "--sso-idp-user-id-claim={{ .Values.auth.sso.userIdClaim }}"
             {{- end }}
@@ -286,6 +299,9 @@ spec:
             {{- if .Values.masterSecret.secretName }}
             - "--master-secret-file={{ include "platforma.path.secrets" . }}/master-secret/{{ .Values.masterSecret.secretKey }}"
             {{- end }}
+            {{- if .Values.auth.sso.clientSecret.secretName }}
+            - "--sso-idp-client-secret-file={{ include "platforma.path.secrets" . }}/sso-client-secret/{{ .Values.auth.sso.clientSecret.secretKey }}"
+            {{- end }}
             # Disable default host data library (exposes entire pod filesystem)
             - "--no-host-data-library"
             # Data library sources
@@ -481,6 +497,11 @@ spec:
               mountPath: {{ include "platforma.path.secrets" . }}/master-secret
               readOnly: true
             {{- end }}
+            {{- if .Values.auth.sso.clientSecret.secretName }}
+            - name: sso-client-secret
+              mountPath: {{ include "platforma.path.secrets" . }}/sso-client-secret
+              readOnly: true
+            {{- end }}
             {{- range .Values.dataSources }}
             {{- if eq .type "pvc" }}
             - name: datasource-{{ .name }}
@@ -540,6 +561,14 @@ spec:
               - key: {{ .Values.masterSecret.secretKey }}
                 path: {{ .Values.masterSecret.secretKey }}
         {{- end }}
+        {{- if .Values.auth.sso.clientSecret.secretName }}
+        - name: sso-client-secret
+          secret:
+            secretName: {{ .Values.auth.sso.clientSecret.secretName }}
+            items:
+              - key: {{ .Values.auth.sso.clientSecret.secretKey }}
+                path: {{ .Values.auth.sso.clientSecret.secretKey }}
+        {{- end }}
         {{- range .Values.dataSources }}
         {{- if eq .type "pvc" }}
         - name: datasource-{{ .name }}
diff --git a/charts/platforma/templates/kueue-clusterqueues.yaml b/charts/platforma/templates/kueue-clusterqueues.yaml
index deeed31..3f22607 100644
--- a/charts/platforma/templates/kueue-clusterqueues.yaml
+++ b/charts/platforma/templates/kueue-clusterqueues.yaml
@@ -54,23 +54,31 @@ spec:
   {{- if .Values.kueue.provisioningRequest.enabled }}
   admissionChecksStrategy:
     admissionChecks:
+      # Scope ProvisioningRequest check to the CPU-only batch flavor.
+      # GPU jobs skip ProvReq because best-effort-atomic-scale-up's strict
+      # capacity planner often refuses GPU scale-ups that a plain MIG resize
+      # would satisfy (per (machine-type, zone) inventory variance), and
+      # atomic reservation is unnecessary for single-pod GPU blocks. Keep
+      # ProvReq on the CPU flavor where multi-pod batch atomicity helps.
       - name: {{ include "platforma.kueue.admissionCheckName" . }}
+        onFlavors:
+          - {{ include "platforma.kueue.clusterResourceName" . }}-batch-flavor
   {{- end }}
   preemption:
     reclaimWithinCohort: Never   # Jobs not retryable, never preempt
     withinClusterQueue: Never    # No preemption within batch queue
   resourceGroups:
 {{- if .Values.kueue.pools.gpu.enabled }}
+    # Flavor declaration order matters: Kueue tries flavors top-down and
+    # assigns the first one whose quota fits the workload. batch-flavor
+    # is listed FIRST so CPU-only jobs land there and go through the
+    # ProvisioningRequest AdmissionCheck (admissionChecksStrategy above).
+    # If gpu-flavor came first, CPU-only jobs requesting < gpu.cpu/memory
+    # would silently be assigned to gpu-flavor and bypass ProvReq (the
+    # check is scoped onFlavors=[batch-flavor]), so multi-pod batch
+    # atomicity would only hold while gpu-flavor was fully saturated.
     - coveredResources: ["cpu", "memory", "nvidia.com/gpu"]
       flavors:
-        - name: {{ include "platforma.kueue.clusterResourceName" . }}-gpu-flavor
-          resources:
-            - name: cpu
-              nominalQuota: {{ .Values.kueue.dedicated.resources.gpu.cpu | quote }}
-            - name: memory
-              nominalQuota: {{ .Values.kueue.dedicated.resources.gpu.memory | quote }}
-            - name: "nvidia.com/gpu"
-              nominalQuota: {{ .Values.kueue.dedicated.resources.gpu.gpu | quote }}
         - name: {{ include "platforma.kueue.clusterResourceName" . }}-batch-flavor
           resources:
             - name: cpu
@@ -81,6 +89,14 @@ spec:
               borrowingLimit: {{ .Values.kueue.dedicated.resources.ui.memory | quote }}
             - name: "nvidia.com/gpu"
               nominalQuota: "0"
+        - name: {{ include "platforma.kueue.clusterResourceName" . }}-gpu-flavor
+          resources:
+            - name: cpu
+              nominalQuota: {{ .Values.kueue.dedicated.resources.gpu.cpu | quote }}
+            - name: memory
+              nominalQuota: {{ .Values.kueue.dedicated.resources.gpu.memory | quote }}
+            - name: "nvidia.com/gpu"
+              nominalQuota: {{ .Values.kueue.dedicated.resources.gpu.gpu | quote }}
 {{- else }}
     - coveredResources: ["cpu", "memory"]
       flavors:
diff --git a/charts/platforma/templates/nvidia-device-plugin.yaml b/charts/platforma/templates/nvidia-device-plugin.yaml
index 195f60f..a7d5f00 100644
--- a/charts/platforma/templates/nvidia-device-plugin.yaml
+++ b/charts/platforma/templates/nvidia-device-plugin.yaml
@@ -1,9 +1,17 @@
-{{- if .Values.kueue.pools.gpu.enabled }}
+{{- if and .Values.kueue.pools.gpu.enabled (eq .Values.kueue.mode "dedicated") .Values.kueue.dedicated.createClusterResources }}
 # =============================================================================
 # NVIDIA Device Plugin DaemonSet
 # =============================================================================
 # Exposes nvidia.com/gpu resources on GPU nodes so the K8s scheduler can
 # assign GPU workloads. Only runs on nodes with NVIDIA GPUs.
+#
+# Gated on the same condition as the cluster-scoped Kueue resources
+# (ClusterQueues, ResourceFlavors): this DaemonSet and its cluster-scoped RBAC
+# are cluster-wide infrastructure, deployed once by the install that owns the
+# Kueue cohort (dedicated mode + createClusterResources). Shared-mode and
+# secondary installs still set kueue.pools.gpu.enabled=true to route GPU jobs
+# and enable the runner GPU flag, but must NOT re-deploy these cluster-scoped
+# objects — two releases doing so collide on the identical resource names.
 # Based on: https://github.com/NVIDIA/k8s-device-plugin v0.17.0
 #
 # Startup taint protocol:
diff --git a/charts/platforma/values.schema.json b/charts/platforma/values.schema.json
index a431902..ebd6707 100644
--- a/charts/platforma/values.schema.json
+++ b/charts/platforma/values.schema.json
@@ -20,11 +20,7 @@
         },
         "pullPolicy": {
           "type": "string",
-          "enum": [
-            "Always",
-            "IfNotPresent",
-            "Never"
-          ]
+          "enum": ["Always", "IfNotPresent", "Never"]
         }
       },
       "additionalProperties": false
@@ -38,9 +34,7 @@
             "type": "string"
           }
         },
-        "required": [
-          "name"
-        ]
+        "required": ["name"]
       }
     },
     "validation": {
@@ -54,11 +48,7 @@
     },
     "environment": {
       "type": "string",
-      "enum": [
-        "",
-        "aws",
-        "gcp"
-      ]
+      "enum": ["", "aws", "gcp"]
     },
     "license": {
       "type": "object",
@@ -70,10 +60,7 @@
           "type": "string"
         }
       },
-      "required": [
-        "secretName",
-        "secretKey"
-      ],
+      "required": ["secretName", "secretKey"],
       "additionalProperties": false
     },
     "auth": {
@@ -97,10 +84,7 @@
                     "type": "string"
                   }
                 },
-                "required": [
-                  "username",
-                  "password"
-                ]
+                "required": ["username", "password"]
               }
             },
             "secretName": {
@@ -204,19 +188,29 @@
             },
             "prompt": {
               "type": "string",
-              "enum": [
-                "",
-                "none",
-                "login",
-                "consent",
-                "select_account"
-              ]
+              "enum": ["", "none", "login", "consent", "select_account"]
+            },
+            "accessType": {
+              "type": "string",
+              "enum": ["", "online", "offline"]
             },
             "userIdClaim": {
               "type": "string"
             },
             "groupsClaim": {
               "type": "string"
+            },
+            "clientSecret": {
+              "type": "object",
+              "properties": {
+                "secretName": {
+                  "type": "string"
+                },
+                "secretKey": {
+                  "type": "string"
+                }
+              },
+              "additionalProperties": false
             }
           },
           "additionalProperties": false
@@ -363,11 +357,7 @@
           "properties": {
             "type": {
               "type": "string",
-              "enum": [
-                "",
-                "s3",
-                "gcs"
-              ]
+              "enum": ["", "s3", "gcs"]
             },
             "s3": {
               "type": "object",
@@ -442,11 +432,7 @@
           },
           "type": {
             "type": "string",
-            "enum": [
-              "pvc",
-              "s3",
-              "gcs"
-            ]
+            "enum": ["pvc", "s3", "gcs"]
           },
           "pvc": {
             "type": "object",
@@ -455,9 +441,7 @@
                 "type": "string"
               }
             },
-            "required": [
-              "existingClaim"
-            ]
+            "required": ["existingClaim"]
           },
           "s3": {
             "type": "object",
@@ -497,9 +481,7 @@
                 "type": "boolean"
               }
             },
-            "required": [
-              "bucket"
-            ]
+            "required": ["bucket"]
           },
           "gcs": {
             "type": "object",
@@ -517,15 +499,10 @@
                 "type": "string"
               }
             },
-            "required": [
-              "bucket"
-            ]
+            "required": ["bucket"]
           }
         },
-        "required": [
-          "name",
-          "type"
-        ]
+        "required": ["name", "type"]
       }
     },
     "kueue": {
@@ -533,25 +510,25 @@
       "properties": {
         "mode": {
           "type": "string",
-          "enum": [
-            "dedicated",
-            "shared"
-          ]
+          "enum": ["dedicated", "shared"]
         },
         "maxJobResources": {
           "type": "object",
           "properties": {
             "cpu": {
-              "type": [
-                "integer",
-                "string"
-              ]
+              "type": ["integer", "string"]
             },
             "memory": {
               "type": "string"
             },
             "gpuMemory": {
               "type": "string"
+            },
+            "gpuCpu": {
+              "type": ["integer", "string"]
+            },
+            "gpuRam": {
+              "type": "string"
             }
           },
           "additionalProperties": false
@@ -612,10 +589,7 @@
                   "type": "object",
                   "properties": {
                     "cpu": {
-                      "type": [
-                        "integer",
-                        "string"
-                      ]
+                      "type": ["integer", "string"]
                     },
                     "memory": {
                       "type": "string"
@@ -626,10 +600,7 @@
                   "type": "object",
                   "properties": {
                     "cpu": {
-                      "type": [
-                        "integer",
-                        "string"
-                      ]
+                      "type": ["integer", "string"]
                     },
                     "memory": {
                       "type": "string"
@@ -640,16 +611,10 @@
                   "type": "object",
                   "properties": {
                     "gpu": {
-                      "type": [
-                        "integer",
-                        "string"
-                      ]
+                      "type": ["integer", "string"]
                     },
                     "cpu": {
-                      "type": [
-                        "integer",
-                        "string"
-                      ]
+                      "type": ["integer", "string"]
                     },
                     "memory": {
                       "type": "string"
@@ -834,11 +799,7 @@
                     "type": "string"
                   }
                 },
-                "required": [
-                  "name",
-                  "secretName",
-                  "secretKey"
-                ]
+                "required": ["name", "secretName", "secretKey"]
               }
             },
             "configMapVariables": {
@@ -856,11 +817,7 @@
                     "type": "string"
                   }
                 },
-                "required": [
-                  "name",
-                  "configMapName",
-                  "configMapKey"
-                ]
+                "required": ["name", "configMapName", "configMapKey"]
               }
             }
           },
diff --git a/charts/platforma/values.yaml b/charts/platforma/values.yaml
index 54f9084..0606ed6 100644
--- a/charts/platforma/values.yaml
+++ b/charts/platforma/values.yaml
@@ -24,7 +24,7 @@ fullnameOverride: ""
 # -- Container image
 image:
   repository: quay.io/milaboratories/platforma
-  tag: ""  # Defaults to Chart.appVersion
+  tag: "" # Defaults to Chart.appVersion
   pullPolicy: IfNotPresent
 
 imagePullSecrets: []
@@ -158,7 +158,7 @@ auth:
       # -- Skip server certificate verification (not recommended for production)
       insecureSkipVerify: false
       # -- CA certificate: safe way to trust self-signed certificate of LDAP server.
-      # If your LDAP server uses self-signed certificate, or all traffic to LDAP server is 
+      # If your LDAP server uses self-signed certificate, or all traffic to LDAP server is
       # intercepted by some network proxy (like ZScaler), provide this certificate's CA here
       # instead of using `insecureSkipVerify: true`
       caSecretRef:
@@ -170,9 +170,9 @@ auth:
         certKey: "tls.crt"
         keyKey: "tls.key"
 
-  # -- SSO / OIDC authentication (public/native OAuth client, no client secret)
-  # Set issuer to enable. Backend defaults apply to any field left empty
-  # (see cmd/platforma/flags/flags_sso.go); advanced flags via app.extraArgs.
+  # -- SSO / OIDC authentication.
+  # Set issuer to enable. Backend defaults apply to any field left empty;
+  # advanced flags are available via app.extraArgs.
   sso:
     # -- OIDC issuer URL (set to enable SSO). Must be https.
     issuer: ""
@@ -184,10 +184,20 @@ auth:
     resource: ""
     # -- OIDC 'prompt' parameter. One of: none, login, consent, select_account. Empty = backend default.
     prompt: ""
+    # -- OIDC 'access_type' parameter. One of: online, offline. Google requires 'offline'
+    # to issue a refresh token; other IdPs ignore it. Empty = backend default.
+    accessType: ""
     # -- JWT claim carrying the stable user id. Empty = backend default ('sub').
     userIdClaim: ""
     # -- JWT claim carrying group memberships. Empty = backend default ('groups').
     groupsClaim: ""
+    # -- OAuth client secret, referenced from an existing secret. Some IdPs (e.g.
+    # Google) require one. The installers create this secret and set secretName.
+    clientSecret:
+      # -- Name of the existing secret holding the client secret. Empty = not set.
+      secretName: ""
+      # -- Key within the secret. Defaults to 'client-secret'.
+      secretKey: "client-secret"
 
 # =============================================================================
 # STORAGE
@@ -265,10 +275,10 @@ storage:
       bucket: ""
       region: ""
       prefix: ""
-      endpoint: ""            # Custom S3-compatible endpoint (leave empty for AWS S3)
-      externalEndpoint: ""    # S3 URL for Desktop App if it differs from backend's (S3-compatible only)
-      noDataIntegrity: false  # Disable SHA256 integrity checks (Ceph, old MinIO)
-      secretRef:              # Only needed for non-IRSA setups or S3-compatible storage
+      endpoint: "" # Custom S3-compatible endpoint (leave empty for AWS S3)
+      externalEndpoint: "" # S3 URL for Desktop App if it differs from backend's (S3-compatible only)
+      noDataIntegrity: false # Disable SHA256 integrity checks (Ceph, old MinIO)
+      secretRef: # Only needed for non-IRSA setups or S3-compatible storage
         name: ""
         accessKeyField: "access-key"
         secretKeyField: "secret-key"
@@ -277,7 +287,7 @@ storage:
       bucket: ""
       projectId: ""
       prefix: ""
-      serviceAccount: ""  # GCP service account for Workload Identity
+      serviceAccount: "" # GCP service account for Workload Identity
 
 # -- External read-only data sources (data library)
 # Makes external datasets available to workflows. Each source appears
@@ -320,16 +330,21 @@ dataSources: []
 
 kueue:
   # -- Mode: dedicated (chart creates all resources) or shared (uses existing queues)
-  mode: dedicated  # dedicated | shared
+  mode: dedicated # dedicated | shared
 
   # -- Maximum resources a single job can request
   # Jobs requesting more than this are rejected. Increase if you have large-memory nodes.
-  # gpuMemory is required when kueue.pools.gpu.enabled=true (chart fails to render otherwise);
-  # it is ignored when GPU is disabled. Set it to the largest VRAM on a single GPU node.
+  # gpuMemory/gpuCpu/gpuRam are required when kueue.pools.gpu.enabled=true (chart fails to
+  # render otherwise); they are ignored when GPU is disabled.
+  # GPU jobs run on a separate node pool, so a job needing a GPU is ceiled to gpuCpu/gpuRam/
+  # gpuMemory instead of cpu/memory. Set each to the largest single GPU node's *allocatable*
+  # (capacity minus kubelet + DaemonSet overhead) so a clamped request actually schedules.
   maxJobResources:
     cpu: 64
     memory: 256Gi
-    gpuMemory: 192Gi
+    gpuMemory: 48Gi
+    gpuCpu: 48
+    gpuRam: 64Gi
 
   # -- Node pool topology — controls where jobs run
   # Set nodeSelector/tolerations to match your node groups.
@@ -413,8 +428,8 @@ kueue:
   shared:
     # -- ClusterQueue names (chart creates LocalQueues pointing at these)
     clusterQueues:
-      ui: ""       # ClusterQueue for UI tasks
-      batch: ""    # ClusterQueue for batch tasks
+      ui: "" # ClusterQueue for UI tasks
+      batch: "" # ClusterQueue for batch tasks
     # -- WorkloadPriorityClass names (must already exist in cluster)
     workloadPriorityClasses:
       uiTasks: ""
@@ -436,7 +451,7 @@ ingress:
   host: ""
   tls:
     enabled: false
-    secretName: ""  # Leave empty when the ingress controller manages TLS (e.g., ALB with ACM)
+    secretName: "" # Leave empty when the ingress controller manages TLS (e.g., ALB with ACM)
   annotations: {}
 
 # -- Additional ingress (e.g., internal network, second region)
@@ -527,11 +542,11 @@ app:
     persistence:
       enabled: true
       size: 20Gi
-      storageClass: ""      # "" = cluster default
-      existingClaim: ""     # Use pre-created PVC instead of creating one
+      storageClass: "" # "" = cluster default
+      existingClaim: "" # Use pre-created PVC instead of creating one
       rotation:
-        size: 1Gi           # Rotate when a log file reaches this size
-        count: 100          # Number of rotated files to keep
+        size: 1Gi # Rotate when a log file reaches this size
+        count: 100 # Number of rotated files to keep
 
   # -- Monitoring
   # Platforma exposes Prometheus metrics. To enable scraping:
@@ -546,7 +561,6 @@ app:
       enabled: false
       interval: 30s
 
-
 # =============================================================================
 # SERVICE ACCOUNTS
 # =============================================================================
diff --git a/infrastructure/gcp/UPGRADE-monolith-to-4.1.5.md b/infrastructure/gcp/UPGRADE-monolith-to-4.2.0.md
similarity index 84%
rename from infrastructure/gcp/UPGRADE-monolith-to-4.1.5.md
rename to infrastructure/gcp/UPGRADE-monolith-to-4.2.0.md
index 70dbeca..5746050 100644
--- a/infrastructure/gcp/UPGRADE-monolith-to-4.1.5.md
+++ b/infrastructure/gcp/UPGRADE-monolith-to-4.2.0.md
@@ -1,10 +1,10 @@
-# Upgrading an existing GCP (monolithic) deployment to Platforma 4.1.5
+# Upgrading an existing GCP (monolithic) deployment to Platforma 4.2.0
 
 This guide is for deployments that were installed with the **single/monolithic**
 GCP Terraform module (`infrastructure/gcp/terraform/`) via Google Cloud
 Infrastructure Manager (`cloudshell/install.sh`).
 
-It brings your running instance up to **chart 4.1.5** and fixes a provider
+It brings your running instance up to **chart 4.2.0** and fixes a provider
 issue that can break re-deploys. The upgrade is **in-place and
 non-destructive**: no node pools are deleted, no data is lost.
 
@@ -18,10 +18,14 @@ non-destructive**: no node pools are deleted, no data is lost.
 
 | Change | Why | Impact on you |
 |---|---|---|
-| Platforma chart **3.5.0 → 4.1.5** | Latest fixes and features | Platforma pod restarts once (rolling) |
-| A **master secret** is now created and injected | Chart 4.x requires it (signs sessions & resources) | **All users must log in again once.** No data loss. |
+| Platforma chart **→ 4.2.0** | Latest fixes and features | Platforma pod restarts once (rolling) |
+| A **master secret** is created and injected | Chart 4.x requires it (signs sessions & resources) | **All users must log in again once** — only if upgrading from 3.5.0. No data loss. |
 | `kubectl` Terraform provider pinned to **< 2.4** | Provider 2.4 breaks this module's plan on re-deploy | Removes a failure you may have already hit |
 
+> Coming from an earlier 4.1.x monolithic build (the master secret and provider
+> pin already applied)? This is a straight chart bump to 4.2.0 — the master
+> secret is unchanged, so **no re-login is required**.
+
 **What is NOT affected:**
 - Your projects, blocks, and all analysis data — the master secret does **not**
   encrypt the main database.
@@ -51,10 +55,10 @@ provide it.
 ```bash
 git clone https://github.com/milaboratory/platforma-helm.git
 cd platforma-helm
-git checkout chore/gcp-monolith-backport
+git checkout chore/gcp-monolith-4.2.0
 ```
 
-(Or, in an existing clone: `git fetch origin && git checkout chore/gcp-monolith-backport`.)
+(Or, in an existing clone: `git fetch origin && git checkout chore/gcp-monolith-4.2.0`.)
 
 ### 2. Re-run the installer with your original settings
 
@@ -70,7 +74,7 @@ Infrastructure Manager detects the existing deployment and applies an **update**
 
 1. Re-resolve providers (now pinned to a working `kubectl` version).
 2. Create the master-secret Secret (in the cluster and in Secret Manager).
-3. Upgrade the Helm release to chart 4.1.5 (rolling restart of the Platforma pod).
+3. Upgrade the Helm release to chart 4.2.0 (rolling restart of the Platforma pod).
 
 Expect the Helm step to take several minutes while the new image is pulled and
 the pod becomes ready.

From e512a961386ab047a49f97e0d34a5175cef162d3 Mon Sep 17 00:00:00 2001
From: Vladimir Antropov <vladimir.antropov@milaboratories.com>
Date: Thu, 16 Jul 2026 20:04:55 +0200
Subject: [PATCH 2/3] chore(gcp): bump monolithic GCP deployment to Platforma
 chart 4.2.2
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Re-vendor the v4.2.2 chart (from the release tag) over 4.2.0. GPU stays
disabled (pools.gpu.enabled=false) — the monolith has no GPU node pools, so
the chart's GPU plumbing remains dormant (--runner-gpu-available=disabled).
Infrastructure/terraform unchanged. Verified: helm lint passes and the chart
renders against the monolith's values (gcs main, Filestore workspace,
dedicated Kueue with the batch ComputeClass nodeSelector, GPU off) — batch
keeps cloud.google.com/compute-class=platforma-batch and the 484Gi cap.

Runbook renamed to UPGRADE-monolith-to-4.2.2.md.
---
 charts/platforma/Chart.yaml                        |  4 ++--
 charts/platforma/templates/deployment.yaml         |  4 ++++
 charts/platforma/values.schema.json                |  3 +++
 charts/platforma/values.yaml                       |  6 ++++++
 ...th-to-4.2.0.md => UPGRADE-monolith-to-4.2.2.md} | 14 +++++++-------
 5 files changed, 22 insertions(+), 9 deletions(-)
 rename infrastructure/gcp/{UPGRADE-monolith-to-4.2.0.md => UPGRADE-monolith-to-4.2.2.md} (92%)

diff --git a/charts/platforma/Chart.yaml b/charts/platforma/Chart.yaml
index d95793d..c78d1eb 100644
--- a/charts/platforma/Chart.yaml
+++ b/charts/platforma/Chart.yaml
@@ -2,8 +2,8 @@ apiVersion: v2
 name: platforma
 description: Platforma - Bioinformatics platform for Kubernetes
 type: application
-version: 4.2.0
-appVersion: "4.2.0"
+version: 4.2.2
+appVersion: "4.2.2"
 kubeVersion: ">=1.31.0-0"
 
 keywords:
diff --git a/charts/platforma/templates/deployment.yaml b/charts/platforma/templates/deployment.yaml
index 87c0a47..f484e06 100644
--- a/charts/platforma/templates/deployment.yaml
+++ b/charts/platforma/templates/deployment.yaml
@@ -221,6 +221,10 @@ spec:
             {{- else }}
             - "--runner-gpu-available=disabled"
             {{- end }}
+            # OOM retries are off by default; set kueue.oomRetry=true to enable them.
+            {{- if .Values.kueue.oomRetry }}
+            - "--k8s-oom-retry"
+            {{- end }}
             # Queue runners (hardcoded — 4 fixed queues, always K8s backend)
             - "--queue-runner=ui-tasks:k8s"
             - "--queue-runner=light:k8s"
diff --git a/charts/platforma/values.schema.json b/charts/platforma/values.schema.json
index ebd6707..8730d2a 100644
--- a/charts/platforma/values.schema.json
+++ b/charts/platforma/values.schema.json
@@ -512,6 +512,9 @@
           "type": "string",
           "enum": ["dedicated", "shared"]
         },
+        "oomRetry": {
+          "type": "boolean"
+        },
         "maxJobResources": {
           "type": "object",
           "properties": {
diff --git a/charts/platforma/values.yaml b/charts/platforma/values.yaml
index 0606ed6..05d1b93 100644
--- a/charts/platforma/values.yaml
+++ b/charts/platforma/values.yaml
@@ -346,6 +346,12 @@ kueue:
     gpuCpu: 48
     gpuRam: 64Gi
 
+  # -- Enable OOM retries for K8s jobs.
+  # Off by default: an out-of-memory job fails immediately with its original error.
+  # When enabled, an OOM-killed job is re-run with doubled memory, capped at
+  # maxJobResources.memory / maxJobResources.gpuMemory.
+  oomRetry: false
+
   # -- Node pool topology — controls where jobs run
   # Set nodeSelector/tolerations to match your node groups.
   # In dedicated mode, these also configure Kueue ResourceFlavors.
diff --git a/infrastructure/gcp/UPGRADE-monolith-to-4.2.0.md b/infrastructure/gcp/UPGRADE-monolith-to-4.2.2.md
similarity index 92%
rename from infrastructure/gcp/UPGRADE-monolith-to-4.2.0.md
rename to infrastructure/gcp/UPGRADE-monolith-to-4.2.2.md
index 5746050..e05839a 100644
--- a/infrastructure/gcp/UPGRADE-monolith-to-4.2.0.md
+++ b/infrastructure/gcp/UPGRADE-monolith-to-4.2.2.md
@@ -1,10 +1,10 @@
-# Upgrading an existing GCP (monolithic) deployment to Platforma 4.2.0
+# Upgrading an existing GCP (monolithic) deployment to Platforma 4.2.2
 
 This guide is for deployments that were installed with the **single/monolithic**
 GCP Terraform module (`infrastructure/gcp/terraform/`) via Google Cloud
 Infrastructure Manager (`cloudshell/install.sh`).
 
-It brings your running instance up to **chart 4.2.0** and fixes a provider
+It brings your running instance up to **chart 4.2.2** and fixes a provider
 issue that can break re-deploys. The upgrade is **in-place and
 non-destructive**: no node pools are deleted, no data is lost.
 
@@ -18,12 +18,12 @@ non-destructive**: no node pools are deleted, no data is lost.
 
 | Change | Why | Impact on you |
 |---|---|---|
-| Platforma chart **→ 4.2.0** | Latest fixes and features | Platforma pod restarts once (rolling) |
+| Platforma chart **→ 4.2.2** | Latest fixes and features | Platforma pod restarts once (rolling) |
 | A **master secret** is created and injected | Chart 4.x requires it (signs sessions & resources) | **All users must log in again once** — only if upgrading from 3.5.0. No data loss. |
 | `kubectl` Terraform provider pinned to **< 2.4** | Provider 2.4 breaks this module's plan on re-deploy | Removes a failure you may have already hit |
 
 > Coming from an earlier 4.1.x monolithic build (the master secret and provider
-> pin already applied)? This is a straight chart bump to 4.2.0 — the master
+> pin already applied)? This is a straight chart bump to 4.2.2 — the master
 > secret is unchanged, so **no re-login is required**.
 
 **What is NOT affected:**
@@ -55,10 +55,10 @@ provide it.
 ```bash
 git clone https://github.com/milaboratory/platforma-helm.git
 cd platforma-helm
-git checkout chore/gcp-monolith-4.2.0
+git checkout chore/gcp-monolith-4.2.2
 ```
 
-(Or, in an existing clone: `git fetch origin && git checkout chore/gcp-monolith-4.2.0`.)
+(Or, in an existing clone: `git fetch origin && git checkout chore/gcp-monolith-4.2.2`.)
 
 ### 2. Re-run the installer with your original settings
 
@@ -74,7 +74,7 @@ Infrastructure Manager detects the existing deployment and applies an **update**
 
 1. Re-resolve providers (now pinned to a working `kubectl` version).
 2. Create the master-secret Secret (in the cluster and in Secret Manager).
-3. Upgrade the Helm release to chart 4.2.0 (rolling restart of the Platforma pod).
+3. Upgrade the Helm release to chart 4.2.2 (rolling restart of the Platforma pod).
 
 Expect the Helm step to take several minutes while the new image is pulled and
 the pod becomes ready.

From 44af59b5123b14736f602df3e20e91733070c963 Mon Sep 17 00:00:00 2001
From: Vladimir Antropov <vladimir.antropov@milaboratories.com>
Date: Thu, 16 Jul 2026 21:11:47 +0200
Subject: [PATCH 3/3] docs(gcp): note 4.2.2 image pairing + OOM-retry-off in
 monolith runbook

Chart 4.2.2 pairs with the 4.2.2 backend image (chart default handles it
unless platforma_image_override is pinned). Also document that OOM job
retries are off by default in 4.2.x, so out-of-memory jobs fail fast with
their original error instead of clogging the batch queue.
---
 infrastructure/gcp/UPGRADE-monolith-to-4.2.2.md | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/infrastructure/gcp/UPGRADE-monolith-to-4.2.2.md b/infrastructure/gcp/UPGRADE-monolith-to-4.2.2.md
index e05839a..8c15a7a 100644
--- a/infrastructure/gcp/UPGRADE-monolith-to-4.2.2.md
+++ b/infrastructure/gcp/UPGRADE-monolith-to-4.2.2.md
@@ -21,11 +21,18 @@ non-destructive**: no node pools are deleted, no data is lost.
 | Platforma chart **→ 4.2.2** | Latest fixes and features | Platforma pod restarts once (rolling) |
 | A **master secret** is created and injected | Chart 4.x requires it (signs sessions & resources) | **All users must log in again once** — only if upgrading from 3.5.0. No data loss. |
 | `kubectl` Terraform provider pinned to **< 2.4** | Provider 2.4 breaks this module's plan on re-deploy | Removes a failure you may have already hit |
+| **OOM job retries off by default** | 4.2.x no longer retries out-of-memory jobs by default | A job that runs out of memory now fails immediately with its original error, instead of being re-run at doubled memory. This prevents oversized retries from piling up and stalling the batch queue. |
 
 > Coming from an earlier 4.1.x monolithic build (the master secret and provider
 > pin already applied)? This is a straight chart bump to 4.2.2 — the master
 > secret is unchanged, so **no re-login is required**.
 
+> **Backend image:** chart 4.2.2 must run against the **4.2.2 Platforma image**.
+> The chart sets this for you — the image tag defaults to the chart's appVersion
+> (4.2.2), so no action is needed **unless** you pinned `platforma_image_override`
+> in your inputs. If you did, point it at a 4.2.2 image (or clear it to use the
+> default). Verify after upgrade with the `kubectl get deploy` command in step 3.
+
 **What is NOT affected:**
 - Your projects, blocks, and all analysis data — the master secret does **not**
   encrypt the main database.