sync: upstream 5f728f7862 into swaplabs/main

.github/workflows/cloud-deploy-relay-asia-topology.yml

@@ -76,6 +76,7 @@ jobs:
             staging:staging-gce-c4) ;;
             production:production-gce-c27,production-gce-c28,production-gce-c29) ;;
             production:production-gce-c30) ;;
+            production:production-gce-c31) ;;
             *) echo "cell-ids do not match the reviewed environment topology" >&2; exit 1 ;;
           esac
           [[ "${TARGET_IMAGE}" =~ ^us-central1-docker\.pkg\.dev/${GCP_PROJECT_ID}/orca-cloud/relay@sha256:[0-9a-f]{64}$ ]]

.github/workflows/cloud-deploy-relay-production-capacity-job.yml

@@ -47,6 +47,9 @@ jobs:
     timeout-minutes: 75
     environment: production
     env:
+      # These drains are unpaced, but restart-safe still needs a quiet runtime for the production
+      # pace window, the same one the same-cap drain uses.
+      RESTART_SAFE_PACE_WINDOW_MS: '300000'
       GCP_PROJECT_ID: onorca-cloud
       GCP_REGION: ${{ vars.PRODUCTION_GCP_REGION }}
       DIRECTOR_SERVICE_NAME: orca-cloud-relay
@@ -565,6 +568,7 @@ jobs:
             --draining required \
             --activity restart-safe \
             --runtime required \
+            --pace-window-ms "${RESTART_SAFE_PACE_WINDOW_MS}" \
             --timeout-ms 450000 \
             --expected-image-digests \
               "${DESIRED_IMAGE_DIGEST},${PREDECESSOR_IMAGE_DIGEST}" \
@@ -615,6 +619,7 @@ jobs:
             --draining required \
             --activity restart-safe \
             --runtime required \
+            --pace-window-ms "${RESTART_SAFE_PACE_WINDOW_MS}" \
             --timeout-ms 450000 \
             --expected-image-digests \
               "${DESIRED_IMAGE_DIGEST},${PREDECESSOR_IMAGE_DIGEST}"
@@ -676,6 +681,8 @@ jobs:
             --draining required \
             --activity restart-safe \
             --runtime required \
+            --pace-window-ms "${RESTART_SAFE_PACE_WINDOW_MS}" \
+            --timeout-ms 480000 \
             --expected-image-digests \
               "${DESIRED_IMAGE_DIGEST},${PREDECESSOR_IMAGE_DIGEST}"
 

.github/workflows/cloud-deploy-relay-production-same-cap-job.yml

@@ -278,7 +278,7 @@ jobs:
               EXPECTED_REGION=us-central1
               EXPECTED_DATABASE_POOL_MAX=
               ;;
-            c27|c28|c29|c30)
+            c27|c28|c29|c30|c31)
               EXPECTED_HARD_CAP=3000
               EXPECTED_REGION=asia-east2
               EXPECTED_DATABASE_POOL_MAX=16
@@ -537,6 +537,17 @@ jobs:
             --draining "${PRECHECK_DRAINING}" --activity allowed \
             --expected-image-digests "${PREDECESSOR_IMAGE_DIGEST}"
 
+      # Hosts drained with nowhere to go keep redialling and pin the cell empty (c28, 2026-10-01).
+      # Every run that drains (apply, rollback) checks first; verify reads the same answer; a resume drains nothing.
+      - name: Require free general-cell slots for the selected cell's hosts
+        if: ${{ env.ROLLBACK_RESUME != 'true' }}
+        env:
+          ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.deploy-auth.outputs.id_token }}
+        run: |
+          node dev/scripts/check-relay-same-cap-headroom.mjs \
+            --director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
+            --cell-id "${TARGET_CELL_ID}" --general-cells "${EXPECTED_GENERAL_CELLS}"
+
       - name: Finish read-only verification
         if: ${{ inputs.mode == 'verify' }}
         run: echo 'Exact same-cap rollback point verified.'
@@ -572,14 +583,15 @@ jobs:
             --director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
             --cell-id "${TARGET_CELL_ID}" --approved-cells same-cap --mode drain \
             --pace-window-ms "${DRAIN_PACE_WINDOW_MS}"
-          # The wait has to outlast the pacing window as well as the leases it waits on.
+          # Restart-safe means an empty runtime for a whole pace window after the drain; leases
+          # held by unplaceable hosts are reported, not waited on (c28, 2026-10-01).
           node dev/scripts/verify-relay-capacity-transition.mjs \
             --director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
             --cell-id "${TARGET_CELL_ID}" --hard-cap "${EXPECTED_HARD_CAP}" \
             --unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" \
             --heartbeat either --admission migration-only --draining required \
             --activity restart-safe --expected-image-digests "${PREDECESSOR_IMAGE_DIGEST}" \
-            --timeout-ms 1200000
+            --pace-window-ms "${DRAIN_PACE_WINDOW_MS}" --timeout-ms 1200000
 
       - id: capacity-auth
         if: ${{ inputs.mode != 'verify' }}

.github/workflows/cloud-operate-relay-asia-admission.yml

@@ -39,7 +39,7 @@ on:
         required: false
         type: string
       evidence-run-id:
-        description: Staging evidence run ID for C27, C27 canary run ID for C28/C29; C30 takes none and proves itself by its own canary
+        description: Staging evidence run ID for C27, C27 canary run ID for C28/C29; C30/C31 take none and each proves itself by its own canary
         required: false
         type: string
       evidence-run-attempt:
@@ -155,9 +155,9 @@ jobs:
                 evidence_kind=c27
                 artifact_name="relay-asia-c27-canary-${EVIDENCE_RUN_ID}-${EVIDENCE_RUN_ATTEMPT}"
                 ;;
-              production-gce-c30)
-                # No earlier proof binds C30's generation; its own canary below rolls it back on failure.
-                canary_cell=production-gce-c30
+              production-gce-c30|production-gce-c31)
+                # No earlier proof binds a later cell's generation; its own canary below rolls it back on failure.
+                canary_cell="${TARGET_CELL_IDS}"
                 ;;
               *) echo "production promotion wave is not reviewed" >&2; exit 1 ;;
             esac
@@ -317,15 +317,15 @@ jobs:
         shell: bash
         run: |
           set -euo pipefail
-          # C30's launch cells were checked general by the promotion; verify reads only C30's digest.
+          # A later cell's launch cells were checked general by the promotion; verify reads only its digest.
           case "${CANARY_CELL}" in
             production-gce-c27)
               verify_cells=production-gce-c27,production-gce-c28,production-gce-c29
               expected_states='{"production-gce-c27":"general","production-gce-c28":"migration-only","production-gce-c29":"migration-only"}'
               ;;
-            production-gce-c30)
-              verify_cells=production-gce-c30
-              expected_states='{"production-gce-c30":"general"}'
+            production-gce-c30|production-gce-c31)
+              verify_cells="${CANARY_CELL}"
+              expected_states="{\"${CANARY_CELL}\":\"general\"}"
               ;;
             *) exit 1 ;;
           esac

.github/workflows/cloud-prove-relay-staging-capacity.yml

@@ -69,6 +69,9 @@ jobs:
       DIRECTOR_SERVICE_NAME: orca-cloud-relay-staging
       DIRECTOR_ORIGIN: https://relay-staging.onorca.dev
       CELL_ORIGIN: https://c3.relay-staging.onorca.dev
+      # These drains are unpaced, but restart-safe still needs a quiet runtime for the production
+      # pace window, the same one the same-cap drain uses.
+      RESTART_SAFE_PACE_WINDOW_MS: '300000'
       TARGET_CELL_ID: staging-gce-c3
       FALLBACK_CELL_ORIGIN: https://c2.relay-staging.onorca.dev
       FALLBACK_CELL_ID: staging-gce-c2
@@ -268,7 +271,8 @@ jobs:
                 --heartbeat either \
                 --admission migration-only \
                 --draining required \
-                --activity restart-safe
+                --activity restart-safe \
+                --pace-window-ms "${RESTART_SAFE_PACE_WINDOW_MS}" --timeout-ms 480000
               TRANSITION_PHASE=director-ready
             fi
           else
@@ -377,7 +381,8 @@ jobs:
             --heartbeat either \
             --admission migration-only \
             --draining required \
-            --activity restart-safe
+            --activity restart-safe \
+            --pace-window-ms "${RESTART_SAFE_PACE_WINDOW_MS}" --timeout-ms 480000
 
       - name: Verify current capacity
         if: ${{ inputs.mode == 'verify' }}
@@ -451,7 +456,8 @@ jobs:
             --heartbeat "${HEARTBEAT_EXPECTATION}" \
             --admission migration-only \
             --draining required \
-            --activity restart-safe
+            --activity restart-safe \
+            --pace-window-ms "${RESTART_SAFE_PACE_WINDOW_MS}" --timeout-ms 480000
 
       - name: Plan and apply only the exact empty cell
         if: ${{ inputs.mode == 'apply' }}