feat(maintenance): maintenance page workflow, two replicas for landing page and service, test + staging on chart 2.4.0

.github/workflows/deploy.yml

@@ -185,6 +185,22 @@ jobs:
             exit 1
           fi
 
+      - name: Warn when this deploy ends maintenance
+        if: ${{ !inputs.dry_run }}
+        run: |
+          set -euo pipefail
+          # maintenance.yml shows the maintenance page by scaling the landing
+          # page to 0. This upgrade scales it back, so say so rather than let
+          # maintenance end without anyone noticing.
+          n=$(kubectl -n "${{ steps.env.outputs.namespace }}" get deploy landing-page-deployment \
+                -o jsonpath='{.spec.replicas}' 2>/dev/null || true)
+          if [[ "$n" == "0" ]]; then
+            msg="The landing page is at 0 replicas (maintenance mode). This deploy brings it back and ends maintenance."
+            echo "::warning::$msg"
+            echo "> [!WARNING]" >> "$GITHUB_STEP_SUMMARY"
+            echo "> $msg" >> "$GITHUB_STEP_SUMMARY"
+          fi
+
       - name: Resolve image overrides
         id: images
         run: |

.github/workflows/maintenance.yml

@@ -0,0 +1,132 @@
+# Show the maintenance page on an environment, or take it down again.
+#
+# The eduide chart (>= 2.4.0) already answers every 502/503/504 on the landing
+# route with a static "EduIDE is currently unavailable" page, served by Envoy.
+# So maintenance mode is nothing more than the landing page having no pods:
+#   on   scales landing-page-deployment to 0
+#   off  scales it back to landingPage.replicas (scripts/landing-replicas.sh)
+#
+# Worth knowing:
+#   - The next deploy ends maintenance, because Helm sets the replicas again.
+#   - Only the landing page goes away. Running sessions and the REST service
+#     are untouched, so students already in an IDE keep working.
+
+name: Maintenance page
+
+on:
+  workflow_dispatch:
+    inputs:
+      environment:
+        required: true
+        type: choice
+        options: [test1.eduide.student.k8s.aet.cit.tum.de, test2.eduide.student.k8s.aet.cit.tum.de, test3.eduide.student.k8s.aet.cit.tum.de, e2e.eduide.student.k8s.aet.cit.tum.de, staging.eduide.student.k8s.aet.cit.tum.de, eduide.artemis.cit.tum.de, bonn.eduide.aet.cit.tum.de, mannheim.eduide.aet.cit.tum.de]
+      mode:
+        required: true
+        type: choice
+        options: ["on", "off"]
+
+permissions:
+  contents: read
+
+jobs:
+  resolve:
+    runs-on: ubuntu-latest
+    outputs:
+      runner: ${{ steps.r.outputs.runner }}
+      namespace: ${{ steps.r.outputs.namespace }}
+      cluster: ${{ steps.r.outputs.cluster }}
+      landing: ${{ steps.r.outputs.landing }}
+      replicas: ${{ steps.r.outputs.replicas }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install yq
+        run: |
+          sudo wget -qO /usr/local/bin/yq https://github.com/mikefarah/yq/releases/download/v4.44.3/yq_linux_amd64
+          sudo chmod +x /usr/local/bin/yq
+      - id: r
+        run: |
+          set -euo pipefail
+          M="environments/${{ inputs.environment }}/env.yaml"
+          [[ -f "$M" ]] || { echo "::error::no such environment: ${{ inputs.environment }}"; exit 1; }
+          C=$(yq -r '.spec.cluster' "$M")
+          V="environments/${{ inputs.environment }}/values.yaml"
+          {
+            echo "cluster=$C"
+            echo "namespace=$(yq -r '.spec.namespace' "$M")"
+            echo "runner=$(yq -r '.spec.runner // "ubuntu-latest"' "clusters/$C.yaml")"
+            echo "landing=$(yq -r '.hosts.configuration | .landing + "." + .baseHost' "$V")"
+            echo "replicas=$(./scripts/landing-replicas.sh "${{ inputs.environment }}")"
+          } >> "$GITHUB_OUTPUT"
+
+  maintenance:
+    needs: resolve
+    runs-on: ${{ needs.resolve.outputs.runner }}
+    environment: ${{ inputs.environment }}
+    concurrency:
+      # Shares a group with Deploy, so it cannot race an upgrade.
+      group: deploy-${{ inputs.environment }}
+      cancel-in-progress: false
+    env:
+      KUBECONFIG: ${{ github.workspace }}/kubeconfig
+      NS: ${{ needs.resolve.outputs.namespace }}
+      URL: https://${{ needs.resolve.outputs.landing }}/
+    steps:
+      - uses: actions/checkout@v4
+      - uses: azure/setup-kubectl@v4
+      - name: Set up kubeconfig
+        run: |
+          set -euo pipefail
+          install -m 600 /dev/null "$KUBECONFIG"
+          printf '%s' '${{ secrets.KUBECONFIG }}' > "$KUBECONFIG"
+
+      - name: Assert we are pointed at the right cluster
+        run: |
+          set -euo pipefail
+          FOUND=$(kubectl -n eduide-system get configmap eduide-cluster-identity \
+                    -o jsonpath='{.data.clusterName}' 2>/dev/null || true)
+          if [[ "$FOUND" != "${{ needs.resolve.outputs.cluster }}" ]]; then
+            echo "::error::expected cluster '${{ needs.resolve.outputs.cluster }}', this KUBECONFIG reaches '${FOUND:-an unbootstrapped cluster}'"
+            exit 1
+          fi
+
+      - name: The chart serves the maintenance page
+        if: ${{ inputs.mode == 'on' }}
+        run: |
+          set -euo pipefail
+          # Without the policy, scaling to 0 shows Envoy's raw error instead.
+          if ! kubectl -n "$NS" get backendtrafficpolicy landing-maintenance-page >/dev/null 2>&1; then
+            echo "::error::no landing-maintenance-page policy in $NS. Deploy chart >= 2.4.0 with maintenancePage.enabled first."
+            exit 1
+          fi
+
+      - name: Turn maintenance on
+        if: ${{ inputs.mode == 'on' }}
+        run: |
+          set -euo pipefail
+          kubectl -n "$NS" scale deploy/landing-page-deployment --replicas=0
+          kubectl -n "$NS" wait --for=delete pod -l app=landing-page --timeout=2m
+
+      - name: Turn maintenance off
+        if: ${{ inputs.mode == 'off' }}
+        run: |
+          set -euo pipefail
+          kubectl -n "$NS" scale deploy/landing-page-deployment --replicas=${{ needs.resolve.outputs.replicas }}
+          kubectl -n "$NS" rollout status deploy/landing-page-deployment --timeout=5m
+
+      - name: Check what visitors see
+        run: |
+          set -euo pipefail
+          # Envoy needs a moment to notice the endpoints changed.
+          want=$([[ "${{ inputs.mode }}" == "on" ]] && echo 503 || echo 200)
+          for _ in $(seq 1 12); do
+            code=$(curl -s -o page.html -w '%{http_code}' --max-time 10 "$URL" || echo 000)
+            if [[ "$code" == "$want" ]]; then break; fi
+            sleep 5
+          done
+          if [[ "$code" != "$want" ]]; then
+            echo "::error::$URL answered $code, expected $want"; exit 1
+          fi
+          if [[ "$want" == 503 ]] && ! grep -q "EduIDE is currently unavailable" page.html; then
+            echo "::error::$URL answered 503 but not with the maintenance page"; exit 1
+          fi
+          echo "### Maintenance ${{ inputs.mode }}: $URL answers $code" >> "$GITHUB_STEP_SUMMARY"

README.md

@@ -69,6 +69,7 @@ file. `Deploy` does the second.
 | Move staging | Actions → **Deploy staging** |
 | Move production | Bump `chartVersion` in `environments/tum-production/env.yaml`, open a PR |
 | Undo a bad deploy | Actions → **Rollback** |
+| Show the maintenance page ("EduIDE is currently unavailable") | Actions → **Maintenance page**, `mode: on`; `mode: off` or any deploy ends it |
 | Bring up a new cluster | Actions → **Bootstrap cluster** |
 
 `e2e.eduide.student.k8s.aet.cit.tum.de` deploys from `main` automatically and
@@ -79,6 +80,10 @@ Every deploy asserts which cluster it reached before touching anything, shows a
 `helm diff` before applying, runs `--wait --atomic`, and prints a summary read
 back from the cluster rather than echoed from its inputs.
 
+The maintenance page needs chart >= 2.4.0. It is what the landing host serves
+whenever the landing page has no ready pod, so `Maintenance page` only scales
+the landing page to 0. The REST service and running sessions are untouched.
+
 ## What an environment does and does not configure
 
 An environment file carries **hosts, gateway routing, Keycloak and branding**.

docs/github-environments.md

@@ -8,7 +8,7 @@ There are **two kinds**, and they are not interchangeable.
 
 | Kind | Named | Used by | Holds |
 |---|---|---|---|
-| Per environment | the installation's landing hostname, which is also its directory under `environments/` | `Deploy`, `Rollback` | `KUBECONFIG`, `THEIA_KEYCLOAK_COOKIE_SECRET` |
+| Per environment | the installation's landing hostname, which is also its directory under `environments/` | `Deploy`, `Rollback`, `Maintenance page` | `KUBECONFIG`, `THEIA_KEYCLOAK_COOKIE_SECRET` |
 | Per cluster | `spec.bootstrapEnvironment` in `clusters/<name>.yaml`, by convention `cluster-<name>` | `Bootstrap cluster` | `KUBECONFIG`, `THEIA_WILDCARD_CERTIFICATE_CERT`, `THEIA_WILDCARD_CERTIFICATE_KEY` |
 
 A deploy never installs anything cluster-scoped, and a bootstrap never installs

environments/_base.yaml

@@ -42,6 +42,9 @@ operator:
   eagerStart: true
 
 landingPage:
+  # Two pods so a node going away or a deploy never leaves the landing page
+  # without a ready one. Needs chart >= 2.4.0; older charts ignore it.
+  replicas: 2
   ephemeralStorage: true
   # Every installation we run is on TUM infrastructure, so TUM is the
   # controller for all of them and this belongs here rather than per
@@ -77,6 +80,9 @@ landingPage:
       text: About
       url: https://aet.cit.tum.de/
 service:
+  # Same reason as landingPage.replicas. The service keeps its state in the
+  # cluster, so two pods are safe.
+  replicas: 2
   # Public spam-mitigation token shared with the landing page, not a secret.
   # Same value the installations have always used; changing it gains nothing.
   authToken: nJV3nKZmpxTD4wu2

environments/bonn.eduide.aet.cit.tum.de/values.yaml

@@ -135,3 +135,9 @@ theia-workspace-garbage-collector:
 # decision rather than an accident.
 monitoring:
   enabled: true
+
+# parma is a single node. A PodDisruptionBudget there makes `kubectl drain`
+# wait forever: the evicted replica cannot be rescheduled on the cordoned node,
+# so the budget never allows evicting the other one.
+podDisruptionBudget:
+  enabled: false

environments/e2e.eduide.student.k8s.aet.cit.tum.de/env.yaml

@@ -12,5 +12,5 @@ spec:
   cluster: tum-student
   namespace: eduide-e2e-test
   platform:
-    chartVersion: 2.2.1
+    chartVersion: 2.4.0
     channel: main

environments/mannheim.eduide.aet.cit.tum.de/values.yaml

@@ -139,3 +139,9 @@ theia-workspace-garbage-collector:
 # decision rather than an accident.
 monitoring:
   enabled: true
+
+# parma is a single node. A PodDisruptionBudget there makes `kubectl drain`
+# wait forever: the evicted replica cannot be rescheduled on the cordoned node,
+# so the budget never allows evicting the other one.
+podDisruptionBudget:
+  enabled: false

environments/staging.eduide.student.k8s.aet.cit.tum.de/env.yaml

@@ -12,5 +12,5 @@ spec:
   cluster: tum-student
   namespace: eduide-staging
   platform:
-    chartVersion: 2.3.0
+    chartVersion: 2.4.0
     channel: main

environments/test1.eduide.student.k8s.aet.cit.tum.de/env.yaml

@@ -12,5 +12,5 @@ spec:
   cluster: tum-student
   namespace: eduide-test1
   platform:
-    chartVersion: 2.2.1
+    chartVersion: 2.4.0
     channel: main

environments/test2.eduide.student.k8s.aet.cit.tum.de/env.yaml

@@ -12,5 +12,5 @@ spec:
   cluster: tum-student
   namespace: eduide-test2
   platform:
-    chartVersion: 2.2.1
+    chartVersion: 2.4.0
     channel: main

environments/test3.eduide.student.k8s.aet.cit.tum.de/env.yaml

@@ -12,5 +12,5 @@ spec:
   cluster: tum-student
   namespace: eduide-test3
   platform:
-    chartVersion: 2.2.1
+    chartVersion: 2.4.0
     channel: main

scripts/landing-replicas.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+# How many landing page pods an environment runs when it is not in maintenance.
+#
+#   ./scripts/landing-replicas.sh <environment>
+#
+# The maintenance workflow scales the landing page to 0 and needs to know what
+# to scale it back to. Same precedence Helm applies: the environment's
+# values.yaml over _base.yaml over the chart default of 1.
+set -euo pipefail
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+ENV="${1:?usage: landing-replicas.sh <environment>}"
+V="$ROOT/environments/$ENV/values.yaml"
+[[ -f "$V" ]] || { echo "no such environment: $ENV" >&2; exit 1; }
+yq -r '.landingPage.replicas // ""' "$V" "$ROOT/environments/_base.yaml" | grep -vE '^(---)?$' | head -1 | grep . || echo 1

scripts/test-deploy-logic.sh

@@ -424,6 +424,42 @@ for v in "$ROOT"/environments/*/values.yaml; do
   fi
 done
 
+# --- maintenance scales the landing page back to what Helm would -----------
+# maintenance.yml turns maintenance off by scaling the landing page back up. It
+# must pick the same count Helm would, or the next deploy changes it again.
+echo
+echo "=== maintenance restores the landing page replicas ==="
+T=$(mktemp -d)
+mkdir -p "$T/scripts" "$T/environments/a" "$T/environments/b"
+cp "$ROOT/scripts/landing-replicas.sh" "$T/scripts/"
+replicas() { "$T/scripts/landing-replicas.sh" "$1" 2>/dev/null; }
+printf 'app:\n  id: x\n' > "$T/environments/_base.yaml"
+printf 'landingPage:\n  appDefinition: y\n' > "$T/environments/a/values.yaml"
+[[ "$(replicas a)" == "1" ]] && ok "chart default when nobody sets it" || bad "chart default" "$(replicas a)"
+printf 'landingPage:\n  replicas: 2\n' > "$T/environments/_base.yaml"
+[[ "$(replicas a)" == "2" ]] && ok "_base.yaml applies" || bad "_base.yaml" "$(replicas a)"
+printf 'landingPage:\n  replicas: 3\n' > "$T/environments/b/values.yaml"
+[[ "$(replicas b)" == "3" ]] && ok "environment overrides _base.yaml" || bad "environment override" "$(replicas b)"
+replicas nope >/dev/null && bad "unknown environment accepted" "" || ok "unknown environment rejected"
+rm -rf "$T"
+for f in "$ROOT"/environments/*/env.yaml; do
+  env=$(basename "$(dirname "$f")")
+  n=$("$ROOT/scripts/landing-replicas.sh" "$env" 2>/dev/null)
+  [[ "$n" =~ ^[1-9][0-9]*$ ]] && ok "$env: landing page returns with $n" || bad "$env: bad landing replica count" "'$n'"
+done
+
+# --- no PodDisruptionBudgets on the single-node cluster ---------------------
+# On parma (cluster `eduide`) a PDB makes every drain hang, see the comment in
+# the bonn and mannheim values files.
+echo
+echo "=== no PodDisruptionBudgets on the single-node cluster ==="
+for f in "$ROOT"/environments/*/env.yaml; do
+  [[ "$(yq -r '.spec.cluster' "$f")" == "eduide" ]] || continue
+  env=$(basename "$(dirname "$f")")
+  pdb=$(yq -r '.podDisruptionBudget.enabled' "$ROOT/environments/$env/values.yaml")
+  [[ "$pdb" == "false" ]] && ok "$env: PDBs off" || bad "$env: PDBs must be off on parma" "podDisruptionBudget.enabled is '$pdb'"
+done
+
 echo
 [[ $FAILED -eq 0 ]] && echo "ALL PASS" || echo "SOME FAILED"
 exit $FAILED