From 3deb642f755e5574af4eab73afc6bd0d031fe3ed Mon Sep 17 00:00:00 2001 From: Matthias Linhuber Date: Tue, 29 Sep 2026 10:10:36 +0200 Subject: [PATCH] feat(maintenance): a workflow to show the maintenance page, and two replicas for landing page and service maintenance.yml scales the landing page to 0 (on) or back to landingPage.replicas (off). Chart 2.4.0 then serves the maintenance page from Envoy. It shares the deploy concurrency group and checks what visitors get. Any deploy ends maintenance, so deploy.yml warns when it finds the landing page at 0. _base.yaml runs two landing page and REST service pods. bonn and mannheim turn PodDisruptionBudgets off: parma is a single node, where a PDB blocks every drain. Test and staging environments move to chart 2.4.0. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/deploy.yml | 16 +++ .github/workflows/maintenance.yml | 132 ++++++++++++++++++ README.md | 5 + docs/github-environments.md | 2 +- environments/_base.yaml | 6 + .../bonn.eduide.aet.cit.tum.de/values.yaml | 6 + .../env.yaml | 2 +- .../values.yaml | 6 + .../env.yaml | 2 +- .../env.yaml | 2 +- .../env.yaml | 2 +- .../env.yaml | 2 +- scripts/landing-replicas.sh | 14 ++ scripts/test-deploy-logic.sh | 36 +++++ 14 files changed, 227 insertions(+), 6 deletions(-) create mode 100644 .github/workflows/maintenance.yml create mode 100755 scripts/landing-replicas.sh diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 965787c..c65e2c7 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -185,6 +185,22 @@ jobs: exit 1 fi + - name: Warn when this deploy ends maintenance + if: ${{ !inputs.dry_run }} + run: | + set -euo pipefail + # maintenance.yml shows the maintenance page by scaling the landing + # page to 0. This upgrade scales it back, so say so rather than let + # maintenance end without anyone noticing. + n=$(kubectl -n "${{ steps.env.outputs.namespace }}" get deploy landing-page-deployment \ + -o jsonpath='{.spec.replicas}' 2>/dev/null || true) + if [[ "$n" == "0" ]]; then + msg="The landing page is at 0 replicas (maintenance mode). This deploy brings it back and ends maintenance." + echo "::warning::$msg" + echo "> [!WARNING]" >> "$GITHUB_STEP_SUMMARY" + echo "> $msg" >> "$GITHUB_STEP_SUMMARY" + fi + - name: Resolve image overrides id: images run: | diff --git a/.github/workflows/maintenance.yml b/.github/workflows/maintenance.yml new file mode 100644 index 0000000..aff08a3 --- /dev/null +++ b/.github/workflows/maintenance.yml @@ -0,0 +1,132 @@ +# Show the maintenance page on an environment, or take it down again. +# +# The eduide chart (>= 2.4.0) already answers every 502/503/504 on the landing +# route with a static "EduIDE is currently unavailable" page, served by Envoy. +# So maintenance mode is nothing more than the landing page having no pods: +# on scales landing-page-deployment to 0 +# off scales it back to landingPage.replicas (scripts/landing-replicas.sh) +# +# Worth knowing: +# - The next deploy ends maintenance, because Helm sets the replicas again. +# - Only the landing page goes away. Running sessions and the REST service +# are untouched, so students already in an IDE keep working. + +name: Maintenance page + +on: + workflow_dispatch: + inputs: + environment: + required: true + type: choice + options: [test1.eduide.student.k8s.aet.cit.tum.de, test2.eduide.student.k8s.aet.cit.tum.de, test3.eduide.student.k8s.aet.cit.tum.de, e2e.eduide.student.k8s.aet.cit.tum.de, staging.eduide.student.k8s.aet.cit.tum.de, eduide.artemis.cit.tum.de, bonn.eduide.aet.cit.tum.de, mannheim.eduide.aet.cit.tum.de] + mode: + required: true + type: choice + options: ["on", "off"] + +permissions: + contents: read + +jobs: + resolve: + runs-on: ubuntu-latest + outputs: + runner: ${{ steps.r.outputs.runner }} + namespace: ${{ steps.r.outputs.namespace }} + cluster: ${{ steps.r.outputs.cluster }} + landing: ${{ steps.r.outputs.landing }} + replicas: ${{ steps.r.outputs.replicas }} + steps: + - uses: actions/checkout@v4 + - name: Install yq + run: | + sudo wget -qO /usr/local/bin/yq https://github.com/mikefarah/yq/releases/download/v4.44.3/yq_linux_amd64 + sudo chmod +x /usr/local/bin/yq + - id: r + run: | + set -euo pipefail + M="environments/${{ inputs.environment }}/env.yaml" + [[ -f "$M" ]] || { echo "::error::no such environment: ${{ inputs.environment }}"; exit 1; } + C=$(yq -r '.spec.cluster' "$M") + V="environments/${{ inputs.environment }}/values.yaml" + { + echo "cluster=$C" + echo "namespace=$(yq -r '.spec.namespace' "$M")" + echo "runner=$(yq -r '.spec.runner // "ubuntu-latest"' "clusters/$C.yaml")" + echo "landing=$(yq -r '.hosts.configuration | .landing + "." + .baseHost' "$V")" + echo "replicas=$(./scripts/landing-replicas.sh "${{ inputs.environment }}")" + } >> "$GITHUB_OUTPUT" + + maintenance: + needs: resolve + runs-on: ${{ needs.resolve.outputs.runner }} + environment: ${{ inputs.environment }} + concurrency: + # Shares a group with Deploy, so it cannot race an upgrade. + group: deploy-${{ inputs.environment }} + cancel-in-progress: false + env: + KUBECONFIG: ${{ github.workspace }}/kubeconfig + NS: ${{ needs.resolve.outputs.namespace }} + URL: https://${{ needs.resolve.outputs.landing }}/ + steps: + - uses: actions/checkout@v4 + - uses: azure/setup-kubectl@v4 + - name: Set up kubeconfig + run: | + set -euo pipefail + install -m 600 /dev/null "$KUBECONFIG" + printf '%s' '${{ secrets.KUBECONFIG }}' > "$KUBECONFIG" + + - name: Assert we are pointed at the right cluster + run: | + set -euo pipefail + FOUND=$(kubectl -n eduide-system get configmap eduide-cluster-identity \ + -o jsonpath='{.data.clusterName}' 2>/dev/null || true) + if [[ "$FOUND" != "${{ needs.resolve.outputs.cluster }}" ]]; then + echo "::error::expected cluster '${{ needs.resolve.outputs.cluster }}', this KUBECONFIG reaches '${FOUND:-an unbootstrapped cluster}'" + exit 1 + fi + + - name: The chart serves the maintenance page + if: ${{ inputs.mode == 'on' }} + run: | + set -euo pipefail + # Without the policy, scaling to 0 shows Envoy's raw error instead. + if ! kubectl -n "$NS" get backendtrafficpolicy landing-maintenance-page >/dev/null 2>&1; then + echo "::error::no landing-maintenance-page policy in $NS. Deploy chart >= 2.4.0 with maintenancePage.enabled first." + exit 1 + fi + + - name: Turn maintenance on + if: ${{ inputs.mode == 'on' }} + run: | + set -euo pipefail + kubectl -n "$NS" scale deploy/landing-page-deployment --replicas=0 + kubectl -n "$NS" wait --for=delete pod -l app=landing-page --timeout=2m + + - name: Turn maintenance off + if: ${{ inputs.mode == 'off' }} + run: | + set -euo pipefail + kubectl -n "$NS" scale deploy/landing-page-deployment --replicas=${{ needs.resolve.outputs.replicas }} + kubectl -n "$NS" rollout status deploy/landing-page-deployment --timeout=5m + + - name: Check what visitors see + run: | + set -euo pipefail + # Envoy needs a moment to notice the endpoints changed. + want=$([[ "${{ inputs.mode }}" == "on" ]] && echo 503 || echo 200) + for _ in $(seq 1 12); do + code=$(curl -s -o page.html -w '%{http_code}' --max-time 10 "$URL" || echo 000) + if [[ "$code" == "$want" ]]; then break; fi + sleep 5 + done + if [[ "$code" != "$want" ]]; then + echo "::error::$URL answered $code, expected $want"; exit 1 + fi + if [[ "$want" == 503 ]] && ! grep -q "EduIDE is currently unavailable" page.html; then + echo "::error::$URL answered 503 but not with the maintenance page"; exit 1 + fi + echo "### Maintenance ${{ inputs.mode }}: $URL answers $code" >> "$GITHUB_STEP_SUMMARY" diff --git a/README.md b/README.md index 8e6c6b7..ed5f661 100644 --- a/README.md +++ b/README.md @@ -69,6 +69,7 @@ file. `Deploy` does the second. | Move staging | Actions → **Deploy staging** | | Move production | Bump `chartVersion` in `environments/tum-production/env.yaml`, open a PR | | Undo a bad deploy | Actions → **Rollback** | +| Show the maintenance page ("EduIDE is currently unavailable") | Actions → **Maintenance page**, `mode: on`; `mode: off` or any deploy ends it | | Bring up a new cluster | Actions → **Bootstrap cluster** | `e2e.eduide.student.k8s.aet.cit.tum.de` deploys from `main` automatically and @@ -79,6 +80,10 @@ Every deploy asserts which cluster it reached before touching anything, shows a `helm diff` before applying, runs `--wait --atomic`, and prints a summary read back from the cluster rather than echoed from its inputs. +The maintenance page needs chart >= 2.4.0. It is what the landing host serves +whenever the landing page has no ready pod, so `Maintenance page` only scales +the landing page to 0. The REST service and running sessions are untouched. + ## What an environment does and does not configure An environment file carries **hosts, gateway routing, Keycloak and branding**. diff --git a/docs/github-environments.md b/docs/github-environments.md index 4796643..1bd8cec 100644 --- a/docs/github-environments.md +++ b/docs/github-environments.md @@ -8,7 +8,7 @@ There are **two kinds**, and they are not interchangeable. | Kind | Named | Used by | Holds | |---|---|---|---| -| Per environment | the installation's landing hostname, which is also its directory under `environments/` | `Deploy`, `Rollback` | `KUBECONFIG`, `THEIA_KEYCLOAK_COOKIE_SECRET` | +| Per environment | the installation's landing hostname, which is also its directory under `environments/` | `Deploy`, `Rollback`, `Maintenance page` | `KUBECONFIG`, `THEIA_KEYCLOAK_COOKIE_SECRET` | | Per cluster | `spec.bootstrapEnvironment` in `clusters/.yaml`, by convention `cluster-` | `Bootstrap cluster` | `KUBECONFIG`, `THEIA_WILDCARD_CERTIFICATE_CERT`, `THEIA_WILDCARD_CERTIFICATE_KEY` | A deploy never installs anything cluster-scoped, and a bootstrap never installs diff --git a/environments/_base.yaml b/environments/_base.yaml index d5de3f0..01af452 100644 --- a/environments/_base.yaml +++ b/environments/_base.yaml @@ -42,6 +42,9 @@ operator: eagerStart: true landingPage: + # Two pods so a node going away or a deploy never leaves the landing page + # without a ready one. Needs chart >= 2.4.0; older charts ignore it. + replicas: 2 ephemeralStorage: true # Every installation we run is on TUM infrastructure, so TUM is the # controller for all of them and this belongs here rather than per @@ -77,6 +80,9 @@ landingPage: text: About url: https://aet.cit.tum.de/ service: + # Same reason as landingPage.replicas. The service keeps its state in the + # cluster, so two pods are safe. + replicas: 2 # Public spam-mitigation token shared with the landing page, not a secret. # Same value the installations have always used; changing it gains nothing. authToken: nJV3nKZmpxTD4wu2 diff --git a/environments/bonn.eduide.aet.cit.tum.de/values.yaml b/environments/bonn.eduide.aet.cit.tum.de/values.yaml index 2bc4829..9283341 100644 --- a/environments/bonn.eduide.aet.cit.tum.de/values.yaml +++ b/environments/bonn.eduide.aet.cit.tum.de/values.yaml @@ -135,3 +135,9 @@ theia-workspace-garbage-collector: # decision rather than an accident. monitoring: enabled: true + +# parma is a single node. A PodDisruptionBudget there makes `kubectl drain` +# wait forever: the evicted replica cannot be rescheduled on the cordoned node, +# so the budget never allows evicting the other one. +podDisruptionBudget: + enabled: false diff --git a/environments/e2e.eduide.student.k8s.aet.cit.tum.de/env.yaml b/environments/e2e.eduide.student.k8s.aet.cit.tum.de/env.yaml index b3d5375..c900600 100644 --- a/environments/e2e.eduide.student.k8s.aet.cit.tum.de/env.yaml +++ b/environments/e2e.eduide.student.k8s.aet.cit.tum.de/env.yaml @@ -12,5 +12,5 @@ spec: cluster: tum-student namespace: eduide-e2e-test platform: - chartVersion: 2.2.1 + chartVersion: 2.4.0 channel: main diff --git a/environments/mannheim.eduide.aet.cit.tum.de/values.yaml b/environments/mannheim.eduide.aet.cit.tum.de/values.yaml index f36674f..c7318af 100644 --- a/environments/mannheim.eduide.aet.cit.tum.de/values.yaml +++ b/environments/mannheim.eduide.aet.cit.tum.de/values.yaml @@ -139,3 +139,9 @@ theia-workspace-garbage-collector: # decision rather than an accident. monitoring: enabled: true + +# parma is a single node. A PodDisruptionBudget there makes `kubectl drain` +# wait forever: the evicted replica cannot be rescheduled on the cordoned node, +# so the budget never allows evicting the other one. +podDisruptionBudget: + enabled: false diff --git a/environments/staging.eduide.student.k8s.aet.cit.tum.de/env.yaml b/environments/staging.eduide.student.k8s.aet.cit.tum.de/env.yaml index e3809a3..4357191 100644 --- a/environments/staging.eduide.student.k8s.aet.cit.tum.de/env.yaml +++ b/environments/staging.eduide.student.k8s.aet.cit.tum.de/env.yaml @@ -12,5 +12,5 @@ spec: cluster: tum-student namespace: eduide-staging platform: - chartVersion: 2.3.0 + chartVersion: 2.4.0 channel: main diff --git a/environments/test1.eduide.student.k8s.aet.cit.tum.de/env.yaml b/environments/test1.eduide.student.k8s.aet.cit.tum.de/env.yaml index cd5c63c..3888028 100644 --- a/environments/test1.eduide.student.k8s.aet.cit.tum.de/env.yaml +++ b/environments/test1.eduide.student.k8s.aet.cit.tum.de/env.yaml @@ -12,5 +12,5 @@ spec: cluster: tum-student namespace: eduide-test1 platform: - chartVersion: 2.2.1 + chartVersion: 2.4.0 channel: main diff --git a/environments/test2.eduide.student.k8s.aet.cit.tum.de/env.yaml b/environments/test2.eduide.student.k8s.aet.cit.tum.de/env.yaml index 063f8c8..9d9e512 100644 --- a/environments/test2.eduide.student.k8s.aet.cit.tum.de/env.yaml +++ b/environments/test2.eduide.student.k8s.aet.cit.tum.de/env.yaml @@ -12,5 +12,5 @@ spec: cluster: tum-student namespace: eduide-test2 platform: - chartVersion: 2.2.1 + chartVersion: 2.4.0 channel: main diff --git a/environments/test3.eduide.student.k8s.aet.cit.tum.de/env.yaml b/environments/test3.eduide.student.k8s.aet.cit.tum.de/env.yaml index 5c65296..6480a64 100644 --- a/environments/test3.eduide.student.k8s.aet.cit.tum.de/env.yaml +++ b/environments/test3.eduide.student.k8s.aet.cit.tum.de/env.yaml @@ -12,5 +12,5 @@ spec: cluster: tum-student namespace: eduide-test3 platform: - chartVersion: 2.2.1 + chartVersion: 2.4.0 channel: main diff --git a/scripts/landing-replicas.sh b/scripts/landing-replicas.sh new file mode 100755 index 0000000..bc46bdf --- /dev/null +++ b/scripts/landing-replicas.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +# How many landing page pods an environment runs when it is not in maintenance. +# +# ./scripts/landing-replicas.sh +# +# The maintenance workflow scales the landing page to 0 and needs to know what +# to scale it back to. Same precedence Helm applies: the environment's +# values.yaml over _base.yaml over the chart default of 1. +set -euo pipefail +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +ENV="${1:?usage: landing-replicas.sh }" +V="$ROOT/environments/$ENV/values.yaml" +[[ -f "$V" ]] || { echo "no such environment: $ENV" >&2; exit 1; } +yq -r '.landingPage.replicas // ""' "$V" "$ROOT/environments/_base.yaml" | grep -vE '^(---)?$' | head -1 | grep . || echo 1 diff --git a/scripts/test-deploy-logic.sh b/scripts/test-deploy-logic.sh index 4e968ff..e9da486 100755 --- a/scripts/test-deploy-logic.sh +++ b/scripts/test-deploy-logic.sh @@ -424,6 +424,42 @@ for v in "$ROOT"/environments/*/values.yaml; do fi done +# --- maintenance scales the landing page back to what Helm would ----------- +# maintenance.yml turns maintenance off by scaling the landing page back up. It +# must pick the same count Helm would, or the next deploy changes it again. +echo +echo "=== maintenance restores the landing page replicas ===" +T=$(mktemp -d) +mkdir -p "$T/scripts" "$T/environments/a" "$T/environments/b" +cp "$ROOT/scripts/landing-replicas.sh" "$T/scripts/" +replicas() { "$T/scripts/landing-replicas.sh" "$1" 2>/dev/null; } +printf 'app:\n id: x\n' > "$T/environments/_base.yaml" +printf 'landingPage:\n appDefinition: y\n' > "$T/environments/a/values.yaml" +[[ "$(replicas a)" == "1" ]] && ok "chart default when nobody sets it" || bad "chart default" "$(replicas a)" +printf 'landingPage:\n replicas: 2\n' > "$T/environments/_base.yaml" +[[ "$(replicas a)" == "2" ]] && ok "_base.yaml applies" || bad "_base.yaml" "$(replicas a)" +printf 'landingPage:\n replicas: 3\n' > "$T/environments/b/values.yaml" +[[ "$(replicas b)" == "3" ]] && ok "environment overrides _base.yaml" || bad "environment override" "$(replicas b)" +replicas nope >/dev/null && bad "unknown environment accepted" "" || ok "unknown environment rejected" +rm -rf "$T" +for f in "$ROOT"/environments/*/env.yaml; do + env=$(basename "$(dirname "$f")") + n=$("$ROOT/scripts/landing-replicas.sh" "$env" 2>/dev/null) + [[ "$n" =~ ^[1-9][0-9]*$ ]] && ok "$env: landing page returns with $n" || bad "$env: bad landing replica count" "'$n'" +done + +# --- no PodDisruptionBudgets on the single-node cluster --------------------- +# On parma (cluster `eduide`) a PDB makes every drain hang, see the comment in +# the bonn and mannheim values files. +echo +echo "=== no PodDisruptionBudgets on the single-node cluster ===" +for f in "$ROOT"/environments/*/env.yaml; do + [[ "$(yq -r '.spec.cluster' "$f")" == "eduide" ]] || continue + env=$(basename "$(dirname "$f")") + pdb=$(yq -r '.podDisruptionBudget.enabled' "$ROOT/environments/$env/values.yaml") + [[ "$pdb" == "false" ]] && ok "$env: PDBs off" || bad "$env: PDBs must be off on parma" "podDisruptionBudget.enabled is '$pdb'" +done + echo [[ $FAILED -eq 0 ]] && echo "ALL PASS" || echo "SOME FAILED" exit $FAILED