diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 965787c..c65e2c7 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -185,6 +185,22 @@ jobs: exit 1 fi + - name: Warn when this deploy ends maintenance + if: ${{ !inputs.dry_run }} + run: | + set -euo pipefail + # maintenance.yml shows the maintenance page by scaling the landing + # page to 0. This upgrade scales it back, so say so rather than let + # maintenance end without anyone noticing. + n=$(kubectl -n "${{ steps.env.outputs.namespace }}" get deploy landing-page-deployment \ + -o jsonpath='{.spec.replicas}' 2>/dev/null || true) + if [[ "$n" == "0" ]]; then + msg="The landing page is at 0 replicas (maintenance mode). This deploy brings it back and ends maintenance." + echo "::warning::$msg" + echo "> [!WARNING]" >> "$GITHUB_STEP_SUMMARY" + echo "> $msg" >> "$GITHUB_STEP_SUMMARY" + fi + - name: Resolve image overrides id: images run: | diff --git a/.github/workflows/maintenance.yml b/.github/workflows/maintenance.yml new file mode 100644 index 0000000..aff08a3 --- /dev/null +++ b/.github/workflows/maintenance.yml @@ -0,0 +1,132 @@ +# Show the maintenance page on an environment, or take it down again. +# +# The eduide chart (>= 2.4.0) already answers every 502/503/504 on the landing +# route with a static "EduIDE is currently unavailable" page, served by Envoy. +# So maintenance mode is nothing more than the landing page having no pods: +# on scales landing-page-deployment to 0 +# off scales it back to landingPage.replicas (scripts/landing-replicas.sh) +# +# Worth knowing: +# - The next deploy ends maintenance, because Helm sets the replicas again. +# - Only the landing page goes away. Running sessions and the REST service +# are untouched, so students already in an IDE keep working. + +name: Maintenance page + +on: + workflow_dispatch: + inputs: + environment: + required: true + type: choice + options: [test1.eduide.student.k8s.aet.cit.tum.de, test2.eduide.student.k8s.aet.cit.tum.de, test3.eduide.student.k8s.aet.cit.tum.de, e2e.eduide.student.k8s.aet.cit.tum.de, staging.eduide.student.k8s.aet.cit.tum.de, eduide.artemis.cit.tum.de, bonn.eduide.aet.cit.tum.de, mannheim.eduide.aet.cit.tum.de] + mode: + required: true + type: choice + options: ["on", "off"] + +permissions: + contents: read + +jobs: + resolve: + runs-on: ubuntu-latest + outputs: + runner: ${{ steps.r.outputs.runner }} + namespace: ${{ steps.r.outputs.namespace }} + cluster: ${{ steps.r.outputs.cluster }} + landing: ${{ steps.r.outputs.landing }} + replicas: ${{ steps.r.outputs.replicas }} + steps: + - uses: actions/checkout@v4 + - name: Install yq + run: | + sudo wget -qO /usr/local/bin/yq https://github.com/mikefarah/yq/releases/download/v4.44.3/yq_linux_amd64 + sudo chmod +x /usr/local/bin/yq + - id: r + run: | + set -euo pipefail + M="environments/${{ inputs.environment }}/env.yaml" + [[ -f "$M" ]] || { echo "::error::no such environment: ${{ inputs.environment }}"; exit 1; } + C=$(yq -r '.spec.cluster' "$M") + V="environments/${{ inputs.environment }}/values.yaml" + { + echo "cluster=$C" + echo "namespace=$(yq -r '.spec.namespace' "$M")" + echo "runner=$(yq -r '.spec.runner // "ubuntu-latest"' "clusters/$C.yaml")" + echo "landing=$(yq -r '.hosts.configuration | .landing + "." + .baseHost' "$V")" + echo "replicas=$(./scripts/landing-replicas.sh "${{ inputs.environment }}")" + } >> "$GITHUB_OUTPUT" + + maintenance: + needs: resolve + runs-on: ${{ needs.resolve.outputs.runner }} + environment: ${{ inputs.environment }} + concurrency: + # Shares a group with Deploy, so it cannot race an upgrade. + group: deploy-${{ inputs.environment }} + cancel-in-progress: false + env: + KUBECONFIG: ${{ github.workspace }}/kubeconfig + NS: ${{ needs.resolve.outputs.namespace }} + URL: https://${{ needs.resolve.outputs.landing }}/ + steps: + - uses: actions/checkout@v4 + - uses: azure/setup-kubectl@v4 + - name: Set up kubeconfig + run: | + set -euo pipefail + install -m 600 /dev/null "$KUBECONFIG" + printf '%s' '${{ secrets.KUBECONFIG }}' > "$KUBECONFIG" + + - name: Assert we are pointed at the right cluster + run: | + set -euo pipefail + FOUND=$(kubectl -n eduide-system get configmap eduide-cluster-identity \ + -o jsonpath='{.data.clusterName}' 2>/dev/null || true) + if [[ "$FOUND" != "${{ needs.resolve.outputs.cluster }}" ]]; then + echo "::error::expected cluster '${{ needs.resolve.outputs.cluster }}', this KUBECONFIG reaches '${FOUND:-an unbootstrapped cluster}'" + exit 1 + fi + + - name: The chart serves the maintenance page + if: ${{ inputs.mode == 'on' }} + run: | + set -euo pipefail + # Without the policy, scaling to 0 shows Envoy's raw error instead. + if ! kubectl -n "$NS" get backendtrafficpolicy landing-maintenance-page >/dev/null 2>&1; then + echo "::error::no landing-maintenance-page policy in $NS. Deploy chart >= 2.4.0 with maintenancePage.enabled first." + exit 1 + fi + + - name: Turn maintenance on + if: ${{ inputs.mode == 'on' }} + run: | + set -euo pipefail + kubectl -n "$NS" scale deploy/landing-page-deployment --replicas=0 + kubectl -n "$NS" wait --for=delete pod -l app=landing-page --timeout=2m + + - name: Turn maintenance off + if: ${{ inputs.mode == 'off' }} + run: | + set -euo pipefail + kubectl -n "$NS" scale deploy/landing-page-deployment --replicas=${{ needs.resolve.outputs.replicas }} + kubectl -n "$NS" rollout status deploy/landing-page-deployment --timeout=5m + + - name: Check what visitors see + run: | + set -euo pipefail + # Envoy needs a moment to notice the endpoints changed. + want=$([[ "${{ inputs.mode }}" == "on" ]] && echo 503 || echo 200) + for _ in $(seq 1 12); do + code=$(curl -s -o page.html -w '%{http_code}' --max-time 10 "$URL" || echo 000) + if [[ "$code" == "$want" ]]; then break; fi + sleep 5 + done + if [[ "$code" != "$want" ]]; then + echo "::error::$URL answered $code, expected $want"; exit 1 + fi + if [[ "$want" == 503 ]] && ! grep -q "EduIDE is currently unavailable" page.html; then + echo "::error::$URL answered 503 but not with the maintenance page"; exit 1 + fi + echo "### Maintenance ${{ inputs.mode }}: $URL answers $code" >> "$GITHUB_STEP_SUMMARY" diff --git a/README.md b/README.md index 8e6c6b7..ed5f661 100644 --- a/README.md +++ b/README.md @@ -69,6 +69,7 @@ file. `Deploy` does the second. | Move staging | Actions → **Deploy staging** | | Move production | Bump `chartVersion` in `environments/tum-production/env.yaml`, open a PR | | Undo a bad deploy | Actions → **Rollback** | +| Show the maintenance page ("EduIDE is currently unavailable") | Actions → **Maintenance page**, `mode: on`; `mode: off` or any deploy ends it | | Bring up a new cluster | Actions → **Bootstrap cluster** | `e2e.eduide.student.k8s.aet.cit.tum.de` deploys from `main` automatically and @@ -79,6 +80,10 @@ Every deploy asserts which cluster it reached before touching anything, shows a `helm diff` before applying, runs `--wait --atomic`, and prints a summary read back from the cluster rather than echoed from its inputs. +The maintenance page needs chart >= 2.4.0. It is what the landing host serves +whenever the landing page has no ready pod, so `Maintenance page` only scales +the landing page to 0. The REST service and running sessions are untouched. + ## What an environment does and does not configure An environment file carries **hosts, gateway routing, Keycloak and branding**. diff --git a/docs/github-environments.md b/docs/github-environments.md index 4796643..1bd8cec 100644 --- a/docs/github-environments.md +++ b/docs/github-environments.md @@ -8,7 +8,7 @@ There are **two kinds**, and they are not interchangeable. | Kind | Named | Used by | Holds | |---|---|---|---| -| Per environment | the installation's landing hostname, which is also its directory under `environments/` | `Deploy`, `Rollback` | `KUBECONFIG`, `THEIA_KEYCLOAK_COOKIE_SECRET` | +| Per environment | the installation's landing hostname, which is also its directory under `environments/` | `Deploy`, `Rollback`, `Maintenance page` | `KUBECONFIG`, `THEIA_KEYCLOAK_COOKIE_SECRET` | | Per cluster | `spec.bootstrapEnvironment` in `clusters/.yaml`, by convention `cluster-` | `Bootstrap cluster` | `KUBECONFIG`, `THEIA_WILDCARD_CERTIFICATE_CERT`, `THEIA_WILDCARD_CERTIFICATE_KEY` | A deploy never installs anything cluster-scoped, and a bootstrap never installs diff --git a/environments/_base.yaml b/environments/_base.yaml index d5de3f0..01af452 100644 --- a/environments/_base.yaml +++ b/environments/_base.yaml @@ -42,6 +42,9 @@ operator: eagerStart: true landingPage: + # Two pods so a node going away or a deploy never leaves the landing page + # without a ready one. Needs chart >= 2.4.0; older charts ignore it. + replicas: 2 ephemeralStorage: true # Every installation we run is on TUM infrastructure, so TUM is the # controller for all of them and this belongs here rather than per @@ -77,6 +80,9 @@ landingPage: text: About url: https://aet.cit.tum.de/ service: + # Same reason as landingPage.replicas. The service keeps its state in the + # cluster, so two pods are safe. + replicas: 2 # Public spam-mitigation token shared with the landing page, not a secret. # Same value the installations have always used; changing it gains nothing. authToken: nJV3nKZmpxTD4wu2 diff --git a/environments/bonn.eduide.aet.cit.tum.de/values.yaml b/environments/bonn.eduide.aet.cit.tum.de/values.yaml index 2bc4829..9283341 100644 --- a/environments/bonn.eduide.aet.cit.tum.de/values.yaml +++ b/environments/bonn.eduide.aet.cit.tum.de/values.yaml @@ -135,3 +135,9 @@ theia-workspace-garbage-collector: # decision rather than an accident. monitoring: enabled: true + +# parma is a single node. A PodDisruptionBudget there makes `kubectl drain` +# wait forever: the evicted replica cannot be rescheduled on the cordoned node, +# so the budget never allows evicting the other one. +podDisruptionBudget: + enabled: false diff --git a/environments/e2e.eduide.student.k8s.aet.cit.tum.de/env.yaml b/environments/e2e.eduide.student.k8s.aet.cit.tum.de/env.yaml index b3d5375..c900600 100644 --- a/environments/e2e.eduide.student.k8s.aet.cit.tum.de/env.yaml +++ b/environments/e2e.eduide.student.k8s.aet.cit.tum.de/env.yaml @@ -12,5 +12,5 @@ spec: cluster: tum-student namespace: eduide-e2e-test platform: - chartVersion: 2.2.1 + chartVersion: 2.4.0 channel: main diff --git a/environments/mannheim.eduide.aet.cit.tum.de/values.yaml b/environments/mannheim.eduide.aet.cit.tum.de/values.yaml index f36674f..c7318af 100644 --- a/environments/mannheim.eduide.aet.cit.tum.de/values.yaml +++ b/environments/mannheim.eduide.aet.cit.tum.de/values.yaml @@ -139,3 +139,9 @@ theia-workspace-garbage-collector: # decision rather than an accident. monitoring: enabled: true + +# parma is a single node. A PodDisruptionBudget there makes `kubectl drain` +# wait forever: the evicted replica cannot be rescheduled on the cordoned node, +# so the budget never allows evicting the other one. +podDisruptionBudget: + enabled: false diff --git a/environments/staging.eduide.student.k8s.aet.cit.tum.de/env.yaml b/environments/staging.eduide.student.k8s.aet.cit.tum.de/env.yaml index e3809a3..4357191 100644 --- a/environments/staging.eduide.student.k8s.aet.cit.tum.de/env.yaml +++ b/environments/staging.eduide.student.k8s.aet.cit.tum.de/env.yaml @@ -12,5 +12,5 @@ spec: cluster: tum-student namespace: eduide-staging platform: - chartVersion: 2.3.0 + chartVersion: 2.4.0 channel: main diff --git a/environments/test1.eduide.student.k8s.aet.cit.tum.de/env.yaml b/environments/test1.eduide.student.k8s.aet.cit.tum.de/env.yaml index cd5c63c..3888028 100644 --- a/environments/test1.eduide.student.k8s.aet.cit.tum.de/env.yaml +++ b/environments/test1.eduide.student.k8s.aet.cit.tum.de/env.yaml @@ -12,5 +12,5 @@ spec: cluster: tum-student namespace: eduide-test1 platform: - chartVersion: 2.2.1 + chartVersion: 2.4.0 channel: main diff --git a/environments/test2.eduide.student.k8s.aet.cit.tum.de/env.yaml b/environments/test2.eduide.student.k8s.aet.cit.tum.de/env.yaml index 063f8c8..9d9e512 100644 --- a/environments/test2.eduide.student.k8s.aet.cit.tum.de/env.yaml +++ b/environments/test2.eduide.student.k8s.aet.cit.tum.de/env.yaml @@ -12,5 +12,5 @@ spec: cluster: tum-student namespace: eduide-test2 platform: - chartVersion: 2.2.1 + chartVersion: 2.4.0 channel: main diff --git a/environments/test3.eduide.student.k8s.aet.cit.tum.de/env.yaml b/environments/test3.eduide.student.k8s.aet.cit.tum.de/env.yaml index 5c65296..6480a64 100644 --- a/environments/test3.eduide.student.k8s.aet.cit.tum.de/env.yaml +++ b/environments/test3.eduide.student.k8s.aet.cit.tum.de/env.yaml @@ -12,5 +12,5 @@ spec: cluster: tum-student namespace: eduide-test3 platform: - chartVersion: 2.2.1 + chartVersion: 2.4.0 channel: main diff --git a/scripts/landing-replicas.sh b/scripts/landing-replicas.sh new file mode 100755 index 0000000..bc46bdf --- /dev/null +++ b/scripts/landing-replicas.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +# How many landing page pods an environment runs when it is not in maintenance. +# +# ./scripts/landing-replicas.sh +# +# The maintenance workflow scales the landing page to 0 and needs to know what +# to scale it back to. Same precedence Helm applies: the environment's +# values.yaml over _base.yaml over the chart default of 1. +set -euo pipefail +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +ENV="${1:?usage: landing-replicas.sh }" +V="$ROOT/environments/$ENV/values.yaml" +[[ -f "$V" ]] || { echo "no such environment: $ENV" >&2; exit 1; } +yq -r '.landingPage.replicas // ""' "$V" "$ROOT/environments/_base.yaml" | grep -vE '^(---)?$' | head -1 | grep . || echo 1 diff --git a/scripts/test-deploy-logic.sh b/scripts/test-deploy-logic.sh index 4e968ff..e9da486 100755 --- a/scripts/test-deploy-logic.sh +++ b/scripts/test-deploy-logic.sh @@ -424,6 +424,42 @@ for v in "$ROOT"/environments/*/values.yaml; do fi done +# --- maintenance scales the landing page back to what Helm would ----------- +# maintenance.yml turns maintenance off by scaling the landing page back up. It +# must pick the same count Helm would, or the next deploy changes it again. +echo +echo "=== maintenance restores the landing page replicas ===" +T=$(mktemp -d) +mkdir -p "$T/scripts" "$T/environments/a" "$T/environments/b" +cp "$ROOT/scripts/landing-replicas.sh" "$T/scripts/" +replicas() { "$T/scripts/landing-replicas.sh" "$1" 2>/dev/null; } +printf 'app:\n id: x\n' > "$T/environments/_base.yaml" +printf 'landingPage:\n appDefinition: y\n' > "$T/environments/a/values.yaml" +[[ "$(replicas a)" == "1" ]] && ok "chart default when nobody sets it" || bad "chart default" "$(replicas a)" +printf 'landingPage:\n replicas: 2\n' > "$T/environments/_base.yaml" +[[ "$(replicas a)" == "2" ]] && ok "_base.yaml applies" || bad "_base.yaml" "$(replicas a)" +printf 'landingPage:\n replicas: 3\n' > "$T/environments/b/values.yaml" +[[ "$(replicas b)" == "3" ]] && ok "environment overrides _base.yaml" || bad "environment override" "$(replicas b)" +replicas nope >/dev/null && bad "unknown environment accepted" "" || ok "unknown environment rejected" +rm -rf "$T" +for f in "$ROOT"/environments/*/env.yaml; do + env=$(basename "$(dirname "$f")") + n=$("$ROOT/scripts/landing-replicas.sh" "$env" 2>/dev/null) + [[ "$n" =~ ^[1-9][0-9]*$ ]] && ok "$env: landing page returns with $n" || bad "$env: bad landing replica count" "'$n'" +done + +# --- no PodDisruptionBudgets on the single-node cluster --------------------- +# On parma (cluster `eduide`) a PDB makes every drain hang, see the comment in +# the bonn and mannheim values files. +echo +echo "=== no PodDisruptionBudgets on the single-node cluster ===" +for f in "$ROOT"/environments/*/env.yaml; do + [[ "$(yq -r '.spec.cluster' "$f")" == "eduide" ]] || continue + env=$(basename "$(dirname "$f")") + pdb=$(yq -r '.podDisruptionBudget.enabled' "$ROOT/environments/$env/values.yaml") + [[ "$pdb" == "false" ]] && ok "$env: PDBs off" || bad "$env: PDBs must be off on parma" "podDisruptionBudget.enabled is '$pdb'" +done + echo [[ $FAILED -eq 0 ]] && echo "ALL PASS" || echo "SOME FAILED" exit $FAILED