diff --git a/deploy/fleet-ctl.sh b/deploy/fleet-ctl.sh index 9b246fe..0fb003b 100755 --- a/deploy/fleet-ctl.sh +++ b/deploy/fleet-ctl.sh @@ -15,7 +15,8 @@ # (ssh host of the node; node commands then run there), FLEET_NODE_PREFIX (its install # dir, default ~/app/fleet) and FLEET_RELEASE_REPO (default ProjectZKM/zkm-fleet). # FLEET_NODE_UNIT names the node's systemd unit; FLEET_PROOF_APP_UNITS the app endpoint -# units a fleet-proof-app upgrade restarts. Every upgrade backs up what it replaces first. +# units a fleet-proof-app upgrade restarts; FLEET_CTL_HEALTH_WAIT how many seconds to wait for +# the scheduler to start listening (default 300). Every upgrade backs up what it replaces first. # Node upgrades and rollbacks require systemd; see docs/maintenance.md for cron nodes. set -euo pipefail SELF=$(readlink -f "$0") @@ -97,10 +98,27 @@ COMPOSE=(docker compose --env-file "$PREFIX/compose.env" -p zkm-fleet -f "$PREFI compose() { run "${COMPOSE[@]}" "$@"; } # Naming a profiled service starts it, so recreate MCP only where install.sh enabled it. control_services() { echo scheduler; grep -q '^COMPOSE_PROFILES=.*mcp' "$PREFIX/compose.env" 2>/dev/null && echo mcp || true; } +scheduler_container() { # STATUS RESTARTS of the scheduler's container, or nothing + local id; id=$("${COMPOSE[@]}" ps -aq scheduler 2>/dev/null) || return 0 + [ -z "$id" ] || docker inspect -f '{{.State.Status}} {{.RestartCount}}' "$id" 2>/dev/null || true +} +# A scheduler reads every job and command record it keeps before it listens, which takes a +# minute or more on a busy control host. Wait up to FLEET_CTL_HEALTH_WAIT seconds for it, but +# give up at once if its container exits or restarts: that is a crash, not a slow start. scheduler_healthy() { [ "$DRY" = 1 ] && return 0 - for _ in $(seq 30); do curl -fs -o /dev/null http://127.0.0.1:9191/healthz && return 0; sleep 2; done - return 1 + local wait=${FLEET_CTL_HEALTH_WAIT:-300} start=$SECONDS first now said= + first=$(scheduler_container) + until curl -fs -o /dev/null http://127.0.0.1:9191/healthz; do + now=$(scheduler_container) + if [ -n "$first" ] && { [ "${now%% *}" != running ] || [ "${now#* }" != "${first#* }" ]; }; then + echo "fleet-ctl: the scheduler container stopped (${now:-gone}); see docker logs zkm-fleet-scheduler-1" >&2 + return 1 + fi + [ $((SECONDS - start)) -lt "$wait" ] || return 1 + [ -n "$said" ] || { echo "waiting up to ${wait}s for the scheduler to load its records and listen"; said=1; } + sleep 2 + done } set_scheduler_image() { # compose.env names the image both services run [ "$DRY" = 1 ] && { echo "+ set FLEET_SCHEDULER_IMAGE=$1 in $PREFIX/compose.env"; return; } diff --git a/deploy/tests/test_fleet_ctl.py b/deploy/tests/test_fleet_ctl.py index e87a43c..51b00b0 100644 --- a/deploy/tests/test_fleet_ctl.py +++ b/deploy/tests/test_fleet_ctl.py @@ -60,6 +60,56 @@ def test_a_failed_upgrade_leaves_no_stage_directory(tmp_path): assert list(tmpdir.iterdir()) == [], "the release stage directory leaked" +def rollback(tmp_path, healthy_after, later_state="running 0", wait=None): + """`fleet-ctl.sh scheduler rollback` with stand-ins: healthz answers on curl's + `healthy_after`th call, and the scheduler's container reports "running 0" first and + `later_state` after that. Sleeps take no time.""" + prefix, commands = tmp_path / "fleet", tmp_path / "commands" + backup = prefix / "backup/control-20260101T000000Z" + for name in ["fleetctl", "docker-compose.yml", "gateway.yaml.in", "compose.env"]: + executable(backup / name, "") + (prefix / "deploy").mkdir(parents=True) + (prefix / "bin").mkdir() + curls, inspects = tmp_path / "curls", tmp_path / "inspects" + executable(commands / "curl", f'n=$(($(cat {curls} 2>/dev/null || echo 0) + 1)); echo $n > {curls}\n' + f'[ "$n" -ge {healthy_after} ]\n') + executable(commands / "docker", f'case "$*" in\n' + f' *" ps -aq scheduler") echo abc123 ;;\n' + f' inspect*) n=$(($(cat {inspects} 2>/dev/null || echo 0) + 1)); echo $n > {inspects}\n' + f' if [ "$n" = 1 ]; then echo "running 0"; else echo {shlex.quote(later_state)}; fi ;;\n' + f'esac\n') + executable(commands / "sleep", "") + environment = {**os.environ, "PATH": f"{commands}:{os.environ['PATH']}", "FLEET_PREFIX": str(prefix), + "FLEET_CTL_ENV": str(tmp_path / "none.env"), "FLEET_NODE_SSH": ""} + if wait is not None: + environment["FLEET_CTL_HEALTH_WAIT"] = str(wait) + result = subprocess.run(["bash", str(ROOT / "deploy/fleet-ctl.sh"), "scheduler", "rollback"], + env=environment, capture_output=True, text=True, timeout=60) + return result, int(curls.read_text()) + + +def test_a_scheduler_that_starts_slowly_is_waited_for(tmp_path): + # Forty checks two seconds apart is longer than the minute the script used to wait. + result, curls = rollback(tmp_path, healthy_after=40) + assert result.returncode == 0, result.stderr + assert curls == 40 + assert result.stdout.count("waiting up to 300s for the scheduler") == 1 + assert "restored" in result.stdout + + +def test_a_scheduler_that_restarts_fails_at_once(tmp_path): + result, curls = rollback(tmp_path, healthy_after=10**6, later_state="restarting 1") + assert result.returncode != 0 + assert "the scheduler container stopped (restarting 1)" in result.stderr + assert curls == 1 + + +def test_a_scheduler_that_never_answers_fails_at_the_deadline(tmp_path): + result, curls = rollback(tmp_path, healthy_after=10**6, wait=0) + assert result.returncode != 0 + assert "scheduler is not healthy after rollback" in result.stderr + assert curls == 1 + def node_fixture(tmp_path): """A node prefix with the old binaries, bridge and launch scripts, plus fakes for sudo and friends.""" prefix = tmp_path / "node" diff --git a/docs/maintenance.md b/docs/maintenance.md index e297a02..1e73e69 100644 --- a/docs/maintenance.md +++ b/docs/maintenance.md @@ -30,8 +30,11 @@ deploy/fleet-ctl.sh scheduler upgrade v0.1.6 It downloads the release package and checks its SHA-256, pulls the image (a missing image changes nothing), backs up `fleetctl`, the Compose files and `compose.env`, records the new image, recreates `scheduler` (and `mcp` when enabled) and waits for -`/healthz`. Recreating takes about 90 seconds while the scheduler finishes in-flight -calls. `scheduler rollback` restores the backup. Keep site changes to the Compose +`/healthz`. The old scheduler can take about 90 seconds to finish in-flight calls, and +the new one reads every job and command record it keeps before it listens, a minute or +more on a busy host. The script waits up to five minutes (`FLEET_CTL_HEALTH_WAIT` +seconds), and gives up at once if the scheduler's container exits or restarts. +`scheduler rollback` restores the backup. Keep site changes to the Compose file, such as another gateway or an existing Postgres volume, in `deploy/docker-compose.override.yml`: an upgrade replaces only `docker-compose.yml`, and every Compose call applies the override after it.