Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 21 additions & 3 deletions deploy/fleet-ctl.sh
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,8 @@
# (ssh host of the node; node commands then run there), FLEET_NODE_PREFIX (its install
# dir, default ~/app/fleet) and FLEET_RELEASE_REPO (default ProjectZKM/zkm-fleet).
# FLEET_NODE_UNIT names the node's systemd unit; FLEET_PROOF_APP_UNITS the app endpoint
# units a fleet-proof-app upgrade restarts. Every upgrade backs up what it replaces first.
# units a fleet-proof-app upgrade restarts; FLEET_CTL_HEALTH_WAIT how many seconds to wait for
# the scheduler to start listening (default 300). Every upgrade backs up what it replaces first.
# Node upgrades and rollbacks require systemd; see docs/maintenance.md for cron nodes.
set -euo pipefail
SELF=$(readlink -f "$0")
Expand Down Expand Up @@ -97,10 +98,27 @@ COMPOSE=(docker compose --env-file "$PREFIX/compose.env" -p zkm-fleet -f "$PREFI
compose() { run "${COMPOSE[@]}" "$@"; }
# Naming a profiled service starts it, so recreate MCP only where install.sh enabled it.
control_services() { echo scheduler; grep -q '^COMPOSE_PROFILES=.*mcp' "$PREFIX/compose.env" 2>/dev/null && echo mcp || true; }
scheduler_container() { # STATUS RESTARTS of the scheduler's container, or nothing
local id; id=$("${COMPOSE[@]}" ps -aq scheduler 2>/dev/null) || return 0
[ -z "$id" ] || docker inspect -f '{{.State.Status}} {{.RestartCount}}' "$id" 2>/dev/null || true
}
# A scheduler reads every job and command record it keeps before it listens, which takes a
# minute or more on a busy control host. Wait up to FLEET_CTL_HEALTH_WAIT seconds for it, but
# give up at once if its container exits or restarts: that is a crash, not a slow start.
scheduler_healthy() {
[ "$DRY" = 1 ] && return 0
for _ in $(seq 30); do curl -fs -o /dev/null http://127.0.0.1:9191/healthz && return 0; sleep 2; done
return 1
local wait=${FLEET_CTL_HEALTH_WAIT:-300} start=$SECONDS first now said=
first=$(scheduler_container)
until curl -fs -o /dev/null http://127.0.0.1:9191/healthz; do
now=$(scheduler_container)
if [ -n "$first" ] && { [ "${now%% *}" != running ] || [ "${now#* }" != "${first#* }" ]; }; then
echo "fleet-ctl: the scheduler container stopped (${now:-gone}); see docker logs zkm-fleet-scheduler-1" >&2
return 1
fi
[ $((SECONDS - start)) -lt "$wait" ] || return 1
[ -n "$said" ] || { echo "waiting up to ${wait}s for the scheduler to load its records and listen"; said=1; }
sleep 2
done
}
set_scheduler_image() { # compose.env names the image both services run
[ "$DRY" = 1 ] && { echo "+ set FLEET_SCHEDULER_IMAGE=$1 in $PREFIX/compose.env"; return; }
Expand Down
50 changes: 50 additions & 0 deletions deploy/tests/test_fleet_ctl.py
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,56 @@ def test_a_failed_upgrade_leaves_no_stage_directory(tmp_path):
assert list(tmpdir.iterdir()) == [], "the release stage directory leaked"


def rollback(tmp_path, healthy_after, later_state="running 0", wait=None):
"""`fleet-ctl.sh scheduler rollback` with stand-ins: healthz answers on curl's
`healthy_after`th call, and the scheduler's container reports "running 0" first and
`later_state` after that. Sleeps take no time."""
prefix, commands = tmp_path / "fleet", tmp_path / "commands"
backup = prefix / "backup/control-20260101T000000Z"
for name in ["fleetctl", "docker-compose.yml", "gateway.yaml.in", "compose.env"]:
executable(backup / name, "")
(prefix / "deploy").mkdir(parents=True)
(prefix / "bin").mkdir()
curls, inspects = tmp_path / "curls", tmp_path / "inspects"
executable(commands / "curl", f'n=$(($(cat {curls} 2>/dev/null || echo 0) + 1)); echo $n > {curls}\n'
f'[ "$n" -ge {healthy_after} ]\n')
executable(commands / "docker", f'case "$*" in\n'
f' *" ps -aq scheduler") echo abc123 ;;\n'
f' inspect*) n=$(($(cat {inspects} 2>/dev/null || echo 0) + 1)); echo $n > {inspects}\n'
f' if [ "$n" = 1 ]; then echo "running 0"; else echo {shlex.quote(later_state)}; fi ;;\n'
f'esac\n')
executable(commands / "sleep", "")
environment = {**os.environ, "PATH": f"{commands}:{os.environ['PATH']}", "FLEET_PREFIX": str(prefix),
"FLEET_CTL_ENV": str(tmp_path / "none.env"), "FLEET_NODE_SSH": ""}
if wait is not None:
environment["FLEET_CTL_HEALTH_WAIT"] = str(wait)
result = subprocess.run(["bash", str(ROOT / "deploy/fleet-ctl.sh"), "scheduler", "rollback"],
env=environment, capture_output=True, text=True, timeout=60)
return result, int(curls.read_text())


def test_a_scheduler_that_starts_slowly_is_waited_for(tmp_path):
# Forty checks two seconds apart is longer than the minute the script used to wait.
result, curls = rollback(tmp_path, healthy_after=40)
assert result.returncode == 0, result.stderr
assert curls == 40
assert result.stdout.count("waiting up to 300s for the scheduler") == 1
assert "restored" in result.stdout


def test_a_scheduler_that_restarts_fails_at_once(tmp_path):
result, curls = rollback(tmp_path, healthy_after=10**6, later_state="restarting 1")
assert result.returncode != 0
assert "the scheduler container stopped (restarting 1)" in result.stderr
assert curls == 1


def test_a_scheduler_that_never_answers_fails_at_the_deadline(tmp_path):
result, curls = rollback(tmp_path, healthy_after=10**6, wait=0)
assert result.returncode != 0
assert "scheduler is not healthy after rollback" in result.stderr
assert curls == 1

def node_fixture(tmp_path):
"""A node prefix with the old binaries, bridge and launch scripts, plus fakes for sudo and friends."""
prefix = tmp_path / "node"
Expand Down
7 changes: 5 additions & 2 deletions docs/maintenance.md
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,11 @@ deploy/fleet-ctl.sh scheduler upgrade v0.1.6
It downloads the release package and checks its SHA-256, pulls the image (a missing
image changes nothing), backs up `fleetctl`, the Compose files and `compose.env`,
records the new image, recreates `scheduler` (and `mcp` when enabled) and waits for
`/healthz`. Recreating takes about 90 seconds while the scheduler finishes in-flight
calls. `scheduler rollback` restores the backup. Keep site changes to the Compose
`/healthz`. The old scheduler can take about 90 seconds to finish in-flight calls, and
the new one reads every job and command record it keeps before it listens, a minute or
more on a busy host. The script waits up to five minutes (`FLEET_CTL_HEALTH_WAIT`
seconds), and gives up at once if the scheduler's container exits or restarts.
`scheduler rollback` restores the backup. Keep site changes to the Compose
file, such as another gateway or an existing Postgres volume, in
`deploy/docker-compose.override.yml`: an upgrade replaces only `docker-compose.yml`,
and every Compose call applies the override after it.
Expand Down
Loading