From 9458ecfd00ac7776029296707011b385511741c5 Mon Sep 17 00:00:00 2001 From: Christopher Plieger <917744+cplieger@users.noreply.github.com> Date: Fri, 10 Jul 2026 18:49:11 +0200 Subject: [PATCH 1/2] feat: start healthy and retry when Plex is unreachable at boot The initial Plex connect + admin resolution previously exited non-zero on any failure, so a Plex that was down at boot crash-looped the container under the restart policy. Now the startup classifies the failure: a fatal misconfiguration (bad token / 4xx, wrong-server 404, or a TLS/cert error) still exits non-zero, but a transient failure (Plex unreachable, DNS/timeout, or 5xx) marks the container healthy-degraded and retries the initial connect with capped exponential backoff (1s->30s) until Plex answers. This mirrors plex-exporter's fatal-vs-transient split and makes the container survive a Plex restart instead of crash-looping. Adds a typed plex.HTTPStatusError (carrying the status code, message unchanged) so the classifier can tell 4xx from 5xx, and a table test for the classifier + backoff. --- README.md | 4 +- internal/plex/client.go | 22 ++++- main.go | 191 ++++++++++++++++++++++++++++++++++------ main_test.go | 54 ++++++++++++ 4 files changed, 241 insertions(+), 30 deletions(-) diff --git a/README.md b/README.md index 8a69936..e9a5883 100644 --- a/README.md +++ b/README.md @@ -121,7 +121,9 @@ your latest preferences immediately. ## Healthcheck -The container includes a built-in CLI health probe (`/plex-language-sync health`) that checks for a marker file written at `/tmp/.healthy` once the initial Plex connection succeeds and the admin user is verified. It requires no shell, HTTP client, or open port. The probe reports unhealthy only if the initial connection to Plex fails or the admin user cannot be resolved — WebSocket disconnects do not cause unhealthy status because the tool automatically reconnects with exponential backoff (1s→30s). +The container includes a built-in CLI health probe (`/plex-language-sync health`) that checks for a marker file written at `/tmp/.healthy`. It requires no shell, HTTP client, or open port. + +Startup distinguishes fatal from transient failures. A **fatal** misconfiguration — a bad `PLEX_TOKEN` (401/403), a wrong-server URL (404), or a TLS/certificate error — exits non-zero, so the container never goes healthy and the problem stays loud. A **transient** failure — Plex unreachable or still starting at boot (connection refused, DNS/timeout, or a 5xx) — starts the container **healthy in a degraded state** and keeps retrying the initial connection with capped exponential backoff (1s→30s) instead of crash-looping under the restart policy; the marker is set as soon as the failure is classified transient, and normal operation begins once Plex answers and the admin user is resolved. WebSocket disconnects after the initial connection never cause unhealthy status either — the listener reconnects with the same backoff. ## Security diff --git a/internal/plex/client.go b/internal/plex/client.go index 57c1b30..cdb117c 100644 --- a/internal/plex/client.go +++ b/internal/plex/client.go @@ -27,6 +27,22 @@ const maxResponseBody = 10 << 20 // 10 MB // session / etc.). Callers detect it with errors.Is(err, plex.ErrNotFound). var ErrNotFound = errors.New("not found") +// HTTPStatusError is returned for a non-200, non-404 Plex response. It carries +// the status code so callers can classify the failure: a 4xx (bad token / auth +// / wrong endpoint) will not self-heal, while a 5xx (Plex up but not ready) +// may recover. The startup connect classifier in package main uses errors.As +// to distinguish fatal (4xx) from transient (5xx) initial-connection failures. +type HTTPStatusError struct { + Method string + Path string + Status string + Code int +} + +func (e *HTTPStatusError) Error() string { + return fmt.Sprintf("plex API %s %s: %s", e.Method, e.Path, e.Status) +} + // errBodyOverCap signals a response exceeded maxResponseBody. Callers map // it to an endpoint-specific error so the message stays specific while the // read-cap WARN + limit live in one place. @@ -230,9 +246,11 @@ func (c *Client) doJSON(ctx context.Context, method, path string, result any) er return ErrNotFound } if resp.StatusCode != http.StatusOK { - // Drain body to allow connection reuse. + // Drain body to allow connection reuse. Return a typed status error so + // the startup connect classifier can distinguish 4xx (fatal) from 5xx + // (transient); the message text is unchanged. drainBody(resp.Body) - return fmt.Errorf("plex API %s %s: %s", method, path, resp.Status) + return &HTTPStatusError{Method: method, Path: path, Status: resp.Status, Code: resp.StatusCode} } if result == nil { drainBody(resp.Body) diff --git a/main.go b/main.go index b48d055..cafb815 100644 --- a/main.go +++ b/main.go @@ -16,8 +16,12 @@ package main import ( "context" + "crypto/tls" + "crypto/x509" "errors" + "fmt" "log/slog" + "net/http" "os" "os/signal" "path/filepath" @@ -80,35 +84,42 @@ func run() int { } plex.WarnIfPlaintextURL(client.BaseURL()) - // Verify connectivity. - identity, err := client.ServerIdentity(ctx) + // Derive the cache encryption key from the admin PLEX_TOKEN up front. It is + // a pure function of the token (no Plex round-trip) and is deterministic for + // a given token, so decryption works offline on restart. Computing it here, + // before the connect loop, means a malformed token is surfaced as a config + // error BEFORE the loop can mark the container healthy-degraded. + encKey, err := cache.DeriveKey(cfg.plexToken) if err != nil { - slog.Error("cannot connect to plex server", "error", err) + slog.Error("cannot derive encryption key", "error", err) return 1 } - slog.Info("connected to plex server", - "name", identity.FriendlyName, - "id", identity.MachineIdentifier, - "version", identity.Version) - // Resolve the admin user. - admin, err := client.LoggedUser(ctx) + // Establish the initial Plex connection and resolve the admin user. A + // transient failure (Plex down or unreachable at boot) marks the container + // healthy-degraded and keeps retrying rather than crash-looping under the + // restart policy; a fatal failure (bad token, wrong server, or a TLS/cert + // misconfiguration) exits non-zero. Blocks until connected, fatal, or a + // shutdown signal. + identity, admin, err := connectAndResolveAdmin(ctx, client, marker) if err != nil { - slog.Error("cannot resolve admin user", "error", err) + marker.Set(false) // clear any degraded-healthy marker before exiting + if ctx.Err() != nil { + slog.Info("shutdown requested during startup", "cause", context.Cause(ctx)) + return 0 + } + slog.Error("cannot establish initial plex connection", "error", err) return 1 } + slog.Info("connected to plex server", + "name", identity.FriendlyName, + "id", identity.MachineIdentifier, + "version", identity.Version) slog.Info("authenticated as admin user", "name", admin.Name, "id", admin.ID) - // Cache — load persistent state, or start fresh on error. + // Cache — load persistent state, or start fresh on error. The encryption + // key (derived above) makes user-token decryption work offline on restart. c := cache.New() - // Derive encryption key from admin PLEX_TOKEN for user-token at-rest - // encryption. The key is deterministic for a given token — decryption - // works offline (no plex.tv round-trip needed on restart). - encKey, err := cache.DeriveKey(cfg.plexToken) - if err != nil { - slog.Error("cannot derive encryption key", "error", err) - return 1 - } c.SetEncryptionKey(encKey) if err := c.LoadFrom(cachePath); err != nil { slog.Warn("cache load failed, starting fresh", "error", err) @@ -125,14 +136,17 @@ func run() int { um.Init(admin, client.BaseURL(), cfg.caCertPath) um.LoadFromCache() - // Core Plex connection is verified, admin resolved, and the cache + user - // manager initialized with any cached tokens: the app can serve. Mark - // healthy BEFORE the plex.tv shared-user refresh so container liveness is - // not gated on that secondary dependency. Per the README, health = - // "initial Plex connection succeeds and admin user verified"; gating on the - // refresh would delay healthy up to ~75s (DefaultRefreshConfig) on a plex.tv - // outage and risk a Docker unhealthy/restart that cannot fix plex.tv. The - // periodic RefreshLoop keeps retrying. + // Connection verified and admin resolved (possibly after a healthy-degraded + // retry period), cache + user manager initialized: the app can serve. + // marker.Set(true) is idempotent — the connect loop already set it if the + // initial connection was degraded; setting it here also covers the + // connected-on-first-try path. Health = "connected, or transiently retrying + // the initial connect"; a fatal config/auth error exits before this point. + // Marked BEFORE the plex.tv shared-user refresh so container liveness is not + // gated on that secondary dependency: gating on the refresh would delay + // healthy up to ~75s (DefaultRefreshConfig) on a plex.tv outage and risk a + // Docker unhealthy/restart that cannot fix plex.tv. The periodic RefreshLoop + // keeps retrying. marker.Set(true) // Shutdown sequence: flag unhealthy first so Docker stops routing health // probes as passing while the (slow) cache save runs, then persist the @@ -258,6 +272,129 @@ func waitForBackgroundLoops(wg *sync.WaitGroup, refreshDone, schedDone <-chan st } } +// --------------------------------------------------------------------------- +// Initial connection (degraded start) +// --------------------------------------------------------------------------- +// +// The app cannot do anything without a Plex connection + a resolved admin +// user (the WebSocket listener, scheduler, syncer, and user manager all +// depend on them), so a "degraded start" is: mark healthy, then retry the +// initial connect until it succeeds, rather than serving in a reduced mode. +// This keeps a Plex-down-at-boot from crash-looping the container under the +// restart policy (the old behaviour was os.Exit(1) on the first failure). +// A fatal config/auth error still exits fast so the misconfiguration is loud. + +const ( + startupBaseBackoff = 1 * time.Second + startupMaxBackoff = 30 * time.Second +) + +// connectAndResolveAdmin verifies the Plex connection and resolves the admin +// user, retrying transient failures with capped exponential backoff. On the +// first transient failure it marks the container healthy (a degraded start), +// then keeps retrying until Plex answers. A fatal error (bad token / 4xx, a +// wrong-server 404, or a TLS/cert misconfiguration) returns immediately so the +// caller can exit non-zero. Returns ctx.Err() when shutdown is signalled +// mid-retry. +func connectAndResolveAdmin(ctx context.Context, client *plex.Client, marker *health.Marker) (*plex.ServerIdentity, *plex.User, error) { + degraded := false + for attempt := 0; ; attempt++ { + identity, admin, err := connectOnce(ctx, client) + if err == nil { + if degraded { + slog.Info("plex connection recovered; leaving degraded state") + } + return identity, admin, nil + } + if isFatalStartupError(err) { + return nil, nil, err + } + if ctx.Err() != nil { + return nil, nil, ctx.Err() + } + if degraded { + slog.Warn("plex still unreachable; retrying", "error", err) + } else { + // First transient failure: mark healthy so the restart policy does + // not crash-loop the container while Plex is unreachable, then keep + // retrying. Recovery needs no counterpart flip — the marker is + // already set and stays set. + marker.Set(true) + degraded = true + slog.Warn("initial plex connection failed; starting in degraded state and retrying", "error", err) + } + select { + case <-ctx.Done(): + return nil, nil, ctx.Err() + case <-time.After(startupBackoff(attempt)): + } + } +} + +// connectOnce performs a single connect + admin-resolve attempt. +func connectOnce(ctx context.Context, client *plex.Client) (*plex.ServerIdentity, *plex.User, error) { + identity, err := client.ServerIdentity(ctx) + if err != nil { + return nil, nil, fmt.Errorf("connecting to plex server: %w", err) + } + admin, err := client.LoggedUser(ctx) + if err != nil { + return nil, nil, fmt.Errorf("resolving admin user: %w", err) + } + return identity, admin, nil +} + +// startupBackoff returns the delay before retry attempt n (0-indexed): +// startupBaseBackoff * 2^n, capped at startupMaxBackoff. The shift is guarded +// so a large attempt count cannot overflow the duration to a negative value. +func startupBackoff(attempt int) time.Duration { + if attempt < 0 || attempt >= 30 { + return startupMaxBackoff + } + d := startupBaseBackoff << attempt + if d <= 0 || d > startupMaxBackoff { + return startupMaxBackoff + } + return d +} + +// isFatalStartupError reports whether an initial Plex connect/admin-resolve +// error is a configuration or authentication problem that will not resolve +// without operator action (so run() should exit) rather than a transient +// connectivity failure (so run() should start degraded and keep retrying). A +// bad token (401/403) or other 4xx, a 404 (wrong server), and TLS/certificate +// errors are fatal; dial/DNS/timeout errors, 5xx (a Plex still starting up), +// and 429/408 (throttle/timeout signals) are treated as transient. +func isFatalStartupError(err error) bool { + var statusErr *plex.HTTPStatusError + if errors.As(err, &statusErr) { + // 429 (Too Many Requests) and 408 (Request Timeout) are throttle/timeout + // signals, not config/auth errors: treat them as transient so a busy or + // slow Plex backs off and retries rather than exiting and crash-looping. + if statusErr.Code == http.StatusTooManyRequests || statusErr.Code == http.StatusRequestTimeout { + return false + } + return statusErr.Code < 500 + } + // 404 on the identity endpoint: reached Plex, wrong server. + if errors.Is(err, plex.ErrNotFound) { + return true + } + // TLS/certificate misconfiguration (e.g. a self-signed cert without + // PLEX_CA_CERT_PATH): will not recover without a config change. + var certErr *tls.CertificateVerificationError + if errors.As(err, &certErr) { + return true + } + var unknownAuthority x509.UnknownAuthorityError + if errors.As(err, &unknownAuthority) { + return true + } + // Transport errors (connection refused, DNS failure, timeout): Plex is + // unreachable now but may come back. + return false +} + // --------------------------------------------------------------------------- // WebSocket listener (adapter) // --------------------------------------------------------------------------- diff --git a/main_test.go b/main_test.go index ce67ac3..c9f3f58 100644 --- a/main_test.go +++ b/main_test.go @@ -38,6 +38,10 @@ package main import ( "bytes" "context" + "crypto/tls" + "crypto/x509" + "errors" + "fmt" "log/slog" "net/http" "net/http/httptest" @@ -1011,3 +1015,53 @@ func TestResolvePlayEventUser_unresolvedSessionFallsBackAdmin(t *testing.T) { t.Errorf("resolvePlayEventUser = (%q, %q), want (1, admin); an unresolved session must fall back to admin", uid, uname) } } + +// --------------------------------------------------------------------------- +// isFatalStartupError / startupBackoff (degraded-start classifier + backoff) +// --------------------------------------------------------------------------- + +func TestIsFatalStartupError(t *testing.T) { + tests := []struct { + name string + err error + want bool + }{ + {"bad token 401 is fatal", &plex.HTTPStatusError{Code: 401, Status: "401 Unauthorized", Method: "GET", Path: "/myplex/account"}, true}, + {"forbidden 403 is fatal", &plex.HTTPStatusError{Code: 403, Status: "403 Forbidden", Method: "GET", Path: "/"}, true}, + {"other 4xx is fatal", &plex.HTTPStatusError{Code: 400, Status: "400 Bad Request", Method: "GET", Path: "/"}, true}, + {"503 is transient", &plex.HTTPStatusError{Code: 503, Status: "503 Service Unavailable", Method: "GET", Path: "/"}, false}, + {"500 is transient", &plex.HTTPStatusError{Code: 500, Status: "500 Internal Server Error", Method: "GET", Path: "/"}, false}, + {"429 rate limited is transient", &plex.HTTPStatusError{Code: 429, Status: "429 Too Many Requests", Method: "GET", Path: "/"}, false}, + {"408 request timeout is transient", &plex.HTTPStatusError{Code: 408, Status: "408 Request Timeout", Method: "GET", Path: "/"}, false}, + {"not found (wrong server) is fatal", fmt.Errorf("connecting to plex server: %w", plex.ErrNotFound), true}, + {"unknown CA is fatal", fmt.Errorf("connecting to plex server: %w", &url.Error{Op: "Get", URL: "https://plex:32400/", Err: x509.UnknownAuthorityError{}}), true}, + {"cert verification error is fatal", fmt.Errorf("connecting to plex server: %w", &url.Error{Op: "Get", URL: "https://plex:32400/", Err: &tls.CertificateVerificationError{Err: errors.New("x509: certificate has expired or is not yet valid")}}), true}, + {"connection refused is transient", fmt.Errorf("connecting to plex server: %w", &url.Error{Op: "Get", URL: "http://127.0.0.1:1/", Err: errors.New("connect: connection refused")}), false}, + {"dns failure is transient", errors.New("connecting to plex server: dial tcp: lookup plex: no such host"), false}, + {"wrapped 401 is fatal", fmt.Errorf("resolving admin user: %w", &plex.HTTPStatusError{Code: 401, Status: "401 Unauthorized", Method: "GET", Path: "/myplex/account"}), true}, + {"nil is not fatal", nil, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := isFatalStartupError(tt.err); got != tt.want { + t.Errorf("isFatalStartupError(%v) = %v, want %v", tt.err, got, tt.want) + } + }) + } +} + +func TestStartupBackoff(t *testing.T) { + if got := startupBackoff(0); got != startupBaseBackoff { + t.Errorf("startupBackoff(0) = %v, want %v", got, startupBaseBackoff) + } + if got := startupBackoff(1); got != 2*startupBaseBackoff { + t.Errorf("startupBackoff(1) = %v, want %v", got, 2*startupBaseBackoff) + } + // Large and negative attempt counts must cap cleanly, never overflowing the + // duration to zero or a negative value. + for _, n := range []int{5, 30, 62, 63, 100, -1} { + if got := startupBackoff(n); got <= 0 || got > startupMaxBackoff { + t.Errorf("startupBackoff(%d) = %v, want in (0, %v]", n, got, startupMaxBackoff) + } + } +} From c5dfec388020e8db4c3b80894a24d39a6f87ae95 Mon Sep 17 00:00:00 2001 From: Christopher Plieger <917744+cplieger@users.noreply.github.com> Date: Fri, 10 Jul 2026 18:49:29 +0200 Subject: [PATCH 2/2] test: adopt the shared image-smoke harness Now that a transient connect failure starts the container healthy-degraded (previous commit), plex-language-sync can reach healthy in CI against an unreachable dummy Plex. Enroll it on the shared harness: tests/image-smoke.conf (env-only, PLEX_URL=http://127.0.0.1:1 + a dummy token) + the canonical tests/image-smoke.sh synced from cplieger/ci. Removes plex-language-sync's documented smoke-test exception. --- tests/image-smoke.conf | 14 ++++++ tests/image-smoke.sh | 108 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 122 insertions(+) create mode 100644 tests/image-smoke.conf create mode 100644 tests/image-smoke.sh diff --git a/tests/image-smoke.conf b/tests/image-smoke.conf new file mode 100644 index 0000000..55a2d69 --- /dev/null +++ b/tests/image-smoke.conf @@ -0,0 +1,14 @@ +# Per-app config for the shared image-smoke harness (tests/image-smoke.sh, +# synced from cplieger/ci). plex-language-sync has no HTTP surface, so this is +# a health-gated test: it waits for the container HEALTHCHECK (the distroless +# `plex-language-sync health` file-marker probe, /tmp/.healthy) to report +# "healthy" - proving the binary runs in the distroless base and the health +# subcommand works. The dummy PLEX_URL is unreachable, which is a TRANSIENT +# connect failure: the app starts healthy-degraded and keeps retrying the +# initial connect (it does not fail-fast), so the marker flips healthy on boot +# without a live Plex. A fatal config error (a bad token -> 401) would instead +# exit non-zero and the container would never go healthy. +SMOKE_APP_NAME=plex-language-sync +# HEALTHCHECK start-period 15s / interval 30s; 90 covers it plus a couple intervals. +SMOKE_TIMEOUT=90 +SMOKE_RUN_ARGS="-e PLEX_URL=http://127.0.0.1:1 -e PLEX_TOKEN=smoke-token" diff --git a/tests/image-smoke.sh b/tests/image-smoke.sh new file mode 100644 index 0000000..86e1b3d --- /dev/null +++ b/tests/image-smoke.sh @@ -0,0 +1,108 @@ +#!/bin/sh +# Runtime image smoke-test harness — CANONICAL COPY in cplieger/ci +# (configs/image-smoke.sh), synced to each serving app's tests/image-smoke.sh +# by scripts/classify-repos.sh (a repo enrolls by committing a +# tests/image-smoke.conf; see below). DO NOT edit the synced copy in an app +# repo — change it here and let the sync land it. +# +# Invoked by the shared CI docker job: sh tests/image-smoke.sh +# +# It starts the assembled image and waits for the container's own HEALTHCHECK +# to report "healthy" — proving the binary runs in the final image, loads its +# config, binds any listener, and its health probe works, catching failures the +# build cannot see (a broken //go:embed frontend, a missing runtime dependency, +# a server that never binds, a broken HEALTHCHECK). It fails fast on an early +# exit (a crash-boot is reported by its exit code, more debuggable than +# "unhealthy") and dumps the container log tail only on failure. +# +# Per-app knobs come from tests/image-smoke.conf beside this script; everything +# below the config block is identical across apps. The .conf is a POSIX-sh +# fragment sourced for these variables (all optional): +# +# SMOKE_APP_NAME label for log lines + container name (default: "image") +# SMOKE_TIMEOUT seconds to wait for "healthy" (default: 120). Size it to +# cover the image's HEALTHCHECK start-period plus a couple of +# intervals; a slow-but-OK cold boot must not be failed early. +# SMOKE_RUN_ARGS extra `docker run` args (env, tmpfs, ...) as a word-split +# string, e.g. "-e FOO=bar --tmpfs /input". Values must not +# contain spaces (these are controlled test configs). +# +# The harness also exports $SMOKE_DIR (this script's own absolute directory) +# before sourcing the .conf, so an app that needs a config/fixture file on disk +# can bind-mount a committed fixture dir, e.g.: +# SMOKE_RUN_ARGS="-e SYNC_INTERVAL=off -v ${SMOKE_DIR}/fixtures:/config:ro" +set -eu + +IMG="${1:?usage: image-smoke.sh }" + +# Absolute directory of this script (also holds image-smoke.conf and any per-app +# fixtures). Exposed to the .conf as $SMOKE_DIR so a .conf can bind-mount a +# committed fixture dir with an absolute source path (docker -v requires one). +SMOKE_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd) + +# Per-app config lives beside this script (repo-local, NOT synced). Pre-set the +# knobs so `set -u` is safe and a repo with no .conf still runs with defaults. +SMOKE_APP_NAME="" +SMOKE_TIMEOUT="" +SMOKE_RUN_ARGS="" +CONF="$SMOKE_DIR/image-smoke.conf" +if [ -f "$CONF" ]; then + # shellcheck disable=SC1090 # per-app config path, resolved at runtime + . "$CONF" +fi + +APP="${SMOKE_APP_NAME:-image}" +TIMEOUT="${SMOKE_TIMEOUT:-120}" +NAME="smoke-${APP}-$$" + +# shellcheck disable=SC2317,SC2329 # invoked indirectly via trap +cleanup() { + code=$? + # Dump container logs only on failure (a passing run stays quiet). + if [ "$code" -ne 0 ]; then + printf '%s\n' "--- container logs (tail) ---" >&2 + docker logs "$NAME" 2>&1 | tail -40 >&2 || true + fi + docker rm -f "$NAME" >/dev/null 2>&1 || true +} +trap cleanup EXIT + +# SMOKE_RUN_ARGS is intentionally word-split (simple test args, no spaces). +# shellcheck disable=SC2086 +docker run -d --name "$NAME" $SMOKE_RUN_ARGS "$IMG" >/dev/null + +i=0 +status=starting +while [ "$i" -lt "$TIMEOUT" ]; do + # Fail fast on an early exit: poll .State.Running before the health status so + # a crash-boot is caught by its exit code (more debuggable than "unhealthy") + # and the verdict never depends on what health a stopped container reports. + if [ "$(docker inspect --format '{{ .State.Running }}' "$NAME" 2>/dev/null || echo missing)" != "true" ]; then + ec=$(docker inspect --format '{{ .State.ExitCode }}' "$NAME" 2>/dev/null || echo '?') + printf 'FAIL: %s container exited early (exit code %s)\n' "$APP" "$ec" >&2 + exit 1 + fi + status=$(docker inspect --format '{{ if .State.Health }}{{ .State.Health.Status }}{{ else }}no-healthcheck{{ end }}' "$NAME" 2>/dev/null || echo gone) + case "$status" in + healthy) + printf '%s image smoke: ok (healthy after %ss)\n' "$APP" "$i" + exit 0 + ;; + unhealthy) + printf 'FAIL: %s reported unhealthy\n' "$APP" >&2 + exit 1 + ;; + no-healthcheck) + printf 'FAIL: image has no HEALTHCHECK to assert against\n' >&2 + exit 1 + ;; + gone) + printf 'FAIL: %s container is gone\n' "$APP" >&2 + exit 1 + ;; + esac + i=$((i + 1)) + sleep 1 +done +printf 'FAIL: %s did not become healthy within %ss (last status: %s)\n' "$APP" "$TIMEOUT" "$status" >&2 +exit 1