From d1bc2016164556ab2e5ea15b550391e27a9f085e Mon Sep 17 00:00:00 2001 From: mhenrixon Date: Sat, 29 Aug 2026 00:40:56 +0200 Subject: [PATCH 1/2] fix(health-check): cap the pre-healthy backoff at 2s MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The pre-healthy backoff doubled from 50ms up to --health-check-interval. With a 20s interval the probe schedule reaches 12.75s then 25.55s, so a Rails app ready at 13s was not noticed until 25.55s — every deploy on cosmos took ~26s regardless of how fast the app booted. The steady-state interval says how often to re-check a target in service. Before the first success nothing is routed there and a probe is cheap, so clamp the pre-healthy delay to min(delay*2, 2s, interval). After the first success the interval governs exactly as before; sub-2s intervals still win. ## Test Coverage - TestHealthCheck_PreHealthyBackoffIsCappedBelowTheInterval: 20s interval, ready at 3.5s, healthy by 5.15s (uncapped: 6.35s); no extra probes after ## Verification - [x] gofmt -l internal/ cmd/ clean - [x] make test passes, go test -race on health check tests - [x] go vet ./... clean Closes #124 Claude-Session: https://claude.ai/code/session_01QkuqoL4xoqxxWkzo7zJgpt --- internal/server/health_check.go | 10 +++- internal/server/health_check_backoff_test.go | 49 ++++++++++++++++++++ 2 files changed, 58 insertions(+), 1 deletion(-) diff --git a/internal/server/health_check.go b/internal/server/health_check.go index 6f333dc..91ad52d 100644 --- a/internal/server/health_check.go +++ b/internal/server/health_check.go @@ -30,6 +30,14 @@ type HealthCheckConsumer interface { // immediately is noticed almost immediately. const initialHealthCheckDelay = 50 * time.Millisecond +// maxPreHealthyDelay caps the backoff before the first success. Two regimes: +// pre-healthy, nothing is routed to the target and a probe is cheap, so the +// delay doubles from initialHealthCheckDelay but never past this ceiling; once +// healthy, the configured interval governs. Without the cap a 20s interval let +// the pre-healthy gap grow to 12.75s and then 25.55s, so a target ready at 13s +// was not noticed until 25.55s. +const maxPreHealthyDelay = 2 * time.Second + type HealthCheck struct { consumer HealthCheckConsumer endpoint *url.URL @@ -99,7 +107,7 @@ func (hc *HealthCheck) run() { if hc.becameHealthy.Load() { delay = hc.interval } else { - delay = min(delay*2, hc.interval) + delay = min(delay*2, maxPreHealthyDelay, hc.interval) } timer.Reset(hc.nextDelay(delay)) diff --git a/internal/server/health_check_backoff_test.go b/internal/server/health_check_backoff_test.go index 0930720..7689100 100644 --- a/internal/server/health_check_backoff_test.go +++ b/internal/server/health_check_backoff_test.go @@ -142,3 +142,52 @@ func TestHealthCheck_BackoffIsBoundedByTheConfiguredInterval(t *testing.T) { assert.Less(t, probes.Load(), int64(12), "the retry must back off rather than hammer a container that is not coming up") } + +// The steady-state interval says how often to re-check a target that is in +// service. Before the first success nothing is routed to the target, so the +// backoff must not be allowed to grow to a large interval: with a 20s interval +// the uncapped schedule is 0.05, 0.15, 0.35, 0.75, 1.55, 3.15, 6.35, 12.75, +// 25.55s, and a Rails app ready at 13s is not noticed until 25.55s. +func TestHealthCheck_PreHealthyBackoffIsCappedBelowTheInterval(t *testing.T) { + var ready atomic.Bool + var probes atomic.Int64 + + backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + probes.Add(1) + if !ready.Load() { + w.WriteHeader(http.StatusServiceUnavailable) + return + } + w.WriteHeader(http.StatusOK) + })) + t.Cleanup(backend.Close) + + endpoint, err := url.Parse(backend.URL) + require.NoError(t, err) + + consumer := newRecordingConsumer() + start := time.Now() + + hc := NewHealthCheck(consumer, endpoint, 20*time.Second, time.Second, "") + t.Cleanup(hc.Close) + + // Ready after the 3.15s probe. Capped at 2s the next probe lands at 5.15s; + // uncapped it would be 6.35s. + time.Sleep(3500 * time.Millisecond) + ready.Store(true) + + select { + case <-consumer.healthy: + case <-time.After(6*time.Second - time.Since(start)): + t.Fatal("readiness waited for the uncapped backoff instead of the 2s ceiling") + } + + assert.Less(t, time.Since(start), 6*time.Second, + "a target that became ready must be noticed within the pre-healthy ceiling") + + // After the first success the configured interval governs again. + settled := probes.Load() + time.Sleep(500 * time.Millisecond) + assert.Equal(t, settled, probes.Load(), + "a healthy target must be probed at its configured interval, not the pre-healthy cadence") +} From d427be856f9a6904e08840d24d2007a89649319e Mon Sep 17 00:00:00 2001 From: mhenrixon Date: Sat, 29 Aug 2026 07:32:36 +0200 Subject: [PATCH 2/2] fix(health-check): bound the pre-healthy fast cadence to a 60s window MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A normal deploy disposes a target that misses its deploy timeout, but `deploy --force` skips that wait and installs the target unhealthy. With the 2s ceiling alone such a target would be probed at boot cadence for as long as it never answered. Past a 60s window the backoff resumes doubling toward the configured interval, as it did before the cap. The ceiling and window are fields with the constants as defaults so the test can shrink them; NewHealthCheck's signature is unchanged. ## Test Coverage - TestHealthCheck_FastWindowExpiresForATargetThatNeverBecomesHealthy: cap 100ms / window 300ms, 503 forever — 7 probes in 1.5s, 16 without the window ## Verification - [x] gofmt -l internal/ cmd/ clean, go vet ./... clean - [x] go test -race ./internal/server/ passes Claude-Session: https://claude.ai/code/session_01QkuqoL4xoqxxWkzo7zJgpt --- internal/server/health_check.go | 27 +++++++++++++++-- internal/server/health_check_backoff_test.go | 31 ++++++++++++++++++++ 2 files changed, 55 insertions(+), 3 deletions(-) diff --git a/internal/server/health_check.go b/internal/server/health_check.go index 91ad52d..722fe17 100644 --- a/internal/server/health_check.go +++ b/internal/server/health_check.go @@ -38,6 +38,13 @@ const initialHealthCheckDelay = 50 * time.Millisecond // was not noticed until 25.55s. const maxPreHealthyDelay = 2 * time.Second +// preHealthyFastWindow bounds how long the ceiling applies. A normal deploy +// disposes a target that misses its deploy timeout, but `deploy --force` skips +// that wait, and a target that never comes up must not be probed at boot +// cadence forever. Past the window the backoff resumes doubling toward the +// configured interval. +const preHealthyFastWindow = 60 * time.Second + type HealthCheck struct { consumer HealthCheckConsumer endpoint *url.URL @@ -45,6 +52,9 @@ type HealthCheck struct { timeout time.Duration host string + maxPreHealthyDelay time.Duration + preHealthyFastWindow time.Duration + ctx context.Context cancel context.CancelFunc @@ -54,6 +64,10 @@ type HealthCheck struct { } func NewHealthCheck(consumer HealthCheckConsumer, endpoint *url.URL, interval time.Duration, timeout time.Duration, host string) *HealthCheck { + return newHealthCheck(consumer, endpoint, interval, timeout, host, maxPreHealthyDelay, preHealthyFastWindow) +} + +func newHealthCheck(consumer HealthCheckConsumer, endpoint *url.URL, interval time.Duration, timeout time.Duration, host string, maxPreHealthyDelay time.Duration, preHealthyFastWindow time.Duration) *HealthCheck { ctx, cancel := context.WithCancel(context.Background()) hc := &HealthCheck{ @@ -63,6 +77,9 @@ func NewHealthCheck(consumer HealthCheckConsumer, endpoint *url.URL, interval ti timeout: timeout, host: host, + maxPreHealthyDelay: maxPreHealthyDelay, + preHealthyFastWindow: preHealthyFastWindow, + ctx: ctx, cancel: cancel, } @@ -90,6 +107,7 @@ func (hc *HealthCheck) Close() { // Once a target is healthy the configured interval governs, so a running target // is not probed any harder than before. func (hc *HealthCheck) run() { + started := time.Now() hc.check() timer := time.NewTimer(hc.nextDelay(initialHealthCheckDelay)) @@ -104,10 +122,13 @@ func (hc *HealthCheck) run() { case <-timer.C: hc.check() - if hc.becameHealthy.Load() { + switch { + case hc.becameHealthy.Load(): delay = hc.interval - } else { - delay = min(delay*2, maxPreHealthyDelay, hc.interval) + case time.Since(started) < hc.preHealthyFastWindow: + delay = min(delay*2, hc.maxPreHealthyDelay, hc.interval) + default: + delay = min(delay*2, hc.interval) } timer.Reset(hc.nextDelay(delay)) diff --git a/internal/server/health_check_backoff_test.go b/internal/server/health_check_backoff_test.go index 7689100..1212e21 100644 --- a/internal/server/health_check_backoff_test.go +++ b/internal/server/health_check_backoff_test.go @@ -191,3 +191,34 @@ func TestHealthCheck_PreHealthyBackoffIsCappedBelowTheInterval(t *testing.T) { assert.Equal(t, settled, probes.Load(), "a healthy target must be probed at its configured interval, not the pre-healthy cadence") } + +// The 2s ceiling is for catching a boot. A target that never comes up -- a +// `--force` deploy skips the wait that would otherwise dispose it -- must not be +// probed at boot cadence forever, so after the fast window the backoff resumes +// doubling toward the configured interval. +func TestHealthCheck_FastWindowExpiresForATargetThatNeverBecomesHealthy(t *testing.T) { + var probes atomic.Int64 + + backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + probes.Add(1) + w.WriteHeader(http.StatusServiceUnavailable) + })) + t.Cleanup(backend.Close) + + endpoint, err := url.Parse(backend.URL) + require.NoError(t, err) + + // Cap 100ms, window 300ms, interval 5s. Inside the window: 0, 50, 150, 250, + // 350ms. Past it the delay doubles: 550, 950, 1750ms. A cap that never + // expired would keep firing every 100ms -- ~15 probes in 1.5s instead of ~7. + hc := newHealthCheck(newRecordingConsumer(), endpoint, 5*time.Second, time.Second, "", + 100*time.Millisecond, 300*time.Millisecond) + t.Cleanup(hc.Close) + + time.Sleep(1500 * time.Millisecond) + + assert.Less(t, probes.Load(), int64(10), + "once the fast window has passed the backoff must resume growing toward the interval") + assert.Greater(t, probes.Load(), int64(3), + "the fast window must still have applied at the start") +}