fix(health-check): cap the pre-healthy backoff at 2s

internal/server/health_check.go

@@ -30,13 +30,31 @@ type HealthCheckConsumer interface {
 // immediately is noticed almost immediately.
 const initialHealthCheckDelay = 50 * time.Millisecond
 
+// maxPreHealthyDelay caps the backoff before the first success. Two regimes:
+// pre-healthy, nothing is routed to the target and a probe is cheap, so the
+// delay doubles from initialHealthCheckDelay but never past this ceiling; once
+// healthy, the configured interval governs. Without the cap a 20s interval let
+// the pre-healthy gap grow to 12.75s and then 25.55s, so a target ready at 13s
+// was not noticed until 25.55s.
+const maxPreHealthyDelay = 2 * time.Second
+
+// preHealthyFastWindow bounds how long the ceiling applies. A normal deploy
+// disposes a target that misses its deploy timeout, but `deploy --force` skips
+// that wait, and a target that never comes up must not be probed at boot
+// cadence forever. Past the window the backoff resumes doubling toward the
+// configured interval.
+const preHealthyFastWindow = 60 * time.Second
+
 type HealthCheck struct {
 	consumer HealthCheckConsumer
 	endpoint *url.URL
 	interval time.Duration
 	timeout  time.Duration
 	host     string
 
+	maxPreHealthyDelay   time.Duration
+	preHealthyFastWindow time.Duration
+
 	ctx    context.Context
 	cancel context.CancelFunc
 
@@ -46,6 +64,10 @@ type HealthCheck struct {
 }
 
 func NewHealthCheck(consumer HealthCheckConsumer, endpoint *url.URL, interval time.Duration, timeout time.Duration, host string) *HealthCheck {
+	return newHealthCheck(consumer, endpoint, interval, timeout, host, maxPreHealthyDelay, preHealthyFastWindow)
+}
+
+func newHealthCheck(consumer HealthCheckConsumer, endpoint *url.URL, interval time.Duration, timeout time.Duration, host string, maxPreHealthyDelay time.Duration, preHealthyFastWindow time.Duration) *HealthCheck {
 	ctx, cancel := context.WithCancel(context.Background())
 
 	hc := &HealthCheck{
@@ -55,6 +77,9 @@ func NewHealthCheck(consumer HealthCheckConsumer, endpoint *url.URL, interval ti
 		timeout:  timeout,
 		host:     host,
 
+		maxPreHealthyDelay:   maxPreHealthyDelay,
+		preHealthyFastWindow: preHealthyFastWindow,
+
 		ctx:    ctx,
 		cancel: cancel,
 	}
@@ -82,6 +107,7 @@ func (hc *HealthCheck) Close() {
 // Once a target is healthy the configured interval governs, so a running target
 // is not probed any harder than before.
 func (hc *HealthCheck) run() {
+	started := time.Now()
 	hc.check()
 
 	timer := time.NewTimer(hc.nextDelay(initialHealthCheckDelay))
@@ -96,9 +122,12 @@ func (hc *HealthCheck) run() {
 		case <-timer.C:
 			hc.check()
 
-			if hc.becameHealthy.Load() {
+			switch {
+			case hc.becameHealthy.Load():
 				delay = hc.interval
-			} else {
+			case time.Since(started) < hc.preHealthyFastWindow:
+				delay = min(delay*2, hc.maxPreHealthyDelay, hc.interval)
+			default:
 				delay = min(delay*2, hc.interval)
 			}
 

internal/server/health_check_backoff_test.go

@@ -142,3 +142,83 @@ func TestHealthCheck_BackoffIsBoundedByTheConfiguredInterval(t *testing.T) {
 	assert.Less(t, probes.Load(), int64(12),
 		"the retry must back off rather than hammer a container that is not coming up")
 }
+
+// The steady-state interval says how often to re-check a target that is in
+// service. Before the first success nothing is routed to the target, so the
+// backoff must not be allowed to grow to a large interval: with a 20s interval
+// the uncapped schedule is 0.05, 0.15, 0.35, 0.75, 1.55, 3.15, 6.35, 12.75,
+// 25.55s, and a Rails app ready at 13s is not noticed until 25.55s.
+func TestHealthCheck_PreHealthyBackoffIsCappedBelowTheInterval(t *testing.T) {
+	var ready atomic.Bool
+	var probes atomic.Int64
+
+	backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		probes.Add(1)
+		if !ready.Load() {
+			w.WriteHeader(http.StatusServiceUnavailable)
+			return
+		}
+		w.WriteHeader(http.StatusOK)
+	}))
+	t.Cleanup(backend.Close)
+
+	endpoint, err := url.Parse(backend.URL)
+	require.NoError(t, err)
+
+	consumer := newRecordingConsumer()
+	start := time.Now()
+
+	hc := NewHealthCheck(consumer, endpoint, 20*time.Second, time.Second, "")
+	t.Cleanup(hc.Close)
+
+	// Ready after the 3.15s probe. Capped at 2s the next probe lands at 5.15s;
+	// uncapped it would be 6.35s.
+	time.Sleep(3500 * time.Millisecond)
+	ready.Store(true)
+
+	select {
+	case <-consumer.healthy:
+	case <-time.After(6*time.Second - time.Since(start)):
+		t.Fatal("readiness waited for the uncapped backoff instead of the 2s ceiling")
+	}
+
+	assert.Less(t, time.Since(start), 6*time.Second,
+		"a target that became ready must be noticed within the pre-healthy ceiling")
+
+	// After the first success the configured interval governs again.
+	settled := probes.Load()
+	time.Sleep(500 * time.Millisecond)
+	assert.Equal(t, settled, probes.Load(),
+		"a healthy target must be probed at its configured interval, not the pre-healthy cadence")
+}
+
+// The 2s ceiling is for catching a boot. A target that never comes up -- a
+// `--force` deploy skips the wait that would otherwise dispose it -- must not be
+// probed at boot cadence forever, so after the fast window the backoff resumes
+// doubling toward the configured interval.
+func TestHealthCheck_FastWindowExpiresForATargetThatNeverBecomesHealthy(t *testing.T) {
+	var probes atomic.Int64
+
+	backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		probes.Add(1)
+		w.WriteHeader(http.StatusServiceUnavailable)
+	}))
+	t.Cleanup(backend.Close)
+
+	endpoint, err := url.Parse(backend.URL)
+	require.NoError(t, err)
+
+	// Cap 100ms, window 300ms, interval 5s. Inside the window: 0, 50, 150, 250,
+	// 350ms. Past it the delay doubles: 550, 950, 1750ms. A cap that never
+	// expired would keep firing every 100ms -- ~15 probes in 1.5s instead of ~7.
+	hc := newHealthCheck(newRecordingConsumer(), endpoint, 5*time.Second, time.Second, "",
+		100*time.Millisecond, 300*time.Millisecond)
+	t.Cleanup(hc.Close)
+
+	time.Sleep(1500 * time.Millisecond)
+
+	assert.Less(t, probes.Load(), int64(10),
+		"once the fast window has passed the backoff must resume growing toward the interval")
+	assert.Greater(t, probes.Load(), int64(3),
+		"the fast window must still have applied at the start")
+}