Skip to content

Commit d4e0e33

Browse files
committed
fix(deploy): health checks probed localhost regardless of the bind address
Docker publishes a container on exactly the address it is given, so an app with a specific `bind:` is not reachable at localhost — but every health probe dialled localhost anyway. The probe therefore never connected, retried to the timeout, and failed the deploy while the container was perfectly healthy. `ingress: host` deploys by recreate, stopping the old container first, so the failure was not a no-op: the deploy tears down the new container and nothing is left running. Setting a bind address turned every subsequent deploy into a full outage. Found by doing it — dash went down moving from 0.0.0.0 to its tailnet address, with "timeout after 30s" as the only clue. The probe now resolves the address from the bind host (0.0.0.0/:: still mean localhost, as does an empty value, which is caddy/external ingress publishing on 127.0.0.1). Both the HTTP probe and the TCP fallback were affected. The same bug sat on the rollback and start/restart paths, where it would have been worse — a rollback that cannot health-check its target aborts and stops what it started, so the escape hatch failed exactly when it was needed. Those read the address back from the running container (docker.HostBindIP) since they have no config in hand. Verified live: the failing dash deploy now passes its health check, and dash answers on the tailnet while the LAN gets nothing.
1 parent 479d01d commit d4e0e33

7 files changed

Lines changed: 154 additions & 19 deletions

File tree

‎internal/deploy/deploy.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -340,7 +340,7 @@ func (d *Deployer) Deploy(ctx context.Context, cfg Config) error {
340340
fmt.Fprintln(d.out, "Running health check...")
341341
healthCfg := cfg.Health.withDefaults()
342342
for i, p := range ports {
343-
if err := d.healthCheck(ctx, p, healthCfg); err != nil {
343+
if err := d.healthCheck(ctx, p, healthCfg, webBindHost); err != nil {
344344
fmt.Fprintf(d.out, " Health check failed for replica %d (port %d): %v\n", i+1, p, err)
345345
return fail(fmt.Errorf("health check failed for replica %d: %w", i+1, err))
346346
}

‎internal/deploy/deploy_test.go‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -533,7 +533,7 @@ func TestHealthCheck_Pass(t *testing.T) {
533533
Interval: 10 * time.Millisecond,
534534
}
535535

536-
if err := d.healthCheck(context.Background(), 49152, cfg); err != nil {
536+
if err := d.healthCheck(context.Background(), 49152, cfg, ""); err != nil {
537537
t.Fatalf("healthCheck: %v", err)
538538
}
539539
}
@@ -553,7 +553,7 @@ func TestHealthCheck_TCPFallback(t *testing.T) {
553553
Interval: 10 * time.Millisecond,
554554
}
555555

556-
if err := d.healthCheck(context.Background(), 49152, cfg); err != nil {
556+
if err := d.healthCheck(context.Background(), 49152, cfg, ""); err != nil {
557557
t.Fatalf("healthCheck with TCP fallback: %v", err)
558558
}
559559
}
@@ -573,7 +573,7 @@ func TestHealthCheck_RedirectFallback(t *testing.T) {
573573
Interval: 10 * time.Millisecond,
574574
}
575575

576-
if err := d.healthCheck(context.Background(), 49152, cfg); err != nil {
576+
if err := d.healthCheck(context.Background(), 49152, cfg, ""); err != nil {
577577
t.Fatalf("healthCheck with redirect TCP fallback: %v", err)
578578
}
579579
}
@@ -590,7 +590,7 @@ func TestHealthCheck_Timeout(t *testing.T) {
590590
Interval: 10 * time.Millisecond,
591591
}
592592

593-
err := d.healthCheck(context.Background(), 49152, cfg)
593+
err := d.healthCheck(context.Background(), 49152, cfg, "")
594594
if err == nil {
595595
t.Fatal("expected timeout error")
596596
}

‎internal/deploy/health.go‎

Lines changed: 32 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -32,24 +32,45 @@ func (h HealthConfig) withDefaults() HealthConfig {
3232
return h
3333
}
3434

35+
// healthProbeHost is the address the server-side probe dials for a container
36+
// published on bindHost.
37+
//
38+
// Docker publishes on exactly the address it was given, so a container bound to
39+
// a specific IP is NOT reachable at localhost — probing there gets a connection
40+
// refused for the life of the timeout. Since `ingress: host` deploys by
41+
// recreate (the old container is stopped first), that turned every deploy of a
42+
// specifically-bound app into a full outage: the new container is healthy, the
43+
// probe cannot see it, the deploy fails, and nothing is left running.
44+
//
45+
// An empty bindHost is caddy/external ingress, which publishes on 127.0.0.1.
46+
func healthProbeHost(bindHost string) string {
47+
switch bindHost {
48+
case "", "0.0.0.0", "::", "[::]":
49+
return "localhost"
50+
default:
51+
return bindHost
52+
}
53+
}
54+
3555
// healthCheck polls the container until it responds healthy or the timeout expires.
3656
//
3757
// Strategy:
38-
// 1. HTTP GET to localhost:{port}{path} — 200 means healthy.
58+
// 1. HTTP GET to {host}:{port}{path} — 200 means healthy.
3959
// 2. If the endpoint returns 404, fall back to a TCP port check.
4060
// 3. Connection refused means the app hasn't started yet — retry.
41-
func (d *Deployer) healthCheck(ctx context.Context, port int, cfg HealthConfig) error {
61+
func (d *Deployer) healthCheck(ctx context.Context, port int, cfg HealthConfig, bindHost string) error {
4262
ctx, cancel := context.WithTimeout(ctx, cfg.Timeout)
4363
defer cancel()
4464

65+
host := healthProbeHost(bindHost)
4566
for {
46-
if d.checkHealth(ctx, port, cfg.Path) {
67+
if d.checkHealth(ctx, host, port, cfg.Path) {
4768
return nil
4869
}
4970

5071
select {
5172
case <-ctx.Done():
52-
return fmt.Errorf("timeout after %s waiting for health check on port %d", cfg.Timeout, port)
73+
return fmt.Errorf("timeout after %s waiting for health check on %s:%d", cfg.Timeout, host, port)
5374
case <-time.After(cfg.Interval):
5475
// retry
5576
}
@@ -59,14 +80,14 @@ func (d *Deployer) healthCheck(ctx context.Context, port int, cfg HealthConfig)
5980
// HealthCheckPublic runs a health check against the given port using default settings.
6081
// This is the public entry point for on-demand health checks.
6182
func (d *Deployer) HealthCheckPublic(ctx context.Context, port int) error {
62-
return d.healthCheck(ctx, port, defaultHealthConfig())
83+
return d.healthCheck(ctx, port, defaultHealthConfig(), "")
6384
}
6485

6586
// checkHealth performs a single health check attempt.
66-
func (d *Deployer) checkHealth(ctx context.Context, port int, path string) bool {
87+
func (d *Deployer) checkHealth(ctx context.Context, host string, port int, path string) bool {
6788
cmd := fmt.Sprintf(
68-
"curl -s -o /dev/null -w '%%{http_code}' http://localhost:%d%s",
69-
port, path,
89+
"curl -s -o /dev/null -w '%%{http_code}' http://%s:%d%s",
90+
host, port, path,
7091
)
7192
output, err := d.exec.Run(ctx, cmd)
7293
if err == nil {
@@ -80,15 +101,15 @@ func (d *Deployer) checkHealth(ctx context.Context, port int, path string) bool
80101
// check rather than failing the deploy. A 5xx or "000" (no response)
81102
// falls through and is retried until the timeout.
82103
if code == "404" || strings.HasPrefix(code, "3") {
83-
return d.checkTCP(ctx, port)
104+
return d.checkTCP(ctx, host, port)
84105
}
85106
}
86107
return false
87108
}
88109

89110
// checkTCP verifies that a TCP connection can be established to the port.
90-
func (d *Deployer) checkTCP(ctx context.Context, port int) bool {
91-
cmd := fmt.Sprintf("bash -c '</dev/tcp/localhost/%d' 2>/dev/null", port)
111+
func (d *Deployer) checkTCP(ctx context.Context, host string, port int) bool {
112+
cmd := fmt.Sprintf("bash -c '</dev/tcp/%s/%d' 2>/dev/null", host, port)
92113
_, err := d.exec.Run(ctx, cmd)
93114
return err == nil
94115
}
Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
package deploy
2+
3+
import (
4+
"context"
5+
"strings"
6+
"testing"
7+
8+
"github.com/useteploy/teploy/internal/ssh"
9+
)
10+
11+
// A container published on a specific IP is NOT reachable at localhost, so a
12+
// health probe hardcoded to localhost can never succeed. Because `ingress:
13+
// host` deploys by recreate (old container stopped first), that turned every
14+
// deploy of a specifically-bound app into a full outage: app healthy, probe
15+
// blind, deploy failed, nothing left running. Observed live on 2026-08-08
16+
// when dash moved from bind 0.0.0.0 to its tailnet address.
17+
func TestHealthProbeHost(t *testing.T) {
18+
cases := map[string]string{
19+
"": "localhost", // caddy/external ingress publishes on 127.0.0.1
20+
"0.0.0.0": "localhost",
21+
"::": "localhost",
22+
"[::]": "localhost",
23+
"100.108.123.49": "100.108.123.49", // tailnet-bound: must be probed there
24+
"192.168.1.84": "192.168.1.84",
25+
"127.0.0.1": "127.0.0.1",
26+
}
27+
for bind, want := range cases {
28+
if got := healthProbeHost(bind); got != want {
29+
t.Errorf("healthProbeHost(%q) = %q, want %q", bind, got, want)
30+
}
31+
}
32+
}
33+
34+
func TestHealthCheck_ProbesTheBoundAddressNotLocalhost(t *testing.T) {
35+
mock := ssh.NewMockExecutor("h", ssh.MockCommand{Match: "curl", Output: "200"})
36+
d := &Deployer{exec: mock, out: nopWriter{}}
37+
38+
if err := d.healthCheck(context.Background(), 3456, defaultHealthConfig(), "100.108.123.49"); err != nil {
39+
t.Fatalf("healthCheck: %v", err)
40+
}
41+
var probed string
42+
for _, c := range mock.Calls {
43+
if strings.HasPrefix(c, "curl") {
44+
probed = c
45+
}
46+
}
47+
if !strings.Contains(probed, "http://100.108.123.49:3456") {
48+
t.Errorf("probe should dial the bound address, got: %s", probed)
49+
}
50+
if strings.Contains(probed, "localhost") {
51+
t.Errorf("probe must not fall back to localhost for a specifically-bound app: %s", probed)
52+
}
53+
}
54+
55+
// The TCP fallback (used when the health path 404s or redirects) has to dial
56+
// the same address, or a bound app with no /health endpoint still fails.
57+
func TestHealthCheck_TCPFallbackAlsoUsesTheBoundAddress(t *testing.T) {
58+
mock := ssh.NewMockExecutor("h",
59+
ssh.MockCommand{Match: "curl", Output: "404"},
60+
ssh.MockCommand{Match: "bash -c", Output: ""},
61+
)
62+
d := &Deployer{exec: mock, out: nopWriter{}}
63+
64+
if err := d.healthCheck(context.Background(), 3456, defaultHealthConfig(), "100.108.123.49"); err != nil {
65+
t.Fatalf("healthCheck with TCP fallback: %v", err)
66+
}
67+
var tcp string
68+
for _, c := range mock.Calls {
69+
if strings.HasPrefix(c, "bash -c") {
70+
tcp = c
71+
}
72+
}
73+
if !strings.Contains(tcp, "/dev/tcp/100.108.123.49/3456") {
74+
t.Errorf("TCP fallback should dial the bound address, got: %s", tcp)
75+
}
76+
}
77+
78+
type nopWriter struct{}
79+
80+
func (nopWriter) Write(p []byte) (int, error) { return len(p), nil }

‎internal/deploy/lifecycle.go‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -75,7 +75,9 @@ func (l *Lifecycle) Start(ctx context.Context, app string) error {
7575
if current.CurrentPort > 0 {
7676
fmt.Fprintln(l.out, "Running health check...")
7777
deployer := &Deployer{exec: l.exec, out: l.out}
78-
if err := deployer.healthCheck(ctx, current.CurrentPort, defaultHealthConfig()); err != nil {
78+
// Probe the bound address, not localhost — see healthProbeHost.
79+
bindHost := l.docker.HostBindIP(ctx, containers[0].Name)
80+
if err := deployer.healthCheck(ctx, current.CurrentPort, defaultHealthConfig(), bindHost); err != nil {
7981
return fmt.Errorf("health check failed after start: %w", err)
8082
}
8183
fmt.Fprintln(l.out, " Health check passed")
@@ -121,7 +123,9 @@ func (l *Lifecycle) Restart(ctx context.Context, app string, timeout int) error
121123
if current.CurrentPort > 0 {
122124
fmt.Fprintln(l.out, "Running health check...")
123125
deployer := &Deployer{exec: l.exec, out: l.out}
124-
if err := deployer.healthCheck(ctx, current.CurrentPort, defaultHealthConfig()); err != nil {
126+
// Probe the bound address, not localhost — see healthProbeHost.
127+
bindHost := l.docker.HostBindIP(ctx, containers[0].Name)
128+
if err := deployer.healthCheck(ctx, current.CurrentPort, defaultHealthConfig(), bindHost); err != nil {
125129
return fmt.Errorf("health check failed after restart: %w", err)
126130
}
127131
fmt.Fprintln(l.out, " Health check passed")

‎internal/deploy/rollback.go‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -210,15 +210,22 @@ func Rollback(ctx context.Context, exec ssh.Executor, out io.Writer, cfg Rollbac
210210
fmt.Fprintln(out, "Running health check...")
211211
deployer := &Deployer{exec: exec, out: out}
212212
healthPorts := make([]int, 0, len(targetWeb))
213+
// Probe the address docker actually bound, not localhost: a container
214+
// published on a specific IP is unreachable there, which would fail every
215+
// rollback of a host-ingress app with a bind address.
216+
healthBindHost := ""
213217
for _, c := range targetWeb {
214218
p, err := dk.HostPort(ctx, c.Name)
215219
if err != nil {
216220
return fmt.Errorf("inspecting target container host port: %w", err)
217221
}
218222
healthPorts = append(healthPorts, p)
223+
if healthBindHost == "" {
224+
healthBindHost = dk.HostBindIP(ctx, c.Name)
225+
}
219226
}
220227
for _, p := range healthPorts {
221-
if err := deployer.healthCheck(ctx, p, healthCfg); err != nil {
228+
if err := deployer.healthCheck(ctx, p, healthCfg, healthBindHost); err != nil {
222229
// Stop what we started and bail.
223230
for _, name := range started {
224231
dk.Stop(ctx, name, 5)

‎internal/docker/docker.go‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -628,6 +628,29 @@ func (c *Client) HostPort(ctx context.Context, name string) (int, error) {
628628
return port, nil
629629
}
630630

631+
// HostBindIP returns the host IP a container's first published port is bound
632+
// to ("0.0.0.0" for all interfaces, or a specific address).
633+
//
634+
// Needed because a container published on a specific IP is not reachable at
635+
// localhost, so anything probing it — health checks on rollback and restart —
636+
// must dial the address docker actually bound. Returns "" when it cannot be
637+
// determined, which callers treat as "assume localhost", the historical
638+
// behavior.
639+
func (c *Client) HostBindIP(ctx context.Context, name string) string {
640+
out, err := c.exec.Run(ctx, fmt.Sprintf(
641+
"docker inspect -f '{{range $p, $b := .NetworkSettings.Ports}}{{range $b}}{{.HostIp}} {{end}}{{end}}' %s",
642+
ssh.ShellQuote(name),
643+
))
644+
if err != nil {
645+
return ""
646+
}
647+
fields := strings.Fields(out)
648+
if len(fields) == 0 {
649+
return ""
650+
}
651+
return fields[0]
652+
}
653+
631654
// InternalPort returns the container's internal listening port — the port
632655
// the app speaks HTTP on inside the Docker network, not the host-mapped
633656
// port. Caddy dials this port when reverse-proxying over the teploy

0 commit comments

Comments
 (0)