fix(health): split liveness from readiness and make reachability configurable

.env.example

@@ -47,6 +47,16 @@ AUTH_DISABLED=false
 # Comma-separated CIDR ranges allowed when AUTH_DISABLED=true
 # TRUSTED_NETWORKS=172.16.0.0/12,10.0.0.0/8,192.168.0.0/16,127.0.0.1,::1
 
+# Comma-separated CIDR ranges, beyond loopback, allowed to reach /health
+# (liveness) and /health/ready (readiness). Empty means loopback only, which is
+# all the container's own HEALTHCHECK needs. Set this to let an external uptime
+# monitor or load balancer probe Capstan.
+#
+# Deliberately separate from TRUSTED_NETWORKS: that value is also the
+# trusted-proxy list and the AUTH_DISABLED bypass list, so reusing it would
+# grant a monitor X-Forwarded-For spoofing and auth bypass just to read health.
+# HEALTH_ALLOWED_NETWORKS=10.1.0.0/16
+
 # CORS origins (comma-separated, empty = same-origin only)
 # CORS_ORIGINS=https://your-domain.com
 

README.md

@@ -574,11 +574,49 @@ npm run build              # production build
 ## API Endpoints
 
 All routes are under `/api/v1` and require authentication (unless
-`AUTH_DISABLED=true`), except `/health` and `/api/v1/version`. The web UI is the
-primary interface; these are the main REST routes.
+`AUTH_DISABLED=true`), except `/health`, `/health/ready` and `/api/v1/version`.
+The web UI is the primary interface; these are the main REST routes.
 
 ### Health
-- `GET /health` — health check (restricted to localhost)
+
+Liveness and readiness are separate endpoints, because Capstan is a separate
+process from Docker. Point restart-on-failure checks at liveness and dependency
+monitoring at readiness.
+
+- `GET /health` — **liveness**. 200 whenever the process is up and serving. Makes
+  no Docker call, so a Docker daemon restart cannot mark the container unhealthy
+  and get Capstan bounced for an outage restarting it would not fix. This is what
+  the container `HEALTHCHECK`, and any orchestrator liveness probe, should use.
+- `GET /health/ready` — **readiness**. Reports dependencies, 503 when any is
+  degraded, naming which. The Docker probe is bounded by a 2-second timeout so a
+  hung daemon cannot pile up goroutines across repeated probes. Point an uptime
+  monitor, a load balancer, or a Kubernetes readiness probe here.
+
+```console
+$ curl -s localhost:5001/health
+{"status":"healthy"}
+
+$ curl -s localhost:5001/health/ready          # Docker up
+{"checks":{"docker":{"status":"ok"}},"status":"ready"}
+
+$ curl -s localhost:5001/health/ready          # Docker daemon stopped -> HTTP 503
+{"checks":{"docker":{"error":"Cannot connect to the Docker daemon at unix:///var/run/docker.sock.",
+"status":"unavailable"}},"degraded":["docker"],"status":"degraded"}
+```
+
+**Reachability.** Loopback reaches both with no configuration, so the container's
+own healthcheck needs no setup. Every other address is denied by default —
+upgrading does not silently widen exposure. To let an external monitor in, list
+its network in `HEALTH_ALLOWED_NETWORKS`:
+
+```bash
+HEALTH_ALLOWED_NETWORKS=10.1.0.0/16,192.168.50.7
+```
+
+This is deliberately **not** `TRUSTED_NETWORKS`. That value is Gin's
+trusted-proxy list *and* the `AUTH_DISABLED` bypass list, so reusing it would
+mean granting an uptime monitor `X-Forwarded-For` spoofing and authentication
+bypass just to let it read a health endpoint.
 
 ### Version
 - `GET /api/v1/version` — build identity of the running binary. Public, so an

backend/cmd/server/main.go

@@ -7,7 +7,6 @@ import (
 	"fmt"
 	"log"
 	"log/slog"
-	"net"
 	"net/http"
 	"os"
 	"os/signal"
@@ -79,14 +78,6 @@ func SecurityHeaders(cfg *config.Config) gin.HandlerFunc {
 	}
 }
 
-func isLocalhost(c *gin.Context) bool {
-	ip := net.ParseIP(c.ClientIP())
-	if ip == nil {
-		return false
-	}
-	return ip.IsLoopback()
-}
-
 func main() {
 	// Bootstrap the logger from the environment before config.Load, so that
 	// config's own startup lines — including the volume-path-identity warning —
@@ -224,32 +215,15 @@ func main() {
 	r.Use(middleware.ValidateInput())
 	r.Use(SecurityHeaders(cfg))
 
-	r.GET("/health", func(c *gin.Context) {
-		if !isLocalhost(c) {
-			c.JSON(http.StatusForbidden, gin.H{"error": "Health endpoint restricted to localhost"})
-			return
-		}
-
-		dockerHealthy := false
-
-		if dockerService != nil {
-			_, err := dockerService.GetContainerList("")
-			if err == nil {
-				dockerHealthy = true
-			}
-		}
-
-		if !dockerHealthy {
-			c.JSON(http.StatusServiceUnavailable, gin.H{
-				"status": "unhealthy",
-			})
-			return
-		}
-
-		c.JSON(http.StatusOK, gin.H{
-			"status": "healthy",
-		})
-	})
+	// Liveness and readiness. dockerService is a typed nil when the daemon was
+	// unreachable at startup; hand the handler an untyped nil so it reports the
+	// outage rather than calling through a non-nil interface holding a nil
+	// pointer (agent-os-69a).
+	var dockerPinger handlers.DockerPinger
+	if dockerService != nil {
+		dockerPinger = dockerService
+	}
+	handlers.NewHealthHandler(dockerPinger, cfg.HealthNetworks).RegisterRoutes(r)
 
 	api := r.Group("/api/v1")
 

backend/internal/config/config.go

@@ -30,6 +30,14 @@ type Config struct {
 	AuthDisabled    bool
 	CORSOrigins     string
 	TrustedNetworks string
+	// HealthNetworks lists the CIDRs, beyond loopback, that may reach /health
+	// and /health/ready. Deliberately separate from TrustedNetworks: that value
+	// is Gin's trusted-proxy list *and* the AUTH_DISABLED bypass list, so
+	// reusing it would force an operator to grant an uptime monitor
+	// X-Forwarded-For spoofing and auth bypass just to read a health endpoint
+	// (agent-os-69a). Empty means loopback only, which is the pre-split
+	// behaviour.
+	HealthNetworks  string
 	ExtraStacksDirs []string
 
 	// Backup / restic env-var fallbacks (DB settings take precedence at runtime).
@@ -57,6 +65,7 @@ func Load() (*Config, error) {
 		GitHTTPSUser:    "git",
 		AuthDisabled:    os.Getenv("AUTH_DISABLED") == "true",
 		TrustedNetworks: os.Getenv("TRUSTED_NETWORKS"),
+		HealthNetworks:  os.Getenv("HEALTH_ALLOWED_NETWORKS"),
 	}
 
 	if stacksDir := os.Getenv("STACKS_DIR"); stacksDir != "" {

backend/internal/handlers/health.go

@@ -0,0 +1,144 @@
+package handlers
+
+import (
+	"context"
+	"log/slog"
+	"net"
+	"net/http"
+	"time"
+
+	"github.com/gin-gonic/gin"
+	"github.com/thinkbig1979/capstan/backend/internal/middleware"
+)
+
+// readinessProbeTimeout bounds the dependency check. A hung Docker daemon
+// otherwise parks a goroutine per probe, and repeated probes pile them up.
+const readinessProbeTimeout = 2 * time.Second
+
+// DockerPinger is the readiness handler's view of the Docker service: one cheap,
+// context-aware round-trip. Narrow on purpose, so the handler is testable
+// without a daemon.
+type DockerPinger interface {
+	Ping(ctx context.Context) error
+}
+
+// HealthHandler serves liveness and readiness.
+//
+// The split exists because Capstan is a separate process from Docker. The old
+// combined endpoint returned 503 whenever the daemon was unreachable, so a
+// daemon restart marked the container unhealthy and anything that restarts on
+// failed health checks would bounce Capstan — which does nothing to fix Docker
+// (agent-os-69a).
+//
+//	GET /health       liveness  — the process is up and serving. No Docker call.
+//	GET /health/ready readiness — reports dependencies; 503 names what is degraded.
+type HealthHandler struct {
+	// docker is nil when the Docker service could not be created at startup.
+	// That is a legitimate degraded state, not an error to hide.
+	docker          DockerPinger
+	allowedNetworks string
+	probeTimeout    time.Duration
+}
+
+// NewHealthHandler builds the handler. Pass a nil docker when the Docker service
+// is unavailable; readiness then reports Docker as degraded rather than
+// pretending to be ready.
+//
+// allowedNetworks is HEALTH_ALLOWED_NETWORKS: comma-separated CIDRs, beyond
+// loopback, permitted to reach either endpoint. Empty means loopback only, which
+// is the pre-split behaviour — an upgrade must not silently widen exposure.
+func NewHealthHandler(docker DockerPinger, allowedNetworks string) *HealthHandler {
+	return &HealthHandler{
+		docker:          docker,
+		allowedNetworks: allowedNetworks,
+		probeTimeout:    readinessProbeTimeout,
+	}
+}
+
+// RegisterRoutes mounts both endpoints on the root router. They sit outside the
+// protected group deliberately: a probe has no session. Both paths are listed in
+// middleware.PublicPaths so auth and CSRF agree with that.
+func (h *HealthHandler) RegisterRoutes(r gin.IRoutes) {
+	r.GET("/health", h.Live)
+	r.GET("/health/ready", h.Ready)
+}
+
+// allowed enforces the network policy shared by both endpoints, writing the 403
+// itself. Loopback always passes, so the container's own HEALTHCHECK needs no
+// configuration.
+func (h *HealthHandler) allowed(c *gin.Context) bool {
+	clientIP := c.ClientIP()
+
+	// The whole loopback range, not just the literal 127.0.0.1/::1 that
+	// middleware.IsTrustedIP matches. The handler this replaced used
+	// net.IP.IsLoopback, and narrowing that here would deny a probe bound to,
+	// say, 127.0.0.2 that works today.
+	if ip := net.ParseIP(clientIP); ip != nil && ip.IsLoopback() {
+		return true
+	}
+
+	if middleware.IsTrustedIP(clientIP, h.allowedNetworks) {
+		return true
+	}
+	c.JSON(http.StatusForbidden, gin.H{
+		"error": "Health endpoint restricted; add this network to HEALTH_ALLOWED_NETWORKS to permit it",
+	})
+	return false
+}
+
+// Live answers whether the process is up and serving. It must not touch any
+// dependency: the container HEALTHCHECK points here, and a dependency failure
+// answered from this endpoint gets the container restarted for someone else's
+// outage.
+//
+// The body keeps the pre-split {"status":"healthy"} so existing monitors that
+// match on it keep working. What changed is that it no longer returns 503.
+func (h *HealthHandler) Live(c *gin.Context) {
+	if !h.allowed(c) {
+		return
+	}
+	c.JSON(http.StatusOK, gin.H{"status": "healthy"})
+}
+
+// Ready reports the dependencies, 503 when any is degraded, naming which.
+func (h *HealthHandler) Ready(c *gin.Context) {
+	if !h.allowed(c) {
+		return
+	}
+
+	timeout := h.probeTimeout
+	if timeout <= 0 {
+		timeout = readinessProbeTimeout
+	}
+	ctx, cancel := context.WithTimeout(c.Request.Context(), timeout)
+	defer cancel()
+
+	dockerCheck := gin.H{"status": "ok"}
+	var degraded []string
+
+	switch {
+	case h.docker == nil:
+		degraded = append(degraded, "docker")
+		dockerCheck = gin.H{"status": "unavailable", "error": "docker service not initialized at startup"}
+	default:
+		if err := h.docker.Ping(ctx); err != nil {
+			degraded = append(degraded, "docker")
+			dockerCheck = gin.H{"status": "unavailable", "error": err.Error()}
+		}
+	}
+
+	body := gin.H{
+		"status": "ready",
+		"checks": gin.H{"docker": dockerCheck},
+	}
+
+	if len(degraded) > 0 {
+		body["status"] = "degraded"
+		body["degraded"] = degraded
+		slog.Warn("Readiness probe degraded", "degraded", degraded)
+		c.JSON(http.StatusServiceUnavailable, body)
+		return
+	}
+
+	c.JSON(http.StatusOK, body)
+}