From cb6928873cfdd2dcffd1ad91c53c992811f82dd8 Mon Sep 17 00:00:00 2001 From: AlisinaDevelo Date: Mon, 7 Sep 2026 12:56:23 +0200 Subject: [PATCH] fix(security): require auth for non-loopback health --- docker-compose.lifecycle.yml | 6 +++ docker-compose.yml | 6 +++ docs/DEPLOYMENT.md | 24 +++++++++--- docs/PROTOCOL.md | 6 ++- docs/TLS_AUTH.md | 8 +++- health_server_acceptance_test.go | 63 +++++++++++++++++++++++++++++++ main.go | 44 +++++++++++++++++++-- scripts/demo-auth.sh | 18 ++++++--- scripts/demo-compose.sh | 12 ++++-- scripts/demo-failure.sh | 12 ++++-- scripts/demo-lifecycle-compose.sh | 14 +++++-- scripts/demo-repair.sh | 14 +++++-- scripts/demo-status.sh | 9 ++++- 13 files changed, 204 insertions(+), 32 deletions(-) diff --git a/docker-compose.lifecycle.yml b/docker-compose.lifecycle.yml index 7eea7b4..89cd93e 100644 --- a/docker-compose.lifecycle.yml +++ b/docker-compose.lifecycle.yml @@ -7,6 +7,8 @@ services: - 0.0.0.0:7070 - -health - 0.0.0.0:8080 + - -health-auth-token + - "${STREAMHIVE_LIFECYCLE_HEALTH_TOKEN:-streamhive-lifecycle-health-demo-token}" - -replicate - -store-dir - /data @@ -43,6 +45,8 @@ services: - 0.0.0.0:7070 - -health - 0.0.0.0:8080 + - -health-auth-token + - "${STREAMHIVE_LIFECYCLE_HEALTH_TOKEN:-streamhive-lifecycle-health-demo-token}" - -replicate - -store-dir - /data @@ -79,6 +83,8 @@ services: - 0.0.0.0:7070 - -health - 0.0.0.0:8080 + - -health-auth-token + - "${STREAMHIVE_LIFECYCLE_HEALTH_TOKEN:-streamhive-lifecycle-health-demo-token}" - -replicate - -store-dir - /data diff --git a/docker-compose.yml b/docker-compose.yml index cf0ba99..43495e4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,6 +7,8 @@ services: - 0.0.0.0:7070 - -health - 0.0.0.0:8080 + - -health-auth-token + - "${STREAMHIVE_HEALTH_TOKEN:-streamhive-health-demo-token}" - -replicate - -store-dir - /data @@ -37,6 +39,8 @@ services: - 0.0.0.0:7070 - -health - 0.0.0.0:8080 + - -health-auth-token + - "${STREAMHIVE_HEALTH_TOKEN:-streamhive-health-demo-token}" - -replicate - -store-dir - /data @@ -67,6 +71,8 @@ services: - 0.0.0.0:7070 - -health - 0.0.0.0:8080 + - -health-auth-token + - "${STREAMHIVE_HEALTH_TOKEN:-streamhive-health-demo-token}" - -replicate - -store-dir - /data diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md index 8ace51a..a981c55 100644 --- a/docs/DEPLOYMENT.md +++ b/docs/DEPLOYMENT.md @@ -6,15 +6,23 @@ Build and run (example flags): ```bash docker build -t streamhive:local . -docker run --rm -p 7070:7070 -p 8080:8080 streamhive:local \ +docker run --rm -e STREAMHIVE_HEALTH_TOKEN=change-me -p 7070:7070 -p 8080:8080 streamhive:local \ -listen 0.0.0.0:7070 \ - -health 0.0.0.0:8080 + -health 0.0.0.0:8080 \ + -health-auth-token change-me ``` - **7070** — P2P TCP listener (example). - **8080** — HTTP `/livez`, `/readyz`, `/version` (aggregate runtime identity), `/peers` (JSON peer metadata), `/metrics` (JSON counters), `/metrics/prometheus` (Prometheus text), `/inventory/status` (aggregate live key fingerprint), `/storage/status` (aggregate live blob integrity), and `/lifecycle/status` (aggregate opt-in lifecycle state). -Use TLS flags (`-tls-cert`, `-tls-key`, `-tls-ca`, `-tls-server-name`) when exposing services beyond a lab network. The verified certificate and application-auth ordering is documented in [TLS_AUTH.md](TLS_AUTH.md) and exercised by `make test-tls-auth`. Reserve `-tls-insecure-skip-verify` for local development. For CLI mTLS, add `-tls-client-ca -tls-require-client-cert` on the listener and `-tls-client-cert -tls-client-key` on outbound peers. Use `-tls-expiry-warning` to set the aggregate short-lived-credential warning window (`720h` by default, `0` disables the warning). For custom trust policy, configure `p2p.TCPTransport.TLSServerConfig` and `TLSClientConfig` in library code. +The health listener is plain HTTP and loopback-first. `/livez` and `/readyz` remain unauthenticated for local and orchestrator probes; diagnostic routes require `Authorization: Bearer ` whenever `-health-auth-token` is configured. A non-loopback `-health` address requires `-health-auth-token` and fails closed without one. For example, an explicitly network-scoped lab listener is: + +```bash +go run . -listen 0.0.0.0:7070 -health 0.0.0.0:8080 \ + -health-auth-token "$STREAMHIVE_HEALTH_TOKEN" +``` + +Use a TLS-terminating reverse proxy or a private, access-controlled network before exposing health beyond a trusted lab; the P2P TLS flags protect the P2P listener, not this HTTP listener. The verified certificate and application-auth ordering is documented in [TLS_AUTH.md](TLS_AUTH.md) and exercised by `make test-tls-auth`. Reserve `-tls-insecure-skip-verify` for local development. For CLI mTLS, add `-tls-client-ca -tls-require-client-cert` on the listener and `-tls-client-cert -tls-client-key` on outbound peers. Use `-tls-expiry-warning` to set the aggregate short-lived-credential warning window (`720h` by default, `0` disables the warning). For custom trust policy, configure `p2p.TCPTransport.TLSServerConfig` and `TLSClientConfig` in library code. Certificate files are loaded at process startup. Use the [TLS rotation runbook](TLS_ROTATION.md) for replacement material, trust overlap, restart ordering, rollback, and reconnect checks. @@ -187,7 +195,13 @@ spec: containers: - name: streamhive image: streamhive:local - args: ["-listen", "0.0.0.0:7070", "-health", "0.0.0.0:8080"] + args: ["-listen", "0.0.0.0:7070", "-health", "0.0.0.0:8080", "-health-auth-token", "$(STREAMHIVE_HEALTH_TOKEN)"] + env: + - name: STREAMHIVE_HEALTH_TOKEN + valueFrom: + secretKeyRef: + name: streamhive-health + key: token ports: - containerPort: 7070 name: p2p @@ -207,7 +221,7 @@ spec: periodSeconds: 10 ``` -Add a `Service` for the health port and (separately) headless or load-balanced service for P2P depending on your topology. Tune resource requests/limits and pod anti-affinity for HA; this manifest is illustrative only. +The health token is intentionally supplied from a Secret. `/livez` and `/readyz` do not require the token, so the illustrative probes remain usable; diagnostic clients must send the bearer header. Add a `Service` for the health port and (separately) headless or load-balanced service for P2P depending on your topology. Tune resource requests/limits and pod anti-affinity for HA; this manifest is illustrative only. ## SLOs diff --git a/docs/PROTOCOL.md b/docs/PROTOCOL.md index 68c0675..ba95c19 100644 --- a/docs/PROTOCOL.md +++ b/docs/PROTOCOL.md @@ -426,7 +426,11 @@ exemplars, peer addresses, blob keys, or certificate metadata. The optional HTTP health server also uses fixed bounds rather than unbounded request handling: 5-second header reads, 10-second request reads, 10-second response writes, 60-second idle connections, and 1 MiB maximum headers. Process cancellation gracefully shuts down the server; -the P2P wire protocol and endpoint paths are unchanged. +the P2P wire protocol and endpoint paths are unchanged. Health binds are loopback-first; a +non-loopback address requires the explicit `-health-auth-token` bearer boundary. `/livez` and +`/readyz` remain probe-safe, while diagnostic routes reject requests without the configured token. +The P2P TLS flags do not apply to this HTTP listener, so deployments beyond a trusted lab must use +a TLS-terminating proxy or an equivalent private network boundary. ## Shutdown and Drain diff --git a/docs/TLS_AUTH.md b/docs/TLS_AUTH.md index 935ac88..2c62e50 100644 --- a/docs/TLS_AUTH.md +++ b/docs/TLS_AUTH.md @@ -35,13 +35,19 @@ The listener loads its certificate and private key: ```bash go run . \ -listen 0.0.0.0:7070 \ - -health 0.0.0.0:8080 \ + -health 127.0.0.1:8080 \ -replicate -store-dir ./streamhive-data \ -tls-cert ./server-cert.pem -tls-key ./server-key.pem \ -peer-auth-token "$STREAMHIVE_PEER_TOKEN" \ -peer-id server -peer-allow-ids client ``` +The P2P TLS flags above do not encrypt or authenticate the separate HTTP health listener. Keep +health on loopback, or provide `-health-auth-token` for a deliberately network-scoped lab bind and +place a TLS-terminating reverse proxy or private access-controlled network in front of it. Diagnostic +health requests then send `Authorization: Bearer `; `/livez` and `/readyz` remain available to +orchestrator probes without the token. + The outbound peer supplies the trusted CA and the DNS name present in the certificate: ```bash diff --git a/health_server_acceptance_test.go b/health_server_acceptance_test.go index ee1da2e..1b7ce33 100644 --- a/health_server_acceptance_test.go +++ b/health_server_acceptance_test.go @@ -60,6 +60,69 @@ func TestRun_healthServerReadOnlyHandlerRejectsBeforeWork(t *testing.T) { assert.False(t, called) } +func TestRun_healthServerRequiresAuthForNonLoopbackAddress(t *testing.T) { + var out, stderr safeBuffer + err := run(context.Background(), []string{ + "-listen", "127.0.0.1:0", + "-health", "0.0.0.0:0", + }, &out, &stderr) + require.Error(t, err) + assert.Contains(t, err.Error(), "requires -health-auth-token") + assert.Empty(t, out.String()) +} + +func TestRun_healthServerBearerAuth(t *testing.T) { + called := false + next := healthAuthHandler("test-token", http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + called = true + w.WriteHeader(http.StatusNoContent) + })) + + for _, test := range []struct { + name string + path string + authority string + statusCode int + wantCalled bool + }{ + {name: "missing", path: "/metrics", statusCode: http.StatusUnauthorized}, + {name: "wrong", path: "/metrics", authority: "Bearer wrong-token", statusCode: http.StatusUnauthorized}, + {name: "valid", path: "/metrics", authority: "Bearer test-token", statusCode: http.StatusNoContent, wantCalled: true}, + {name: "liveness probe", path: "/livez", statusCode: http.StatusNoContent, wantCalled: true}, + } { + t.Run(test.name, func(t *testing.T) { + called = false + req := httptest.NewRequest(http.MethodGet, "http://127.0.0.1"+test.path, nil) + if test.authority != "" { + req.Header.Set("Authorization", test.authority) + } + resp := httptest.NewRecorder() + next.ServeHTTP(resp, req) + assert.Equal(t, test.statusCode, resp.Code) + assert.Equal(t, test.wantCalled, called) + }) + } +} + +func TestRun_healthAddrIsLoopback(t *testing.T) { + for _, test := range []struct { + addr string + loopback bool + }{ + {addr: "127.0.0.1:8080", loopback: true}, + {addr: "[::1]:8080", loopback: true}, + {addr: "localhost:8080", loopback: true}, + {addr: ":8080"}, + {addr: "0.0.0.0:8080"}, + {addr: "[::]:8080"}, + {addr: "health.internal:8080"}, + } { + t.Run(test.addr, func(t *testing.T) { + assert.Equal(t, test.loopback, healthAddrIsLoopback(test.addr)) + }) + } +} + func TestRun_healthServerReadOnlyEndpoints(t *testing.T) { node := startHealthAcceptanceNode(t) t.Cleanup(func() { node.stop(t) }) diff --git a/main.go b/main.go index f81df21..ea93f9a 100644 --- a/main.go +++ b/main.go @@ -3,6 +3,7 @@ package main import ( "bytes" "context" + "crypto/subtle" "crypto/tls" "crypto/x509" "encoding/hex" @@ -80,6 +81,7 @@ func run(ctx context.Context, args []string, stdout, stderr io.Writer) error { peerReconnectMax := fs.Duration("peer-reconnect-max", 30*time.Second, "maximum reconnect backoff for -peer-reconnect") syncInterval := fs.Duration("sync-interval", 0, "periodically advertise local blob keys to connected peers (0 = startup only)") health := fs.String("health", "", "optional HTTP listen addr for /livez /readyz /peers /metrics (e.g. :8080)") + healthAuthToken := fs.String("health-auth-token", "", "bearer token required when -health binds a non-loopback address") shutdownGrace := fs.Duration("shutdown-grace", defaultShutdownGrace, "bounded graceful shutdown deadline for health and P2P drain") maxPeers := fs.Int("max-peers", 0, "max simultaneous peers (0 = unlimited)") peerAuthToken := fs.String("peer-auth-token", "", "optional shared token required before peer registration") @@ -136,6 +138,9 @@ func run(ctx context.Context, args []string, stdout, stderr io.Writer) error { if err := fs.Parse(args); err != nil { return err } + if *health != "" && !healthAddrIsLoopback(*health) && strings.TrimSpace(*healthAuthToken) == "" { + return fmt.Errorf("health: non-loopback address %q requires -health-auth-token", *health) + } log := slog.New(slog.NewTextHandler(stderr, &slog.HandlerOptions{Level: slog.LevelInfo})) @@ -700,7 +705,7 @@ func run(ctx context.Context, args []string, stdout, stderr io.Writer) error { if *health != "" { var err error - hsrv, err = startHealth(*health, tr, replMetrics, blobStore, keyLister, tlsHealth, lifecycleState, reconnectMetrics, log) + hsrv, err = startHealth(*health, *healthAuthToken, tr, replMetrics, blobStore, keyLister, tlsHealth, lifecycleState, reconnectMetrics, log) if err != nil { return fmt.Errorf("health: %w", err) } @@ -2422,7 +2427,39 @@ func readOnlyHealthHandler(handler http.HandlerFunc) http.HandlerFunc { } } -func startHealth(addr string, tr *p2p.TCPTransport, replMetrics *replicationMetrics, blobStore storage.BlobStore, keyLister storage.BlobKeyLister, tlsHealth *tlsCredentialHealth, lifecycleState *lifecycleRuntime, reconnectMetrics *peerReconnectMetrics, log *slog.Logger) (*http.Server, error) { +func healthAddrIsLoopback(addr string) bool { + host, _, err := net.SplitHostPort(addr) + if err != nil { + return false + } + host = strings.Trim(host, "[]") + if strings.EqualFold(host, "localhost") { + return true + } + ip := net.ParseIP(host) + return ip != nil && ip.IsLoopback() +} + +func healthAuthHandler(token string, next http.Handler) http.Handler { + if token == "" { + return next + } + expected := "Bearer " + token + return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + if req.URL.Path == "/livez" || req.URL.Path == "/readyz" { + next.ServeHTTP(w, req) + return + } + if subtle.ConstantTimeCompare([]byte(req.Header.Get("Authorization")), []byte(expected)) != 1 { + w.Header().Set("WWW-Authenticate", `Bearer realm="streamhive-health"`) + http.Error(w, "unauthorized", http.StatusUnauthorized) + return + } + next.ServeHTTP(w, req) + }) +} + +func startHealth(addr, authToken string, tr *p2p.TCPTransport, replMetrics *replicationMetrics, blobStore storage.BlobStore, keyLister storage.BlobKeyLister, tlsHealth *tlsCredentialHealth, lifecycleState *lifecycleRuntime, reconnectMetrics *peerReconnectMetrics, log *slog.Logger) (*http.Server, error) { mux := http.NewServeMux() mux.HandleFunc("/livez", readOnlyHealthHandler(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) @@ -2512,6 +2549,7 @@ func startHealth(addr string, tr *p2p.TCPTransport, replMetrics *replicationMetr } writePrometheusMetrics(w, snapshot) })) + protected := healthAuthHandler(authToken, mux) ln, err := net.Listen("tcp", addr) if err != nil { @@ -2519,7 +2557,7 @@ func startHealth(addr string, tr *p2p.TCPTransport, replMetrics *replicationMetr } srv := &http.Server{ - Handler: mux, + Handler: protected, ReadHeaderTimeout: defaultHealthReadHeaderTimeout, ReadTimeout: defaultHealthReadTimeout, WriteTimeout: defaultHealthWriteTimeout, diff --git a/scripts/demo-auth.sh b/scripts/demo-auth.sh index 2a69989..a5a63de 100755 --- a/scripts/demo-auth.sh +++ b/scripts/demo-auth.sh @@ -6,12 +6,14 @@ DATA_DIR="${STREAMHIVE_DATA_DIR:-$ROOT_DIR/.streamhive-auth}" COMPOSE="docker compose" EXPECTED_KEY="cd13ac0817f0f8ba2f29fba23617ef0191a6193ed0311298163834199398ee05" TOKEN="${STREAMHIVE_PEER_TOKEN:-streamhive-compose-demo-token}" +HEALTH_TOKEN="${STREAMHIVE_HEALTH_TOKEN:-streamhive-health-demo-token}" WRONG_TOKEN="${STREAMHIVE_WRONG_PEER_TOKEN:-streamhive-invalid-token}" if [ "$TOKEN" = "$WRONG_TOKEN" ]; then WRONG_TOKEN="${TOKEN}!" fi export STREAMHIVE_DATA_DIR="$DATA_DIR" export STREAMHIVE_PEER_TOKEN="$TOKEN" +export STREAMHIVE_HEALTH_TOKEN="$HEALTH_TOKEN" export STREAMHIVE_NODE1_ID=node1 export STREAMHIVE_NODE1_ALLOW_IDS=node2,node3,seed export STREAMHIVE_NODE2_ID=node2 @@ -20,6 +22,10 @@ export STREAMHIVE_NODE3_ID=node3 export STREAMHIVE_NODE3_ALLOW_IDS=node1,node2 export STREAMHIVE_SEED_ID=seed +curl_health() { + curl -fsS -H "Authorization: Bearer $HEALTH_TOKEN" "$@" +} + cleanup() { $COMPOSE -f "$ROOT_DIR/docker-compose.yml" down --remove-orphans >/dev/null 2>&1 || true } @@ -29,7 +35,7 @@ wait_ready() { name="$1" url="$2" i=0 - until curl -fsS "$url/readyz" >/dev/null 2>&1; do + until curl_health "$url/readyz" >/dev/null 2>&1; do i=$((i + 1)) if [ "$i" -gt 80 ]; then echo "$name did not become ready" >&2 @@ -45,11 +51,11 @@ wait_metric() { url="$2" metric="$3" i=0 - until curl -fsS "$url/metrics" | grep "\"$metric\": [1-9]" >/dev/null; do + until curl_health "$url/metrics" | grep "\"$metric\": [1-9]" >/dev/null; do i=$((i + 1)) if [ "$i" -gt 80 ]; then echo "$name did not report a positive $metric counter" >&2 - curl -fsS "$url/metrics" >&2 || true + curl_health "$url/metrics" >&2 || true $COMPOSE -f "$ROOT_DIR/docker-compose.yml" logs "$name" >&2 || true exit 1 fi @@ -80,7 +86,7 @@ wait_key_present() { metric_value() { url="$1" metric="$2" - curl -fsS "$url/metrics" | awk -F': ' -v metric="\"$metric\"" 'index($1, metric) > 0 {gsub(/,/, "", $2); print $2; exit}' + curl_health "$url/metrics" | awk -F': ' -v metric="\"$metric\"" 'index($1, metric) > 0 {gsub(/,/, "", $2); print $2; exit}' } wait_identity() { @@ -88,11 +94,11 @@ wait_identity() { url="$2" identity="$3" i=0 - until curl -fsS "$url/peers" | grep "\"auth_identity\": \"$identity\"" >/dev/null; do + until curl_health "$url/peers" | grep "\"auth_identity\": \"$identity\"" >/dev/null; do i=$((i + 1)) if [ "$i" -gt 80 ]; then echo "$name did not expose authenticated peer identity $identity" >&2 - curl -fsS "$url/peers" >&2 || true + curl_health "$url/peers" >&2 || true $COMPOSE -f "$ROOT_DIR/docker-compose.yml" logs "$name" >&2 || true exit 1 fi diff --git a/scripts/demo-compose.sh b/scripts/demo-compose.sh index ba65fe6..125d5ee 100755 --- a/scripts/demo-compose.sh +++ b/scripts/demo-compose.sh @@ -5,7 +5,13 @@ ROOT_DIR="$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)" DATA_DIR="${STREAMHIVE_DATA_DIR:-$ROOT_DIR/.streamhive-compose}" COMPOSE="docker compose" EXPECTED_KEY="cd13ac0817f0f8ba2f29fba23617ef0191a6193ed0311298163834199398ee05" +HEALTH_TOKEN="${STREAMHIVE_HEALTH_TOKEN:-streamhive-health-demo-token}" export STREAMHIVE_DATA_DIR="$DATA_DIR" +export STREAMHIVE_HEALTH_TOKEN="$HEALTH_TOKEN" + +curl_health() { + curl -fsS -H "Authorization: Bearer $HEALTH_TOKEN" "$@" +} cleanup() { $COMPOSE -f "$ROOT_DIR/docker-compose.yml" down --remove-orphans >/dev/null 2>&1 || true @@ -16,7 +22,7 @@ wait_ready() { name="$1" url="$2" i=0 - until curl -fsS "$url/readyz" >/dev/null 2>&1; do + until curl_health "$url/readyz" >/dev/null 2>&1; do i=$((i + 1)) if [ "$i" -gt 80 ]; then echo "$name did not become ready" >&2 @@ -31,7 +37,7 @@ wait_stored() { name="$1" url="$2" i=0 - until curl -fsS "$url/metrics" | grep '"replication_blobs_stored": [1-9]' >/dev/null; do + until curl_health "$url/metrics" | grep '"replication_blobs_stored": [1-9]' >/dev/null; do i=$((i + 1)) if [ "$i" -gt 80 ]; then echo "$name did not store replicated blob" >&2 @@ -78,6 +84,6 @@ esac echo "3-node compose demo passed: node3 rehydrated the blob after restart" echo "rehydrated key: $EXPECTED_KEY" -curl -fsS http://127.0.0.1:18083/metrics +curl_health http://127.0.0.1:18083/metrics echo "$ROOT_DIR/scripts/demo-status.sh" diff --git a/scripts/demo-failure.sh b/scripts/demo-failure.sh index a21d0f7..973aa45 100755 --- a/scripts/demo-failure.sh +++ b/scripts/demo-failure.sh @@ -5,7 +5,13 @@ ROOT_DIR="$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)" DATA_DIR="${STREAMHIVE_DATA_DIR:-$ROOT_DIR/.streamhive-failure}" COMPOSE="docker compose" EXPECTED_KEY="cd13ac0817f0f8ba2f29fba23617ef0191a6193ed0311298163834199398ee05" +HEALTH_TOKEN="${STREAMHIVE_HEALTH_TOKEN:-streamhive-health-demo-token}" export STREAMHIVE_DATA_DIR="$DATA_DIR" +export STREAMHIVE_HEALTH_TOKEN="$HEALTH_TOKEN" + +curl_health() { + curl -fsS -H "Authorization: Bearer $HEALTH_TOKEN" "$@" +} cleanup() { $COMPOSE -f "$ROOT_DIR/docker-compose.yml" down --remove-orphans >/dev/null 2>&1 || true @@ -16,7 +22,7 @@ wait_ready() { name="$1" url="$2" i=0 - until curl -fsS "$url/readyz" >/dev/null 2>&1; do + until curl_health "$url/readyz" >/dev/null 2>&1; do i=$((i + 1)) if [ "$i" -gt 80 ]; then echo "$name did not become ready" >&2 @@ -31,7 +37,7 @@ wait_unready() { name="$1" url="$2" i=0 - while curl -fsS "$url/readyz" >/dev/null 2>&1; do + while curl_health "$url/readyz" >/dev/null 2>&1; do i=$((i + 1)) if [ "$i" -gt 40 ]; then echo "$name stayed ready after stop" >&2 @@ -46,7 +52,7 @@ wait_peers() { name="$1" url="$2" i=0 - until curl -fsS "$url/peers" | grep '"active_peers": [1-9]' >/dev/null; do + until curl_health "$url/peers" | grep '"active_peers": [1-9]' >/dev/null; do i=$((i + 1)) if [ "$i" -gt 80 ]; then echo "$name did not observe any active peers" >&2 diff --git a/scripts/demo-lifecycle-compose.sh b/scripts/demo-lifecycle-compose.sh index 13f967e..e26d345 100755 --- a/scripts/demo-lifecycle-compose.sh +++ b/scripts/demo-lifecycle-compose.sh @@ -5,9 +5,15 @@ ROOT_DIR="$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)" COMPOSE_FILE="$ROOT_DIR/docker-compose.lifecycle.yml" DATA_DIR="${STREAMHIVE_LIFECYCLE_DATA_DIR:-$ROOT_DIR/.streamhive-lifecycle}" TOKEN="${STREAMHIVE_LIFECYCLE_TOKEN:-streamhive-lifecycle-demo-token}" +HEALTH_TOKEN="${STREAMHIVE_LIFECYCLE_HEALTH_TOKEN:-streamhive-lifecycle-health-demo-token}" EXPECTED_KEY="2ec046e61b07981f2e389edfeafbfd5d08ffde38faf4512fc7dd725d507b94f2" export STREAMHIVE_LIFECYCLE_DATA_DIR="$DATA_DIR" export STREAMHIVE_LIFECYCLE_TOKEN="$TOKEN" +export STREAMHIVE_LIFECYCLE_HEALTH_TOKEN="$HEALTH_TOKEN" + +curl_health() { + curl -fsS -H "Authorization: Bearer $HEALTH_TOKEN" "$@" +} compose() { docker compose -f "$COMPOSE_FILE" "$@" @@ -22,7 +28,7 @@ wait_ready() { name="$1" url="$2" i=0 - until curl -fsS "$url/readyz" >/dev/null 2>&1; do + until curl_health "$url/readyz" >/dev/null 2>&1; do i=$((i + 1)) if [ "$i" -gt 120 ]; then echo "$name did not become ready" >&2 @@ -39,7 +45,7 @@ wait_status() { pattern="$3" i=0 while :; do - payload="$(curl -fsS "$url/lifecycle/status" 2>/dev/null || true)" + payload="$(curl_health "$url/lifecycle/status" 2>/dev/null || true)" if printf '%s\n' "$payload" | grep -F "$pattern" >/dev/null 2>&1; then return 0 fi @@ -56,7 +62,7 @@ wait_status() { floor_version() { url="$1" - curl -fsS "$url/lifecycle/status" | awk -F: ' + curl_health "$url/lifecycle/status" | awk -F: ' /"journal_floor": \{/ { in_floor = 1; next } in_floor && /"epoch":/ { gsub(/[^0-9]/, "", $2); epoch = $2; next } in_floor && /"sequence":/ { gsub(/[^0-9]/, "", $2); print epoch "/" $2; exit } @@ -214,4 +220,4 @@ fi echo "3-node lifecycle compose demo passed: stale target restored compacted snapshot after source restart" echo "checkpoint floor: $source_floor" echo "retained raw key: $EXPECTED_KEY" -curl -fsS http://127.0.0.1:18183/lifecycle/status +curl_health http://127.0.0.1:18183/lifecycle/status diff --git a/scripts/demo-repair.sh b/scripts/demo-repair.sh index ff1b9b9..8c60be1 100755 --- a/scripts/demo-repair.sh +++ b/scripts/demo-repair.sh @@ -5,7 +5,13 @@ ROOT_DIR="$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)" DATA_DIR="${STREAMHIVE_DATA_DIR:-$ROOT_DIR/.streamhive-repair}" COMPOSE="docker compose" EXPECTED_KEY="cd13ac0817f0f8ba2f29fba23617ef0191a6193ed0311298163834199398ee05" +HEALTH_TOKEN="${STREAMHIVE_HEALTH_TOKEN:-streamhive-health-demo-token}" export STREAMHIVE_DATA_DIR="$DATA_DIR" +export STREAMHIVE_HEALTH_TOKEN="$HEALTH_TOKEN" + +curl_health() { + curl -fsS -H "Authorization: Bearer $HEALTH_TOKEN" "$@" +} cleanup() { $COMPOSE -f "$ROOT_DIR/docker-compose.yml" down --remove-orphans >/dev/null 2>&1 || true @@ -16,7 +22,7 @@ wait_ready() { name="$1" url="$2" i=0 - until curl -fsS "$url/readyz" >/dev/null 2>&1; do + until curl_health "$url/readyz" >/dev/null 2>&1; do i=$((i + 1)) if [ "$i" -gt 80 ]; then echo "$name did not become ready" >&2 @@ -32,11 +38,11 @@ wait_metric() { url="$2" metric="$3" i=0 - until curl -fsS "$url/metrics" | grep "\"$metric\": [1-9]" >/dev/null; do + until curl_health "$url/metrics" | grep "\"$metric\": [1-9]" >/dev/null; do i=$((i + 1)) if [ "$i" -gt 80 ]; then echo "$name did not report a positive $metric counter" >&2 - curl -fsS "$url/metrics" >&2 || true + curl_health "$url/metrics" >&2 || true $COMPOSE -f "$ROOT_DIR/docker-compose.yml" logs "$name" >&2 || true exit 1 fi @@ -95,5 +101,5 @@ fi echo "3-node repair demo passed: node3 detected and repaired corrupted blob" echo "repaired key: $EXPECTED_KEY" -curl -fsS http://127.0.0.1:18083/metrics +curl_health http://127.0.0.1:18083/metrics echo diff --git a/scripts/demo-status.sh b/scripts/demo-status.sh index 0c581be..ec56527 100755 --- a/scripts/demo-status.sh +++ b/scripts/demo-status.sh @@ -4,18 +4,23 @@ set -eu ROOT_DIR="$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)" DATA_DIR="${STREAMHIVE_DATA_DIR:-$ROOT_DIR/.streamhive-compose}" COMPOSE="docker compose" +HEALTH_TOKEN="${STREAMHIVE_HEALTH_TOKEN:-streamhive-health-demo-token}" export STREAMHIVE_DATA_DIR="$DATA_DIR" +curl_health() { + curl -fsS -H "Authorization: Bearer $HEALTH_TOKEN" "$@" +} + print_node() { name="$1" url="$2" echo "== $name ==" echo "-- peers --" - curl -fsS "$url/peers" + curl_health "$url/peers" echo echo "-- metrics --" - curl -fsS "$url/metrics" + curl_health "$url/metrics" echo echo "-- durable keys --" keys="$($COMPOSE -f "$ROOT_DIR/docker-compose.yml" --profile tools run --rm --no-deps -v "$DATA_DIR/$name:/data" seed -store-dir /data -list-keys)"